I'm trying to run a simple script that will scrape some data using x-ray and insert it into my Events collection.
if (Meteor.isServer) {
var Xray = Meteor.npmRequire('x-ray');
var xray = new Xray({
version: "2.0.3"
});
xray('http://www.events12.com/seattle/january/', '.qq', [{
title: '.title',
date: '.date',
link: 'a #href',
allContent: '#html'
}])(function(err, content) {
for (var i = 0; i < content.length; i++) {
(function() {
console.log(i);
var newEvent = {
owner: 'me',
name: content[i].title,
date: content[i].date,
url: content[i].link,
createdAt: new Date(),
description: 'none'
};
console.log(newEvent);
Events.insert(newEvent, function(err, data) {
console.log(err);
console.log(data);
});
})();
}
});
}
The callback from x-ray that takes in content has all the scraped data in an array of objects, each with several properties. When I try to insert this data into my Events collection, the for loop iterates once and then exits, but no error is shown. If I remove the Events.insert() the loop iterates all the way through.
What am I missing? What is the proper way to execute such a task?
The Events.insert() was being called outside of any Meteor fibers. Adding Meteor.bindEnvironment() and feeding the entire function in as a callback fixed this problem.
Related
I have this array of objects.
let links = [
{
url: 'some url 1',
status: 200
},
{
url: 'some url 2',
status: 200
}
]
Which is the result of calling LinkFunction asyncronously inside before:
before(async () => {
try {
links = await LinkFunction();
} catch (err) {
assert.fail(err);
}
});
I would like to check if the url and status properties exist and if their types are correspondingly string and number.
Note: The specified object is just a sample of a big response. So the loop is required for iteration in any case.
I've done this iteration:
it('Array to contain some props', () => {
links.map(property => {
expect(property).to.have.property('url').to.be.a('string');
expect(property).to.have.property('status').to.be.a('number');
});
});
But I would like to have something like this:
it('Array to contain some props', () => {//or describe would be better here
links.map(property => {
it('link array to contain url string', () => {
expect(property).to.have.property('url').to.be.a('string');
});
it('link array to contain status number', () => {
expect(property).to.have.property('status').to.be.a('number');
});
});
});
Unfortunately it statements are ignored inside map. Maybe it's because of the several nested it statements. So How can I implement a similar logic?
Update:
My full code:
You might want to use forEach instead of map.
Also, "Passing arrow functions (aka "lambdas") to Mocha is discouraged" so you will probably want to change those to normal functions.
Having said that, it works fine if links is defined as mocha initially runs the test file and collects the individual it tests:
const expect = require('chai').expect;
describe('links', function() {
let links = [
{
url: 'some url 1',
status: 200
},
{
url: 'some url 2',
status: 200
}
]
links.forEach(function(property) {
it('link array to contain url string', function() {
expect(property).to.have.property('url').to.be.a('string');
});
it('link array to contain status number', function() {
expect(property).to.have.property('status').to.be.a('number');
});
});
});
..results in:
> mocha
links
√ link array to contain url string
√ link array to contain status number
√ link array to contain url string
√ link array to contain status number
4 passing (14ms)
Update
As you have found, it only works at the top level or with a describe:
before(function() {
it('will NOT work here', function() { });
});
it('will work here', function() {
it('will NOT work here', function() { });
});
Also, links must be available while the test is first running and the it tests are being collected by mocha so this doesn't work either:
describe('links', function() {
let links = [];
before(function() {
links = [
{
url: 'some url 1',
status: 200
},
{
url: 'some url 2',
status: 200
}
];
});
// This won't work...
links.forEach(function(property) {
// .. since links is still an empty array when this runs
it('should...', function() { /* ... */ });
});
});
From your question update it looks like your code retrieves links from an async function call in before. Because of this, there is no way to have links populated at the time the test is first running and the it tests are collected.
So it looks like you won't be able to map across the items in links to create your it tests, and will instead need to take the approach you described by mapping across the items in links within a single test.
My goal is to create a service appointment record associated with multiple resources.
For this purpose I have followed this MSDN example.
The problem rises when I try to associate multiple resources for one particular service appointment, CRM server will store only the last one(1 record). Following demonstrates my code:
//attendees =[... array of resource ids]
var serviceAppointment = {
ScheduledStart: new Date("12/22/2014 4:53 PM"),
ScheduledEnd: new Date("12/22/2014 5:53 PM"),
Subject: "test service",
ServiceId:
{
//// hardcoded id for simplicity
Id: "6f795012-ca55-e411-aa38-00155d0a0948",
LogicalName: "service"
}
};
SDK.JQuery.createRecord(serviceAppointment,"ServiceAppointment"
,function(sa){
for(var i=0;i<attendees.length;i++)
{
var activityParty = {
PartyId:
{
Id: attendees[i],
LogicalName: "systemuser",
},
ActivityId:
{
Id: sa.ActivityId,
LogicalName: "serviceappointment",
},
ParticipationTypeMask:
{
//See http://msdn.microsoft.com/en-us/library/gg328549.aspx
//10 is for resource
Value: 10
}
};
SDK.JQuery.createRecord(activityParty,"ActivityParty", function(ap){debugger;},errorHandler);
}
}
,errorHandler);
As far as I debugged the code the create record is being executed properly without no exception. I believe I'm missing a configuration flag somewhere in my code and CRM is considering one to one association rather than one to many.
Any clues?
I could solve this issue by passing array of parties in service appointment record at the first place rather than inserting them one-by-one at the end. Here is the code which works:
var parties =[];
for(var i=0;i< e.event.attendees.length;i++)
{
var activityParty = {
PartyId:
{
Id: e.event.attendees[i],
LogicalName: "systemuser",
},
ParticipationTypeMask:
{
//See http://msdn.microsoft.com/en-us/library/gg328549.aspx
//10 is for resource
Value: 10
}
}
parties.push(activityParty);
}
var serviceAppointment = {
ScheduledStart: e.event.start,
ScheduledEnd: e.event.end,
Subject: e.event.title,
ServiceId:
{
Id: "6f795012-ca55-e411-aa38-00155d0a0948",
LogicalName: "service"
},
serviceappointment_activity_parties: parties,
};
SDK.JQuery.createRecord(serviceAppointment,"ServiceAppointment"
,function(sa){
debugger;
e.event.ActivityId = serviceAppointmentActivityId = sa.ActivityId;
}
,errorHandler);
}
hope it helps somebody out there.
A webpage sends JSON data via POST to my Node.js App (MEAN-environment using Mongoose). The JSON file looks like this (excerpt):
Firstname: 'XY',
Surname: 'asd',
Articles:
[ { title: '1', description: 'XY' },
{ title: '2', description: 'XY' },
{ title: '3', description: 'XY' }
The purpose is to create an Author in a mongodb database, add the author to an additional directory, store associated articles and create references to those articles in the author document (code excluded). This is the code that handles the request:
[...]
async.waterfall([
//Create random code for author (besides mongodb-specific id)
function(callback){
newAuthorCode.randomAuthorCode(function(err, code) {
callback(null, code);
});
},
//Save new author to db
function(code, callback){
var newAuthor = new Author({firstname: req.body.Firstname,
surname: req.body.Surname,
identcode: code
});
newAuthor.save(function (err){
callback(null, newAuthor);
});
},
//Add new author to an additional directory
function(newAuthor, callback){
Directory.update({_id: req.user._id}, {$push: {authorids: newAuthor._id }}, function(err, update){
if (update){
callback(null);
}
});
},
//saves articles to db
function(callback){
var keys = Object.keys(req.body.articles);
for(var i=0, length=keys.length; i<length; i++){
var newArticle = new Article({title: req.body.Articles[keys[i]].title,
description: req.body.Articles[keys[i]].description
});
newArticle.save(function (err){
console.log(newArticle._id); // <--- !!!!!!!
});
}
callback(null);
}
], function (err, result) {
console.log('DONE!!!');
res.send('200');
});
My problems:
1) The marked line of code where I try to output all IDs of the generated articles only delivers i-times the ID of the last article stored (in this case of article 3).
2) Problem 1 leads to the issue that I can not create references of the newly created articles in the author document (stored in a different collection) as I can't access no article IDs but the last one!?
3) Sometimes the author, as well as the articles, are created multiple times in the database (with huge time gap in between)!?
Thanks for any advice as I am running out of ideas.
Igor
Not sure what newLink is in your code, but you can try this instead:
newArticle.save(function (err, newDoc){
console.log(newDoc._id); // <--- !!!!!!!
});
I have the following models in my Sailsjs application with a many-to-many relationship:
event.js:
attributes: {
title : { type: 'string', required: true },
description : { type: 'string', required: true },
location : { type: 'string', required: true },
maxMembers : { type: 'integer', required: true },
currentMembers : { collection: 'user', via: 'eventsAttending', dominant: true },
creator : { model: 'user', required: true },
invitations : { collection: 'invitation', via: 'eventID' },
tags : { collection: 'tag', via: 'taggedEvents', dominant: true },
lat : { type: 'float' },
lon : { type: 'float' },
},
tags.js:
attributes: {
tagName : { type: 'string', unique: true, required: true },
taggedEvents : { collection: 'event', via: 'tags' },
},
Based on the documentation, this relationship looks correct. I have the following method in tag.js that accepts an array of tag strings, and an event id, and is supposed to add or remove the tags that were passed in:
modifyTags: function (tags, eventId) {
var tagRecords = [];
_.forEach(tags, function(tag) {
Tag.findOrCreate({tagName: tag}, {tagName: tag}, function (error, result) {
tagRecords.push({id: result.id})
})
})
Event.findOneById(eventId).populate('tags').exec(function(error, event){
console.log(event)
var currentTags = event.tags;
console.log(currentTags)
delete currentTags.add;
delete currentTags.remove;
if (currentTags.length > 0) {
currentTags = _.pluck(currentTags, 'id');
}
var modifiedTags = _.pluck(tagRecords, 'id');
var tagsToAdd = _.difference(modifiedTags, currentTags);
var tagsToRemove = _.difference(currentTags, modifiedTags);
console.log('current', currentTags)
console.log('remove', tagsToRemove)
console.log('add', tagsToAdd)
if (tagsToAdd.length > 0) {
_.forEach(tagsToAdd, function (tag) {
event.tags.add(tag);
})
event.save(console.log)
}
if (tagsToRemove.length > 0) {
_.forEach(tagsToRemove, function (tagId) {
event.tags.remove(tagId)
})
event.save()
}
})
}
This is how the method is called from the event model:
afterCreate: function(record, next) {
Tag.modifyTags(tags, record.id)
next();
}
When I post to event/create, I get this result: http://pastebin.com/PMiqBbfR.
It looks as if the method call itself is looped over, rather than just the tagsToAdd or tagsToRemove array. Whats more confusing is that at the end, in the last log of the event, it looks like the event has the correct tags. When I then post to event/1, however, the tags array is empty. I've also tried saving immediately after each .add(), but still get similar results.
Ideally, I'd like to loop over both the tagsToAdd and tagsToRemove arrays, modify their ids in the model's collection, and then call .save() once on the model.
I have spent a ton of time trying to debug this, so any help would be greatly appreciated!
There are a few problems with your implementation, but the main issue is that you're treating certain methods--namely .save() and .findOrCreate as synchronous methods, when they are (like all Waterline methods) asynchronous, requiring a callback. So you're effectively running a bunch of code in parallel and not waiting for it to finish before returning.
Also, since it seems like what you're trying to do is replace the current event tags with this new list, the method you came up with is a bit over-engineered--you don't need to use event.tags.add and event.tags.remove. You can just use plain old update.
So you could probably rewrite the modifyTags method as:
modifyTags: function (tags, eventId, mainCb) {
// Asynchronously transform the `tags` array into an array of Tag records
async.map(tags, function(tag, cb) {
// For each tag, find or create a new record.
// Since the async.map `cb` argument expects a function with
// the standard (error, result) node signature, this will add
// the new (or existing) Tag instance to the resulting array.
// If an error occurs, async.map will exit early and call the
// "done()" function below
Tag.findOrCreate({tagName: tag}, {tagName: tag}, cb);
}, function done (err, tagRecords) {
if (err) {return mainCb(err);}
// Update the event with the new tags
Event.update({id: eventId}, {tags: tagRecords}).exec(mainCb);
});
}
See the full docs for async.map here.
If you wanted to stick with your implementation using .add and .remove, you would still want to use async.map, and do the rest of your logic in the done method. You don't need two .save calls; just do run all the .add and .remove code first, then do a single .save(mainCb) to finish it off.
And I don't know what you're trying to accomplish by deleting the .add and .remove methods from currentTags (which is a direct reference to event.tags), but it won't work and will just cause confusion later!
So I've been puzzling about this and just can't figure out how to fix this.
I've a nested function that loops through an array of objects and scrapes the social links of each object's URL.
After that, I'd like to update each object by including { social: [array of social urls] }. However, I get { social: [] } instead.
I've tried placing the contacts[i].social = results on other sections of this, but I get an "object not found" error.
Help would be much appreciated...
var contacts = [
{ title: 'Healthy breakfast: Quick, flexible options to grab at home - Mayo Clinic',
url: 'http://www.mayoclinic.org/healthy-living/nutrition-and-healthy-eating/in-depth/food-and-nutrition/art-20048294' },
{ title: 'Healthy Breakfast Ideas from Dr. Weil\'s Facebook Readers',
url: 'http://www.drweil.com/drw/u/ART03113/Healthy-Breakfast-Ideas-from-Facebook-Readers.html' },
{ title: '8 Healthy Breakfast Ideas - Prevention.com',
url: 'http://www.prevention.com/food/healthy-eating-tips/8-healthy-breakfast-ideas' },
{ title: 'Quick & Easy Healthy Breakfast Ideas! - YouTube',
url: 'http://www.youtube.com/watch?v=mD4YSD8LDiQ' },
{ title: 'The Best Foods to Eat for Breakfast - Health.com',
url: 'http://www.health.com/health/gallery/0,,20676415,00.html' },
{ title: 'Healthy Breakfast Recipes - Secrets To Cooking Healthier - YouTube',
url: 'http://www.youtube.com/watch?v=7jH0xe1XKxI' },
{ title: 'Healthy Breakfast Ideas You Can Make the Night Before - FitSugar',
url: 'http://www.fitsugar.com/Healthy-Breakfast-Ideas-You-Can-Make-Night-Before-20048633' },
{ title: '10 Easy, 5-Minute Breakfast Ideas - Diet and ... - Everyday Health',
url: 'http://www.everydayhealth.com/diet-and-nutrition-pictures/easy-5-minute-breakfast-ideas.aspx' },
{ title: 'Healthy Breakfast Ideas for Kids | Parenting - Parenting.com',
url: 'http://www.parenting.com/gallery/healthy-breakfast-ideas-kids' },
{ title: 'Fruits & Veggies More Matters : Healthy Breakfast Ideas : Health ...',
url: 'http://www.fruitsandveggiesmorematters.org/healthy-breakfast-ideas' }
];
scraper(contacts);
// loops through contacts database and scrapes requested information
function scraper(contacts) {
// Adds the domain of each contact
for(var i=0;i<contacts.length;i++){
contacts[i].domain = contacts[i].url.split(/\//, 3).join().replace(/,/g, '/');
};
//
for(var i=0;i<contacts.length;i++){
var homepage = contacts[i].domain;
var results = [];
function socialScrape(homepage) {
request(homepage, function(err, resp, html) {
var $ = cheerio.load(html);
if(!err && resp.statusCode == 200) {
$('a').each(function(i, el){
var a = $(el).attr('href');
for(var key in socialURLS){
if(socialURLS[key].test(a) && results.indexOf(a) < 0){
results.push(a);
}
}
});
} else { console.log(err); }
})
}
contacts[i].social = results;
socialScrape(homepage);
}
console.log(contacts);
}
Your first issue is that your request call is asynchronous and hasn't yet returned by the time contacts[i].social = results is executed, so contacts[i].results is getting assigned an empty array, []. (Variations of this issue are posted on SO multiple times every day, a good explanation of the problem can be found here: How do I return the response from an asynchronous call?) The solution to this is not as simple as just moving contacts[i].social = results; into inside the request call success handler because the value of i will have changed before the handler is called.
Your second issue is that results is defined outside of the socialScrape function definition - so instead of having an array of items per request call, you have one array with all request results. The best way to resolve your scoping issues is with a closure, which we can achieve by removing the call to socialScrape(homepage); and making socialScrape a self-invoking function:
(function socialScrape(homepage) {
var results = [];
var index = i;
request(homepage, function(err, resp, html) {
/* do error and status check stuff and build our results array */
contacts[index].social = results;
});
}(homepage));
Notice how we capture the current value of i within the closure and assign it to index. This will allow us to get the correct contact by index when our result is delivered.