Develop Reference

Promises structure misunderstood

I have a problem with understanding Promises syntax.
So, what I am trying to do:
getPosts() gets some data from a DB then, I want to get some metadata for each row with another promise call, addMetadata(). Then, once all the metadata is fetched, I want to console it out.
See my attempt below:
var getPosts = function(){
return new Promise(function(resolve, reject){
postModel.find()
.exec(function(err, posts) {
resolve(posts);
);
});
};
var addMetadata = function(posts){
var options = {
host: 'localhost',
port: 3000,
path: '',
method: 'GET'
};
var postPromises = posts.map(function(post) {
return new Promise(function(resolve) {
options.path = '/api/user?id=' + post.profileID;
var req = http.get(options, function(res) {
var bodyChunks = [];
res.on('data', function(chunk) {
bodyChunks.push(chunk);
}).on('end', function() {
var body = Buffer.concat(bodyChunks);
var parsedBody = JSON.parse(body);
post.fullname = parsedBody.user.fullname;
post.profilePic = parsedBody.user.profilePic;
// resolve the promise with the updated post
resolve(post);
});
});
});
});
// Is this the right place to put Promise.all???
Promise.all(postPromises)
.then(function(posts) {
//What should I put here
});
};
getPosts()
.then(function(posts){
return addMetadata(posts);
})
.then(function(posts){//I get a little lost here
console.log();//posts is undefined
});
Of course, my understanding is wrong but I thought I was going the right way. Can someone please guide me to the right direction?
Thanks

Change
// Is this the right place to put Promise.all???
Promise.all(postPromises)
.then(function (posts) {
//What should I put here
});
into
// Is this the right place to put Promise.all???
return Promise.all(postPromises);
This way your addMetadata function will return Promise that resolve when all promises from postPromises resolves or reject if any of postPromises rejects.

The key point to understand the async concept of it and what time the content is available.
Reading this will help to put you in the right direction.
For instance:
var promise = new Promise(function(resolve, reject) {
resolve(1);
});
promise
.then(function(val) {
console.log(val); // 1
return val + 2;
})
.then(function(val) {
console.log(val); // 3
})
After as per your scenario, in order to have all the metadata Promise.all is the way to go.
Promise.all(arrayOfPromises).then(function(arrayOfResults) {
// One result per each promise of AddMetadata
})

What you wanna do here, if I am correct, is called streams, as you wanna call multiple paralel promises as your concept of looping through list of posts using map is not going to work this way
Take a look at this short video introducing streams Streams - FunFunFunction, he is using library for workin with streams called Baconjs
Here is a short example on streams
const stupidNumberStream = {
each: (callback) => {
setTimeout( () => callback(1), 3000 )
setTimeout( () => callback(2), 2000 )
setTimeout( () => callback(3), 1000 )
}
}
stupidNumberStream.each(console.log)

Your getPosts function is good in the sense that its only job is to promisfy the database find. (Though, I think if it's mongo, the exec produces a promise for you).
Your addMetadataToAPost is less good, because it mixes up processing an array of posts and "promisifying" the http.get. Use the same pattern you applied correctly in the first function and return a promise to do a single get and add metadata. (It would be even better to just wrap the get, which you can reuse, and build a simple add-metadata function that returns - rather than creates - a promise)
// renamed pedantically
var addMetadataToAPost = function(post) {
return new Promise(function(resolve) {
options.path = '/api/user?id=' + post.profileID;
var req = http.get(options, function(res) {
var bodyChunks = [];
res.on('data', function(chunk) {
bodyChunks.push(chunk);
}).on('end', function() {
var body = Buffer.concat(bodyChunks);
var parsedBody = JSON.parse(body);
post.fullname = parsedBody.user.fullname;
post.profilePic = parsedBody.user.profilePic;
// resolve the promise with the updated post
resolve(post);
});
});
});
}
Now your batching function is simple:
// also renamed pedantically
var addMetadataToAllPosts = function(posts){
var postPromises = posts.map(function(post) {
return addMetadataToAPost(post)
})
return Promise.all(postPromises)
};
Your original code should work...
getPosts().then(function(posts){
return addMetadataToAllPosts(posts);
})
.then(function(posts){
console.log(posts);//posts should be an array of posts with metadata added
});

Promise.map within a Promise.map

I'm trying to do a return of JSON object using a Bluebird's Promise.mapSeries/Promise.map within a Promise.mapSeries/Promise.map but I am unable to return them correctly. The function is as follows:
function getMovieDetails(link){
return new Promise(function(resolve, reject) {
request(link, function(error, response, body){
var request = Promise.promisifyAll(require("request"), {multiArgs: true});
var $ = cheerio.load(body);
var movieYears = "Years";
var movieYearLinks = [];
movieYearLinks.each(function(index, item) {
var movieYear = $(item).text();
var movieYearLink = $(item).attr("href");
movieYearLinks.push(movieYearLink);
});
Promise.mapSeries(movieYearLinks, function(url) {
return request.getAsync(url).spread(function(response,body) {
var $ = cheerio.load(body);
var movie = {};
var year = "YEAR".text();
movie["year"] = year;
var movieActorsArray = [];
movieActors.each(function(index, item){
var movieActor = $(item).text();
movieActorsArray.push(movieActor);
});
movie["movieActors"] = movieActorsArray;
var recommendedMovies = //SOME ROWS;
var recommendedMoviesLinks = [];
recommendedMovies.each(function(jndex, jtem){
var recommendedRowObject = {};
var recommendedText = .text();
var recommendedLink = .attr("href");
recommendedRowObject["recommendedText"] = recommendedText
recommendedRowObject["recommendedLink"] = recommendedLink
recommendedMoviesLinks.push(recommendedLink);
});
Promise.mapSeries(recommendedMoviesLinks, function(url) {
return request.getAsync(url).spread(function(response,body) {
var $ = cheerio.load(body);
var obj = {};
// GET SOME OTHER DESCRIPTION FROM THE RECOMMENDED MOVIE LINK
return obj;
});
}).then(function(results) {
// results here returns an array of recommended movie links objects.
}).catch(function(err) {
});
return main;
});
}).then(function(results) {
// results here returns an array of movies
// I want to be able to link each movie with its recommended movies.
}).catch(function(err) {
console.log("Big Error " + err);
});
});
});
}
Some explanation of my code and context.
The home page is a page with movie years. I've looped through them to get the links of the years and put into an array movieYearLinks.
Next, I've used Promise.mapSeries to get some basic info on the movies. Within that basic info, there is are recommended movie link listed in a table. I've looped them through and put them into an array, recommendedMoviesLinks.
After which, I'm going in to grab some other recommended movie (recommendedMovieObject) info before.
I want to create a JSON object with all these info. The JSON object should be something like
{ movieYear: 2017, movieLink: ..., recommendedMovies (This is an array
of recommendedMovieObjects): }
I'm open to any solution to achieve this. Thank you in advance.

You don't need to nest promises, the whole point of promises is not having to nest callbacks. A new promise need to either be rejected with an error or resolved with a value. Your code should be something like :
let getMovieDetails = function(link){
return new Promise(function(resolve,reject){
request(link,function(err,response,body){
err ? reject(err):resolve(body)
}
}).then(function(body){
// do some cherio stuff
return dataList
}).then(function(dataList){
return Promise.all(dataList.map(function(d){return // a promise of d})
})
}
let finalData = getMovieDetails(link).then(function(data){
//do some computation
}).catch(function(e){ // handle errors })
Depending on your version of node there is no need for bluebird at all. Javascript supports promises.

how to can i handle multiple callbacks return values in nodejs?

I am trying to perform sql queries based on the callback results in if conditions but i am unable to write the code .so please provide som information in code
app.get('/resell-property', function(req, res) {
var data = {}
data.unit_price_id = 1;
function callback(error, result) {
if (result.count == 0) {
return hp_property_sell_request.create(data)
}
else if (result.count > 0) {
return hp_unit_price.findAll({
where: {
unit_price_id: data.unit_price_id,
hp_property_id: data.property_id,
hp_unit_details_id: data.unit_details_id
}
})
}
}
hp_property_sell_request.findAndCountAll({
where: {
unit_price_id: data.unit_price_id
}
}).then(function (result) {
if (result) {
callback(null, result);
}
});
});
In this how can i write the callbacks for
hp_property_sell_request.create(data) ,hp_unit_price.findAll({
where: {
unit_price_id: data.unit_price_id,
hp_property_id: data.property_id,
hp_unit_details_id: data.unit_details_id
}
})
In that after returning result again i have to handle callbacks and perform this query
if(result.request_id){
return hp_unit_price.findAll({
where:{
unit_price_id:result.unit_price_id,
hp_property_id:result.property_id,
hp_unit_details_id:result.unit_details_id
}
}).then(function (result){
if(result.is_resale_unit==0 && result.sold_out==0){
return Sequelize.query('UPDATE hp_unit_price SET resale_unit_status=1 WHERE hp_unit_details_id='+result.unit_details_id+' and hp_property_id='+result.property_id)
}
})
}

The promise resolve function takes only one input argument, so if you need to pass in multiple stuff, you have to enclose them in a single object. Like, if you have to go with something like:
database.openCollection()
.then(function(collection){
var result = collection.query(something);
var resultObject = { result: result, collection: collection };
})
.then(function(resultObject){
doSomethingSyncronousWithResult(resultObject.result);
resultObject.collection.close();
});
You can't use Promise all if all of your stuff isn't a result of a promise resolve, you might need to go with something like this.
Disclaimer: The code example is a very poor one, but it explains the concept.

I would suggest you to learn about Promises, particularly Bluebird.
You can promisify traditional callback methods.
I would also create model level functions in different files. Here's an example.
parent.js
const db = require("./connections/database"); // connection to database
const getChildForParent = function (parentId, childId, callback) {
db.find({parent: parentId, child_id: childId}, "childrenTable", function(err, result) {
if (err) {
return callback(err);
}
return callback(null, result);
});
};
children.js
const db = require("./connections/database"); // connection to database
const getToysForChild = function (childId, callback) {
db.find({toy_belongs_to: parentId}, "toysTable", function(err, result) {
if (err) {
return callback(err);
}
return callback(null, result);
});
};
Then in controller you can do something like this:
const Bluebird = require("bluebird");
const Parent = require("./parent.js");
const Child = require("./child.js");
// Promisifying adds "Async" at the end of your methods' names (these are promisified)
Bluebird.promisifyAll(Parent);
Bluebird.promisifyAll(Child);
// Just an example.
app.get("/parent/:parentId/children/:childId", function(req, res) {
return Bluebird.try(function() {
return User.getChildForParentAsync(req.params.parentId, req.params.childId);
}).then(function(child) {
return Child.getToysForChildAsync(child.child_id);
}).then(function(toys) {
// Do something with toys.
});
});
Of course you can do much more with this and this is not the only way.
Also you can use Promise.all(). This method is useful for when you want to wait for more than one promise to complete.
Let's say you have a list of urls that you want to fetch and process the results after all the data has been fetched.
var urls = [url1, url2, url3, url4, url5 .......... ];
var Bluebird = require("bluebird");
var request = require("request"); // callback version library
Bluebird.promisifyAll(request);
// create a list which will keep all the promises
var promises = [];
urls.forEach(function(url) {
promises.push(request.getAsync(url1));
});
// promises array has all the promises
// Then define what you want to do on completion.
Bluebird.all(promises).then(function(results) {
// results is an array with result a url in an index
// process results.
});

I would recommend to use Promises to solve that. If you need all results of all Requests, when they are all done Promise.all() will do that for you. Your basic could look like that:
var req1 = new Promise(function(res, rej){
var req = new XMLHttpRequest()
…
req.addEventListener('load', function (e) {
res(e);
})
var req2 = //similar to the above
Promise.all([req1, req2, …]).then(function(values){
//all requests are done here and you can do your stuff
});
You can also use the new fetch api, which creates Promises like so:
var req1 = fetch(…);
var req2 = fetch(…);
Promise.all([req1, re2, …]).then(…);

Node.js web scraping with loop on array of URLs

I'm trying to build a little script to scrap some data. I'm some basics knowledge in javascript however I'm kind of lost with all the async callback or promises stuff. Here is what I have now :
url = "http://Blablablabla.com";
var shares = function(req, res) {
request(url, function (error, response, body) {
if (!error) {
var $ = cheerio.load(body),
share = $(".theitemIwant").html();
return res.send(url + ":" + share);
} else {
console.log("We've encountered an error: " + error);
}
})
}
So everything is fine with this piece of code. What I would like to do is :
Using an array of url var urls = [url1,url2,url3,etc...]
Storing my scrapped data into another array, something like this data = [{url: url1, shares: share},{url: url2, shares: share},etc...]
I know I need to use something like this data.push({ urls: url, shares: share})})
and I understand that I need to loop over my first url array to push data into my second data array.
however I'm kind of lost with the request method and the way I should deal with async issue in my situation.
thanks !
edit#1 :
I tried this to use promises :
var url = "www.blablabla.com"
var geturl = request(url, function (error, response, body) {
if (!error) { return $ = cheerio.load(body) } else
{ console.log("We've encountered an error: " + error); }
});
var shares = geturl.then( function() {
return $(".nb-shares").html();
})
but got the following error geturl.then is not a function

I think you should use async:
var async = require('async');
var urls = ["http://example.com", "http://example.com", "http://example.com"];
var data = [];
var calls = urls.map((url) => (cb) => {
request(url, (error, response, body) => {
if (error) {
console.error("We've encountered an error:", error);
return cb();
}
var $ = cheerio.load(body),
share = $(".theitemIwant").html();
data.push({ url, share })
})
})
async.parallel(calls, () => { /* YOUR CODE HERE */ })
You could do the same with promises, but I don't see why.

I took a stab at it. You need to install the q library and require it to
var Q = require('q');
//... where ever your function is
//start with an array of string urls
var urls = [ "http://Blablablabla.com", '...', '...'];
//store results in this array in the form:
// {
// url: url,
// promise: <will be resolved when its done>,
// share:'code that you wanted'
// }
var results = [];
//loop over each url and perform the request
urls.forEach(processUrl);
function processUrl(url) {
//we use deferred object so we can know when the request is done
var deferred = Q.defer();
//create a new result object and add it to results
var result = {
url: url,
promise: deferred.promise
};
results.push(result);
//perform the request
request(url, function (error, response, body) {
if (!error) {
var $ = cheerio.load(body),
share = $(".theitemIwant").html();
//resolve the promise so we know this request is done.
// no one is using the resolve, but if they were they would get the result of share
deferred.resolve(share);
//set the value we extracted to the results object
result.share = share;
} else {
//request failed, reject the promise to abort the chain and fall into the "catch" block
deferred.reject(error)
console.log("We've encountered an error: " + error);
}
});
}
//results.map, converts the "array" to just promises
//Q.all takes in an array of promises
//when they are all done it rull call your then/catch block.
Q.all(results.map(function(i){i.promise}))
.then(sendResponse) //when all promises are done it calls this
.catch(sendError); //if any promise fails it calls this
function sendError(error){
res.status(500).json({failed: error});
}
function sendResponse(data){ //data = response from every resolve call
//process results and convert to your response
return res.send(results);
}

Here is another solution I like a lot :
const requestPromise = require('request-promise');
const Promise = require('bluebird');
const cheerio = require('cheerio');
const urls = ['http://google.be', 'http://biiinge.konbini.com/series/au-dela-des-murs-serie-herve-hadmar-marc-herpoux-critique/?src=konbini_home']
Promise.map(urls, requestPromise)
.map((htmlOnePage, index) => {
const $ = cheerio.load(htmlOnePage);
const share = $('.nb-shares').html();
let shareTuple = {};
shareTuple[urls[index]] = share;
return shareTuple;
})
.then(console.log)
.catch((e) => console.log('We encountered an error' + e));

Chaining Promises while maintaining data (angular js)

Ive seen that there are questions about chaining promises, but this one is a little bit different.
I'm making http get requests in my code. The first call returns an array. For each object in the array, i need to make another http call which returns another array and so on (this chains 3 levels deep).
The problem is, I need to keep track of which array element was used for to make each http call, and I dont know how to do this using promises.
I also want to end the chain by returning a promise.
I have the code for what I want to do written in nodejs without promises:
var https = require('https');
var fs = require('fs');
function makeRequest(options){
var httpopts = {
host: 'soc.courseoff.com',
path: '/gatech/terms/201601/majors/' + options.p,
method: 'GET'
};
var response = "";
var req = https.request(httpopts, function(res) {
res.on('data', function(d) {
response += d;
});
res.on('end',function(){
options.cb(response,options)
})
});
req.end();
req.on('error', function(e) {
console.error(e);
});
}
var classData = {};
function getCourses(m){
var majors = JSON.parse(m);
majors.forEach(function(maj){
classData[maj] = {};
var options = {
p:maj.ident +'/courses',
cb:getSections,
major:maj
};
makeRequest(options);
});
}
var classCount = 0;
function getSections(c,opts){
var courses = JSON.parse(c);
courses.forEach(function(course){
classCount++;
var options = JSON.parse(JSON.stringify(opts));
options.p += '/'+course.ident+'/sections';
options.course = course
options.cb = buildData
makeRequest(options)
});
}
var sectionCount = 0;
function buildData(r, options){
var major = options.major.ident;
sectionCount++;
if(!classData[major]){
classData[major] = {
name: options.major.name,
classes:{}
};
}
classData[major].classes[options.course.ident] = {
name:options.course.name,
sections:JSON.parse(r)
};
console.log('classCount-sectionCount '+classCount + '---'+sectionCount);
if(classCount === sectionCount){
writeIt();
}
}
makeRequest({
p:'',
cb:getCourses
});
function writeIt(){
fs.writeFileSync('./classData.js', 'module.exports = ' + JSON.stringify(classData));
}
EDIT:
I managed to get the promises to nest while keeping track of the data, but how can i return a promise that eventually resolves with the final data object?
My code:
Thanks four your help! I've managed to code it so that the promises work, my only problem now is in returning the final data as a promise
fact.factory('ClassFactory', ['$http',function ($http) {
var eventData = {};
var promise;
var courseData = [];
var baseURL ='https://soc.courseoff.com/gatech/terms/201601/majors/';
eventData.getClasses = function (event) {
if(!promise){
promise = $http.get(baseURL).then(
function(majors){
Promise.all(majors.data.map(m => $http.get(baseURL + m.ident+'/courses')
.then(
function(courses){
if(!m.courses) m.courses = [];
courses.data.map(c => $http.get(baseURL+ m.ident+'/courses/' +c.ident+'/sections' )
.then(
function(sections){
c.sections = sections.data;
m.courses.push(c);
}
));
courseData.push(m);
}
)));
}
)
}
return promise;
}
return eventData;
}]);

Almost certainly, each time you deal with an array of Promises, you'll want to use Promise.all in order to connect and merge your promises into a new promise. That promise will then contain an array of the results from each call. Nested Promise.alls can thus return Arrays of Arrays with all your levels of results as long as you use something like a map and a closure to capture the outer levels.
var fakeCall = x => Promise.resolve(x||Math.random());
Promise.all([fakeCall(1),fakeCall(2)])
.then(
results => Promise.all(results.map( x => fakeCall(5).then( results2 => [x, results2]) ))
)
.then( x => console.log(x));//-> [[1,5],[2,5]]
The first array of calls generates an array of results, and mapping over those with a function that makes yet more calls will return a single result that can be paired with its parent.
Explicitly nesting things in this way will work for even deeper levels, but is not going to be pretty. There's probably an abstraction you can create using Array.reduce which can generalize this pattern.

You forgot some returns in your code. The function you pass to .then should always return something. Also you are modifying majors but then throw it away without using it. When working with promises - especially when they are complex and nested - it's not a good idea to modify any data structures contained in those promises unless you are sure nothing bad can possibly happen.
I would split it into several functions.
e.g.
var baseURL ='https://soc.courseoff.com/gatech/terms/201601/majors/';
function getSections(major, course) {
return $http.get(baseURL+ major.ident+'/courses/' +course.ident+'/sections')
.then(sections => sections.data)
.catch(e => []);
}
function getCourses(major) {
return $http.get(baseURL + major.ident+'/courses')
.then(courses => Promise.all(courses.data.map(course =>
getSections(major, course).then(sections => ({[course.ident]: {name: course.name, sections: sections}})))))
.then(courses => angular.extend({}, ...courses))
.catch(e => ({}));
}
function getClassData() {
return $http.get(baseURL)
.then(majors => Promise.all(majors.data.map(major =>
getCourses(major).then(courses => ({[major.ident]: {name: major.name, classes: courses}})))))
.then(majors => angular.extend({}, ...majors))
.catch(e => ({}));
}
getClassData().then(data => console.log(data));