Develop Reference

Javascript - Chaining 2 (or more) arrays of promises

I have some code that does this: First scrape this array of webpages. After that, scrape another array of webpages.
The following code does what I expect:
let bays=[];
let promises=promisesN=[];
for (let y=2019;y>=2015;y--)
promises.push(new Promise(resolve=>
curl.get(`/*url*/${y}.html`,null, (error,resp,body)=>
resp.statusCode==200? resolve(parse(body)):reject(error)
)));
Promise.all(promises).then(()=>{
bays.forEach(bay=>{
if (bay.no.match(/\d+/)<=103) return;
promisesN.push(new Promise(resolve=>
curl.get(`/*url*/${bay.code}/`,null, (error,resp,body)=>
resp.statusCode==200? resolve(image(bey,body)):reject(error)
)))});
Promise.all(promisesN).then(()=>{
bays.sort((a,b)=>{return parseInt(a.no.match(/\d+/))<parseInt(b.no.match(/\d+/))? -1:1});
console.log(bays);
});
}).catch(error=>console.log(error));`
So I've read you can write a simplier nesting-free syntax:
doSomething()
.then(function(result) {
return doSomethingElse(result);
})
.then(function(newResult) {
return doThirdThing(newResult);
})
.then(function(finalResult) {
console.log('Got the final result: ' + finalResult);
})
.catch(failureCallback);
How to apply this to the code above?

correctness
let promises=promisesN=[];
This is really incorrect. It makes both variables reference the same array, and makes promisesN an implicit global. The fact that it appears to work means you aren’t in strict mode. Always use strict mode. The correct version of what you intended is:
let promises = [];
let promisesN = [];
cleanliness
new Promise(resolve=>
curl.get(`/*url*/${y}.html`,null, (error,resp,body)=>
resp.statusCode==200? resolve(parse(body)):reject(error)
))
You’re repeating this pattern, so make it into a function, or use a package that does the job for you, like request-promise[-native] or axios. (Also, please show your real code. reject isn’t defined here.)
const getAsync = url => new Promise((resolve, reject) => {
curl.get(url, null, (error, resp, body) => {
if (resp.statusCode === 200) {
resolve(body);
} else {
reject(error);
}
});
});
Notice how you’re free to make the function more readable when it isn’t repeated, and to extend it later.
let promises = [];
let promisesN = [];
for (let y = 2019; y >= 2015; y--) {
promises.push(getAsync(`/*url*/${y}.html`).then(parse));
}
Promise.all(promises).then(bays => {
bays.forEach(bay => {
if (bay.no.match(/\d+/) <= 103) return;
promisesN.push(getAsync(`/*url*/${bay.code}/`).then(body => image(bay, body)));
});
Promise.all(promisesN).then(() => {
bays.sort((a, b) => {return parseInt(a.no.match(/\d+/)) < parseInt(b.no.match(/\d+/)) ? -1 : 1;});
console.log(bays);
});
}).catch(error => console.log(error));
I had to take a few guesses at what your real code looks like again, because you’re surely doing something with the resolved value of Promise.all(promises). It doesn’t have any easily-accessible side-effects. bey also seemed likely enough to be bay.
Now you can give promisesN a more appropriate scope:
let promises = [];
for (let y = 2019; y >= 2015; y--) {
promises.push(getAsync(`/*url*/${y}.html`).then(parse));
}
Promise.all(promises).then(bays => {
let promisesN = bays
.filter(bay => bay.no.match(/\d+/) > 103)
.map(bay => getAsync(`/*url*/${bay.code}/`).then(body => image(bay, body)));
Promise.all(promisesN).then(() => {
bays.sort((a, b) => {return parseInt(a.no.match(/\d+/)) < parseInt(b.no.match(/\d+/)) ? -1 : 1;});
console.log(bays);
});
}).catch(error => console.log(error));
and use an expression-bodied arrow function where appropriate, since you’re already using them whenever they aren’t appropriate:
bays.sort((a, b) => parseInt(a.no.match(/\d+/)) < parseInt(b.no.match(/\d+/)) ? -1 : 1);
Now, if my guess about bays is right, then you can’t unnest. If it comes from somewhere else then you can. Normally I would leave a comment about that but I already wrote all this, so… please clarify that for further cleanup.

If you're looking to simplify your code, you might consider the use of async/await instead of promises.
The async/await syntax will greatly simplify the presentation and ease comprehension of the code, especially given that your logic relies on asynchronous iteration of arrays.
Consider the following code revision of your code:
/* Define local helper that wraps curl() in async function declaration */
function async doRequest(url) {
return (await new Promise(resolve=> curl.get(url, null, (error,resp,body) =>
resp.statusCode==200 ? resolve(res) : reject(error))))
}
/* Re-define simplified scrape logic using await/async */
function async doScrape() {
try {
var bays = []
/* Iterate date range asynchronously */
for (let y=2019; y>=2015; y--) {
/* Use doRequest helper function to fetch html */
const response = await doRequest(`/*url*/${y}.html`)
const bay = parse(response)
bays.push(bay)
}
/* Iterate bays array that was obtained */
for(const bay of bays) {
/* Use doRequest helper again to fetch data */
const response = await doRequest(`/*url*/${bay.code}/`)
/* Await may not be needed here */
await image(bay, response)
}
/* Perform your sort (which is non asynchronous) */
bays.sort((a,b)=> parseInt(a.no.match(/\d+/))<parseInt(b.no.match(/\d+/))? -1:1);
console.log("Result", bays);
}
catch(err) {
/* If something goes wrong we arrive here - this is
essentially equivalent to your catch() block */
console.error('Scrape failed', err);
}
}
/* Usage */
doScrape()
Hope that helps!

Not entirely sure if this is what you want, but I've separated your code out a bit because I found it easier for me to read.
let bays = [];
let promises = [];
let promisesN = [];
for (let y = 2019; y >= 2015; y--) {
const promiseOne = new Promise((resolve, reject) => {
return curl.get(`/*url*/${y}.html`, null, (error, resp, body) => {
resp.statusCode === 200 ? resolve(parse(body)) : reject(error);
});
});
promises.push(promiseOne);
}
Promise.all(promises)
.then(() => {
bays.forEach((bay) => {
if (bay.no.match(/\d+/) <= 103) {
return;
}
const promiseTwo = new Promise((resolve, reject) => {
return curl.get(`/*url*/${bay.code}/`, null, (error, resp, body) => {
resp.statusCode === 200 ? resolve(image(bay, body)) : reject(error);
});
});
promisesN.push(promiseTwo);
});
return Promise.all(promisesN);
})
.then(() => {
bays.sort((a, b) => {
return parseInt(a.no.match(/\d+/), 10) < parseInt(b.no.match(/\d+/), 10) ? -1 : 1;
});
console.log(bays);
})
.catch((error) => {
console.log(error);
});
I am wondering though, you are firing the promises instantly on each iteration of your for loop. This might be intentional, but it means if those promises resolve before the code gets to execute Promise.all you may run into issues. I personally would do something like, e.g. const promiseOne = () => somePromise, that way you can create a bunch of promises, and then once they're all created, map over that array and fire them at once. Same thing goes for the second promises.
Not sure if this is helpful, let me know if it is. Feel free to ask more questions too.

How can I turn my javascript callback flow into a Promise?

function getMentionedUsers(str, next){
var array = getUsernamesFromString(str); //['john','alex','jess'];
if(array.length > 0){
var users = [];
var pending = array.length;
array.forEach(function(username){
getUserByUsername(username).then(function(model){
users.push(model.key);
--pending || next(users); //this is a callback model
});
});
}
};
function getUserByUsername(username){
return admin.database().ref('/users').orderByChild('username').equalTo(username).once('value').then(function(snapshot) {
return snapshot.val(); //this is the firebase example of a promise
});
};
Right now, I'm doing this:
getMentionedUsers(model.body, function(users){
console.log("Mentions", users);
});
However, I'd like to turn getMentionedUsers into a promise. How can I do that? I'm new to Promises

You could use Promise.all and Array#map:
function getMentionedUsers(str) {
return Promise.all(getUsernamesFromString(str).map((username) => {
return getUserByUsername(username).then((model) => model.key);
}));
}
A more readable version broken into two functions:
function getUserKeyByUsername(username) {
return getUserByUsername(username).then((user) => user.key);
}
function getMentionedUsers(str) {
const promises = getUsernamesFromString(str).map(getUserKeyByUsername);
return Promise.all(promises);
}

Use Promise.all.
const getMentionedUsers = str =>
Promise.all(
getUsernamesFromString(str).map(
username => getUserByUsername(username)
.then(model => model.key)
)
);
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all

You can have the best of both. If you pass a next function, it will get called with the results. If not, your method will return a promise.
function getMentionedUsers(str, next){
var array = getUsernamesFromString(str); //['john','alex','jess'];
var promise = Promise.resolve([]); // default
var hasNext = typeof next === 'function';
if(array.length > 0){
promise = Promise.all(array.map(function(username){
return getUserByUsername(username);
}));
}
promise = promise.then(models => {
var users = models.map(model => model.key);
if (hasNext) next(null, users);
return users;
});
if (hasNext) promise.catch(next);
else return promise;
};
UPDATE: Though not part of your original question, this is still a good point and worth pointing out. Your existing code is using a non-standard callback technique. The standard callback technique expects an error as the first parameter and results as a second parameter:
next(new Error(...)); //-> when something fails
next(null, results); //-> when something succeeds
As such, I have updated my code to show the "standard" callback behavior alongside promises. Using the hybrid approach above allows for existing code to stay in place while allowing new code to use the new Promise technique. This would be considered a "non-breaking change".

using native ES6 promises, written in a functional style:
// Returns array of usernames
function getUsernamesFromString(str = '') {
return str.split(',').map(s => s.trim())
}
// returns promise of user
function getUserByUserName(username) {
// Lets say this is a slow async function and returns a promise
return Promise.resolve({
id: (Math.random() * 10 ** 10).toFixed(0),
username
});
}
function getMentionedUsers(str) {
return Promise.all(
getUsernamesFromString(str).map(getUserByUserName)
);
}
getMentionedUsers('kai, ava, mia, nova').then(console.log.bind(console))
However, there are also libraries like bluebird that can promisify objects and functions automatically as long as the follow the NODE convention of (err, result) as the callback arguments.
You can also just return a new Promise((resolve, reject) => { /* all your code */ }) and just call resolve(dataToResolveWith) if it succeeds and reject(new Error()) if it fails, but you rarely have to do that and in fact, it's an anti-pattern.

How to read array object in angularjs

I got this array objects to be read:
These was my sample codes:
$scope.obj_qst_local_study_main = tbl_qst_local_study_main.all();
$scope.quesion_id_allocated = $scope.obj_qst_local_study_main[0];
$timeout(function(){
console.log('----------all objects----------');
console.log($scope.obj_qst_local_study_main);
console.log('-----------one object-----------');
console.log($scope.quesion_id_allocated);
},200);
When I used:
$scope.obj_qst_local_study_main[0];
The result was: undefined
My angularjs services:
.service('tbl_qst_local_study_main', function($cordovaSQLite, DATABASE_LOCAL_NAME){
var self = this;
var qst_local_study_main_array = [];
self.all = function() {
var db = $cordovaSQLite.openDB({name: DATABASE_LOCAL_NAME,location:'default'});
$cordovaSQLite.execute(db, "SELECT * FROM qst_local_study_main")
.then(function (res) {
console.log('--------Successfully read from qst_local_study_main---------');
for (var i = 0; i < res.rows.length; i++) {
qst_local_study_main_array.push(res.rows.item(i));
}
},
function (err) {
console.log(err);
});
return qst_local_study_main_array;
};
})

Your service should return a Promise. This is a super common case, because (don't be offended please) people do not understand how Promises work.
Please search the internet for an article, like this one: https://developers.google.com/web/fundamentals/getting-started/primers/promises
tl;dr Your service should return a Promise. In your case $cordovaSQLite.execute Then you can correctly handle the response by chaining thens. You also do not need the timeout. Using a timeout is super bad here!
tbl_qst_local_study_main.all()
.then(function(result) {
console.log(result);
})

Chaining Promises while maintaining data (angular js)

Ive seen that there are questions about chaining promises, but this one is a little bit different.
I'm making http get requests in my code. The first call returns an array. For each object in the array, i need to make another http call which returns another array and so on (this chains 3 levels deep).
The problem is, I need to keep track of which array element was used for to make each http call, and I dont know how to do this using promises.
I also want to end the chain by returning a promise.
I have the code for what I want to do written in nodejs without promises:
var https = require('https');
var fs = require('fs');
function makeRequest(options){
var httpopts = {
host: 'soc.courseoff.com',
path: '/gatech/terms/201601/majors/' + options.p,
method: 'GET'
};
var response = "";
var req = https.request(httpopts, function(res) {
res.on('data', function(d) {
response += d;
});
res.on('end',function(){
options.cb(response,options)
})
});
req.end();
req.on('error', function(e) {
console.error(e);
});
}
var classData = {};
function getCourses(m){
var majors = JSON.parse(m);
majors.forEach(function(maj){
classData[maj] = {};
var options = {
p:maj.ident +'/courses',
cb:getSections,
major:maj
};
makeRequest(options);
});
}
var classCount = 0;
function getSections(c,opts){
var courses = JSON.parse(c);
courses.forEach(function(course){
classCount++;
var options = JSON.parse(JSON.stringify(opts));
options.p += '/'+course.ident+'/sections';
options.course = course
options.cb = buildData
makeRequest(options)
});
}
var sectionCount = 0;
function buildData(r, options){
var major = options.major.ident;
sectionCount++;
if(!classData[major]){
classData[major] = {
name: options.major.name,
classes:{}
};
}
classData[major].classes[options.course.ident] = {
name:options.course.name,
sections:JSON.parse(r)
};
console.log('classCount-sectionCount '+classCount + '---'+sectionCount);
if(classCount === sectionCount){
writeIt();
}
}
makeRequest({
p:'',
cb:getCourses
});
function writeIt(){
fs.writeFileSync('./classData.js', 'module.exports = ' + JSON.stringify(classData));
}
EDIT:
I managed to get the promises to nest while keeping track of the data, but how can i return a promise that eventually resolves with the final data object?
My code:
Thanks four your help! I've managed to code it so that the promises work, my only problem now is in returning the final data as a promise
fact.factory('ClassFactory', ['$http',function ($http) {
var eventData = {};
var promise;
var courseData = [];
var baseURL ='https://soc.courseoff.com/gatech/terms/201601/majors/';
eventData.getClasses = function (event) {
if(!promise){
promise = $http.get(baseURL).then(
function(majors){
Promise.all(majors.data.map(m => $http.get(baseURL + m.ident+'/courses')
.then(
function(courses){
if(!m.courses) m.courses = [];
courses.data.map(c => $http.get(baseURL+ m.ident+'/courses/' +c.ident+'/sections' )
.then(
function(sections){
c.sections = sections.data;
m.courses.push(c);
}
));
courseData.push(m);
}
)));
}
)
}
return promise;
}
return eventData;
}]);

Almost certainly, each time you deal with an array of Promises, you'll want to use Promise.all in order to connect and merge your promises into a new promise. That promise will then contain an array of the results from each call. Nested Promise.alls can thus return Arrays of Arrays with all your levels of results as long as you use something like a map and a closure to capture the outer levels.
var fakeCall = x => Promise.resolve(x||Math.random());
Promise.all([fakeCall(1),fakeCall(2)])
.then(
results => Promise.all(results.map( x => fakeCall(5).then( results2 => [x, results2]) ))
)
.then( x => console.log(x));//-> [[1,5],[2,5]]
The first array of calls generates an array of results, and mapping over those with a function that makes yet more calls will return a single result that can be paired with its parent.
Explicitly nesting things in this way will work for even deeper levels, but is not going to be pretty. There's probably an abstraction you can create using Array.reduce which can generalize this pattern.

You forgot some returns in your code. The function you pass to .then should always return something. Also you are modifying majors but then throw it away without using it. When working with promises - especially when they are complex and nested - it's not a good idea to modify any data structures contained in those promises unless you are sure nothing bad can possibly happen.
I would split it into several functions.
e.g.
var baseURL ='https://soc.courseoff.com/gatech/terms/201601/majors/';
function getSections(major, course) {
return $http.get(baseURL+ major.ident+'/courses/' +course.ident+'/sections')
.then(sections => sections.data)
.catch(e => []);
}
function getCourses(major) {
return $http.get(baseURL + major.ident+'/courses')
.then(courses => Promise.all(courses.data.map(course =>
getSections(major, course).then(sections => ({[course.ident]: {name: course.name, sections: sections}})))))
.then(courses => angular.extend({}, ...courses))
.catch(e => ({}));
}
function getClassData() {
return $http.get(baseURL)
.then(majors => Promise.all(majors.data.map(major =>
getCourses(major).then(courses => ({[major.ident]: {name: major.name, classes: courses}})))))
.then(majors => angular.extend({}, ...majors))
.catch(e => ({}));
}
getClassData().then(data => console.log(data));

Is it possible to asynchronously collect items from a generator into an array?

I'm playing around with writing a web service using Node.js/Express which generates some objects based on templates and then returns the generated data. I'm using Bluebird promises to manage all the async logic. After stripping out all the unimportant stuff, my code looks something like this[1].
My problem is the core logic can block for several seconds if the requested number of output elements is large. Since I've been playing with ES6 for this project, my first thought was to factor out the element creation into a generator[2]. However, the only way I can find to get all the results from this generator is Array.from, which doesn't help with the blocking.
I've played around with .map, .all, .coroutine, and a couple of other things, in an attempt to asynchronously collect the results from the generator, but I haven't had any luck. Is there any nice way to do this with Bluebird? (Or perhaps a better way of doing it altogether?)
Native ES6 Promise.all can take an iterator and give back an array of values, but V8 doesn't support this yet. Also, in my experimentation with polyfills/Firefox, it seems to be synchronous.
This is not-too-common operation, so I don't care much about absolute performance. I just want to avoid blocking the event queue, and I would prefer a nice, easy to read and maintain solution.
[1]:
let Bluebird = require('bluebird');
let templates = ...; // logic to load data templates
function createRandomElementFromRandomTemplate(templates) {
let el;
// synchronous work that can take a couple of milliseconds...
return el;
};
api.createRandomElements = function(req, res) {
let numEls = req.params.numEls;
Bluebird.resolve(templates)
.then(templates => {
let elements = [];
// numEls could potentially be several thousand
for(let i = 0; i < numEls; ++i) {
elements.push(createRandomElementFromRandomTemplate(templates));
}
return elements;
})
.then(elements => {
res.json(elements);
})
.error(err => {
res.status(500).json(err);
});
}
[2]:
function* generateRandomElementsFromRandomTemplate(templates, numEls) {
for(let i = 0; i < numEls; ++i) {
let el;
// synchronous work that can take a couple of milliseconds...
yield el;
}
}
api.createRandomElements = function(req, res) {
let numEls = req.params.numEls;
Bluebird.resolve(templates)
.then(templates => {
// this still blocks
return Array.from(generateRandomElementsFromRandomTemplate(templates, numEls));
})
.then(elements => {
res.json(elements);
})
.error(err => {
res.status(500).json(err);
});
}

Here's a halfway-decent solution I found after looking more closely at Bluebird's .map() as Benjamin suggested. I still have the feeling I'm missing something, though.
The main reason I started with Bluebird was because of Mongoose, so I left a bit of that in for a more realistic sample.
let Bluebird = require('bluebird');
let mongoose = require('mongoose');
Bluebird.promisifyAll(mongoose);
const Template = mongoose.models.Template,
UserPref = mongoose.models.UserPref;
// just a normal function that generates one element with a random choice of template
function createRandomElementFromRandomTemplate(templates, userPrefs) {
let el;
// synchronous work that can take a couple of milliseconds...
return el;
}
api.generate = function(req, res) {
let userId = req.params.userId;
let numRecord = req.params.numRecords
let data;
Bluebird.props({
userprefs: UserPref.findOneAsync({userId: userId}),
templates: Template.findAsync({})
})
.then(_data => {
data = _data;
// use a sparse array to convince .map() to loop the desired number of times
return Array(numRecords);
})
.map(() => {
// ignore the parameter map passes in - we're using the exact same data in each iteration
// generate one item each time and let Bluebird collect them into an array
// I think this could work just as easily with a coroutine
return Bluebird.delay(createRandomElementFromRandomTemplate(data.templates, data.userprefs), 0);
}, {concurrency: 5})
.then(generated => {
return Generated.createAsync(generated);
})
.then(results => {
res.json(results);
})
.catch(err => {
console.log(err);
res.status(500);
});
};