I need to make a request to get a list of html, and I need to scan it and loop through it and make more requests for each item in the list found, and those might have lists inside them, and so on till theres none left.
I need a way to keep track of all the requests called and call another function when they're done. The tricky bit is the function calls itself over and over for any list items found in the HTML.
The problem I'm having is using Q promises the only promises it waits for are from first request made, and I cant understand why assuming node works like I think it does, see code:
var _ = require('underscore');
var request = require('request');
var allPromises = [];
var finalArray = [];
var start = function(id) {
var deferred = Q.defer();
request.get({
url: 'http://www.example.com/id/' + id
}, function() {
_.each(body.items, function(index) {
var item = this;
finalArray.push(item);
if(item.hasMore) {
start(item.id);
}
}
deferred.resolve();
});
allPromises.push(deferred.promise);
}
console.log('Starting');
start(1);
Q.all(allPromises).done(function (values) {
console.log('All Done');
});
What I thought happens is:
1 - starts() is called for the first time and the first deferred var is created
2 - the first request is made and the first created deferred variable is pushed to the promises array
3 - Q.all is called and waits
4 - The first request's callback is called
5 - if the request contains body.x, start() is called again with a new id
6 - a new promises is created and pushed and a new request is made
7 - the first promise is resolved
assuming this only went one level deep
8 - the second promise is resolved
9 - Q.all calls its callback
but in practice, Q.all calls its callback after the first promise, it doesn't wait for any others even though the second promise is pushed before the first promise is resolved.
Why? And how can I make this work?
Update forgot to add the loop inside the request callback.
Answer to edited question:
var request = require('request');
var finalArray = [];
var start = function(id) {
var deferred = Q.defer();
request.get({
url: 'http://www.example.com/id/' + id
}, function() {
var subitems = [];
_.each(body.items, function(index) {
var item = this;
finalArray.push(item);
if(item.hasMore) {
subitems.push(start(item.id));
}
}
if (subitems.length) {
deferred.resolve(Q.all(subitems)); // resolve result of Q.all
} else {
deferred.resolve();
}
});
return deferred.promise;
}
start(1).done(function() {
console.log('All Done');
});
#Bergi's code
var request = require('request');
var start = function(id) {
var deferred = Q.defer();
request.get({
url: 'http://www.example.com/id/' + id
}, function(err, body) {
if (err) deferred.reject(err);
else deferred.resolve(body);
});
return deferred.promise.then(function(body) {
var finalArray = [];
return Q.all(_.map(body.items, function(index) {
var item = this;
finalArray.push(item);
if(item.hasMore)
return start(item.id);
else
return [];
})).then(function(moreResults) {
return finalArray.concat.apply(finalArray, moreResults);
});
});
}
start(1).then(function(finalArray) {
console.log('All Done');
});
Related
Problem is I'm not able to get the promises to return anything. they.. just come empty.
Every answer I see here on SO is telling me to do just this, though for some reason this is not working. I'm at my wits end, pulling hair and smashing keyboards; Can someone pin-point my dumbness?
var q = require('q');
var request = require('request-promise'); // https://www.npmjs.com/package/request-promise
function findSynonym(searchList) {
var defer = q.defer();
var promises = [];
var url = "http://thesaurus.altervista.org/service.php?word=%word%&language=en_US&output=json&key=awesomekeyisawesome";
var wURL;
searchList.forEach(function(word){
wURL = url.replace('%word%',word);
promises.push(request(wURL));
});
q.all(promises).then(function(data){
console.log('after all->', data); // data is empty
defer.resolve();
});
return defer;
}
var search = ['cookie', 'performance', 'danger'];
findSynonym(search).then(function(supposedDataFromAllPromises) { // TypeError: undefined is not a function [then is not a function]
console.log('->',supposedDataFromAllPromises); // this never happens
});
You're returning the Deferred object defer, which does not have a .then method, instead of the Promise object defer.promise.
But anyway, that's the deferred antipattern, there's no need of using deferreds here. Just return the promise that Promise.all gets you:
function findSynonym(searchList) {
var url = "http://thesaurus.altervista.org/service.php?word=%word%&language=en_US&output=json&key=awesomekeyisawesome";
var promises = searchList.map(function(word) {
return request(url.replace('%word%', word));
});
return q.all(promises).then(function(data){
console.log('after all->', data); // data is empty
return undefined; // that's what you were resolve()ing with
});
}
So, turns out I was resolving the promises or something something. returning the q.all() worked pretty well :)
function findSynonym(searchList) {
var promises = [];
var url = "http://thesaurus.altervista.org/service.php?word=%word%&language=en_US&output=json&key=REDACTED";
var wURL;
searchList.forEach(function(word){
wURL = url.replace('%word%',word);
promises.push(request({url:wURL}));
});
return q.all(promises);
}
var search = ['cookie', 'performance', 'danger'];
findSynonym(search)
.then(function(a){
console.log('->',a);
});
I use the node request ajax package. So, i have an loop, in every iteration it makes an request to my server.
// realItems needs the complete value of items assigned
var realItems;
var items = [];
_.forEach(JSON.parse(body), (value, key) => {
request('myurl/' + id, (error, response, body) => {
items = JSON.parse(body)
});
});
How can i bundle all my requests from request package, so I can assign the value of items variable to the realItems at the end?
// edit:
I use react js, so in this case realItems is an state, and i can't trigger it in every loop iteration, because render triggers on every setState
There are a number of ways to approach this. Here's a brute force method that does not preserve the order of the results:
var items = [];
var cnt = 0;
_.forEach(JSON.parse(body), (value, key) => {
++cnt;
request('myurl/' + value.id, (error, response, body) => {
items.push(JSON.parse(body));
// if all requesets are done
if (--cnt === 0) {
// process items here as all results are done now
}
});
});
Here's a version that uses Bluebird promises:
var Promise = require('bluebird');
var request = Promise.promisify(require("request"));
Promise.promisifyAll(request);
var promises = [];
_.forEach(JSON.parse(body), (value, key) => {
promises.push(request('myurl/' + value.id));
});
Promise.all(promises).then(function(results) {
// all requests are done, data from all requests is in the results array
// and are in the order that the requests were originally made
});
And, here's a little bit simpler Bluebird promises method that uses a Bluebird iterator:
var Promise = require('bluebird');
var request = Promise.promisify(require("request"));
Promise.promisifyAll(request);
Promise.map(JSON.parse(body), function(value) {
return request('myurl/' + value.id);
}).then(function(results) {
// all requests are done, data is in the results array
});
Is it a requirement that you use the request package? I use async which is similar and comes with a parallel method which does exactly what you're asking -
https://github.com/caolan/async#parallel
example:
async.parallel([
function(callback){
setTimeout(function(){
callback(null, 'one');
}, 200);
},
function(callback){
setTimeout(function(){
callback(null, 'two');
}, 100);
}
],
// optional callback
function(err, results){
// the results array will equal ['one','two'] even though
// the second function had a shorter timeout.
});
I have this code in a factory:
getAyahsByJuz: function (juzIndex) {
var response = [];
var promises = [];
var self = this;
var deferred = $q.defer();
$timeout(function () {
$http.get('data/quran.json').success(function (data) {
var ayahs = Quran.ayah.listFromJuz(juzIndex);
angular.forEach(ayahs, function (value, key) {
var promise = self.getVerse(value.surah, value.ayah).then(function (res) {
var verse = {
surah: value.surah,
ayah: value.ayah,
text: res
};
response.push(verse);
}, function (err) {
console.log(err);
});
promises.push(promise);
});
});
}, 30);
$q.all(promises).then(function() {
deferred.resolve(response);
});
return deferred.promise;
},
Please note that everything is working fine the verse object is returning properly. However, when I use this in a controller using .then(res). res returns [] instead of the array filled with the verse objects.
Can anyone point out why? Thanks!
The short answer is because your $q.all runs before $timeout & before the $http embedded in $timeout. Let's boil your original code down to its relevant components:
getAyahsByJuz: function (juzIndex) {
var response = [];
var promises = [];
var deferred = $q.defer();
// ...irrelevant stuff that will happen after a $timeout
// this happens IMMEDIATELY (before $timeout):
$q.all(promises).then(function() { // wait for empty promise array
deferred.resolve(response); // resolve with empty response array
}); // side note: this is a broken chain! deferred.promise can't reject
return deferred.promise; // send promise for empty array
}
See the problem? If for some odd reason you need to keep that $timeout, here's the fix with substantial promise refactoring & removing the awful jquery-inspired non-promisy success syntax):
getAyahsByJuz: function (juzIndex) {
var self = this;
// $timeout itself returns a promise which we can post-process using its callback return value
return $timeout(function () {
// returning the $http promise modifies the $timeout promise
return $http.get('data/quran.json').then(function (response) { // you never used this response!
var versePromises = [];
var ayahs = Quran.ayah.listFromJuz(juzIndex);
angular.forEach(ayahs, function (value, key) {
// we'll push all versePromises into an array…
var versePromise = self.getVerse(value.surah, value.ayah).then(function (res) {
// the return value of this `then` modifies `versePromise`
return {
surah: value.surah,
ayah: value.ayah,
text: res
};
});
versePromises.push(versePromise);
});
return $q.all(versePromises); // modifies $http promise — this is our ultimate promised value
// if a versePromise fails, $q.all will fail; add a `catch` when using getAyahsByJuz!
});
}, 30);
}
However, there is still a huge issue here… why aren't you using the server response of your $http call anywhere? What is the point of that first call?
Also I find that $timeout to be extremely suspicious. If you need it then it's likely there's something bad going on elsewhere in the code.
I have done a lot of reading around this, but ultimately the tutorials and guides I have found differ too much for me to get a decent grasp on this concept.
This is what I want to achieve:
1) Simple http request from our server [Any API for demonstration]
2) Run a function with data from (1). [Remove a property from the object]
3) Use result and length of (2) to run a loop of $http requests to our server. [Or any server]
4) This will result in 6 different objects. Run a function on these 6 objects. [Add a property]
5) Once ALL of this is done, run a separate function [Log "finished"]
How can this be achieved using promises? How do I pass data from (1) via a promise to (2)? Is this the right way to achieve what I need to do?
If anyone can show me how this should be structured it would be immensely helpful; I have kept the functions as simple as possible for this question.
Yes, promises are very nice to structure solutions for this kind of problems.
Simplified solution (more or less pseudo-code):
$http(...)
.then(function(response) {
// do something with response, for example:
var list = reponse.data.list;
// return it so that you can use it in the next 'then'.
return list;
})
.then(function(list) {
var promises = [];
angular.forEach(list, function(item) {
// perform a request for each item
var promise = $http(...).then(function(itemResponse) {
itemResponse.extraProperty = true;
return itemResponse;
});
// we make an array of promises
promises.push(promise);
});
// combine all promises into one and return it for the next then()
return $q.all(promises);
})
.then(function(itemsList) {
// itemsList is now an array of all parsed item responses.
console.log(itemsList);
});
(Hopefully this is right, I did not tested it.)
As you can see, you can return values in a callback to pass it to the next then(), or you can pass a promise, and this will result in calling the next callback when it resolves. $q.all() is used to combine multiple promises into one and resolve if all are resolved.
Edit: I realised that you can optionally leave out these three lines:
return list;
})
.then(function(list) {
But it is nice syntax though, because the separation of tasks is more visible.
Check code below, it could contains syntax error, the important is the structure. Step3 contains multiple(6) $http requests, it waits until the last request response to return a unique response object (array) containing response for each $http requets.
//Step 1
var Step1 = function () {
$http.get('api/controller').success(function (resp) {
var object1 = resp;
Step2(object1);
Step3(object1).then(function (resp) {
//resp.data is an array containing the response of each $http request
Step4(resp);
Step5();
});
});
}
//Step2
var Step2 = function(obj){
//do whatever with the object
}
//Step3
var Step3 = function (object1) {
var call = $q.defer();
var get1 = $http.get(object1[0].url);
var get2 = $http.get(object[1].url2);
//...
var get6 = $http.get(object[5].url6);
$q.all([get1, get2,..get6]).then(function (resp) {
call.resolve(resp);
});
return call.promise;
}
//Step4
var Step4 = function (resp) {
for (var i=0; i<resp.data.lenght;i++){
DoWhatEver(resp.data[i]);
};
}
//Step5
var Step5 = function () {
alert("Finished");
}
Step1(); //Call Step1 function
Don't know why you have difficulty implementing this, but maybe $q.all() is what you're missing:
var config1={method:'GET',url:'/api/...'};
$http(config1).success(function(resultsFrom1){
functionForResultsOf1(resultsFrom1);
})
var functionForResultsOf1 = function(resultsOf1){
//remove something from the result, assuming this is a synchronous operation
resultsOf1.splice()...;
var promises=makePromises(*pass whatever you want*);
$q.all(promises).then(function(aggregateOfAllCallsToServer){
angular.forEach(aggregateOfAllCallsToServer,function(data){
//do something to data from each call to the server
})
console.log("finished");
})
}
var makePromises = function(serverUrls){
var promises = [];
angular.forEach(serverUrls, function(url) {
var promise=$http({
url : '/api/'+url,
method: 'GET',
})
promises.push(promise);
});
return $q.all(promises);
}
I have a service SQLService on my PhoneGap/AngularJS app that runs when the app is loading. It iterates through a long array of Guidelines, and makes a DB transaction for each one. How can I signal that the final transaction has been completed?
What I want to have happen is something like:
In the controller, call `SQLService.ParseJSON`
ParseJSON calls `generateIntersectionSnippets`
`generateIntersectionSnippets` makes multiple calls to `getKeywordInCategory``
When the final call to getKeywordInCategory is called, resolve the whole chain
SQLService.ParseJSON is complete, fire `.then`
I really don't understand how to combine the multiple asynchronous calls here. ParseJSON returns a promise which will be resolved when generateIntersectionSnippets() is completed, but generateIntersectionSnippets() makes multiple calls to getKeywordInCategory which also returns promises.
Here's a simplified version of what's not working (apologies for any misplaced brackets, this is very stripped down).
What I want to happen is for $scope.ready = 2 to run at the completion of all of the transactions. Right now, it runs as soon as the program has looped through generateIntersectionSnippets once.
in the controller:
SQLService.parseJSON().then(function(d) {
console.log("finished parsing JSON")
$scope.ready = 2;
});
Service:
.factory('SQLService', ['$q',
function ($q) {
function parseJSON() {
var deferred = $q.defer();
function generateIntersectionSnippets(guideline, index) {
var snippet_self, snippet_other;
for (var i = 0; i < guideline.intersections.length; i++) {
snippet_self = getKeywordInCategory(guideline.line_id, snippets.keyword).then(function() {
//Should something go here?
});
snippet_other = getKeywordInCategory(guideline.intersections[i].line_id, snippets.keyword).then(function() {
//Should something go here?
});
}
}
deferred.resolve(); //Is fired before the transactions above are complete
}
generateIntersectionSnippets();
return deferred.promise;
} //End ParseJSON
function getKeywordInCategory(keyword, category) {
var deferred = $q.defer();
var query = "SELECT category, id, chapter, header, snippet(guidelines, '<b>', '</b>', '...', '-1', '-24' ) AS snip FROM guidelines WHERE content MATCH '" + keyword + "' AND id='" + category + "';",
results = [];
db.transaction(function(transaction) {
transaction.executeSql(query, [],
function(transaction, result) {
if (result != null && result.rows != null) {
for (var i = 0; i < result.rows.length; i++) {
var row = result.rows.item(i);
results.push(row);
}
}
},defaultErrorHandler);
deferred.resolve(responses);
},defaultErrorHandler,defaultNullHandler);
return deferred.promise;
}
return {
parseJSON : parseJSON
};
}]);
I'd appreciate any guidance on what the correct model is to doing a chain of promises that includes an iteration across multiple async transactions- I know that how I have it right now is not correct at all.
You can use $q.all() to wait for a list of promises to be resolved.
function parseJSON() {
var deferred = $q.defer();
var promiseList = [];
for (var i = 0; i < guideline.intersections.length; i++) {
promiseList.push(getKeywordInCategory(guideline.line_id, snippets.keyword));
promiseList.push(getKeywordInCategory(guideline.intersections[i].line_id, snippets.keyword));
}
$q.all(promiseList).then(function() {
deferred.resolve();
});
return deferred.promise;
} //End ParseJSON