I have a promise call that is supposed to do the following:
1) Get a link from completedLinks
2) Use cheerio to load the html into the $, like jQuery.
3) get the title, and other details from the page
4) save those details to the object siteObj
JS:
completedLinks = ["http://www.example.com/1","http://www.example.com/2","http://www.example.com/3"...]
function fetchSiteDetails(arr) {
return arr.reduce(function(promise, link) {
console.log(link);
var siteObj = {};
var $ = cheerio.load(link);
var titles = $("title").text().trim();
var html = $("#content").html().trim();
var mainContent = $(".left-side").html().trim();
var allLinks = $('a');
siteObj.title = titles;
siteObj.html = html;
siteObj.mainContent = mainContent;
siteObj.subPages = [];
allLinks.each(function(i, elem) {
siteObj.subPages.push($(this).attr('href'));
});
return promise;
}, Promise.resolve());
}
fetchSiteDetails(completedLinks).then(function() {
console.log('all done');
});
Right now it is not completing each task before it starts the next, and finishes before it's done anything.
As I can't see what is asynchronous in your code, here's a generic pattern to run asynchronous operations in series using promises and array.reduce
return arr.reduce(function(promise, link) {
return promise.then(function(resultOfPreviousPromise) {
// your code goes here
// return a promise that resolves when the asynchronous function completes
});
}, Promise.resolve());
What will interest you is the bit where that says "your code goes here" - which is where you should put your code. The return value needs to be a promise that is resolved when the asynchronous operation completes
Related
Let's say I have a file containing certain promises, that when executed in order, prepares an input file input.txt.
// prepareInput.js
var step1 = function() {
var promise = new Promise(function(resolve, reject) {
...
});
return promise;
};
var step2 = function() {
var promise = new Promise(function(resolve, reject) {
...
});
return promise;
};
var step3 = function() {
var promise = new Promise(function(resolve, reject) {
...
});
return promise;
};
step1().then(step2).then(step3);
exports.fileName = "input.txt";
If I run node prepareInput.js, the line step1().then(step2).then(step3) gets executed and creates the file.
How can I alter this so that when other files attempt to retrieve fileName from this module, step1().then(step2).then(step3); is run and completed before fileName is exposed? Something along the line of:
// prepareInput.js
...
exports.fileName =
step1().then(step2).then(step3).then(function() {
return "input.txt";
});
// main.js
var prepareInput = require("./prepareInput");
var inputFileName = require(prepareInput.fileName);
Node.js beginner here; apologize beforehand if my approach makes completely no sense... :)
You can't directly export the results of something retrieved asynchronously because export is synchronous, so it happens before any async results have been retrieved. The usual solution here is to export a method that returns a promise. The caller can then call that method and use that promise to get the desired async result.
module.exports = function() {
return step1().then(step2).then(step3).then(function() {
// based on results of above three operations,
// return the filename here
return ...;
});
}
The caller then does this:
require('yourmodule')().then(function(filename) {
// use filename here
});
One thing to keep is mind is that if any operation in a sequence of things is asynchronous, then the whole operation becomes asynchronous and no caller can then fetch the result synchronously. Some people refer to asynchronous as "contagious" in this way. So, if any part of your operation is asynchronous, then you must create an asynchronous interface for the eventual result.
You can also cache the promise so it is only ever run once per app:
module.exports = step1().then(step2).then(step3).then(function() {
// based on results of above three operations,
// return the filename here
return ...;
});
The caller then does this:
require('yourmodule').then(function(filename) {
// use filename here
});
I need to make a request to get a list of html, and I need to scan it and loop through it and make more requests for each item in the list found, and those might have lists inside them, and so on till theres none left.
I need a way to keep track of all the requests called and call another function when they're done. The tricky bit is the function calls itself over and over for any list items found in the HTML.
The problem I'm having is using Q promises the only promises it waits for are from first request made, and I cant understand why assuming node works like I think it does, see code:
var _ = require('underscore');
var request = require('request');
var allPromises = [];
var finalArray = [];
var start = function(id) {
var deferred = Q.defer();
request.get({
url: 'http://www.example.com/id/' + id
}, function() {
_.each(body.items, function(index) {
var item = this;
finalArray.push(item);
if(item.hasMore) {
start(item.id);
}
}
deferred.resolve();
});
allPromises.push(deferred.promise);
}
console.log('Starting');
start(1);
Q.all(allPromises).done(function (values) {
console.log('All Done');
});
What I thought happens is:
1 - starts() is called for the first time and the first deferred var is created
2 - the first request is made and the first created deferred variable is pushed to the promises array
3 - Q.all is called and waits
4 - The first request's callback is called
5 - if the request contains body.x, start() is called again with a new id
6 - a new promises is created and pushed and a new request is made
7 - the first promise is resolved
assuming this only went one level deep
8 - the second promise is resolved
9 - Q.all calls its callback
but in practice, Q.all calls its callback after the first promise, it doesn't wait for any others even though the second promise is pushed before the first promise is resolved.
Why? And how can I make this work?
Update forgot to add the loop inside the request callback.
Answer to edited question:
var request = require('request');
var finalArray = [];
var start = function(id) {
var deferred = Q.defer();
request.get({
url: 'http://www.example.com/id/' + id
}, function() {
var subitems = [];
_.each(body.items, function(index) {
var item = this;
finalArray.push(item);
if(item.hasMore) {
subitems.push(start(item.id));
}
}
if (subitems.length) {
deferred.resolve(Q.all(subitems)); // resolve result of Q.all
} else {
deferred.resolve();
}
});
return deferred.promise;
}
start(1).done(function() {
console.log('All Done');
});
#Bergi's code
var request = require('request');
var start = function(id) {
var deferred = Q.defer();
request.get({
url: 'http://www.example.com/id/' + id
}, function(err, body) {
if (err) deferred.reject(err);
else deferred.resolve(body);
});
return deferred.promise.then(function(body) {
var finalArray = [];
return Q.all(_.map(body.items, function(index) {
var item = this;
finalArray.push(item);
if(item.hasMore)
return start(item.id);
else
return [];
})).then(function(moreResults) {
return finalArray.concat.apply(finalArray, moreResults);
});
});
}
start(1).then(function(finalArray) {
console.log('All Done');
});
Problem is I'm not able to get the promises to return anything. they.. just come empty.
Every answer I see here on SO is telling me to do just this, though for some reason this is not working. I'm at my wits end, pulling hair and smashing keyboards; Can someone pin-point my dumbness?
var q = require('q');
var request = require('request-promise'); // https://www.npmjs.com/package/request-promise
function findSynonym(searchList) {
var defer = q.defer();
var promises = [];
var url = "http://thesaurus.altervista.org/service.php?word=%word%&language=en_US&output=json&key=awesomekeyisawesome";
var wURL;
searchList.forEach(function(word){
wURL = url.replace('%word%',word);
promises.push(request(wURL));
});
q.all(promises).then(function(data){
console.log('after all->', data); // data is empty
defer.resolve();
});
return defer;
}
var search = ['cookie', 'performance', 'danger'];
findSynonym(search).then(function(supposedDataFromAllPromises) { // TypeError: undefined is not a function [then is not a function]
console.log('->',supposedDataFromAllPromises); // this never happens
});
You're returning the Deferred object defer, which does not have a .then method, instead of the Promise object defer.promise.
But anyway, that's the deferred antipattern, there's no need of using deferreds here. Just return the promise that Promise.all gets you:
function findSynonym(searchList) {
var url = "http://thesaurus.altervista.org/service.php?word=%word%&language=en_US&output=json&key=awesomekeyisawesome";
var promises = searchList.map(function(word) {
return request(url.replace('%word%', word));
});
return q.all(promises).then(function(data){
console.log('after all->', data); // data is empty
return undefined; // that's what you were resolve()ing with
});
}
So, turns out I was resolving the promises or something something. returning the q.all() worked pretty well :)
function findSynonym(searchList) {
var promises = [];
var url = "http://thesaurus.altervista.org/service.php?word=%word%&language=en_US&output=json&key=REDACTED";
var wURL;
searchList.forEach(function(word){
wURL = url.replace('%word%',word);
promises.push(request({url:wURL}));
});
return q.all(promises);
}
var search = ['cookie', 'performance', 'danger'];
findSynonym(search)
.then(function(a){
console.log('->',a);
});
I have done a lot of reading around this, but ultimately the tutorials and guides I have found differ too much for me to get a decent grasp on this concept.
This is what I want to achieve:
1) Simple http request from our server [Any API for demonstration]
2) Run a function with data from (1). [Remove a property from the object]
3) Use result and length of (2) to run a loop of $http requests to our server. [Or any server]
4) This will result in 6 different objects. Run a function on these 6 objects. [Add a property]
5) Once ALL of this is done, run a separate function [Log "finished"]
How can this be achieved using promises? How do I pass data from (1) via a promise to (2)? Is this the right way to achieve what I need to do?
If anyone can show me how this should be structured it would be immensely helpful; I have kept the functions as simple as possible for this question.
Yes, promises are very nice to structure solutions for this kind of problems.
Simplified solution (more or less pseudo-code):
$http(...)
.then(function(response) {
// do something with response, for example:
var list = reponse.data.list;
// return it so that you can use it in the next 'then'.
return list;
})
.then(function(list) {
var promises = [];
angular.forEach(list, function(item) {
// perform a request for each item
var promise = $http(...).then(function(itemResponse) {
itemResponse.extraProperty = true;
return itemResponse;
});
// we make an array of promises
promises.push(promise);
});
// combine all promises into one and return it for the next then()
return $q.all(promises);
})
.then(function(itemsList) {
// itemsList is now an array of all parsed item responses.
console.log(itemsList);
});
(Hopefully this is right, I did not tested it.)
As you can see, you can return values in a callback to pass it to the next then(), or you can pass a promise, and this will result in calling the next callback when it resolves. $q.all() is used to combine multiple promises into one and resolve if all are resolved.
Edit: I realised that you can optionally leave out these three lines:
return list;
})
.then(function(list) {
But it is nice syntax though, because the separation of tasks is more visible.
Check code below, it could contains syntax error, the important is the structure. Step3 contains multiple(6) $http requests, it waits until the last request response to return a unique response object (array) containing response for each $http requets.
//Step 1
var Step1 = function () {
$http.get('api/controller').success(function (resp) {
var object1 = resp;
Step2(object1);
Step3(object1).then(function (resp) {
//resp.data is an array containing the response of each $http request
Step4(resp);
Step5();
});
});
}
//Step2
var Step2 = function(obj){
//do whatever with the object
}
//Step3
var Step3 = function (object1) {
var call = $q.defer();
var get1 = $http.get(object1[0].url);
var get2 = $http.get(object[1].url2);
//...
var get6 = $http.get(object[5].url6);
$q.all([get1, get2,..get6]).then(function (resp) {
call.resolve(resp);
});
return call.promise;
}
//Step4
var Step4 = function (resp) {
for (var i=0; i<resp.data.lenght;i++){
DoWhatEver(resp.data[i]);
};
}
//Step5
var Step5 = function () {
alert("Finished");
}
Step1(); //Call Step1 function
Don't know why you have difficulty implementing this, but maybe $q.all() is what you're missing:
var config1={method:'GET',url:'/api/...'};
$http(config1).success(function(resultsFrom1){
functionForResultsOf1(resultsFrom1);
})
var functionForResultsOf1 = function(resultsOf1){
//remove something from the result, assuming this is a synchronous operation
resultsOf1.splice()...;
var promises=makePromises(*pass whatever you want*);
$q.all(promises).then(function(aggregateOfAllCallsToServer){
angular.forEach(aggregateOfAllCallsToServer,function(data){
//do something to data from each call to the server
})
console.log("finished");
})
}
var makePromises = function(serverUrls){
var promises = [];
angular.forEach(serverUrls, function(url) {
var promise=$http({
url : '/api/'+url,
method: 'GET',
})
promises.push(promise);
});
return $q.all(promises);
}
I have a service SQLService on my PhoneGap/AngularJS app that runs when the app is loading. It iterates through a long array of Guidelines, and makes a DB transaction for each one. How can I signal that the final transaction has been completed?
What I want to have happen is something like:
In the controller, call `SQLService.ParseJSON`
ParseJSON calls `generateIntersectionSnippets`
`generateIntersectionSnippets` makes multiple calls to `getKeywordInCategory``
When the final call to getKeywordInCategory is called, resolve the whole chain
SQLService.ParseJSON is complete, fire `.then`
I really don't understand how to combine the multiple asynchronous calls here. ParseJSON returns a promise which will be resolved when generateIntersectionSnippets() is completed, but generateIntersectionSnippets() makes multiple calls to getKeywordInCategory which also returns promises.
Here's a simplified version of what's not working (apologies for any misplaced brackets, this is very stripped down).
What I want to happen is for $scope.ready = 2 to run at the completion of all of the transactions. Right now, it runs as soon as the program has looped through generateIntersectionSnippets once.
in the controller:
SQLService.parseJSON().then(function(d) {
console.log("finished parsing JSON")
$scope.ready = 2;
});
Service:
.factory('SQLService', ['$q',
function ($q) {
function parseJSON() {
var deferred = $q.defer();
function generateIntersectionSnippets(guideline, index) {
var snippet_self, snippet_other;
for (var i = 0; i < guideline.intersections.length; i++) {
snippet_self = getKeywordInCategory(guideline.line_id, snippets.keyword).then(function() {
//Should something go here?
});
snippet_other = getKeywordInCategory(guideline.intersections[i].line_id, snippets.keyword).then(function() {
//Should something go here?
});
}
}
deferred.resolve(); //Is fired before the transactions above are complete
}
generateIntersectionSnippets();
return deferred.promise;
} //End ParseJSON
function getKeywordInCategory(keyword, category) {
var deferred = $q.defer();
var query = "SELECT category, id, chapter, header, snippet(guidelines, '<b>', '</b>', '...', '-1', '-24' ) AS snip FROM guidelines WHERE content MATCH '" + keyword + "' AND id='" + category + "';",
results = [];
db.transaction(function(transaction) {
transaction.executeSql(query, [],
function(transaction, result) {
if (result != null && result.rows != null) {
for (var i = 0; i < result.rows.length; i++) {
var row = result.rows.item(i);
results.push(row);
}
}
},defaultErrorHandler);
deferred.resolve(responses);
},defaultErrorHandler,defaultNullHandler);
return deferred.promise;
}
return {
parseJSON : parseJSON
};
}]);
I'd appreciate any guidance on what the correct model is to doing a chain of promises that includes an iteration across multiple async transactions- I know that how I have it right now is not correct at all.
You can use $q.all() to wait for a list of promises to be resolved.
function parseJSON() {
var deferred = $q.defer();
var promiseList = [];
for (var i = 0; i < guideline.intersections.length; i++) {
promiseList.push(getKeywordInCategory(guideline.line_id, snippets.keyword));
promiseList.push(getKeywordInCategory(guideline.intersections[i].line_id, snippets.keyword));
}
$q.all(promiseList).then(function() {
deferred.resolve();
});
return deferred.promise;
} //End ParseJSON