I have a problem with promises in a loop. The whole promise thing is completely new to me, so I try to learn it with very simple examples.
In my example, I have 2 text files on a server and I want to save the content of the text files into an array.
It works with a setTimeout, but this is not the solution that I want. Here is the example wit setTimeout
var http = require('http'),
Q = require('q');
var urls = ["http://localhost:8000/1.txt", "http://localhost:8000/2.txt"]
var txts = [];
function getData(url) {
http.get(url, function(res) {
var data = "";
res.on('data',function(chunk){
data+=chunk;
});
res.on('end',function(){
txts.push(data);
});
}).on('error',function(e){
console.log("Error Request: "+e.message);
})
}
function getTxts() {
for(a in urls) {
var url = urls[a];
getData(url);
}
// is not working
console.log(txts);
// is working
setTimeout(function() {
console.log(txts);
}, 1000)
}
getTxts();
I now tried doing it with Q, but I am stuck at some point. There is some point where I am going into the wrong direction, but I can't see where it is.
var http = require('http'),
Q = require('q');
var urls = ["http://localhost:8000/1.txt", "http://localhost:8000/2.txt"]
var txts = [];
function getData(url) {
return Q.promise(function(respond,reject){
http.get(url, function(res) {
var data = "";
res.on('data',function(chunk){
data+=chunk;
});
res.on('end',function(){
txts.push(data);
});
}).on('error',function(e){
console.log("Error Request: "+e.message);
})
});
}
function getTxts() {
var promises = [];
for(a in urls) {
var url = urls[a];
var promise = getData(url);
promises.push(promise);
}
return promises;
}
function start() {
Q.fcall(function() {
getTxts();
}).then(function() {
console.log(txts);
})
}
start();
Thanks for your help!
You could use just regular promises for this
var http = require('http');
var urls = ["http://localhost:8000/1.txt", "http://localhost:8000/2.txt"]
function getData(url) {
return new Promise(function(resolve, reject) {
http.get(url, function(res) {
var data = "";
res.on('data',function(chunk){
data+=chunk;
});
res.on('end',function(){
resolve(data);
});
}).on('error',function(err){
reject(err);
});
});
}
function getTxts() {
return Promise.all(
urls.map(function(url) {
return getData(url);
})
);
}
getTxts().then(function(texts) {
// "texts" is an array of the returned data
}).catch(function(err) {
// epic fail
});
Issue is you are not resolving or rejecting the promise you created in the getData function
function getData(url) {
return Q.promise(function(resolve,reject){
http.get(url, function(res) {
var data = "";
res.on('data',function(chunk){
data+=chunk;
});
res.on('end',function(){
txts.push(data);
resolve(); // resolve the promise when done
});
}).on('error',function(e){
console.log("Error Request: "+e.message);
reject(); // reject the promise if there is an error
})
});
}
Related
I have a promise that is meant to collect metadata on a file and then resolve it with the metadata that was collected. Here is how I am trying to get its result:
getMetadata: function(text, url){
return this.getpageno(url).then(function(pagecount){
return new Promise(function(accept, reject){
var result = []
var dataMatch = rx.exec(text)
var produktDaten = dataMatch[1].split("&&&").filter(Boolean);
var dokuArr = dataMatch[2].split("&&&").filter(Boolean);
console.log('the produktdaten are ' + produktDaten)
for (var i=0; i<produktDaten.length; i+=4){
var entry = {}
for(var j=0; j<dokuArr.length; j+=3){
var seite = dokuArr[j+2];
// make sure seite is a digit
if (!(/^\d+$/.test(seite))){
console.log(seite + ' was not a valid page number')
throw err
}
if (/(A|a)lle?/i.test(nummer)){
entry.kks.pages[beschreibung] = seite;
// })
}
else if (nummer === kksNummer) {
entry.kks.pages[beschreibung] = seite;
}
}
entry.hersteller = produktDaten[i+1]
entry.typ = produktDaten[i+2]
entry.artikelNummer = produktDaten[i+3]
result.push(entry)
}
if (result.length>0){
return accept(result)
}
return reject()
})
})
},
getpageno: function(url){
var self=this
var pdf = pdfjs.getDocument(url);
return pdf.then(function(pdf){
var maxPages = pdf.pdfInfo.numPages;
var countPromises = []; // collecting all page promises
for (var j = 1; j <= maxPages; j++) {
try {
var page = pdf.getPage(j);
var txt = "";
countPromises.push(page.then(function(page) { // add page promise
var textContent = page.getTextContent();
return textContent.then(function(text){ // return content promise
return text.items.map(function (s) { return s.str; }).join('&&&'); // value page text
});
}));
}
catch(e){
console.log(e)
}
}
// Wait for all pages and join text
return Promise.all(countPromises).then(function (texts) {
// since doumentation pages do not add the final '&&&', must add one manually (only after rx has been found)
var fulltext = texts.reduce(function(full, text){
if (rx.test(full)){
var next = '&&&'+text
return full+=next
}
return full+=text
}, '')
return [fulltext, texts]
});
}).then(function(textarr){
var fulltext = textarr[0]
self.fulltext = fulltext;
var texts = textarr[1]
try {
var partialmatch = rx.exec(fulltext)[0]
var count = texts.reduce(function(pageno, text){
var tomatch = text.replace(/.*Typ&&&/, '')
if (tomatch.length>0 && partialmatch.indexOf(tomatch) > -1){
pageno++
}
return pageno;
}, 0)
}
catch(e){
console.log(e)
}
return count;
})
}
Edited the entry to show that I'm now returning the value of the gepageno function.
The data that I am expecting is logged but not available as a result of accept(). Can anyone tell what could be going wrong?
I have tried to simulate and give a solution of your promise. I have posted pseudo code.
Promise flow
var testFunction = ((result) => {
return new Promise(function (resolve, reject) {
var err = null;
if (!!err) {
reject(err);
} else {
console.log("2. testFunction resolved");
resolve("Final response");
}
});
});
var getpageno = ((url) => {
return new Promise((resolve, reject) => {
var err = null;
if (!!err) {
reject(err);
} else {
console.log("1. getpageno resolved");
resolve(45);
}
});
});
var getMetadata = ((text, url) => {
console.log("0. getmetadata resolved");
var self = this;
getpageno(url)
.then((pageCount) => {
console.log("> pageCount :", pageCount);
return testFunction(pageCount);
})
.then((data) => {
console.log(">", data);
});
});
getMetadata("hell", "https://www.google.com");
output
0. getmetadata resolved
1. getpageno resolved
> pageCount : 452. testFunction resolved
> Final response
you need to return the promise on the third line
return self.getpageno(url).then
to access its eventual fulfillment value.
I am developing a backend server code with NodeJS. What the code does is, periodically connect to a REST API, request updates and then write to my database.
I have no way of getting delta of the data from the API, so I drop my collection from MongoDB and then just insert the newly pulled data.
I implemented promises to make sure that the dependent methods are executed only after the previous methods resolve. This however doesn't seem to work as I anticipated.
So, I drop the collection and insert, this works. But the following method seems to execute before the new data is populated. It sometime works, when I have some new console.log statements which seems to induce a slight delay ever so slightly to make it all work.
setTimeout function didn't seem to help. Any suggestions?
Here is a sanitized version of the code: https://jsfiddle.net/ppbfrozg/
var request = require("request");
var q = require('q');
function authenticate() {
var deferred = q.defer();
request(options, function(error, response, body) {
if (error) throw new Error(error);
deferred.resolve(JSON.parse(body).token);
});
return deferred.promise;
}
function getData(token) {
var deferred = q.defer();
request(options, function(error, response, body) {
if (error) throw new Error(error);
deferred.resolve(JSON.parse(body).token);
});
return deferred.promise;
}
function insertDataInMongo(a) {
var deferred = q.defer();
var MongoClient = require('mongodb').MongoClient;
var url = 'mongodb://localhost/myDB';
var token = a[1];
MongoClient.connect(url, function(err, db) {
if (err) return deferred.reject(new Error(err))
console.log("connected for insert");
var apiData = JSON.parse(a[0]).data;
if (JSON.parse(a[0]).data) {
db.collection('MediaData').insert(apiData);
console.log("Records Inserted");
} else {
db.collection('Details').drop();
db.collection('Details').insert(JSON.parse(a[0]));
console.log("Records Inserted");
}
deferred.resolve(token);
});
return deferred.promise;
}
function getMedia(dataContext) {
var deferred = q.defer();
var cursor = dataContext[0];
var token = dataContext[1];
if (cursor !== null) {
console.log("Inside cursor not null");
cursor.forEach(function(data) {
insertDataInMongo(data);
})
}
return deferred.promise;
}
function check(array, attr, value) {
for (var i = 0; i < array.length; i += 1) {
if (array[i][attr] === value) {
return false;
}
}
return true;
}
function get_value(array, attr) {
for (var i = 0; i < array.length; i += 1) {
if (array[i].hasOwnProperty(attr)) {
return array[i][attr];
}
}
}
function getNames(token) {
var deferred = q.defer();
var MongoClient2 = require('mongodb').MongoClient;
var url = 'mongodb://localhost/myDB';
console.log("going to get Data");
MongoClient2.connect(url, function(err, db) {
if (err) return deferred.reject(new Error(err));
console.log("connected for select");
var data = db.collection('Details').find();
var dataContext = [data, token, 0, 0, 0, 0, 0, 0, 0, null];
deferred.resolve(dataContext);
});
return deferred.promise;
}
function convertDate(date) {
var yyyy = date.getFullYear().toString();
var mm = (date.getMonth() + 1).toString();
var dd = (date.getDate() - 3).toString();
var mmChars = mm.split('');
var ddChars = dd.split('');
return yyyy + '-' + (mmChars[1] ? mm : "0" + mmChars[0]) + '-' + (ddChars[1] ? dd : "0" + ddChars[0]);
}
authenticate()
.then(getData)
.then(insertDataInMongo)
.then(getNames)
.then(getMedia);
This should work. Let me know if any issue.
function insertDataInMongo(a) {
var deferred = q.defer();
var MongoClient = require('mongodb').MongoClient;
var url = 'mongodb://localhost/myDB';
var token = a[1];
MongoClient.connect(url, function(err, db) {
if (err) return deferred.reject(new Error(err))
console.log("connected for insert");
var apiData = JSON.parse(a[0]).data;
if (JSON.parse(a[0]).data) {
db.collection('MediaData').insert(apiData, function(){
console.log("Records Inserted");
return deferred.resolve(token);
});
} else {
db.collection('Details').drop(function(error, result){//Callback function that executes after drop operation has completed.
if(error){
return deferred.reject(error);//Reject the promise if there was an error
}
db.collection('Details').insert(JSON.parse(a[0]), function(err, res){//Callback function that executes after insert operation has completed.
if(err){
return deferred.reject(err);//Reject the promise if there was an error
}
console.log("Records Inserted");
return deferred.resolve(token);
});
});
}
});
return deferred.promise;
}
From what I see from nodeJS driver API for MongoDB :
https://mongodb.github.io/node-mongodb-native/api-generated/collection.html
hint : db.collection.drop is asynchronous, so you have to use a callback with it
db.collection('Details').drop(function(err, result){
// Do anything AFTER you dropped your collection
}
but since you are using promises, you should use something like that:
authenticate()
.then(getData)
.then(db.collection('Details').drop)
.then(insertDataInMongo)
.then(getNames)
.then(getMedia);
or if you want to really keep the same code format :
function dropCollectionInMongo() {
db.collection('Details').drop()
}
authenticate()
.then(getData)
.then(dropCollectionInMongo)
.then(insertDataInMongo)
.then(getNames)
.then(getMedia);
having fun with promises in JS and trying to craft simple xpath website parser, but I am struggling with logic on finishing overall parsing process, my code is:
var request = require('request');
var xpath = require('xpath');
var dom = require('xmldom').DOMParser;
var olxMain = 'https://www.some.site/';
var xpathRoot = '//a[contains(#href, "https://www.some.site/mask/")]';
var linksXpath = '//a';
var allGlobalLinks = [];
var getLink = function (node) {
for (key in node['attributes']) {
if (node['attributes'][key]['name'] === 'href') {
return node['attributes'][key]['value'];
}
}
}
var getData = function (url, xpathPattern) {
return new Promise(function (resolve, reject) {
console.log("Opening " + url);
var processResponse = function (error, response, body) {
var doc = new dom().parseFromString(body);
var childNodes = xpath.select(xpathPattern, doc);
var links = childNodes.map(function (n) {
return getLink(n);
});
resolve(links);
};
request({url: url}, processResponse);
}
);
}
var arrayUnique = function (x, i, a) {
return a.indexOf(x) == i;
};
var main = function () {
getData(olxMain, xpathRoot).then(function (links) {
links = links.filter(arrayUnique);
var maxThreads = 10, n = 0;
var chunks = [];
for (k in links) {
var url = links[k];
n++;
if (n <= maxThreads)
chunks.push(url);
else {
n = 0;
// console.log(chunks);
Promise.all(chunks.map(function (url) {
return getData(url, linksXpath);
})).then(function (links) {
// add these links to global scope list here
});
console.log("Finished mappings iteration");
});
chunks = [];
}
}
;
});
}
main();
So what I want is basically some kind of threadPool with promises, how to I manage these 10 promises, when they all are finished, I should spawn another 10 more, until list is finished and all Promises have finished ?
So I have read a lot of the different answers about asynchronous functions on here but I think I am over thinking my problem, or I have been staring at it just to long and I cant figure it out. So your help is greatly appreciated.
So I am parsing a csv file and then trying to get the lat/long through another api. but I cant access the lat/lng outside of the function. below is my code I have commented it to the best of my ability please let me know if there are any questions or a much better way to do this.
Thanks,
var location = []
function run() {
http.get(url, function(res) {
if(res.statusCode === 200) {
res.pipe(parse(function(err, data) {
for(i = 1; i < data.length; i++) {
var info = data[i];
passLoc = info[6].replace('block ', '')
passLoc = passLoc.replace(/ /g, "+")
getLoc(passLoc, function(loc) {
location.push(loc);
//If I console.log(location) here I get all the info I want but.....it is printed 100 times becuase it is printed for each i in data.length
})
}
console.log(location) // loging this here gives me an empty array
}))
}else {
console.error('The address is unavailable. (%d)', res.statusCode);
}
})
}
function getLoc(x, callback) {
var url = "http://geodata.alleghenycounty.us/arcgis/rest/services/Geocoders/EAMS_Composite_Loc/GeocodeServer/findAddressCandidates?Street=" + x + "&City=Pittsburgh&State=PA&ZIP=&SingleLine=&outFields=&outSR=4326&searchExtent=&f=pjson";
http.get(url, function(res) {
var data = '';
res.on('data', function(chunk) {
data += chunk;
});
res.on('end', function() {
var d = JSON.parse(data);
var obj = d.candidates;
if(obj != '') {
var loc = obj[0].location
var lat = loc.x
var lng = loc.y
var location = [lat, lng];
callback(location)
} else {
callback(x);
}
});
res.on('error', function(err) {
callback("error!")
});
});
}
Your code tries to synchronously consume asynchronous data -- you're synchronously trying to access the results (location) before any of the asynchronous operations have finished.
As you have multiple async operations running in parallel, you can make use of async.parallel to aid in controlling the asynchronous flow:
var async = require('async');
function run() {
http.get(url, function(res) {
if(res.statusCode === 200) {
res.pipe(parse(function(err, data) {
// array of async tasks to execute
var tasks = [];
data.slice(1).forEach(function(info) {
var passLoc = info[6].replace('block ', '').replace(/ /g, '+');
// push an async operation to the `tasks` array
tasks.push(function(cb) {
getLoc(passLoc, function(loc) {
cb(null, loc);
});
});
});
// run all async tasks in parallel
async.parallel(tasks, function(err, locations) {
// consume data when all async tasks are finished
console.log(locations);
});
}));
}else {
console.error('The address is unavailable. (%d)', res.statusCode);
}
});
}
Also, for loops don't create a scope, so I've swapped it by a forEach in order to scope the info and passLoc variables inside each iteration.
Here's a slightly more condensed version using ES5's Array#map:
var async = require('async');
function run() {
http.get(url, function(res) {
if(res.statusCode === 200) {
res.pipe(parse(function(err, data) {
async.parallel(
// map data items to async tasks
data.slice(1).map(function(info) {
return function(cb) {
var passLoc = info[6].replace('block ', '').replace(/ /g, '+');
getLoc(passLoc, function(loc) {
cb(null, loc);
});
};
}),
function(err, locations) {
// consume data when all async tasks are finished
console.log(locations);
}
);
}));
} else {
console.error('The address is unavailable. (%d)', res.statusCode);
}
});
}
I have a Node app that I'm writing where I need to use promises for async calls.
I currently have a foreach loop being called from within a .then(function()) of a promise, but when I return the end result of the foreach, I get nothing.
In the foreach I can console.log the value of data and retrieve it, but not outside the for loop before the return?
var Feeds = function(){
this.reddit = new Reddit();
}
Feeds.prototype.parseRedditData = function(){
var _this = this;
this.getData(this.reddit.endpoint).then(function(data){
return _this.reddit.parseData(data, q);
});
}
Feeds.prototype.getData = function(endpoint){
var deferred = q.defer();
https.get(endpoint, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
deferred.resolve(JSON.parse(body));
});
}).on('error', function(e) {
deferred.reject(e);
});
return deferred.promise;
}
var Reddit = function(){
this.endpoint = "https://www.reddit.com/r/programming/hot.json?limit=10";
}
Reddit.prototype.parseData = function(json, q){
var dataLength = json.data.children.length,
data = [];
for(var i = 0; i <= dataLength; i++){
var post = {};
post.url = json.data.children[i].data.url;
post.title = json.data.children[i].data.title;
post.score = json.data.children[i].data.score;
data.push(post);
}
return data;
}
Feeds.prototype.parseRedditData = function(){
var _this = this;
this.getData(this.reddit.endpoint).then(function(data){
return _this.reddit.parseData(data, q);
});
}
When i see this I see a "return" in the callback of the promise... I don't know why you're doing this, but I just want to be sure:
I you want this "return" to be the returned value of the function 'parseRedditData', this won't work.
The only way to return your data here is by using a callback, or a promise, like this:
Feeds.prototype.parseRedditData = function(callack){
var _this = this;
this.getData(this.reddit.endpoint).then(function(data){
callback(_this.reddit.parseData(data, q));
});
}