I'm importing a json file and loop through the object.
Each object has an array with eventIDs, which I loop through as well.
In this forEach I push a wrapper function that returns a Promise into an array promises
After looping through all objects I perform a promise.all() on the promises array. But strangely anything I do after this loop is not executed.
const fs = require('fs')
const promisify = require('util').promisify
const rp = require('request-promise')
const readFile = promisify(fs.readFile)
const filePath = 'json/results.json'
let promises = []
function init() {
readFile(filePath, 'utf8')
.then(file => {
const json = JSON.parse(file)
for (const country of json) {
if (country.eventIDs.length === 0) return
country.eventIDs.forEach(id => promises.push(getEventData(country, id)))
}
// nothing here is executed,
console.log('this is not fired')
Promise.all(promises)
.then(results => writeFile(results))
.catch( err => console.log(err))
})
.catch(err => console.log(err))
}
getEventData = (country, id) => new Promise((resolve, reject) => {
setTimeout(() => {
rp(url)
.then((results) => {
resolve ({
...results
})
})
.catch((err) => reject(`getEventData Error:\n ${err}`))
}, 2000)
})
writeFile = (results) => {
const json = JSON.stringify(results)
// const date = new Date()
// const filename = `${date.getTime()}-all-event-ids.json`
const filename = `results-allevents.json`
fs.writeFile(`json/${filename}`, json, 'utf8', () => console.log(`Succesfully written: ${filename}`))
}
init()
After the investigation, the line:
if (country.eventIDs.length === 0) return
was the main issue (and many others solved through comments).
The issue, to explain that, is that the return is not skipping the looped item (as perhaps expected), but rather returning void in the then callback, so skipping further executions of that block.
In order to "skip" the item if that condition is true, just do this instead
for (const country of json) {
if (country.eventIDs.length > 0) {
country.eventIDs.forEach(id => promises.push(getEventData(country, id)))
}
}
better is to capture each result from each promise. Because if you want array of responses of the .All . If one of those promises reject you will get no results.
const results = [];
country.eventIDs.forEach(id => promises.push(getEventData(country, id)
.then(res => results.push(res));
Promise.all(promises).then(()=> writeFile(results))
Stackblitz Example
Related
I have a function that reads files in a directory asynchronously (readdir) and filters for csv files. I also have an async function that calls readdir filtered for csv files and then iterates through them with fast-csv. Logging to the console the list and its length within the .on('end') function, I can see that they produce the desired results. however, my async call only resolves the first iteration.
const fs = require(`fs`);
const path = require(`path`);
const csv = require(`fast-csv`);
var ofsActivities = [];
const currDir = path.join(__dirname + `/../Downloads/`);
const readdir = async dirname => {
return new Promise((resolve, reject) => {
fs.readdir(dirname, (error, filenames) => {
error ? reject(error) : resolve(filenames);
});
});
};
const filtercsvFiles = (filename) => {
return filename.split(`.`)[1] == `csv`;
};
const ofsDataObjectArray = async () => {
return readdir(currDir).then(async filenames => {
return await new Promise((resolve, reject) => {
filenames = filenames.filter(filtercsvFiles);
for (let i = 0; i < filenames.length; i++) {
let currFilePath = currDir + filenames[i];
console.log(`Reading File: ${filenames[i]}`);
csv
.parseFile(currFilePath)
.on(`data`, (data) => {
//Doing stuff
})
.on(`error`, error => reject(error))
.on(`end`, () => resolve(ofsActivities)); //Inserting a console.log(ofsActivities.length) logs the correct and expected length on the last iteration
}
});
});
};
(async () => {
let list = await ofsDataObjectArray(); // This seems to only resolve the first iteration within the promise
console.log(list.length);
})();
You need to call resolve() only when the LAST csv.parseFile() is done. You're calling it when the FIRST one is done, thus the promise doesn't wait for all the others to complete. I'd suggest you promisify csv.parseFile() by itself and then await that inside the loop or accumulate all the promises from csv.parseFile() and use Promise.all() with all of them.
Here's using await on each csv.parseFile():
const ofsDataObjectArray = async () => {
return readdir(currDir).then(async filenames => {
filenames = filenames.filter(filtercsvFiles);
for (let i = 0; i < filenames.length; i++) {
let currFilePath = currDir + filenames[i];
console.log(`Reading File: ${filenames[i]}`);
await new Promise((resolve, reject) => {
csv.parseFile(currFilePath)
.on(`data`, (data) => {
//Doing stuff
})
.on(`error`, reject)
.on(`end`, () => resolve(ofsActivities));
});
}
return ofsActivities;
});
};
Or, here's running them in parallel with Promise.all():
const ofsDataObjectArray = async () => {
return readdir(currDir).then(filenames => {
filenames = filenames.filter(filtercsvFiles);
return Promise.all(filenames.map(file => {
let currFilePath = currDir + file;
console.log(`Reading File: ${file}`);
return new Promise((resolve, reject) => {
csv.parseFile(currFilePath)
.on(`data`, (data) => {
//Doing stuff
})
.on(`error`, error => reject(error))
.on(`end`, () => resolve(ofsActivities));
});
}))
});
};
P.S. It's unclear from your question what final result you're trying to accumulate (you have left that out) so you will have to add that to this code in the "doing stuff" code or by modifying the resolve(something) code.
I am trying to replace the loop that send out http requests using Axios. http is an Axios object and returns a promise. I want to change the code so that I use Promises.all() instead of a loop. I am trying to create a Promise, push into an array and then pass on to Promises.all. I only get empty arrays in my promises array.
I would appreciate any pointers on what I am doing wrong.
// Converting this
responseData = [];
for (const record of response.records) {
let response = await http.get('/records/' + record.id);
responseData.push(response.data.data);
}
// I am trying to convert to this ..
let promises = [];
for (const record of response.data.data) {
let promise = new Promise((resolve, reject) => {
let response = http.get('/records/' + record.id)
.then(response => {
return response.json();
})
.then(resp => {
// console.log(resp.data.data);
//return resp.data.data
resolve(resp.data.data);
});
return response;
});
promises.push(promise);
}
Promise.all(promises).then(records);
Not entirely sure of whether response.json() is required, and where the .records and .data.data should go ... but this might work:
const promises = [];
for (const record of response.records) {
promises.push(
http.get('/records/' + record.id)
.then(response => response.json()) //maybe?
.then(response => response.data.data)
)
}
Promise.all(promises).then(records => { /* do something */ } );
or use map:
const promises = response.records.map(record =>
http.get('/records/' + record.id)
.then(response => response.json()) //maybe?
.then(response => response.data.data)
)
Promise.all(promises).then(responseData => {/* do something */} );
I have the following code that is used to get JSON data from an Amazon Web Server API.
var json1 = new Promise((resolve, reject) => {
fetch(url[0])
.then(r => {
resolve(r.json())
})
.catch(err => {
reject(err)
})
})
I have this repeating 14 times using different urls and json vars and have it return the promises at the end using.
return Promise.all([json1,json2,json3,json4,json5,json6,json7,json8,json9,json10,json11,json12,json13,json14]).then(function(values) {
return values;
});
This works, but it takes up 150+ lines. I want to make a for loop that runs through the same code using a for loop. I created this...
for(var jsonCount = 0;jsonCount<url.length-1;jsonCount++){
jsonArr[jsonCount] = new Promise((resolve, reject) => {
fetch(url[jsonCount])
.then(r => {
resolve(r.json())
})
.catch(err => {
reject(err)
})
})
}
This doesn't work because the promise functions come back as undefined even though it is called by an await function.
const data = await fetchURL(urlToQuery())
Does anyone have suggestions to make this work? There is JSON being returned.
Thanks for your help.
Edit: Here is a larger chunk of the code.
function fetchURL(urls) {
let fetchJson = url => fetch(url).then(response => response.json());
Promise.all(urls.map(fetchJson)).then(arr => {
return arr;
});
(async function() {
const data = await fetchURL(urlToQuery())
console.log(data);
for(var r=0;r<numStations;r++){
if (data[r] == ""){
onlineArr[r] = false;
wdDataArr[r].push(cardinalToDeg(stationHistAvgArr[r]));
wsDataArr[r].push(0);
You can use .map for the loop. But don't use new Promise. You don't need a new promise when fetch already provides you with one.
Also, call your array urls instead of url. A plural will be a good indication for the reader of your code that indeed it is a collection of URLs.
Here is how it could look:
let fetchJson = url => fetch(url).then(response => response.json());
Promise.all(urls.map(fetchJson)).then(arr => {
// process your data
for (let obj of arr) {
console.log(obj);
}
});
I think this example can helps you:
// Mock async function
const getDataAsync = callback => {
setTimeout(
() => callback(Math.ceil(Math.random() * 100)),
Math.random() * 1000 + 2000
)
}
// Create the promise
const getDataWithPromise = () => {
return new Promise((resolve, reject) => {
try {
getDataAsync(resolve);
} catch(e) {
reject(e);
}
});
}
// Using the promise one time
getDataWithPromise()
.then(data => console.log("Simple promise:",data))
.catch(error => console.error(`Error catched ${error}`));
// Promises compound: Promise.all
const promise1 = getDataWithPromise();
promise1.then(data => console.log("promise1 ends:",data));
const promise2 = getDataWithPromise();
promise2.then(data => console.log("promise2 ends:",data));
const promise3 = getDataWithPromise();
promise3.then(data => console.log("promise3 ends:",data));
const promise4 = getDataWithPromise();
promise4.then(data => console.log("promise4 ends:",data));
const promise5 = getDataWithPromise();
promise5.then(data => console.log("promise5 ends:",data));
Promise.all([promise1,promise2,promise3,promise4,promise5])
.then(data => console.log("Promise all ends !!",data));
Hope this helps
you will have issues with closure and var variable capture.
You may want to change var to let to capture the right value in the closure so that url[jsonCount] is actually what you want.
also I think it would be much easier to do something like that in one line :)
let results = [];
for(let i = 0; i < urls.length; ++i) results.push(await (await fetch[urls[i]]).json());
This is a good use for map, mapping urls to promises...
function fetchUrls(urls) {
let promises = urls.map(url => fetch(url))
return Promise.all(promises).then(results => {
return results.map(result => result.json())
})
}}
// url is your array of urls (which would be better named as a plural)
fetchUrls(url).then(results => {
// results will be the fetched json
})
Using the async/await syntax (equivalent meaning)
// this can be called with await from within another async function
async function fetchUrls(urls) {
let promises = urls.map(url => fetch(url))
let results = await Promise.all(promises)
return results.map(result => result.json())
}
I Have a array of objects which i need to clone with different values.
Those values i'll get from each promise finally after preparing main modified array of object i'll have to save this. So i many need this as one single promise.
I not sure how to do it.
Here is the example we need to clone oldUser data. Say old user has credit score = 100; but for new user default credit will be created randomly by system.
For each user in the array of users few details has to get updated using async call.
This is the requirement
function getUserCreditScore(user){
var url = '/someurl';
return $http.get(url).then(function(res){
user.creditScore = (res.data) ? res.data : 0;
});
}
function getUserRecomandations(user){
var url = '/someurl';
return $http.get(url).then(function(res){
user.recommendation = (res.data) ? res.data : 'basic recommendation';
});
}
function getUserHelpInfo(user){
var url = '/someurl';
return $http.get(url).then(function(res){
user.helpInfo = (res.data) ? res.data : 'Help Info';
});
}
function clone(){
var newUsers = angular.copy(oldUsers);
for (var i=0; i<newUsers.length; i++){
newUsers[i].id = undefined;
getUserCreditScore(newUsers[i]);
getUserRecommendation(newUsers[i]);
getUserHelpInfo(newUsers[i]);
}
var promises = _.map(newUsers, user => user.save());
$q.all(promises).then(function (data) {
console.log(data);
}
}
You'll need to Promise.all on an array of Promises that are returned by getScreditScore
something like
function getCreditScore(){
var url = '/someurl';
return $http.get(url).then(res => (res && res.data) ? res.data : res);
}
function clone(){
var newUsers = angular.copy(oldUsers);
Promise.all(
newUsers.map(newUser => {
newUser.id = undefined;
return getCreditScore()
.then(result => newUser.creditScore = result);
})
).then(results => // results will be an array of values returned by the get in getCreditScore(newUser)
Promise.all(newUsers.map(user => user.save()))
).then(data =>
console.log(data); // this will be the result of all the user.save
);
}
Note: the newUser.creditScore is set in the .then in the newUsers.map callback - (minimal change to my original answer)
Alternatively, passing user to getCreditScore
function getCreditScore(user){
var url = '/someurl';
return $http.get(url)
.then(res => (res && res.data) ? res.data : res)
.then(score => user.creditScore = score);
}
function clone(){
var newUsers = angular.copy(oldUsers);
Promise.all(
newUsers.map(newUser => {
newUser.id = undefined;
return getCreditScore(newUser);
})
).then(results => // results will be an array of values returned by the get in getCreditScore(newUser)
Promise.all(newUsers.map(user => user.save()))
).then(data =>
console.log(data); // this will be the result of all the user.save
);
}
Personally, I'd write the code
function getCreditScore(){
var url = '/someurl';
return $http.get(url).then(res => (res && res.data) ? res.data : res);
}
function clone(){
var newUsers = angular.copy(oldUsers);
Promise.all(
newUsers.map(newUser => {
newUser.id = undefined;
return getCreditScore()
.then(result => newUser.creditScore = result)
.then(() => newUser.save())
.then(() => newUser);
})
).then(data =>
console.log(data); // this will be the newUsers Array
);
}
This assumes, though, that you don't need to wait for all the $http.get before running the user.save() - in fact this may be a little (very little) more performant as the newUser.save and $http.get will run in tandem
Ok, I know your meaning, you want your every element of your array do something that is async.
So you can use map and Promise.all. Here is my code:
const asyncFunction = (item, cb) => {
setTimeout(() => {
console.log(`done with ${item}`);
cb();
}, 1000);
}
let requests = [1, 2, 3].map((item) => {
return new Promise((resolve) =>{
asyncFunction(item, resolve);
});
});
Promise.all(requests).then(() => console.log('done'));
I have the promise function that execute async function in the loop few times for different data. I want to wait till all async functions will be executed and then resolve(), (or call callback function in non-promise function):
var readFiles = ()=>{
return new Promise((resolve,reject)=>{
var iterator = 0;
var contents = {};
for(let i in this.files){
iterator++;
let p = path.resolve(this.componentPath,this.files[i]);
fs.readFile(p,{encoding:'utf8'},(err,data)=>{
if(err){
reject(`Could not read ${this.files[i]} file.`);
} else {
contents[this.files[i]] = data;
iterator--;
if(!iterator) resolve(contents);
}
});
}
if(!iterator) resolve(contents); //in case of !this.files.length
});
};
I increase iterator on every loop repetition, then, in async function's callback decrease iterator and check if all async functions are done (iterator===0), if so - call resolve().
It works great, but seems not elegant and readable. Do you know any better way for this issue?
Following up the comment with some code and more detail!
Promise.all() takes an iterator, and waits for all promises to either resolve or reject. It will then return the results of all the promises. So instead of keeping track of when all promises resolve, we can create little promises and add them to an array. Then, use Promise.all() to wait for all of them to resolve.
const readFiles = () => {
const promises = [];
for(let i in files) {
const p = path.resolve(componentPath, files[i]);
promises.push(new Promise((resolve, reject) => {
fs.readFile(p, {encoding:'utf8'}, (err, data) => {
if(err) {
reject(`Could not read ${files[i]} file.`);
} else {
resolve(data);
}
});
}));
}
return Promise.all(promises);
};
const fileContents = readFiles().then(contents => {
console.log(contents)
})
.catch(err => console.error(err));
You only need push all the Promises into an array to then pass it as argument to Promise.all(arrayOfPromises)
try something like this:
var readFiles = () => {
var promises = [];
let contents = {};
var keys_files = Object.keys(this.files);
if (keys_files.length <= 0) {
var promise = new Promise((resolve, reject) => {
resolve(contents);
});
promises.push(promise);
}
keys_files.forEach((key) => {
var file = this.files[key];
var promise = new Promise((resolve, reject) => {
const currentPath = path.resolve(this.componentPath, file);
fs.readFile(p,{encoding:'utf8'},(err, data) => {
if (err) {
return reject(`Could not read ${file} file.`);
}
contents[file] = data;
resolve(contents)
});
});
});
return Promises.all(promises);
}
Then you should use the function like so:
// this will return a promise that contains an array of promises
var readAllFiles = readFiles();
// the then block only will execute if all promises were resolved if one of them were reject so all the process was rejected automatically
readAllFiles.then((promises) => {
promises.forEach((respond) => {
console.log(respond);
});
}).catch((error) => error);
If you don't care if one of the promises was rejected, maybe you should do the following
var readFiles = () => {
var promises = [];
let contents = {};
var keys_files = Object.keys(this.files);
if (keys_files.length <= 0) {
var promise = new Promise((resolve, reject) => {
resolve(contents);
});
promises.push(promise);
}
keys_files.forEach((key) => {
var file = this.files[key];
var promise = new Promise((resolve, reject) => {
const currentPath = path.resolve(this.componentPath, file);
fs.readFile(p,{encoding:'utf8'},(err, data) => {
// create an object with the information
let info = { completed: true };
if (err) {
info.completed = false;
info.error = err;
return resolve(info);
}
info.data = data;
contents[file] = info;
resolve(contents)
});
});
});
return Promises.all(promises);
}
Copied from comments:
Also - you might want to use fs-extra, a drop-in replacement for fs, but with promise support added.
Here's how that goes:
const fs = require('fs-extra');
var readFiles = ()=>{
let promises = files
.map(file => path.resolve(componentPath, file))
.map(path => fs.readFile(path));
return Promise.all(promises);
});
Nice and clean. You can then get contents like this:
readFiles()
.then(contents => { ... })
.catch(error => { ... });
This will fail on first error though (because that's what Promise.all does). If you want individual error handling, you can add another map line:
.map(promise => promise.catch(err => err));
Then you can filter the results:
let errors = contents.filter(content => content instanceof Error)
let successes = contents.filter(content => !(content instanceof Error))