I have the promise function that execute async function in the loop few times for different data. I want to wait till all async functions will be executed and then resolve(), (or call callback function in non-promise function):
var readFiles = ()=>{
return new Promise((resolve,reject)=>{
var iterator = 0;
var contents = {};
for(let i in this.files){
iterator++;
let p = path.resolve(this.componentPath,this.files[i]);
fs.readFile(p,{encoding:'utf8'},(err,data)=>{
if(err){
reject(`Could not read ${this.files[i]} file.`);
} else {
contents[this.files[i]] = data;
iterator--;
if(!iterator) resolve(contents);
}
});
}
if(!iterator) resolve(contents); //in case of !this.files.length
});
};
I increase iterator on every loop repetition, then, in async function's callback decrease iterator and check if all async functions are done (iterator===0), if so - call resolve().
It works great, but seems not elegant and readable. Do you know any better way for this issue?
Following up the comment with some code and more detail!
Promise.all() takes an iterator, and waits for all promises to either resolve or reject. It will then return the results of all the promises. So instead of keeping track of when all promises resolve, we can create little promises and add them to an array. Then, use Promise.all() to wait for all of them to resolve.
const readFiles = () => {
const promises = [];
for(let i in files) {
const p = path.resolve(componentPath, files[i]);
promises.push(new Promise((resolve, reject) => {
fs.readFile(p, {encoding:'utf8'}, (err, data) => {
if(err) {
reject(`Could not read ${files[i]} file.`);
} else {
resolve(data);
}
});
}));
}
return Promise.all(promises);
};
const fileContents = readFiles().then(contents => {
console.log(contents)
})
.catch(err => console.error(err));
You only need push all the Promises into an array to then pass it as argument to Promise.all(arrayOfPromises)
try something like this:
var readFiles = () => {
var promises = [];
let contents = {};
var keys_files = Object.keys(this.files);
if (keys_files.length <= 0) {
var promise = new Promise((resolve, reject) => {
resolve(contents);
});
promises.push(promise);
}
keys_files.forEach((key) => {
var file = this.files[key];
var promise = new Promise((resolve, reject) => {
const currentPath = path.resolve(this.componentPath, file);
fs.readFile(p,{encoding:'utf8'},(err, data) => {
if (err) {
return reject(`Could not read ${file} file.`);
}
contents[file] = data;
resolve(contents)
});
});
});
return Promises.all(promises);
}
Then you should use the function like so:
// this will return a promise that contains an array of promises
var readAllFiles = readFiles();
// the then block only will execute if all promises were resolved if one of them were reject so all the process was rejected automatically
readAllFiles.then((promises) => {
promises.forEach((respond) => {
console.log(respond);
});
}).catch((error) => error);
If you don't care if one of the promises was rejected, maybe you should do the following
var readFiles = () => {
var promises = [];
let contents = {};
var keys_files = Object.keys(this.files);
if (keys_files.length <= 0) {
var promise = new Promise((resolve, reject) => {
resolve(contents);
});
promises.push(promise);
}
keys_files.forEach((key) => {
var file = this.files[key];
var promise = new Promise((resolve, reject) => {
const currentPath = path.resolve(this.componentPath, file);
fs.readFile(p,{encoding:'utf8'},(err, data) => {
// create an object with the information
let info = { completed: true };
if (err) {
info.completed = false;
info.error = err;
return resolve(info);
}
info.data = data;
contents[file] = info;
resolve(contents)
});
});
});
return Promises.all(promises);
}
Copied from comments:
Also - you might want to use fs-extra, a drop-in replacement for fs, but with promise support added.
Here's how that goes:
const fs = require('fs-extra');
var readFiles = ()=>{
let promises = files
.map(file => path.resolve(componentPath, file))
.map(path => fs.readFile(path));
return Promise.all(promises);
});
Nice and clean. You can then get contents like this:
readFiles()
.then(contents => { ... })
.catch(error => { ... });
This will fail on first error though (because that's what Promise.all does). If you want individual error handling, you can add another map line:
.map(promise => promise.catch(err => err));
Then you can filter the results:
let errors = contents.filter(content => content instanceof Error)
let successes = contents.filter(content => !(content instanceof Error))
Related
I have a function that reads files in a directory asynchronously (readdir) and filters for csv files. I also have an async function that calls readdir filtered for csv files and then iterates through them with fast-csv. Logging to the console the list and its length within the .on('end') function, I can see that they produce the desired results. however, my async call only resolves the first iteration.
const fs = require(`fs`);
const path = require(`path`);
const csv = require(`fast-csv`);
var ofsActivities = [];
const currDir = path.join(__dirname + `/../Downloads/`);
const readdir = async dirname => {
return new Promise((resolve, reject) => {
fs.readdir(dirname, (error, filenames) => {
error ? reject(error) : resolve(filenames);
});
});
};
const filtercsvFiles = (filename) => {
return filename.split(`.`)[1] == `csv`;
};
const ofsDataObjectArray = async () => {
return readdir(currDir).then(async filenames => {
return await new Promise((resolve, reject) => {
filenames = filenames.filter(filtercsvFiles);
for (let i = 0; i < filenames.length; i++) {
let currFilePath = currDir + filenames[i];
console.log(`Reading File: ${filenames[i]}`);
csv
.parseFile(currFilePath)
.on(`data`, (data) => {
//Doing stuff
})
.on(`error`, error => reject(error))
.on(`end`, () => resolve(ofsActivities)); //Inserting a console.log(ofsActivities.length) logs the correct and expected length on the last iteration
}
});
});
};
(async () => {
let list = await ofsDataObjectArray(); // This seems to only resolve the first iteration within the promise
console.log(list.length);
})();
You need to call resolve() only when the LAST csv.parseFile() is done. You're calling it when the FIRST one is done, thus the promise doesn't wait for all the others to complete. I'd suggest you promisify csv.parseFile() by itself and then await that inside the loop or accumulate all the promises from csv.parseFile() and use Promise.all() with all of them.
Here's using await on each csv.parseFile():
const ofsDataObjectArray = async () => {
return readdir(currDir).then(async filenames => {
filenames = filenames.filter(filtercsvFiles);
for (let i = 0; i < filenames.length; i++) {
let currFilePath = currDir + filenames[i];
console.log(`Reading File: ${filenames[i]}`);
await new Promise((resolve, reject) => {
csv.parseFile(currFilePath)
.on(`data`, (data) => {
//Doing stuff
})
.on(`error`, reject)
.on(`end`, () => resolve(ofsActivities));
});
}
return ofsActivities;
});
};
Or, here's running them in parallel with Promise.all():
const ofsDataObjectArray = async () => {
return readdir(currDir).then(filenames => {
filenames = filenames.filter(filtercsvFiles);
return Promise.all(filenames.map(file => {
let currFilePath = currDir + file;
console.log(`Reading File: ${file}`);
return new Promise((resolve, reject) => {
csv.parseFile(currFilePath)
.on(`data`, (data) => {
//Doing stuff
})
.on(`error`, error => reject(error))
.on(`end`, () => resolve(ofsActivities));
});
}))
});
};
P.S. It's unclear from your question what final result you're trying to accumulate (you have left that out) so you will have to add that to this code in the "doing stuff" code or by modifying the resolve(something) code.
I have the following code that is used to get JSON data from an Amazon Web Server API.
var json1 = new Promise((resolve, reject) => {
fetch(url[0])
.then(r => {
resolve(r.json())
})
.catch(err => {
reject(err)
})
})
I have this repeating 14 times using different urls and json vars and have it return the promises at the end using.
return Promise.all([json1,json2,json3,json4,json5,json6,json7,json8,json9,json10,json11,json12,json13,json14]).then(function(values) {
return values;
});
This works, but it takes up 150+ lines. I want to make a for loop that runs through the same code using a for loop. I created this...
for(var jsonCount = 0;jsonCount<url.length-1;jsonCount++){
jsonArr[jsonCount] = new Promise((resolve, reject) => {
fetch(url[jsonCount])
.then(r => {
resolve(r.json())
})
.catch(err => {
reject(err)
})
})
}
This doesn't work because the promise functions come back as undefined even though it is called by an await function.
const data = await fetchURL(urlToQuery())
Does anyone have suggestions to make this work? There is JSON being returned.
Thanks for your help.
Edit: Here is a larger chunk of the code.
function fetchURL(urls) {
let fetchJson = url => fetch(url).then(response => response.json());
Promise.all(urls.map(fetchJson)).then(arr => {
return arr;
});
(async function() {
const data = await fetchURL(urlToQuery())
console.log(data);
for(var r=0;r<numStations;r++){
if (data[r] == ""){
onlineArr[r] = false;
wdDataArr[r].push(cardinalToDeg(stationHistAvgArr[r]));
wsDataArr[r].push(0);
You can use .map for the loop. But don't use new Promise. You don't need a new promise when fetch already provides you with one.
Also, call your array urls instead of url. A plural will be a good indication for the reader of your code that indeed it is a collection of URLs.
Here is how it could look:
let fetchJson = url => fetch(url).then(response => response.json());
Promise.all(urls.map(fetchJson)).then(arr => {
// process your data
for (let obj of arr) {
console.log(obj);
}
});
I think this example can helps you:
// Mock async function
const getDataAsync = callback => {
setTimeout(
() => callback(Math.ceil(Math.random() * 100)),
Math.random() * 1000 + 2000
)
}
// Create the promise
const getDataWithPromise = () => {
return new Promise((resolve, reject) => {
try {
getDataAsync(resolve);
} catch(e) {
reject(e);
}
});
}
// Using the promise one time
getDataWithPromise()
.then(data => console.log("Simple promise:",data))
.catch(error => console.error(`Error catched ${error}`));
// Promises compound: Promise.all
const promise1 = getDataWithPromise();
promise1.then(data => console.log("promise1 ends:",data));
const promise2 = getDataWithPromise();
promise2.then(data => console.log("promise2 ends:",data));
const promise3 = getDataWithPromise();
promise3.then(data => console.log("promise3 ends:",data));
const promise4 = getDataWithPromise();
promise4.then(data => console.log("promise4 ends:",data));
const promise5 = getDataWithPromise();
promise5.then(data => console.log("promise5 ends:",data));
Promise.all([promise1,promise2,promise3,promise4,promise5])
.then(data => console.log("Promise all ends !!",data));
Hope this helps
you will have issues with closure and var variable capture.
You may want to change var to let to capture the right value in the closure so that url[jsonCount] is actually what you want.
also I think it would be much easier to do something like that in one line :)
let results = [];
for(let i = 0; i < urls.length; ++i) results.push(await (await fetch[urls[i]]).json());
This is a good use for map, mapping urls to promises...
function fetchUrls(urls) {
let promises = urls.map(url => fetch(url))
return Promise.all(promises).then(results => {
return results.map(result => result.json())
})
}}
// url is your array of urls (which would be better named as a plural)
fetchUrls(url).then(results => {
// results will be the fetched json
})
Using the async/await syntax (equivalent meaning)
// this can be called with await from within another async function
async function fetchUrls(urls) {
let promises = urls.map(url => fetch(url))
let results = await Promise.all(promises)
return results.map(result => result.json())
}
I'm trying to return a custom object from a async function that works as wrapper for a put using indexdb.
Using Promises this is easy.
However, using async/await became more challenging...
const set = async (storeName, key, value) => {
if (!db)
throw new Error("no db!");
try {
const result = {};
let tx = db.transaction(storeName, "readwrite");
let store = tx.objectStore(storeName);
let r = store.put({ data: key, value: value });
console.log(r);
r.onsuccess = async () => {
console.log('onsuccess');
result.something = true;
}
r.onerror = async () => {
console.log('onerror');
result.something = false;
}
await r.transaction.complete; // ok... this don't work
// how can I await until onsuccess or onerror runs?
return result;
} catch (error) {
console.log(error);
}
}
The ideia is to return a composed object... however all my attemps fails as onsuccess runs after returning the result.
I googled a lot and could't find a way to proper await for onsuccess/onerror events.
I know that returning a Promise is more easy as resolve(result) would end returning what I want... but i'm trying to learn to make same code using async/await.
Thank you so much,
Try this:
function set(db, storeName, key, value) {
return new Promise((resolve, reject) => {
let result;
const tx = db.transaction(storeName, 'readwrite');
tx.oncomplete = _ => resolve(result);
tx.onerror = event => reject(event.target.error);
const store = tx.objectStore(storeName);
const request = store.put({data: key, value: value});
request.onsuccess = _ => result = request.result;
});
}
async function callIt() {
const db = ...;
const result = await set(db, storeName, key, value);
console.log(result);
}
Edit, since you insist on using the async qualifier for the set function, you can do this instead. Please note I find this pretty silly:
async function set(db, storeName, key, value) {
// Wrap the code that uses indexedDB in a promise because that is
// the only way to use indexedDB together with promises and
// async/await syntax. Note this syntax is much less preferred than
// using the promise-returning function pattern I used in the previous
// section of this answer.
const promise = new Promise((resolve, reject) => {
let result;
const tx = db.transaction(storeName, 'readwrite');
tx.oncomplete = _ => resolve(result);
tx.onerror = event => reject(event.target.error);
const store = tx.objectStore(storeName);
const request = store.put({data: key, value: value});
request.onsuccess = _ => result = request.result;
});
// We have executed the promise, but have not awaited it yet. So now we
// await it. We can use try/catch here too, if we want, because the
// await will translate the promise rejection into an exception. Of course,
// this is also rather silly because we are doing the same thing as just
// allowing an uncaught exception to exit the function early.
let result;
try {
result = await promise;
} catch(error) {
console.log(error);
return;
}
// Now do something with the result
console.debug('The result is', result);
}
Ultimately you'll end up wrapping IDB in a promise-friend library, but for your specific need, you could use something like this:
function promiseForTransaction(tx) {
return new Promise((resolve, reject) => {
tx.oncomplete = e => resolve();
tx.onabort = e => reject(tx.error);
});
}
And then in your code you can write things such as:
await promiseForTransaction(r.tx);
... which will wait until the transaction completes, and throw an exception if it aborts. (Note that this requires calling the helper
before the transaction could possibly have completed/aborted, since
it won't ever resolve if the events have already fired)
I can't confirm it right now but I think it should be await tx.complete instead of await r.transaction.complete;.
But a general solution that would work even if the API would not support Promises directly would be to wrap a new Promise around the onsuccess and onerror and use await to wait for that Promise to resolve, and in your onsuccess and onerror you then call the resolve function:
const set = async (storeName, key, value) => {
if (!db)
throw new Error("no db!");
try {
const result = {};
let tx = db.transaction(storeName, "readwrite");
let store = tx.objectStore(storeName);
let r = store.put({
data: key,
value: value
});
console.log(r);
await new Promise((resolve, reject) => {
r.onsuccess = () => {
console.log('onsuccess');
result.something = true;
resolve()
}
r.onerror = () => {
console.log('onerror');
result.something = false;
// I assume you want to resolve the promise even if you get an error
resolve()
}
})
return result;
} catch (error) {
console.log(error);
}
}
I would furhter change it to:
try {
await new Promise((resolve, reject) => {
r.onsuccess = resolve
r.onerror = reject
})
console.log('success');
result.something = true;
} catch(err) {
console.log('error');
result.something = false;
}
The function here creates a bunch of files but does not write to them. How ever, if I remove the Promise.all in the end and don't resolve the function at all it DOES write the data to the files. It doesn't matter what I try to write, I can comment out everything and just write 'hello world' and it still won't write anything. The question is simple, why?
const writeSmilFiles = (smilInfo) => {
return new Promise ((resolve, reject) => {
const p1 = smilPartOne();
const p2 = smilPartTwo();
let promises = dataTypes.new.camera_set.map(instance => {
return new Promise((resolve, reject) => {
const smilEntries = smilInfo.filter(smil => smil.BroadcastAndKlippTupleId == instance.BroadcastAndKlippTupleId && smil.CameraId == instance.CameraId);
try {
const fileName = `${__dirname}/newSmilFiles/${smilEntries[0].Smil}`;
const file = fs.createWriteStream(fileName);
file.write(p1);
smilEntries.forEach(entry => {
const smilEntry = smilSwitch(entry.Filename, entry.Bitrate, entry.Width, entry.Height);
file.write(smilEntry);
console.log(smilEntry);
file.write('\n');
});
file.write(p2);
file.end();
resolve(`Smil written.`);
} catch (ex) {
reject(ex);
}
});
});
Promise.all(promises).then(msg => {
resolve(msg);
});
});
};
Resolve when the stream has actually finished:
file.on("finish", () => resolve(`Smil written.`));
How can I iterate through an array of data using Promises and returning data? I have seen some promises.push(asyncFunc) methods but some of the entries from my array will fail so from what I gather I can't use that.
var filesFromDisk = [
'41679_4_2015-09-06_17-02-12.mp4',
'41679_4_2015-09-06_17-02-12.smil',
'41680_4_2015-09-09_10-44-05.mp4'
];
start(filesFromDisk)
.then((data) => {
console.log(data); // Want my data here
});
I start start(dbFiles) from another file which is why I want the data returned there.
function start(dbFiles) {
var listOfFiles = [],
promises = [];
return new Promise((fulfill, reject) => {
for (var i = 0; i < dbFiles.length; i++) {
getMp4(dbFiles[i])
.then((data) => {
listOfFiles = listOfFiles.concat(data);
console.log(listOfFiles);
})
}
fulfill(listOfFiles) // Need to happen AFTER for loop has filled listOfFiles
});
}
So for every entry in my array I want to check if the file with the new extension exists and read that file. If the file with extension does not exist I fulfill the original file. My Promise.all chain works and all the data is returned in for loop above (getMp4(dbFiles[i]))
function getMp4(filename) {
var mp4Files = [];
var smil = privateMethods.setSmileExt(localData.devPath + filename.toString());
return new Promise((fulfill, reject) => {
Promise.all([
privateMethods.fileExists(smil),
privateMethods.readTest(smil)
]).then(() => {
readFile(filename).then((files) => {
fulfill(files)
});
}).catch((err) => {
if (!err.exists) fulfill([filename]);
});
});
}
function readFile(filename){
var filesFromSmil = [];
return new Promise((fulfill, reject) => {
fs.readFile(localData.devPath + filename, function (err, res){
if (err) {
reject(err);
}
else {
xmlParser(res.toString(), {trim: true}, (err, result) => {
var entry = JSON.parse(JSON.stringify(result.smil.body[0].switch[0].video));
for (var i = 0; i < entry.length; i++) {
filesFromSmil.push(privateMethods.getFileName(entry[i].$.src))
}
});
fulfill(filesFromSmil);
}
});
});
};
Methods in the Promise.all chain in getMp4 - have no problems with these that I know.
var privateMethods = {
getFileName: (str) => {
var rx = /[a-zA-Z-1\--9-_]*.mp4/g;
var file = rx.exec(str);
return file[0];
},
setSmileExt: (videoFile) => {
return videoFile.split('.').shift() + '.smil';
},
fileExists: (file) => {
return new Promise((fulfill, reject) => {
try {
fs.accessSync(file);
fulfill({exists: true})
} catch (ex) {
reject({exists: false})
}
})
},
readTest: (file) => {
return new Promise((fulfill, reject) => {
fs.readFile(file, (err, res) => {
if (err) reject(err);
else fulfill(res.toString());
})
})
}
}
If you need them to run in parallel, Promise.all is what you want:
function start(dbFiles) {
return Promise.all(dbFiles.map(getMp4));
}
That starts the getMp4 operation for all of the files and waits until they all complete, then resolves with an array of the results. (getMp4 will receive multiple arguments — the value, its index, and a a reference to the dbFiles arary — but since it only uses the first, that's fine.)
Usage:
start(filesFromDisk).then(function(results) {
// `results` is an array of the results, in order
});
Just for completeness, if you needed them to run sequentially, you could use the reduce pattern:
function start(dbFiles) {
return dbFiles.reduce(function(p, file) {
return p.then(function(results) {
return getMp4(file).then(function(data) {
results.push(data);
return results;
});
});
}, Promise.resolve([]));
}
Same usage. Note how we start with a promise resolved with [], then queue up a bunch of then handlers, each of which receives the array, does the getMp4 call, and when it gets the result pushes the result on the array and returns it; the final resolution value is the filled array.