Develop Reference

Run multiple functions in succession while waiting for previous one to complete

I have a script which involves data processing and then uploading/downloading files in mass.
I've been trying to find different ways to sort out an issue I'm having and I've decided I want to try something like this (it appears to be the easiest solution), but I'm not sure if it's achievable with JavaScript.
I'm not that great with async and I have no clue what to do.
Thanks in advance, I've left a vague idea of what I'm doing below.
function one() {
>> do something | data processing
};
function two() {
>> do something | file upload/download
};
function thr() {
>> do something | process two arrays for an output that will be pushed to a global var
};
one();//wait for one to finish
two();//wait for two to finish
thr();//wait for thr to finish
function two() {
return new Promise( async (resolve, reject) => {
fs.createReadStream('data.csv')
.pipe(csv())
.on('data', (row) => {
if (row[removed] === '') return;
const
url1 = row[removed],
url2 = row[removed]),
https = require('https'),
id = row[removed]);
let upload = [];
const fetchRealUrl = request(url1, (e, response) => {//need to await completion
const FILENAMEREMOVED1= fs.createWriteStream(`${id}-FILENAMEREMOVED1`),
downloadRealFile = https.get(response.request.uri.href, function(response) {
response.pipe(FILENAMEREMOVED1);
console.log(`FILENAMEREMOVED file ${id} downloaded.`);
});
}),
fetchRealUrl2 = request(url2, (e, response) => {//need to await completion
const FILENAMEREMOVED2= fs.createWriteStream(`${id}-FILENAMEREMOVED2`),//need to extract into funciton instead of repeating
downloadRealFile2= https.get(response.request.uri.href, function(response) {
response.pipe(FILENAMEREMOVED2);
console.log(`FILENAMEREMOVEDfile ${id} downloaded.`);
});
});
//getData(url);
upload.push(`{"id":"${id}",REMOVED}`);
})
.on('end', (row) => {
console.info('Completed .');
resolve(upload);//completes while downloads still running
});
});
}

Try looking at Promises.
function one() {
return new Promise((resolve, reject) => {
//Do something...
let x = 10
resolve(x)
}
}
function two() {
return new Promise((resolve, reject) => {
//Do something else...
let y = 20
resolve(y)
}
}
one().then(x => { //The value resolved in one() is passed here
//then() is executed only when one() resolves its promise
return two()
}).then(y => { //The value resolved in two() is passed here
//etc...
})

const func1 = new Promise(res =>{
setTimeout(() => {
//do some asynchronous work
res()
}, 1000)
})
const func2 = new Promise(res =>{
setTimeout(() => {
//do some asynchronous work
res()
}, 1000)
})
//To run async functions in a waterfall pattern:
func1()
.then(resultOfFunc1 => {
//do something with resultOfFunc1
return func2()
})
.then(resultOfFunc2 => {
//do something with resultOfFunc2
})
//To run async functions in a parallel pattern:
let promiseFunctions = [func1(), func2()]
let result = Promise.all(promiseFunctions)
//result will be an array of resolved promises

Save base64 data into a variable

I have seen couple of examples here and based on those examples I have prepared my solution. But still I am getting Promise
__proto__: Promise
[[PromiseStatus]]: "pending
[[PromiseValue]]: undefined
I am trying to store my base 64 data into my base64 variable. Please let me know what's wrong with my code and a working solution.
Below is my try:
import React, { Component } from 'react'
import { DropzoneArea, DropzoneDialog } from 'material-ui-dropzone'
import Button from '#material-ui/core/Button';
async function getBaseData(selectedFile) {
let base64;
var fileToLoad = selectedFile[0];
base64 = await getBase64(fileToLoad);
return base64; //This returns me PromiseStatus Pending
}
function getBase64(file) {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.readAsDataURL(file);
reader.onload = () => resolve(reader.result);
reader.onerror = error => reject(error);
return Promise.resolve(reader.result)
});
}
export class ImageUploader extends Component {
constructor(props) {
super(props);
this.state = {
files: [],
image: null,
};
}
handleImageChange(files) {
const { imageCallback } = this.props
imageCallback(getBaseData(files)); // By this callback i am trying to pass my base64 string to other component
}
render() {
return (
<DropzoneArea
acceptedFiles={['image/*']}
filesLimit={1}
maxFileSize={10000000}
//showPreviews={false}
onChange={this.handleImageChange.bind(this)}
/>
)
}
}

your problem is how you are managing your Promise. you dont need to return anything from a Promise as the documentation says or it will resolve inmediatly.
in this case when you do:
function getBase64(file) {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.readAsDataURL(file);
reader.onload = () => resolve(reader.result);
reader.onerror = error => reject(error);
return Promise.resolve(reader.result) // <--- this is wrong.
});
}
it will resolve to Promise.resolve(reader.result) which is undefined, because maybe it didn't finish, maybe and just maybe you could get an accurate result but it is just luck that the reader actually resolves before that return(race condition)
basically in order to make it work, you need to use the resolver and the rejecter of the promise so it can resolve/reject.
EDIT: I noticed that you are also calling the reader function before setting the callbacks, then you are reading the file, adding the callbacks and then the callbacks are not called.
just change your code to this:
function getBase64(file) {
return new Promise((resolve, reject) => {
const reader = new FileReader();
//set the callbacks before reading the object
reader.onload = () => resolve(reader.result);
reader.onerror = error => reject(error);
reader.readAsDataURL(file);
});
}

The problem is getBaseData is an async function but you still need to wait for it to resolve.
Try "awaiting" for it as well like this:
handleImageChange(files) {
const { imageCallback } = this.props;
async function wrap(){
const result = await getBaseData(files); //result should be a value here and not promise
imageCallback(result);
}
wrap(); // call the async function which will await the getBaseData and use your callback
}

There are two problems here: the creation of the Promise is incorrect, and you don't handle the resolved Promise as you should. There is no need for any async or await like other answers suggest.
The Promise creation:
function getBaseData(selectedFiles) {
var fileToLoad = selectedFiles[0];
return getBase64(fileToLoad);
}
function getBase64(file) {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => resolve(reader.result);
reader.onerror = error => reject(error);
reader.readAsDataURL(file);
});
}
Resolving the Promise:
handleImageChange(files) {
const { imageCallback } = this.props
getBaseData(files).then(imageCallback);
}

How to update Promise.all with new array?

In my code below, I would like to execute a, b and c but then a1 since a1 is added within a. However, it doesn't look like Promise.all is updated with the new a1 added. What's the best way to do this? Is there Promise all update? I try not to do await a1 within a.
var arr = [];
async function a() {
return new Promise(resolve => {
setTimeout(() => {
console.log("Resolve a");
arr.push(a1);
resolve(1);
}, 2000);
});
}
async function b() {
return new Promise(resolve => {
setTimeout(() => {
console.log("Resolve b");
resolve(2);
}, 4000);
});
}
async function c() {
return new Promise(resolve => {
setTimeout(() => {
console.log("Resolve c " + arr.length);
resolve(3);
}, 6000);
});
}
async function a1() {
return new Promise(resolve => {
setTimeout(() => {
console.log("Resolve a1");
resolve(11);
}, 2000);
});
}
arr = [a(), b(), c()];
(async function run() {
await Promise.all(arr);
})();
console.log('Done');

For a start, you push a1 where I think you'd want to push a1() - otherwise you're pushing a function, not a promise
But that won't change anything, because the array passed to Promise.all is (in every library I've seen) copied (using Array#slice) so any changes to the passed in array won't be "visible" to the Promise.all (internal) code
But, you can create a function that recursively calls Promise.all on the array until the length of the results is equal to the (current, new) length of the array
var recursiveAll = a => Promise.all(a).then(r => r.length == a.length ? r : recursiveAll(a));
var arr = [];
async function a() {
return new Promise(resolve => {
setTimeout(() => {
console.log("Resolve a");
arr.push(a1());
resolve(1);
}, 200);
});
}
async function b() {
return new Promise(resolve => {
setTimeout(() => {
console.log("Resolve b");
resolve(2);
}, 400);
});
}
async function c() {
return new Promise(resolve => {
setTimeout(() => {
console.log("Resolve c " + arr.length);
resolve(3);
}, 600);
});
}
async function a1() {
return new Promise(resolve => {
setTimeout(() => {
console.log("Resolve a1");
resolve(11);
}, 200);
});
}
arr = [a(), b(), c()];
(async function run() {
let results = await recursiveAll(arr);
console.log(results);
})();

Interesting question...
You may indeed modify Promise.all() on the run by replacing a promise with another one even if the newly inserted one resolves the latest.
In the below example promise three will replace itself with a new promise which will resolve in an additional 2000ms later and Promise.all() will wait until it to resolves to include it's resolving value to the array provided to the .then() stage.
var fun = _ => new Promise((v,x) => setTimeout(v,2000,5)),
one = new Promise((v,x) => setTimeout(v,1000,1)),
two = new Promise((v,x) => setTimeout(v,2000,2)),
three = new Promise((v,x) => setTimeout(v,3000,fun())),
four = new Promise((v,x) => setTimeout(v,4000,4));
Promise.all([one,two,three,four])
.then(a => console.log(a));
Please wait for 5 seconds to see the result.

waiting for many async functions execution

I have the promise function that execute async function in the loop few times for different data. I want to wait till all async functions will be executed and then resolve(), (or call callback function in non-promise function):
var readFiles = ()=>{
return new Promise((resolve,reject)=>{
var iterator = 0;
var contents = {};
for(let i in this.files){
iterator++;
let p = path.resolve(this.componentPath,this.files[i]);
fs.readFile(p,{encoding:'utf8'},(err,data)=>{
if(err){
reject(`Could not read ${this.files[i]} file.`);
} else {
contents[this.files[i]] = data;
iterator--;
if(!iterator) resolve(contents);
}
});
}
if(!iterator) resolve(contents); //in case of !this.files.length
});
};
I increase iterator on every loop repetition, then, in async function's callback decrease iterator and check if all async functions are done (iterator===0), if so - call resolve().
It works great, but seems not elegant and readable. Do you know any better way for this issue?

Following up the comment with some code and more detail!
Promise.all() takes an iterator, and waits for all promises to either resolve or reject. It will then return the results of all the promises. So instead of keeping track of when all promises resolve, we can create little promises and add them to an array. Then, use Promise.all() to wait for all of them to resolve.
const readFiles = () => {
const promises = [];
for(let i in files) {
const p = path.resolve(componentPath, files[i]);
promises.push(new Promise((resolve, reject) => {
fs.readFile(p, {encoding:'utf8'}, (err, data) => {
if(err) {
reject(`Could not read ${files[i]} file.`);
} else {
resolve(data);
}
});
}));
}
return Promise.all(promises);
};
const fileContents = readFiles().then(contents => {
console.log(contents)
})
.catch(err => console.error(err));

You only need push all the Promises into an array to then pass it as argument to Promise.all(arrayOfPromises)
try something like this:
var readFiles = () => {
var promises = [];
let contents = {};
var keys_files = Object.keys(this.files);
if (keys_files.length <= 0) {
var promise = new Promise((resolve, reject) => {
resolve(contents);
});
promises.push(promise);
}
keys_files.forEach((key) => {
var file = this.files[key];
var promise = new Promise((resolve, reject) => {
const currentPath = path.resolve(this.componentPath, file);
fs.readFile(p,{encoding:'utf8'},(err, data) => {
if (err) {
return reject(`Could not read ${file} file.`);
}
contents[file] = data;
resolve(contents)
});
});
});
return Promises.all(promises);
}
Then you should use the function like so:
// this will return a promise that contains an array of promises
var readAllFiles = readFiles();
// the then block only will execute if all promises were resolved if one of them were reject so all the process was rejected automatically
readAllFiles.then((promises) => {
promises.forEach((respond) => {
console.log(respond);
});
}).catch((error) => error);
If you don't care if one of the promises was rejected, maybe you should do the following
var readFiles = () => {
var promises = [];
let contents = {};
var keys_files = Object.keys(this.files);
if (keys_files.length <= 0) {
var promise = new Promise((resolve, reject) => {
resolve(contents);
});
promises.push(promise);
}
keys_files.forEach((key) => {
var file = this.files[key];
var promise = new Promise((resolve, reject) => {
const currentPath = path.resolve(this.componentPath, file);
fs.readFile(p,{encoding:'utf8'},(err, data) => {
// create an object with the information
let info = { completed: true };
if (err) {
info.completed = false;
info.error = err;
return resolve(info);
}
info.data = data;
contents[file] = info;
resolve(contents)
});
});
});
return Promises.all(promises);
}

Copied from comments:
Also - you might want to use fs-extra, a drop-in replacement for fs, but with promise support added.
Here's how that goes:
const fs = require('fs-extra');
var readFiles = ()=>{
let promises = files
.map(file => path.resolve(componentPath, file))
.map(path => fs.readFile(path));
return Promise.all(promises);
});
Nice and clean. You can then get contents like this:
readFiles()
.then(contents => { ... })
.catch(error => { ... });
This will fail on first error though (because that's what Promise.all does). If you want individual error handling, you can add another map line:
.map(promise => promise.catch(err => err));
Then you can filter the results:
let errors = contents.filter(content => content instanceof Error)
let successes = contents.filter(content => !(content instanceof Error))

Writing to file in a promise

The function here creates a bunch of files but does not write to them. How ever, if I remove the Promise.all in the end and don't resolve the function at all it DOES write the data to the files. It doesn't matter what I try to write, I can comment out everything and just write 'hello world' and it still won't write anything. The question is simple, why?
const writeSmilFiles = (smilInfo) => {
return new Promise ((resolve, reject) => {
const p1 = smilPartOne();
const p2 = smilPartTwo();
let promises = dataTypes.new.camera_set.map(instance => {
return new Promise((resolve, reject) => {
const smilEntries = smilInfo.filter(smil => smil.BroadcastAndKlippTupleId == instance.BroadcastAndKlippTupleId && smil.CameraId == instance.CameraId);
try {
const fileName = `${__dirname}/newSmilFiles/${smilEntries[0].Smil}`;
const file = fs.createWriteStream(fileName);
file.write(p1);
smilEntries.forEach(entry => {
const smilEntry = smilSwitch(entry.Filename, entry.Bitrate, entry.Width, entry.Height);
file.write(smilEntry);
console.log(smilEntry);
file.write('\n');
});
file.write(p2);
file.end();
resolve(`Smil written.`);
} catch (ex) {
reject(ex);
}
});
});
Promise.all(promises).then(msg => {
resolve(msg);
});
});
};

Resolve when the stream has actually finished:
file.on("finish", () => resolve(`Smil written.`));