I have this code:
const fs = require("fs");
const saveFile = (fileName, data) => {
return new Promise((resolve) => {
fs.writeFile(fileName, data, (err) => {
resolve(true);
});
});
};
const readFile = (fileName) => {
return new Promise((resolve) => {
fs.readFile(fileName, "utf8", (err, data) => {
resolve(data);
});
});
};
const filename = "test.txt";
saveFile(filename, "first");
readFile(filename).then((contents) => {
saveFile(filename, contents + " second");
});
readFile(filename).then((contents) => {
saveFile(filename, contents + " third");
});
I'm hoping to obtain in 'test.txt'
first second third
but instead, I get
first thirdd
The idea is that every time I receive a certain post request. I have to add more text to the file
Does someone have any solution for this?
Thank you so much!
Edit:
The problem of using async await or a chain of .then( ) is that I have to add more text to the file every time I receive a certain post request. So I don't have control over what is written or when. The Idea is that everything is written and nothing is overwritten even if two post requests are received at the same time.
I'm going to share the solution with a linked list I came up with yesterday. But I still want to know if someone has a better solution.
const saveFile = (fileName, data) => {
return new Promise((resolve) => {
fs.writeFile(fileName, data, (err) => {
resolve(true);
});
});
};
const readFile = (fileName) => {
return new Promise((resolve) => {
fs.readFile(fileName, "utf8", (err, data) => {
resolve(data);
});
});
};
class LinkedCommands {
constructor(head = null) {
this.head = head;
}
getLast() {
let lastNode = this.head;
if (lastNode) {
while (lastNode.next) {
lastNode = lastNode.next;
}
}
return lastNode;
}
addCommand(command, description) {
let lastNode = this.getLast();
const newNode = new CommandNode(command, description);
if (lastNode) {
return (lastNode.next = newNode);
}
this.head = newNode;
this.startCommandChain();
}
startCommandChain() {
if (!this.head) return;
this.head
.command()
.then(() => {
this.pop();
this.startCommandChain();
})
.catch((e) => {
console.log("Error in linked command\n", e);
console.log("command description:", this.head.description);
throw e;
});
}
pop() {
if (!this.head) return;
this.head = this.head.next;
}
}
class CommandNode {
constructor(command, description = null) {
this.command = command;
this.description = description;
this.next = null;
}
}
const linkedCommands = new LinkedCommands();
const filename = "test.txt";
linkedCommands.addCommand(() => saveFile(filename, "first"));
linkedCommands.addCommand(() =>
readFile(filename).then((contents) =>
saveFile(filename, contents + " second")
)
);
linkedCommands.addCommand(() =>
readFile(filename).then((contents) => saveFile(filename, contents + " third"))
);
Because these are async functions they notify you that the work is completed in the then function.
That means you want to use a then chain (or an async function) like so:
readFile(filename).then((contents) => {
return saveFile(filename, contents + " second");
}).then(() => {
return readFile(filename)
}).then((contents) => {
saveFile(filename, contents + " third");
});
You can use a FIFO queue of functions that return promises for this.
const { readFile, writeFile } = require("fs/promises");
let queue = [];
let lock = false;
async function flush() {
lock = true;
let promise;
do {
promise = queue.shift();
if (promise) await promise();
} while (promise);
lock = false;
}
function createAppendPromise(filename, segment) {
return async function append() {
const contents = await readFile(filename, "utf-8");
await writeFile(
filename,
[contents.toString("utf-8"), segment].filter((s) => s).join(" ")
);
};
}
async function sequentialWrite(filename, segment) {
queue.push(createAppendPromise(filename, segment));
if (!lock) await flush();
}
async function start() {
const filename = "test.txt";
// Create all three promises right away
await Promise.all(
["first", "second", "third"].map((segment) =>
sequentialWrite(filename, segment)
)
);
}
start();
So how's this work? Well, we use a FIFO queue of promise functions. As requests come in we create them and add them to the queue.
Every time we add to the array we attempt to flush it. If there's a lock in place, we know we're already flushing, so just leave.
The flushing mechanism will grab the first function in the queue, delete it from the queue, invoke it, and await on the promise that returns. It will continue to do this until the queue is empty. Because all of this is happening asynchronously, the queue can continue to get populated while flushing.
Please keep in mind that if this file is shared on a server somewhere and you have multiple processes reading from this file (such as with horizontal scaling) you will lose data. You should instead use some kind of distributed mutex somewhere. A popular way of doing this is using Redis and redlock.
Hope this helps!
Edit: by the way, if you want to prove that this indeed works, you can add a completely random setTimeout to the createAppendPromise function.
function createAppendPromise(filename, segment) {
const randomTime = () =>
new Promise((resolve) => setTimeout(resolve, Math.random() * 1000));
return async function append() {
await randomTime();
const contents = await readFile(filename, "utf-8");
await writeFile(
filename,
[contents.toString("utf-8"), segment].filter((s) => s).join(" ")
);
};
}
Chaining is fine, even if you don't know in advance when or how many promises will be created. Just keep the end of the chain handy, and chain to it whenever you create a promise...
// this context must persist across posts
let appendChain = Promise.resolve();
const filename = "test.txt";
// assuming op's readFile and saveFile work...
const appendToFile = (filename, data) =>
return readFile(filename).then(contents => {
return saveFile(filename, contents + data);
});
}
function processNewPost(data) {
return appendChain = appendChain.then(() => {
return appendToFile(filename, data);
});
}
Here's a demonstration. The async functions are pretend read, write and append. The <p> tag is the simulated contents of a file. Press the button to add new data to the pretend file.
The button is for you to simulate the external event that triggers the need to append. The append function has a 1s delay, so, if you want, you can get in several button clicks before all the appends on the promise chain are write is done.
function pretendReadFile() {
return new Promise(resolve => {
const theFile = document.getElementById('the-file');
resolve(theFile.innerText);
})
}
function pretendWriteFile(data) {
return new Promise(resolve => {
const theFile = document.getElementById('the-file');
theFile.innerText = data;
resolve();
})
}
function pretendDelay(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
function appendFile(data) {
return pretendDelay(1000).then(() => {
return pretendReadFile()
}).then(result => {
return pretendWriteFile(result + data);
});
}
document.getElementById("my-button").addEventListener("click", () => click());
let chain = Promise.resolve();
let count = 0
function click() {
chain = chain.then(() => appendFile(` ${count++}`));
}
<button id="my-button">Click Fast and At Random Intervals</button>
<h3>The contents of the pretend file:</h3>
<p id="the-file">empty</p>
Related
I have a function that reads files in a directory asynchronously (readdir) and filters for csv files. I also have an async function that calls readdir filtered for csv files and then iterates through them with fast-csv. Logging to the console the list and its length within the .on('end') function, I can see that they produce the desired results. however, my async call only resolves the first iteration.
const fs = require(`fs`);
const path = require(`path`);
const csv = require(`fast-csv`);
var ofsActivities = [];
const currDir = path.join(__dirname + `/../Downloads/`);
const readdir = async dirname => {
return new Promise((resolve, reject) => {
fs.readdir(dirname, (error, filenames) => {
error ? reject(error) : resolve(filenames);
});
});
};
const filtercsvFiles = (filename) => {
return filename.split(`.`)[1] == `csv`;
};
const ofsDataObjectArray = async () => {
return readdir(currDir).then(async filenames => {
return await new Promise((resolve, reject) => {
filenames = filenames.filter(filtercsvFiles);
for (let i = 0; i < filenames.length; i++) {
let currFilePath = currDir + filenames[i];
console.log(`Reading File: ${filenames[i]}`);
csv
.parseFile(currFilePath)
.on(`data`, (data) => {
//Doing stuff
})
.on(`error`, error => reject(error))
.on(`end`, () => resolve(ofsActivities)); //Inserting a console.log(ofsActivities.length) logs the correct and expected length on the last iteration
}
});
});
};
(async () => {
let list = await ofsDataObjectArray(); // This seems to only resolve the first iteration within the promise
console.log(list.length);
})();
You need to call resolve() only when the LAST csv.parseFile() is done. You're calling it when the FIRST one is done, thus the promise doesn't wait for all the others to complete. I'd suggest you promisify csv.parseFile() by itself and then await that inside the loop or accumulate all the promises from csv.parseFile() and use Promise.all() with all of them.
Here's using await on each csv.parseFile():
const ofsDataObjectArray = async () => {
return readdir(currDir).then(async filenames => {
filenames = filenames.filter(filtercsvFiles);
for (let i = 0; i < filenames.length; i++) {
let currFilePath = currDir + filenames[i];
console.log(`Reading File: ${filenames[i]}`);
await new Promise((resolve, reject) => {
csv.parseFile(currFilePath)
.on(`data`, (data) => {
//Doing stuff
})
.on(`error`, reject)
.on(`end`, () => resolve(ofsActivities));
});
}
return ofsActivities;
});
};
Or, here's running them in parallel with Promise.all():
const ofsDataObjectArray = async () => {
return readdir(currDir).then(filenames => {
filenames = filenames.filter(filtercsvFiles);
return Promise.all(filenames.map(file => {
let currFilePath = currDir + file;
console.log(`Reading File: ${file}`);
return new Promise((resolve, reject) => {
csv.parseFile(currFilePath)
.on(`data`, (data) => {
//Doing stuff
})
.on(`error`, error => reject(error))
.on(`end`, () => resolve(ofsActivities));
});
}))
});
};
P.S. It's unclear from your question what final result you're trying to accumulate (you have left that out) so you will have to add that to this code in the "doing stuff" code or by modifying the resolve(something) code.
When ever I run this code the content_2 function runs first instead of content_1. The code below runs asynchronously and the second function uses a variable in the first function through "node.js store" to run so I need content_2 to wait for content_1 to finish before it starts running, I want it to run synchronously.
const content_1 = function main_Content(req, res, callback) {
const assert = require('assert');
const fs = require('fs');
const mongodb = require('mongodb');
const mv = require('mv');
var filename = req.body.Filename + Math.ceil((Math.random() * 1000000000000) + 10);
console.log(req.body.Filename)
//CREATE A FILE
fs.writeFile(filename + '.html', req.body.Content, (err) => {
if (err) throw err;
console.log('File was created successfully...');
});
//MOVE TO UPLOADS
const currentPath = path.join(__dirname, "../", filename + ".html");
const destinationPath = path.join(__dirname, "../uploads", filename + ".html");
mv(currentPath, destinationPath, function(err) {
if (err) {
throw err
} else {
console.log("Successfully moved the file!");
}
});
const uri = 'mongodb://localhost:27017';
const dbName = 'registration';
const client = new mongodb.MongoClient(uri);
client.connect(function(error) {
assert.ifError(error);
const db = client.db(dbName);
var bucket = new mongodb.GridFSBucket(db);
//UPLOAD FILE TO DB THROUGH STREAMING
fs.createReadStream('./uploads/' + filename + '.html').
pipe(bucket.openUploadStream(filename + ".html")).
on('error', function(error) {
assert.ifError(error);
}).
on('finish', function(res) {
var result = res._id
store.set('id', result);
//process.exit(0);
});
});
}
const content_2 = function metaData(req, res, callback) {
const obj = new ObjectId()
var filename = req.body.Filename + Math.ceil((Math.random() * 1000000000000) + 10);
const slice = require('array-slice')
var id = store.get('id');
console.log(id)
var objID = slice(id, 14, 24)
console.log(objID + '2nd')
Key.findByIdAndUpdate(req.body.id, {$push: {Articles:{Title: req.body.Title, Desc:req.body.Desc, Content: {_id: `ObjectId("${objID}")`}}} }, (err, docs) => {
if(err){
console.log(err)
}else{
console.log('done' + obj)
}
});
}
I will give you two examples. First one will be using promises. Second one will be using async await. The output though is exactly the same.
Supose those three methods. Someone cleans a room, "then" someone checks if the room is clean, "then" a payment is done.
Promises version
const cleanRoom = new Promise((res, rej) => {
console.log("Cleaning room...");
setTimeout(() => {
console.log("Room clean!");
res();
}, 2000);
});
const cleanCheckup = () => {
return new Promise((res, rej) => {
console.log("Clean checkup...");
setTimeout(() => {
console.log("Checkup complete!");
res();
}, 2000);
});
}
const payMoney = () => {
console.log("Open Wallet!");
return new Promise((res, rej) => {
setTimeout(() => {
res("50€");
}, 2000);
});
}
cleanRoom
.then(cleanCheckup)
.then(payMoney)
.then(console.log);
Async/Await version
const sleep = mils => {
return new Promise(r => setTimeout(r, mils));
};
const cleanRoomAW = async () => {
console.log("Cleaning room...");
await sleep(2000);
console.log("Room clean!");
};
const cleanCheckupAW = async () => {
console.log("Clean checkup...");
await sleep(2000);
console.log("Checkup complete!");
};
const payMonney = async () => {
console.log("Open Wallet!");
await sleep(2000);
return "50€";
};
async function run() {
await cleanRoomAW();
await cleanCheckupAW();
const value = await payMonney();
console.log(value);
};
run();
Please be aware of the helper method sleep, since you can't await for setTimeout in the browser yet. (In node js >=16 I believe you dont need this helper method).
You can copy/paste any of the two versions into the browser's console and confirm that both versions run synchronously despite the asynchronous nature of the setTimeout.
I am trying to run the node js Lighthouse function serially (one at a time) with an array of URLs. My problem is that whenever I loop through the array, Lighthouse runs all the URLs at once, which I imagine is problematic if you have a very large array of URLs.
The code:
for(let url of urls) {
function launchChromeAndRunLighthouse(url, opts, config = null) {
return chromeLauncher.launch({chromeFlags: opts.chromeFlags}).then(chrome => {
opts.port = chrome.port;
return lighthouse(url, opts, config).then(results => {
return chrome.kill().then(() => results.lhr)
});
});
}
}
launchChromeAndRunLighthouse('https://example.com', opts).then(results => {
// Use results!
});
Please help! And thank you for your time!
Your answer is correct but it can be improved. Since you have access to async and await, you should fully utilize it to make your code cleaner:
async function launchChromeAndRunLighthouse (url, opts, config = null) {
const chrome = await chromeLauncher.launch({chromeFlags: opts.chromeFlags});
opts.port = chrome.port;
const { lhr } = await lighthouse(url, opts, config);
await chrome.kill();
return lhr;
}
async function launchAudit (urls) {
for (const url of urls) {
const results = await launchChromeAndRunLighthouse(url, opts);
// Use results!
};
}
launchAudit(urls);
I believe I figured it out. What I did is below. Please continue to send feedback if you think this is wrong.
function launchChromeAndRunLighthouse(url, opts, config = null) {
return chromeLauncher.launch({chromeFlags: opts.chromeFlags}).then(chrome => {
opts.port = chrome.port;
return lighthouse(url, opts, config).then(results => {
return chrome.kill().then(() => results.lhr)
});
});
};
async function launchAudit(urls) {
for (let url of urls) {
await launchChromeAndRunLighthouse(url, opts).then(results => {
// Use results!
});
};
};
launchAudit(urls);
A variation on Patric Roberts answer (which should be the accepted answer).
I was wondering if it was necessary to kill chrome every iteration.
const lighthouse = require('lighthouse');
const chromeLauncher = require('chrome-launcher');
function launchChromeAndRunLighthouse(sites, opts, config = null) {
return chromeLauncher.launch({chromeFlags: opts.chromeFlags}).then(chrome => {
opts.port = chrome.port;
const siteResults = [];
return new Promise((resolve, reject) => {
// batch async functions.
// C/O https://stackoverflow.com/questions/43082934/how-to-execute-promises-sequentially-passing-the-parameters-from-an-array
const runBatch = async (iterable, action) => {
for (const x of iterable) {
await action(x)
}
}
// func to run lighthouse
const doLightHouse = (site) => new Promise((resolve, reject) => {
lighthouse(site, opts, config).then(results => {
siteResults.push(results.lhr);
resolve();
});
});
// go go go
runBatch(sites, doLightHouse).then(d => {
chrome.kill().then((result) => {
resolve(siteResults)
})
});
});
});
}
const opts = {
chromeFlags: ['--show-paint-rects'],
onlyCategories: ['performance']
};
const sites = ['https://www.example.com', 'https://www.test.com']
launchChromeAndRunLighthouse(sites, opts).then(results => {
// Use results!
console.log(results);
});
Just to execute your code as test, we'll use async/await and IIFE
Then, will create function which will put all our request to array of non resolved promises, so we could use it with Promise.all()
You need to rewrite code in something like this:
(async() => {
const promisesToExecute = [];
const launchChromeAndRunLighthouse = async (url, opts, config = null) => {
const chrome = await return chromeLauncher.launch({chromeFlags: opts.chromeFlags});
opts.port = chrome.port;
promisesToExecute.push(lighthouse(url, opts, config));
}
const results = await Promise.all(promisesToExecute);
for(const result of results) {
const resolvedResult = await result.kill();
// here you can access your results.lhr
console.log(resolvedResult.lhr);
}
})()
Please note, this code wasn't tested, so there might be problems with kill() on result. But, the main goal is to answer your question and explain how to execute promises.
Also, if you don't want to execute all promises at the same time, you could use Promise.waterfall with some npm package, like this
I've written a script that recursively asynchronously modifies js files in a directory. It's made up of a search(dirname) function which searches a directory for js files, and a modify(filename) which does the modifying.
let totalFilesSearched = 0;
const search = (dir) => {
fs.readdir(dir, (err, list) => {
if (err) return;
list.forEach((filename) => {
const filepath = path.join(dir, filename);
if (filename.endsWith('.js')) {
modify(filepath);
} else if (fs.lstatSync(filepath).isDirectory()) {
search(filepath);
}
})
});
}
const modify = (filename) => {
fs.readFile(filename, 'utf8', (err, data) => {
if (err) console.log(err);
// ... my modification code ...
totalFilesSearched++;
});
}
search(args[0])
console.log(`Total files searched: ${totalFilesSearched}`);
I want to print out the totalFilesSearched at the end of my script but because my code is asynchronous, it just prints Total files searched: 0 right away.
Does anyone know how I'd wait until the script is about to end to print this out? I'm having trouble because both my search() and modify() functions are asynchronous.
Use Promises instead, and then call console.log when everything is resolved. Use promisify to turn the callbacks into promises:
const { promisify } = require('util');
const readFile = promisify(fs.readFile);
const readDir = util.promisify(fs.readdir);
const search = (dir) => (
readDir(dir).then((list) => (
Promise.all(list.map((filename) => {
const filepath = path.join(dir, filename);
if (filename.endsWith('.js')) {
return modify(filepath); // recursively return the promise
} else if (fs.lstatSync(filepath).isDirectory()) {
return search(filepath); // recursively return the promise
}
}))
))
.catch(err => void 0)
);
const modify = (filename) => (
readFile(filename, 'utf8')
.then((data) => {
// other code
totalFilesSearched++;
}).catch(err => console.log(err))
)
search(args[0])
.then(() => {
console.log(`Total files searched: ${totalFilesSearched}`);
});
Self answer:
Just use process.on('exit', callback_function_to_execute_at_end)
Its built into node, your callback will get executed right before the process exits.
I have the promise function that execute async function in the loop few times for different data. I want to wait till all async functions will be executed and then resolve(), (or call callback function in non-promise function):
var readFiles = ()=>{
return new Promise((resolve,reject)=>{
var iterator = 0;
var contents = {};
for(let i in this.files){
iterator++;
let p = path.resolve(this.componentPath,this.files[i]);
fs.readFile(p,{encoding:'utf8'},(err,data)=>{
if(err){
reject(`Could not read ${this.files[i]} file.`);
} else {
contents[this.files[i]] = data;
iterator--;
if(!iterator) resolve(contents);
}
});
}
if(!iterator) resolve(contents); //in case of !this.files.length
});
};
I increase iterator on every loop repetition, then, in async function's callback decrease iterator and check if all async functions are done (iterator===0), if so - call resolve().
It works great, but seems not elegant and readable. Do you know any better way for this issue?
Following up the comment with some code and more detail!
Promise.all() takes an iterator, and waits for all promises to either resolve or reject. It will then return the results of all the promises. So instead of keeping track of when all promises resolve, we can create little promises and add them to an array. Then, use Promise.all() to wait for all of them to resolve.
const readFiles = () => {
const promises = [];
for(let i in files) {
const p = path.resolve(componentPath, files[i]);
promises.push(new Promise((resolve, reject) => {
fs.readFile(p, {encoding:'utf8'}, (err, data) => {
if(err) {
reject(`Could not read ${files[i]} file.`);
} else {
resolve(data);
}
});
}));
}
return Promise.all(promises);
};
const fileContents = readFiles().then(contents => {
console.log(contents)
})
.catch(err => console.error(err));
You only need push all the Promises into an array to then pass it as argument to Promise.all(arrayOfPromises)
try something like this:
var readFiles = () => {
var promises = [];
let contents = {};
var keys_files = Object.keys(this.files);
if (keys_files.length <= 0) {
var promise = new Promise((resolve, reject) => {
resolve(contents);
});
promises.push(promise);
}
keys_files.forEach((key) => {
var file = this.files[key];
var promise = new Promise((resolve, reject) => {
const currentPath = path.resolve(this.componentPath, file);
fs.readFile(p,{encoding:'utf8'},(err, data) => {
if (err) {
return reject(`Could not read ${file} file.`);
}
contents[file] = data;
resolve(contents)
});
});
});
return Promises.all(promises);
}
Then you should use the function like so:
// this will return a promise that contains an array of promises
var readAllFiles = readFiles();
// the then block only will execute if all promises were resolved if one of them were reject so all the process was rejected automatically
readAllFiles.then((promises) => {
promises.forEach((respond) => {
console.log(respond);
});
}).catch((error) => error);
If you don't care if one of the promises was rejected, maybe you should do the following
var readFiles = () => {
var promises = [];
let contents = {};
var keys_files = Object.keys(this.files);
if (keys_files.length <= 0) {
var promise = new Promise((resolve, reject) => {
resolve(contents);
});
promises.push(promise);
}
keys_files.forEach((key) => {
var file = this.files[key];
var promise = new Promise((resolve, reject) => {
const currentPath = path.resolve(this.componentPath, file);
fs.readFile(p,{encoding:'utf8'},(err, data) => {
// create an object with the information
let info = { completed: true };
if (err) {
info.completed = false;
info.error = err;
return resolve(info);
}
info.data = data;
contents[file] = info;
resolve(contents)
});
});
});
return Promises.all(promises);
}
Copied from comments:
Also - you might want to use fs-extra, a drop-in replacement for fs, but with promise support added.
Here's how that goes:
const fs = require('fs-extra');
var readFiles = ()=>{
let promises = files
.map(file => path.resolve(componentPath, file))
.map(path => fs.readFile(path));
return Promise.all(promises);
});
Nice and clean. You can then get contents like this:
readFiles()
.then(contents => { ... })
.catch(error => { ... });
This will fail on first error though (because that's what Promise.all does). If you want individual error handling, you can add another map line:
.map(promise => promise.catch(err => err));
Then you can filter the results:
let errors = contents.filter(content => content instanceof Error)
let successes = contents.filter(content => !(content instanceof Error))