On NodeJS I need to make a grep like function for log research purposes and I'm trying to make it using readline (since I don't want readline-sync). I've read many messages, tutorials, documentation, stackoverflow posts, and so on, but I couldn't understood how to make it works.
const grep = async function(pattern, filepath){
return new Promise((resolve, reject)=>{
let regex = new RegExp(pattern);
let fresult = ``;
let lineReader = require(`readline`).createInterface({
input: require(`fs`).createReadStream(filepath)
});
lineReader.on(`line`, function (line) {
if(line.match(regex)){
fresult += line;
}
});
resolve(fresult);
});
}
let getLogs = await grep(/myregex/gi, filepath);
console.log(getLogs);
Which gives me:
SyntaxError: await is only valid in async functions and the top level bodies of modules
Where am I wrong? I feel like I'm good but did a beginner mistake which is dancing just under my eyes.
What the other answers have missed is that you have two problems in your code.
The error in your question:
Top level await (await not wrapped in an async function) is only possible when running Node.js "as an ES Module", which is when you are using import {xyz} from 'module' rather than const {xyz} = require('module').
As mentioned, one way to fix this is to wrap it in an async function:
// Note: You omitted the ; on line 18 (the closing } bracket)
// which will cause an error, so add it.
(async () => {
let getLogs = await grep(/myregex/gi, filepath);
console.log(getLogs);
})();
A different option is to save your file as .mjs, not .js. This means you must use import rather than require.
The third option is to create a package.json and specify "type": "module". Same rules apply.
A more fundamental issue:
When you call resolve() the event handler in lineReader.on('line') will not have been executed yet. This means that you will be resolving to an empty string and not the user's input. Declaring a function async does nothing to wait for events/callbacks, after all.
You can solve this by waiting for the 'close' event of Readline & only then resolve the promise.
const grep = async function(pattern, filepath){
return new Promise((resolve, reject)=>{
let regex = new RegExp(pattern);
let fresult = ``;
let lineReader = require(`readline`).createInterface({
input: require(`fs`).createReadStream(filepath)
});
lineReader.on(`line`, function (line) {
if(line.match(regex)){
fresult += line;
}
});
// Wait for close/error event and resolve/reject
lineReader.on('close', () => resolve(fresult));
lineReader.on('error', reject);
});
}; // ";" was added
(async () => {
let getLogs = await grep(/myregex/gi, 'foo');
console.log("output", getLogs);
})();
A tip
The events module has a handy utility function named once that allows you to write your code in a shorter and clearer way:
const { once } = require('events');
// If you're using ES Modules use:
// import { once } from 'events'
const grep = async function(pattern, filepath){
// Since the function is 'async', no need for
// 'new Promise()'
let regex = new RegExp(pattern);
let fresult = ``;
let lineReader = require(`readline`).createInterface({
input: require(`fs`).createReadStream(filepath)
});
lineReader.on(`line`, function (line) {
if(line.match(regex)){
fresult += line;
}
});
// Wait for the first 'close' event
// If 'lineReader' emits an 'error' event, this
// will throw an exception with the error in it.
await once(lineReader, 'close');
return fresult;
};
(async () => {
let getLogs = await grep(/myregex/gi, 'foo');
console.log("output", getLogs);
})();
// If you're using ES Modules:
// leave out the (async () => { ... })();
Wrap the function call into an async IIFE:
(async()=>{
let getLogs = await grep(/myregex/gi, filepath);
console.log(getLogs);
})()
try this:
async function run(){
let getLogs = await grep(/myregex/gi, `/`);
console.log(getLogs);
}
run();
Related
I wrote this code in lib/helper.js:
var myfunction = async function(x,y) {
....
return [variableA, variableB]
}
exports.myfunction = myfunction;
Then I tried to use it in another file :
var helper = require('./helper.js');
var start = function(a,b){
....
const result = await helper.myfunction('test','test');
}
exports.start = start;
I got an error:
await is only valid in async function
What is the issue?
The error is not refering to myfunction but to start.
async function start() {
....
const result = await helper.myfunction('test', 'test');
}
// My function
const myfunction = async function(x, y) {
return [
x,
y,
];
}
// Start function
const start = async function(a, b) {
const result = await myfunction('test', 'test');
console.log(result);
}
// Call start
start();
I use the opportunity of this question to advise you about an known anti pattern using await which is : return await.
WRONG
async function myfunction() {
console.log('Inside of myfunction');
}
// Here we wait for the myfunction to finish
// and then returns a promise that'll be waited for aswell
// It's useless to wait the myfunction to finish before to return
// we can simply returns a promise that will be resolved later
// useless async here
async function start() {
// useless await here
return await myfunction();
}
// Call start
(async() => {
console.log('before start');
await start();
console.log('after start');
})();
CORRECT
async function myfunction() {
console.log('Inside of myfunction');
}
// Here we wait for the myfunction to finish
// and then returns a promise that'll be waited for aswell
// It's useless to wait the myfunction to finish before to return
// we can simply returns a promise that will be resolved later
// Also point that we don't use async keyword on the function because
// we can simply returns the promise returned by myfunction
function start() {
return myfunction();
}
// Call start
(async() => {
console.log('before start');
await start();
console.log('after start');
})();
Also, know that there is a special case where return await is correct and important : (using try/catch)
Are there performance concerns with `return await`?
To use await, its executing context needs to be async in nature
As it said, you need to define the nature of your executing context where you are willing to await a task before anything.
Just put async before the fn declaration in which your async task will execute.
var start = async function(a, b) {
// Your async task will execute with await
await foo()
console.log('I will execute after foo get either resolved/rejected')
}
Explanation:
In your question, you are importing a method which is asynchronous in nature and will execute in parallel. But where you are trying to execute that async method is inside a different execution context which you need to define async to use await.
var helper = require('./helper.js');
var start = async function(a,b){
....
const result = await helper.myfunction('test','test');
}
exports.start = start;
Wondering what's going under the hood
await consumes promise/future / task-returning methods/functions and async marks a method/function as capable of using await.
Also if you are familiar with promises, await is actually doing the same process of promise/resolve. Creating a chain of promise and executes your next task in resolve callback.
For more info you can refer to MDN DOCS.
When I got this error, it turned out I had a call to the map function inside my "async" function, so this error message was actually referring to the map function not being marked as "async". I got around this issue by taking the "await" call out of the map function and coming up with some other way of getting the expected behavior.
var myfunction = async function(x,y) {
....
someArray.map(someVariable => { // <- This was the function giving the error
return await someFunction(someVariable);
});
}
I had the same problem and the following block of code was giving the same error message:
repositories.forEach( repo => {
const commits = await getCommits(repo);
displayCommit(commits);
});
The problem is that the method getCommits() was async but I was passing it the argument repo which was also produced by a Promise. So, I had to add the word async to it like this: async(repo) and it started working:
repositories.forEach( async(repo) => {
const commits = await getCommits(repo);
displayCommit(commits);
});
If you are writing a Chrome Extension and you get this error for your code at root, you can fix it using the following "workaround":
async function run() {
// Your async code here
const beers = await fetch("https://api.punkapi.com/v2/beers");
}
run();
Basically you have to wrap your async code in an async function and then call the function without awaiting it.
The current implementation of async / await only supports the await keyword inside of async functions Change your start function signature so you can use await inside start.
var start = async function(a, b) {
}
For those interested, the proposal for top-level await is currently in Stage 2: https://github.com/tc39/proposal-top-level-await
async/await is the mechanism of handling promise, two ways we can do it
functionWhichReturnsPromise()
.then(result => {
console.log(result);
})
.cathc(err => {
console.log(result);
});
or we can use await to wait for the promise to full-filed it first, which means either it is rejected or resolved.
Now if we want to use await (waiting for a promise to fulfil) inside a function, it's mandatory that the container function must be an async function because we are waiting for a promise to fulfiled asynchronously || make sense right?.
async function getRecipesAw(){
const IDs = await getIds; // returns promise
const recipe = await getRecipe(IDs[2]); // returns promise
return recipe; // returning a promise
}
getRecipesAw().then(result=>{
console.log(result);
}).catch(error=>{
console.log(error);
});
If you have called async function inside foreach update it to for loop
Found the code below in this nice article: HTTP requests in Node using Axios
const axios = require('axios')
const getBreeds = async () => {
try {
return await axios.get('https://dog.ceo/api/breeds/list/all')
} catch (error) {
console.error(error)
}
}
const countBreeds = async () => {
const breeds = await getBreeds()
if (breeds.data.message) {
console.log(`Got ${Object.entries(breeds.data.message).length} breeds`)
}
}
countBreeds()
Or using Promise:
const axios = require('axios')
const getBreeds = () => {
try {
return axios.get('https://dog.ceo/api/breeds/list/all')
} catch (error) {
console.error(error)
}
}
const countBreeds = async () => {
const breeds = getBreeds()
.then(response => {
if (response.data.message) {
console.log(
`Got ${Object.entries(response.data.message).length} breeds`
)
}
})
.catch(error => {
console.log(error)
})
}
countBreeds()
In later nodejs (>=14), top await is allowed with { "type": "module" } specified in package.json or with file extension .mjs.
https://www.stefanjudis.com/today-i-learned/top-level-await-is-available-in-node-js-modules/
This in one file works..
Looks like await only is applied to the local function which has to be async..
I also am struggling now with a more complex structure and in between different files. That's why I made this small test code.
edit: i forgot to say that I'm working with node.js.. sry. I don't have a clear question. Just thought it could be helpful with the discussion..
function helper(callback){
function doA(){
var array = ["a ","b ","c "];
var alphabet = "";
return new Promise(function (resolve, reject) {
array.forEach(function(key,index){
alphabet += key;
if (index == array.length - 1){
resolve(alphabet);
};
});
});
};
function doB(){
var a = "well done!";
return a;
};
async function make() {
var alphabet = await doA();
var appreciate = doB();
callback(alphabet+appreciate);
};
make();
};
helper(function(message){
console.log(message);
});
A common problem in Express:
The warning can refer to the function, or where you call it.
Express items tend to look like this:
app.post('/foo', ensureLoggedIn("/join"), (req, res) => {
const facts = await db.lookup(something)
res.redirect('/')
})
Notice the => arrow function syntax for the function.
The problem is NOT actually in the db.lookup call, but right here in the Express item.
Needs to be:
app.post('/foo', ensureLoggedIn("/join"), async function (req, res) {
const facts = await db.lookup(something)
res.redirect('/')
})
Basically, nix the => and add async function .
"await is only valid in async function"
But why? 'await' explicitly turns an async call into a synchronous call, and therefore the caller cannot be async (or asyncable) - at least, not because of the call being made at 'await'.
Yes, await / async was a great concept, but the implementation is completely broken.
For whatever reason, the await keyword has been implemented such that it can only be used within an async method. This is in fact a bug, though you will not see it referred to as such anywhere but right here. The fix for this bug would be to implement the await keyword such that it can only be used TO CALL an async function, regardless of whether the calling function is itself synchronous or asynchronous.
Due to this bug, if you use await to call a real asynchronous function somewhere in your code, then ALL of your functions must be marked as async and ALL of your function calls must use await.
This essentially means that you must add the overhead of promises to all of the functions in your entire application, most of which are not and never will be asynchronous.
If you actually think about it, using await in a function should require the function containing the await keyword TO NOT BE ASYNC - this is because the await keyword is going to pause processing in the function where the await keyword is found. If processing in that function is paused, then it is definitely NOT asynchronous.
So, to the developers of javascript and ECMAScript - please fix the await/async implementation as follows...
await can only be used to CALL async functions.
await can appear in any kind of function, synchronous or asynchronous.
Change the error message from "await is only valid in async function" to "await can only be used to call async functions".
Working on a nodejs implementation of JSZip to create a zip file in binary string format and store its value in a global variable however in spite of using async/await the zipstring is being printed before the zipping process is finished,
Here is my code so far:
const JSZip = require("jszip");
const zip = new JSZip();
let s="not yet done";
let zipstring = "";
async function dozipping() {
zip.file("Hello.txt", "Hello World\n");
zip.generateAsync({ type: "binarystring" })
.then(function(content) {
console.log("doing");
zipstring = content;
});
}
async function status() {
console.log(s)
await dozipping();
console.log(zipstring)
s = "done";
console.log(s)
}
status();
Expected Output
not yet done
doing
*zipstring value*
done
Output I am currently getting
not yet done
done
doing
I am new to node and working with jszip for the 1st time and the documentation is a little confusing, sorry in advance if the question is too trivial
The reason you got nothing is promises are async and they are called once the async task is completed with the status as fulfilled or rejected. I have made some changes to it:
const zip = new JSZip();
let s="not yet done";
let zipstring = "";
async function dozipping(){
zip.file("Hello.txt", "Hello World\n");
try {
console.log("doing");
const response = await zip.generateAsync({type:"binarystring"});
zipstring = response;
return response;
} catch (error) {
console.error(error);
}
}
async function status(){
try {
console.log(s)
await dozipping();
console.log(zipstring)
s="done";
console.log(s)
} catch (error) {
console.error(error);
}
}
await status();
I would suggest you to first go through how promises work and how async\await should be used. They are just a syntactic sugar on it. I am attaching the link to through them.
This is because the .then from zip.generateAsync does not halt the execution. You are gonna need to use
zipstring = await zip.generateAsync(...);
Await halts the execution of an asynchronous function until the promise is resolved.
const JSZip = require('jszip');
const zip = new JSZip();
const content = fs.readFileSync("something.txt");
zip.file("something.txt", content);
let zip_file_path = "something.zip";
zip.generateNodeStream({ type: 'nodebuffer', streamFiles: true })
.pipe(fs.createWriteStream(zip_file_path))
.on('finish', () => {
/**/
});
I have 2 function: the first one is: readCSV that reads a csv file line by line, every lines contains a site name and for each one I have to call a web crawler.
the first function is:
async function readCSV(){
const fileStream = fs.createReadStream('./topm.csv');
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity
});
for await (const line of rl) {
var currentline=line.split(",");
var current_site = "http://www."+currentline[1];
await crawling("http://www."+currentline[1])
}
}
and the other one is: crawling(web_page) that crawls the page.
async function crawling(web_page){
try{
new Crawler().configure({depth: 1})
.crawl(web_page, function onSuccess(page) {
console.log(page.url);
});
}catch(error){
console.log("Error: "+error.message);
}
}
every function is async, but when I call readCSV I noticed that the line with await crawling(.....) doesn't wait the end of each one and execute a lot of pages in parallel, giving me for some pages this waring:
(node:757) MaxListenersExceededWarning: Possible EventEmitter memory leak detected. 11 pipe listeners added to [Request]. Use emitter.setMaxListeners() to increase limit
I also noticed that with a file having 100 lines (100 web pages) the crawler is calling just 84 times... I don't understand why
At the end I tried to add a promise inside readCSV function, in this way:
async function readCSV(){
const fileStream = fs.createReadStream('./topm.csv');
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity
});
for await (const line of rl) {
var currentline=line.split(",");
var current_site = "http://www."+currentline[1];
//await crawling("http://www."+currentline[1])
await (new Promise( resolve => {
new Crawler().configure({depth: 1})
.crawl(current_site, async (page) => {
console.log(page.url);
resolve();
});
}));
}
}
But in this way it works only for the first 4 site (to compile I need to add this --unhandled-rejections=strict)
From https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await
The await operator is used to wait for a Promise.
I think your problem is that your crawling function doesn't actually return a promise. So the code await crawling("http://www."+currentline[1]) evaluates immediately, returning the return value of the crawling function, which is undefined.
It is possible that this is the fix:
await new Crawler().configure({depth: 1})
//^^^
.crawl(web_page, function onSuccess(page) {
console.log(page.url);
});
It depends on what is returned by the crawl method, but if it returns a promise, that should be right.
I wrote this code in lib/helper.js:
var myfunction = async function(x,y) {
....
return [variableA, variableB]
}
exports.myfunction = myfunction;
Then I tried to use it in another file :
var helper = require('./helper.js');
var start = function(a,b){
....
const result = await helper.myfunction('test','test');
}
exports.start = start;
I got an error:
await is only valid in async function
What is the issue?
The error is not refering to myfunction but to start.
async function start() {
....
const result = await helper.myfunction('test', 'test');
}
// My function
const myfunction = async function(x, y) {
return [
x,
y,
];
}
// Start function
const start = async function(a, b) {
const result = await myfunction('test', 'test');
console.log(result);
}
// Call start
start();
I use the opportunity of this question to advise you about an known anti pattern using await which is : return await.
WRONG
async function myfunction() {
console.log('Inside of myfunction');
}
// Here we wait for the myfunction to finish
// and then returns a promise that'll be waited for aswell
// It's useless to wait the myfunction to finish before to return
// we can simply returns a promise that will be resolved later
// useless async here
async function start() {
// useless await here
return await myfunction();
}
// Call start
(async() => {
console.log('before start');
await start();
console.log('after start');
})();
CORRECT
async function myfunction() {
console.log('Inside of myfunction');
}
// Here we wait for the myfunction to finish
// and then returns a promise that'll be waited for aswell
// It's useless to wait the myfunction to finish before to return
// we can simply returns a promise that will be resolved later
// Also point that we don't use async keyword on the function because
// we can simply returns the promise returned by myfunction
function start() {
return myfunction();
}
// Call start
(async() => {
console.log('before start');
await start();
console.log('after start');
})();
Also, know that there is a special case where return await is correct and important : (using try/catch)
Are there performance concerns with `return await`?
To use await, its executing context needs to be async in nature
As it said, you need to define the nature of your executing context where you are willing to await a task before anything.
Just put async before the fn declaration in which your async task will execute.
var start = async function(a, b) {
// Your async task will execute with await
await foo()
console.log('I will execute after foo get either resolved/rejected')
}
Explanation:
In your question, you are importing a method which is asynchronous in nature and will execute in parallel. But where you are trying to execute that async method is inside a different execution context which you need to define async to use await.
var helper = require('./helper.js');
var start = async function(a,b){
....
const result = await helper.myfunction('test','test');
}
exports.start = start;
Wondering what's going under the hood
await consumes promise/future / task-returning methods/functions and async marks a method/function as capable of using await.
Also if you are familiar with promises, await is actually doing the same process of promise/resolve. Creating a chain of promise and executes your next task in resolve callback.
For more info you can refer to MDN DOCS.
When I got this error, it turned out I had a call to the map function inside my "async" function, so this error message was actually referring to the map function not being marked as "async". I got around this issue by taking the "await" call out of the map function and coming up with some other way of getting the expected behavior.
var myfunction = async function(x,y) {
....
someArray.map(someVariable => { // <- This was the function giving the error
return await someFunction(someVariable);
});
}
I had the same problem and the following block of code was giving the same error message:
repositories.forEach( repo => {
const commits = await getCommits(repo);
displayCommit(commits);
});
The problem is that the method getCommits() was async but I was passing it the argument repo which was also produced by a Promise. So, I had to add the word async to it like this: async(repo) and it started working:
repositories.forEach( async(repo) => {
const commits = await getCommits(repo);
displayCommit(commits);
});
If you are writing a Chrome Extension and you get this error for your code at root, you can fix it using the following "workaround":
async function run() {
// Your async code here
const beers = await fetch("https://api.punkapi.com/v2/beers");
}
run();
Basically you have to wrap your async code in an async function and then call the function without awaiting it.
The current implementation of async / await only supports the await keyword inside of async functions Change your start function signature so you can use await inside start.
var start = async function(a, b) {
}
For those interested, the proposal for top-level await is currently in Stage 2: https://github.com/tc39/proposal-top-level-await
async/await is the mechanism of handling promise, two ways we can do it
functionWhichReturnsPromise()
.then(result => {
console.log(result);
})
.cathc(err => {
console.log(result);
});
or we can use await to wait for the promise to full-filed it first, which means either it is rejected or resolved.
Now if we want to use await (waiting for a promise to fulfil) inside a function, it's mandatory that the container function must be an async function because we are waiting for a promise to fulfiled asynchronously || make sense right?.
async function getRecipesAw(){
const IDs = await getIds; // returns promise
const recipe = await getRecipe(IDs[2]); // returns promise
return recipe; // returning a promise
}
getRecipesAw().then(result=>{
console.log(result);
}).catch(error=>{
console.log(error);
});
If you have called async function inside foreach update it to for loop
Found the code below in this nice article: HTTP requests in Node using Axios
const axios = require('axios')
const getBreeds = async () => {
try {
return await axios.get('https://dog.ceo/api/breeds/list/all')
} catch (error) {
console.error(error)
}
}
const countBreeds = async () => {
const breeds = await getBreeds()
if (breeds.data.message) {
console.log(`Got ${Object.entries(breeds.data.message).length} breeds`)
}
}
countBreeds()
Or using Promise:
const axios = require('axios')
const getBreeds = () => {
try {
return axios.get('https://dog.ceo/api/breeds/list/all')
} catch (error) {
console.error(error)
}
}
const countBreeds = async () => {
const breeds = getBreeds()
.then(response => {
if (response.data.message) {
console.log(
`Got ${Object.entries(response.data.message).length} breeds`
)
}
})
.catch(error => {
console.log(error)
})
}
countBreeds()
In later nodejs (>=14), top await is allowed with { "type": "module" } specified in package.json or with file extension .mjs.
https://www.stefanjudis.com/today-i-learned/top-level-await-is-available-in-node-js-modules/
This in one file works..
Looks like await only is applied to the local function which has to be async..
I also am struggling now with a more complex structure and in between different files. That's why I made this small test code.
edit: i forgot to say that I'm working with node.js.. sry. I don't have a clear question. Just thought it could be helpful with the discussion..
function helper(callback){
function doA(){
var array = ["a ","b ","c "];
var alphabet = "";
return new Promise(function (resolve, reject) {
array.forEach(function(key,index){
alphabet += key;
if (index == array.length - 1){
resolve(alphabet);
};
});
});
};
function doB(){
var a = "well done!";
return a;
};
async function make() {
var alphabet = await doA();
var appreciate = doB();
callback(alphabet+appreciate);
};
make();
};
helper(function(message){
console.log(message);
});
A common problem in Express:
The warning can refer to the function, or where you call it.
Express items tend to look like this:
app.post('/foo', ensureLoggedIn("/join"), (req, res) => {
const facts = await db.lookup(something)
res.redirect('/')
})
Notice the => arrow function syntax for the function.
The problem is NOT actually in the db.lookup call, but right here in the Express item.
Needs to be:
app.post('/foo', ensureLoggedIn("/join"), async function (req, res) {
const facts = await db.lookup(something)
res.redirect('/')
})
Basically, nix the => and add async function .
"await is only valid in async function"
But why? 'await' explicitly turns an async call into a synchronous call, and therefore the caller cannot be async (or asyncable) - at least, not because of the call being made at 'await'.
Yes, await / async was a great concept, but the implementation is completely broken.
For whatever reason, the await keyword has been implemented such that it can only be used within an async method. This is in fact a bug, though you will not see it referred to as such anywhere but right here. The fix for this bug would be to implement the await keyword such that it can only be used TO CALL an async function, regardless of whether the calling function is itself synchronous or asynchronous.
Due to this bug, if you use await to call a real asynchronous function somewhere in your code, then ALL of your functions must be marked as async and ALL of your function calls must use await.
This essentially means that you must add the overhead of promises to all of the functions in your entire application, most of which are not and never will be asynchronous.
If you actually think about it, using await in a function should require the function containing the await keyword TO NOT BE ASYNC - this is because the await keyword is going to pause processing in the function where the await keyword is found. If processing in that function is paused, then it is definitely NOT asynchronous.
So, to the developers of javascript and ECMAScript - please fix the await/async implementation as follows...
await can only be used to CALL async functions.
await can appear in any kind of function, synchronous or asynchronous.
Change the error message from "await is only valid in async function" to "await can only be used to call async functions".
In https://stackoverflow.com/a/18658613/779159 is an example of how to calculate the md5 of a file using the built-in crypto library and streams.
var fs = require('fs');
var crypto = require('crypto');
// the file you want to get the hash
var fd = fs.createReadStream('/some/file/name.txt');
var hash = crypto.createHash('sha1');
hash.setEncoding('hex');
fd.on('end', function() {
hash.end();
console.log(hash.read()); // the desired sha1sum
});
// read all file and pipe it (write it) to the hash object
fd.pipe(hash);
But is it possible to convert this to using ES8 async/await instead of using the callback as seen above, but while still keeping the efficiency of using streams?
The await keyword only works on promises, not on streams. There are ideas to make an extra stream-like data type that would get its own syntax, but those are highly experimental if at all and I won't go into details.
Anyway, your callback is only waiting for the end of the stream, which is a perfect fit for a promise. You'd just have to wrap the stream:
var fd = fs.createReadStream('/some/file/name.txt');
var hash = crypto.createHash('sha1');
hash.setEncoding('hex');
// read all file and pipe it (write it) to the hash object
fd.pipe(hash);
var end = new Promise(function(resolve, reject) {
hash.on('end', () => resolve(hash.read()));
fd.on('error', reject); // or something like that. might need to close `hash`
});
There also exists a helper function to do just that in more recent versions of nodejs - pipeline from the stream/promises module:
import { pipeline } from 'node:stream/promises';
const fd = fs.createReadStream('/some/file/name.txt');
const hash = crypto.createHash('sha1');
hash.setEncoding('hex');
// read all file and pipe it (write it) to the hash object
const end = pipeline(fd, hash);
Now you can await that promise:
(async function() {
let sha1sum = await end;
console.log(sha1sum);
}());
If you are using node version >= v10.0.0 then you can use stream.pipeline and util.promisify.
const fs = require('fs');
const crypto = require('crypto');
const util = require('util');
const stream = require('stream');
const pipeline = util.promisify(stream.pipeline);
const hash = crypto.createHash('sha1');
hash.setEncoding('hex');
async function run() {
await pipeline(
fs.createReadStream('/some/file/name.txt'),
hash
);
console.log('Pipeline succeeded');
}
run().catch(console.error);
Node V15 now has a promisfiy pipeline in stream/promises.
This is the cleanest and most official way.
const { pipeline } = require('stream/promises');
async function run() {
await pipeline(
fs.createReadStream('archive.tar'),
zlib.createGzip(),
fs.createWriteStream('archive.tar.gz')
);
console.log('Pipeline succeeded.');
}
run().catch(console.error);
We all should appreciate how much works it's done here:
Capture errors in all the streams.
Destroy unfinished streams when error is raised.
Only return when the last writable stream is finished.
This pipe thing is one of the most powerful feature Node.JS has. Making it fully async is not easy. Now we have it.
Something like this works:
for (var res of fetchResponses){ //node-fetch package responses
const dest = fs.createWriteStream(filePath,{flags:'a'});
totalBytes += Number(res.headers.get('content-length'));
await new Promise((resolve, reject) => {
res.body.pipe(dest);
res.body.on("error", (err) => {
reject(err);
});
dest.on("finish", function() {
resolve();
});
});
}
2021 Update:
New example from Node documentation:
async function print(readable) {
readable.setEncoding('utf8');
let data = '';
for await (const chunk of readable) {
data += chunk;
}
console.log(data);
}
see https://nodejs.org/api/stream.html#stream_readable_symbol_asynciterator
I would comment, but don't have enough reputation.
A WORD OF CAUTION:
If you have an application that is passing streams around AND doing async/await, be VERY CAREFUL to connect ALL pipes before you await. You can end up with streams not containing what you thought they did. Here's the minimal example
const { PassThrough } = require('stream');
async function main() {
const initialStream = new PassThrough();
const otherStream = new PassThrough();
const data = [];
otherStream.on('data', dat => data.push(dat));
const resultOtherStreamPromise = new Promise(resolve => otherStream.on('end', () => { resolve(Buffer.concat(data)) }));
const yetAnotherStream = new PassThrough();
const data2 = [];
yetAnotherStream.on('data', dat => data2.push(dat));
const resultYetAnotherStreamPromise = new Promise(resolve => yetAnotherStream.on('end', () => { resolve(Buffer.concat(data2)) }));
initialStream.pipe(otherStream);
initialStream.write('some ');
await Promise.resolve(); // Completely unrelated await
initialStream.pipe(yetAnotherStream);
initialStream.end('data');
const [resultOtherStream, resultYetAnotherStream] = await Promise.all([
resultOtherStreamPromise,
resultYetAnotherStreamPromise,
]);
console.log('other stream:', resultOtherStream.toString()); // other stream: some data
console.log('yet another stream:', resultYetAnotherStream.toString()); // yet another stream: data
}
main();
I believe it will be helpful for someone:
async function readFile(filename) {
let records = []
return new Promise(resolve => {
fs.createReadStream(filename)
.on("data", (data) => {
records.push(data);
})
.on("end", () => {
resolve(records)
});
})
}