Based on a code snippet found here on stackoverflow, I want to read all files in a directory and then proceed.
I've added a promise, but this somehow doesn't work.
My directory contains 2 files and the console log output is:
promise resolved
inside filenames
inside filenames
inside readFiles
inside readFiles
function readFiles(dirname, onFileContent, onError) {
return new Promise((resolve, reject) => {
fs.readdir(dirname, function(err, filenames) {
filenames.forEach(function(filename) {
console.log('inside filenames');
fs.readFile(dirname + filename, 'utf-8', function(err, content) {
onFileContent(filename, content);
});
});
});
});
}
var data = [];
readFiles('datadir/', function(filename, content) {
console.log('inside readFiles');
data.push(filename);
}).then(
console.log('promise resolved');
//proceed handling the data-array
);
The promise does not "resolve first". The call to console.log executes before your first file is read.
You're never calling resolve on your promise, so the then is never being called. However you're passing the result of console.log to the then. the result of console.log is void.
You can test this by correcting the problem:
readFiles('datadir/', function(filename, content) {
console.log('inside readFiles');
data.push(filename);
}).then(function(){ // NOTE: addition of function(){..}
console.log('promise resolved');
//proceed handling the data-array
});
And you'll notice the message is never written to the console.
So that's whats wrong - how to fix it. It takes some thinking to wrap your head round the totally async/promise based code in node.
I'm assuming that you want to wait for all the files to have the contents read before resolving your promise. This is a little tricky as you have 2 async calls (reading the list of files, and then individually reading their contents). It may well be easier to wrap the reading of the file into its own promise. Something like this:
function readFile(filePath){
return new Promise((resolve,reject) => {
fs.readFile(filePath, "utf-8", function(err,content) => {
if(err) reject(err)
else resolve({path:filePath, content:content})
});
});
}
Do the same for readdir so as to make that also chainable:
function readDirectory(dir){
return new Promise((resolve,reject) => {
fs.readdir(dirname, function(err, filenames) {
if(err) reject(err);
else{
resolve(filenames.map(fn => dir + fn));
}
});
});
}
The reason to do this is that you can then chain on a Promise.all to wait for all the file contents too.
function readFileContents(dirname) {
return readDirectory(dirname)
.then(files =>
Promise.all(files.map(file => readFile(file))
);
}
Usage:
readFileContents('datadir/').then(files => {
files.forEach(file => {
console.log(file.path, file.content.length);
});
});
Related
I'm using a csv-parser npm module to read a csv file, process it, and then create a statistical model based on the data. The issue I'm having is that the other file that uses this isn't waiting for the model to finish before moving on, so it ends up trying to use values/methods that are not yet defined. Based on other posts, this is what I have:
this.read = async function(){
return new Promise((resolve, reject) => {
console.log("in mv read");
fs.createReadStream("./assets/stats-csv.csv")
.pipe(csv({
mapValues: ({ header, index, value }) => this.toNumber(header, value)
}))
.on('data', (data) => this.process(data))
.on('error', err => {
reject(err);
})
.on('end', () => {
this.model();
console.log('finished mv model');
resolve(true);
});
})
}
And then the other file uses the method the following way:
this.train_mv = async function(){
console.log("in train mv wrapper")
const success = await this.mvReg.read();
return success;
//console.log(success);
}
I added the "success" bit just to see if returning and using a value from the promise would help, but it doesn't. The function just moves on and doesn't even go to the "return success" line. Am I missing something about async/await? Shouldn't the train_mv function pause and wait until the promise resolves? I would appreciate any help. Thanks!
I'm likely reinventing the wheel here, but I want to make a function that takes a filename and returns the base64 encoded string asynchronously.
I'm really only familiar with callbacks, but trying to understand promises as well. My initial callback function looks like this --
blob2Base64callback.js
module.exports = {
blob2base64 : function(file) {
fs.readFile(file, 'utf8', function(err, contents){
if (err) {
console.log('file not found');
return '';
} else {
var base64str = Buffer.from(contents).toString('base64');
console.log(base64str.length);
return base64str;
}
});
}, //..other functions
};
This callback doesn't work because I'm not actually returning something from the blob2base4 function and if I modify this to something like the following:
module.exports = {
blob2base64 : function(file) {
return fs.readFile(file, 'utf8', function(err, contents){
//same implementation as above, cut to improve readability
});
}
};
It returns undefined because it doesn't wait to be executed. As far as I know, there is no possible way to return something to blob2base64 using callbacks.
So I looked into promises and tried the following:
blob2Base64promise.js
blob2base64 : function(file) {
console.log('function called');
const getFile = new Promise(function(resolve, reject){
fs.readFile(file, 'utf8', function(err, contents){
if (err) {
console.log('file not found');
reject(err)
} else {
//console.log(contents);
resolve(contents);
}
});
}).then(contents =>{
var base64str = Buffer.from(contents).toString('base64');
return base64str;
}).catch(err => console.log(err));
}
Is there anyway I can return the promise to the function and extract the resolve portion to get base64str? Am I right in thinking, this behavior is only possible with promises, but not callbacks. Or is it not possible with either syntax?
Also, I thought callbacks and promises couldn't be used together, yet this promise is just a wrapper around a callback.
(Answering in reverse order)
Typically callbacks and Promises are not mixed, just because it's messy. They are possible to intertwine, if necessary. There are utilities to turn callbacks into Promises ("promisify"), and Promises are becoming much more common, especially with the async/await syntax of ES6.
For Promises, be sure to return the Promise itself. In your example, return getFile. What the Promise chain eventually returns you must await. That could be the async/await keywords in ES6. You're call could be:
async function() {
let result = await blob2base64('myFile.png')
}
Which is just a nicer syntax for knowing your are returning a Promise. You could also do it like this:
function () {
blob2base64('myFile.png')
.then( result => {
// use result here
})
// you could catch as well.
}
For callbacks, you need to also pass a callback to your function. That is what makes it asynchronous and where you give your results:
blob2base64 : function(file, callback) {
fs.readFile(file, 'utf8', function(err, contents){
if (err) {
console.log('file not found');
callback(err)
} else {
var base64str = Buffer.from(contents).toString('base64');
console.log(base64str.length);
callback(null, base64str)
}
});
And you would call it like: blob2base64( 'myFile.png', function (err,result) {
})
I am attempting to open a 3rd party generated PDF that I know will fail occasionally. I am trying both pdf2json and pdfreader, and am encountering the same issue, which I'm not sure if it how I am attempting to handle the libraries in a promise.
When I receive an PDF, I would like to open it, to ensure that it is a valid PDF before passing it on for processing.
I am doing it like so:
function printRawItems(filename, callback){
new pdfReader.PdfReader().parseBuffer(filename, function(err, item) {
if (err) {
callback(err);
} else if (!item) {
callback();
} else if (item.text) {
callback(null, item)
} else if (item.page){
console.log("page =", item.page);
callback(null, item);
} else if (item.x){
console.log([item.x, item.y, item.oc, item.A, Math.floor(item.w), item.text].join("\t"));
callback(null, item);
} else {
console.warn(item);
}
});
}
function isValidPdf(buffer) {
return new Promise((resolve, reject) => {
printRawItems(buffer, function(err, item){
if (err) {
return reject(err);
} else if (item) {
return resolve(item);
}
return reject();
})
}).catch(err => {throw err})
}
The buffer being passed in to the "isValidPdf" is from an http request.
Now from what I can tell the callback I'm passing into the parseBuffer appears to get run twice. Once when the file is opened (and so item is "file"), and a second when it is parsed. After the first pass the promise in "isValidPdf" is resolved and the callback being passed in is never called, so it isn't rejected. The second run of the parseBuffer callback displays errors, which throws the exception, but by that time the promise is resolved and bad things happen.
Am I misunderstanding how the callbacks work, or are these libraries doing something wrong, and I should open a support ticket?
You're not misunderstanding how callbacks work. Just using them in the wrong way. I had a quick look at pdf2json and it seems you first create the parser, then do .parseBuffer() and wait for events to fire, e.g.:
function printRawItems (buffer, cb) {
const parser = new PDFParser()
parser.on('pdfParser_dataError', errData => {
cb(errData.parserError)
})
parser.on('pdfParser_dataReady', pdfData => {
cb(null, pdfData)
})
parser.parseBuffer(buffer)
}
I am creating a function in node.js that loops through the files of a directory. It is supposed to add the file name to the returnData variable, then return the returnData. However, it keeps returning nothing. I've put a few console.log statements in the function to help me debug, but I can't figure out why it won't work.
function loopMusic (directory) {
var returnData = "";
fs.readdir (directory, function (err, files) {
if (err) {
console.log (err);
}
files.forEach (function (file, index) {
returnData += file;
console.log (returnData);
});
});
console.log (returnData);
return returnData;
}
The first console.log statement is able to print the files, but the one right before the return just prints a new line.
You can make the function return a promise:
function loopMusic (directory) {
return new Promise((resolve, reject) => {
fs.readdir (directory, function (err, files) {
if (err) {
reject(err);
return;
}
files.forEach (function (file, index) {
returnData += file;
console.log (returnData);
});
resolve(returnData);
});
}
You would use in that way:
loopMusic('...')
.then((data) => console.log(data))
.catch((err) => ...);
fs.readdir is asynchronous, meaning it does not return with the result when you call it. Instead the result is provided to the callback, which is called when the command finishes processing. It "calls-back" to the function you provided when it's done (hence the name).
If you wanted to do this synchronously you can do the following:
function loopMusic (directory) {
var returnData = "";
var files = fs.readdirSync(directory);
files.forEach (function (file, index) {
returnData += file;
console.log (returnData);
});
console.log(files);
return returnData;
}
That would return a string of mushed together file paths, as in your question.
However, blocking isn't usually a good idea and you should use the asynchronous version. I like to return a Promise in these situations. Here's an example that returns a promise filled with that string. This technically isn't necessary since the callback could just be used...but lets just pretend.
function loopMusic (directory) {
return new Promise(function(resolve, reject) {
fs.readdir (directory, function (err, files) {
if (err) {
return reject(err);
}
let returnData = "";
files.forEach (function (file, index) {
returnData += file;
});
resolve(returnData);
});
});
}
Usage:
var musicPromise = loopMusic(dir);
musicPromise.then((musicStr) => console.log(musicStr)), (err) => console.log(err));
The asynchronous nature of this makes it a bit hard to follow since things don't happen in order, but when using Promises the then() is used to handle what happens on success (or failure) when it does complete later on.
Finally, if you're using ES2017+ (the newest version of Node) you can use the async/await pattern. Keep in mind my promise example above:
async function loopMusicAsync(directory) {
try{
return await loopMusic(directory); //promise returned
}
catch(error) {
console.log(error); //promise rejected
return null;
}
}
I have two promises. One that reads a sample.txt file and another that reads all the files from a /books/ folder. The second promise uses a function called readFiles, which takes the dirnames and uses them to look though each file. When all the promises are ready the code inside then should run:
const p1 = new Promise((resolve, reject) => {
fs.readdir(__dirname + '/books/', (err, archives) => {
// archives = [ 'archive1.txt', 'archive2.txt']
readFiles(archives, result => {
if (archives.length === result.length) resolve(result)
else reject(result)
})
})
})
const p2 = new Promise((resolve, reject) => {
fs.readFile('sample.txt', 'utf-8', (err, sample) => {
resolve(sample)
})
})
Promise.all([p1, p2]).then(values => {
console.log('v:', values)
}).catch(reason => {
console.log('reason:', reason)
})
function readFiles (archives, callback) {
const result = []
archives.forEach(archive => {
fs.readFile(__dirname + '/books/' + archive, 'utf-8', (err, data) => {
result.push(data)
callback(result)
})
})
}
However, Promise.all always get rejected:
reason: [ 'archive 1\n' ]
What am I doing wrong?
Promises are one-shot devices. Once they've been rejected or resolved, their state can never change. With that in mind, readFiles() calls its callback for every file that it reads and you reject or resolve every time that callback is called, but the way you are using it, you check:
if (archives.length === result.length)
which will never be true on the first one and then you reject. Once that promise is rejected, its state cannot change. Subsequent calls to the callback will also call reject() and then the last one will call resolve(), but the state is long since set so only the first call to reject() or resolve() actually does anything. The others are simply ignored. So, p1 will always reject, thus Promise.all() that uses p1 will always reject.
You need to change readFiles() to either only call its callback once when it is done with all the files or change it to return a single promise that resolves when all the files are read or change how you're using the callback so you don't reject the first time it is called.
In general, if you're going to use promises, then you want to promisify at the lowest level and use the advantages of promises (particular for error propagation) everywhere rather than mix callbacks and promises. To that end, I'd suggest:
fs.readFileP = function(fname, encoding) {
return new Promise(function(resolve, reject) {
fs.readFile(fname, encoding, function(err, data) {
if (err) return reject(err);
resolve(data);
});
});
}
function readFiles(archives, encoding, callback) {
return Promise.all(archives.map(function(file) {
return fs.readFileP(file, encoding);
}));
}
Or, going a level deeper and promisifying fs.readdir() also, you'd get this:
// helper functions
fs.readdirP = function(dir) {
return new Promise(function(resolve, reject) {
fs.readdir(dir, function(err, files) {
if (err) return reject(err);
resolve(files);
});
});
}
fs.readFileP = function(fname, encoding) {
return new Promise(function(resolve, reject) {
fs.readFile(fname, encoding, function(err, data) {
if (err) return reject(err);
resolve(data);
});
});
}
function readFiles(archives, encoding) {
encoding = encoding || 'utf8';
return Promise.all(archives.map(function(file) {
return fs.readFileP(file, encoding);
}));
}
// actual logic for your operation
const p1 = fs.readdirP(__dirname + '/books/').then(readFiles);
const p2 = fs.readFileP('sample.txt', 'utf-8');
Promise.all([p1, p2]).then(values => {
console.log('v:', values);
}).catch(reason => {
console.log('reason:', reason);
});
If you use the Bluebird promise library which makes it easy to promisify whole modules at once and has some extra functions for managing Promise flow control, then the above code simplifies to this:
const Promise = require('bluebird');
const fs = Promise.promisifyAll(require('fs'));
const p1 = fs.readdirAsync(__dirname + '/books/').then(files => {
return Promise.map(archives, file => {
return fs.readFileAsync(file, 'utf8');
});
});
const p2 = fs.readFileAsync('sample.txt', 'utf-8');
Promise.all([p1, p2]).then(values => {
console.log('v:', values);
}).catch(reason => {
console.log('reason:', reason);
});
In this block of code, the Promise.promisifyAll() line of code creates promisified versions of every method on the fs module with the Async suffix on them. Here, we use fs.readFileAsync() and fs.readdirAsync() so we can use promises for everything.