I have this code that serves every markdown file in the './markdown' folder. At '/api/markdown/filename'.
var apiRouter = express.Router();
markdownFolder = './markdown/';
apiRouter.get('/:markdown_file_noext', function(req, res) {
fs.readdir(markdownFolder, function(err, markdown) {
if (err) throw err;
markdown.forEach(function(file) {
fs.readFile(markdownFolder + file, 'utf8', function(err, file_content) {
if (err) throw err;
fileNoExtension = file.slice(0, file.indexOf('.'));
if (req.params.markdown_file_noext == fileNoExtension) {
res.json({
'title': fileNoExtension,
'markdown': marked(file_content)
});
};
});
});
});
});
But i end having a ton of callbacks do the the nature of the 'fs' methods. How do i avoid this?
Using Q as promise library:
const Q = require('q');
const fs = require('fs');
const markdownFolder = './markdown/';
const readdir = Q.nfbind(fs.readdir);
const readFile = Q.nfbind(fs.readFile);
readdir(markdownFolder).then(markdown => {
const promises = [];
markdown.forEach(file => promises.push(readFile(markdownFolder + file, 'utf8')));
return Q.all(promises);
}).then(files => {
// Do your magic.
}).catch(error => {
// Do something with error.
});
You have different option.
Use named Function instead of anonymus functinos. It would make it a little bit more readable but you will still be using callbacks.
Use Promises, but you will need to use bluebird to wrap the fs module.
For a more advance option, you can use generators and Promises to make your code look more like a sync way. Take a look at co or bluebird.coroutine.
With Promises you could do like this:
const path = require('path');
var apiRouter = express.Router();
markdownFolder = './markdown/';
apiRouter.get('/:markdown_file_noext', function(req, res) {
readdir(markdownFolder)
.then((files) => {
const tasks = files.map((file) => {
const filePath = path.resolve(markdownFolder, file);
return readFile(filePath);
});
return Promise.all(tasks); // Read all files
})
.then((fileContents) => {
return fileContents.map((content) => {
fileNoExtension = file.slice(0, file.indexOf('.'));
if (req.params.markdown_file_noext == fileNoExtension) {
return {
'title': fileNoExtension,
'markdown': marked(content)
};
};
})
})
.then((results) => {
// It's better if you aggregate all results in one array and return it,
// instead of calling res.json for each result
res.json(results);
})
.catch((err) => {
// All errors are catched here
console.log(err);
})
});
function readdir(folderPath) {
return new Promise((resolve, reject) => {
fs.readdir(folderPath, (err, files) {
if (err) {
return reject(err);
}
resolve(files);
});
});
}
function readFile(filePath) {
return new Promise((resolve, reject) => {
fs.readFile(filePath, 'utf8', (err, file_content) => {
if (err) {
return reject(err);
}
resolve(file_content);
});
});
}
Related
I have a little problem, I need the makeZip function to wait for the takeScreenshot function to take all the screenshots it needs, how do I do this while taking care of best practices?
(I know at this point "then" doesn't make sense with the post method, I just tried it my way before but it didn't work the way I wanted)
Function:
const takeScreenshot = (url) => {
const resolutionsArray = Object.values(resolutions);
resolutionsArray.map(async (mediaSize) => {
webshot(url, setFileName(url, mediaSize), setOptions(mediaSize), (err) => {
if (!err) {
console.log("screenshot taken!");
}
});
});
};
calling functions:
app.post("/", async (req, res) => {
const { url } = req.body;
takeScreenshot(url)
.then((url) => makeZip(url))
.then((url) => sendEmail(url))
.then((message) => res.send(message))
.catch((err) => console.log(err));
});
My suggestion is:
to use Promise.all or Promise.allSettled when you need to handle several promises
extract callback of map fn
const makeWebshot = (argsHere) => new Promise((reselove, reject) => {
webshot(url, setFileName(url, mediaSize), setOptions(mediaSize), (err) => {
if (err) return reject(err);
return resolve();
});
});
Update takeScreenshot to
const takeScreenshot = (url) => {
const resolutionsArray = Object.values(resolutions);
return Promise.all(resolutionsArray.map((mediaSize) => makeWebshot(argsHere)));
};
When dealing with a list of Promises you will want to use Promise.all to wait for them all to resolve. Here is a simple example:
const list = [1,2,3];
const all = list.map(i => new Promise((resolve, reject) => {
setTimeout(() => {
console.log(i);
resolve(i*2);
}, 100);
}));
Promise.all(all).then(console.log)
In your case it would be something like this:
const takeScreenshot = (url) =>
Object.values(resolutions).map(async (mediaSize) => {
webshot(url, setFileName(url, mediaSize), setOptions(mediaSize), (err) => {
if (!err) {
console.log("screenshot taken!");
}
});
});
app.post("/", async (req, res) => {
const { url } = req.body;
Promise.all(takeScreenshot(url))
.then((listOfUrls) => ...
});
But since I don't know what webshot returns, I can't tell you what the processing of the listOfUrls should look like.
I'm trying to develop a simple app that if you pass a parameter in command line the application will search inside a directory and if the text match in some of the files the file should be save in a list, but when I put the console.log the value is not updated
here is my code:
const folder = "./movies/data";
const fs = require("fs");
var args = process.argv.slice(2);
console.log("myArgs: ", args);
var count = 0;
var list = [];
fs.readdir(folder, (err, files) => {
files.forEach((file) => {
fs.readFile(`movies/data/${file}`, "utf8", function (err, data) {
if (err) console.log(err);
if (data.includes("walt disney")) {
count++;
list.push(data);
console.log("Found in: ", data);
}
});
});
console.log(`Foram encontradas ${count} ocorrências pelo termo ${args}.`);
});
any suggestions about what i'm doing wrong?
For your program to work, you will have to add some Promise / async/await logic. On the moment you try to read from the files, the files are still undefined so the fs.readDir() function will not provide the wanted result.
This should work:
const { resolve } = require('path');
const { readdir } = require('fs').promises;
const fs = require("fs");
var args = process.argv.slice(2);
const pathToFiles = "./movies/";
async function getFiles(dir) {
const dirents = await readdir(dir, { withFileTypes: true });
const files = await Promise.all(dirents.map((dirent) => {
const res = resolve(dir, dirent.name);
return dirent.isDirectory() ? getFiles(res) : res;
}));
return Array.prototype.concat(...files);
}
getFiles(pathToFiles)
.then(files => {
console.log(files)
files.forEach((file) => {
fs.readFile(file, 'utf8', (err, data) => {
if (err) console.log(err);
if (data.includes(args)) {
console.log(`${args} found in ${file}.`);
} else {
console.log(`${args} not found.`);
}
});
})
})
.catch (e => console.error(e));
I am using Node.js file system to build an array of file paths. I would like to know when all files have been read, so I could work further with my array.
Sequence of events:
Go into a folder
Get a path of each file
Put each path into an array
Let me know once you're done
Code:
'use strict';
const fs = require('fs');
function readDirectory(path) {
return new Promise((resolve, reject) => {
const files = [];
fs.readdir(path, (err, contents) => {
if (err) {
reject(err);
}
contents.forEach((file) => {
const pathname = `${ path }/${ file }`;
getFilesFromPath(pathname).then(() => {
console.log('pathname', pathname);
files.push(pathname);
});
resolve(files);
});
});
});
}
function getFilesFromPath(path) {
return new Promise((resolve, reject) => {
const stat = fs.statSync(path);
if (stat.isFile()) {
fs.readFile(path, 'utf8', (err, data) => {
if (err) {
reject(err);
} else {
resolve(data);
}
});
} else if (stat.isDirectory()) {
readDirectory(path);
}
});
}
getFilesFromPath('./dist');
Would be great to glue with:
Promise.all(files).then(() => {
// do stuff
})
Your suggestion pretty much works - did you try it? Here's a typical way of doing it:
getFilesFromPath( path ).then( files => {
const filePromises = files.map( readFile );
return Promises.all( filePromises );
}).then( fileContentsArray => {
//do stuff - the array will contain the contents of each file
});
You'll have to write the "readFile()" function yourself, but looks like you got that covered.
I'm building a server in Node that will search a folder to see if an XML file exists (glob), and if it does, read the file in (fs) as a JSON object (xml2js) and eventually store it in a database somewhere. I'm want to get the results OUT of the parser and into another variable so I can do other things with the data. From what I can tell, something is running synchronously, but I can't figure out how to stop it and for me to wait until it's finished to continue moving on.
I'm separating my function out into a controller elsewhere from app.js:
app.controller.js
const fs = require('fs-extra');
const glob = require('glob');
const xml2js = require('xml2js');
exports.requests = {};
exports.checkFileDrop = async () => {
console.log('Checking for xml in filedrop...');
// this is the only place await works...
await glob('./filedrop/ALLREQUESTS-*.xml', (err, files) => {
var parser = new xml2js.Parser();
// this is looking for a specific file now, which I'll address later once I can figure out this issue
fs.readFile('./filedrop/ALLREQUESTS-20170707.xml', 'utf16le', function (err, data) {
if (err) {
console.log('ERROR: ', err);
} else {
parser.parseString(data, (err, result) => {
if (err) {
console.log('ERROR: ', err);
} else {
console.log('data found');
exports.requests = JSON.stringify(result.Records.Record);
// data is outputted here correctly
console.log(exports.requests);
// this doesn't even seem to want to save to exports.requests anyways...
}
});
}
});
});
}
app.js
const appController = require('./controllers/app.controller');
// check if there is file in filedrop
appController.checkFileDrop();
// prints out an empty object
console.log(appController.requests);
// can't do anything if it doesn't exist yet
appController.saveToDB(appController.requests);
await will wait for a Promise value to resolve, otherwise it'll just wrap the value it is given in a promise and resolve the promise right away. In your example,
await glob('./filedrop/ALLREQUESTS-*.xml', (err, files) => {
the call to glob does not return a Promise, so the await is essentially useless. So you need to create the promise yourself.
exports.checkFileDrop = async () => {
console.log('Checking for xml in filedrop...');
const files = await new Promise((resolve, reject) => glob('./filedrop/ALLREQUESTS-*.xml', (err, files) => {
if (err) reject(err);
else resolve(files);
});
const parser = new xml2js.Parser();
const data = await new Promise((resolve, reject) => fs.readFile('./filedrop/ALLREQUESTS-20170707.xml', 'utf16le', function (err, data) {
if (err) reject(err);
else resolve(data);
});
const result = await new Promise((resolve, reject) => parser.parseString(data, (err, result) => {
if (err) reject(err);
else resolve(result);
});
console.log('data found');
const requests = JSON.stringify(result.Records.Record);
console.log(requests);
}
Note that now this function will reject the promise it returns instead of force-logging the error.
You can also condense this down with a helper. Node 8 for instance includes util.promisify to make code like this easier to write, e.g.
const util = require('util');
exports.checkFileDrop = async () => {
console.log('Checking for xml in filedrop...');
const files = await util.promisify(glob)('./filedrop/ALLREQUESTS-*.xml');
const parser = new xml2js.Parser();
const data = await util.promisify(fs.readFile)('./filedrop/ALLREQUESTS-20170707.xml', 'utf16le');
const result = await util.promisify(parser.parseString.bind(parser))(data);
console.log('data found');
const requests = JSON.stringify(result.Records.Record);
console.log(requests);
}
You can use async/await
import fs from 'fs';
import { promisify } from 'util';
const xmlToJson = async filePath => {
const parser = new xml2js.Parser
try {
const data = await fs.promises.readFile(filePath, 'utf8')
const result = await promisify(parser.parseString)(data);
const requests = JSON.stringify(result.merchandiser.product);
return requests
}
catch(err) {
console.log(err)
}
}
I wrote a function running recursively to find out files whose name include given world. I do not understand how promises works and cannot find a way to write this function with promises despite trying hard.
I tried returning a promise inside findPath function but I couldn't use it since extractFiles calls findPath. I tried to create a list of promises and return all but couldn't succeed neither.
So how could I write these functions with promises?
const fs = require('fs');
const path = require('path');
function findPath(targetPath, targetWord, done) {
if (!fs.existsSync(targetPath)) return;
fs.readdir(targetPath, (err, allPaths) => {
if (err) done(err, null);
for (aPath of allPaths) {
aPath = path.join(targetPath, aPath);
extractFiles(aPath, targetWord, done);
}
});
function extractFiles(aPath, targetWord, done) {
fs.lstat(aPath, (err, stat) => {
if (err) done(err, null);
if (stat.isDirectory()) {
findPath(aPath, targetWord, done);
}
else if (aPath.indexOf(targetWord) >= 0) {
let fileName = aPath.split('.')[0];
done(null, fileName);
}
});
}
}
findPath('../modules', 'routes', file => {
console.log(file);
});
Firstly, to make the "core" code more readable, I'd promisify the fs functions
const promisify1p = fn => p1 => new Promise((resolve, reject) => {
fn(p1, (err, result) => {
if(err) {
reject(err);
} else {
resolve(result);
}
});
});
const readdirAsync = promisify1p(fs.readdir);
const lstatAsync = promisify1p(fs.lstat);
Then, just chain the promises as you would with any other promises
const fs = require('fs');
const path = require('path');
function findPath(targetPath, targetWord) {
const readPath = target =>
readdirAsync(target)
.then(allPaths =>
Promise.all(allPaths.map(aPath => extractFiles(path.join(target, aPath))))
.then(x => x.filter(x=>x)) // remove all the "false" entries - i.e. that don't match targetWord
.then(x => [].concat.apply([], x)) // flatten the result
);
const extractFiles = aPath =>
lstatAsync(aPath).then(stat => {
if (stat.isDirectory()) {
return readPath(aPath);
} else if (aPath.includes(targetWord)) {
return aPath.split('.')[0];
}
return false;
});
return readPath(targetPath);
}
findPath('../modules', 'routes')
.then(results => {
// do things with the results - which is an array of files that contain the targetWord
})
.catch(err => console.error(err));
Not much to it at all.