I'm trying to develop a simple app that if you pass a parameter in command line the application will search inside a directory and if the text match in some of the files the file should be save in a list, but when I put the console.log the value is not updated
here is my code:
const folder = "./movies/data";
const fs = require("fs");
var args = process.argv.slice(2);
console.log("myArgs: ", args);
var count = 0;
var list = [];
fs.readdir(folder, (err, files) => {
files.forEach((file) => {
fs.readFile(`movies/data/${file}`, "utf8", function (err, data) {
if (err) console.log(err);
if (data.includes("walt disney")) {
count++;
list.push(data);
console.log("Found in: ", data);
}
});
});
console.log(`Foram encontradas ${count} ocorrências pelo termo ${args}.`);
});
any suggestions about what i'm doing wrong?
For your program to work, you will have to add some Promise / async/await logic. On the moment you try to read from the files, the files are still undefined so the fs.readDir() function will not provide the wanted result.
This should work:
const { resolve } = require('path');
const { readdir } = require('fs').promises;
const fs = require("fs");
var args = process.argv.slice(2);
const pathToFiles = "./movies/";
async function getFiles(dir) {
const dirents = await readdir(dir, { withFileTypes: true });
const files = await Promise.all(dirents.map((dirent) => {
const res = resolve(dir, dirent.name);
return dirent.isDirectory() ? getFiles(res) : res;
}));
return Array.prototype.concat(...files);
}
getFiles(pathToFiles)
.then(files => {
console.log(files)
files.forEach((file) => {
fs.readFile(file, 'utf8', (err, data) => {
if (err) console.log(err);
if (data.includes(args)) {
console.log(`${args} found in ${file}.`);
} else {
console.log(`${args} not found.`);
}
});
})
})
.catch (e => console.error(e));
Related
I have the following code.
However, it's not catching all errors and I am still getting "throw er; // Unhandled 'error' event".
Why is this?
app.post('/api/properties/zip/:zip/bedrooms/:bedrooms', async (req, res, next) => {
try {
const file = await apiCall(req.params.zip, req.params.bedrooms);
const records = await parse(file);
const seq = await sequelize();
const result = await dbImport(seq, records);
return await res.status(200).json(`${result.length} properties successfully imported to the database`);
} catch (err) {
return next(err);
}
});
// Middleware error handling
app.use((err, req, res, next) => {
console.error(err.message);
if (!err.statusCode) err.statusCode = 500;
return res.status(err.statusCode).json(err.message);
});
For example, it didn't catch the error in the parse() function, until I added a specific error handler. Shouldn't my try/catch catch this error even without adding this?
const fs = require('fs');
const parse = filename => new Promise(((resolve, reject) => {
// Converts a line from the file, parses it to JSON, and stores it an array
const func = (data, records) => {
const json = JSON.parse(data);
records.push(json);
};
// Read in each line of the file and pass that line to func
const readLines = (input) => {
const records = [];
let remaining = '';
// ******** HAD TO ADD THIS *********
input.on('error', (err) => {
reject(err);
});
input.on('data', (data) => {
remaining += data;
let index = remaining.indexOf('\n');
let last = 0;
while (index > -1) {
const line = remaining.substring(last, index);
last = index + 1;
func(line, records);
index = remaining.indexOf('\n', last);
}
remaining = remaining.substring(last);
});
input.on('end', () => {
if (remaining.length > 0) {
func(remaining, records);
}
resolve(records);
});
};
const input = fs.createReadStream(filename);
readLines(input, func);
}));
module.exports = parse;
Thanks in advance!
Perhaps this will demonstrate for you how a try/catch will work with when using await. When a promise is rejected it will throw the resulting value. If the underlying promise resolves it will return that value.
(async () => {
try {
const val1 = await Promise.resolve('resolved val');
const val2 = await Promise.reject('reject val');
console.log(val1);
} catch (err) {
console.error(err);
}
})();
I'm a bit new to async/await. I can get my code to work if not chaining, but I'm trying to keep everything as modular as possible. readFiles() is returning the correct value according to the log, but the "files" log is returning undefined. Nothing breaks, but the ultimate value from saveRecords() is "undefined."
const fs = require("fs");
const {
convertCurrencyStr,
getFiles,
getMsFromDate,
parseJSON
} = require("./utils");
const saveResults = data => {
const writePath = path.join(__dirname, "filteredRecords.json");
try {
fs.writeFileSync(writePath, JSON.stringify(data));
} catch (err) {
console.log("Could not write data.", err);
}
};
const filterRecords = data => {
console.log("filtered", data);
const deltaDateMs = getMsFromDate("January 1, 2016");
try {
const results = data.filter(item => {
const registeredDateMs = getMsFromDate(item.registered);
const afterJan16 = registeredDateMs > deltaDateMs;
const isActive = item.isActive === true;
const moreThan2000 = convertCurrencyStr(item.balance) > 2000;
if (afterJan16 && isActive && moreThan2000) {
return true;
}
});
return results;
} catch (err) {
"No data available.", err;
}
};
const readFiles = async () => {
const directoryPath = path.join(__dirname, "data");
try {
const files = await getFiles(directoryPath);
files.forEach(file => {
const filePath = path.join(directoryPath, file);
const data = fs.readFileSync(filePath);
const results = parseJSON(data);
return results;
});
} catch (err) {
console.log("Could not read files.", err);
}
};
(async () => {
const files = await readFiles();
const records = await filterRecords(files);
saveResults(records);
})();
Your readFiles need to have a return after the forEach, the return inside is for the array element you need another one to return the results.
And I am not sure if your filter will work, the if its not necessary, You should use only a
return (afterJan16 && isActive && moreThan2000)
Put your return outside the forEach
try {
const files = await getFiles(directoryPath);
let results;
files.forEach(file => {
const filePath = path.join(directoryPath, file);
const data = fs.readFileSync(filePath);
results = parseJSON(data);
});
return results;
} catch (err) {
console.log("Could not read files.", err);
}
I have this code that serves every markdown file in the './markdown' folder. At '/api/markdown/filename'.
var apiRouter = express.Router();
markdownFolder = './markdown/';
apiRouter.get('/:markdown_file_noext', function(req, res) {
fs.readdir(markdownFolder, function(err, markdown) {
if (err) throw err;
markdown.forEach(function(file) {
fs.readFile(markdownFolder + file, 'utf8', function(err, file_content) {
if (err) throw err;
fileNoExtension = file.slice(0, file.indexOf('.'));
if (req.params.markdown_file_noext == fileNoExtension) {
res.json({
'title': fileNoExtension,
'markdown': marked(file_content)
});
};
});
});
});
});
But i end having a ton of callbacks do the the nature of the 'fs' methods. How do i avoid this?
Using Q as promise library:
const Q = require('q');
const fs = require('fs');
const markdownFolder = './markdown/';
const readdir = Q.nfbind(fs.readdir);
const readFile = Q.nfbind(fs.readFile);
readdir(markdownFolder).then(markdown => {
const promises = [];
markdown.forEach(file => promises.push(readFile(markdownFolder + file, 'utf8')));
return Q.all(promises);
}).then(files => {
// Do your magic.
}).catch(error => {
// Do something with error.
});
You have different option.
Use named Function instead of anonymus functinos. It would make it a little bit more readable but you will still be using callbacks.
Use Promises, but you will need to use bluebird to wrap the fs module.
For a more advance option, you can use generators and Promises to make your code look more like a sync way. Take a look at co or bluebird.coroutine.
With Promises you could do like this:
const path = require('path');
var apiRouter = express.Router();
markdownFolder = './markdown/';
apiRouter.get('/:markdown_file_noext', function(req, res) {
readdir(markdownFolder)
.then((files) => {
const tasks = files.map((file) => {
const filePath = path.resolve(markdownFolder, file);
return readFile(filePath);
});
return Promise.all(tasks); // Read all files
})
.then((fileContents) => {
return fileContents.map((content) => {
fileNoExtension = file.slice(0, file.indexOf('.'));
if (req.params.markdown_file_noext == fileNoExtension) {
return {
'title': fileNoExtension,
'markdown': marked(content)
};
};
})
})
.then((results) => {
// It's better if you aggregate all results in one array and return it,
// instead of calling res.json for each result
res.json(results);
})
.catch((err) => {
// All errors are catched here
console.log(err);
})
});
function readdir(folderPath) {
return new Promise((resolve, reject) => {
fs.readdir(folderPath, (err, files) {
if (err) {
return reject(err);
}
resolve(files);
});
});
}
function readFile(filePath) {
return new Promise((resolve, reject) => {
fs.readFile(filePath, 'utf8', (err, file_content) => {
if (err) {
return reject(err);
}
resolve(file_content);
});
});
}
I'm building a server in Node that will search a folder to see if an XML file exists (glob), and if it does, read the file in (fs) as a JSON object (xml2js) and eventually store it in a database somewhere. I'm want to get the results OUT of the parser and into another variable so I can do other things with the data. From what I can tell, something is running synchronously, but I can't figure out how to stop it and for me to wait until it's finished to continue moving on.
I'm separating my function out into a controller elsewhere from app.js:
app.controller.js
const fs = require('fs-extra');
const glob = require('glob');
const xml2js = require('xml2js');
exports.requests = {};
exports.checkFileDrop = async () => {
console.log('Checking for xml in filedrop...');
// this is the only place await works...
await glob('./filedrop/ALLREQUESTS-*.xml', (err, files) => {
var parser = new xml2js.Parser();
// this is looking for a specific file now, which I'll address later once I can figure out this issue
fs.readFile('./filedrop/ALLREQUESTS-20170707.xml', 'utf16le', function (err, data) {
if (err) {
console.log('ERROR: ', err);
} else {
parser.parseString(data, (err, result) => {
if (err) {
console.log('ERROR: ', err);
} else {
console.log('data found');
exports.requests = JSON.stringify(result.Records.Record);
// data is outputted here correctly
console.log(exports.requests);
// this doesn't even seem to want to save to exports.requests anyways...
}
});
}
});
});
}
app.js
const appController = require('./controllers/app.controller');
// check if there is file in filedrop
appController.checkFileDrop();
// prints out an empty object
console.log(appController.requests);
// can't do anything if it doesn't exist yet
appController.saveToDB(appController.requests);
await will wait for a Promise value to resolve, otherwise it'll just wrap the value it is given in a promise and resolve the promise right away. In your example,
await glob('./filedrop/ALLREQUESTS-*.xml', (err, files) => {
the call to glob does not return a Promise, so the await is essentially useless. So you need to create the promise yourself.
exports.checkFileDrop = async () => {
console.log('Checking for xml in filedrop...');
const files = await new Promise((resolve, reject) => glob('./filedrop/ALLREQUESTS-*.xml', (err, files) => {
if (err) reject(err);
else resolve(files);
});
const parser = new xml2js.Parser();
const data = await new Promise((resolve, reject) => fs.readFile('./filedrop/ALLREQUESTS-20170707.xml', 'utf16le', function (err, data) {
if (err) reject(err);
else resolve(data);
});
const result = await new Promise((resolve, reject) => parser.parseString(data, (err, result) => {
if (err) reject(err);
else resolve(result);
});
console.log('data found');
const requests = JSON.stringify(result.Records.Record);
console.log(requests);
}
Note that now this function will reject the promise it returns instead of force-logging the error.
You can also condense this down with a helper. Node 8 for instance includes util.promisify to make code like this easier to write, e.g.
const util = require('util');
exports.checkFileDrop = async () => {
console.log('Checking for xml in filedrop...');
const files = await util.promisify(glob)('./filedrop/ALLREQUESTS-*.xml');
const parser = new xml2js.Parser();
const data = await util.promisify(fs.readFile)('./filedrop/ALLREQUESTS-20170707.xml', 'utf16le');
const result = await util.promisify(parser.parseString.bind(parser))(data);
console.log('data found');
const requests = JSON.stringify(result.Records.Record);
console.log(requests);
}
You can use async/await
import fs from 'fs';
import { promisify } from 'util';
const xmlToJson = async filePath => {
const parser = new xml2js.Parser
try {
const data = await fs.promises.readFile(filePath, 'utf8')
const result = await promisify(parser.parseString)(data);
const requests = JSON.stringify(result.merchandiser.product);
return requests
}
catch(err) {
console.log(err)
}
}
I wrote a function running recursively to find out files whose name include given world. I do not understand how promises works and cannot find a way to write this function with promises despite trying hard.
I tried returning a promise inside findPath function but I couldn't use it since extractFiles calls findPath. I tried to create a list of promises and return all but couldn't succeed neither.
So how could I write these functions with promises?
const fs = require('fs');
const path = require('path');
function findPath(targetPath, targetWord, done) {
if (!fs.existsSync(targetPath)) return;
fs.readdir(targetPath, (err, allPaths) => {
if (err) done(err, null);
for (aPath of allPaths) {
aPath = path.join(targetPath, aPath);
extractFiles(aPath, targetWord, done);
}
});
function extractFiles(aPath, targetWord, done) {
fs.lstat(aPath, (err, stat) => {
if (err) done(err, null);
if (stat.isDirectory()) {
findPath(aPath, targetWord, done);
}
else if (aPath.indexOf(targetWord) >= 0) {
let fileName = aPath.split('.')[0];
done(null, fileName);
}
});
}
}
findPath('../modules', 'routes', file => {
console.log(file);
});
Firstly, to make the "core" code more readable, I'd promisify the fs functions
const promisify1p = fn => p1 => new Promise((resolve, reject) => {
fn(p1, (err, result) => {
if(err) {
reject(err);
} else {
resolve(result);
}
});
});
const readdirAsync = promisify1p(fs.readdir);
const lstatAsync = promisify1p(fs.lstat);
Then, just chain the promises as you would with any other promises
const fs = require('fs');
const path = require('path');
function findPath(targetPath, targetWord) {
const readPath = target =>
readdirAsync(target)
.then(allPaths =>
Promise.all(allPaths.map(aPath => extractFiles(path.join(target, aPath))))
.then(x => x.filter(x=>x)) // remove all the "false" entries - i.e. that don't match targetWord
.then(x => [].concat.apply([], x)) // flatten the result
);
const extractFiles = aPath =>
lstatAsync(aPath).then(stat => {
if (stat.isDirectory()) {
return readPath(aPath);
} else if (aPath.includes(targetWord)) {
return aPath.split('.')[0];
}
return false;
});
return readPath(targetPath);
}
findPath('../modules', 'routes')
.then(results => {
// do things with the results - which is an array of files that contain the targetWord
})
.catch(err => console.error(err));
Not much to it at all.