I have the following code.
However, it's not catching all errors and I am still getting "throw er; // Unhandled 'error' event".
Why is this?
app.post('/api/properties/zip/:zip/bedrooms/:bedrooms', async (req, res, next) => {
try {
const file = await apiCall(req.params.zip, req.params.bedrooms);
const records = await parse(file);
const seq = await sequelize();
const result = await dbImport(seq, records);
return await res.status(200).json(`${result.length} properties successfully imported to the database`);
} catch (err) {
return next(err);
}
});
// Middleware error handling
app.use((err, req, res, next) => {
console.error(err.message);
if (!err.statusCode) err.statusCode = 500;
return res.status(err.statusCode).json(err.message);
});
For example, it didn't catch the error in the parse() function, until I added a specific error handler. Shouldn't my try/catch catch this error even without adding this?
const fs = require('fs');
const parse = filename => new Promise(((resolve, reject) => {
// Converts a line from the file, parses it to JSON, and stores it an array
const func = (data, records) => {
const json = JSON.parse(data);
records.push(json);
};
// Read in each line of the file and pass that line to func
const readLines = (input) => {
const records = [];
let remaining = '';
// ******** HAD TO ADD THIS *********
input.on('error', (err) => {
reject(err);
});
input.on('data', (data) => {
remaining += data;
let index = remaining.indexOf('\n');
let last = 0;
while (index > -1) {
const line = remaining.substring(last, index);
last = index + 1;
func(line, records);
index = remaining.indexOf('\n', last);
}
remaining = remaining.substring(last);
});
input.on('end', () => {
if (remaining.length > 0) {
func(remaining, records);
}
resolve(records);
});
};
const input = fs.createReadStream(filename);
readLines(input, func);
}));
module.exports = parse;
Thanks in advance!
Perhaps this will demonstrate for you how a try/catch will work with when using await. When a promise is rejected it will throw the resulting value. If the underlying promise resolves it will return that value.
(async () => {
try {
const val1 = await Promise.resolve('resolved val');
const val2 = await Promise.reject('reject val');
console.log(val1);
} catch (err) {
console.error(err);
}
})();
Related
I'm trying to develop a simple app that if you pass a parameter in command line the application will search inside a directory and if the text match in some of the files the file should be save in a list, but when I put the console.log the value is not updated
here is my code:
const folder = "./movies/data";
const fs = require("fs");
var args = process.argv.slice(2);
console.log("myArgs: ", args);
var count = 0;
var list = [];
fs.readdir(folder, (err, files) => {
files.forEach((file) => {
fs.readFile(`movies/data/${file}`, "utf8", function (err, data) {
if (err) console.log(err);
if (data.includes("walt disney")) {
count++;
list.push(data);
console.log("Found in: ", data);
}
});
});
console.log(`Foram encontradas ${count} ocorrências pelo termo ${args}.`);
});
any suggestions about what i'm doing wrong?
For your program to work, you will have to add some Promise / async/await logic. On the moment you try to read from the files, the files are still undefined so the fs.readDir() function will not provide the wanted result.
This should work:
const { resolve } = require('path');
const { readdir } = require('fs').promises;
const fs = require("fs");
var args = process.argv.slice(2);
const pathToFiles = "./movies/";
async function getFiles(dir) {
const dirents = await readdir(dir, { withFileTypes: true });
const files = await Promise.all(dirents.map((dirent) => {
const res = resolve(dir, dirent.name);
return dirent.isDirectory() ? getFiles(res) : res;
}));
return Array.prototype.concat(...files);
}
getFiles(pathToFiles)
.then(files => {
console.log(files)
files.forEach((file) => {
fs.readFile(file, 'utf8', (err, data) => {
if (err) console.log(err);
if (data.includes(args)) {
console.log(`${args} found in ${file}.`);
} else {
console.log(`${args} not found.`);
}
});
})
})
.catch (e => console.error(e));
I can't figure this one out. I have one function that connects to an SFTP server and downloads files. Then, I have a second function that reads the contents, puts the data in an array, and returns the array.
The problem is that the second function always runs first. I tried different methods but I can't get it to work. That connection to SFTP is quite slow, it can take like 10+ seconds to finish. But I need to somehow wait for it to finish before doing anything else.
const SFTPConfig = require('../config/keys').sftpconfig;
const getCSATFiles = async function(targetDate) {
try {
let Client = require('ssh2-sftp-client');
let sftp = new Client();
const date = moment(targetDate);
var dir = `../csv/${targetDate}/`;
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir);
}
sftp
.connect(SFTPConfig, 'once')
.then(() => {
return sftp.list('/In/Archives/');
})
.then(data => {
data.forEach(item => {
const fileName = item.name;
const remotePath = '/In/Archives/' + fileName;
const localePath = path.join(dir + fileName);
if (
moment(item.modifyTime)
.format('YYYY-MM-DD hh:mm')
.toString()
.slice(0, 10) ===
date
.format('YYYY-MM-DD hh:mm')
.toString()
.slice(0, 10)
) {
sftp
.fastGet(remotePath, localePath, {})
.then(() => {
console.log('finished getting the files!');
sftp.end();
})
.catch(err => {
sftp.end();
console.log(err, 'fastGet method error');
});
}
});
});
} catch (error) {
console.log(error);
}
};
const readCSVFiles = async function(targetDate) {
try {
const casesBO = [];
var dir = `../csv/${targetDate}/`;
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir);
}
const allLocalFiles = path.join(__dirname, dir);
const readDir = util.promisify(fs.readdir);
const files = await readDir(allLocalFiles);
for (let file of files) {
fs.createReadStream(allLocalFiles + file)
.pipe(csv.parse({ headers: true, delimiter: ';' }))
.on('error', error => console.error(error))
.on('data', row => {
if (row['[REGION2]'] !== 'FR') {
casesBO.push(row['[CALLERNO_EMAIL_SOCIAL]']);
console.log(
`${row['[AGENT]']} is ${row['[REGION2]']} and case = ${
row['[CALLERNO_EMAIL_SOCIAL]']
}`
);
}
})
.on('end', rowCount => {
console.log(`Parsed ${rowCount} rows`);
});
}
return casesBO;
} catch (error) {
console.log(error);
}
};
const testFunc = async () => {
const csatfiles = await getCSATFiles('2021-02-03');
const boData = await readCSVFiles('2021-02-03');
console.log(boData);
};
testFunc();
#1 as #messerbill suggested, you need to return the promise from your function.
#2 your promise has a loop inside of it that have more promises. In this case, you need to collect those promises and use Promise.all to resolve them before you second function runs. I put comments on the lines you need to change below. Try this:
const SFTPConfig = require('../config/keys').sftpconfig;
const getCSATFiles = function(targetDate) {
try {
let Client = require('ssh2-sftp-client');
let sftp = new Client();
const date = moment(targetDate);
var dir = `../csv/${targetDate}/`;
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir);
}
// I return the promise here
return sftp
.connect(SFTPConfig, 'once')
.then(() => {
return sftp.list('/In/Archives/');
})
.then(data => {
// I set up my promises as a blank array
const promises = [];
data.forEach(item => {
const fileName = item.name;
const remotePath = '/In/Archives/' + fileName;
const localePath = path.join(dir + fileName);
if (
moment(item.modifyTime)
.format('YYYY-MM-DD hh:mm')
.toString()
.slice(0, 10) ===
date
.format('YYYY-MM-DD hh:mm')
.toString()
.slice(0, 10)
) {
// I collect the promises here
promises.push(sftp
.fastGet(remotePath, localePath, {})
.then(() => {
console.log('finished getting the files!');
sftp.end();
})
.catch(err => {
sftp.end();
console.log(err, 'fastGet method error');
}));
}
});
// I resolve them here
return Promise.all(promises);
});
} catch (error) {
console.log(error);
}
};
const readCSVFiles = async function(targetDate) {
try {
const casesBO = [];
var dir = `../csv/${targetDate}/`;
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir);
}
const allLocalFiles = path.join(__dirname, dir);
const readDir = util.promisify(fs.readdir);
const files = await readDir(allLocalFiles);
for (let file of files) {
fs.createReadStream(allLocalFiles + file)
.pipe(csv.parse({ headers: true, delimiter: ';' }))
.on('error', error => console.error(error))
.on('data', row => {
if (row['[REGION2]'] !== 'FR') {
casesBO.push(row['[CALLERNO_EMAIL_SOCIAL]']);
console.log(
`${row['[AGENT]']} is ${row['[REGION2]']} and case = ${
row['[CALLERNO_EMAIL_SOCIAL]']
}`
);
}
})
.on('end', rowCount => {
console.log(`Parsed ${rowCount} rows`);
});
}
return casesBO;
} catch (error) {
console.log(error);
}
};
const testFunc = async () => {
const csatfiles = await getCSATFiles('2021-02-03');
const boData = await readCSVFiles('2021-02-03');
console.log(boData);
};
testFunc();
You need to take care, that you return the promise you want to resolve inside the function body in order to get the promise resolved at the right time.
async function promiseNotReturned() {
new Promise(resolve => setTimeout(resolve.bind(null), 5000))
}
async function promiseReturned() {
return new Promise(resolve => setTimeout(resolve.bind(null), 5000))
}
async function run() {
await promiseNotReturned()
console.log("does not wait for 5 seconds")
await promiseReturned()
console.log("waits for 5 seconds")
}
run()
I know that old school for loop works in the traditional way - that it waits for the await to finish getting results.
But in my use case, I need to read a file from local/s3 and process it line by line, and for each line I need to call an External API.
Generally I use await inside the loop because all are running inside a lambda and I don't want to use all memory for running it parallelly.
Here I am reading the file using a stream.on() method, and in order to use await inside that, I need to add async in read method, like so:
stream.on('data',async () =>{
while(data=stream.read()!==null){
console.log('line');
const requests = getRequests(); // sync code,no pblms
for(let i=0;i<requests.length;i++){
const result = await apiCall(request[i);
console.log('result from api')
const finalResult = await anotherapiCall(result.data);
}
}
});
This is working but order in which the lines are processed is not guaranteed. I need all in a sync manner. Any help?
Complete Code
async function processSOIFileLocal (options, params) {
console.log('Process SOI file');
const readStream = byline.createStream(fs.createReadStream(key));
readStream.setEncoding('utf8');
const pattern = /^UHL\s|^UTL\s/;
const regExp = new RegExp(pattern);
readStream.on('readable', () => {
let line;
while (null !== (line = readStream.read())) {
if (!regExp.test(line.toString())) {
totalRecordsCount++;
dataObject = soiParser(line);
const { id } = dataObject;
const XMLRequests = createLoSTRequestXML(
options,
{ mapping: event.mapping, row: dataObject }
);
console.log('Read line');
console.log(id);
try {
for (let i = 0;i < XMLRequests.length;i++) {
totalRequestsCount++;
console.log('Sending request');
const response = await sendLoSTRequest(
options,
{ data: XMLRequests[i],
url: LOST_URL }
);
console.log("got response");
const responseObj = await xml2js.
parseStringPromise(response.data);
if (Object.keys(responseObj).indexOf('errors') !== -1) {
fs.writeFileSync(`${ERR_DIR}/${generateKey()}-${id}.xml`, response.data);
failedRequestsCount++;
} else {
successRequestsCount++;
console.log('Response from the Lost Server');
console.log(response[i].data);
}
}
} catch (err) {
console.log(err);
}
}
}
})
.on('end', () => {
console.log('file processed');
console.log(`
************************************************
Total Records Processed:${totalRecordsCount}
Total Requests Sent: ${totalRequestsCount}
Success Requests: ${successRequestsCount}
Failed Requests: ${failedRequestsCount}
************************************************
`);
});
}
async function sendLoSTRequest (options, params) {
const { axios } = options;
const { url, data } = params;
if (url) {
return axios.post(url, data);
// eslint-disable-next-line no-else-return
} else {
console.log('URL is not found');
return null;
}
}
Code needs to flow like so:
read a line in a sync way
process the line and transform the line into an array of two members
for every member call API and do stuff
once line is complete, look for another line, all done in order
UPDATE: Got a workaround..but it fires stream.end() without waiting stream to finish read
async function processSOIFileLocal (options, params) {
console.log('Process SOI file');
const { ERR_DIR, fs, xml2js, LOST_URL, byline, event } = options;
const { key } = params;
const responseObject = {};
let totalRecordsCount = 0;
let totalRequestsCount = 0;
let failedRequestsCount = 0;
let successRequestsCount = 0;
let dataObject = {};
const queue = (() => {
let q = Promise.resolve();
return fn => (q = q.then(fn));
})();
const readStream = byline.createStream(fs.createReadStream(key));
readStream.setEncoding('utf8');
const pattern = /^UHL\s|^UTL\s/;
const regExp = new RegExp(pattern);
readStream.on('readable', () => {
let line;
while (null !== (line = readStream.read())) {
if (!regExp.test(line.toString())) {
totalRecordsCount++;
dataObject = soiParser(line);
const { id } = dataObject;
const XMLRequests = createLoSTRequestXML(
options,
{ mapping: event.mapping, row: dataObject }
);
// eslint-disable-next-line no-loop-func
queue(async () => {
try {
for (let i = 0;i < XMLRequests.length;i++) {
console.log('Sending request');
console.log(id);
totalRequestsCount++;
const response = await sendLoSTRequest(
options,
{ data: XMLRequests[i],
url: LOST_URL }
);
console.log('got response');
const responseObj = await xml2js.
parseStringPromise(response.data);
if (Object.keys(responseObj).indexOf('errors') !== -1) {
// console.log('Response have the error:');
// await handleError(options, { err: responseObj, id });
failedRequestsCount++;
fs.writeFileSync(`${ERR_DIR}/${generateKey()}-${id}.xml`, response.data);
} else {
console.log('Response from the Lost Server');
console.log(response[i].data);
successRequestsCount++;
}
}
} catch (err) {
console.log(err);
}
});
}
}
})
.on('end', () => {
console.log('file processed');
console.log(`
************************************************
Total Records Processed:${totalRecordsCount}
Total Requests Sent: ${totalRequestsCount}
Success Requests: ${successRequestsCount}
Failed Requests: ${failedRequestsCount}
************************************************
`);
Object.assign(responseObject, {
failedRequestsCount,
successRequestsCount,
totalRecordsCount,
totalRequestsCount
});
});
}
Thank You
The sample code at the top of your question could be rewritten like
const queue = (() => {
let q = Promise.resolve();
return (fn) => (q = q.then(fn));
})();
stream.on('data', async() => {
while (data = stream.read() !== null) {
console.log('line');
const requests = getRequests(); // sync code,no pblms
queue(async () => {
for (let i = 0; i < requests.length; i++) {
const result = await apiCall(request[i]);
console.log('result from api');
const finalResult = await anotherapiCall(result.data);
}
});
}
});
Hopefully that will be useful for the complete code
If anyone want a solution for synchronisely process the file, ie, linebyline read and execute some Async call, it's recommended to use inbuilt stream transform. There we can create a transform function and return a callback when finishes.
That's will help of any one face this issues.
Through2 is a small npm library that also can be used for the same.
I'm a bit new to async/await. I can get my code to work if not chaining, but I'm trying to keep everything as modular as possible. readFiles() is returning the correct value according to the log, but the "files" log is returning undefined. Nothing breaks, but the ultimate value from saveRecords() is "undefined."
const fs = require("fs");
const {
convertCurrencyStr,
getFiles,
getMsFromDate,
parseJSON
} = require("./utils");
const saveResults = data => {
const writePath = path.join(__dirname, "filteredRecords.json");
try {
fs.writeFileSync(writePath, JSON.stringify(data));
} catch (err) {
console.log("Could not write data.", err);
}
};
const filterRecords = data => {
console.log("filtered", data);
const deltaDateMs = getMsFromDate("January 1, 2016");
try {
const results = data.filter(item => {
const registeredDateMs = getMsFromDate(item.registered);
const afterJan16 = registeredDateMs > deltaDateMs;
const isActive = item.isActive === true;
const moreThan2000 = convertCurrencyStr(item.balance) > 2000;
if (afterJan16 && isActive && moreThan2000) {
return true;
}
});
return results;
} catch (err) {
"No data available.", err;
}
};
const readFiles = async () => {
const directoryPath = path.join(__dirname, "data");
try {
const files = await getFiles(directoryPath);
files.forEach(file => {
const filePath = path.join(directoryPath, file);
const data = fs.readFileSync(filePath);
const results = parseJSON(data);
return results;
});
} catch (err) {
console.log("Could not read files.", err);
}
};
(async () => {
const files = await readFiles();
const records = await filterRecords(files);
saveResults(records);
})();
Your readFiles need to have a return after the forEach, the return inside is for the array element you need another one to return the results.
And I am not sure if your filter will work, the if its not necessary, You should use only a
return (afterJan16 && isActive && moreThan2000)
Put your return outside the forEach
try {
const files = await getFiles(directoryPath);
let results;
files.forEach(file => {
const filePath = path.join(directoryPath, file);
const data = fs.readFileSync(filePath);
results = parseJSON(data);
});
return results;
} catch (err) {
console.log("Could not read files.", err);
}
Here is the index.ts script I am running (based on something I found on reddit):
const path = require("path");
const sql = require("mssql");
const config = require(path.resolve("./config.json"));
let db1;
const connect = () => {
return new Promise((resolve, reject) => {
db1 = new sql.ConnectionPool(config.db, err => {
if (err) {
console.error("Connection failed.", err);
reject(err);
} else {
console.log("Database pool #1 connected.");
resolve();
}
});
});
};
const selectProjects = async (name) => {
const query = `
select * from [Time].ProjectData where [Name] like concat('%', concat(#name, '%'))`;
const request = new sql.Request(db1);
const result = await request
.input("name", name)
.query(query);
return result.recordset;
};
module.exports = {
connect,
selectProjects
};
connect().then(function() {
console.log(selectProjects('General'));
}).catch(function(err) {
console.log(err);
});
When I run the script using node index (after compiling it of course), I get this in the console:
Database pool #1 connected.
Promise { <pending> }
And then the script hangs.
Apparently the await keyword creates an implicit promise; I had to change the last function call to:
connect().then(function() {
selectProjects('General').then(function(data) {
console.log(data);
});
}).catch(function(err) {
console.log(err);
});