I have the code below that runs fine when run in a standalone file. But when I try to use it in my API endpoint and send a request with the postman, it can't seem to work.
I can't understand why. I also have the same issue when trying to write an excel file with the same API endpoint. If I specify a path, it won't find it. If I just use the filename, it will write fine in the current directory.
What am I missing? the code below is from when I use it in a standalone file. If I use it inside this route below, it won't work.
exports.download_excel = async (req, res) => {....
full code below
// modules import
const path = require('path');
const fs = require('fs');
const util = require('util');
const csv = require('#fast-csv/parse');
const { forEach } = require('p-iteration');
// const path = '../csv/';
const casesBO = [];
const readCSVFiles = async function() {
try {
const allLocalFiles = path.join('csv/');
const readDir = util.promisify(fs.readdir);
await readDir(allLocalFiles, async function(err, file) {
forEach(file, async item => {
fs.createReadStream('csv/' + item)
.pipe(csv.parse({ headers: true, delimiter: ';' }))
.on('error', error => console.error(error))
.on('data', async row => {
if (row['[REGION2]'] !== 'FR') {
casesBO.push(row['[CALLERNO_EMAIL_SOCIAL]']);
console.log(
`${row['[AGENT]']} is ${row['[REGION2]']} and case = ${
row['[CALLERNO_EMAIL_SOCIAL]']
}`
);
}
})
.on('end', async rowCount => {});
});
});
} catch (error) {
console.log(error);
}
};
You are awaiting readDir, but you are also giving it a callback function. You can't both await a Promise and also give it a callback. For that matter, Promises don't take a callback as argument at all.
Also you are writing async everywhere, this is useless for functions that don't await anything inside.
const readCSVFiles = async function () {
try {
const allLocalFiles = path.join('csv/');
const readDir = util.promisify(fs.readdir);
const files = await readDir(allLocalFiles);
for (let file of files) {
fs.createReadStream('csv/' + file)
.pipe(csv.parse({ headers: true, delimiter: ';' }))
.on('error', error => console.error(error))
.on('data', row => {
if (row['[REGION2]'] !== 'FR') {
casesBO.push(row['[CALLERNO_EMAIL_SOCIAL]']);
console.log(
`${row['[AGENT]']} is ${row['[REGION2]']} and case = ${row['[CALLERNO_EMAIL_SOCIAL]']
}`
);
}
})
.on('end', rowCount => { });
}
} catch (error) {
console.log(error);
}
};
Related
I'm using node 18.7 on ubuntu. I'm trying to parse a bunch of csv files to objects (using csv-parse), ultimately to load into a db. Because there are large numbers of these I decided to try streams and I'd like to use the async await style.
So far I have:
const { parse } = require('csv-parse');
const path = __dirname + '/file1.csv';
const opt = { columns: true, relax_column_count: true, skip_empty_lines: true, skip_records_with_error: true };
console.log(path);
const { pipeline } = require('stream');
// const pipeline = stream.pipeline;
async function readByLine(path, opt) {
const readFileStream = fs.createReadStream(path);
var csvParser = parse(opt, function (err, records) {
if (err) throw err;
});
await pipeline(readFileStream, csvParser, (err) => {
if (err) {
console.error('Pipeline failed.', err);
} else {
console.log('Pipeline succeeded.');
}
});
for await (const record of csvParser) {
console.log(record);
}
}
readByLine(path, opt)
When I run this I see:
Pipeline succeeded.
But the parsed objects are not sent to the console. What am I doing wrong?
edit1:
I changed the code to :
async function readByLine(path, opt) {
const readFileStream = fs.createReadStream(path);
var csvParser = parse(opt, function (err, records) {
if (err) throw err;
});
await pipeline(readFileStream, csvParser, (err) => {
if (err) {
console.error('Pipeline failed.', err);
} else {
console.log('Pipeline succeeded.');
}
});
// for await (const record of csvParser) {
// console.log(record);
// }
return csvParser;
}
(async function () {
const o = await readByLine(path, opt);
console.log(o);
})();
The result is an object which has a million properties, but some look set like in the screenshot.
You can only useful await a promise.
The pipeline function you are using doesn't return a promise.
If you look at the documentation you will see:
The pipeline API provides a promise version, which can also receive an options argument as the last parameter with a signal <AbortSignal> property. When the signal is aborted, destroy will be called on the underlying pipeline, with an AbortError.
const { pipeline } = require('node:stream/promises');
const fs = require('node:fs');
const zlib = require('node:zlib');
async function run() {
await pipeline(
fs.createReadStream('archive.tar'),
zlib.createGzip(),
fs.createWriteStream('archive.tar.gz')
);
console.log('Pipeline succeeded.');
}
run().catch(console.error);
Note the different value passed to require. Use that version of pipeline instead.
I am seeing the following error when trying to extract an image's metadata information with the Sharp module: "Input file contains unsupported image format".
This is only happening for certain signed image urls, particularly ones that contain xmp information in the metadata.
I am hoping someone can help me spot where the issue might be occurring in this code snippet.
Here is the exact code snippet I am using (insert the signed image URL where specified in the doStuff function to test):
const sharp = require("sharp");
const fs = require('fs');
const fetch = require('node-fetch');
async function storeUrlToLocal(sourceUrl) {
const destPath = './';
const request = {
method: 'GET',
encoding: null,
};
response = await fetch(sourceUrl, request);
if (response.status >= 400)
throw new Error(`Failed to fetch data from ${sourceUrl}, status returned = ${response.status}`);
const localPath = `${destPath}test.png`;
const fileStream = fs.createWriteStream(localPath);
return new Promise((resolve, reject) => {
response.body.pipe(fileStream);
response.body.on("error", reject);
response.body.on("end", async () => {
const fileExists = fs.existsSync(localPath);
console.log(`All the data in the file has been read ${localPath} = ${fileExists}`);
resolve(localPath);
});
response.body.on("finish",() => {
console.log('All writes are now complete.');
});
}).catch(error => {
console.log(error);
});
}
async function doStuff() {
const localFilePath = await storeUrlToLocal('<INSERT_IMAGE_URL_HERE>');
// Read image file and extract metadata
let manipulator;
let imageMetadata;
try {
manipulator = sharp(localFilePath, { limitInputPixels: 5000000000 });
console.log('Manipulator = ', manipulator);
imageMetadata = await manipulator.metadata();
console.log("ImageMetadata = ", imageMetadata);
} catch (error) {
console.log(`Image Metadata Extraction Error: ${error.message}`);
throw error;
}
}
doStuff();
This code snippet above fails with the "Input file contains unsupported image format" on the line that extracts metadata (imageMetadata = await manipulator.metadata();)
So the strange thing is, I am able to properly extract the metadata (with no errors) with this same code if I add a short Sleep after this line: const fileStream = fs.createWriteStream(localPath);
So this code snippet (all I'm doing here is adding a short sleep after fs.createWriteSteam) allows the image metadata to be extracted without issue:
const sharp = require("sharp");
const fs = require('fs');
const fetch = require('node-fetch');
async function storeUrlToLocal(sourceUrl) {
const destPath = './';
const request = {
method: 'GET',
encoding: null,
};
response = await fetch(sourceUrl, request);
if (response.status >= 400)
throw new Error(`Failed to fetch data from ${sourceUrl}, status returned = ${response.status}`);
const localPath = `${destPath}test.png`;
const fileStream = fs.createWriteStream(localPath);
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
await sleep(1000);
return new Promise((resolve, reject) => {
response.body.pipe(fileStream);
response.body.on("error", reject);
response.body.on("end", async () => {
const fileExists = fs.existsSync(localPath);
console.log(`All the data in the file has been read ${localPath} = ${fileExists}`);
resolve(localPath);
});
response.body.on("finish",() => {
console.log('All writes are now complete.');
});
}).catch(error => {
console.log(error);
});
}
async function doStuff() {
const localFilePath = await storeUrlToLocal('<INSERT_IMAGE_URL_HERE>');
// Read image file and extract metadata
let manipulator;
let imageMetadata;
try {
manipulator = sharp(localFilePath, { limitInputPixels: 5000000000 });
console.log('Manipulator = ', manipulator);
imageMetadata = await manipulator.metadata();
console.log("ImageMetadata = ", imageMetadata);
} catch (error) {
console.log(`Image Metadata Extraction Error: ${error.message}`);
throw error;
}
}
doStuff();
Why would this Sleep resolve my issues? I don't see any asynchronous calls being run that I would need to be waiting for to complete. Perhaps fs.createWriteStream didn't have enough time to complete its operation? But I do not have the option to await the call to fs.createWriteStream, as it is not async.
I'm trying to develop a simple app that if you pass a parameter in command line the application will search inside a directory and if the text match in some of the files the file should be save in a list, but when I put the console.log the value is not updated
here is my code:
const folder = "./movies/data";
const fs = require("fs");
var args = process.argv.slice(2);
console.log("myArgs: ", args);
var count = 0;
var list = [];
fs.readdir(folder, (err, files) => {
files.forEach((file) => {
fs.readFile(`movies/data/${file}`, "utf8", function (err, data) {
if (err) console.log(err);
if (data.includes("walt disney")) {
count++;
list.push(data);
console.log("Found in: ", data);
}
});
});
console.log(`Foram encontradas ${count} ocorrências pelo termo ${args}.`);
});
any suggestions about what i'm doing wrong?
For your program to work, you will have to add some Promise / async/await logic. On the moment you try to read from the files, the files are still undefined so the fs.readDir() function will not provide the wanted result.
This should work:
const { resolve } = require('path');
const { readdir } = require('fs').promises;
const fs = require("fs");
var args = process.argv.slice(2);
const pathToFiles = "./movies/";
async function getFiles(dir) {
const dirents = await readdir(dir, { withFileTypes: true });
const files = await Promise.all(dirents.map((dirent) => {
const res = resolve(dir, dirent.name);
return dirent.isDirectory() ? getFiles(res) : res;
}));
return Array.prototype.concat(...files);
}
getFiles(pathToFiles)
.then(files => {
console.log(files)
files.forEach((file) => {
fs.readFile(file, 'utf8', (err, data) => {
if (err) console.log(err);
if (data.includes(args)) {
console.log(`${args} found in ${file}.`);
} else {
console.log(`${args} not found.`);
}
});
})
})
.catch (e => console.error(e));
I just started working with Express and am currently lost on how to make an Axios request using route parameters and change some locals based on what the request returns. This is what I have so far:
helpers.js
const axios = require('axios');
const {
titleSuffix,
organizationPath,
varietyPath
} = require('./constants');
let organizationData = {};
let varietyData = {};
const Helpers = {
fetchOrganization: (organizationID) => {
axios.get(organizationPath + organizationID)
.then( (response) => {
//console.log(response);
organizationData = response.data.data;
})
.catch( (error) => {
//console.log(error);
});
return organizationData;
},
fetchVariety: (varietyID) => {
axios.get(varietyPath + varietyID)
.then( (response) => {
//console.log(response);
varietyData = response.data.data;
})
.catch( (error) => {
//console.log(error);
});
return varietyData;
},
setOrgOpenGraphTags: (growerHash, res) => {
Helpers.fetchOrganization(growerHash);
res.locals.meta.og.title = organizationData.name + titleSuffix;
console.log('Org = ' + organizationData.name);
},
setVarOpenGraphTags: (contextualId, res) => {
Helpers.fetchVariety(contextualId);
res.locals.meta.og.title = varietyData.name + titleSuffix;
console.log('Var = ' + varietyData.name);
}
};
module.exports = Helpers;
server.js
// Express
const express = require('express');
const app = express();
// Helpers
const {
setOrgOpenGraphTags,
setVarOpenGraphTags
} = require('./helpers');
// Organization
app.get(['/org/:growerHash/*', '/_org/:growerHash/*'], (req, res) => {
setOrgOpenGraphTags(req.params.growerHash, res);
res.render('org');
});
I'm fairly certain I'm missing something small but can't seem to get the following local changed based on the response from Axios:
res.locals.meta.og.title
Based on what I have so far how do I properly access the response from Axios in Express and change the locals? I really need an answer based around the code I've provided. Currently in my dev environment the request works but in production it returns "undefined". Thanks so much in advance.
The duplicate that I linked, Why is my variable unaltered after I modify it inside of a function? - Asynchronous code reference, discusses why and how writing asynchronous code means that you have to propagate being asynchronous.
Your code, as it is written right now, does not propagate asynchronicity. axios.get() returns a Promise. Unless everything that depends on the value that that Promise resolves to actually waits for the Promise chain to resolve, you aren't going to get what you are expecting.
Consider your code which I have commented below:
const axios = require('axios');
const Helpers = {
fetchOrganization: (organizationID) => {
// axios.get() will return a Promise
// You have to wait for the Promise to finish before
// you can use any data that it produces
// You must propogate the Proise of data up
// You should return axios.get(...)
axios.get(organizationPath + organizationID)
.then((response) => {
//console.log(response);
organizationData = response.data.data;
})
.catch((error) => {
//console.log(error);
});
// This won't be populated by the time you try to use it
return organizationData;
// Instead do
return axios
.get(organizationPath + organizationID)
.then(response => {
const organizationData = response.data.data;
return organizationData
})
.catch(err => console.error(err));
// Better yet, do
/*
return axios.get(organizationPath + organizationID)
.then(res => response.data.data) // Return is implied
.catch(err => console.error(err));
*/
},
setOrgOpenGraphTags: (growerHash, res) => {
// Nothing is coming out of this function and you aren't waiting on it
Helpers.fetchOrganization(growerHash);
// Instead do
return Helpers.fetchOrganization(growerHash)
.then(org => {
return org.name + titleSuffix;
});
//res.locals.meta.og.title = organizationData.name + titleSuffix;
//console.log('Org = ' + organizationData.name);
}
}
// Organization
app.get(['/org/:growerHash/*', '/_org/:growerHash/*'], (req, res) => {
// Below, you are starting the async process
// but you don't wait for the async to finish
// you just immediately res.render()
setOrgOpenGraphTags(req.params.growerHash, res);
res.render('org');
// Instead
setOrgOpenGraphTags(req.params.growerHash, res)
.then(orgTitle => {
res.locals.meta.og.title = orgTitle;
res.render('org');
});
});
After considering that, let's consider a distilled version of your code that will wait for the Promise chain to resolve:
// Let's boil your app down to it's core
const SOME_SUFFIX = "foobar";
// fetchOrganization
function getSomeData(id) {
return axios
.get(`http://www.example.com/things/${id}`)
.then(thatThing => thatThing.nested.property.i.want)
.catch(err => console.error(err));
}
// setOrgOpenGraphTags
function calculateDerivedData(id) {
return getSomeData(id)
.then(thatThingsProperty => `${thatThingsProperty}-${SOME_SUFFIX}`)
}
// Route
app.get("/some/endpoint/:id", (req, res) => {
calculateDerivedData(req.params.id)
.then(thatDerivedDataWeNeed => {
res.locals.whatever = thatDerivedDataWeNeed;
res.render("someTemplate");
})
});
If you want to write something that looks arguably cleaner, you can also consider async/await:
// Let's boil your app down to it's core
const SOME_SUFFIX = "foobar";
// fetchOrganization
async function getSomeData(id) {
try {
const thatThing = await axios.get(`http://www.example.com/things/${id}`);
return thatThing.nested.property.i.want;
} catch(err){
console.error(err);
}
}
// setOrgOpenGraphTags
async function calculateDerivedData(id) {
const thatThingsProperty = await getSomeData(id);
return `${thatThingsProperty}-${SOME_SUFFIX}`;
}
// Route
app.get("/some/endpoint/:id", async function(req, res) => {
res.locals.whatever = await calculateDerivedData(req.params.id);
res.render("someTemplate");
});
I've written a script that recursively asynchronously modifies js files in a directory. It's made up of a search(dirname) function which searches a directory for js files, and a modify(filename) which does the modifying.
let totalFilesSearched = 0;
const search = (dir) => {
fs.readdir(dir, (err, list) => {
if (err) return;
list.forEach((filename) => {
const filepath = path.join(dir, filename);
if (filename.endsWith('.js')) {
modify(filepath);
} else if (fs.lstatSync(filepath).isDirectory()) {
search(filepath);
}
})
});
}
const modify = (filename) => {
fs.readFile(filename, 'utf8', (err, data) => {
if (err) console.log(err);
// ... my modification code ...
totalFilesSearched++;
});
}
search(args[0])
console.log(`Total files searched: ${totalFilesSearched}`);
I want to print out the totalFilesSearched at the end of my script but because my code is asynchronous, it just prints Total files searched: 0 right away.
Does anyone know how I'd wait until the script is about to end to print this out? I'm having trouble because both my search() and modify() functions are asynchronous.
Use Promises instead, and then call console.log when everything is resolved. Use promisify to turn the callbacks into promises:
const { promisify } = require('util');
const readFile = promisify(fs.readFile);
const readDir = util.promisify(fs.readdir);
const search = (dir) => (
readDir(dir).then((list) => (
Promise.all(list.map((filename) => {
const filepath = path.join(dir, filename);
if (filename.endsWith('.js')) {
return modify(filepath); // recursively return the promise
} else if (fs.lstatSync(filepath).isDirectory()) {
return search(filepath); // recursively return the promise
}
}))
))
.catch(err => void 0)
);
const modify = (filename) => (
readFile(filename, 'utf8')
.then((data) => {
// other code
totalFilesSearched++;
}).catch(err => console.log(err))
)
search(args[0])
.then(() => {
console.log(`Total files searched: ${totalFilesSearched}`);
});
Self answer:
Just use process.on('exit', callback_function_to_execute_at_end)
Its built into node, your callback will get executed right before the process exits.