Develop Reference

Using async/await syntax with node stream

I'm using node 18.7 on ubuntu. I'm trying to parse a bunch of csv files to objects (using csv-parse), ultimately to load into a db. Because there are large numbers of these I decided to try streams and I'd like to use the async await style.
So far I have:
const { parse } = require('csv-parse');
const path = __dirname + '/file1.csv';
const opt = { columns: true, relax_column_count: true, skip_empty_lines: true, skip_records_with_error: true };
console.log(path);
const { pipeline } = require('stream');
// const pipeline = stream.pipeline;
async function readByLine(path, opt) {
const readFileStream = fs.createReadStream(path);
var csvParser = parse(opt, function (err, records) {
if (err) throw err;
});
await pipeline(readFileStream, csvParser, (err) => {
if (err) {
console.error('Pipeline failed.', err);
} else {
console.log('Pipeline succeeded.');
}
});
for await (const record of csvParser) {
console.log(record);
}
}
readByLine(path, opt)
When I run this I see:
Pipeline succeeded.
But the parsed objects are not sent to the console. What am I doing wrong?
edit1:
I changed the code to :
async function readByLine(path, opt) {
const readFileStream = fs.createReadStream(path);
var csvParser = parse(opt, function (err, records) {
if (err) throw err;
});
await pipeline(readFileStream, csvParser, (err) => {
if (err) {
console.error('Pipeline failed.', err);
} else {
console.log('Pipeline succeeded.');
}
});
// for await (const record of csvParser) {
// console.log(record);
// }
return csvParser;
}
(async function () {
const o = await readByLine(path, opt);
console.log(o);
})();
The result is an object which has a million properties, but some look set like in the screenshot.

You can only useful await a promise.
The pipeline function you are using doesn't return a promise.
If you look at the documentation you will see:
The pipeline API provides a promise version, which can also receive an options argument as the last parameter with a signal <AbortSignal> property. When the signal is aborted, destroy will be called on the underlying pipeline, with an AbortError.
const { pipeline } = require('node:stream/promises');
const fs = require('node:fs');
const zlib = require('node:zlib');
async function run() {
await pipeline(
fs.createReadStream('archive.tar'),
zlib.createGzip(),
fs.createWriteStream('archive.tar.gz')
);
console.log('Pipeline succeeded.');
}
run().catch(console.error);
Note the different value passed to require. Use that version of pipeline instead.

I called then() on a TypeScript promise but it is still pending. Why is this? How can I get it to resolve?

Here is the index.ts script I am running (based on something I found on reddit):
const path = require("path");
const sql = require("mssql");
const config = require(path.resolve("./config.json"));
let db1;
const connect = () => {
return new Promise((resolve, reject) => {
db1 = new sql.ConnectionPool(config.db, err => {
if (err) {
console.error("Connection failed.", err);
reject(err);
} else {
console.log("Database pool #1 connected.");
resolve();
}
});
});
};
const selectProjects = async (name) => {
const query = `
select * from [Time].ProjectData where [Name] like concat('%', concat(#name, '%'))`;
const request = new sql.Request(db1);
const result = await request
.input("name", name)
.query(query);
return result.recordset;
};
module.exports = {
connect,
selectProjects
};
connect().then(function() {
console.log(selectProjects('General'));
}).catch(function(err) {
console.log(err);
});
When I run the script using node index (after compiling it of course), I get this in the console:
Database pool #1 connected.
Promise { <pending> }
And then the script hangs.

Apparently the await keyword creates an implicit promise; I had to change the last function call to:
connect().then(function() {
selectProjects('General').then(function(data) {
console.log(data);
});
}).catch(function(err) {
console.log(err);
});

NodeJS: Wait for all foreach with Promises to finish but never actually finishes

I am working with Nodejs. I have a forEach which is async as I have to wait for a result inside the forEach. As a result, I need to wait for the forEach to finish and then carry on with the result of the loop. I found several solutions for waiting for the forEach, one of them is using Promises. I did though, and these promises are created, however, the code after the forEach (and therefore the promises) are finished, is never actually executed (console.log is not printed). And the NodeJS function just ends without any errors.
Here is my Code:
var Client = require('ssh2').Client;
// eslint-disable-next-line no-undef
var csv = require("csvtojson");
// eslint-disable-next-line no-undef
var fs = require("fs");
// eslint-disable-next-line no-undef
const config = require('./config.json');
// eslint-disable-next-line no-undef
const os = require('os');
let headerRow = [];
let sumTxAmount = 0;
const filenameShortened = 'testFile';
let csvLists = [];
let csvFile;
const options = {
flags: 'r',
encoding: 'utf8',
handle: null,
mode: 0o664,
autoClose: true
}
var conn = new Client();
async function start() {
const list = await getCSVList();
let content = fs.readFileSync('./temp.json', 'utf8');
content = JSON.parse(content);
var promises = list.map(function(entry) {
return new Promise(async function (resolve, reject) {
if (!content['usedFiles'].includes(entry.filename)) {
const filename = entry.filename;
csvFile = await getCsv(filename);
csvLists.push(csvFile);
console.log('here');
resolve();
} else {
resolve();
}
})
});
console.log(promises)
Promise.all(promises)
.then(function() {
console.log(csvLists.length, 'length');
})
.catch(console.error);
}
start();
The "here" is printed once (not 8 times as the arrays length is 8), but there are 8 promises created. The lower part where I am printing the length of the array is not executed.
Can anyone tell me what I am doing wrong? Am I using Promises and forEach falsely as I have to do an await inside the forEach?
Note: getCSVList() and getCsv() are functions to get Csvs from an sftp server:
function getCSVList() {
return new Promise((resolve, reject) => {
conn.on('ready', function () {
conn.sftp(function (err, sftp) {
if (err) throw err;
sftp.readdir(config.development.pathToFile, function (err, list) {
if(err) {
console.log(err);
conn.end();
reject(err);
} else {
console.log('resolved');
conn.end();
resolve(list);
}
})
})
}).connect({
host: config.development.host,
port: config.development.port, // Normal is 22 port
username: config.development.username,
password: config.development.password
// You can use a key file too, read the ssh2 documentation
});
})
}
function getCsv(filename) {
return new Promise((resolve, reject) => {
conn.on('ready', function () {
conn.sftp(function (err, sftp) {
if (err) reject(err);
let csvFile = sftp.createReadStream(`${config.development.pathToFile}/${filename}`, options);
// console.log(csvFile);
conn.end();
resolve(csvFile);
})
}).connect({
host: config.development.host,
port: config.development.port, // Normal is 22 port
username: config.development.username,
password: config.development.password
// You can use a key file too, read the ssh2 documentation
});
});
}
The output in my console from all the console logs is:
`➜ node server.js
resolved
[ Promise { <pending> },
Promise { <pending> },
Promise { <pending> },
Promise { <pending> },
Promise { <pending> },
Promise { <pending> },
Promise { <pending> },
Promise { <pending> } ]
here`

Break up your problem into pieces, confirming they work along the way.
You are not using the stream correctly, among other things.
I made a working example with ssh2-sftp-client so you can maybe use it as a starting point.
Working example :
var fs = require('fs'); var _ = require('underscore');
var SFTPClient = require('ssh2-sftp-client');
const CONFIG = {
"SSH_CONN_OPTS":{"host":"XXXXXXXX","port":22,"username":"XXXXXXXX","password":"XXXXXXXX"},
"CSV_DIRECTORY":"/var/www/html"
}
//---------------
//.:The order-logic of the script is here
function StartScript(){
console.log("[i] SSH Connection")
LoadValidationFile(()=>{
InitializeSFTP(()=>{ console.log("[+] SSH Connection Established")
ListRemoteDirectory((list)=>{ console.log(`[i] Total Files # ${CONFIG.CSV_DIRECTORY} : ${list.length}`)
//console.log(list) //:now you have a 'list' of file_objects, you can iterate over to check the filename
var csvFileList = [] //store the names of the files you will request after
_.each(list,(list_entry)=>{ console.log(list_entry)
if(!CONFIG.USED_FILES.includes(list_entry.name)){ csvFileList.push(list_entry.name) }
})
//:now loop over the new final list of files you have just validated for future fetch
GenerateFinalOutput(csvFileList)
})
})
})
}
//.:Loads your validation file
function LoadValidationFile(cb){
fs.readFile(__dirname+'/temp.json','utf8',(err,data)=>{ if(err){throw err}else{
var content = JSON.parse(data)
CONFIG.USED_FILES = content.usedFiles
cb()
}})
}
//.:Connects to remote server using CONFIG.SSH_CONN_OPTS
function InitializeSFTP(cb){
global.SFTP = new SFTPClient();
SFTP.connect(CONFIG.SSH_CONN_OPTS)
.then(()=>{cb()})
.catch((err)=>{console.log("[!] InitializeSFTP :",err)})
}
//.:Get a list of files from a remote directory
function ListRemoteDirectory(cb){
SFTP.list(`${CONFIG.CSV_DIRECTORY}`)
.then((list)=>{cb(list)})
.catch((err)=>{console.log("[!] ListRemoteDirectory :",err)})
}
//.:Get target file from remote directory
function GetRemoteFile(filename,cb){
SFTP.get(`${CONFIG.CSV_DIRECTORY}/${filename}`)
.then((data)=>{cb(data.toString("utf8"))}) //convert it to a parsable string
.catch((err)=>{console.log("[!] ListRemoteDirectory :",err)})
}
//-------------------------------------------
var csvLists = []
function GenerateFinalOutput(csv_files,current_index){ if(!current_index){current_index=0}
if(current_index!=csv_files.length){ //:loop
var csv_file = csv_files[current_index]
console.log(`[i] Loop Step #${current_index+1}/${csv_files.length} : ${csv_file}`)
GetRemoteFile(csv_file,(csv_data)=>{
if(csv_data){csvLists.push(csv_data)}
current_index++
GenerateFinalOutput(csv_files,current_index)
})
}else{ //:completed
console.log("[i] Loop Completed")
console.log(csvLists)
}
}
//------------
StartScript()
Good luck!

Promise.all is a method that will return a promise object, but you are not waiting for your start method to execute.
function getCSVList() {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve([1, 2, 3, 4]);
}, 1000);
});
}
function getCsv(params) {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve(params);
}, 1000);
});
}
async function start() {
const list = await getCSVList();
const promises = list.map(item => {
return new Promise(async function (resolve, reject) {
const csvFile = await getCsv(item);
console.log('here');
resolve(csvFile);
});
});
return Promise.all(promises);
}
start().then(res => {
console.log(res);
});

How to pass a variable from a Request as the return value of the global function js

Hello and thank you in advance. I'm using npm tedious package to interact with a database. I'm also using Meteor.call and methods, for which I need to pass a variable (newdata in the example below) that carries the data taken from the database as the return value of the function "rr", so that I would be able to use the result of the function in the client by a Meteor call.
function rr(){
var newdata = [];
var ahora = new Request("SELECT * FROM prueba", function (err) {
if (err) {
console.log("err1");
} else {
}
})
ahora.on('row', function(columns) {
columns.forEach(function(column) {
newdata.push(column.value);
});
});
}
I want "newdata" to be the result of the rr function. How can I do that? If I write "return newdata" it's undefined, I can't use await because newdata is not the return value of any function...
Thank you very much.

Tedious doesn't seem to support promises natively, but you can wrap your function in a promise:
function rr() {
return new Promise((resolve, reject) => {
var ahora = new Request("SELECT * FROM prueba", function (err) {
if (err) {
reject(err);
}
});
var newdata = [];
ahora.on('row', function(columns) {
columns.forEach(function(column) {
newdata.push(column.value);
});
});
resolve(newdata);
}
}
Or slightly shorter:
function rr() {
return new Promise((resolve, reject) => {
new Request("SELECT * FROM prueba")
.on("error", reject)
.on("row", function(columns) {
resolve(columns.map(column => column.value))
});
}
}
If you'd rather not make promises manually, you can try Bluebird's promisify function. I also found a tedious-specific promisifying package tedious-promises, but it doesn't seem to be properly maintained.

You could do something like this:
function rr(){
return await new Promise((resolve, reject) => {
new Request("SELECT * FROM prueba", (err, rowCount) => {err && reject(err);})
.on('row', columns => resolve(columns.map(c => c.value)));
});
}

variable undefined outside youtube search api function

I am fetching playlistId from youtube api .
It gives correct output when console output within the youtube search function.
It gives undefined outside youtube search api function.
var playlistId;
async function suggestTrack(genre) {
youtube.search.list({
auth: config.youtube.key,
part: 'id,snippet',
q: genre
}, function (err, data) {
if (err) {
console.error('Error: ' + err);
}
if (data) {
console.log(data.items[0].id.playlistId); //getting the id
playlistId = data.items[0].id.playlistId;
}
//process.exit();
});
console.log(playlistId);// undefined
const tracks = await youtube_api.getPlaylistTracks(playlistId);
return tracks[Math.floor(tracks.length * Math.random())];
}

The API call is asynchronous. And you are printing the value of playlistId before the response of the api even comes back. You have to wait for the response to come. And since you are using async wrap the api call in a Promise and use await. To promisify the search.list method, you have a lot of options, or you can do it yourself, like below
function search(key, part, genre) {
return new Promise((resolve, reject) => {
youtube.search.list({
auth: key,
part: part,
q: genre
}, function (err, data) {
if (err) {
reject(err);
return;
}
// use better check for playlistId here
resolve(data ? data.items[0].id.playlistId : null);
})
});
}
// then use it here
async function suggestTrack(genre) {
const playlistId = await search(config.youtube.key, 'id,snippet', genre);
const tracks = await youtube_api.getPlaylistTracks(playlistId);
return tracks[Math.floor(tracks.length * Math.random())];
}

youtube.search.list is asynchronous. You are trying to access to playlistId as it was a part of a synchronous process.
You can wrap the youtube.search.list inside a Promise to simplify it's use.
OLD WAY
function wrappedSearch() {
return new Promise((resolve, reject) => {
youtube.search.list({
auth: config.youtube.key,
part: 'id,snippet',
q: genre
}, (err, data) => {
if (err) {
console.error('Error: ' + err);
return reject(err);
}
return resolve((data && data.items[0].id.playlistId) || false);
});
});
}
async function suggestTrack(genre) {
const playlistId = await wrappedSearch();
// Here playlistId is either the playlistId, either false
console.log(playlistId);
const tracks = await youtube_api.getPlaylistTracks(playlistId);
return tracks[Math.floor(tracks.length * Math.random())];
}
NEW WAY
available in node v8 doc
tutorial
const {
promisify,
} = require('util');
const youtubeSearchAsync = promisify(youtube.search.list);
async function suggestTrack(genre) {
const data = await youtubeSearchAsync({
auth: config.youtube.key,
part: 'id,snippet',
q: genre
});
const playlistId = (data && data.items[0].id.playlistId) || false;
console.log(playlistId);
const tracks = await youtube_api.getPlaylistTracks(playlistId);
return tracks[Math.floor(tracks.length * Math.random())];
}