Develop Reference

How to stream x-ndjson content using Express and parse the streamed data?

I have a TS library using Node v19.1.0. The library has a function that observes streamed server events.
The server provides a /events route streaming 'application/x-ndjson' content which might be an event/ping/... ( sending a ping every x seconds is important to keep the connection alive )
My observe function parses the streamed data and inspects it. If it is a valid event it will pass it to a callback function. The caller also receives an abort function to abort the streaming on demand.
Whenever I run tests locally or via CI I get the following error
Warning: Test "observes events." generated asynchronous activity after the test ended. This activity created the error "AbortError: The operation was aborted." and would have caused the test to fail, but instead triggered an unhandledRejection event.
I tried to minimize the example code using plain JavaScript
const assert = require('assert/strict');
const express = require('express');
const { it } = require('node:test');
it('observes events.', async () => {
const expectedEvent = { type: 'event', payload: { metadata: { type: 'entity-created', commandId: 'commandId' } } };
const api = express();
const server = api
.use(express.json())
.post('/events', (request, response) => {
response.writeHead(200, {
'content-type': 'application/x-ndjson',
});
const line = JSON.stringify(expectedEvent) + '\n';
response.write(line);
})
.listen(3000);
let stopObserving = () => {
throw new Error('should never happen');
};
const actualEventPayload = await new Promise(async resolve => {
stopObserving = await observeEvents(async newEvent => {
resolve(newEvent);
});
});
stopObserving();
server.closeAllConnections();
server.close();
assert.deepEqual(actualEventPayload, expectedEvent.payload);
});
const observeEvents = async function (onReceivedFn) {
const abortController = new AbortController();
const response = await fetch('http://localhost:3000/events', {
method: 'POST',
headers: { 'content-type': 'application/json' },
signal: abortController.signal,
});
if (!response.ok) {
throw new Error('error handling goes here - request failed');
}
Promise.resolve().then(async () => {
if (!response.body) {
throw new Error('error handling goes here - missing response body');
}
for await (const item of parseStream(response.body, abortController)) {
switch (item.type) {
case 'event': {
await onReceivedFn(item.payload);
break;
}
case 'ping':
// Intentionally left blank
break;
case 'error':
throw new Error('error handling goes here - stream failed');
default:
throw new Error('error handling goes here - should never happen');
}
}
});
return () => { abortController.abort(); };
};
const parseLine = function () {
return new TransformStream({
transform(chunk, controller) {
try {
const data = JSON.parse(chunk);
// ... check if this is a valid line...
controller.enqueue(data);
} catch (error) {
controller.error(error);
}
},
});
};
const splitLines = function () {
let buffer = '';
return new TransformStream({
transform(chunk, controller) {
buffer += chunk;
const lines = buffer.split('\n');
for (let i = 0; i < lines.length - 1; i++) {
controller.enqueue(lines[i]);
}
buffer = lines.at(-1) ?? '';
},
flush(controller) {
if (buffer.length > 0) {
controller.enqueue(buffer);
}
},
});
};
const parseStream = async function* (stream, abortController) {
let streamReader;
try {
const pipedStream = stream
.pipeThrough(new TextDecoderStream())
.pipeThrough(splitLines())
.pipeThrough(parseLine());
streamReader = pipedStream.getReader();
while (true) {
const item = await streamReader.read();
if (item.done) {
break;
}
yield item.value;
}
} finally {
await streamReader?.cancel();
abortController.abort();
}
};
Unfortunately, when running node --test, the test does not finish. I have to cancel it manually.
The test breaks with these lines
const actualEventPayload = await new Promise(async resolve => {
stopObserving = await observeEvents(async newEvent => {
resolve(newEvent);
});
});
and I think that's because the Promise never resolves. I thought the stream parsing might have a bug but if you remove all the stream parsing stuff and replace
Promise.resolve().then(async () => {
/* ... */
});
with
Promise.resolve().then(async () => {
await onReceivedFn({ metadata: { type: 'entity-created', commandId: 'commandId' }});
});
it doesn't work neither. Does someone know what's wrong or missing?

The problem here has nothing to do with your promise not resolving since you never even get to that point.
The problem here is that observeEvents is not yet initialized when the test is being run and thus throws a ReferenceError: Cannot access 'observeEvents' before initialization error.
To see that for yourself you can add a simple const it = (name, fn) => fn(); stub to the top of the file and run it without the --test.
There are multiple ways to fix this and the simplest one is to move the test function to the bottom of the file.
If you don't want to do that you can also define the observeEvents function like this: async function observeEvents(onReceivedFn) {...}. This way it will be available immediately.

How to convert Promise to async await in node js

i am trying to convert different code with promise to async await. as i am new to node and dont have much idea about promise can anybody help me with the following example. my vs code doesn't highlight the promise syntax.
module.exports = {
execSqlQuery: function (procedure, parameters) {
return new Promise(function (fulfill, reject) {
var request = new sql.Request(connPool);
parameters.forEach(function (parameter) {
request.input(parameter.name, parameter.type, parameter.value);
});
request.execute(procedure, function (err, recordSets) {
if (err) {
reject(err);
}
else {
fulfill(recordSets);
}
});
});
},

First, check whether request.execute has a promise version rather than a callback version. If not you can use promisify library to make it a promise.
const util = require('util');
const execute= util.promisify(sql.execute);
const execSqlQuery = async (procedure, parameters) => {
var request = new sql.Request(connPool);
parameters.forEach(function (parameter) {
request.input(parameter.name, parameter.type, parameter.value);
});
var recordSets = await request.execute(procedure)
return recordSets;
}

I'm guessing you using Microsoft SQL then if it is true: https://www.npmjs.com/package/mssql#asyncawait
function async () {
var request = new sql.Request(connPool);
parameters.forEach(function (parameter) {
request.input(parameter.name, parameter.type, parameter.value);
});
var res = await request.execute(procedure, function (err, recordSets);
// now check the result for success or errors!
}

this one worked
const execSqlQuery = async (procedure, parameters) => {
var request = new sql.Request(connPool);
parameters.forEach(function (parameter) {
request.input(parameter.name, parameter.type, parameter.value);
});
var recordSets = await request.execute(procedure)
return recordSets;
}

NodeJS: Wait for all foreach with Promises to finish but never actually finishes

I am working with Nodejs. I have a forEach which is async as I have to wait for a result inside the forEach. As a result, I need to wait for the forEach to finish and then carry on with the result of the loop. I found several solutions for waiting for the forEach, one of them is using Promises. I did though, and these promises are created, however, the code after the forEach (and therefore the promises) are finished, is never actually executed (console.log is not printed). And the NodeJS function just ends without any errors.
Here is my Code:
var Client = require('ssh2').Client;
// eslint-disable-next-line no-undef
var csv = require("csvtojson");
// eslint-disable-next-line no-undef
var fs = require("fs");
// eslint-disable-next-line no-undef
const config = require('./config.json');
// eslint-disable-next-line no-undef
const os = require('os');
let headerRow = [];
let sumTxAmount = 0;
const filenameShortened = 'testFile';
let csvLists = [];
let csvFile;
const options = {
flags: 'r',
encoding: 'utf8',
handle: null,
mode: 0o664,
autoClose: true
}
var conn = new Client();
async function start() {
const list = await getCSVList();
let content = fs.readFileSync('./temp.json', 'utf8');
content = JSON.parse(content);
var promises = list.map(function(entry) {
return new Promise(async function (resolve, reject) {
if (!content['usedFiles'].includes(entry.filename)) {
const filename = entry.filename;
csvFile = await getCsv(filename);
csvLists.push(csvFile);
console.log('here');
resolve();
} else {
resolve();
}
})
});
console.log(promises)
Promise.all(promises)
.then(function() {
console.log(csvLists.length, 'length');
})
.catch(console.error);
}
start();
The "here" is printed once (not 8 times as the arrays length is 8), but there are 8 promises created. The lower part where I am printing the length of the array is not executed.
Can anyone tell me what I am doing wrong? Am I using Promises and forEach falsely as I have to do an await inside the forEach?
Note: getCSVList() and getCsv() are functions to get Csvs from an sftp server:
function getCSVList() {
return new Promise((resolve, reject) => {
conn.on('ready', function () {
conn.sftp(function (err, sftp) {
if (err) throw err;
sftp.readdir(config.development.pathToFile, function (err, list) {
if(err) {
console.log(err);
conn.end();
reject(err);
} else {
console.log('resolved');
conn.end();
resolve(list);
}
})
})
}).connect({
host: config.development.host,
port: config.development.port, // Normal is 22 port
username: config.development.username,
password: config.development.password
// You can use a key file too, read the ssh2 documentation
});
})
}
function getCsv(filename) {
return new Promise((resolve, reject) => {
conn.on('ready', function () {
conn.sftp(function (err, sftp) {
if (err) reject(err);
let csvFile = sftp.createReadStream(`${config.development.pathToFile}/${filename}`, options);
// console.log(csvFile);
conn.end();
resolve(csvFile);
})
}).connect({
host: config.development.host,
port: config.development.port, // Normal is 22 port
username: config.development.username,
password: config.development.password
// You can use a key file too, read the ssh2 documentation
});
});
}
The output in my console from all the console logs is:
`➜ node server.js
resolved
[ Promise { <pending> },
Promise { <pending> },
Promise { <pending> },
Promise { <pending> },
Promise { <pending> },
Promise { <pending> },
Promise { <pending> },
Promise { <pending> } ]
here`

Break up your problem into pieces, confirming they work along the way.
You are not using the stream correctly, among other things.
I made a working example with ssh2-sftp-client so you can maybe use it as a starting point.
Working example :
var fs = require('fs'); var _ = require('underscore');
var SFTPClient = require('ssh2-sftp-client');
const CONFIG = {
"SSH_CONN_OPTS":{"host":"XXXXXXXX","port":22,"username":"XXXXXXXX","password":"XXXXXXXX"},
"CSV_DIRECTORY":"/var/www/html"
}
//---------------
//.:The order-logic of the script is here
function StartScript(){
console.log("[i] SSH Connection")
LoadValidationFile(()=>{
InitializeSFTP(()=>{ console.log("[+] SSH Connection Established")
ListRemoteDirectory((list)=>{ console.log(`[i] Total Files # ${CONFIG.CSV_DIRECTORY} : ${list.length}`)
//console.log(list) //:now you have a 'list' of file_objects, you can iterate over to check the filename
var csvFileList = [] //store the names of the files you will request after
_.each(list,(list_entry)=>{ console.log(list_entry)
if(!CONFIG.USED_FILES.includes(list_entry.name)){ csvFileList.push(list_entry.name) }
})
//:now loop over the new final list of files you have just validated for future fetch
GenerateFinalOutput(csvFileList)
})
})
})
}
//.:Loads your validation file
function LoadValidationFile(cb){
fs.readFile(__dirname+'/temp.json','utf8',(err,data)=>{ if(err){throw err}else{
var content = JSON.parse(data)
CONFIG.USED_FILES = content.usedFiles
cb()
}})
}
//.:Connects to remote server using CONFIG.SSH_CONN_OPTS
function InitializeSFTP(cb){
global.SFTP = new SFTPClient();
SFTP.connect(CONFIG.SSH_CONN_OPTS)
.then(()=>{cb()})
.catch((err)=>{console.log("[!] InitializeSFTP :",err)})
}
//.:Get a list of files from a remote directory
function ListRemoteDirectory(cb){
SFTP.list(`${CONFIG.CSV_DIRECTORY}`)
.then((list)=>{cb(list)})
.catch((err)=>{console.log("[!] ListRemoteDirectory :",err)})
}
//.:Get target file from remote directory
function GetRemoteFile(filename,cb){
SFTP.get(`${CONFIG.CSV_DIRECTORY}/${filename}`)
.then((data)=>{cb(data.toString("utf8"))}) //convert it to a parsable string
.catch((err)=>{console.log("[!] ListRemoteDirectory :",err)})
}
//-------------------------------------------
var csvLists = []
function GenerateFinalOutput(csv_files,current_index){ if(!current_index){current_index=0}
if(current_index!=csv_files.length){ //:loop
var csv_file = csv_files[current_index]
console.log(`[i] Loop Step #${current_index+1}/${csv_files.length} : ${csv_file}`)
GetRemoteFile(csv_file,(csv_data)=>{
if(csv_data){csvLists.push(csv_data)}
current_index++
GenerateFinalOutput(csv_files,current_index)
})
}else{ //:completed
console.log("[i] Loop Completed")
console.log(csvLists)
}
}
//------------
StartScript()
Good luck!

Promise.all is a method that will return a promise object, but you are not waiting for your start method to execute.
function getCSVList() {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve([1, 2, 3, 4]);
}, 1000);
});
}
function getCsv(params) {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve(params);
}, 1000);
});
}
async function start() {
const list = await getCSVList();
const promises = list.map(item => {
return new Promise(async function (resolve, reject) {
const csvFile = await getCsv(item);
console.log('here');
resolve(csvFile);
});
});
return Promise.all(promises);
}
start().then(res => {
console.log(res);
});

Waiting for 2 promises to come back to use simultaneously in a function

First-off: I hope my question has not already been asked and answered elsewhere... I've been looking SO and the web on that and I can't seem to find anything (maybe it's not even possible).
Basically, I'm trying to build a DB (in my case with Mongo) get info from an webAPI (the crypto exchange Kraken).
The objective is to build my collections with various objects that I get from requests.
Here the gitub repo: MongoNode
Here's the current code (I'm using simple callbacks and request at the moment: chaining up everything):
// External Modules
var request = require('request');
var yargs = require('yargs');
var _ = require('underscore');
var async = require('async');
var MongoClient = require('mongodb').MongoClient,
co = require('co'),
assert = require('assert');
// Own Modules
// Variables
var tickerArr= []
// Generator function for connection
co(function*() {
// Connection URL
var url = 'mongodb://localhost:27017/CryptoDB';
// Use connect method to connect to the Server
var db = yield MongoClient.connect(url);
// Inside the DB connection
console.log('Connection up and running...');
// Getting the list of the tickers for USD and EUR exclusively
request({
url:'https://api.kraken.com/0/public/AssetPairs',
json: true
}, (error, response, body) => {
var jsonObject = body;
_.map(jsonObject, function(content) {
_.map(content, function(data) {
if(data.altname.indexOf('EUR') !== -1 || data.altname.indexOf('USD') !== -1)
tickerArr.push(data.altname);
});
});
// Getting the ticker info for each USD && EUR ticker available
async.forEach(tickerArr, (item) => {
request({
url:`https://api.kraken.com/0/public/Ticker?pair=${item}`,
json: true
}, (error, response, body) => {
db.collection('Assets').insertOne(
body.result,
request({
url:'https://api.kraken.com/0/public/Time',
json: true
}, (error, response, body) => {
return body.result;
})
);
console.log('Asset Added!', body.result);
});
});
});
// Closing the DB connection
// db.close();
}).catch(function(err) {
console.log(err.stack);
});
I've trying to use promises chaining 'em up as well but can't seem to find a way for it to work.
Here's my promise playground:
var somePromise1 = new Promise((resolve, reject) => {
resolve('Hey! It worked!');
});
var somePromise2 = new Promise((resolve, reject) => {
resolve('Hey! It worked the second time!');
});
somePromise1.then(somePromise2.then((message1, message2) => {
console.log('Success', message1, message2);
})
);
and here's what it print in the console when executed:
$Success Hey! It worked the second time! undefined
Thanks in advance for your help and I hope the question hasn't been answered already a million times!

var somePromise1 = new Promise((resolve, reject) => {
resolve('Hey! It worked!');
});
var somePromise2 = new Promise((resolve, reject) => {
resolve('Hey! It worked the second time!');
});
somePromise1.then((message1) => somePromise2.then((message2) => {
console.log('Success', message1, message2);
})
);
Promises are quite convoluted though. async/await makes things a lot clearer:
var somePromise1 = new Promise((resolve, reject) => {
resolve('Hey! It worked!');
});
var somePromise2 = new Promise((resolve, reject) => {
resolve('Hey! It worked the second time!');
});
async function exec() {
const message1 = await somePromise1
const message2 = await somePromise2
console.log('Success', message1, message2);
}
exec()

Promisify streams

I'm trying to promisify streams but it appears harder than I expected. Here is my attempt:
'use strict'
const Promise = require('bluebird')
const Twitter = require('twitter')
const TwitterStream = module.exports = function TwitterStream (config) {
// init Twitter Streaming API for OAuth
this.stream = new Twitter({
consumer_key: config.get('/twitter/consumerKey'),
consumer_secret: config.get('/twitter/consumerSecret'),
access_token_key: config.get('/twitter/accessTokenKey'),
access_token_secret: config.get('/twitter/accessTokenSecret')
})
.stream('statuses/filter', {
track: config.get('/twitter/track')
})
}
TwitterStream.prototype.receive = function () {
return new Promise((resolve, reject) => {
this.stream.on('data', resolve).on('error', reject)
})
}
TwitterStream.prototype.destroy = function () {
this.stream.destroy()
}
The main problem is that when I create the object
const stream = new TwitterStream(config)
stream.receive().then((data) => console.log(data))
when I execute only one object is read. no other data are streamed.
TwitterStream.prototype.receive = function () {
return new Promise((resolve, reject) => {
this.stream
.on('data', (data) => resolve(data)
.on('error', (error) => reject(error))
})
}

By using Rx extensions, it's pretty straightforward:
TwitterStream.prototype.receive = function () {
return Rx.Observable.create((observer) => {
this.stream
.on('data', (data) => observer.onNext(data))
.on('error', (err) => observer.onError(err));
});
}
And then
const stream = new TwitterStream(config)
stream.receive().subscribe((data) => console.log(data));

You need to return a promise in the callback of the stream.on function. Right now, the receive method when being called just returns a promise which once resolved returns the value or error.

Here is a not tested and most likely still buggy code to illustrate how you could do it with promises:
function defer() {
var resolve, reject;
var promise = new Promise(function() {
resolve = arguments[0];
reject = arguments[1];
});
return {
resolve: resolve,
reject: reject,
promise: promise
};
}
TwitterStream.prototype.receive = function() {
this.stream
.on('data', data => {
this.dataCache = this.dataCache || [];
this.dataCache.push(data);
this.tryToSendData()
})
.on('end', () => {
this.finished = true;
this.tryToSendData()
})
.on('error', err => {
this.lastError = err;
// error handling still missing
})
return this;
}
TwitterStream.prototype.tryToSendData = function() {
if (this.defered) {
let defered = this.defered;
this.defered = null;
// if data is available or finished then pass the first element of buffer (or undefined)
defered.resolve(this.dataCache.shift())
}
}
TwitterStream.prototype.getNextData = function() {
if (this.dataCache.length > 0 || this.finished) {
// if data is available or finished then pass the first element of buffer (or undefined)
return Promise.resolve(this.dataCache.shift());
} else {
// otherwise we need a defered object
this.defered = defer();
}
}
The usage could then look like this:
stream.receive().getNextData()
.then(function processData(data) {
if (data) {
console.dir(data);
// if data is available then continue requestin the data
return stream.getNextData().then(processData);
}
})
It is a rare case where you could use Deferreds.

I think you might want to take a look at my, already promisified streams in scramjet.
For your Twitter example this code should work well:
const stream = new Twitter({
consumer_key: config.get('/twitter/consumerKey'),
consumer_secret: config.get('/twitter/consumerSecret'),
access_token_key: config.get('/twitter/accessTokenKey'),
access_token_secret: config.get('/twitter/accessTokenSecret')
})
.stream('statuses/filter', {
track: config.get('/twitter/track')
})
.pipe(new scramjet.DataStream)
Then perform any transformations you like... for example map the stream somehow and accumulate the stream into an array when you're done.
stream.map(
function (a) { return modifyTheTweetSomehow(a); } // a Promise can be returned here
).accumulate(
function(a, i) { a.push(i); },
[]
) // this returns a Promise that will be resolved on stream end.
I hope you like it. :)