Develop Reference

How to stream x-ndjson content using Express and parse the streamed data?

I have a TS library using Node v19.1.0. The library has a function that observes streamed server events.
The server provides a /events route streaming 'application/x-ndjson' content which might be an event/ping/... ( sending a ping every x seconds is important to keep the connection alive )
My observe function parses the streamed data and inspects it. If it is a valid event it will pass it to a callback function. The caller also receives an abort function to abort the streaming on demand.
Whenever I run tests locally or via CI I get the following error
Warning: Test "observes events." generated asynchronous activity after the test ended. This activity created the error "AbortError: The operation was aborted." and would have caused the test to fail, but instead triggered an unhandledRejection event.
I tried to minimize the example code using plain JavaScript
const assert = require('assert/strict');
const express = require('express');
const { it } = require('node:test');
it('observes events.', async () => {
const expectedEvent = { type: 'event', payload: { metadata: { type: 'entity-created', commandId: 'commandId' } } };
const api = express();
const server = api
.use(express.json())
.post('/events', (request, response) => {
response.writeHead(200, {
'content-type': 'application/x-ndjson',
});
const line = JSON.stringify(expectedEvent) + '\n';
response.write(line);
})
.listen(3000);
let stopObserving = () => {
throw new Error('should never happen');
};
const actualEventPayload = await new Promise(async resolve => {
stopObserving = await observeEvents(async newEvent => {
resolve(newEvent);
});
});
stopObserving();
server.closeAllConnections();
server.close();
assert.deepEqual(actualEventPayload, expectedEvent.payload);
});
const observeEvents = async function (onReceivedFn) {
const abortController = new AbortController();
const response = await fetch('http://localhost:3000/events', {
method: 'POST',
headers: { 'content-type': 'application/json' },
signal: abortController.signal,
});
if (!response.ok) {
throw new Error('error handling goes here - request failed');
}
Promise.resolve().then(async () => {
if (!response.body) {
throw new Error('error handling goes here - missing response body');
}
for await (const item of parseStream(response.body, abortController)) {
switch (item.type) {
case 'event': {
await onReceivedFn(item.payload);
break;
}
case 'ping':
// Intentionally left blank
break;
case 'error':
throw new Error('error handling goes here - stream failed');
default:
throw new Error('error handling goes here - should never happen');
}
}
});
return () => { abortController.abort(); };
};
const parseLine = function () {
return new TransformStream({
transform(chunk, controller) {
try {
const data = JSON.parse(chunk);
// ... check if this is a valid line...
controller.enqueue(data);
} catch (error) {
controller.error(error);
}
},
});
};
const splitLines = function () {
let buffer = '';
return new TransformStream({
transform(chunk, controller) {
buffer += chunk;
const lines = buffer.split('\n');
for (let i = 0; i < lines.length - 1; i++) {
controller.enqueue(lines[i]);
}
buffer = lines.at(-1) ?? '';
},
flush(controller) {
if (buffer.length > 0) {
controller.enqueue(buffer);
}
},
});
};
const parseStream = async function* (stream, abortController) {
let streamReader;
try {
const pipedStream = stream
.pipeThrough(new TextDecoderStream())
.pipeThrough(splitLines())
.pipeThrough(parseLine());
streamReader = pipedStream.getReader();
while (true) {
const item = await streamReader.read();
if (item.done) {
break;
}
yield item.value;
}
} finally {
await streamReader?.cancel();
abortController.abort();
}
};
Unfortunately, when running node --test, the test does not finish. I have to cancel it manually.
The test breaks with these lines
const actualEventPayload = await new Promise(async resolve => {
stopObserving = await observeEvents(async newEvent => {
resolve(newEvent);
});
});
and I think that's because the Promise never resolves. I thought the stream parsing might have a bug but if you remove all the stream parsing stuff and replace
Promise.resolve().then(async () => {
/* ... */
});
with
Promise.resolve().then(async () => {
await onReceivedFn({ metadata: { type: 'entity-created', commandId: 'commandId' }});
});
it doesn't work neither. Does someone know what's wrong or missing?

The problem here has nothing to do with your promise not resolving since you never even get to that point.
The problem here is that observeEvents is not yet initialized when the test is being run and thus throws a ReferenceError: Cannot access 'observeEvents' before initialization error.
To see that for yourself you can add a simple const it = (name, fn) => fn(); stub to the top of the file and run it without the --test.
There are multiple ways to fix this and the simplest one is to move the test function to the bottom of the file.
If you don't want to do that you can also define the observeEvents function like this: async function observeEvents(onReceivedFn) {...}. This way it will be available immediately.

Promises not working on websocket when listening to messages on js

Upon receptiton of a message from a websocket server, I would like to call a promise function and wait for a respond to decide if it needs to threat another messages or not.
I tried to wrap a connection.on('message', cb) into a Promise but it doesn't help either. Here is the code:
let alreadyBought = false
const client = new WebSocketClient()
client.connect('wss://...')
client.on('connect', async (connection) => {
connection.on('message', async event => {
if (event.type === 'utf8') {
const data = JSON.parse(event.utf8Data)
if (!alreadyBought) {
await trigger(data.p) // <--- not working
}
}
})
})
async function trigger(price) {
const order = await exchange.createLimitBuyOrder(config.currency, amount, price)
console.log(order)
alreadyBought = true
}
If I do a console.log(event), I get this, check a timestamp:
{
d: '{"t":1924698,"p":"1541.86", "T":1662043735929}'
}
{
d: '{"t":1924699,"p":"1541.86","T":1662043735955}' // <-- At the same timestamp
}
{
d: '{"t":1924700,"p":"1541.21","T":1662043735955}' // <-- At the same timestamp
}
{
d: '{"t":1924701,"p":"1540.91","T":1662043735955}' // <-- At the same timestamp
}

Ok, I finally found a solution, by using a pricelocks while a Promise is processing.
let pricelock = false
const client = new WebSocketClient()
client.connect('wss://...')
client.on('connect', async (connection) => {
connection.on('message', async event => {
if (event.type === 'utf8') {
const data = JSON.parse(event.utf8Data)
if (!pricelock) {
pricelock = true
await trigger(data.p).then(() => {
pricelock = false
})
}
}
})
})
async function trigger(price) {
await exchange.createLimitBuyOrder(config.currency, amount, price)
}
I will leave a post open in case there is a better solution.

How to do cleanup in useEffect React?

Usually I do useEffect cleanups like this:
useEffect(() => {
if (!openModal) {
let controller = new AbortController();
const getEvents = async () => {
try {
const response = await fetch(`/api/groups/`, {
signal: controller.signal,
});
const jsonData = await response.json();
setGroupEvents(jsonData);
controller = null;
} catch (err) {
console.error(err.message);
}
};
getEvents();
return () => controller?.abort();
}
}, [openModal]);
But I don't know how to do in this situation:
I have useEffect in Events.js file that get events from function and function in helpers.js file that create events on given dates except holidays (holiday dates fetch from database).
Events.js
useEffect(() => {
if (groupEvents.length > 0) {
const getGroupEvents = async () => {
const passed = await passEvents(groupEvents); // function in helpers.js (return array of events)
if (passed) {
setEvents(passed.concat(userEvents));
} else {
setEvents(userEvents);
}
};
getGroupEvents();
}
}, [groupEvents, userEvents]);
helpers.js
const passEvents = async (e) => {
try {
const response = await fetch(`/api/misc/holidays`, {
credentials: 'same-origin',
});
const jsonData = await response.json();
const holidays = jsonData.map((x) => x.date.split('T')[0]); // holiday dates
return getEvents(e, holidays); // create events
} catch (err) {
console.error(err.message);
}
};

You can either not clean up, which really is also fine in many situations, but if you definitely want to be able to abort the in-flight request, you will need to create the signal from the top-level where you want to be able to abort, and pass it down to every function.
This means adding a signal parameter to passEvents.

Remove ?:
return () => controller.abort();
Like to this: https://www.youtube.com/watch?v=aKOQtGLT-Yk&list=PL4cUxeGkcC9gZD-Tvwfod2gaISzfRiP9d&index=25
Is it what you want?

Repeat async function until true

I have an async function that checks for the status of an order (checkOrderStatus()). I would like to repeat this function until it returns either "FILLED" or "CANCELED", then use this return value in another function to decide to continue or stop the code. Every order goes through different status before being "FILLED" or "CANCELED", therefore the need to repeat the checkOrderStatus() function (it is an API call).
What I have now is this, to repeat the checkOrderStatus() function:
const watch = filter => {
return new Promise(callback => {
const interval = setInterval(async () => {
if (!(await filter())) return;
clearInterval(interval);
callback();
}, 1000);
});
};
const watchFill = (asset, orderId) => {
return watch(async () => {
const { status } = await checkOrderStatus(asset, orderId);
console.log(`Order status: ${status}`);
if (status === 'CANCELED') return false;
return status === 'FILLED';
});
};
I then call watchFill() from another function, where I would like to check its return value (true or false) and continue the code if true or stop it if false:
const sellOrder = async (asset, orderId) => {
try {
const orderIsFilled = await watchFill(asset, orderId);
if (orderIsFilled) {
//… Continue the code (status === 'FILLED'), calling other async functions …
}
else {
//… Stop the code
return false;
}
}
catch (err) {
console.error('Err sellIfFilled() :', err);
}
};
However, this does not work. I can see the status being updated in the terminal via the console.log in watchFill(), but it never stops and most importantly, the value in the orderIsFilled variable in sellOrder() does not get updated, whatever the value returned by watchFill() becomes.
How can I achieve the desired behavior?

watch never calls resolve (in the original code, this is misleadingly named callback()) with any value, so there's no way const orderIsFilled = await watchFill(asset, orderId); will populate orderIsFilled with anything but undefined.
If you save the result of await filter() in a variable and pass it to
callback as callback(result), your code seems like it should work.
That said, the code can be simplified by using a loop and writing a simple wait function. This way, you can return a value (more natural than figuring out how/when to call resolve), keep the new Promise pattern away from the logic and avoid dealing with setInterval and the bookkeeping that goes with that.
const wait = ms =>
new Promise(resolve => setTimeout(resolve, ms))
;
const watch = async (predicate, ms) => {
for (;; await wait(ms)) {
const result = await predicate();
if (result) {
return result;
}
}
};
/* mock the API for demonstration purposes */
const checkOrderStatus = (() => {
let calls = 0;
return async () => ({
status: ++calls === 3 ? "FILLED" : false
});
})();
const watchFill = (asset, orderId) =>
watch(async () => {
const {status} = await checkOrderStatus();
console.log(`Order status: ${status}`);
return status === "CANCELLED" ? false : status === "FILLED";
}, 1000)
;
const sellOrder = async () => {
try {
const orderIsFilled = await watchFill();
console.log("orderIsFilled:", orderIsFilled);
}
catch (err) {
console.error('Err sellIfFilled() :', err);
}
};
sellOrder();

You can use recursive functionality like this:
const checkOrderStatus = async () => {
// ... function does some work ...
await someOtherFunction() // you can use here the other async function as well
// ... function does some more work after returning from await ...
if(/* if status is FILLED or CANCELED */) {
// return true or false or some info about response for your needs
} else {
checkOrderStatus();
}
}
// this will response back when status will be FILLED or CANCELED
await checkOrderStatus();

The watch function clears the interval timer after the first call if filter resolves with false. setInterval doesn't wait for an async function to finish executing either so you'll have to create a loop yourself. Try this:
const delay = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds));
const watch = async check => {
while (true) {
if (await check()) {
return;
}
await delay(1000);
}
};
Because watch only resolves when check succeeds, it is not possible to fail so you don't need to check for it (this might be a bug in your code):
const sellOrder = async (asset, orderId) => {
try {
await watchFill(asset, orderId);
//… Continue the code (status === 'FILLED'), calling other async functions …
}
catch (err) {
console.error('Err sellIfFilled() :', err);
}
};
p-wait-for contains an excellent implementation of this. You can use it like so:
import pWaitFor from 'p-wait-for';
const watchFill = (asset, orderId) => pWaitFor(async () => {
const { status } = await checkOrderStatus(asset, orderId);
console.log(`Order status: ${status}`);
if (status === 'CANCELED') return false;
return status === 'FILLED';
}, {
interval: 1000,
leadingCheck: false
});

Async/Await in for loop NodeJS Not blocking the loop execuation

I know that old school for loop works in the traditional way - that it waits for the await to finish getting results.
But in my use case, I need to read a file from local/s3 and process it line by line, and for each line I need to call an External API.
Generally I use await inside the loop because all are running inside a lambda and I don't want to use all memory for running it parallelly.
Here I am reading the file using a stream.on() method, and in order to use await inside that, I need to add async in read method, like so:
stream.on('data',async () =>{
while(data=stream.read()!==null){
console.log('line');
const requests = getRequests(); // sync code,no pblms
for(let i=0;i<requests.length;i++){
const result = await apiCall(request[i);
console.log('result from api')
const finalResult = await anotherapiCall(result.data);
}
}
});
This is working but order in which the lines are processed is not guaranteed. I need all in a sync manner. Any help?
Complete Code
async function processSOIFileLocal (options, params) {
console.log('Process SOI file');
const readStream = byline.createStream(fs.createReadStream(key));
readStream.setEncoding('utf8');
const pattern = /^UHL\s|^UTL\s/;
const regExp = new RegExp(pattern);
readStream.on('readable', () => {
let line;
while (null !== (line = readStream.read())) {
if (!regExp.test(line.toString())) {
totalRecordsCount++;
dataObject = soiParser(line);
const { id } = dataObject;
const XMLRequests = createLoSTRequestXML(
options,
{ mapping: event.mapping, row: dataObject }
);
console.log('Read line');
console.log(id);
try {
for (let i = 0;i < XMLRequests.length;i++) {
totalRequestsCount++;
console.log('Sending request');
const response = await sendLoSTRequest(
options,
{ data: XMLRequests[i],
url: LOST_URL }
);
console.log("got response");
const responseObj = await xml2js.
parseStringPromise(response.data);
if (Object.keys(responseObj).indexOf('errors') !== -1) {
fs.writeFileSync(`${ERR_DIR}/${generateKey()}-${id}.xml`, response.data);
failedRequestsCount++;
} else {
successRequestsCount++;
console.log('Response from the Lost Server');
console.log(response[i].data);
}
}
} catch (err) {
console.log(err);
}
}
}
})
.on('end', () => {
console.log('file processed');
console.log(`
************************************************
Total Records Processed:${totalRecordsCount}
Total Requests Sent: ${totalRequestsCount}
Success Requests: ${successRequestsCount}
Failed Requests: ${failedRequestsCount}
************************************************
`);
});
}
async function sendLoSTRequest (options, params) {
const { axios } = options;
const { url, data } = params;
if (url) {
return axios.post(url, data);
// eslint-disable-next-line no-else-return
} else {
console.log('URL is not found');
return null;
}
}
Code needs to flow like so:
read a line in a sync way
process the line and transform the line into an array of two members
for every member call API and do stuff
once line is complete, look for another line, all done in order
UPDATE: Got a workaround..but it fires stream.end() without waiting stream to finish read
async function processSOIFileLocal (options, params) {
console.log('Process SOI file');
const { ERR_DIR, fs, xml2js, LOST_URL, byline, event } = options;
const { key } = params;
const responseObject = {};
let totalRecordsCount = 0;
let totalRequestsCount = 0;
let failedRequestsCount = 0;
let successRequestsCount = 0;
let dataObject = {};
const queue = (() => {
let q = Promise.resolve();
return fn => (q = q.then(fn));
})();
const readStream = byline.createStream(fs.createReadStream(key));
readStream.setEncoding('utf8');
const pattern = /^UHL\s|^UTL\s/;
const regExp = new RegExp(pattern);
readStream.on('readable', () => {
let line;
while (null !== (line = readStream.read())) {
if (!regExp.test(line.toString())) {
totalRecordsCount++;
dataObject = soiParser(line);
const { id } = dataObject;
const XMLRequests = createLoSTRequestXML(
options,
{ mapping: event.mapping, row: dataObject }
);
// eslint-disable-next-line no-loop-func
queue(async () => {
try {
for (let i = 0;i < XMLRequests.length;i++) {
console.log('Sending request');
console.log(id);
totalRequestsCount++;
const response = await sendLoSTRequest(
options,
{ data: XMLRequests[i],
url: LOST_URL }
);
console.log('got response');
const responseObj = await xml2js.
parseStringPromise(response.data);
if (Object.keys(responseObj).indexOf('errors') !== -1) {
// console.log('Response have the error:');
// await handleError(options, { err: responseObj, id });
failedRequestsCount++;
fs.writeFileSync(`${ERR_DIR}/${generateKey()}-${id}.xml`, response.data);
} else {
console.log('Response from the Lost Server');
console.log(response[i].data);
successRequestsCount++;
}
}
} catch (err) {
console.log(err);
}
});
}
}
})
.on('end', () => {
console.log('file processed');
console.log(`
************************************************
Total Records Processed:${totalRecordsCount}
Total Requests Sent: ${totalRequestsCount}
Success Requests: ${successRequestsCount}
Failed Requests: ${failedRequestsCount}
************************************************
`);
Object.assign(responseObject, {
failedRequestsCount,
successRequestsCount,
totalRecordsCount,
totalRequestsCount
});
});
}
Thank You

The sample code at the top of your question could be rewritten like
const queue = (() => {
let q = Promise.resolve();
return (fn) => (q = q.then(fn));
})();
stream.on('data', async() => {
while (data = stream.read() !== null) {
console.log('line');
const requests = getRequests(); // sync code,no pblms
queue(async () => {
for (let i = 0; i < requests.length; i++) {
const result = await apiCall(request[i]);
console.log('result from api');
const finalResult = await anotherapiCall(result.data);
}
});
}
});
Hopefully that will be useful for the complete code

If anyone want a solution for synchronisely process the file, ie, linebyline read and execute some Async call, it's recommended to use inbuilt stream transform. There we can create a transform function and return a callback when finishes.
That's will help of any one face this issues.
Through2 is a small npm library that also can be used for the same.