Develop Reference

How to stream x-ndjson content using Express and parse the streamed data?

I have a TS library using Node v19.1.0. The library has a function that observes streamed server events.
The server provides a /events route streaming 'application/x-ndjson' content which might be an event/ping/... ( sending a ping every x seconds is important to keep the connection alive )
My observe function parses the streamed data and inspects it. If it is a valid event it will pass it to a callback function. The caller also receives an abort function to abort the streaming on demand.
Whenever I run tests locally or via CI I get the following error
Warning: Test "observes events." generated asynchronous activity after the test ended. This activity created the error "AbortError: The operation was aborted." and would have caused the test to fail, but instead triggered an unhandledRejection event.
I tried to minimize the example code using plain JavaScript
const assert = require('assert/strict');
const express = require('express');
const { it } = require('node:test');
it('observes events.', async () => {
const expectedEvent = { type: 'event', payload: { metadata: { type: 'entity-created', commandId: 'commandId' } } };
const api = express();
const server = api
.use(express.json())
.post('/events', (request, response) => {
response.writeHead(200, {
'content-type': 'application/x-ndjson',
});
const line = JSON.stringify(expectedEvent) + '\n';
response.write(line);
})
.listen(3000);
let stopObserving = () => {
throw new Error('should never happen');
};
const actualEventPayload = await new Promise(async resolve => {
stopObserving = await observeEvents(async newEvent => {
resolve(newEvent);
});
});
stopObserving();
server.closeAllConnections();
server.close();
assert.deepEqual(actualEventPayload, expectedEvent.payload);
});
const observeEvents = async function (onReceivedFn) {
const abortController = new AbortController();
const response = await fetch('http://localhost:3000/events', {
method: 'POST',
headers: { 'content-type': 'application/json' },
signal: abortController.signal,
});
if (!response.ok) {
throw new Error('error handling goes here - request failed');
}
Promise.resolve().then(async () => {
if (!response.body) {
throw new Error('error handling goes here - missing response body');
}
for await (const item of parseStream(response.body, abortController)) {
switch (item.type) {
case 'event': {
await onReceivedFn(item.payload);
break;
}
case 'ping':
// Intentionally left blank
break;
case 'error':
throw new Error('error handling goes here - stream failed');
default:
throw new Error('error handling goes here - should never happen');
}
}
});
return () => { abortController.abort(); };
};
const parseLine = function () {
return new TransformStream({
transform(chunk, controller) {
try {
const data = JSON.parse(chunk);
// ... check if this is a valid line...
controller.enqueue(data);
} catch (error) {
controller.error(error);
}
},
});
};
const splitLines = function () {
let buffer = '';
return new TransformStream({
transform(chunk, controller) {
buffer += chunk;
const lines = buffer.split('\n');
for (let i = 0; i < lines.length - 1; i++) {
controller.enqueue(lines[i]);
}
buffer = lines.at(-1) ?? '';
},
flush(controller) {
if (buffer.length > 0) {
controller.enqueue(buffer);
}
},
});
};
const parseStream = async function* (stream, abortController) {
let streamReader;
try {
const pipedStream = stream
.pipeThrough(new TextDecoderStream())
.pipeThrough(splitLines())
.pipeThrough(parseLine());
streamReader = pipedStream.getReader();
while (true) {
const item = await streamReader.read();
if (item.done) {
break;
}
yield item.value;
}
} finally {
await streamReader?.cancel();
abortController.abort();
}
};
Unfortunately, when running node --test, the test does not finish. I have to cancel it manually.
The test breaks with these lines
const actualEventPayload = await new Promise(async resolve => {
stopObserving = await observeEvents(async newEvent => {
resolve(newEvent);
});
});
and I think that's because the Promise never resolves. I thought the stream parsing might have a bug but if you remove all the stream parsing stuff and replace
Promise.resolve().then(async () => {
/* ... */
});
with
Promise.resolve().then(async () => {
await onReceivedFn({ metadata: { type: 'entity-created', commandId: 'commandId' }});
});
it doesn't work neither. Does someone know what's wrong or missing?

The problem here has nothing to do with your promise not resolving since you never even get to that point.
The problem here is that observeEvents is not yet initialized when the test is being run and thus throws a ReferenceError: Cannot access 'observeEvents' before initialization error.
To see that for yourself you can add a simple const it = (name, fn) => fn(); stub to the top of the file and run it without the --test.
There are multiple ways to fix this and the simplest one is to move the test function to the bottom of the file.
If you don't want to do that you can also define the observeEvents function like this: async function observeEvents(onReceivedFn) {...}. This way it will be available immediately.

Poll API until a path in the response object is Successful | Failure - Typescript

I have a function which accepts a string and a path value and checks whether the path at the result returns a 1 or a -1. I fire the function with multiple requests and everything seems to be successful except for one. For example, if I call the function with 10 different URL's continously (one by one, not in an array), the promise is resolved for 9 but not for the 10th one.
This is my code:
enum Status {
Queued = 0,
Started = 1,
Finished = 2,
Failed = -1,
}
let dataFetchingTimer: number;
export const getDataAtIntervals = (url: string, path: string): Promise<any> => {
clearTimeout(dataFetchingTimer);
return new Promise<any>((resolve: Function, reject: Function): void => {
try {
(async function loop() {
const result = await API.load(url);
console.log(`${url} - ${JSON.stringify(result)}`)
if (
get(result, path) &&
(get(result, path) === Status.Finished ||
get(result, path) === Status.Failed)
) {
return resolve(result); // Resolve with the data
}
dataFetchingTimer = window.setTimeout(loop, 2500);
})();
} catch (e) {
reject(e);
}
});
};
export const clearGetDataAtIntervals = () => clearTimeout(dataFetchingTimer);
Please advice. In the above image, 4535 is called only once. and is not called until 2 or -1 is returned.

Using a single timeout for all your calls might be the cause of weird behaviors. A solution to avoid collisions between your calls might be to use a timeout per call. You could do something along these lines (I used simple JS because I'm not used to Typescript):
const Status = {
Queued: 0,
Started: 1,
Finished: 2,
Failed: -1,
}
let dataFetchingTimerMap = {
// Will contain data like this:
// "uploads/4541_status": 36,
};
const setDataFetchingTimer = (key, cb, delay) => {
// Save the timeout with a key
dataFetchingTimerMap[key] = window.setTimeout(() => {
clearDataFetchingTimer(key); // Delete key when it executes
cb(); // Execute the callback
}, delay);
}
const clearDataFetchingTimer = (key) => {
// Clear the timeout
clearTimeout(dataFetchingTimerMap[key]);
// Delete the key
delete dataFetchingTimerMap[key];
}
const getDataAtIntervals = (url, path) => {
// Create a key for the timeout
const timerKey = `${url}_${path}`;
// Clear it making sure you're only clearing this one (by key)
clearDataFetchingTimer(timerKey);
return new Promise((resolve, reject) => {
// A try/catch is useless here (https://jsfiddle.net/4wpezauc/)
(async function loop() {
// It should be here (https://jsfiddle.net/4wpezauc/2/)
try {
const result = await API.load(url);
console.log(`${url} - ${JSON.stringify(result)}`);
if ([Status.Finished, Status.Failed].includes(get(result, path))) {
return resolve(result); // Resolve with the data
}
// Create your timeout and save it with its key
setDataFetchingTimer(timerKey, loop, 2500);
} catch (e) {
reject(e);
}
})();
});
};
const clearGetDataAtIntervals = () => {
// Clear every timeout
Object.keys(dataFetchingTimerMap).forEach(clearDataFetchingTimer);
};

Axios requests with express, node, ejs

I am working on a site using Express.js, node.js, Axios, and ejs. I am making REST calls to a Oracle SQL REST services using Axios. I am having trouble working with Promises or Async/Await. I could use some guidance, if possible.
I have a repository layer to interface with the Oracle DB. For example:
dataaccess.js
const axios = require('axios');
exports.IsManufacturerCategory = function (categoryId) {
axios.get(`DB ADDRESS ${categoryId}`)
.then(response => {
console.error('GET IsManufacturerCategory categoryId = ' + categoryId);
console.error('Response = ' + JSON.stringify(response.data));
return (response.data);
})
.catch(rej => {
console.error('ERROR IsManufacturerCategory categoryId = ' + categoryId);
console.error('ERR = \n' + rej.data);
return (rej.data);
});
}
Which is called in my middleware. When I call var isManufacturerCat = exports.IsManufacturerCategory(categoryId); it is undefined. I am attempting to use the data retrieved from the Axios call to return a ejs view to my router, which I can provide if needed.
category.js
var isManufacturerCat = exports.IsManufacturerCategory(categoryId);
if (isManufacturerCat) {
var models = dataaccess.GetCategorySubCategories(categoryId);
return ("manufacturers", {
data: {
Canonical: cononical,
Category: category,
IsAManufacturerCategory: iAManufacturerCat,
Models: models
}
});
}
I am open to any advice in my project structure, usage of Promises, Async/Await, etc.
Thank you in advance.
EDIT
After working with some of the answers given, I have made some progress but I am having issues with layers of async calls. I end up getting into a spot where I need to await a call, but I am in a function that I am not able/do not want to do so (i.e. my router).
indexMiddleware.js
exports.getRedirectURL = async function (fullOrigionalpath) {
if (fullOrigionalpath.split('.').length == 1 || fullOrigionalpath.indexOf(".aspx") != -1) {
if (fullOrigionalpath.indexOf(".aspx") != -1) {
//some string stuff to get url
}
else if (fullOrigionalpath.indexOf("/solutions/") != -1) {
if (!fullOrigionalpath.match("/solutions/$")) {
if (fullOrigionalpath.indexOf("/t-") != -1) {
//some stuff
}
else {
var solPart = fullOrigionalpath.split("/solutions/");
solPart = solPart.filter(function (e) { return e });
if (solPart.length > 0) {
var solParts = solPart[solPart.length - 1].split("/");
solParts = solParts.filter(function (e) { return e });
if (solParts.length == 1) {
waitForRespose = true;
const isASolutionCategory = await dataaccess.isASolutionCategory(solParts[0]); // returns void
if (isASolutionCategory != undefined && isASolutionCategory.length > 0 && isASolutionCategory[0].Count == 1) {
// set redirecturl
}
}
else {
redirecturl = "/solutions/solutiontemplate";
}
}
}
}
}
else if (URL STUFF) {
// finally if none of the above fit into url condition then verify current url with Category URL or product url into database and if that matches then redirect to proper internal URL
if (fullOrigionalpath.lastIndexOf('/') == (fullOrigionalpath.length - 1)) {
fullOrigionalpath = fullOrigionalpath.substring(0, fullOrigionalpath.lastIndexOf('/'));
}
waitForRespose = true;
const originalURL = await exports.getOriginalUrl(fullOrigionalpath); //returns string
redirecturl = originalURL;
return redirecturl;
}
}
if (!waitForRespose) {
return redirecturl;
}
}
exports.getOriginalUrl = async function (friendlyUrl) {
var originalUrl = '';
var urlParts = friendlyUrl.split('/');
urlParts = urlParts.filter(function (e) { return e });
if (urlParts.length > 0) {
var skuID = urlParts[urlParts.length - 1];
const parts = await dataaccess.getFriendlyUrlParts(skuID); //returns void
console.log("Inside GetOriginalUrl (index.js middleware) FriendlyUrlParts: " + parts);//undefined
if (parts != undefined && parts != null && parts.length > 0) {
//some stuff
}
else {
// verify whether it's category URL then return the category local URL
console.log('Getting CategoryLocalUrl');
const categoryLocalUrl = await dataaccess.getCategoryLocalUrl(friendlyUrl); // returns void
console.log('CategoryLocalUrl Gotten ' + JSON.stringify(categoryLocalUrl)); //undefined
if (categoryLocalUrl != undefined && categoryLocalUrl.length > 0) {
//set originalUrl
return originalUrl;
}
}
}
else { return ''; }
}
index.js router
router.use(function (req, res, next) {
//bunch of stuff
index.getRedirectURL(url)
.then(res => {
req.url = res;
})
.catch(error => {
console.error(error);
})
.finally(final => {
next();
});
}
I am getting undefined in my console.logs after the awaits. I'm not really sure what I'm doing I guess.

Let's start with dataaccess.js. Basically, you're exporting a function that's doing async work, but the function isn't async. Most people want to be able to utilize async/await, so rather than have IsManufacturerCategory accept a callback function, it would better to have the function return a promise. The easiest way to do that is to make the function an async function. Async functions return promises which are resolved/rejected more easily than by returning an explicit promise. Here's how that could be rewritten:
const axios = require('axios');
exports.IsManufacturerCategory = async function (categoryId) {
try {
const response = await axios.get(`DB ADDRESS ${categoryId}`);
console.log('GET IsManufacturerCategory categoryId = ' + categoryId);
console.log('Response = ' + JSON.stringify(response.data));
} catch (err) {
console.error('ERROR IsManufacturerCategory categoryId = ' + categoryId);
console.error('ERR = \n' + rej.data);
throw err;
}
}
Note that I'm using console.error only for errors. Because you wanted to log some error related details, I used a try/catch statement. If you didn't care about doing that, the function could be simplified to this:
const axios = require('axios');
exports.IsManufacturerCategory = async function (categoryId) {
const response = await axios.get(`DB ADDRESS ${categoryId}`);
console.log('GET IsManufacturerCategory categoryId = ' + categoryId);
console.log('Response = ' + JSON.stringify(response.data));
}
This is because async functions automatically swallow errors and treat them as rejections - so the promise returned by IsManufacturerCategory would be rejected. Similarly, when an async function returns a value, the promise is resolved with the value returned.
Moving on to category.js... This looks strange because you're accessing IsManufacturerCategory from the exports of that module when I think you mean to access it from the import of the dataaccess module, right?
Inside this function, you should put any async work in an async function so that you can use await with function that return promises. Here's how it could be rewritten:
const dataaccess = require('dataccess.js');
async function validateManufacturerCat(categoryId) {
const isManufacturerCat = await dataaccess.IsManufacturerCategory(categoryId);
if (isManufacturerCat) {
const models = await dataaccess.GetCategorySubCategories(categoryId);
return ({
manufacturers: {
data: {
Canonical: cononical,
Category: category,
IsAManufacturerCategory: iAManufacturerCat,
Models: models
}
}
});
}
}
validateManufacturerCat(categoryId)
.then(res => {
console.log(res);
})
.catch(err => {
console.error(err);
});
A couple notes:
I changed the return value in the if statement to be a single value. You should try to always return a single value when working with promises and async/await (since you can only resolve/return one value).
I see lots of functions starting with capital letters. There's a convention in JavaScript where functions with capital letters are constructor functions (meant to be invoked with the new keyword).

Here is a working example. Note that you need a callback function to get the data when it is available from the promise
//client.js
const axios = require("axios")
exports.IsManufacturerCategory = function (url, callback) {
axios.get(url)
.then(response=>
{
callback(response.data);
})
.catch(error=>{
callback( error);
})
};
//app.js
const client = require('./client');
function process(data) {
console.log(data)
}
client.IsManufacturerCategory("http://localhost:3000/persons",process);
documentation

Request result from async function

I need the value of the variable 'result' to make a comparison. Im having problem to get this 'result' value.
Does anyone know how to fix it?
**Im using node to make an REST application, and I need this to result in my GET method
async function get(req, res, next) {
var result = 0;
try {
const context = {};
const login = {};
login.user = req.params.user, 100;
login.pass = req.params.pass, 200;
var optionsgetmsg = {
host: 'URL', // here only the domain name
// (no http/https !)
path: '/controller/verifica.asp?a=' + login.user + '&b=' + login.pass, // the rest of the url with parameters if needed
method: 'GET' // do GET
};
var reqGet = https.request(optionsgetmsg, function (res) {
res.on('data', function (d) {
jsonResponse = JSON.parse(d);
result = jsonResponse.message;
});
});
reqGet.end();
reqGet.on('error', function (e) {
console.error(e);
});
context.id = parseInt(req.params.id, 10);
console.log(result);
//problem
if(result == "yes"){
const rows = await operadores.find(context);
if (req.params.id) {
if (rows.length === 1) {
res.status(200).json(rows[0]);
} else {
res.status(404).end();
}
} else {
res.status(200).json(rows);
}
}
}

You can use a library that does HTTP requests and supports promises, something like axios or refactor your current https.request and convert it to a promise, here is an example:
function makeRequest({ user, password }) {
return new Promise((resolve, reject) => {
const options = {
host: 'URL',
path: `/controller/verifica.asp?a=${encodeURIComponent(user)}&b=${encodeURIComponent(pass)}`
method: 'GET'
};
const reqGet = https.request(options, function (res) {
res.on('data', function (d) {
result = JSON.parse(d);
resolve(result.message);
});
});
reqGet.on('error', function (e) {
reject(e);
});
reqGet.end();
});
}
Then you can simply do this:
const result = await makeRequest(req.params);
I've used some concepts that you may not be familiar with, template literal and object
destructuring

Updating many(100k+) documents in the most efficient way MongoDB

I have a function that runs periodically, that updates the item.price of some Documents in my Prices Collection. The Price Collection has 100k+ items. The function looks like this:
//Just a helper function for multiple GET requests with request.
let _request = (urls, cb) => {
let results = {}, i = urls.length, c = 0;
handler = (err, response, body) => {
let url = response.request.uri.href;
results[url] = { err, response, body };
if (++c === urls.length) {
cb(results);
}
};
while (i--) {
request(urls[i], handler);
}
};
// function to update the prices in our Prices collection.
const update = (cb) => {
Price.remove({}, (err, remove) => {
if (err) {
return logger.error(`Error removing items...`);
}
logger.info(`Removed all items... Beginning to update.`);
_request(urls, (responses) => {
let url, response, gameid;
for (url in responses) {
id = url.split('/')[5].split('?')[0];
response = responses[url];
if (response.err) {
logger.error(`Error in request to ${url}: ${err}`);
return;
}
if (response.body) {
logger.info(`Request to ${url} successful.`)
let jsonResult = {};
try {
jsonResult = JSON.parse(response.body);
} catch (e) {
logger.error(`Could not parse.`);
}
logger.info(`Response body for ${id} is ${Object.keys(jsonResult).length}.`);
let allItemsArray = Object.keys(jsonResult).map((key, index) => {
return {
itemid: id,
hash_name: key,
price: jsonResult[key]
}
});
Price.insertMany(allItemsArray).then(docs => {
logger.info(`Saved docs for ${id}`)
}, (e) => {
logger.error(`Error saving docs.`);
});
}
}
if (cb && typeof cb == 'function') {
cb();
}
})
});
}
As you can see, to avoid iterating through 100k+ Documents, and updating each and every one of them separately, I delete them all at the beginning, and just call the API that gives me these Items with prices, and use InsertMany to Insert all of them into my Prices Collection.
This updating process will happen every 30 minutes.
But I just now realised, what if some user wants to check the Prices and my Prices Collection is currently empty because it's in the middle of updating itself?
The Question
So do I have to iterate through all of them in order to not delete it? (Remember, there are MANY documents to be updated every 30 mins.) Or is there another solution?
Here's a picture of how my Prices Collection looks (there are 100k docs like these, I just want to update the price property):
Update:
I have re-written my update function a bit and now it looks like this:
const update = (cb = null) => {
Price.remove({}, (err, remove) => {
if (err) {
return logger.error(`Error removing items...`);
}
logger.info(`Removed all items... Beginning to update.`);
_request(urls, (responses) => {
let url, response, gameid;
for (url in responses) {
gameid = url.split('/')[5].split('?')[0];
response = responses[url];
if (response.err) {
logger.error(`Error in request to ${url}: ${err}`);
return;
}
if (response.body) {
logger.info(`Request to ${url} successful.`)
let jsonResult = {};
try {
jsonResult = JSON.parse(response.body);
} catch (e) {
logger.error(`Could not parse.`);
}
logger.info(`Response body for ${gameid} is ${Object.keys(jsonResult).length}.`);
let allItemsArray = Object.keys(jsonResult).map((key, index) => {
return {
game_id: gameid,
market_hash_name: key,
price: jsonResult[key]
}
});
let bulk = Price.collection.initializeUnorderedBulkOp();
allItemsArray.forEach(item => {
bulk.find({market_hash_name: item.market_hash_name})
.upsert().updateOne(item);
});
bulk.execute((err, bulkers) => {
if (err) {
return logger.error(`Error bulking: ${e}`);
}
logger.info(`Updated Items for ${gameid}`)
});
// Price.insertMany(allItemsArray).then(docs => {
// logger.info(`Saved docs for ${gameid}`)
// }, (e) => {
// logger.error(`Error saving docs.`);
// });
}
}
if (cb && typeof cb == 'function') {
cb();
}
})
});
}
Notice the bulk variable now (Thanks #Rahul) but now, the collection takes ages to update. My processor is burning up and it literally takes 3+ minutes to update 60k+ documents. I honestly feel like the previous method, while it might delete all of them and then reinserting them, it also takes 10x faster.
Anyone?

From my experience (updating millions of mongo docs on a hourly basis), here's a realistic approach to very large bulk updates:
do all your API calls separately and write results in as bson into a file
invoke mongoimport and import that bson file into a new empty collection prices_new. Javascript, let alone high-level OO wrappers, are just too slow for that
rename prices_new -> prices dropTarget=true (this will be atomic hence no downtime)
Schematically, it would look like this in JS
let fname = '/tmp/data.bson';
let apiUrls = [...];
async function doRequest(url) {
// perform a request and return an array of records
}
let responses = await Promise.all(apiUrls.map(doRequest));
// if the data too big to fit in memory, use streams instead of this:
let data = flatMap(responses, BSON.serialize).join('\n'));
await fs.writeFile(fname, data);
await child_process.exec(`mongoimport --collection prices_new --drop ${fname}`);
await db.prices_new.renameCollection('prices', true);

There's no need to clear the database and do a fresh insert. You can use the bulkWrite() method for this or use the updateMany() method to do the updates.
You can refactor the existing code to
const update = (cb) => {
_request(urls, responses => {
let bulkUpdateOps = [], gameid;
responses.forEach(url => {
let response = responses[url];
gameid = url.split('/')[5].split('?')[0];
if (response.err) {
logger.error(`Error in request to ${url}: ${response.err}`);
return;
}
if (response.body) {
logger.info(`Request to ${url} successful.`)
let jsonResult = {};
try {
jsonResult = JSON.parse(response.body);
} catch (e) {
logger.error(`Could not parse.`);
}
Object.keys(jsonResult).forEach(key => {
bulkUpdateOps.push({
"updateOne": {
"filter": { market_hash_name: key },
"update": { "$set": {
game_id: gameid,
price: jsonResult[key]
} },
"upsert": true
}
});
});
}
if (bulkUpdateOps.length === 1000) {
Price.bulkWrite(bulkUpdateOps).then(result => {
logger.info(`Updated Items`)
}).catch(e => logger.error(`Error bulking: ${e}`));
bulkUpdateOps = [];
}
});
if (bulkUpdateOps.length > 0) {
Price.bulkWrite(bulkUpdateOps).then(result => {
logger.info(`Updated Items`)
}).catch(e => logger.error(`Error bulking: ${e}`));
}
});
if (cb && typeof cb == 'function') {
cb();
}
}

I have not tested anything but you can try this, might be helpful. I am using bluebird library for concurrency.
let _request = (url) => {
return new Promise((resolve, reject) => {
request(url, (err, response, body) => {
if (err) {
reject(err);
}
resolve(body);
});
});
};
const formatRespose = async (response) => {
// do stuff
return {
query: {}, // itemid: id,
body: {}
};
}
const bulkUpsert = (allItemsArray) => {
let bulk = Price.collection.initializeUnorderedBulkOp();
return new Promise((resolve, reject) => {
allItemsArray.forEach(item => {
bulk.find(item.query).upsert().updateOne(item.body);
});
bulk.execute((err, bulkers) => {
if (err) {
return reject(err);
}
return resolve(bulkers);
});
});
}
const getAndUpdateData = async (urls) => {
const allItemsArray = urls.map((url) => {
const requestData = await _request(url); // you can make this also parallel
const formattedData = formatRespose(requestData); // return {query: {},body: {} };
return formattedData;
});
return await (bulkUpsert(allItemsArray));
};
function update() {
// split urls into as per your need 100/1000
var i, j, chunkUrls = [],
chunk = 100;
for (i = 0, j = urls.length; i < j; i += chunk) {
chunkUrls.push(getAndUpdateData(urls.slice(i, i + chunk)));
}
Bluebird.map(chunkUrls, function (chunk) {
return await chunk;
}, {
concurrency: 1 // depends on concurrent request change 1 = 100 request get and insert in db at time
}).then(function () {
console.log("done");
}).catch(function () {
console.log("error");
});
}