Develop Reference

Node.js how to synchronously read lines from stream.Readable

I'm interacting with a child process through stdio, and I need to wait for a line from childProcess.stdout each time I write some command to childProcess.stdin.
It's easy to wrap an asynchronous method for writing like below:
async function write(data){
return new Promise(resolve=>{
childProcess.stdin.write(data,()=>resolve());
})
}
However, it turns out quite difficult when it comes to reading, since data from stdout must be processed using listeners. I've tried below:
const LineReader = require("readline")
const reader = LineReader.createInterface(childProcess.stdout);
async function read(){
return new Promise(resolve=>{
reader.once("line",line=>resolve(line));
})
}
But it always returns the first line.
I know I may achieve this using setInterval, And I've already implemented the functionality this way. But it obviously has an impact on the performance, so now I'm trying to optimize it by wrapping it into an asynchronous method.
Any suggestions and solutions will be appreciated!

Well, I ended up with something pretty similar to what you were trying. It makes some assumptions that are mentioned in the code and needs more complete error handling:
const cp = require('child_process');
const readline = require('readline');
const child = cp.spawn("node", ["./echo.js"]);
child.on('error', err => {
console.log(err);
}).on('exit', () => {
console.log("child exited");
});
const reader = readline.createInterface({ input: child.stdout });
// this will miss line events that occurred before this is called
// so this only really works if you know the output comes one line at a time
function nextLine() {
return new Promise(resolve => {
reader.once('line', resolve);
});
}
// this does not check for stdin that is full and wants us to wait
// for a drain event
function write(str) {
return new Promise(resolve => {
let ready = child.stdin.write(str, resolve);
if (!ready) {
console.log("stream isn't ready yet");
}
});
}
async function sendCmd(cmd) {
// get line reader event handler installed so there's no race condition
// on missing the return event
let p = nextLine();
// send the command
await write(cmd);
return p;
}
// send a sequence of commands and get their results
async function run() {
let result1 = await sendCmd("hi\n");
console.log(`Got '${result1}'`);
let result2 = await sendCmd("goodbye\n");
console.log(`Got '${result2}'`);
let result3 = await sendCmd("exit\n");
console.log(`Got '${result3}'`);
}
run().then(() => {
console.log("done");
}).catch(err => {
console.log(err);
});
And, for testing purposes, I ran it with this echo app:
process.stdin.on("data", data => {
let str = data.toString();
let ready = process.stdout.write("return: " + str, () => {
if (str.startsWith("exit")) {
process.exit();
}
});
if (!ready) {
console.log("echo wasn't ready");
}
});

A Promise with child_process running a Python script in Node.js - Process exits before all data

I'm trying to get sublist3r to run in a node app. It runs, however, it only shows the banner and then exits out in about 5 seconds. The script is supposed to reach out to the web and takes about 30 seconds to run. If I don't use a promise, it will work just fine. Does it have something to do with pyrog.stdout.on('data') not waiting before it outputs? I've read around and tried 'end' with no luck.
Tried everything on this article if it didn't involve editing python script (I don't think I should need to ?!)
Also read the Node.js Child Process but it doesn't mention using promises with spawn, and I believe that's what I need to use for running a Python script.
sublist3r screen shot
Am I using Promises wrong?
Any help is much appreciated
Edit: Added await to runPy, verified same issue. Have also tried making it a variable, let test = await runPy ... with no success
var express = require('express');
var router = express.Router();
let runPy = new Promise((success, nosuccess) => {
const {spawn} = require('child_process')
const pyprog = spawn('python',['/path/to/sublist3r.py', '-d', 'domain.com'] )
pyprog.stdout.on('data', (data) => {
success(data)
})
pyprog.stderr.on('data', (data) => {
nosuccess(data)
})
pyprog.on('close', (code) => {
console.log(`child process ended with ${code}`);
})
})
/* GET home page. */
router.get('/', async (req, res) => {
// EDIT: Added await and verified same issue.
await runPy.then(fromRunPy => {
console.log(fromRunPy.toString());
})
// It works fine below, but I want a promise for no other reason that I can't get it to work...
// const { spawn } = require('child_process');
// const pyProg = spawn('python', ['/home/wayne/BugHunterJS/controllers/Sublist3r/sublist3r.py', '-d', req.body.domain]);
// console.log('inside post');
// pyProg.stdout.on('data', function(data) {
// let sublist3rData = data.toString();
// console.log(sublist3rData);
// });
});
module.exports = router

Not shortly after I asked, I found a solution if anyone is looking. I'm still not really sure how it works, got to figure that out later. I think pushing it to the array is the key.
const {
spawn
} = require('child_process')
const logOutput = (name) => (data) => console.log(`[${name}] ${data}`)
function run() {
return new Promise((resolve, reject) => {
const process = spawn('python', ['/path/sublist3r.py', '-d', 'domain.com']);
const out = []
process.stdout.on(
'data',
(data) => {
out.push(data.toString());
logOutput('stdout')(data);
}
);
const err = []
process.stderr.on(
'data',
(data) => {
err.push(data.toString());
logOutput('stderr')(data);
}
);
process.on('exit', (code, signal) => {
logOutput('exit')(`${code} (${signal})`)
if (code === 0) {
resolve(out);
} else {
reject(new Error(err.join('\n')))
}
});
});
}
(async () => {
try {
const output = await run()
logOutput('main')(output)
process.exit(0)
} catch (e) {
console.error('Error during script execution ', e.stack);
process.exit(1);
}
})();

Halt Execution of Network Request If It Takes Too Long?

I have some code that basically calls fetch in Javascript. The third party services sometimes take too long to return a response and in an attempt to be more user-friendly, I want to be able to either post a message or stop the connection from being open after N milliseconds.
I had recently come across this post:
Skip the function if executing time too long. JavaScript
But did not have much luck and had issues getting it to work with the below code. I was also hoping that there was a more modern approach to do such a task, maybe using async/await?
module.exports = (url, { ...options } = {}) => {
return fetch(url, {
...options
})
}

You can use a combination of Promise.race and AbortController, here is an example:
function get(url, timeout) {
const controller = new AbortController();
return Promise.race([fetch(url, {
signal: controller.signal
}), new Promise(resolve => {
setTimeout(() => {
resolve("request was not fulfilled in time");
controller.abort();
}, timeout)
})]);
}
(async() => {
const result = await get("https://example.com", 1);
console.log(result);
})();

The native Fetch API doesn't have a timeout built in like something like axios does, but you can always create a wrapper function that wraps the fetch call to implement this.
Here is an example:
const fetchWithTimeout = (timeout, fetchConfig) => {
const FETCH_TIMEOUT = timeout || 5000;
let didTimeOut = false;
return new Promise(function(resolve, reject) {
const timeout = setTimeout(function() {
didTimeOut = true;
reject(new Error('Request timed out'));
}, FETCH_TIMEOUT);
fetch('url', fetchConfig)
.then(function(response) {
// cleanup timeout
clearTimeout(timeout);
if(!didTimeOut) {
// fetch request was good
resolve(response);
}
})
.catch(function(err) {
// Rejection already happened with setTimeout
if(didTimeOut) return;
// Reject with error
reject(err);
});
})
.then(function() {
// Request success and no timeout
})
.catch(function(err) {
//error
});
}
from here https://davidwalsh.name/fetch-timeout

Why isn't Await blocking the next line of code?

I'm using an async function to start a server, check if the url returns 200, and then run my tests, but my checkUrl function doesn't complete before running the next line of code.
I'm using the axios package to ping the url for the status code and I've been trying several variations of async/await and Promise.resolve().
function checkUrl(url) {
console.log(`Checking for ${url}`);
return axios
.get(url)
.then(function(res) {
const message = `${url} is status code: ${res.status}`;
return res;
})
.catch(function(err) {
console.log("Will check again in 5 seconds");
setTimeout(() => {
return checkUrl(url);
}, 5000);
});
}
async function init() {
let val;
console.log("Running tests:::");
// start server in react-integration directory
shell.exec("BROWSER=none yarn start", {
cwd: "sampleIntegration/react-integration",
async: true
});
// check server has started
val = await checkUrl("http://localhost:3000");
console.log(`value from checkUrl promise: ${val}`);
}
I'm expecting the val variable to be the message returned from the Promise.resolve() in my checkUrl function.
val is coming back undefined.

The problem is, the setTimeout in the catch block. This is asynchronous, but the catch block immediately returns. As it does not have anything to return, it returns undefined. To resolve this (and also keep your desired behaviour of waiting) you could do something like this:
function checkUrl(url) {
console.log(`Checking for ${url}`);
return axios.get(url)
.then(res => {
const message = `${url} is status code: ${res.status}`;
return res;
})
.catch(err => {
console.log('Will check again in 5 seconds');
return new Promise((resolve) => {
setTimeout(resolve, 5000);
}).then(() => checkUrl(url));
});
}
This will make a Promise which resolves after 5 seconds and on resolve it calls checkUrl.

how to throw and catch error if function runs longer than n seconds

I have a function of the form:
async someFunction () {
try {
someStuff()
} catch (err) {
doSomeCleanup()
}
}
Which gets called multiple times with workers from a queue. Now I have a new requirement that a worker should fail if it takes longer than 30 seconds to run this function with a given worker. My gut instinct was to whip up something like:
const timeoutCheck;
const time = new Date();
async someFunction () {
try {
timeoutCheck = setInterval(() => {
if (new Date() - time > 30000) throw new Error("timeout");
}, 2000);
someStuff()
} catch (err) {
doSomeCleanup()
} finally {
clearInterval(timeoutCheck);
}
}
But as I quickly learned the setInterval gets run outside the try/catch block so the error isn't caught inside the catch block but in the main app and the whole app crashes rather than just a worker failing. Is there a good way to contain this error in the try/catch block or another way to accomplish erroring into the try/catch block if the function runs more than 30 seconds?

You can use Promise.race()
let stop = () => new Promise((_, reject) =>
setTimeout(reject, 2999, "rejected"));
let fn = () => new Promise((resolve, _) =>
setTimeout(resolve, 3000, "resolved"));
let someStuff = data => console.log(data);
let doSomeCleanup = err => console.error(err);
Promise.race([fn(), stop()])
.then(doSomeStuff, doSomeCleanup);