I am new to node js. I have used the python call where data comes from. Since the process is a little complex so it taking time to execute and node js to receive a response. The code I mentioned here tried to wait for the function to complete, but the function is repeating the call from the start and goes repeating until the python function finished and to get a response.
So after completion of function
console.log("Image Conversation::")
await bot.say(dumy_msg);
await bot.beginDialog('classfication_done_');
These statements are repeating for two times.
How to resolve the issue?
convo1.ask('Post Your message', async(answer, convo1, bot,message) => {
if(message.type=='file_share') {
console.log("Image Conversation::")
const botToken = bot.api.token;
const destination_path = '/_classification/upload/' + message.files[0].name;
console.log("URl :::",message.files[0].url_private)
console.log('Testing file share' + message.files[0].name);
const url = message.files[0].url_private;
const opts = {
method: 'GET',
url: url,
headers: {
Authorization: 'Bearer ' + botToken,
}
};
request(opts, function (err, res, body) {
console.log('FILE RETRIEVE STATUS', res.statusCode);
}).pipe(fs.createWriteStream(destination_path));
const img = '/_classification/upload/' + message.files[0].name;
spawn = require("child_process").spawn;
const getResponse = () => {
return new Promise((resolve, reject) => {
const process = spawn('python',["/_classification/tensorflow-image-detection-master/classify.py",img] );
process.stdout.on('data', data => {
resolve(data);
});
});
}
const response = await getResponse();
var dumy_msg = `${response}`;
await bot.say(dumy_msg);
await bot.beginDialog('classfication_done_');
}
else{
await bot.say(message,"Please provide jpeg file format, other files are not allowed");
await bot.beginDialog('classfication_done_');
}
});
I think the callback function(async () => {}) itself has been called twice.
I don't know exactly what the role convo1.ask do, but make sure it's not in a sitution call a callback twice.
Related
I have a TS library using Node v19.1.0. The library has a function that observes streamed server events.
The server provides a /events route streaming 'application/x-ndjson' content which might be an event/ping/... ( sending a ping every x seconds is important to keep the connection alive )
My observe function parses the streamed data and inspects it. If it is a valid event it will pass it to a callback function. The caller also receives an abort function to abort the streaming on demand.
Whenever I run tests locally or via CI I get the following error
Warning: Test "observes events." generated asynchronous activity after the test ended. This activity created the error "AbortError: The operation was aborted." and would have caused the test to fail, but instead triggered an unhandledRejection event.
I tried to minimize the example code using plain JavaScript
const assert = require('assert/strict');
const express = require('express');
const { it } = require('node:test');
it('observes events.', async () => {
const expectedEvent = { type: 'event', payload: { metadata: { type: 'entity-created', commandId: 'commandId' } } };
const api = express();
const server = api
.use(express.json())
.post('/events', (request, response) => {
response.writeHead(200, {
'content-type': 'application/x-ndjson',
});
const line = JSON.stringify(expectedEvent) + '\n';
response.write(line);
})
.listen(3000);
let stopObserving = () => {
throw new Error('should never happen');
};
const actualEventPayload = await new Promise(async resolve => {
stopObserving = await observeEvents(async newEvent => {
resolve(newEvent);
});
});
stopObserving();
server.closeAllConnections();
server.close();
assert.deepEqual(actualEventPayload, expectedEvent.payload);
});
const observeEvents = async function (onReceivedFn) {
const abortController = new AbortController();
const response = await fetch('http://localhost:3000/events', {
method: 'POST',
headers: { 'content-type': 'application/json' },
signal: abortController.signal,
});
if (!response.ok) {
throw new Error('error handling goes here - request failed');
}
Promise.resolve().then(async () => {
if (!response.body) {
throw new Error('error handling goes here - missing response body');
}
for await (const item of parseStream(response.body, abortController)) {
switch (item.type) {
case 'event': {
await onReceivedFn(item.payload);
break;
}
case 'ping':
// Intentionally left blank
break;
case 'error':
throw new Error('error handling goes here - stream failed');
default:
throw new Error('error handling goes here - should never happen');
}
}
});
return () => { abortController.abort(); };
};
const parseLine = function () {
return new TransformStream({
transform(chunk, controller) {
try {
const data = JSON.parse(chunk);
// ... check if this is a valid line...
controller.enqueue(data);
} catch (error) {
controller.error(error);
}
},
});
};
const splitLines = function () {
let buffer = '';
return new TransformStream({
transform(chunk, controller) {
buffer += chunk;
const lines = buffer.split('\n');
for (let i = 0; i < lines.length - 1; i++) {
controller.enqueue(lines[i]);
}
buffer = lines.at(-1) ?? '';
},
flush(controller) {
if (buffer.length > 0) {
controller.enqueue(buffer);
}
},
});
};
const parseStream = async function* (stream, abortController) {
let streamReader;
try {
const pipedStream = stream
.pipeThrough(new TextDecoderStream())
.pipeThrough(splitLines())
.pipeThrough(parseLine());
streamReader = pipedStream.getReader();
while (true) {
const item = await streamReader.read();
if (item.done) {
break;
}
yield item.value;
}
} finally {
await streamReader?.cancel();
abortController.abort();
}
};
Unfortunately, when running node --test, the test does not finish. I have to cancel it manually.
The test breaks with these lines
const actualEventPayload = await new Promise(async resolve => {
stopObserving = await observeEvents(async newEvent => {
resolve(newEvent);
});
});
and I think that's because the Promise never resolves. I thought the stream parsing might have a bug but if you remove all the stream parsing stuff and replace
Promise.resolve().then(async () => {
/* ... */
});
with
Promise.resolve().then(async () => {
await onReceivedFn({ metadata: { type: 'entity-created', commandId: 'commandId' }});
});
it doesn't work neither. Does someone know what's wrong or missing?
The problem here has nothing to do with your promise not resolving since you never even get to that point.
The problem here is that observeEvents is not yet initialized when the test is being run and thus throws a ReferenceError: Cannot access 'observeEvents' before initialization error.
To see that for yourself you can add a simple const it = (name, fn) => fn(); stub to the top of the file and run it without the --test.
There are multiple ways to fix this and the simplest one is to move the test function to the bottom of the file.
If you don't want to do that you can also define the observeEvents function like this: async function observeEvents(onReceivedFn) {...}. This way it will be available immediately.
I hope everyone is well.
I have the following code that brings the value of a json:
const https = require('https');
var http = require("https");
var AWS = require('aws-sdk');
exports.handler = async (event, context, callback) => {
console.log('Event: ', JSON.stringify(event, null, 2));
const request = event.Records[0].cf.request;
const https = require('https');
const http_get = require('https');
var options = {
host: 'www.local.dev',
method: 'GET',
timeout: 2000,
path: '/list.json',
headers: {
'Accept': 'application/json'
}
};
const res = await new Promise(resolve => {
http_get.get(options, resolve);
});
let data = await new Promise((resolve, reject) => {
let data = '';
res.on('data', chunk => data += chunk);
res.on('error', err => reject(err));
res.on('end', () => resolve(data));
});
data = JSON.parse(data)
// CONTINUE FUNCTION
return request;
};
I would like to set a timeout, If the request does not return in 2 seconds, I continue the function, The function has other parts, so even if nothing comes from the json endpoint it can continue.
It has no dependency on the json return.
What is happening and that the timeout is not validated and the function is running indefinitely.
I need to use this http module from node.js, I can't use axios here.
To simulate the timeout I am using this app: httpstat.us/501?sleep=60000
Change this code
const res = await new Promise(resolve => {
http_get.get(options, resolve);
});
to this:
const res = http_get.get(options);
https.get returns a stream that is ready to use, and there is no reason to wrap that into a promise (where the syntax you are showing would be incorrect anyway).
I am seeing the following error when trying to extract an image's metadata information with the Sharp module: "Input file contains unsupported image format".
This is only happening for certain signed image urls, particularly ones that contain xmp information in the metadata.
I am hoping someone can help me spot where the issue might be occurring in this code snippet.
Here is the exact code snippet I am using (insert the signed image URL where specified in the doStuff function to test):
const sharp = require("sharp");
const fs = require('fs');
const fetch = require('node-fetch');
async function storeUrlToLocal(sourceUrl) {
const destPath = './';
const request = {
method: 'GET',
encoding: null,
};
response = await fetch(sourceUrl, request);
if (response.status >= 400)
throw new Error(`Failed to fetch data from ${sourceUrl}, status returned = ${response.status}`);
const localPath = `${destPath}test.png`;
const fileStream = fs.createWriteStream(localPath);
return new Promise((resolve, reject) => {
response.body.pipe(fileStream);
response.body.on("error", reject);
response.body.on("end", async () => {
const fileExists = fs.existsSync(localPath);
console.log(`All the data in the file has been read ${localPath} = ${fileExists}`);
resolve(localPath);
});
response.body.on("finish",() => {
console.log('All writes are now complete.');
});
}).catch(error => {
console.log(error);
});
}
async function doStuff() {
const localFilePath = await storeUrlToLocal('<INSERT_IMAGE_URL_HERE>');
// Read image file and extract metadata
let manipulator;
let imageMetadata;
try {
manipulator = sharp(localFilePath, { limitInputPixels: 5000000000 });
console.log('Manipulator = ', manipulator);
imageMetadata = await manipulator.metadata();
console.log("ImageMetadata = ", imageMetadata);
} catch (error) {
console.log(`Image Metadata Extraction Error: ${error.message}`);
throw error;
}
}
doStuff();
This code snippet above fails with the "Input file contains unsupported image format" on the line that extracts metadata (imageMetadata = await manipulator.metadata();)
So the strange thing is, I am able to properly extract the metadata (with no errors) with this same code if I add a short Sleep after this line: const fileStream = fs.createWriteStream(localPath);
So this code snippet (all I'm doing here is adding a short sleep after fs.createWriteSteam) allows the image metadata to be extracted without issue:
const sharp = require("sharp");
const fs = require('fs');
const fetch = require('node-fetch');
async function storeUrlToLocal(sourceUrl) {
const destPath = './';
const request = {
method: 'GET',
encoding: null,
};
response = await fetch(sourceUrl, request);
if (response.status >= 400)
throw new Error(`Failed to fetch data from ${sourceUrl}, status returned = ${response.status}`);
const localPath = `${destPath}test.png`;
const fileStream = fs.createWriteStream(localPath);
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
await sleep(1000);
return new Promise((resolve, reject) => {
response.body.pipe(fileStream);
response.body.on("error", reject);
response.body.on("end", async () => {
const fileExists = fs.existsSync(localPath);
console.log(`All the data in the file has been read ${localPath} = ${fileExists}`);
resolve(localPath);
});
response.body.on("finish",() => {
console.log('All writes are now complete.');
});
}).catch(error => {
console.log(error);
});
}
async function doStuff() {
const localFilePath = await storeUrlToLocal('<INSERT_IMAGE_URL_HERE>');
// Read image file and extract metadata
let manipulator;
let imageMetadata;
try {
manipulator = sharp(localFilePath, { limitInputPixels: 5000000000 });
console.log('Manipulator = ', manipulator);
imageMetadata = await manipulator.metadata();
console.log("ImageMetadata = ", imageMetadata);
} catch (error) {
console.log(`Image Metadata Extraction Error: ${error.message}`);
throw error;
}
}
doStuff();
Why would this Sleep resolve my issues? I don't see any asynchronous calls being run that I would need to be waiting for to complete. Perhaps fs.createWriteStream didn't have enough time to complete its operation? But I do not have the option to await the call to fs.createWriteStream, as it is not async.
I have a 2 identical Firebase functions that batch write data to Firestore. One is wrapped in a scheduled/onRun trigger, and the other is a HTTP onRequest trigger.
Both functions work fine and throw no errors.
They have the same amount of memory and timeout as well.
When invoking the http trigger, the function runs through and completes in about 30 seconds.
When invoking the scheduled onRun trigger, the function takes 5+ minutes to complete.
Is there something different about the runtimes that is not documented or something?
Edit: It works now - I made processMentions await totalMentions and return null.
processMentions does not have to return a promise, only a value because the actual scheduledPull/onRun function is returning the processMentions async function, which resolves the promise by returning a value.
Cheers for the help #dougstevenson
Triggers:
/**
* Get manual mentions
*/
exports.get = functions.https.onRequest((req, res) => {
const topic = 'topic'
const query = 'queryString'
processMentions(res, query, topic)
})
/**
* Get schedule mentions
*/
exports.scheduledPull = functions.pubsub.schedule('every day 1:00').onRun((context) => {
const topic = 'topic'
const query = 'queryString'
return processMentions('sched', query, topic)
})
Logic:
const functions = require('firebase-functions')
const admin = require('firebase-admin')
admin.initializeApp()
const db = admin.firestore()
const axios = require('axios')
const moment = require('moment')
// Globals
const auth = 'token'
const url = 'https://apiurl.com/'
async function totalMentions(nextPage, start, end, query) {
try {
let config = {
headers: {
Authorization: auth,
Accept: 'text/html',
}
}
const response = await axios.get(url, config)
const total = response.data.results.total
const loops = Math.ceil(total / 500)
return loops
} catch (error) {
console.log('error 1', error)
}
}
async function allMentions(nextPage, start, end, query) {
try {
let config = {
headers: {
Authorization: auth,
Accept: 'text/html',
},
}
const response = await axios.get(url, config)
return response
} catch (error) {
console.log('error 2', error)
}
}
async function saveData(response, end, topic) {
try {
let data = await response.data.results.clips
let batch = db.batch()
data.forEach((c) => {
delete c.localTime
let reff = db.collection(collection).doc(date).collection(collection).doc(c.id.toString())
batch.set(reff, c)
})
let batches = await batch.commit()
return batches
} catch (error) {
console.log('error3 ', error)
}
}
async function processMentions(res, query, topic) {
try {
totalMentions(1, start, end, query)
.then(async (loops) => {
let endbatch = 0
for (let i = 1; i <= loops; i++) {
await allMentions(i, start, end, query)
.then(async (response) => {
await saveData(response, end, topic)
return ++endbatch
})
.catch((err) => {
console.log('error 4 ' + err)
})
if (endbatch === loops) {
if (res !== 'sched') {
console.log('http trigger finished')
return res.status(200).end()
} else {
return console.log('schedule finished')
}
}
}
})
.catch((err) => {
console.log('error5 ' + err)
})
} catch (error) {
console.log('error6 ' + error)
}
}
For the pubsub trigger to work correctly, processMentions needs to return a promise that resovles when all of the async work is complete. Right now, it's returning nothing, which (since it's declared async) translates into a promise that's resolved immediately with no value. Calling then/catch on a promise isn't doing what you expect - you need to return a promise chain from your async work.
I'm not sure why you have it declared async, without also using await inside of it to manage the promises much more easily.
I am calling an API where I can only fetch 1000 records per request,
I was able to achieve this using recursion.
I am now trying to achieve the same using promises, I am fairly new to Node.js and JavaScript too.
I tried adding the recursion code in an if else block but failed
var requestP = require('request-promise');
const option = {
url: 'rest/api/2/search',
json: true,
qs: {
//jql: "project in (FLAGPS)",
}
}
const callback = (body) => {
// some code
.
.
.//saving records to file
.
//some code
if (totlExtractedRecords < total) {
requestP(option, callback).auth('api-reader', token, true)
.then(callback)
.catch((err) => {
console.log('Error Observed ' + err)
})
}
}
requestP(option).auth('api-reader', token, true)
.then(callback)
.catch((err) => {
console.log('Error Observed ' + err)
})
I want to execute the method using promise and in a synchronous way,
i.e. I want to wait until the records are all exported to a file and continue with my code
I think its better to create your own promise and simply resolve it when your done with your recursion. Here's a simply example just for you to understand the approach
async function myRecursiveLogic(resolveMethod, ctr = 0) {
// This is where you do the logic
await new Promise((res) => setTimeout(res, 1000)); // wait - just for example
ctr++;
console.log('counter:', ctr);
if (ctr === 5) {
resolveMethod(); // Work done, resolve the promise
} else {
await myRecursiveLogic(resolveMethod, ctr); // recursion - continue work
}
}
// Run the method with a single promise
new Promise((res) => myRecursiveLogic(res)).then(r => console.log('done'));
Here's a clean and nice solution using the latest NodeJS features.
The recursive function will continue executing until a specific condition is met (in this example asynchronously getting some data).
const sleep = require('util').promisify(setTimeout)
const recursive = async () => {
await sleep(1000)
const data = await getDataViaPromise() // you can replace this with request-promise
if (!data) {
return recursive() // call the function again
}
return data // job done, return the data
}
The recursive function can be used as follows:
const main = async () => {
const data = await recursive()
// do something here with the data
}
Using your code, I'd refactored it as shown below. I hope it helps.
const requestP = require('request-promise');
const option = {
url: 'rest/api/2/search',
json: true,
qs: {
//jql: "project in (FLAGPS)",
}
};
/*
NOTE: Add async to the function so you can udse await inside the function
*/
const callback = async (body) => {
// some code
//saving records to file
//some code
try {
const result = await requestP(option, callback).auth('api-reader', token, true);
if (totlExtractedRecords < total) {
return callback(result);
}
return result;
} catch (error) {
console.log('Error Observed ' + err);
return error;
}
}
Created this code using feed back from Amir Popovich
const rp = require('Request-Promise')
const fs = require('fs')
const pageSize = 200
const options = {
url: 'https://jira.xyz.com/rest/api/2/search',
json: true,
qs: {
jql: "project in (PROJECT_XYZ)",
maxResults: pageSize,
startAt: 0,
fields: '*all'
},
auth: {
user: 'api-reader',
pass: '<token>',
sendImmediately: true
}
}
const updateCSV = (elment) => {
//fs.writeFileSync('issuedata.json', JSON.stringify(elment.body, undefined, 4))
}
async function getPageinatedData(resolve, reject, ctr = 0) {
var total = 0
await rp(options).then((body) => {
let a = body.issues
console.log(a)
a.forEach(element => {
console.log(element)
//updateCSV(element)
});
total = body.total
}).catch((error) => {
reject(error)
return
})
ctr = ctr + pageSize
options.qs.startAt = ctr
if (ctr >= total) {
resolve();
} else {
await getPageinatedData(resolve, reject, ctr);
}
}
new Promise((resolve, reject) => getPageinatedData(resolve, reject))
.then(() => console.log('DONE'))
.catch((error) => console.log('Error observed - ' + error.name + '\n' + 'Error Code - ' + error.statusCode));