Develop Reference

Cannot await for sqlite3.Database.get() function completion in Node.js

I'm struggling with some basic async/await problem in node.js using node-sqlite3.
My objective is to select some value from SQLite DB, check it for some condition and take some actions in case the condition is met. Here's the code:
const sqlite3 = require('sqlite3').verbose();
main();
async function main() {
let ordersDb = await createDbConnection('./ProcessedOrders.db');
var orderProcessed = await orderAlreadyProcessed(ordersDb, "555");
console.log("orderProcessed = " + orderProcessed);
if (!orderProcessed) {
console.log("So condition is met!");
}
}
async function orderAlreadyProcessed(ordersDb, orderNumberStr) {
console.log('starting orderAlreadyProcessed function'); //DEBUG
var result;
var query = 'select count(SoldOrderNumber) as "recsCount" from ProcessedSoldOrders where SoldOrderNumber = ?;';
await ordersDb.get(query
,[orderNumberStr]
,(err, row) => {
console.log('Row with count = ' + row); //DEBUG
console.log('row.recsCount = ' + row.recsCount); //DEBUG
result = typeof row !== 'undefined' && row.recsCount > 0;
});
console.log('Returning ' + result); //DEBUG
return result;
}
async function createDbConnection(dbFileName) {
let db = new sqlite3.Database(dbFileName, (err) => {
if (err) {
console.log(err.message);
}
});
return db;
}
But what I get is code executing further, not awaiting for Database.get() method at all! As a result, here's what I see printing in console:
starting orderAlreadyProcessed function
Returning undefined
orderProcessed = undefined
So IF condition met!
Row with count = [object Object]
row.recsCount = 1
As we can see, we return from orderAlreadyProcessed too early with return value = 'undefined'. So condition is met, actions taken, and only then Database.get() returns. But if it was properly awaited, condition would not be met.
How can I make it await for result value?

Since you want to use async/await, and the node-sqlite3 (sqlite3) library does not support the Promise API, you need to use the node-sqlite (sqlite) library, which is a wrapper over sqlite3 and adds support for the Promise API. Then, your code will look something like this:
const sqlite3 = require('sqlite3');
const { open } = require('sqlite');
async function main() {
try {
sqlite3.verbose();
const ordersDb = await createDbConnection('./ProcessedOrders.db');
const orderProcessed = await orderAlreadyProcessed(ordersDb, "555");
console.log("orderProcessed = " + orderProcessed);
if (!orderProcessed) {
console.log("So condition is met!");
}
} catch (error) {
console.error(error);
}
}
async function orderAlreadyProcessed(ordersDb, orderNumberStr) {
try {
console.log('Starting orderAlreadyProcessed function');
const query = 'SELECT COUNT(SoldOrderNumber) as `recsCount` from ProcessedSoldOrders where SoldOrderNumber = ?;'
const row = await ordersDb.get(query, [orderNumberStr]);
console.log('Row with count =', row);
console.log('row.recsCount =', row.recsCount);
const result = typeof row !== 'undefined' && row.recsCount > 0;
console.log('Returning ' + result);
return result;
} catch (error) {
console.error(error);
throw error;
}
}
function createDbConnection(filename) {
return open({
filename,
driver: sqlite3.Database
});
}
main();
I specifically did not remove your console.log and other parts of the code so as not to confuse the original logic of your program.

If we don't to use another library
then we can return a new Promise function & use await, as below:
Note: Below has example for INSERT/run, instead of SELECT/get, but promise/await works same
const sqlite3 = require("sqlite3").verbose();
let db;
db = new sqlite3.Database('./Chinook.db');
function insert() {
return new Promise((resolve, reject) => { // return new Promise here <---
const userId = uuid4();
let sql = `INSERT INTO Users(id) VALUES (?)`; // INSERT <----
let params = [userId];
return db.run(sql, params, function (err, res) { // .run <----
if (err) {
console.error("DB Error: Insert failed: ", err.message);
return reject(err.message);
}
return resolve("done");
});
});
}
let result = await insert(); // now await works fine <------
res.send({ result });

Promise condition equal

I want to wait in my code until two values are the same. For this I use
await new Promise((resolve, reject) => {
if(curCount == maxTests) resolve;
});
But I think, this is only called one time. How I can make it that if both values are the same the promise is resolved? How to avoid that it will never send a resolve?
UPDATE:
Some requested the function that makes trouble. Here is the whole function, without it sub function. The function will fill the q-queue to fullfill the tests sync. The problem ist that req.body.selection.forEach immediately returns but I want to wait until the whole queue is ready. So my idea was to add a promise to the end and hit until current and max are the same.
router.post('/imgtest', async (req, res) => {
console.log('Start Image Test');
//Er kommt an.
req.setTimeout(5000000); // Nach Adam Riese 83 Minuten.
process.setMaxListeners(0);
io = req.app.get('socketio');
//Caluclate the max amounnt of tests
const maxTests = req.body.selection.length * req.body.servers.length;
var curCount = 0;
//RETURN IF THIS IS READY. CURRENTLY IT RETURNS IMMEDIATLY
req.body.selection.forEach(async function(entry) {
//Jetzt erstmal die Domain aus der DB holen
var dbUrl = await getUrl(entry);
console.log('tapsi');
var bildFormat = '';
var arrQuestionmark = dbUrl.split('?');
if(arrQuestionmark.length==2){
if(arrQuestionmark[1].includes('&')){
var arrAnd = arrQuestionmark[1].split('&');
arrAnd.forEach(function(entry) {
if(entry.includes('format=')){
var arrFormat = entry.split('=');
bildFormat = arrFormat[1];
}
});
}
}
var masterName = uuidv1();
const orgpath = path.resolve(__basedir, 'tests/downloads', masterName + '-.' + bildFormat);
//Download the MAsterimage
(async () => {
await queue.add(() =>downloadImage(dbUrl, 'c11', req.body.domain, bildFormat, orgpath) );
})();
req.body.servers.forEach(async function(xserver) {
var fileName = masterName + '-' + xserver + '.' + bildFormat;
const dpath = path.resolve(__basedir, 'tests/downloads', fileName);
(async () => {
await queue.add(() => downloadImage(dbUrl, xserver, req.body.domain, bildFormat, dpath));
//console.log('Done ' + entry);
})();
(async () => {
await queue.add(async() => startCompare(orgpath, dpath, 'null:').then(function(result) {
console.log(result);
curCount++;
messageIO(curCount,maxTests);
}));
//console.log('done compare ' + entry);
//fs.unlinkSync(dpath);
})();
});
});
console.log('Need to wait');
res.sendStatus(200);
});

You're correct in assuming that will only be called once. A way around that is to, within the function, create a loop via setInterval - doing a regular check and resolving if true and clearing the loop.

Not too sure what trying to achieve but one thing is for certain, Array.prototype.forEach() will not await even if its callback is async and performing a test inside a new Promise(...) constructor won't help.
Good news though, for loops will await.
Here's the code with for loops instead of .forEach() (twice), unnecessary stuff removed, and otherwise tidied up.
First a suggestion ...
// https://nodejs.org/api/querystring.html
// The querystring module provides utilities for parsing and formatting URL query strings.
const querystring = require('querystring');
... then:
router.post('/imgtest', async (req, res) => {
req.setTimeout(5000000); // Nach Adam Riese 83 Minuten.
process.setMaxListeners(0);
// io = req.app.get('socketio'); // not used
const masterName = uuidv1(); // moved from inner loop
for (i=0; i<req.body.selection.length; i++) {
let entry = req.body.selection[i];
let dbUrl = await getUrl(entry);
let bildFormat = querystring.parse(dbUrl).format;
let orgpath = path.resolve(__basedir, 'tests/downloads', `${masterName}-.${bildFormat}`);
await queue.add(() => downloadImage(dbUrl, 'c11', req.body.domain, bildFormat, orgpath));
for (j=0; j<req.body.servers.length; j++) {
let xserver = req.body.servers[j];
let dpath = path.resolve(__basedir, 'tests/downloads', `${masterName}-${xserver}.${bildFormat}`);
await queue.add(() => downloadImage(dbUrl, xserver, req.body.domain, bildFormat, dpath));
await queue.add(async() => startCompare(orgpath, dpath, 'null:'); // probably safe to remove `async`
// fs.unlinkSync(dpath); // ???
}
}
res.sendStatus(200);
});
This should get you started. I expect that you still have some way to go. At the very least you need to add a try/catch structure and be prepared to send error/status back to the client.

Don't understand this async await call with axios

I am using node 10+, and I have this function where I do a database query and wait for the result and return it:
var test3 = (req,res,query) => {
var conn = new sql.ConnectionPool(dbconfig);
var req = new sql.Request(conn);
var result;
return conn.connect().then(async() => {
result = await req.query(query);
conn.close();
return result;
}).catch(e => {
return e;
}).finally(() => {
conn.close();
});
}
First, I would like to know why I have to return the conn.connect() block..
return conn.connect().then(async() => {...
I know it has something to do with promise chaining I think, but I dont understand why, because my async db call is already resolved from the await dbcall function... and I just return the result from inside the function
Then, I have a router where I call the api function here:
router.get("/api/compareCount", function(req,res) {
var query = `SELECT COUNT(*) as count
FROM [DublettenReferenzmenge].[dbo].[DocumentForElasticsearch] where lastChange < dateadd(day,-1,getdate())`;
var query2 = `SELECT COUNT(*) as count
FROM [DublettenReferenzmenge].[dbo].[DocumentForElasticsearch] where lastChange < dateadd(hour,-8,getdate())`;
var query3 =`SELECT COUNT(*) as count
FROM [DublettenReferenzmenge].[dbo].[DocumentForElasticsearch]`;
axios.all([searchES(req,res), test3(req,res,query), test3(req,res,query2) , test3(req,res,query3)])
.then(axios.spread(function (esCount, mssqlCount1, mssqlCount2, mssqlCount3) {
totalES = esCount.hits.total;
totalMSSQL = mssqlCount1.recordset[0].count;
totalMSSQL2 = mssqlCount2.recordset[0].count;
totalMSSQL3 = mssqlCount3.recordset[0].count;totalMSSQL, " mssqlCount2: ", totalMSSQL2, "mssqlCount3: ", totalMSSQL3);
var msg = "ES Dokumente total: " + totalES + " MSSQL Dokumente total: " + totalMSSQL + "<br>";
if ( totalES != totalMSSQL) {
msg += "Critical: " + totalES != totalMSSQL + "<br>";
} if ((totalES != totalMSSQL2)) {
msg += "Warning: " + (totalES != totalMSSQL2) + "<br>";
} if ((totalES > totalMSSQL3)) {
msg += "Achtung es gibt ungelöschte Dokumente im Elasticsearch Index!";
}
res.set('Content-Type', 'text/html');
res.send(msg);
})).catch((err) => {
res.send(err);
});
})
router.get("/api/test3", async function (req,res) {
var query = `SELECT COUNT(*) as count
FROM [DublettenReferenzmenge].[dbo].[DocumentForElasticsearch] where lastChange < dateadd(day,-1,getdate())`;
var result = await test3(req,res,query);
res.json(result);
})
The api/test3 route returns me the result as usual, but the api/compareCount does return me correct results as well...
Furthermore, I have to use the async function ... await test3(..) async-await syntax structure to resolve my result into a variable... But I do not have to use that same structure for my api/compareCount function above, the result is returned anyways in the .then(axios.spread(function(...))). Why is that? I am quite confused as I don't really know the inner workings of the Promise chaining and calls...
EDIT: before my test3() function, I had something like this:
async function testQuery(query) {
try {
let pool = await sql.connect(dbconfig);
let result1 = await pool.request()
//.input('input_parameter', sql.Int, value)
.query(query);
sql.close();
return result1;
} catch (err) {
console.log(err);
sql.close();
} finally {
sql.close();
}
};
I got also results with that function, however, I got some kind of race condition where it told me that the sql- connection already exists, and do sql.close() first if I reload the page too quickly... I dont get this with the test3() function anymore...

To start with, test3() needs to return a promise so the caller knows when it's done and whether it had an error or not. It's probably easiest to use async/await for that:
async function test3(query) => {
const conn = new sql.ConnectionPool(dbconfig);
const request = new sql.Request(conn);
await conn.connect();
try {
const result = await request.query(query);
return result;
} finally {
conn.close();
}
}
Various changes:
Only pass arguments that are going to be used
Use await to simplify async logic flow
Use try/finally to catch any error after connected so we can always close the connection and not leak a connection
Switch from var to const.
Make function async so it returns a promise that is hooked to when the internals are done or have an error
Then, if you adjust how test3() is called to only pass the query argument (since that's all that is used), your use of that function the other places you are using it should work.
Your code was doing this:
var result = await test3(req,res,query);
But, test3() had no return value so the await had nothing useful to do. await works with a promise and you weren't returning a promise that was linked to the async operations inside of test3(). That's what my changes above do.

Push into an array from foreach and make it available outside foreach

I stuck by looping through an array that receive values from a promise and push values into a new array which is available outside the foreach.
What i have:
app.post('/submit', function (req, res) {
uploadPics(req, res, function (err) {
if (err instanceof multer.MulterError) {
res.send(JSON.stringify({UploadResult: err.message}));
console.log(err.message + ' ' +'Redirect /home');
} else if (err) {
console.log(err);
} else {
res.send(JSON.stringify({UploadResult: 'Success'}));
var filesarray = req.files;
var picinfos = [];
filesarray.forEach(function(file){
GetFileMetaInfo.filemetainfo(file.path).then(function (metadata){
//Stuck here! Can push values into an array (picinfos) but only available in the foreach. not outside..
})
})
//I need picinfos array here....
}
})
})
How i receive my metadata:
var exif = require('exif-parser');
var fs = require('fs');
exports.filemetainfo = function (filepath) {
return new Promise((resolve) => {
var file = filepath;
var buffer = fs.readFileSync(file);
var parser = exif.create(buffer);
var result = parser.parse();
resolve (result);
}).then(function (metadata){
if (metadata.tags.CreateDate !== undefined){
date = new Date (metadata.tags.CreateDate*1000);
datevalues = [
date.getFullYear(),
date.getMonth()+1,
date.getDate(),
date.getHours(),
date.getMinutes(),
date.getSeconds(),
];
CreateDate = date.getFullYear()+'-'+(date.getMonth()+1)+'-'+date.getDate();
CreateTime = date.getHours()+':'+date.getMinutes()+':'+date.getSeconds();
console.log("CrDate:" +CreateDate, "CrTime:" +CreateTime );
} else {
console.log("No Metadata Creation Infos found in " +filepath);
CreateDate = "";
CretaeTime = "";
}
if (metadata.tags.GPSLatitude !== undefined){
GPSLat = metadata.tags.GPSLatitude;
GPSLon = metadata.tags.GPSLongitude;
console.log("GPSLat:" + GPSLat , "GPSLon:" +GPSLon);
}
else {
console.log("No Metadata GPS Infos found in " +filepath)
GPSLat = "";
GPSLon = "";
}
return MetaData = {
GPSLat: GPSLat ,
GPSLon: GPSLon,
CreateDate: CreateDate,
CreateTime: CreateTime,
}
})
}
May i ask someone to give a hand. How can i make my array available outside the foreach. thank you very much!

The reason you're getting empty array at the end of forEach is because, GetFileMetaInfo.filemetainfo() returns a promise and forEach won't wait for async actions.
You could use async/await with for...of loop to get your desired result.
app.post('/submit', function (req, res) {
uploadPics(req, res, async function (err) { // note async here
if (err instanceof multer.MulterError) {
res.send(JSON.stringify({UploadResult: err.message}));
console.log(err.message + ' ' +'Redirect /home');
} else if (err) {
console.log(err);
} else {
res.send(JSON.stringify({UploadResult: 'Success'}));
var filesarray = req.files;
var picinfos = [];
for(let file of filesarray) {
const metadata = await GetFileMetaInfo.filemetainfo(file.path);
// push metadata into your array here
picinfos.push(metadata);
}
// You will have picinfos here
}
})
})

Although the question is already answered by Dinesh Pandiyan there are still some adjustments that can be made. The following code in his answer runs sequential, meaning that every async request is made after the previously returned result is resolved.
for(let file of filesarray) {
const metadata = await GetFileMetaInfo.filemetainfo(file.path);
// ^- pauses the execution of the current running code
// push metadata into your array here
picinfos.push(metadata);
}
async call #1 ╌╌await╌╌> async call #2 ╌╌await╌╌> async call #3 ╌╌await╌╌> result
You could make the code concurrent by first executing all async statements and then wait until all results are resolved. This can be done by simply changing the following:
// execute all the async functions first, reducing the wait time
for(let file of filesarray) {
const metadata = GetFileMetaInfo.filemetainfo(file.path);
// ^- remove the await
// push metadata into your array here
picinfos.push(metadata);
}
// wait for all results to be resolved
picinfos = await Promise.all(picinfos);
// ^- instead await here
async call #1 ╌╌┐
async call #2 ╌╌┼╌╌await all╌╌> result
async call #3 ╌╌┘
The above could be further simplified by simply using an Array.map() in combination with the already shown Promise.all().
var filesarray = req.files;
var picinfos = await Promise.all(filesarray.map(file => {
return GetFileMetaInfo.filemetainfo(file.path);
}));
// picinfos should be present
Or if you want to avoid working with async/await:
var filesarray = req.files;
Promise.all(filesarray.map(file => {
return GetFileMetaInfo.filemetainfo(file.path);
})).then(picinfos => {
// picinfos should be present
});

JS Asynchronous undefined

I have read the following stack overflow post and this one, but i still can't find the solution to my problem.
I have function defined in a utility.js file as follows:
let geocodeOptions = {
provider: 'google',
httpAdapter: 'https',
apiKey: 'GMAPKEY_HERE',
formatter: null
};
async function postInformation(message, bot) {
try {
let messageContent = message.content;
let EncId = await getDetails(messageContent), msg
const NodeGeocoder = require('node-geocoder');
geocoder = NodeGeocoder(geocodeOptions);
geocoder.reverse({ lat: EncId[0], lon: EncId[1] }).then(res => {
msg = "*" + res[0]['city'] + "*";
})
let post;
let delayValue = await Bluebird.delay(2000)
if (delayValue) {
post = getMsg1() + '\n' + getMsg2() + '\n' + EncId;
return await post
}
} catch (error) {
console.log(error)
}
}
And in the main file, I am calling it as follows:
const util = require('./utility')
util.postInformation(message, client).then(value => {
console.log("value from main file")
console.log(value) // Always prints undefined
})
I also tried following:
let value = await util.postInformation(message, client);
console.log("value from main file")
console.log(value) // Always prints undefined
Instead of redirecting to other posts, pls provide a solution to this problem, where is the mistake i am making here.

I'm just going to leave this here for now
const NodeGeocoder = require('node-geocoder');
let geocodeOptions = {
provider: 'google',
httpAdapter: 'https',
apiKey: 'GMAPKEY_HERE',
formatter: null
};
async function postInformation(message, bot) {
try {
let messageContent = message.content;
let [lat, lon] = await getDetails(messageContent);
geocoder = NodeGeocoder(geocodeOptions);
let msg = await geocoder.reverse({lat, lon})
.then(([res]) => "*" + res.city + "*")
return getMsg1() + '\n' + getMsg2() + '\n' + EncId;
} catch (error) {
console.log(error)
}
}
OK - let me explain some (relevant) changes to your code
As NodeGeocoder returns a Promise, we can await, however, you need to return something in the .then, otherwise you get undefined
awaiting geocoder.reverse means no need for the (hacky) pause for 2 seconds, so, that's gone. Besides
let x = await Bluebird.delay(2000)
will always result in x being undefined - you'd want
let x = await Bluebird.delay(2000, true)
at least, if you want x to be truthy - but since it's just a delay, there's really no reason for testing the result of Bluebird.delay - because we already know it
The not so relevant changes:
I made a few little (ES2015+) changes, like shorthand arrow notation, and ([res]) => res.city instead of res => res[0]['city'] just to make the code neater
Also
let [lat, lon] = await getDetails(messageContent);
instead of
let EncId = await getDetails(messageContent)
and accessing lat as EndId[0] and lon as EndId[1] - also means
{ lat: EncId[0], lon: EncId[1] }
is just
{ lat, lon }