Develop Reference

Promises resolving to undefined with node.js mysql queries

I'm writing a route that takes in user-input data from a form and uses that data to execute a few separate queries on my database. That is all well-tested, and I know I can get the user input, query the database, and return the results. The part giving me trouble, however, is that I want to randomly select a few results out of all of the returned data, so I am pushing the results into one big array. But since the queries execute asynchronously, the only way to access the data is in the callback and of course if I tried to push the rows in the callback, I still can't access that array outside of the callback functions because that code will execute before the queries are even run. So I'm attempting to use JavaScript async/await syntax to do the queries, as follows:
module.exports = {
search: (req, res) => {
function saveResults(resultsObj) {
Object.keys(resultsObj).forEach(function (key) {
var row = resultsObj[key];
var song = row.song_name
resultsArray.push(song);
return resultsArray;
});
}
async function queryForSong(){
var songGenre;
getSongGenre = "The queries aren't important, I know they work as expected";
await mysqlConnection.query(getSongGenre, (err, rows) => {
if (err) throw err;
songGenre = row[0].song_genre;
songSearchQuery = "SELECT query";
mysqlConnection.query(songSearchQuery, (err, rows) => {
if (err) throw err;
return saveResults(rows);
});
});
}
async function queryForArtist() {
var artistID;
getArtistID = "SELECT query";
await mysqlConnection.query(getArtistID, (err, rows) => {
if (err) throw err;
artistID = rows[0].artist_id;
console.log(rows, artistID);
artistSearchQuery = "SELECT query";
mysqlConnection.query(artistSearchQuery, (err, rows) => {
if (err) throw err;
return saveResults(rows);
});
});
}
async function queryForGenre() {
genreSearchQuery = "SELECT query";
await mysqlConnection.query(genreSearchQuery, (err, rows) => {
if (err) throw err;
return saveResults(rows);
});
}
async function queryForDecade() {
decadeSearchQuery = "SELECT query";
await mysqlConnection.query(decadeSearchQuery, (err, rows) => {
if (err) throw err;
return saveResults(rows);
});
}
var resultsArray = [];
var artist = req.body.searchArtist;
var song = req.body.searchSong;
var genre = req.body.searchGenre;
var decade = req.body.searchDecade;
if (song) {
queryForSong().then((value) => console.log(value)); //these all log undefined
}
if (artist) {
queryForArtist().then((value) => console.log(value));
}
if (genre) {
queryForGenre().then((value) => console.log(value));
}
if (decade) {
queryForDecade().then((value) => console.log(value));
}
}
}
Then the idea is to save the resolved rows from the promises into one big array and randomize it from there. My only thought of why this isn't working is that it could be because the return statements are located inside the callback functions? I have also tried it using promises directly as this post explains, by writing all of my async functions like this instead:
function queryForGenre() {
genreSearchQuery = "SELECT query";
return new Promise(function (resolve, reject) {
mysqlConnection.query(genreSearchQuery, (err, rows) => {
if (err) reject(err);
resolve(saveResults(rows));
});
});
}
But either way, the console.log statements in the .then still print undefined, meaning my promises aren't resolving to the saveResults(rows) value that I give it. Do I need to make the saveResults function asynchronous as well? From my understanding of promises, this seems like it should work. What am I missing? Or is there a simpler way to do this?

Your functions queryForSong, queryForDecade, etc are using await, but the mysql driver query function accepts a callback and does not return a promise. Also, the return inside your callback is scoped within the callback, not scoped with respect to your outside async function. Instead wrap the function within a promise, resolve and reject where necessary. Also avoid using await unless absolutely necessary because using it blocks the event loop. I have given an example for how to wrap with promise for queryForSong
function queryForSong(){
return new Promise((resolve, reject) => {
let songGenre;
const getSongGenre = "The queries aren't important, I know they work as expected";
mysqlConnection.query(getSongGenre, (err, rows) => {
if (err) reject(err);
else {
songGenre = row[0].song_genre;
const songSearchQuery = "SELECT query";
mysqlConnection.query(songSearchQuery, (err, rows) => {
if (err) reject(err);
else
resolve(saveResults(rows));
});
}
});
});
}

So it turns out it was the saveResults function causing me issues. I don't totally understand why this would fix it, but instead of resolving to the value of the saveResults function, I instead tried just including the saveResults body in each function that returns a promise and that works. For example,
function queryForSong() {
var songResultsArray = [];
var songGenre;
getSongGenre = "SELECT query";
return new Promise(function (resolve, reject) {
mysqlConnection.query(getSongGenre, (err, rows) => {
if (err) reject(err);
var row = rows[0];
songGenre = row.song_genre;
songSearchQuery = "SELECT query #2";
mysqlConnection.query(songSearchQuery, (err, rows) => {
if (err) throw err;
//this is the body of the saveResults function
Object.keys(rows).forEach(function (key) {
var row = rows[key];
var song = row.song_name;
songResultsArray.push(song);
});
resolve(songResultsArray);
});
});
});
}

How to make sure that the onUpdate method of the firebase database only runs after the onCreate method completes

I am trying to sync Firebase database data with a Google spreadsheet. For this, I have two methods that run onCreate and onUpdate in the database in a specific path. The problem is that sometimes the method called by the onUpdate callback is executed before the method called by the onCreate callback. How can I make sure the onCreate method is complete before doing something in onUpdate Method?
By the way, here is the code. Maybe I got some advice about the code in general, because now is the most difficult time to work with the Firebase and JavaScript functions.
exports.appendrecordtospreadsheet = functions.database.ref(`${CONFIG_DATA_PATH}/{USERID}/{GAMENUMBER}`).onCreate(
(snap) => {
var firstRecord = [];
console.log(snap.ref.parent.key);
const key = snap.ref.parent.key;
firstRecord.unshift(key);
firstRecord.push(snap.val()[0])
return appendPromise({
spreadsheetId: CONFIG_SHEET_ID,
range: 'A:Z',
valueInputOption: 'USER_ENTERED',
insertDataOption: 'INSERT_ROWS',
resource: {
values: [firstRecord],
},
});
});
function appendPromise(requestWithoutAuth)
{
return new Promise(async (resolve, reject) =>
{
const client = await getAuthorizedClient();
const sheets = google.sheets('v4');
const request = requestWithoutAuth;
request.auth = client;
return sheets.spreadsheets.values.append(request, (err, response) => {
if (err) {
console.log(`The API returned an error: ${err}`);
return reject(err);
}
return resolve(response.data);
});
});
}
exports.UpdateRecordInSpreadSheet = functions.database.ref(`${CONFIG_DATA_PATH}/{USERID}/{GAMENUMBER}`).onUpdate(
(change) => {
const key = change.after.ref.parent.key;
const updatedRecord = change.after.val();
updatedRecord.unshift(key);
return updatePromise({
spreadsheetId: CONFIG_SHEET_ID,
range: 'A:X',
valueInputOption: 'USER_ENTERED',
resource: {
values: [updatedRecord],
},
});
});
function updatePromise(requestWithoutAuth)
{
return new Promise(async (resolve, reject) =>
{
const client = await getAuthorizedClient();
const sheets = google.sheets('v4');
const updateRequest = requestWithoutAuth;
updateRequest.auth = client;
const getRequest = {
spreadsheetId: CONFIG_SHEET_ID,
range: 'A:A'
};
getRequest.auth = client;
return sheets.spreadsheets.values.get(getRequest, (err, response) => {
if (err) {
console.log(`The API returned an error: ${err}`);
return reject(err);
}
else {
const index = response.data.values[0].lastIndexOf(updateRequest.resource.values[0][0]) + 1;
updateRequest.range = `A${index}:Z${index}`;
return sheets.spreadsheets.values.update(updateRequest, (err, response) => {
if (err) {
console.log(`The API returned an error: ${err}`);
return reject(err);
}
return resolve(response.data);
});
}
});
});
}

Cloud Functions doesn't guarantee order of events. In fact, two events might be processed at the same time. You will need to account for that in your code. Typically it requires use of a transaction to make sure that two concurrently running bits of code do not collide with each other.
There are no workarounds to this - it is just the way Cloud Functions works, and it ensures the product is able to scale up demand without losing events.

Querying multiple promises with a callback

In node.js i have a databaseMapper.js file, that uses the Ojai node MapR api. to extract data. So far i have it working with single documents, but since this is an async api, i have a bit of issues with querying multiple documents.
This is what i have so far:
function queryResultPromise(queryResult) {
//this should handle multiple promises
return new Promise((resolve, reject) => {
queryResult.on("data", resolve);
// ...presumably something here to hook an error event and call `reject`...
});
}
const getAllWithCondition = async (connectionString, tablename, condition) =>{
const connection = await ConnectionManager.getConnection(connectionString);
try {
const newStore = await connection.getStore(tablename);
const queryResult = await newStore.find(condition);
return await queryResultPromise(queryResult);
} finally {
connection.close();
}
}
here it will only return the first because queryResultPromise will resolve on the first document.. however the callback with "data" may occur multiple times, before the queryResult will end like this queryResult.on('end', () => connection.close())
i tried using something like Promise.all() to resolve all of them, but I'm not sure how i include the queryResult.on callback into this logic

This will work
const queryResultPromise = (queryResult) => {
return new Promise((resolve, reject) => {
let result = [];
queryResult.on('data', (data) => {
result.push(data)
});
queryResult.on('end', (data) => {
resolve(result);
});
queryResult.on('error', (err) => {
reject(err);
})
});
};

How do I send multiple queries from one endpoint with Express?

I am trying to query my database several times and construct an object which stores every response from my database in a field. Here is my code:
router.post('/search', (req, res) => {
var collection = db.get().collection('styles')
var data = [];
collection.distinct('make.name', (err, docs) => {
data.push({'make': docs });
});
collection.distinct('model', (function (err, docs) {
data.push({'model': docs });
}))
res.send(data);
});
Since NodeJS/Express is asynchronous, this isn't working as I would like. How can I reconstruct this endpoint to make several database calls (from the same collection) and return an object containing it?

There's more than one way to do it:
Nested callbacks
Without promises you could nest the callbacks:
router.post('/search', (req, res) => {
var collection = db.get().collection('styles')
var data = [];
collection.distinct('make.name', (err, docs) => {
if (err) {
// ALWAYS HANDLE ERRORS!
}
data.push({'make': docs });
collection.distinct('model', (function (err, docs) {
if (err) {
// ALWAYS HANDLE ERRORS!
}
data.push({'model': docs });
res.send(data);
}))
});
});
This would be the easiest way, but note that it is not efficient if those two requests could be done in parallel.
The async module
You can use the async module:
router.post('/search', (req, res) => {
var collection = db.get().collection('styles')
var data = [];
async.parallel({
make: cb => collection.distinct('make.name', cb),
model: cb => collection.distinct('model', cb),
}, (err, responses) => {
if (err) {
// ALWAYS HANDLE ERRORS!
}
data.push({'make': responses.make });
data.push({'model': responses.model });
res.send(data);
});
});
See: https://caolan.github.io/async/docs.html#parallel
But this may still not be the most convenient method.
ES2017 async/await
The most flexible way of doing that if you have 30 calls to make would be to:
Use functions that return promises instead of functions that take callbacks
Use async/await if you can or at least generator based coroutines
Await on promises (or yield promises) when the logic needs to run in sequence
Use Promise.all() for anything that can be done in parallel
With async/await your code could look like this:
// in sequence:
var make = await collection.distinct('make.name');
var model = await collection.distinct('model');
// use 'make' and 'model'
Or:
// in parallel:
var array = await Promise.all([
collection.distinct('make.name'),
collection.distinct('model'),
]);
// use array[0] and array[1]
A big advantage of async/await is the error handling:
try {
var x = await asyncFunc1();
var array = await Promise.all([asyncFunc2(x), asyncFunc3(x)]);
var y = asyncFunc4(array);
console.log(await asyncFunc5(y));
} catch (err) {
// handle any error here
}
You can only use it inside of a function created with the async keyword. For more info, see:
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/async_function
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await
For support in browsers, see:
http://caniuse.com/async-functions
For support in Node, see:
http://node.green/#ES2017-features-async-functions
In places where you don't have native support for async and await you can use Babel:
https://babeljs.io/docs/plugins/transform-async-to-generator/
or with a slightly different syntax a generator based approach like in co or Bluebird coroutines:
https://www.npmjs.com/package/co
http://bluebirdjs.com/docs/api/promise.coroutine.html
See those answers for more info:
try/catch blocks with async/await
node.js ~ constructing chained sequence of Promise resolves
How to run Generator Functions in Parallel?
node.js ~ constructing chained sequence of Promise resolves
Using async/await + Bluebird to promisifyAll
jQuery: Return data after ajax call success

You can do it with Promises
router.post('/search', (req, res) => {
var collection = db.get().collection('styles');
// Create promise for "make.name" query
let firstQuery = new Promise((resolve, reject) => {
collection.distinct('make.name', (err, docs) => {
if (!err) {
resolve(docs);
} else {
reject(err);
}
});
});
// Create promise for "model" query
let secondQuery = new Promise((resolve, reject) => {
collection.distinct('model', (function (err, docs) {
if (!err) {
resolve(docs);
} else {
reject(err);
}
}))
})
// Run both queries at the same time and handle both resolve results or first reject
Promise.all([firstQuery, secondQuery])
.then((results) => {
res.send({ "make.name": results[0], "model": results[1] });
})
.catch((err) => {
// Catch error
res.send({});
});
});
Also you can use destructuring in callback functions like that:
Promise.all([firstQuery, secondQuery])
.then(([makeName, model]) => res.send({ "make.name": makeName, model }))
UPD: If you have a bunch of collection to request you can create an array of collections name, map it to promise requests and handle with Promise.all, for example
let collections = ["firstCollection", "secondCollection", "nCollection"];
let promises = collections.map((collectionName) => {
return new Promise((resolve, reject) => {
collection.distinct(collectionName, (err, docs) => {
if (!err) {
resolve(docs)
} else {
reject(err);
}
});
})
});
Promise.all(promises)
.then(results => {
// Do what you want to do
})
.catch(error => {
// or catch
});

How to properly chain Promises with nesting

My node project currently contains a sideway christmas tree of nested callbacks in order to fetch data and process them in the right order. Now I'm trying refactor that using Promises, but I'm unsure how to do it properly.
Let's say I'm fetching a list of offices, then for each office all their employees and then each employees' salary. In the end all entities (offices, employees and salaries) should be linked together and stored in a database.
Some pseudo-code illustrating my current code (error handling omitted):
fetch(officesEndpoint, function (data, response) {
parse(data, function (err, offices) {
offices.forEach(function (office) {
save(office);
fetch(employeesEndPoint, function (data, response) {
parse(data, function (err, employees) {
// link each employee to office
save(office);
save(employee);
employees.forEach(function () {
fetch(salaryEndpoint, function (data, response) {
parse(data, function (err, salaries) {
// link salary to employee
save(employee);
});
});
});
});
});
});
});
});
I tried solving this with promises, but I have a couple of problems:
kind of verbose?
each office needs to be linked to their respective employees, but in the saveEmployees function I only have access to the employees, not the office from further up in the chain:
var restClient = require('node-rest-client');
var client = new restClient.Client();
var xml2js = require('xml2js');
// some imaginary endpoints
var officesEndpoint = 'http://api/offices';
var employeesEndpoint = 'http://api/offices/employees';
var salaryEndpoint = 'http://api/employees/:id/salary';
function fetch (url) {
return new Promise(function (resolve, reject) {
client.get(url, function (data, response) {
if (response.statusCode !== 200) {
reject(statusCode);
}
resolve(data);
});
});
}
function parse (data) {
return new Promise(function (resolve, reject) {
xml2js.parseString(data, function (err, result) {
if (err) {
reject(err);
}
resolve(result);
});
});
}
function saveOffices (offices) {
var saveOffice = function (office) {
return new Promise(function (resolve, reject) {
setTimeout(function () { // simulating async save()
console.log('saved office in mongodb');
resolve(office);
}, 500);
})
}
return Promise.all(offices.map(saveOffice));
}
function saveEmployees (employees) {
var saveEmployee = function (employee) {
return new Promise(function (resolve, reject) {
setTimeout(function () { // simulating async save()
console.log('saved employee in mongodb');
resolve(office);
}, 500);
})
}
return Promise.all(offices.map(saveEmployee));
}
fetch(officesEndpoint)
.then(parse)
.then(saveOffices)
.then(function (savedOffices) {
console.log('all offices saved!', savedOffices);
return savedOffices;
})
.then(function (savedOffices) {
fetch(employeesEndPoint)
.then(parse)
.then(saveEmployees)
.then(function (savedEmployees) {
// repeat the chain for fetching salaries?
})
})
.catch(function (error) {
console.log('something went wrong:', error);
});

You don't necesseraly have to nest, this would work too:
fetch(officesEndpoint)
.then(parse)
.then(saveOffices)
.then(function(savedOffices) {
console.log('all offices saved!', savedOffices);
return savedOffices;
})
.then(function(savedOffices) {
// return a promise
return fetch(employeesEndPoint); // the returned promise can be more complex, like a Promise.all of fetchEmployeesOfThisOffice(officeId)
})
// so you can chain at this level
.then(parse)
.then(saveEmployees)
.then(function(savedEmployees) {
return fetch(salariesEndPoint);
})
.catch(function(error) {
console.log('something went wrong:', error);
});

Your promisified functions fetch, parse, saveOffices and saveEmployees are fine. With those, you can refactor your current code to use promises, chain instead of nest where applicable, and leave out a bunch of error handling boilerplate:
fetch(officesEndpoint)
.then(parse)
.then(function(offices) {
return Promise.all(offices.map(function(office) {
return save(office)
.then(function(){ return fetch(employeesEndPoint); })
.then(parse)
.then(function(employees) {
// link each employee to office
// throw in a Promise.all([save(office), save(employee)]) if needed here
return Promise.all(employees.map(function(employee) {
return fetch(salaryEndpoint)
.then(parse)
.then(function(salaries) {
return Promise.all(salaries.map(function(salary) {
// link salary to employee
return save(employee);
}));
});
}));
});
}));
});
In the innermost loop callback, you've got all of office, employee and salary available to interlink them to your liking. You cannot really avoid this kind of nesting.
You'll get back a promise for a huge array of arrays of arrays of save results, or for any error in the whole process.

It is good approach to change this
if (response.statusCode !== 200) {
reject(statusCode);
}
resolve(data);
to this
if (response.statusCode !== 200) {
return reject(statusCode);
}
resolve(data);
In your example, the result will be same, but if you are making more things (like doing something in database) the unexpected result may occure, because without return the whole method will be executed.
This example
var prom = new Promise((resolve,reject) => {
reject(new Error('error'));
console.log('What? It did not end');
resolve('Ok, promise will not be called twice');
});
prom.then(val => {
console.log(val);
}).catch(err => {
console.log(err.message);
});
is having this output
What? It did not end
error
To the question - if you need access to more than one returned value (i.e. offices AND employies), you have basically two options :
Nested promises - this is not generally bad, if it "makes sense". Altought promises are great to avoid huge callback nesting, it is ok to nest the promises, if the logic needs it.
Having "global" variables - you can define variable in the scope of the promise itself and save results to it, therefore the promises are using these variables as "global" (in their scope).