Develop Reference

Wait for callback to be returned in a controller

I want to upload some files, add them to a database and return the ids of the new objects.
fn: async function (inputs) {
let docIds = []
let settings = {...}
await inputs.filesToUpload.upload(settings, async (err, files) => {
if (err)
throw {'invalid': 'The provided data is invalid.'}
for (let i = 0; i < files.length; i += 1) {
let newDocument = await Document.create({
name: file.filename
}).fetch()
docIds.push(newDocument.id)
}
})
return {
ids: docIds
}
})
Unfortunately, the controller doesn't wait for the objects to be created in the database and returns {ids: []} immediately, only then the documents are uploaded and objects created. I have tried using passing the ids as a callback and promise but the controller always executes return without waiting for the results.

inputs.filesToUpload.upload is making a callback which is always going to be asynchronous.
async keyword before a function with callback can not make it wait.
async works only if the function is returning a promise
check following code where I have extracted the upload flow in a separate function that returns a promise
you can then await on this promise and get generated ids..
async function test(inputs) {
const docIds = await upload(inputs, {});
return { ids: docIds };
}
function upload(inputs, settings) {
return new Promise((resolve, reject) => {
const ids = [];
inputs.filesToUpload.upload(settings, async (err, files) => {
if (err) {
return reject({ 'invalid': 'The provided data is invalid.' });
}
for (let i = 0; i < files.length; i += 1) {
let newDocument = await Document.create({ name: file.filename }).fetch();
ids.push(newDocument.id);
}
resolve(ids);
});
});
}
Please note that the above function is just clarifying the use of promises.
It can be implemented in various ways specifically if we want to optimise it.
EDIT
As an example, Promise.all can be used to optimise the upload if order is not a concern, something like like this -
function upload(inputs, settings) {
return new Promise((resolve, reject) => {
const ids = [];
inputs.filesToUpload.upload(settings, async (err, files) => {
if (err) {
return reject({ 'invalid': 'The provided data is invalid.' });
}
const newDocuments = await Promise.all(files.map(file =>
Document.create({ name: file.filename }).fetch())
);
resolve(newDocuments.map(newDocument => newDocument.id));
});
});
}
I hope this helps.

NodeJS: Wait for all foreach with Promises to finish but never actually finishes

I am working with Nodejs. I have a forEach which is async as I have to wait for a result inside the forEach. As a result, I need to wait for the forEach to finish and then carry on with the result of the loop. I found several solutions for waiting for the forEach, one of them is using Promises. I did though, and these promises are created, however, the code after the forEach (and therefore the promises) are finished, is never actually executed (console.log is not printed). And the NodeJS function just ends without any errors.
Here is my Code:
var Client = require('ssh2').Client;
// eslint-disable-next-line no-undef
var csv = require("csvtojson");
// eslint-disable-next-line no-undef
var fs = require("fs");
// eslint-disable-next-line no-undef
const config = require('./config.json');
// eslint-disable-next-line no-undef
const os = require('os');
let headerRow = [];
let sumTxAmount = 0;
const filenameShortened = 'testFile';
let csvLists = [];
let csvFile;
const options = {
flags: 'r',
encoding: 'utf8',
handle: null,
mode: 0o664,
autoClose: true
}
var conn = new Client();
async function start() {
const list = await getCSVList();
let content = fs.readFileSync('./temp.json', 'utf8');
content = JSON.parse(content);
var promises = list.map(function(entry) {
return new Promise(async function (resolve, reject) {
if (!content['usedFiles'].includes(entry.filename)) {
const filename = entry.filename;
csvFile = await getCsv(filename);
csvLists.push(csvFile);
console.log('here');
resolve();
} else {
resolve();
}
})
});
console.log(promises)
Promise.all(promises)
.then(function() {
console.log(csvLists.length, 'length');
})
.catch(console.error);
}
start();
The "here" is printed once (not 8 times as the arrays length is 8), but there are 8 promises created. The lower part where I am printing the length of the array is not executed.
Can anyone tell me what I am doing wrong? Am I using Promises and forEach falsely as I have to do an await inside the forEach?
Note: getCSVList() and getCsv() are functions to get Csvs from an sftp server:
function getCSVList() {
return new Promise((resolve, reject) => {
conn.on('ready', function () {
conn.sftp(function (err, sftp) {
if (err) throw err;
sftp.readdir(config.development.pathToFile, function (err, list) {
if(err) {
console.log(err);
conn.end();
reject(err);
} else {
console.log('resolved');
conn.end();
resolve(list);
}
})
})
}).connect({
host: config.development.host,
port: config.development.port, // Normal is 22 port
username: config.development.username,
password: config.development.password
// You can use a key file too, read the ssh2 documentation
});
})
}
function getCsv(filename) {
return new Promise((resolve, reject) => {
conn.on('ready', function () {
conn.sftp(function (err, sftp) {
if (err) reject(err);
let csvFile = sftp.createReadStream(`${config.development.pathToFile}/${filename}`, options);
// console.log(csvFile);
conn.end();
resolve(csvFile);
})
}).connect({
host: config.development.host,
port: config.development.port, // Normal is 22 port
username: config.development.username,
password: config.development.password
// You can use a key file too, read the ssh2 documentation
});
});
}
The output in my console from all the console logs is:
`➜ node server.js
resolved
[ Promise { <pending> },
Promise { <pending> },
Promise { <pending> },
Promise { <pending> },
Promise { <pending> },
Promise { <pending> },
Promise { <pending> },
Promise { <pending> } ]
here`

Break up your problem into pieces, confirming they work along the way.
You are not using the stream correctly, among other things.
I made a working example with ssh2-sftp-client so you can maybe use it as a starting point.
Working example :
var fs = require('fs'); var _ = require('underscore');
var SFTPClient = require('ssh2-sftp-client');
const CONFIG = {
"SSH_CONN_OPTS":{"host":"XXXXXXXX","port":22,"username":"XXXXXXXX","password":"XXXXXXXX"},
"CSV_DIRECTORY":"/var/www/html"
}
//---------------
//.:The order-logic of the script is here
function StartScript(){
console.log("[i] SSH Connection")
LoadValidationFile(()=>{
InitializeSFTP(()=>{ console.log("[+] SSH Connection Established")
ListRemoteDirectory((list)=>{ console.log(`[i] Total Files # ${CONFIG.CSV_DIRECTORY} : ${list.length}`)
//console.log(list) //:now you have a 'list' of file_objects, you can iterate over to check the filename
var csvFileList = [] //store the names of the files you will request after
_.each(list,(list_entry)=>{ console.log(list_entry)
if(!CONFIG.USED_FILES.includes(list_entry.name)){ csvFileList.push(list_entry.name) }
})
//:now loop over the new final list of files you have just validated for future fetch
GenerateFinalOutput(csvFileList)
})
})
})
}
//.:Loads your validation file
function LoadValidationFile(cb){
fs.readFile(__dirname+'/temp.json','utf8',(err,data)=>{ if(err){throw err}else{
var content = JSON.parse(data)
CONFIG.USED_FILES = content.usedFiles
cb()
}})
}
//.:Connects to remote server using CONFIG.SSH_CONN_OPTS
function InitializeSFTP(cb){
global.SFTP = new SFTPClient();
SFTP.connect(CONFIG.SSH_CONN_OPTS)
.then(()=>{cb()})
.catch((err)=>{console.log("[!] InitializeSFTP :",err)})
}
//.:Get a list of files from a remote directory
function ListRemoteDirectory(cb){
SFTP.list(`${CONFIG.CSV_DIRECTORY}`)
.then((list)=>{cb(list)})
.catch((err)=>{console.log("[!] ListRemoteDirectory :",err)})
}
//.:Get target file from remote directory
function GetRemoteFile(filename,cb){
SFTP.get(`${CONFIG.CSV_DIRECTORY}/${filename}`)
.then((data)=>{cb(data.toString("utf8"))}) //convert it to a parsable string
.catch((err)=>{console.log("[!] ListRemoteDirectory :",err)})
}
//-------------------------------------------
var csvLists = []
function GenerateFinalOutput(csv_files,current_index){ if(!current_index){current_index=0}
if(current_index!=csv_files.length){ //:loop
var csv_file = csv_files[current_index]
console.log(`[i] Loop Step #${current_index+1}/${csv_files.length} : ${csv_file}`)
GetRemoteFile(csv_file,(csv_data)=>{
if(csv_data){csvLists.push(csv_data)}
current_index++
GenerateFinalOutput(csv_files,current_index)
})
}else{ //:completed
console.log("[i] Loop Completed")
console.log(csvLists)
}
}
//------------
StartScript()
Good luck!

Promise.all is a method that will return a promise object, but you are not waiting for your start method to execute.
function getCSVList() {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve([1, 2, 3, 4]);
}, 1000);
});
}
function getCsv(params) {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve(params);
}, 1000);
});
}
async function start() {
const list = await getCSVList();
const promises = list.map(item => {
return new Promise(async function (resolve, reject) {
const csvFile = await getCsv(item);
console.log('here');
resolve(csvFile);
});
});
return Promise.all(promises);
}
start().then(res => {
console.log(res);
});

Calling async function multiple times

So I have a method, which I want to call multiple times in a loop. This is the function:
function PageSpeedCall(callback) {
var pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${websites[0]}&strategy=mobile&key=${keys.pageSpeed}`;
// second call
var results = '';
https.get(pagespeedCall, resource => {
resource.setEncoding('utf8');
resource.on('data', data => {
results += data;
});
resource.on('end', () => {
callback(null, results);
});
resource.on('error', err => {
callback(err);
});
});
// callback(null, );
}
As you see this is an async function that calls the PageSpeed API. It then gets the response thanks to the callback and renders it in the view. Now how do I get this to be work in a for/while loop? For example
function PageSpeedCall(websites, i, callback) {
var pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${websites[i]}&strategy=mobile&key=${keys.pageSpeed}`;
// second call
var results = '';
https.get(pagespeedCall, resource => {
resource.setEncoding('utf8');
resource.on('data', data => {
results += data;
});
resource.on('end', () => {
callback(null, results);
});
resource.on('error', err => {
callback(err);
});
});
// callback(null, );
}
var websites = ['google.com','facebook.com','stackoverflow.com'];
for (let i = 0; i < websites.length; i++) {
PageSpeedCall(websites, i);
}
I want to get a raport for each of these sites. The length of the array will change depending on what the user does.
I am using async.parallel to call the functions like this:
let freeReportCalls = [PageSpeedCall, MozCall, AlexaCall];
async.parallel(freeReportCalls, (err, results) => {
if (err) {
console.log(err);
} else {
res.render('reports/report', {
title: 'Report',
// bw: JSON.parse(results[0]),
ps: JSON.parse(results[0]),
moz: JSON.parse(results[1]),
// pst: results[0],
// mozt: results[1],
// bw: results[1],
al: JSON.parse(results[2]),
user: req.user,
});
}
});
I tried to use promise chaining, but for some reason I cannot put it together in my head. This is my attempt.
return Promise.all([PageSpeedCall,MozCall,AlexaCall]).then(([ps,mz,al]) => {
if (awaiting != null)
var areAwaiting = true;
res.render('admin/', {
title: 'Report',
// bw: JSON.parse(results[0]),
ps: JSON.parse(results[0]),
moz: JSON.parse(results[1]),
// pst: results[0],
// mozt: results[1],
// bw: results[1],
al: JSON.parse(results[2]),
user: req.user,
});
}).catch(e => {
console.error(e)
});
I tried doing this:
return Promise.all([for(let i = 0;i < websites.length;i++){PageSpeedCall(websites, i)}, MozCall, AlexaCall]).
then(([ps, mz, al]) => {
if (awaiting != null)
var areAwaiting = true;
res.render('admin/', {
title: 'Report',
// bw: JSON.parse(results[0]),
ps: JSON.parse(results[0]),
moz: JSON.parse(results[1]),
// pst: results[0],
// mozt: results[1],
// bw: results[1],
al: JSON.parse(results[2]),
user: req.user,
});
}).catch(e => {
console.error(e)
});
But node just said it's stupid.
And this would work if I didn't want to pass the websites and the iterator into the functions. Any idea how to solve this?
To recap. So far the functions work for single websites. I'd like them to work for an array of websites.
I'm basically not sure how to call them, and how to return the responses.

It's much easier if you use fetch and async/await
const fetch = require('node-fetch');
async function PageSpeedCall(website) {
const pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${website}&strategy=mobile&key=${keys.pageSpeed}`;
const result = await fetch(pagespeeddCall);
return await result.json();
}
async function callAllSites (websites) {
const results = [];
for (const website of websites) {
results.push(await PageSpeedCall(website));
}
return results;
}
callAllSites(['google.com','facebook.com','stackoverflow.com'])
.then(results => console.log(results))
.error(error => console.error(error));
Which is better with a Promise.all
async function callAllSites (websites) {
return await Promise.all(websites.map(website => PageSpeedCall(website));
}

Starting on Node 7.5.0 you can use native async/await:
async function PageSpeedCall(website) {
var pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${website}&strategy=mobile&key=${keys.pageSpeed}`;
return await promisify(pagespeedCall);
}
async function getResults(){
const websites = ['google.com','facebook.com','stackoverflow.com'];
return websites.map(website => {
try {
return await PageSpeedCall(website);
}
catch (ex) {
// handle exception
}
})
}
Node http "callback" to promise function:
function promisify(url) {
// return new pending promise
return new Promise((resolve, reject) => {
// select http or https module, depending on reqested url
const lib = url.startsWith('https') ? require('https') : require('http');
const request = lib.get(url, (response) => {
// handle http errors
if (response.statusCode < 200 || response.statusCode > 299) {
reject(new Error('Failed to load page, status code: ' + response.statusCode));
}
// temporary data holder
const body = [];
// on every content chunk, push it to the data array
response.on('data', (chunk) => body.push(chunk));
// we are done, resolve promise with those joined chunks
response.on('end', () => resolve(body.join('')));
});
// handle connection errors of the request
request.on('error', (err) => reject(err))
})
}

Make PageSpeedCall a promise and push that promise to an array as many times as you need, e.g. myArray.push(PageSpeedCall(foo)) then myArray.push(PageSpeedCall(foo2)) and so on. Then you Promise.all the array.
If subsequent asynch calls require the result of a prior asynch call, that is what .then is for.
Promise.all()
Promise.all([promise1, promise2, promise3]).then(function(values) {
console.log(values);
});

Bluebird.js Promise not chaining .then()s

socket.on('new video', function(data) {
console.log("New video")
var addToCueP = Promise.promisify(addToCue)
var getCueFromDbP = Promise.promisify(getCueFromDb)
addToCueP(data.id, socket.nickname)
.then(function() {
return getCueFromDbP();
})
.then(function() {
console.log("Emit change video")
io.sockets.emit('change video', {id: data.id, title: data.title, nick: socket.nickname});
})
});
I am using Bluebird for promises but I'm having issues, the addToCue function gets called but the other functions after for example the getCuefromDb and the Socket emitter isnt getting triggered, does anyone have any idea where I'm going wrong?
addToCue()
var addToCue = function(id, user) {
console.log("Add to cue")
var video = new Video();
video.id = id;
video.user = user;
video.save(function(err, data) {
if (err) {
console.log(err)
} else {
return true
}
});
}
getCueFromDb()
var getCueFromDb = function(callback) {
Video.find({}).exec(function(err, videos) {
if (err) {
return err;
}
if (videos.length) {
cue = []; // empty array
videos.forEach(function(video) {
cue.push(video.id) // push all the videos from db into cue array
});
io.sockets.emit('send cue', {cue: cue});
console.log("getCueFromDb", cue)
if (callback) {
callback();
return
}
else {
return
}
}
else {
cue = [];
io.sockets.emit('send cue', {cue: cue});
console.log("getCueFromDb (no videos)", cue)
if (callback)
callback()
else
return
}
})
}

addToCue() does not accept the right types of arguments in order to call .promisify() on it- therefore promisify does not work properly with it.
The last argument to addToCue() MUST be a callback that is called when the async operation is done and the first argument to that callback must be an error value and the second argument can be a result.
Now, since you control the code to addToCue(), you should just change its code to return a promise that is resolved/rejected based on the underlying async operation and not use .promisify() at all.
var addToCue = function(id, user) {
return new Promise(function(resolve, reject) {
console.log("Add to cue")
var video = new Video();
video.id = id;
video.user = user;
video.save(function(err, data) {
if (err) {
reject(err);
} else {
resolve(data);
}
});
});
}
Alternatively, you could use .promisify() on video.save() or .promisifyAll() on the Video object.
Or, if your Video interface already supports promises, you can just use it's native support and return that promise.

Parse Promise 'when' returns undefined (Javascript)

I'm trying to follow the 'when' example on Parse JavaScript SDK: Parse.Promise
with the following code:
GetFromDB2 = function(id) {
var DB2 = Parse.Object.extend("DB2");
var q = new Parse.Query(DB2);
q.get(id, {
success: function(res) {
return Parse.Promise.as(10);
},
error: function(res, err) {
console.log( err);
}
});
}
GetData = function() {
var DB1 = Parse.Object.extend("DB1");
var query = new Parse.Query(DB1);
query.equalTo("param", "close");
query.find().then(function(results) {
var promises = [];
_.each(results, function(res) {
promises.push(GetFromDB2(res.get("user")));
});
Parse.Promise.when(promises).then(function() {
console.log(arguments); // expect: [10, 10, ...]
})
});
};
The length of the array arguments is correct but not sure why its values are undefined.

As written, GetFromDB2() returns undefined. To deliver a value, it must return either a value or a promise.
At its simplest, to deliver 10, you could write :
function GetFromDB2(id) {
return 10;
}
But to be asynchronous, and still deliver 10, you need to return a promise that will resolve to 10 :
function GetFromDB2(id) {
return Parse.Promise.as(10);
}
Or the .get(id) query you really want :
function GetFromDB2(id) {
var DB2 = Parse.Object.extend('DB2');
var q = new Parse.Query(DB2);
return q.get(id).then(function(res) {
return 10;
});
//omit `.then(...)` entirely to deliver `res`
}
Now GetData() can be written as follows :
function GetData() {
var DB1 = Parse.Object.extend('DB1');
var query = new Parse.Query(DB1);
query.equalTo('param', 'close');
return query.find().then(function(results) {
var promises = _.map(results, function(res) {
return GetFromDB2(res.get('user'));
});
Parse.Promise.when(promises).then(function() {
console.log(arguments); // expect: [10, 10, ...]
}, function(err) {
console.log(err);
return err;
});
});
};
Notes:
promises = _.map(...) is more elegant than _.each(...) plus `promises.push(...).
moving the error handler into GetData() allows a greater range of possible errors to be handled, eg an error arising from query.find().
By returning a promise, GetData()'s caller is also informed of the eventual asynchronous outcome.