Develop Reference

Who can explain nodejs promises in a loop to a PHP programmer

I have searched high and low but can't get my head around promises. What I do understand is how to define one promise and use its result by using .then.
What I do not understand is how I can create a loop to query the database for different blocks of records. This is needed due to a limit set on the number of records to query.
The predefined promise api call is used like this:
let getRecords = (name) => {
return new Promise((resolve,reject) => {
xyz.api.getRecords(name, 1000, 1000, function(err, result){
// result gets processed here.
resolve(//somevariables here);
});
)};
going with what I am used to, I tried:
for (let i=1; i<90000; i+=500) {
xyz.api.getRecords('james', i, 500, function(err, result){
// result gets processed here.
});
}
But then I can't access the information (could be my wrong doing)
Also tried something like this:
function getRecords(name,i){
xyz.api.getRecords(name, i, 500, function(err, result){
// result gets processed here.
});
};
for (let i=1; i<90000; i+=500) {
var someThing = getRecords('james', i);
}
All tutorials only seem to use one query, and process the data.
How do I call the api function multiple times with different arguments, collect the data and process it once everything is retrieved?
Only thing I can think of is, to write info to a file, terrible thought.

Using async/await
(async () => {
function getRecords(name,i){
// create new Promise so you can await for it later
return new Promise((resolve, reject) => {
xyz.api.getRecords(name, i, 500, function(err, result){
if(err) {
return reject(err);
}
resolve(result);
});
});
}
for (let i = 1; i < 90000; i += 500) {
// wait for the result in every loop iteration
const someThing = await getRecords('james', i);
}
})();
To handle errors you need to use try/catch block
try {
const someThing = await getRecords('james', i);
} catch(e) {
// handle somehow
}
Using only Promises
function getRecords(name, i) {
// create Promise so you can use Promise.all
return new Promise((resolve, reject) => {
xyz.api.getRecords(name, i, 500, function (err, result) {
if (err) {
return reject(err);
}
resolve(result);
});
});
}
const results = [];
for (let i = 1; i < 90000; i += 500) {
// push Promise's to array without waiting for results
results.push(getRecords("james", i));
}
// wait for all pending Promise's
Promise.all(results).then((results) => {
console.log(results);
});
let count = 0;
function getRecords(name, i) {
return new Promise((resolve, reject) => {
setTimeout(() => {
// example results
resolve((new Array(10)).fill(0).map(() => ++count));
}, 100);
});
}
const results = [];
for (let i = 1; i < 9000; i += 500) {
results.push(getRecords("james", i));
}
Promise.all(results).then((results) => {
console.log("Results:", results);
console.log("Combined results:",[].concat(...results));
});
To handle errors you need to use .catch() block
Promise.all(results).then((results) => { ... }).catch((error) => {
// handle somehow
});

By returning a promise and calling your asynchronous function inside, you can resolve the result and then use it this way:
function getRecords (name, i) {
return new Promise((resolve, reject) => {
xyz.api.getRecords(name, i, 500, (err, result) => {
if (err) {
reject(err);
} else {
resolve(result);
}
});
});
}
for (let i = 1; i < 90000; i * 500) {
getRecords('james', i)
.then(result => {
// do stuff with result
})
}
Or, using async / await syntax:
async function getRecords (name, i) {
return new Promise((resolve, reject) => {
xyz.api.getRecords(name, i, 500, (err, result) => {
if (err) {
reject(err);
} else {
resolve(result);
}
});
});
}
// this needs to happen inside a function, node does not allow top level `await`
for (let i = 1; i < 90000; i *= 500) {
const result = await getRecords('james', i);
// do stuff
}
Get all of your records at once
const requests = [];
for (let i = 1; i < 90000; i * 500) {
requests.push(getRecords('james', i));
}
const results = await Promise.all(requests);

How to do a callback in nodeJS

I'm working on a nodeJS script and I would like to know how to execute a function after an another one.
Because actually i need to save in my database some data and then retrieve them. However for the moment my retrieve is executed before my save :/
Have already looked on internet there is a lot of example I tried them but for the moment no one worked ... I should probably do something wrong if somebody could help me on it :)
function persistMAP(jsonData, callback) {
console.log(jsonData);
//Deck persistance
for (var i = 0; i < 1; i++) {
(function(i) {
var rowData = new DeckDatabase({
_id: new mongoose.Types.ObjectId(),
DeckNumber: Number(jsonData.Deck[i].DeckNumber),
x: Number(jsonData.Deck[i].x),
y: Number(jsonData.Deck[i].y),
});
rowData.save(function(err) {
if (err) return console.log(err);
for (var i = 0; j = jsonData.Units.length, i < j; i++) {
(function(i) {
var unit = new MapDatabase({
//UnitID: mongoose.ObjectId(jsonData.Units[i].UnitID),
UnitID: jsonData.Units[i].UnitID,
TypeID: Number(jsonData.Units[i].TypeID),
x: Number(jsonData.Units[i].x),
y: Number(jsonData.Units[i].y),
_id: mongoose.Types.ObjectId(jsonData.Units[i].Code + 'dd40c86762e0fb12000003'),
MainClass: jsonData.Units[i].MainClass,
Orientation: jsonData.Units[i].Orientation,
Postion: jsonData.Units[i].Postion,
Deck: String(rowData._id)
});
unit.save(function(err) {
if (err) return console.log(err);
console.log('save');
});
})(i);
}
});
})(i);
}
callback();
};
app.get("/Map", function(req, res) {
console.log("got");
var urlTempBox = 'http://localhost:3000/MapCreate';
DeckDatabase.find(null, function(err, data) {
if (err) {
throw (err);
}
if (data.length != 0) {
MapDatabase.find()
.populate('Deck')
.exec(function(err, finalData) {
res.send(finalData);
});
} else {
request(urlTempBox, data, function(error, response, body) {
if (error) {
throw (error);
} else {
var jobj = JSON.parse(response.body);
console.log("persist begin");
persistMAP(jobj, function() {
console.log('retrieve Done');
});
}
});
}
});

You can use javascript callbacks like this:
User.findById(user_id,function(err,data){
//Inside this you can call another function.
});
console.log("Hello"); //this statement won't wait for the above statement
The more good approach would be use promises.
You can also use async await function to handle asynchronus tasks.
Async Await Style
async function getData(){
let user_data=await User.findById(user_id);
let user_videos=await Videos.findById(user_data._id); //user_data._id is coming from the above statement
}
But you can only use await in async methods.
Hope it helps.

why my array is empty?

i can not figure out why my array is empty while the console is showing output:
I'm fetching data from dynamodb and trying to make a modified response json array i can print the values of dynamodb response but when i push it into array the returned array is empty here is my code.
Here is Util.js file
const dynamo = require('./dynamo.js')
var myjson = [];
exports.myjson = [];
exports.maketimetabledata = function(callback) {
var table = "timetable";
for (i = 1; i <= 2; i++) {
dynamo.docClient.get({
TableName: table,
Key: {
"id": i
}
}, function(err, data) {
if (err) {
console.error("Unable to read item. Error JSON:", JSON.stringify(err, null, 2));
return err;
} else {
dynamores = JSON.parse(JSON.stringify(data));
myjson.push(i);
}
});
console.log(myjson);
}
callback("done");
};
and here is my partial index file:
app.get('/', (req, res) => {
var myjson = [];
util.maketimetabledata(function(returnvalue) {
if (returnvalue) {
console.log(util.myjson);
}
});
});
output :
[] [] []
undefined
{ response from dynamo db for debugging purpose}

Assuming dynamo.docClient.get() returns a Promise, You can use Promise.all() to wait for all the promise to complete then invoke the callback method.
//Use Promise.all()
exports.maketimetabledata = function (callback) {
var table = "timetable";
for (i = 1; i <= 2; i++) {
var table = "timetable";
var promiseArray = [];
for (i = 1; i <= 2; i++) {
var promise = dynamo.docClient.get({
//Your code
});
promiseArray.push(promise)
}
console.log(myjson);
}
Promise.all(promiseArray).then(function () {
callback(myjson);
});
};
//Usage
util.maketimetabledata(function (myjson) {
if (returnvalue) {
console.log(myjson);
}
});
If the method is not returning Promise
var promise = new Promise(function (resolve, reject) {
dynamo.docClient.get({
TableName: table,
Key: {
"id": i
}
}, function (err, data) {
if (err) {
console.error("Unable to read item. Error JSON:", JSON.stringify(err, null, 2));
reject();
return err;
} else {
dynamores = JSON.parse(JSON.stringify(data));
myjson.push(i);
resolve();
}
});
});

Iterate array and wait for promises

How can I iterate through an array of data using Promises and returning data? I have seen some promises.push(asyncFunc) methods but some of the entries from my array will fail so from what I gather I can't use that.
var filesFromDisk = [
'41679_4_2015-09-06_17-02-12.mp4',
'41679_4_2015-09-06_17-02-12.smil',
'41680_4_2015-09-09_10-44-05.mp4'
];
start(filesFromDisk)
.then((data) => {
console.log(data); // Want my data here
});
I start start(dbFiles) from another file which is why I want the data returned there.
function start(dbFiles) {
var listOfFiles = [],
promises = [];
return new Promise((fulfill, reject) => {
for (var i = 0; i < dbFiles.length; i++) {
getMp4(dbFiles[i])
.then((data) => {
listOfFiles = listOfFiles.concat(data);
console.log(listOfFiles);
})
}
fulfill(listOfFiles) // Need to happen AFTER for loop has filled listOfFiles
});
}
So for every entry in my array I want to check if the file with the new extension exists and read that file. If the file with extension does not exist I fulfill the original file. My Promise.all chain works and all the data is returned in for loop above (getMp4(dbFiles[i]))
function getMp4(filename) {
var mp4Files = [];
var smil = privateMethods.setSmileExt(localData.devPath + filename.toString());
return new Promise((fulfill, reject) => {
Promise.all([
privateMethods.fileExists(smil),
privateMethods.readTest(smil)
]).then(() => {
readFile(filename).then((files) => {
fulfill(files)
});
}).catch((err) => {
if (!err.exists) fulfill([filename]);
});
});
}
function readFile(filename){
var filesFromSmil = [];
return new Promise((fulfill, reject) => {
fs.readFile(localData.devPath + filename, function (err, res){
if (err) {
reject(err);
}
else {
xmlParser(res.toString(), {trim: true}, (err, result) => {
var entry = JSON.parse(JSON.stringify(result.smil.body[0].switch[0].video));
for (var i = 0; i < entry.length; i++) {
filesFromSmil.push(privateMethods.getFileName(entry[i].$.src))
}
});
fulfill(filesFromSmil);
}
});
});
};
Methods in the Promise.all chain in getMp4 - have no problems with these that I know.
var privateMethods = {
getFileName: (str) => {
var rx = /[a-zA-Z-1\--9-_]*.mp4/g;
var file = rx.exec(str);
return file[0];
},
setSmileExt: (videoFile) => {
return videoFile.split('.').shift() + '.smil';
},
fileExists: (file) => {
return new Promise((fulfill, reject) => {
try {
fs.accessSync(file);
fulfill({exists: true})
} catch (ex) {
reject({exists: false})
}
})
},
readTest: (file) => {
return new Promise((fulfill, reject) => {
fs.readFile(file, (err, res) => {
if (err) reject(err);
else fulfill(res.toString());
})
})
}
}

If you need them to run in parallel, Promise.all is what you want:
function start(dbFiles) {
return Promise.all(dbFiles.map(getMp4));
}
That starts the getMp4 operation for all of the files and waits until they all complete, then resolves with an array of the results. (getMp4 will receive multiple arguments — the value, its index, and a a reference to the dbFiles arary — but since it only uses the first, that's fine.)
Usage:
start(filesFromDisk).then(function(results) {
// `results` is an array of the results, in order
});
Just for completeness, if you needed them to run sequentially, you could use the reduce pattern:
function start(dbFiles) {
return dbFiles.reduce(function(p, file) {
return p.then(function(results) {
return getMp4(file).then(function(data) {
results.push(data);
return results;
});
});
}, Promise.resolve([]));
}
Same usage. Note how we start with a promise resolved with [], then queue up a bunch of then handlers, each of which receives the array, does the getMp4 call, and when it gets the result pushes the result on the array and returns it; the final resolution value is the filled array.

Syncano Codebox - Call API - parse JSON - get Reference - Save new Objects

I am using Syncano as a baas, where I am trying to call an external API to receive a JSON array. This JSON needs to be parsed and afterwards stored in syncano. Before that I need to receive the reference object from the DB to link it to the new team object.
I receive the team (json) array & reference object successfully. But I am unable to get the new data stored, as only 12-14 teams (has to be 18) get saved.
I tried this & that with promises but it didn´t work out. Anyone a good advise how to rewrite the code to store all data? Thank you - here is what I have so far...
//TODO: get from ARGS when executing this codebox
var teamKey = 394;
var requestURL = 'http://api.football-data.org/v1/soccerseasons/' + teamKey + "/teams";
var request = require("request");
var Syncano = require('syncano');
var Promise = require('bluebird');
var account = new Syncano({
accountKey: "abc"
});
var promises = [];
//from: http://docs.syncano.io/v1.0/docs/data-objects-filtering
//"_eq" means equals to
var filter = {
"query": {
"apikey": {
"_eq": apiKey
}
}
};
request({
headers: {
'X-Auth-Token': 'abc'
},
url: requestURL,
'Content-Type': 'application/json;charset=utf-8;',
method: 'GET',
}, function(error, response, body) {
if (error) {
console.log(error);
} else {
var json = JSON.parse(body);
var teamArray = json.teams;
var newObject;
account.instance('instance').class('competition').dataobject().list(filter)
.then(function(compRes) {
var competitionID = compRes.objects[0].id;
for (var i = 0; i < teamArray.length; i++) {
newObject = {
"name": teamArray[i].name,
"nameshort": teamArray[i].code,
"logo": teamArray[i].crestUrl,
"competition": competitionID
};
(account.instance('instance').class('teams').dataobject().add(newObject).then(function(res) {
console.log(res);
}).catch(function(err) {
console.log("Error eq: " + err);
})
);
}
}).catch(function(err) {
console.log(err);
});
}
});

The issue might be you are finishing the request process before all the save calls are made, you could try Promise.all():
account.instance('instance').class('competition').dataobject().list(filter)
.then(function(compRes) {
var competitionID = compRes.objects[0].id, promises=[];
for (var i = 0; i < teamArray.length; i++) {
newObject = {
"name": teamArray[i].name,
"nameshort": teamArray[i].code,
"logo": teamArray[i].crestUrl,
"competition": competitionID
};
promises.push(account.instance('instance').class('teams').dataobject().add(newObject).then(function(res) {
console.log(res);
}).catch(function(err) {
console.log("Error eq: " + err);
})
);
}
return Promise.all(promises);
}).catch(function(err) {
console.log(err);
});
if too many parallel calls at a time is the issue then chain one call after other:
account.instance('instance').class('competition').dataobject().list(filter)
.then(function(compRes) {
var competitionID = compRes.objects[0].id, promise = Promise.resolve();
function chainToPromise(promise, teamObj, waitTime){
waitTime = waitTime || 500;
return promise.then(function(){
return new Promise(function(resolve, reject){
setTimeout(resolve, waitTime);
});
}).then(function(){
return account.instance('instance').class('teams').dataobject().add(teamObj);
}).then(function(res) {
console.log(res);
}).catch(function(err) {
console.log("Error eq: " + err);
});
}
for (var i = 0; i < teamArray.length; i++) {
newObject = {
"name": teamArray[i].name,
"nameshort": teamArray[i].code,
"logo": teamArray[i].crestUrl,
"competition": competitionID
};
promise = chainToPromise(promise, newObject);
}
return promise;
}).catch(function(err) {
console.log(err);
});