I am developing a backend server code with NodeJS. What the code does is, periodically connect to a REST API, request updates and then write to my database.
I have no way of getting delta of the data from the API, so I drop my collection from MongoDB and then just insert the newly pulled data.
I implemented promises to make sure that the dependent methods are executed only after the previous methods resolve. This however doesn't seem to work as I anticipated.
So, I drop the collection and insert, this works. But the following method seems to execute before the new data is populated. It sometime works, when I have some new console.log statements which seems to induce a slight delay ever so slightly to make it all work.
setTimeout function didn't seem to help. Any suggestions?
Here is a sanitized version of the code: https://jsfiddle.net/ppbfrozg/
var request = require("request");
var q = require('q');
function authenticate() {
var deferred = q.defer();
request(options, function(error, response, body) {
if (error) throw new Error(error);
deferred.resolve(JSON.parse(body).token);
});
return deferred.promise;
}
function getData(token) {
var deferred = q.defer();
request(options, function(error, response, body) {
if (error) throw new Error(error);
deferred.resolve(JSON.parse(body).token);
});
return deferred.promise;
}
function insertDataInMongo(a) {
var deferred = q.defer();
var MongoClient = require('mongodb').MongoClient;
var url = 'mongodb://localhost/myDB';
var token = a[1];
MongoClient.connect(url, function(err, db) {
if (err) return deferred.reject(new Error(err))
console.log("connected for insert");
var apiData = JSON.parse(a[0]).data;
if (JSON.parse(a[0]).data) {
db.collection('MediaData').insert(apiData);
console.log("Records Inserted");
} else {
db.collection('Details').drop();
db.collection('Details').insert(JSON.parse(a[0]));
console.log("Records Inserted");
}
deferred.resolve(token);
});
return deferred.promise;
}
function getMedia(dataContext) {
var deferred = q.defer();
var cursor = dataContext[0];
var token = dataContext[1];
if (cursor !== null) {
console.log("Inside cursor not null");
cursor.forEach(function(data) {
insertDataInMongo(data);
})
}
return deferred.promise;
}
function check(array, attr, value) {
for (var i = 0; i < array.length; i += 1) {
if (array[i][attr] === value) {
return false;
}
}
return true;
}
function get_value(array, attr) {
for (var i = 0; i < array.length; i += 1) {
if (array[i].hasOwnProperty(attr)) {
return array[i][attr];
}
}
}
function getNames(token) {
var deferred = q.defer();
var MongoClient2 = require('mongodb').MongoClient;
var url = 'mongodb://localhost/myDB';
console.log("going to get Data");
MongoClient2.connect(url, function(err, db) {
if (err) return deferred.reject(new Error(err));
console.log("connected for select");
var data = db.collection('Details').find();
var dataContext = [data, token, 0, 0, 0, 0, 0, 0, 0, null];
deferred.resolve(dataContext);
});
return deferred.promise;
}
function convertDate(date) {
var yyyy = date.getFullYear().toString();
var mm = (date.getMonth() + 1).toString();
var dd = (date.getDate() - 3).toString();
var mmChars = mm.split('');
var ddChars = dd.split('');
return yyyy + '-' + (mmChars[1] ? mm : "0" + mmChars[0]) + '-' + (ddChars[1] ? dd : "0" + ddChars[0]);
}
authenticate()
.then(getData)
.then(insertDataInMongo)
.then(getNames)
.then(getMedia);
This should work. Let me know if any issue.
function insertDataInMongo(a) {
var deferred = q.defer();
var MongoClient = require('mongodb').MongoClient;
var url = 'mongodb://localhost/myDB';
var token = a[1];
MongoClient.connect(url, function(err, db) {
if (err) return deferred.reject(new Error(err))
console.log("connected for insert");
var apiData = JSON.parse(a[0]).data;
if (JSON.parse(a[0]).data) {
db.collection('MediaData').insert(apiData, function(){
console.log("Records Inserted");
return deferred.resolve(token);
});
} else {
db.collection('Details').drop(function(error, result){//Callback function that executes after drop operation has completed.
if(error){
return deferred.reject(error);//Reject the promise if there was an error
}
db.collection('Details').insert(JSON.parse(a[0]), function(err, res){//Callback function that executes after insert operation has completed.
if(err){
return deferred.reject(err);//Reject the promise if there was an error
}
console.log("Records Inserted");
return deferred.resolve(token);
});
});
}
});
return deferred.promise;
}
From what I see from nodeJS driver API for MongoDB :
https://mongodb.github.io/node-mongodb-native/api-generated/collection.html
hint : db.collection.drop is asynchronous, so you have to use a callback with it
db.collection('Details').drop(function(err, result){
// Do anything AFTER you dropped your collection
}
but since you are using promises, you should use something like that:
authenticate()
.then(getData)
.then(db.collection('Details').drop)
.then(insertDataInMongo)
.then(getNames)
.then(getMedia);
or if you want to really keep the same code format :
function dropCollectionInMongo() {
db.collection('Details').drop()
}
authenticate()
.then(getData)
.then(dropCollectionInMongo)
.then(insertDataInMongo)
.then(getNames)
.then(getMedia);
Related
This is the async function:
async function getpackages(conn, childId, callback) {
var childId = childId;
var request6 = new sql.Request(conn);
var packageQuery = "select OrderId,ChildID from dbo.tbl_Scheduler where NoOfMealsLeft>0 and ChildId=" + childId;
await request6.query(packageQuery, function (err, packagelist) {
if (!err && packagelist.recordsets.length > 0) {
console.log("Error:" + err + "Result:" + util.inspect(packagelist.recordsets[0]));
var orderdetail_ = [];
for (i = 0; i < packagelist.recordsets[0].length; i++) {
orderdetail_.push(packagelist.recordsets[0][i].OrderId);
}
console.log("-->" + orderdetail_);
callback(null, packagelist.recordsets[0]);
} else if (packagelist.recordsets.length < 1) {
callback("Not a valid id input", null);
}
});
};
I need to call the orderdetails_ array in the query. The array contains four data and I need to iterate over 4 data one by one, using the or in the SQL query.
module.exports.newscheduledmeal = function (req, res, next, callback) {
let entered_date = req.query.date;
let childId = req.query.id;
let current_date = new Date().toISOString().slice(0, 10);
if (entered_date < current_date) {
return callback('Please enter date more than or equal to current date.', null);
} else
var conn = new sql.ConnectionPool(dbConfig);
try {
conn.connect().then(function () {
var request = new sql.Request(conn);
getpackages(conn, childId, function (err, orderid) {
if (err) {
callback(err, null);
} else
var PackageidQuery = "select PackageId from dbo.tbl_Order where OrderId=";
request.query(PackageidQuery, function (err, packagelist) {
if (!err) {
conn.close();
callback(null, packagelist.recordsets);
} else {
conn.close();
callback("Error", null);
}
});
});
});
} catch (err) {
console.log("Exception occured:" + err);
conn.close();
callback(err, null);
}
};
I want to get the details of the array which is in getpackages to be used in the module section and specifically in the SQL query section.
I'm pretty new to async on node js. I use the waterfall method while parsing a xml file like this:
$('situation').each( function(){
var situation = [];
$(this).find('situationRecord').each( function(i){
var record = this;
async.waterfall([
function (callback){
var filter = {
startLocationCode: $(record).find('alertCMethod2SecondaryPointLocation').find('specificLocation').text(),
endLocationCode: $(record).find('alertCMethod2PrimaryPointLocation').find('specificLocation').text(),
overallStartTime: $(record).find('overallStartTime').text(),
overallEndTime: $(record).find('overallEndTime').text()
}
callback(null, filter, record);
},
function (filter, record, callback){
var timestamp = new Date().toDateInputValue();
var startDbResponse = 0;
var endDbResponse = 0;
if((filter.startLocationCode != '') && new Date(timestamp) >= new Date(filter.overallStartTime) && new Date(timestamp) <= new Date(filter.overallEndTime) ){
startDbResponse = locationCodeToGeodataRequst.geodataByLocationcode(filter.startLocationCode);
endDbResponse = locationCodeToGeodataRequst.geodataByLocationcode(filter.endLocationCode);
}
console.log("startDbResponse: ", startDbResponse);
console.log("endDbResponse: ", endDbResponse);
callback(null, filter, record, startDbResponse, endDbResponse);
},
function (filter, record, startDbResponse, endDbResponse, callback){
console.log("startDbResponse: ", startDbResponse);
console.log("endDbResponse: ", endDbResponse);
var situationRecord = createSituationRecord($, record, filter.startLocationCode, filter.endLocationCode, startDbResponse, endDbResponse);
console.log(situationRecord);
},
function (situationRecord, callback){
situation[i] = { situationRecord };
}
],
function(err, results){
console.error("There was an error by filtering the xml file");
console.error(err);
});
})
if(situation.length > 0){ //if situation is not empty
locations.push(situation);
}
})
console.log(locations);
}
In this part of the waterfall I make a request to my database with locationCodeToGeodataRequst.geodataByLocationcode(filter.startLocationCode); but startDbResponse and endDbResponse is undefined :
....
function (filter, record, callback){
var timestamp = new Date().toDateInputValue();
var startDbResponse = 0;
var endDbResponse = 0;
if((filter.startLocationCode != '') && new Date(timestamp) >= new Date(filter.overallStartTime) && new Date(timestamp) <= new Date(filter.overallEndTime) ){
startDbResponse = locationCodeToGeodataRequst.geodataByLocationcode(filter.startLocationCode);
endDbResponse = locationCodeToGeodataRequst.geodataByLocationcode(filter.endLocationCode);
}
console.log("startDbResponse: ", startDbResponse);
console.log("endDbResponse: ", endDbResponse);
callback(null, filter, record, startDbResponse, endDbResponse);
},
....
The request by it self works because a console.log in the request module show the correct data. So I don't understand why its undefined.
This is the request module:
exports.geodataByLocationcode = function geodataByLocationcode(locationcode){
let sql = "SELECT * FROM tmc WHERE LOCATION_CODE = " + locationcode;
let query = db.query(sql, (err, result) =>{
if(err == null){
//console.log(result);
return result;
}else{
console.log("Error by getting item from db: " + err);
throw err;
}
});
}
This is a snipped of the console.log:
....
startDbResponse: undefined
endDbResponse: undefined
startDbResponse: undefined
endDbResponse: undefined
startDbResponse: 0
endDbResponse: 0
startDbResponse: 0
endDbResponse: 0
[]
//here comes the output of the requests as json objects
....
move the console log to the last function of async. As Manjeet pointed out async takes time and console prints early.
function (situationRecord, callback){
situation[i] = { situationRecord };
if(situation.length > 0){ //if situation is not empty
locations.push(situation);
}
})
console.log(locations);
}
}
I'm building a script to handle some data modifications via the Shopify API, through JS.
My strategy is pull ALL the orders from one date, and to run a function against each order, then PUT the new data up.
I've gotten pretty close, I have an array containing all the orders from a specific date. I am returning this data through promises.
I need help with the async nature of this function to work as expected. Currently, my PUT request is being ignored completely, and I'm not sure why. Here's my code:
(function main() {
var yesterday = moment().add('-1', 'days').format('YYYY-MM-DD');
var today = moment().format('YYYY-MM-DD');
var scopeStart = 'created_at_min=' + yesterday + "%2000:00";
var scopeEnd = '&created_at_max=' + today + '%2000:00';
var promise = new Promise(function(resolve, reject) {
var apiEndpoint = '/admin/orders/count.json?limit=250&status=any&' + scopeStart + scopeEnd;
Shopify.get(apiEndpoint, function(err, data) {
if (err) reject(err);
else resolve(data);
});
});
promise.then(function(data) {
console.log('promise data', data.count);
if (data.count <= 250) {
var secondPromise = new Promise(function(resolve, reject) {
var apiEndpoint = '/admin/orders.json?limit=250&status=any&' + scopeStart + scopeEnd;
Shopify.get(apiEndpoint, function(err, data) {
if (err) reject(err);
else resolve(data);
});
});
secondPromise.then(function(data) {
var array = data.orders;
async.each(array, function(order, callback) {
console.log(order.name, order.id);
var thirdPromise = new Promise(function(resolve, reject) {
var fields = generateFields(order.id);
console.log('fields', fields);
console.log('----------------');
Shopify.put('/admin/orders/' + order.id, fields, function(err, data) {
if (err) reject(err);
else resolve(data);
})
});
thirdPromise.then(function(data) {
console.log('Updated', data);
});
});
});
}
});
}());
My third promise never returns a console log. The put request does not seem to go through. Any suggestions?
Here's what I've got:
function nextAvailableFilename(path) {
return new Promise(function (resolve, reject) {
FileSystem.exists(path, function (exists) {
if (!exists) return resolve(path);
var ext = Path.extname(path);
var pathWithoutExt = path.slice(0, -ext.length);
var match = /\d+$/.exec(pathWithoutExt);
var number = 1;
if (match) {
number = parseInt(match[0]);
pathWithoutExt = pathWithoutExt.slice(0, -match[0].length);
}
++number;
nextAvailableFilename(pathWithoutExt + number + ext).then(function () {
return resolve.apply(undefined, arguments);
}, function () {
return reject.apply(undefined, arguments);
});
});
});
}
But I don't like that block at the end -- isn't there a way to 'replace' the current promise with the next one in the stack rather than having one promise resolve the next like I've done here?
Here's a version that uses promise chaining and file create to avoid the race condition. I used the bluebird promise library so I can use promises with the fs library just to simplify the code and error handling:
var Promise = require('bluebird');
var fs = Promise.promisifyAll(require('fs'));
var path = require('path');
// Creates next available xxx/yyy/foo4.txt numeric sequenced file that does
// not yet exist. Returns the new filename in the promise
// Calling this function will create a new empty file.
function nextAvailableFilename(filename) {
return fs.openAsync(filename, "wx+").then(function(fd) {
return fs.closeAsync(fd).then(function() {
return filename;
});
}, function(err) {
if (err.code !== 'EEXIST') {
// unexpected file system error
// to avoid possible looping forever, we must bail
// and cause rejected promise to be returned
throw err;
}
// Previous file exists so reate a new filename
// xxx/yyy/foo4.txt becomes xxx/yyy/foo5.txt
var ext = path.extname(filename);
var filenameWithoutExt = filename.slice(0, -ext.length);
var number = 0;
var match = filenameWithoutExt.match(/\d+$/);
if (match) {
number = parseInt(match[0], 10);
filenameWithoutExt = filenameWithoutExt.slice(0, -match[0].length);
}
++number;
// call this function again, returning the promise
// which will cause it to chain onto previous promise
return nextAvailableFilename(filenameWithoutExt + number + ext);
});
}
I came up with a solution too that doesn't depend on bluebird.promisify. It should handle the case where file creation fails for a reason other than it already exists.
function createFile(path) {
return new Promise(function (resolve, reject) {
FileSystem.open(path, 'wx', function (err, fd) {
if (err) return reject(err);
FileSystem.close(fd, function (err) {
if (err) return reject(err);
return resolve();
});
});
});
}
// todo: make more efficient by multiplying numbers by 2 or something like http://stackoverflow.com/a/1078898/65387
function nextFile(path) {
return createFile(path).then(function () {
return path;
}, function (err) {
if (err.code !== 'EEXIST') throw err; // error other than "file exists"
var ext = Path.extname(path);
var pathWithoutExt = path.slice(0, -ext.length);
var match = /\d+$/.exec(pathWithoutExt);
var number = 2;
if (match) {
number = parseInt(match[0]) + 1;
pathWithoutExt = pathWithoutExt.slice(0, -match[0].length);
}
return nextFile(pathWithoutExt + number + ext);
});
}
I have this function:
function tryStartLocalTrendsFetch(woeid) {
var userIds = Object.keys(twitClientsMap);
var isStarted = false;
for (var i = 0; i < userIds.length; i++) {
var userId = userIds[i];
var twitClientData = twitClientsMap[userId];
var isWoeidMatch = (woeid === twitClientData.woeid);
if (isWoeidMatch) {
startLocalTrendsFetch(woeid, twitClientData, function (err, data) {
if (err) {
// Couldn't start local trends fetch for userId: and woeid:
isStarted = false;
} else {
isStarted = true;
}
});
// This will not obviously work because startLocalTrendsFetch method is async and will execute immediately
if (isStarted) {
break;
}
}
}
console.log("No users are fetching woeid: " + woeid);
}
The gist of this method is that I want the line if (isStarted) { break; } to work. The reason is that if it's started it should not continue the loop and try to start another one.
I'm doing this in NodeJS.
try to use a recursive definition instead
function tryStartLocalTrendsFetch(woeid) {
var userIds = Object.keys(twitClientsMap);
recursiveDefinition (userIds, woeid);
}
function recursiveDefinition (userIds, woeid, userIndex)
var userId = userIds[userIndex = userIndex || 0];
var twitClientData = twitClientsMap[userId];
var isWoeidMatch = (woeid === twitClientData.woeid);
if (isWoeidMatch && userIndex<userIds.length) {
startLocalTrendsFetch(woeid, twitClientData, function (err, data) {
if (err) {
recursiveDefinition(userIds, woeid, userIndex + 1)
} else {
console.log("No users are fetching woeid: " + woeid);
}
});
} else {
console.log("No users are fetching woeid: " + woeid);
}
}
You may also use async (npm install async):
var async = require('async');
async.forEach(row, function(col, callback){
// Do your magic here
callback(); // indicates the end of loop - exit out of loop
}, function(err){
if(err) throw err;
});
More material to help you out: Node.js - Using the async lib - async.foreach with object