I'm having trouble with some asynchronous database queries. I'm using the Async library for Node.js. I have some code forming an object before it hits the add() function. In my code, I have a loop the executes to create the object multiple times.
for(var i = 0; i < 10; i++){
var obj.id = i + '_idstring';
add(obj);
}
As you can see, each object is slight different. I need to perform a lookup in my database to see if the object already exists (1st waterfall function) and if it does not, add it to my database (2nd waterfall function). My issue is, loaObj.id always equals 10_idstring. Is this a problem with how I'm passing the object through the callbacks? How should I be handling this?
function add(loaObj) {
async.waterfall([
function(loaObj, callback) {
var sql = "";
connection.query(sql, function(error, results, fields) {
callback(error, results, loaObj);
});
},
function(results, loaObj, callback) {
if (results.length > 0) {
sendMessage();
} else {
var sql = "";
connection.query(sql, function(error, results, fields) {
callback(error, loaObj);
});
}
}
], function(err, loaObj) {
console.log(err);
if (err) {
sendMessage();
} else {
sendMessage();
}
});
}
Because you are using Object, which will be passed by "copy of a reference", then obj.id will be overridden with every loop and it will settle down to 10_idstring -where the loop stops-.
One simple solution is to pass new Object every time like below:
for(var i = 0; i < 10; i++){
add({id: `${i}_idstring`});
}
Related
I'm trying to write a function which i can use in an if statement for logic. I've used callback functions to avoid having undefined response due to the asynchronous nature of Javascript. Am i going about this all wrong? is there a better way?
if(emailExists(Email, someCallback)){
//email does exist, do stuff
}
Basically i just want a function to tell me if an email exists in my database to return true, But even with all the precaution i took i'm still getting undefined when i run this code.
function someCallback(e){
console.log(e);
return e;
}
function emailExists(input, callback) {
pg.connect(conString, function(err, client, done){
//Connect to database
client.query('select email from users', function(err, result){
//select all emails from database
var tempArray = [];
for(var x = 0; x < result.rows.length; x++){
tempArray.push(result.rows[x].email)
} //create array of emails
callback(tempArray.includes(input));
});
});
}
I would recommend using a Promise instead, that makes working with async much more convenient. I didn't use pg-js before and don't know if it does support promises by itself - if so, you can simply use the promise returned by it; if not, below code will work for you:
function emailExists(input) {
return new Promise((resolve, reject) => {
pg.connect(conString, function (err, client, done) {
client.query('select email from users', function (err, result) {
var tempArray = [];
for (var x = 0; x < result.rows.length; x ++) {
tempArray.push(result.rows[x].email)
}
resolve(tempArray.includes(email));
});
});
});
}
You can then use it like this:
emailExists("foo#bar.baz")
.then(exists => {
if (exists) {
// email exists
} else {
// not
}
});
You just need to put your if statement in the callback.
emailExists(Email, function(hasMail){
if(hasMail){
//email does exist, do stuff
}
});
Promise would be good but not required.
I am trying to implement a for loop that iterates through a list and subsequently calls two functions, only if the first function results are found.
The issue is that the second function (search.similar) might be taking longer to fetch results.
With the code below, when I run, all of the appropriate output from (search.locate) is correct, but only the last element's results from myList are stored from the (search.similar) function.
ie. all_results = [[cat_res1,mouse_res2],[dog_res1,mouse_res2],[mouse_res1,mouse_res2]]
How do I fix this to append the right results in the right order?
ie. all_results = [[cat_res1,cat_res2],[dog_res1,dog_res2],[mouse_res1,mouse_res2]]
var search = require('./search');
var myList = ['cat','dog','mouse'];
var all_results = [];
for (i=0; i<myList.length; i++){
/* locate function*/
search.locate(myList[i], function (err, searchResult){
if (err){
console.log("Error");
return;
}
if (!searchResult){
console.log("Cannot find it");
return;
}
/*similarity function*/
/* seems to take longer*/
search.similar(myList[i], function (err, similarResult){
if (err){
return;
}
if (!similarResult){
return;
}
var res1 = searchResult.data;
var res2 = similarResult.data;
/* append results to array*/
all_results.push([res1,res2]);
}
});
}
Javascript can be thought of as asynchronous, in that the execution of particular functions do not necessarily happen synchronously, however "describing JavaScript as asynchronous is perhaps misleading. It's more accurate to say that JavaScript is synchronous and single-threaded with various callback mechanisms"
In order to accomplish your goal, though you may still get some ordering issues with the top array, you will need to wrap your .similar() call in another function that takes both arguments. Your reference to the "item" on the top search is changing:
function searchNestedSimilar(item, topRes) {
search.similar(item, function (err, similarResult) {
if (err){
return;
}
if (!topRes){
return;
}
var res1 = topRes.data
var res2 = similarResult.data
// append results to array
all_results.push([res1,res2])
}
}
function searchLocate(item) {
search.locate(item, function (err, searchResult) {
if (err){
console.log("Error");
return;
}
if (!searchResult){
console.log("Cannot find it");
return;
}
searchNestedSimilar(item, searchResults);
}
I encapsulated both calls to keep it modular, but since "item" is in the closure, you really only need the searchLocate() function to wrap your capture your item reference during iteration.
This is a good case for Promises (see Bluebird JS for example http://bluebirdjs.com/docs/getting-started.html) or you could do it with
async.map().
This page talks about it well, too. http://promise-nuggets.github.io/articles/14-map-in-parallel.html
There are many Stack Overflows discussing Promises as well. Understanding JS Promises for example.
A rough example of how to write this with a Promise:
var search = require('./search');
var myList = ['cat','dog','mouse']
var all_results = []
var Promise = require('bluebird');
var locate = Promise.promisify(search.locate);
var similar = Promise.promisify(search.similar);
for (i = 0; i < myList.length; i++){
// locate function
locate(myList[i], function (err, searchResult) {
if (err) {
console.log("Error");
return;
}
if (!searchResult){
console.log("Cannot find it");
return;
}
}).then(function(result) {
//similarity function
similar(myList[i], function (err, similarResult) {
if (err){
return;
}
if (!similarResult){
return;
}
var res1 = searchResult.data
var res2 = similarResult.data
// append results to array
all_results.push([res1,res2])
}).finally(function() {
// NOP
});
});
}
This question already has answers here:
How do I return the response from an asynchronous call?
(41 answers)
Closed 6 years ago.
Problem resolved on Asynchronously solution to check data from database kinds of loop clause
with this below code i can generate simple array as jsonArray into checkUserMobileNumberAsEwallet function, but i can't pass it out of that to send inside client,
with socket.emit('syncContacts', accountNumbers) i get [] result on accountNumbers, but into if (success) { statement array successful created and pushed into accountNumbers array
socket.on('syncContacts', function (data) {
var accountNumbers = [];
for (var i = 0; i < data.length; i++) {
checkUserMobileNumberAsEwallet(data[i].mobileNumber, function (success) {
if (success) {
accountNumbers.push({ewalletNumber: this.mobileNumber});
console.log(accountNumbers);
}
}.bind({mobileNumber: data[i].mobileNumber}));
}
console.log(accountNumbers);
socket.emit('syncContacts', accountNumbers);
});
function checkUserMobileNumberAsEwallet(mobileNumber, callback) {
var mobileNumber = mobileNumber.substr(1, mobileNumber.length);
var query = "SELECT id FROM userEwallets WHERE ewalletNumber LIKE '%" + mobileNumber + "'";
connection.query(query, function (err, results) {
if (err) return callback(false);
if (results.length === 0)
return callback(false);
else {
return callback(true);
}
});
}
Updated after post comments:
socket.on('syncContacts', function (data) {
//console.log(accountNumbers);
//socket.emit('syncContacts', accountNumbers);
async.parallel(
[
function (callback) {
var accountNumbers = [];
for (var i = 0; i < data.length; i++) {
checkUserMobileNumberAsEwallet(data[i].mobileNumber, function (success) {
if (success) {
accountNumbers.push({ewalletNumber: this.mobileNumber});
console.log(accountNumbers);
}
}.bind({mobileNumber: data[i].mobileNumber}));
}
callback(success, accountNumbers);
}
],
function (success, results) {
console.log("results " + results.toString());
socket.emit('syncContacts', results);
});
});
function checkUserMobileNumberAsEwallet(mobileNumber, callback) {
var mobileNumber = mobileNumber.substr(1, mobileNumber.length);
var query = "SELECT id FROM userEwallets WHERE ewalletNumber LIKE '%" + mobileNumber + "'";
connection.query(query, function (err, results) {
if (err) return callback(false);
if (results.length === 0)
return callback(false);
else {
return callback(true);
}
});
}
Based on our code, it seems like checkUserMobileNumberAsEwallet is an asynchronous event, which accepts a function callback. That would mean that the for-loop would execute, which would queue up executions to checkUserMobileNumberAsEwallet. Immediately after the for-loop executes, console.log would correctly output the empty array accountNumbers and emit the event through the socket. Then the callback functions to each checkUserMobileNumberAsEwallet execution would begin to execute, and log the accountNumbers array which now has data.
This can be solved in a few ways, but likely the easiest and most readable would be to create Promises, and act on the Promises when they resolve. Personally I like the 'when' promise library, but many libraries could help to solve this problem. https://github.com/cujojs/when/blob/master/docs/api.md#whensettle
The problem is that you have no control over when the code inside the callback executes, and it ends up executing after you've already called socket.emit.
You can either a) use an async aggregator to call a callback for each successful mobile number and then, when all of those have finished, call the socket.emit call OR b) do something like what I did below. Alter your checkUserMobileNumberAsEwallet function to accept and verify an array of mobile numbers, and pass the successful numbers to your callback function
Try the following:
socket.on('syncContacts', function (data) {
var accountNumbers = [];
var mobileNumbers = [];
for (var i = 0; i < data.length; i++) {
mobileNumbers.push(data[i].mobileNumber);
}
checkUserMobileNumberAsEwallet(mobileNumbers, function (successfulNumbers) {
if (successfulNumbers.length > 0) {
for (var i = 0; i < successfulNumbers.length; i++) {
accountNumbers.push({ewalletNumber: successfulNumbers[i]});
}
socket.emit('syncContacts', accountNumbers);
}
}.bind({mobileNumbers: mobileNumbers}));
});
I created a Node.js script that creates a large array of randomly generated test data and I want to write it to a Redis DB. I am using the redis client library and the async library. Initially, I tried executing a redisClient.hset(...) command within the for loop that generates my test data, but after some Googling, I learned the Redis method is asynchronous while the for loop is synchronous. After seeing some questions on StackOverflow, I can't get it to work the way I want.
I can write to Redis without a problem with a small array or larger, such as one with 100,000 items. However, it does not work well when I have an array of 5,000,000 items. I end up not having enough memory because the redis commands seem to be queueing up, but aren't executed until after async.each(...) is complete and the node process does not exit. How do I get the Redis client to actually execute the commands, as I call redisClient.hset(...)?
Here a fragment of the code I am working with.
var redis = require('redis');
var async = require('async');
var redisClient = redis.createClient(6379, '192.168.1.150');
var testData = generateTestData();
async.each(testData, function(item, callback) {
var someData = JSON.stringify(item.data);
redisClient.hset('item:'+item.key, 'hashKey', someData, function(err, reply) {
console.log("Item was persisted. Result: " +reply);
});
callback();
}, function(err) {
if (err) {
console.error(err);
} else {
console.log.info("Items have been persisted to Redis.");
}
});
You could call eachLimit to ensure you are not executing too many redisClient.hset calls at the same time.
To avoid overflowing the call stack you could do setTimeout(callback, 0); instead of calling the callback directly.
edit:
Forget what I said about setTimeout. All you need to do is call the callback at the right place. Like so:
redisClient.hset('item:'+item.key, 'hashKey', someData, function(err, reply) {
console.log("Item was persisted. Result: " +reply);
callback();
});
You may still want to use eachLimit and try out which limit works best.
By the way - async.each is supposed to be used only on code that schedules the invocation of the callback in the javascript event queue (e.g. timer, network, etc) . Never use it on code that calls the callback immediately as was the case in your original code.
edit:
You can implement your own eachLimit function that instead of an array takes a generator as it's first argument. Then you write a generator function to create the test data. For that to work, node needs to be run with "node --harmony code.js".
function eachLimit(generator, limit, iterator, callback) {
var isError = false, j;
function startNextSetOfActions() {
var elems = [];
for(var i = 0; i < limit; i++) {
j = generator.next();
if(j.done) break;
elems.push(j.value);
}
var activeActions = elems.length;
if(activeActions === 0) {
callback(null);
}
elems.forEach(function(elem) {
iterator(elem, function(err) {
if(isError) return;
else if(err) {
callback(err);
isError = true;
return;
}
activeActions--;
if(activeActions === 0) startNextSetOfActions();
});
});
}
startNextSetOfActions();
}
function* testData() {
while(...) {
yield new Data(...);
}
}
eachLimit(testData(), 10, function(item, callback) {
var someData = JSON.stringify(item.data);
redisClient.hset('item:'+item.key, 'hashKey', someData, function(err, reply) {
if(err) callback(err);
else {
console.log("Item was persisted. Result: " +reply);
callback();
}
});
}, function(err) {
if (err) {
console.error(err);
} else {
console.log.info("Items have been persisted to Redis.");
}
});
I have a for loop that pulls data from a MySQL server. I would like the four values to be put into variables so I can use them later. Here's the code I have; for some reason, it says thev is undefined?
create();
function create(){
for(var i=0;i<4;i++){
var thev=[];
client.query('SELECT curattend FROM table1 WHERE ind=?',[i], function(err,result){
thev[i] = result[0].curattend;
});
}
return thev;
}
console.log(thev[2]);
I would appreciate any advice on this problem.
There are a lot of problems here.
thev is local to create. You don’t assign the return value of create to anything, so it’s still not going to be defined.
var thev = []; should not be inside the for loop. It’ll only end up containing one element. Or it would, but…
The callback to query is not just there for fun; it’s an asynchronous call, and is 100% sure to not have happened by the time you actually return from the function.
I would just do it using the async library:
function range(start, end) {
var result = [];
while(start < end) {
result.push(start);
start++;
}
return result;
}
async.map(range(0, 4), function(i, callback) {
client.query('SELECT curattend FROM table1 WHERE ind = ?', [i], function(err, result) {
if(err) return callback(err);
callback(null, result[0].curattend);
});
}, function(err, thev) {
// Continue
});