Is there a good practical example of how to use _.after method in lodash library?
Use it whenever you need to invoke a callback after it's been called n number of times.
var fn = _.after(3, function () {
console.log('done');
});
fn(); // Nothing
fn(); // Nothing
fn(); // Prints "done"
It's useful for invoking callback when all async calls are complete.
var done = _.after(3, function () {
console.log('all 3 requests done!');
});
$.get('https://example.com', done);
$.get('https://example.com', done);
$.get('https://example.com', done);
Basic game example where player dies after getting shot 3 times.
var isDead = _.after(3, function () {
console.log('Player died!');
});
player1.shoot(player2, isDead); // undefined
player1.shoot(player2, isDead); // undefined
player1.shoot(player2, isDead); // "Player died!"
Basically you use _.after in place of a manual counter.
There are not many examples out there but see the following for something that I use for my internal tooling.
Basically the following is a script to be run using Node. It removes documents from given Mongodb collections. That is it. But the idea is to close the DB connection only after all collections are cleaned up. We will use _.after method for that. You can read about after function here
var Db = require('mongodb').Db,
MongoClient = require('mongodb').MongoClient,
Server = require('mongodb').Server;
_ = require('lodash');
var db = new Db('mydb', new Server('localhost', 27017));
db.open(function(err, db) {
var collectionsToClean = ['COLLECTIONA', 'COLLECTIONB', 'COLLECTIONC'];
var closeDB = _.after(collectionsToClean.length, function() {
db.close();
console.log('Connection closed');
});
_.forEach(collectionsToClean, function(collectionName) {
db.collection(collectionName, function(err, collection) {
collection.remove({}, function(err) {
if (err) {
console.log('Could not remove documents from ' + collectionName);
} else {
console.log("All documents removed from " + collectionName);
}
closeDB();
});
})
});
});
You can now use this as a template for other Mongodb shell methods.
Related
I am unable to execute the sql, when using the global database connection in node.js.
I have followed the steps as in Azure documentation: https://learn.microsoft.com/en-us/azure/mysql/connect-nodejs and able to display the output on the console. But, I want to put all my Azure SQL database connection in a separate file, but the select query is not printing the output on the console.
DatabaseManager.js
var Connection = require('tedious').Connection;
var Request = require('tedious').Request;
var sqlConnection = function sqlConnection() {
// Create connection to database
var config =
{
userName: 'uname',
password: 'password',
server: 'dbserver.database.windows.net',
options:
{
database: 'mydatabase',
encrypt: true
}
}
var connection = new Connection(config);
// Attempt to connect and execute queries if connection goes through
connection.on('connect', function(err) {
if (err)
{
console.log(err)
}
else
{
console.log('CONNECTED TO DATABASE');
}
}
);
}
module.exports = sqlConnection;
app.js
var restify = require('restify');
var builder = require('botbuilder');
var botbuilder_azure = require("botbuilder-azure");
var azure = require('azure-storage');
var dbconnection = require('./DatabaseManager');
bot.dialog('profileDialog',
(session) => {
session.send('You reached the profile intent. You said \'%s\'.', session.message.text);
console.log('Reading rows from the Table...');
dbconnection("select FNAME from StudentProfile where ID=1"),
function (err, result, fields) {
if (err) throw err;
console.log(result);
}
session.endDialog();
}
Console Output:
Reading rows from the Table...
CONNECTED TO DATABASE
I was expecting the output of FNAME, but nothing is printing on the console. Is there anything, I am missing?
Thank you.
There's a couple of problems here. First off, you should only ever import a module once per file. This is just a performance consideration and won't actually break your code.
Next, pay attention to what you're exporting from your DatabaseManager module. Right now, you're exporting a function that creates the connection and then doesn't do anything with it. We can fix this by using a pattern called a "callback" which lets us provide a function that will then be called with the connection as an argument.
I added a ton of comments to the code explaining things. This code won't run as-is - there's a couple places where I have "do this or this". You'll have to choose one.
var Tedious = require('tedious'); // Only require a library once per file
var Connection = Tedious.Connection;
var Request = Tedious.Request;
// Or using the object spread operator
var { Connection, Request } = require('tedious');
// You called this `sqlConnection`. I'm going to use a verb since it's a
// function and not a variable containing the connection. I'm also going
// to change the declaration syntax to be clearer.
function connect(cb) { // cb is short for callback. It should be a function.
var config = {
userName: 'uname',
password: 'password',
server: 'dbserver.database.windows.net',
options: {
database: 'mydatabase',
encrypt: true
}
}; // Put a semi-colon on your variable assignments
var connection = new Connection(config);
// Attempt to connect and execute queries if connection goes through
connection.on('connect', function(err) {
if (err) {
console.log(err);
return; // Stop executing the function if it failed
}
// We don't need an "else" because of the return statement above
console.log('CONNECTED TO DATABASE');
// We have a connection, now let's do something with it. Call the
// callback and pass it the connection.
cb(connection);
});
}
module.exports = connect; // This exports a function that creates the connection
Then back in your main file, you can use it like so.
var restify = require('restify');
var builder = require('botbuilder');
var botbuilder_azure = require('botbuilder-azure');
var azure = require('azure-storage');
var connect = require('./DatabaseManager'); // renamed to be a verb since it's a function.
bot.dialog('profileDialog', (session) => { // Hey, this is a callback too!
session.send('You reached the profile intent. You said \'%s\'.', session.message.text);
console.log('Creating a connection');
connect((connection) => {
// or with the traditional function notation
connect(function(connection) {
console.log('Reading rows from the Table...');
// Execute your queries here using your connection. This code is
// taken from
// https://github.com/tediousjs/tedious/blob/master/examples/minimal.js
request = new Request("select FNAME from StudentProfile where ID=1", function(err, rowCount) { // Look another callback!
if (err) {
console.log(err);
} else {
console.log(rowCount + ' rows');
}
connection.close();
});
request.on('row', function(columns) { // Iterate through the rows using a callback
columns.forEach(function(column) {
if (column.value === null) {
console.log('NULL');
} else {
console.log(column.value);
}
});
});
connection.execSql(request);
});
For the life of me I cannot work this one out. Have look around and tried many many different ways of trying to get this to go. Currently have the following code.
var config = require("./config.js");
var cradle = require('cradle')
var MikroNode = require('mikronode');
var WebServer = require('./bin/www');
var Routers = "Hasnt changed";
var conndb = new(cradle.Connection)(config.couchdb.host);
var db = conndb.database(config.couchdb.db);
db.exists(function(err, exists){
if (err) { console.log('error', err);}
else if (exists) { console.log('Seems the Force is with you - Database Exists');}
else { db.create(); }
});
db.temporaryView({
map: function (doc){
if (doc.type=='ConfigRouter') emit(doc.name, doc);
}
}, function (err, res){
Routers = JSON.stringify(res);
}
);
console.log(Routers);
As it stands it will respond with:
E:\Dev\MM>npm start
> MM#0.0.1 start E:\Dev\MM
> node ./Start.js
Hasnt changed
Seems the Force is with you - Database Exists
I am assuming it is an asynchronous call to the CouchDB and is not filling the result in time before it displays the result. How do I get around this issue?
You are right, the call is asynchronous so when console.log(Routers); is processed, Routers is "Hasnt changed".
One way of doing it would be to use promises thanks to the Q npm module:
var Q = require('q');
var deferred = Q.defer();
db.temporaryView({
map: function (doc) {
if (doc.type=='ConfigRouter') emit(doc.name, doc);
}
}, function (err, res) {
deferred.resolve(JSON.stringify(res));
});
deferred.promise
.then(function (data) {
Routers = data;
console.log(Routers);
// do some stuff...
})
.done();
Maybe it's possible to do something better without using Q.defer and adapting directly the callback:
https://github.com/kriskowal/q#adapting-node
I'm using the SerialPort npm package with meteor. I've used wrapAsync to list Serial ports but i don't know how to do with the serialPort.on method.
I've an error when i want to inser datas in my Cars collection :
Meteor code must always run within a Fiber. Try wrapping callbacks
that you pass to non-Meteor libraries with Meteor.bindEnvironment.
Code :
Meteor.startup(function () {
SerialPort = Meteor.npmRequire('serialport');
// Wrap method SerialPort.list to call it Synchronously
listSerialPorts = function(callback) {
SerialPort.list(function (err, ports) {
callback(null, ports);
});
}
// Reset cars collection
});
Meteor.methods({
serialPortsRefresh: function () {
// TODO : problem when several arduinos ?
Config.remove({key:'serialPorts'});
// Call SerialPort.list
var asyncListSerialPorts = Meteor.wrapAsync(listSerialPorts);
var resultsListSerialPorts = asyncListSerialPorts();
// Insert results in database
var configSerialPorts = {key: "serialPorts", value: resultsListSerialPorts[0].comName };
Config.insert(configSerialPorts);
},
// Connect Serial port
serialPortConnect: function (port) {
// debugger;
// serialPort = new SerialPort(port.value, {baudrate: 9600});
serialPort = new SerialPort.SerialPort("/dev/ttyUSB0", {baudrate: 9600, parser: SerialPort.parsers.readline("\n")});
// connectSerialPort(port);
serialPort.on('open', function() {
console.log('Port ' + port.value + ' open');
});
serialPort.on('data', function(data) {
dispatchMessages(data);
//Watchdog.insert({key: "Receiving data", value: data })
});
sendToArduino = function(message) {
console.log(message);
serialPort.write(message);
};
dispatchMessages = function(data) {
console.log(data);
//Split data
var datas = data.split(" ");
if (datas[1] == "OK") {
console.log("Car " + datas[0] + " is here");
// Add car to database
Cars.insert({
cid: datas[0],
active: true
});
}
};
},
// Ping bridge
ping: function () {
sendToArduino("LED13\n");
}
});
The problem is that the callbacks you're passing to serialPort.on will not run within the same fiber as your method when they're invoked. In fact, they won't run within a fiber at all, unless you wrap them appropriately.
Meteor.bindEnvironment runs the passed function within a fiber, but also copies in the surrounding environment, which is necessary as Meteor stores all sorts of variables within the current fiber which might be required to run the callback in question.
So, if you do this it should work:
serialPort.on('open', Meteor.bindEnvironment(function() {
// Wrapping this one is unnecessary at present as it doesn't
// do anything that needs to be run in a fiber, but you should
// probably wrap it anyway so that you can safely add more code
// if required.
console.log('Port ' + port.value + ' open');
}, function(e) {
// This is an error-handler - you don't have to pass one, but
// if you don't it can make debugging a nightmare.
throw e;
}));
serialPort.on('data', Meteor.bindEnvironment(function(data) {
dispatchMessages(data);
//Watchdog.insert({key: "Receiving data", value: data })
}, function(e) {
throw e;
}));
Note that you also need to wrap callbacks within callbacks, etc., which can become quite verbose (and makes putting something like var mBE = Meteor.bindEnvironment at the top of your methods file quite a good idea).
I created a Node.js script that creates a large array of randomly generated test data and I want to write it to a Redis DB. I am using the redis client library and the async library. Initially, I tried executing a redisClient.hset(...) command within the for loop that generates my test data, but after some Googling, I learned the Redis method is asynchronous while the for loop is synchronous. After seeing some questions on StackOverflow, I can't get it to work the way I want.
I can write to Redis without a problem with a small array or larger, such as one with 100,000 items. However, it does not work well when I have an array of 5,000,000 items. I end up not having enough memory because the redis commands seem to be queueing up, but aren't executed until after async.each(...) is complete and the node process does not exit. How do I get the Redis client to actually execute the commands, as I call redisClient.hset(...)?
Here a fragment of the code I am working with.
var redis = require('redis');
var async = require('async');
var redisClient = redis.createClient(6379, '192.168.1.150');
var testData = generateTestData();
async.each(testData, function(item, callback) {
var someData = JSON.stringify(item.data);
redisClient.hset('item:'+item.key, 'hashKey', someData, function(err, reply) {
console.log("Item was persisted. Result: " +reply);
});
callback();
}, function(err) {
if (err) {
console.error(err);
} else {
console.log.info("Items have been persisted to Redis.");
}
});
You could call eachLimit to ensure you are not executing too many redisClient.hset calls at the same time.
To avoid overflowing the call stack you could do setTimeout(callback, 0); instead of calling the callback directly.
edit:
Forget what I said about setTimeout. All you need to do is call the callback at the right place. Like so:
redisClient.hset('item:'+item.key, 'hashKey', someData, function(err, reply) {
console.log("Item was persisted. Result: " +reply);
callback();
});
You may still want to use eachLimit and try out which limit works best.
By the way - async.each is supposed to be used only on code that schedules the invocation of the callback in the javascript event queue (e.g. timer, network, etc) . Never use it on code that calls the callback immediately as was the case in your original code.
edit:
You can implement your own eachLimit function that instead of an array takes a generator as it's first argument. Then you write a generator function to create the test data. For that to work, node needs to be run with "node --harmony code.js".
function eachLimit(generator, limit, iterator, callback) {
var isError = false, j;
function startNextSetOfActions() {
var elems = [];
for(var i = 0; i < limit; i++) {
j = generator.next();
if(j.done) break;
elems.push(j.value);
}
var activeActions = elems.length;
if(activeActions === 0) {
callback(null);
}
elems.forEach(function(elem) {
iterator(elem, function(err) {
if(isError) return;
else if(err) {
callback(err);
isError = true;
return;
}
activeActions--;
if(activeActions === 0) startNextSetOfActions();
});
});
}
startNextSetOfActions();
}
function* testData() {
while(...) {
yield new Data(...);
}
}
eachLimit(testData(), 10, function(item, callback) {
var someData = JSON.stringify(item.data);
redisClient.hset('item:'+item.key, 'hashKey', someData, function(err, reply) {
if(err) callback(err);
else {
console.log("Item was persisted. Result: " +reply);
callback();
}
});
}, function(err) {
if (err) {
console.error(err);
} else {
console.log.info("Items have been persisted to Redis.");
}
});
I need help.
I've been trying to wrap my head around async programming with node.js and socket.io for a day now. I understand that I need some flow control but I don't seem to understand how to properly implement it.
I have a redis datastore that has modules stored in a set let's say 'moda','modb'
instances of those modules are 'moda:instances' and in 'modb:instances' the properties of those instances are stored in 'moda:instancea' and 'modb:instanceb' as a hash.
I am trying to get the following json:
"moda": {"instancea": {"property1": "value1", "property2", "value2"}}, "modb": {"instanceb": {"property1": "value1"}}
Could someone give me a little push in the right direction?
Here is my current code:
var io = require('socket.io').listen(2000);
var redis = require('redis').createClient();
var http = require('http');
var async = require('async');
var step = require('step');
io.sockets.on('connection', function (socket) {
var notifications = require('redis').createClient();
notifications.subscribe("notification");
notifications.on("message", function (channel, message) {
socket.send(message);
console.log(channel + ':' + message);
});
socket.on('modules', function(params, callback) {
var response = {};
async.series([
function (callback) {
console.log('1>');
redis.smembers('modules', function (err, modules) {
async.forEachSeries(modules, function(module, moduleCallback) {
response[module] = {}
redis.smembers(module + ':instances', function(err, instances) {
async.forEachSeries(instances, function(instance, instanceCallback) {
response[module][instance] = {}
console.log('2>' + module + ':' +instance);
instanceCallback();
});
moduleCallback();
});
});
callback();
});
},
function (callback) {
console.log('3');
callback();
}
], function() {
console.log(JSON.stringify(response));
});
});
});
The output from this code is:
info - socket.io started
debug - client authorized
info - handshake authorized JMMn1I8aiOMGCMPOhC11
debug - setting request GET /socket.io/1/websocket/JMMn1I8aiOMGCMPOhC11
debug - set heartbeat interval for client JMMn1I8aiOMGCMPOhC11
debug - client authorized for
debug - websocket writing 1::
1>
3
{"moda":{}}
2>moda:instancea
2>moda:instanceb
2>modb:instancea
The problem comes from the fact forEachSeries requires an additional callback as a third parameter (to be called when the whole processing is complete). You cannot just put some code after forEachSeries hoping it will be called once it is complete.
Here is your code modified:
var response = {};
async.series([
function (callback) {
console.log('1>');
redis.smembers('modules', function (err, modules) {
async.forEachSeries(modules, function(module, moduleCallback) {
response[module] = {}
redis.smembers(module + ':instances', function(err, instances) {
async.forEachSeries(instances, function(instance, instanceCallback) {
response[module][instance] = {}
console.log('2>' + module + ':' +instance);
instanceCallback();
}, moduleCallback );
});
}, callback );
});
},
function (callback) {
console.log('3');
callback();
}],
function() {
console.log(JSON.stringify(response));
});
Note how the callback, and moduleCallback are used as a third parameter. The output is:
1>
2>moda:instancea
2>moda:instanceb
2>modb:instancea
3
{"moda":{"instancea":{},"instanceb":{}},"modb":{"instancea":{}}}
which I guess is what you expected.
Additional remark: forEachSeries will process everything in sequence, the next operation waiting for the previous one to complete. This will generate plenty of roundtrips to Redis. forEach should be much more efficient here to leverage pipelining.
Take a look also at promise concept, with promises you configure the flow much more readable way.
First you need to prepare promise versions of redis methods that you're using:
var promisify = require('deferred').promisify;
var RedisClient = require('redis').RedisClient;
RedisClient.prototype.psmembers = promisify(RedisClient.prototype.smembers);
Then you can construct your flow as:
console.log('1>');
redis.psmembers('modules').map(function (module) {
response[module] = {};
return redis.psmembers(module + ':instances').map(function (instance) {
response[module][instance] = {};
console.log('2>' + module + ':' +instance);
});
}).end(function () {
console.log('3');
console.log(JSON.stringify(response));
});
Check: https://github.com/medikoo/deferred