Develop Reference

This code doesn't seem to fire in order?

My problem is that the code does not seem to be running in order, as seen below.
This code is for my discord.js bot that I am creating.
var Discord = require("discord.js");
var bot = new Discord.Client();
var yt = require("C:/Users/username/Documents/Coding/Discord/youtubetest.js");
var youtubetest = new yt();
var fs = require('fs');
var youtubedl = require('youtube-dl');
var prefix = "!";
var vidid;
var commands = {
play: {
name: "!play ",
fnc: "Gets a Youtube video matching given tags.",
process: function(msg, query) {
youtubetest.respond(query, msg);
var vidid = youtubetest.vidid;
console.log(typeof(vidid) + " + " + vidid);
console.log("3");
}
}
};
bot.on('ready', () => {
console.log('I am ready!');
});
bot.on("message", msg => {
if(!msg.content.startsWith(prefix) || msg.author.bot || (msg.author.id === bot.user.id)) return;
var cmdraw = msg.content.split(" ")[0].substring(1).toLowerCase();
var query = msg.content.split("!")[1];
var cmd = commands[cmdraw];
if (cmd) {
var res = cmd.process(msg, query, bot);
if (res) {
msg.channel.sendMessage(res);
}
} else {
let msgs = [];
msgs.push(msg.content + " is not a valid command.");
msgs.push(" ");
msgs.push("Available commands:");
msgs.push(" ");
msg.channel.sendMessage(msgs);
msg.channel.sendMessage(commands.help.process(msg));
}
});
bot.on('error', e => { console.error(e); });
bot.login("mytoken");
The youtubetest.js file:
var youtube_node = require('youtube-node');
var ConfigFile = require("C:/Users/username/Documents/Coding/Discord/json_config.json");
var mybot = require("C:/Users/username/Documents/Coding/Discord/mybot.js");
function myyt () {
this.youtube = new youtube_node();
this.youtube.setKey(ConfigFile.youtube_api_key);
this.vidid = "";
}
myyt.prototype.respond = function(query, msg) {
this.youtube.search(query, 1, function(error, result) {
if (error) {
msg.channel.sendMessage("There was an error finding requested video.");
} else {
vidid = 'http://www.youtube.com/watch?v=' + result.items[0].id.videoId;
myyt.vidid = vidid;
console.log("1");
}
});
console.log("2");
};
module.exports = myyt;
As the code shows, i have an object for the commands that the bot will be able to process, and I have a function to run said commands when a message is received.
Throughout the code you can see that I have put three console.logs with 1, 2 and 3 showing in which order I expect the parts of the code to run. When the code is run and a query is found the output is this:
I am ready!
string +
2
3
1
This shows that the code is running in the wrong order that I expect it to.
All help is very highly appreciated :)
*Update! Thank you all very much to understand why it isn't working. I found a solution where in the main file at vidid = youtubetest.respond(query, msg) when it does that the variable is not assigned until the function is done so it goes onto the rest of my code without the variable. To fix I simply put an if statement checking if the variable if undefined and waiting until it is defined.*

Like is mentioned before, a lot of stuff in javascript runs in async, hence the callback handlers. The reason it runs in async, is to avoid the rest of your code being "blocked" by remote calls. To avoid ending up in callback hell, most of us Javascript developers are moving more and more over to Promises. So your code could then look more like this:
myyt.prototype.respond = function(query, msg) {
return new Promise(function(resolve, reject) {
this.youtube.search(query, 1, function(error, result) {
if (error) {
reject("There was an error finding requested video."); // passed down to the ".catch" statement below
} else {
vidid = 'http://www.youtube.com/watch?v=' + result.items[0].id.videoId;
myyt.vidid = vidid;
console.log("1");
resolve(2); // Resolve marks the promises as successfully completed, and passes along to the ".then" method
}
});
}).then(function(two) {
// video is now the same as myyt.vidid as above.
console.log(two);
}).catch(function(err) {
// err contains the error object from above
msg.channel.sendMessage(err);
})
};
This would naturally require a change in anything that uses this process, but creating your own prototypes seems.. odd.
This promise returns the vidid, so you'd then set vidid = youtubetest.response(query, msg);, and whenever that function gets called, you do:
vidid.then(function(id) {
// id is now the vidid.
});
Javascript runs async by design, and trying to hack your way around that leads you to dark places fast. As far as I can tell, you're also targetting nodeJS, which means that once you start running something synchronously, you'll kill off performance for other users, as everyone has to wait for that sync call to finish.
Some suggested reading:
http://callbackhell.com/
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise
https://stackoverflow.com/a/11233849/3646975
I'd also suggest looking up ES6 syntax, as it shortens your code and makes life a hellofalot easier (native promises were only introduced in ES6, which NodeJS 4 and above supports (more or less))

In javascript, please remember that any callback function you pass to some other function is called asynchronously. I.e. the calls to callback function may not happen "in order". "In order" in this case means the order they appear on the source file.
The callback function is simply called on certain event:
When there is data to be processed
on error
in your case for example when the youtube search results are ready,
'ready' event is received or 'message' is received.
etc.

Async.js - ETIMEDOUT and Callback was already called

I keep getting an ETIMEDOUT or ECONNRESET error followed by a Callback was already called error when I run index.js.
At first I thought it was because I was not including return prior to calling the onEachLimitItem callback. So I included it per the async multiple callbacks documentation. Still not solving it. I've also tried removing the error event and removing the callback to onEachLimit in the error event, but neither has worked. I've looked at the other SO questions around the issue of Callback already called, but because they aren't concerned with streams, I didn't find a solution.
My understanding is that if the stream encounters an error like ECONNRESET, it will return the callback in the error event and move on to the next stream, but this doesn't seem to be the case. It almost seems if the error resolves itself i.e. it re-connects and tries sending the errored steam up to Azure again and it works, then it triggers the 'finish' event, and we get the Callback already called.
Am I handling the callbacks within the stream events correctly?
var Q = require('q');
var async = require('async');
var webshot = require('webshot');
var Readable = require('stream').Readable;
var azure = require('azure-storage');
var blob = azure.createBlobService('123', '112244');
var container = 'awesome';
var countries = [
'en-us', 'es-us', 'en-au', 'de-at', 'pt-br', 'en-ca', 'fr-ca', 'cs-cz', 'ar-ly', 'es-ve',
'da-dk', 'fi-fi', 'de-de', 'hu-hu', 'ko-kr', 'es-xl', 'en-my', 'nl-nl', 'en-nz', 'nb-no',
'nn-no', 'pl-pl', 'ro-ro', 'ru-ru', 'ca-es', 'es-es', 'eu-es', 'gl-es', 'en-gb', 'es-ar',
'nl-be', 'bg-bg', 'es-cl', 'zh-cn', 'es-co', 'es-cr', 'es-ec', 'et-ee', 'fr-fr', 'el-gr',
'zh-hk', 'en-in', 'id-id', 'en-ie', 'he-il', 'it-it', 'ja-jp', 'es-mx', 'es-pe', 'en-ph'
];
var uploadStreamToStorage = function (fileName, stream, onEachLimitItem) {
var readable = new Readable().wrap(stream);
var writeable = blob.createWriteStreamToBlockBlob(container, fileName);
readable.pipe(writeable);
writeable.on('error', function (error) {
return onEachLimitItem.call(error);
});
writeable.on('finish', function () {
onEachLimitItem.call(null);
});
};
var takeIndividualScreenshot = function (ID, country, onEachLimitItem) {
var fileName = ID + '-' + country + '.jpg';
var url = 'https://example.com/' + country + '/' + ID;
webshot(url, function (error, stream) {
if (error) { throw 'Screenshot not taken'; }
uploadStreamToStorage(fileName, stream, onEachLimitItem);
});
};
var getAllCountriesOfId = function (ID) {
var deferred = Q.defer();
var limit = 5;
function onEachCountry(country, onEachLimitItem) {
takeIndividualScreenshot(ID, country, onEachLimitItem);
}
async.eachLimit(countries, limit, onEachCountry, function (error) {
if (error) { deferred.reject(error); }
deferred.resolve();
});
return deferred.promise;
};
var createContainer = function () {
var df = Q.defer();
var self = this;
blob.createContainerIfNotExists(this.container, this.containerOptions, function (error) {
if (error) { df.reject(error); }
df.resolve(self.container);
});
return df.promise;
};
createContainer()
.then(function () {
return getAllCountriesOfId('211007');
})
.then(function () {
return getAllCountriesOfId('123456');
})
.fail(function (error) {
log.info(error);
});

You are letting your callback get called twice, as you already know. The question is; do you want to stop on all errors as you are iterating the stream or do you want to accumulate all errors from the stream?
There are multiple ways to catch and handle the errors which you are already doing, but because you aren't throwing the error object leading to additional calls from your data stream to fatally error.
The actual problem in your code is due to the scope of your return. When you are handling the error and trying to return the callback and halt script execution the scope of hour return is local to the streams error handler, not the global script hence the script continuing and catching moving on to the next valid stream.
writeable.on('error', function (error) {
// This 'return' is in the local scope of 'writable.on('error')'
return onEachLimitItem.call(error);
});
It could perhaps set an array, then handle the error outside of that functions local scope. i.e.
// Set the array's scope as global to the writable.on() error
var errResults = [];
writeable.on('error', function (error) {
// Push the local scoped 'error' into the global scoped 'errResults' array
errResults.push(error);
});
writeable.on('finish', function () {
// Are there any errors?
return (errResults.length > 0) ?
onEachLimitItem.call(errors) : onEachLimitItem.call(null);
});
The above is just one way you could tackle the problem.
I am not sure if you have read the error handling help provided from Joyent (original node.js language backers) but it should give you a good idea of your options when handling the error(s).
https://www.joyent.com/developers/node/design/errors

Is there a good example of lodash's _.after method

Is there a good practical example of how to use _.after method in lodash library?

Use it whenever you need to invoke a callback after it's been called n number of times.
var fn = _.after(3, function () {
console.log('done');
});
fn(); // Nothing
fn(); // Nothing
fn(); // Prints "done"
It's useful for invoking callback when all async calls are complete.
var done = _.after(3, function () {
console.log('all 3 requests done!');
});
$.get('https://example.com', done);
$.get('https://example.com', done);
$.get('https://example.com', done);
Basic game example where player dies after getting shot 3 times.
var isDead = _.after(3, function () {
console.log('Player died!');
});
player1.shoot(player2, isDead); // undefined
player1.shoot(player2, isDead); // undefined
player1.shoot(player2, isDead); // "Player died!"
Basically you use _.after in place of a manual counter.

There are not many examples out there but see the following for something that I use for my internal tooling.
Basically the following is a script to be run using Node. It removes documents from given Mongodb collections. That is it. But the idea is to close the DB connection only after all collections are cleaned up. We will use _.after method for that. You can read about after function here
var Db = require('mongodb').Db,
MongoClient = require('mongodb').MongoClient,
Server = require('mongodb').Server;
_ = require('lodash');
var db = new Db('mydb', new Server('localhost', 27017));
db.open(function(err, db) {
var collectionsToClean = ['COLLECTIONA', 'COLLECTIONB', 'COLLECTIONC'];
var closeDB = _.after(collectionsToClean.length, function() {
db.close();
console.log('Connection closed');
});
_.forEach(collectionsToClean, function(collectionName) {
db.collection(collectionName, function(err, collection) {
collection.remove({}, function(err) {
if (err) {
console.log('Could not remove documents from ' + collectionName);
} else {
console.log("All documents removed from " + collectionName);
}
closeDB();
});
})
});
});
You can now use this as a template for other Mongodb shell methods.

Calling a closure inside a promise callback

I'm implementing queue system for a webapp so that HTTP request that are failed are locally stored to be re-executed later on.
I have read Mozilla's documentation about closure in loops and then created inner closures.
If running this with ids [1,2,3], the output will be:
makeRequestRecovery 1
makeRequestRecovery 2
makeRequestRecovery 3
failure
recover id 1
failure
recover id 1
failure
recover id 1
code:
var recoverRequest = function(entry) {
console.log('recover id', entry.data.id);
$scope.storage.entries.push(entry);
};
var makeRequestRecovery = function(entry) {
console.log('makeRequestRecovery', entry.data.id);
return function() {
recoverRequest(entry);
}
};
$scope.syncEntries = function() {
var initialLength = $scope.storage.entries.length;
var requestCount = 0;
while ($scope.storage.entries.length > 0) {
var entry = $scope.storage.entries.pop(0);
var recover = makeRequestRecovery(entry);
// restangular stuff...
// restangular HTTP REMOVE
entry.remove().then(function(data) {
console.log('success!', data);
}, function(data) {
console.log('failure', data);
recover();
});
requestCount++;
if (requestCount > 2 * initialLength) {
// too many requests failed
break;
}
}
}
What changes should I make so recover() is executed with the right value?

I actually didn't use the closure properly.
recover needs to be passed as callback:
entry.remove().then(function(data) {
console.log('success!', data);
}, recover);

node.js + socket.io + multiple database calls needed for result

I need help.
I've been trying to wrap my head around async programming with node.js and socket.io for a day now. I understand that I need some flow control but I don't seem to understand how to properly implement it.
I have a redis datastore that has modules stored in a set let's say 'moda','modb'
instances of those modules are 'moda:instances' and in 'modb:instances' the properties of those instances are stored in 'moda:instancea' and 'modb:instanceb' as a hash.
I am trying to get the following json:
"moda": {"instancea": {"property1": "value1", "property2", "value2"}}, "modb": {"instanceb": {"property1": "value1"}}
Could someone give me a little push in the right direction?
Here is my current code:
var io = require('socket.io').listen(2000);
var redis = require('redis').createClient();
var http = require('http');
var async = require('async');
var step = require('step');
io.sockets.on('connection', function (socket) {
var notifications = require('redis').createClient();
notifications.subscribe("notification");
notifications.on("message", function (channel, message) {
socket.send(message);
console.log(channel + ':' + message);
});
socket.on('modules', function(params, callback) {
var response = {};
async.series([
function (callback) {
console.log('1>');
redis.smembers('modules', function (err, modules) {
async.forEachSeries(modules, function(module, moduleCallback) {
response[module] = {}
redis.smembers(module + ':instances', function(err, instances) {
async.forEachSeries(instances, function(instance, instanceCallback) {
response[module][instance] = {}
console.log('2>' + module + ':' +instance);
instanceCallback();
});
moduleCallback();
});
});
callback();
});
},
function (callback) {
console.log('3');
callback();
}
], function() {
console.log(JSON.stringify(response));
});
});
});
The output from this code is:
info - socket.io started
debug - client authorized
info - handshake authorized JMMn1I8aiOMGCMPOhC11
debug - setting request GET /socket.io/1/websocket/JMMn1I8aiOMGCMPOhC11
debug - set heartbeat interval for client JMMn1I8aiOMGCMPOhC11
debug - client authorized for
debug - websocket writing 1::
1>
3
{"moda":{}}
2>moda:instancea
2>moda:instanceb
2>modb:instancea

The problem comes from the fact forEachSeries requires an additional callback as a third parameter (to be called when the whole processing is complete). You cannot just put some code after forEachSeries hoping it will be called once it is complete.
Here is your code modified:
var response = {};
async.series([
function (callback) {
console.log('1>');
redis.smembers('modules', function (err, modules) {
async.forEachSeries(modules, function(module, moduleCallback) {
response[module] = {}
redis.smembers(module + ':instances', function(err, instances) {
async.forEachSeries(instances, function(instance, instanceCallback) {
response[module][instance] = {}
console.log('2>' + module + ':' +instance);
instanceCallback();
}, moduleCallback );
});
}, callback );
});
},
function (callback) {
console.log('3');
callback();
}],
function() {
console.log(JSON.stringify(response));
});
Note how the callback, and moduleCallback are used as a third parameter. The output is:
1>
2>moda:instancea
2>moda:instanceb
2>modb:instancea
3
{"moda":{"instancea":{},"instanceb":{}},"modb":{"instancea":{}}}
which I guess is what you expected.
Additional remark: forEachSeries will process everything in sequence, the next operation waiting for the previous one to complete. This will generate plenty of roundtrips to Redis. forEach should be much more efficient here to leverage pipelining.

Take a look also at promise concept, with promises you configure the flow much more readable way.
First you need to prepare promise versions of redis methods that you're using:
var promisify = require('deferred').promisify;
var RedisClient = require('redis').RedisClient;
RedisClient.prototype.psmembers = promisify(RedisClient.prototype.smembers);
Then you can construct your flow as:
console.log('1>');
redis.psmembers('modules').map(function (module) {
response[module] = {};
return redis.psmembers(module + ':instances').map(function (instance) {
response[module][instance] = {};
console.log('2>' + module + ':' +instance);
});
}).end(function () {
console.log('3');
console.log(JSON.stringify(response));
});
Check: https://github.com/medikoo/deferred