Develop Reference

node.js / express app - which async method to use to replace my nested calls to HGET?

Background
I'm just learning node js and have run into a situation where I need to make up to two back to back calls to my redis database, depending on the results of the first query.
The code I have right now works.. but it's very ugly. I wrote it this way because I'm not good with async 'stuff'. But now that it's working... I want to refactor in a way that is readable and of course, in a way that works.
Here's the code, along with an explanation of what I'm trying to do:
Code
router.get('/:ip', function(req, res, next) {
var ip = req.params.ip;
if ( ! validate_ipV4(ip) ) {
res.status(400).send("Invalid IP");
return;
}
var three_octets = extract_octets(ip, 3);
var two_octets = extract_octets(ip, 2);
if (debug) { winston.log('info', 'emergency router.get() attempting hget using :' + three_octets); }
redis.hget("e:" + three_octets, 'ccid', function (e, d) {
if (e){
winston.log('error', 'hget using key: ' + octets + ' failed with error: ' + e);
res.status(500).send("Database query failed");
return;
}
if (d) {
if (debug) { winston.log('info', 'HGET query using ip: ' + ip + ' returning data: ' + d ) };
res.status(200).send(JSON.stringify(d));
return;
} else {
//retry using only 2 octets
redis.hget("e:" + two_octets, 'ccid', function (e, d) {
if (e){
winston.log('error', 'hget using key: ' + octets + ' failed with error: ' + e);
res.status(500).send("Database query failed");
return;
}
if (d) {
if (debug) { winston.log('info', 'HGET query using ip: ' + ip + ' returning data: ' + d ) };
res.status(200).send(JSON.stringify(d));
return;
}else {
res.status(404).send("Unknown IP");
return;
}
});//end hget
}
});//end hget
});
Explanation:
Accept an ip address as input. 10.1.1.1
Try to query the database for a hash that matches the first three octets. For example: "hget e:10.1.1 ccid"
If i have a match, I can return the db results and exit. otherwise, if the query came back with no results, then I need to retry using the first two octets: "hget e:10.1 ccid"
if that returns nothing, then i can exit the GET method.
ASYNC
I know that there is an async module... and i've tried to use MAP before. But from what I understand, you cannot force MAP to exit early.
So for example, if I did something like this:
async.map(ipOctets, hash_iterator, function (e, r) {
})
where ipOctets was an array with both 10.1.1. and 10.1 in it, if the first query found a match in the database, there's no way I can stop it from running the second query.
Can you give me some pointers on how to improve this code so that I don't have to repeat the same code twice?
I also thought of putting the redis.hget call into a separate function... like this:
var hash_get = function (hash, key, field) {
if (debug) { winston.log('info', 'hash_get() invoked with : ' + hash + ' ' + key + ' ' + field);}
redis.hget(hash + key, field, function (e, d) {
if (e){
winston.log('hash_get() failed with: ' + e);
return 500;
}
if (d) {
return (d);
}else {
return 404;
}
});
}
But again, I'm not sure how to do the following in a synchronous way:
call it from router.get
check results
repeat if necessary
Sorry for the noob questions.. but any pointers would be appreciated.
EDIT 1
Since posting, i found this http://caolan.github.io/async/docs.html#some
and I'm currently testing to see if this will work for me.
But please comment if you have some suggestions!
Thanks.

You could use the waterfall method which cascades functions into each other. I really only like to use it when I have 3 nested callbacks or more, otherwise I don't feel like it simplifies it enough.
After looking at your code and seeing how much you can reuse I think I would use async.until though.
router.get('/:ip', function(req, res, next) {
var ip = req.params.ip;
if (!validate_ipV4(ip)) {
res.status(400).send("Invalid IP");
return;
}
let success = false;
let octets_num = 3;
async.until(
// Test this for each iteration
function() { return success == true || octets < 2}, // You would adjust the test to set limits
// Do this until above
function(callback) {
let octets = extract_octets(ip, octets_num);
redis.hget("e:" + octets, 'ccid', function(e, d) {
if(e) {
winston.log('error', 'hget using key: ' + octets + ' failed with error: ' + e);
res.status(500).send("Database query failed");
}
else if(id) {
if (debug) { winston.log('info', 'HGET query using ip: ' + ip + ' returning data: ' + d ) };
res.status(200).send(JSON.stringify(d));
success == true;
}
else
{
octects_num--;
}
callback(null);
});
}
// After success or not found within 3 or 2 octets
function(err, result) {
if(success == false) {
res.status(404).send("Unknown IP");
return;
}
}
...
}
This permits you to reuse the same chunk of code with minimal variation. It's rough and I don't have the rest of your application to test it, but I hope you get the idea.

Maybe like this:
router.get('/:ip', function (req, res, next) {
var ip = req.params.ip;
if (!validate_ipV4(ip)) {
res.status(400).send("Invalid IP");
return;
}
var three_octets = extract_octets(ip, 3);
var two_octets = extract_octets(ip, 2);
//if (debug) { winston.log('info', 'emergency router.get() attempting hget using :' + three_octets); }
var hash = "e:"
var field = 'ccid';
async.waterfall([
function (callback) {
hash_get(hash, three_octets, field, callback)
},
function (d, callback) {
if (d) {
callback(null, d);
return;
}
hash_get(hash, two_octets, field, callback)
}
], function (err, result) {
if (err) {
winston.log('error', err.message);
res.status(err.status).send(err.message);
return;
}
if (result) {
res.status(200).send(JSON.stringify(result));
return;
}
res.status(404).send("Unknown IP");
return;
});
});
var hash_get = function (hash, key, field, callback) {
if (debug) { winston.log('info', 'hash_get() invoked with : ' + hash + ' ' + key + ' ' + field); }
redis.hget(hash + key, field, function (e, d) {
if (e) {
callback({ status: 500, message: 'hget using key: ' + key + ' failed with error: ' + e });
return;
}
if (d) {
if (debug) { winston.log('info', 'HGET query using ip: ' + ip + ' returning data: ' + d) };
callback(null, d);
} else {
callback(null, null);
}
});
}

Check Async.waterfall() for this as you want the result of one callback into another (http://caolan.github.io/async/docs.html#waterfall).
Async.map could not be used as it will hit both the octets at the same time
which you don't want .
Code
router.get('/:ip', function(req, res, next) {
var ip = req.params.ip;
if ( ! validate_ipV4(ip) ) {
res.status(400).send("Invalid IP");
return;
}
var three_octets = extract_octets(ip, 3);
var two_octets = extract_octets(ip, 2);
var redis_hget=function(octets){
redis.hget("e:"+octets,'ccid',function(e,d){
callback(null,d)
})
}
if (debug) { winston.log('info', 'emergency router.get() attempting hget using :' + three_octets); }
async.waterfall([
function(callback){
redis_hget(three_octets)
},
function(d,callback){
if(d)
callback(d)
else
redis_hget(two_octets)
}
],function(err,result){
if(err){
winston.log('error', 'hget using key: ' + octets + ' failed with error: ' + e);
res.status(500).send("Database query failed");
return;
}else{
if(result){
if (debug) { winston.log('info', 'HGET query using ip: ' + ip + ' returning data: ' + d ) };
res.status(200).send(JSON.stringify(d));
return;
}else{
res.status(404).send("Unknown IP");
return;
}
}
})
}

How can I connect mosquitto server between raspberry pi and windows7?

Hello I'm trying to use mosquitto server in Raspberry Pi using MQTT to send a json data from r-pi to window.
Before I use mosquitto server, I used "test.mosquitto.org" It worked well.
I mean It sended some json data to windows.
However, when I turned mosquitto server in r-pi on, the windows put some error message which is
opts.protocol = opts.protocol.replace, cannot read property 'replace' of null.
Would you mind telling me what it is going on and fix it?
this is javascript on windows code (I use python in raspberry pi)
console.log("start");
var mqtt = require('mqtt');
var client = mqtt.connect('mqtt://test.mosquitto.org');
var client = mqtt.connect('192.168.1.2'); // IP of main-broker
client.on('connect', function () {
client.subscribe('sensor_A');
});
client.on('message', function (topic, message) {
console.log("Topic: " + topic);
var parsedData = JSON.parse(message);
var dataLen = parsedData.length
console.log('dataLen: ' + dataLen);
for (var i = 0; i < dataLen; i++) {
var data = JSON.parse(parsedData[i]);
console.log('data ' + i + ': ' + data.time + ' ' + data.tem + ' ' + data.hum + ' ' + data.gas);
}
});
I am using two r-pi which is sub-borker and main-broker.
sub-broker just send some sensor data as json and main-broker controls the json data and send again as json to windows.
I think my writing is quite complex to understand.
In short, I don't want to use "test.mosquitto.org" in r-pi so I turn mosquitto server on in r-pi to send data to window, however, there a error in window.

First you need to remove the line connecting to test.mosquitto.org as that will just confuse things.
Secondly you have missed out the mqtt:// from the URL for the local instance of mosquitto. The error is points out it can not find the protocol from the url.
console.log("start");
var mqtt = require('mqtt');
var client = mqtt.connect('mqtt://192.168.1.2'); // IP of main-broker
client.on('connect', function () {
client.subscribe('sensor_A');
});
client.on('message', function (topic, message) {
console.log("Topic: " + topic);
var parsedData = JSON.parse(message);
var dataLen = parsedData.length
console.log('dataLen: ' + dataLen);
for (var i = 0; i < dataLen; i++) {
var data = JSON.parse(parsedData[i]);
console.log('data ' + i + ': ' + data.time + ' ' + data.tem + ' ' + data.hum + ' ' + data.gas);
}
});

Node.js Downloading multiples files asynchronously

In trying to get a hang of node.js asynchronous coding style, I decided to write a program that would read a text file containing a bunch of URLS to download and download each file. I started out writing a function to download just one file (which works fine), but having trouble extending the logic to download multiple files.
Here's the code:
var http = require("http"),
fs = require("fs"),
input = process.argv[2],
folder = "C:/Users/Wiz/Downloads/",
regex = /(https?:\/\/)?([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w \.-]*)*\/?/,
urls = null,
url = "",
filename = "";
fs.readFile(input, "utf8", function(e, data) {
console.log("Reading file: " + input);
if (e) console.log("Got error:" + e.message);
urls = data.split("\n");
for (var i = urls.length; i--;) {
url = urls[i];
if (!url.match(regex)) continue;
filename = folder + url.substring(url.lastIndexOf('/') + 1);
downloadQueue.addItem(url, filename);
}
});
var downloadQueue = {
queue: [],
addItem: function(p_sSrc, p_sDest) {
this.queue.push({
src: p_sSrc,
dest: p_sDest
});
if (this.queue.length === 1) {
this.getNext();
}
},
getNext: function() {
var l_oItem = this.queue[0];
http.get(l_oItem.src, function(response) {
console.log("Downloading: " + l_oItem.dest);
var file = fs.createWriteStream(l_oItem.dest);
response.on("end", function() {
file.end();
console.log("Download complete.");
downloadQueue.removeItem();
}).on("error", function(error) {
console.log("Error: " + error.message);
fs.unlink(l_oItem.dest);
});
response.pipe(file);
});
},
removeItem: function() {
this.queue.splice(0, 1);
if (this.queue.length != 0) {
this.getNext();
} else {
console.log("All items downloaded");
}
}
};
How do I structure the code so that the completion of the first download can signal the initiation of the next one. Please note that this exercise is just for learning purposes, to understand how asynchronous coding works. In practice, I'm sure there are much better tools out there to download multiple files.

Try simple at first, it look like you copy paste codes and quite don't understand what they do.
Do a simple loop, that get the url, and print something.
var http = require('http');
URL = require('url').parse('http://www.timeapi.org/utc/now?format=%25F%20%25T%20-%20%25N')
URL['headers'] = {'User-Agent': 'Hello World'}
// launch 20 queries asynchronously
for(var i = 0; i < 20; i++) {
(function(i) {
console.log('Query ' + i + ' started');
var req = http.request(URL, function(res) {
console.log('Query ' + i + ' status: ' + res.statusCode + ' - ' + res.statusMessage);
res.on('data', function(content){
console.log('Query ' + i + ' ended - ' + content);
});
});
req.on('error', function(err) {
console.log('Query ' + i + ' return error: ' + err.message);
});
req.end();
})(i);
}
All the urls will be fetched asynchronously. You can observe that the response does not arrive in order, but are still processed correctly.
The difficulty with async is not to do the things is parallel, because you just write like a single task, and execute multiple time. It becomes complicated when you need for instance to wait for all tasks to finished before continuing. And for that, have a look at promises

Here is what I started out with. Figuring that each download was invoked asynchronously, they would all be independent of each other.
var http = require("http"),
fs = require("fs"),
input = process.argv[2],
folder = "C:/Users/Wiz/Downloads/",
regex = /(https?:\/\/)?([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w \.-]*)*\/?/,
urls = null,
url = "",
filename = "";
fs.readFile(input, "utf8",
function(e, data) {
console.log("Reading file: " + input);
if (e) console.log("Got error:" + e.message);
urls = data.split("\n");
for (var i = urls.length; i--;) {
url = urls[i];
if (!url.match(regex)) continue;
filename = folder + url.substring(url.lastIndexOf('/') + 1);
http.get(url, function(response) {
var file = fs.createWriteStream(filename);
response.on("end", function() {
file.end();
});
response.pipe(file);
})
}
});

I'm trying to reply to a tweet using Twit on Node.js

Not sure how to input the in_reply_to_status_id.
It's tweeting out fine, just not replying to the tweet with the mention in it.
The in_reply_to_status_id is part of the Twitter API, which Twit accesses, but can I use this in this context?
Any help would be greatly appreciated.
Here's the code:
var stream = T.stream('statuses/filter', { track: '#example'});
io.on('connection', function (socket) {
socket.on('chat message', function (msg) {
console.log('message: ' + msg);
P.post('statuses/update', { status: '#example' + ' ' + msg}, function (err, data, response) {
socket.emit('info', data.text);
socket.emit('userPic', data.user.profile_image_url);
console.log(data.user.profile_image_url);
});
});
stream.start();
stream.on('tweet', function (tweet) {
console.log(tweet);
// console.log('listening to tweets');
if (tweet.text.indexOf('#example') > -1) {
console.log("there is a tweet");
var number = Date.now();
var reply = replies[Math.floor(Math.random() * replies.length)];
var name = '#' + tweet.user.screen_name;
T.post('statuses/update', {in_reply_to_status_id: [name], status: reply + ' ' + number + ' ' + name}, function (err, data, response) {
console.log(reply + number);
socket.emit('reply', data.text);
});
}
});
});

The user name ID string was not being parsed correctly. The solution:
var nameID = tweet.id_str;
var name = tweet.user.screen_name;
T.post('statuses/update', {in_reply_to_status_id: nameID, status: reply + ' ' + number + ' #' + name}, function(err, data, response) {

GET http://js:port/socket.io/1/ 404 not found

Socket.io is having issues connecting up.
This is my Server code:
function chatserver(){
var express = require('express'),
app = express(),
server = require('http').createServer(app).listen(app.get('port'),function(){
console.log('Express server listening on port '+ app.get('port'));
});
var io = require('socket.io').listen(server),
users = {};
io.sockets.on('connection', function(socket){
socket.on('new user', function(data, callback){
if (data in users){
callback(false);
} else{
callback(true);
socket.nickname = data;
users[socket.nickname] = socket;
updateNicknames();
}
});
function updateNicknames(){
io.sockets.emit('usernames', Object.keys(users));
}
socket.on('send message', function(data, callback){
var msg = data.trim();
console.log('after trimming message is: ' + msg);
if(msg.substr(0,3) === '/w '){
msg = msg.substr(3);
var ind = msg.indexOf(' ');
if(ind !== -1){
var name = msg.substring(0, ind);
var msg = msg.substring(ind + 1);
if(name in users){
users[name].emit('whisper', {msg: msg, nick: socket.nickname});
console.log('message sent is: ' + msg);
console.log('Whisper!');
} else{
callback('Error! Enter a valid user.');
}
} else{
callback('Error! Please enter a message for your whisper.');
}
} else{
io.sockets.emit('new message', {msg: msg, nick: socket.nickname});
}
});
socket.on('disconnect', function(data){
if(!socket.nickname) return;
delete users[socket.nickname];
updateNicknames();
});
});
}
My HTML script code:
jQuery(function($){
var socket = io.connect('js/node_modules/socket.io/node_modules/socket.io-client/dist/socket.io.js');
var $nickForm = $('#setNick');
var $nickError = $('#nickError');
var $nickBox = $('#nickname');
var $users = $('#users');
var $messageForm = $('#send-message');
var $messageBox = $('#message');
var $chat = $('#chat');
$nickForm.submit(function(e){
e.preventDefault();
socket.emit('new user', $nickBox.val(), function(data){
if(data){
$('#nickWrap').hide();
$('#contentWrap').show();
} else{
$nickError.html('That username is already taken! Try again.');
}
});
$nickBox.val('');
});
socket.on('usernames', function(data){
var html = '';
for(i=0; i < data.length; i++){
html += data[i] + '<br/>'
}
$users.html(html);
});
$messageForm.submit(function(e){
e.preventDefault();
socket.emit('send message', $messageBox.val(), function(data){
$chat.append('<span class="error">' + data + "</span><br/>");
});
$messageBox.val('');
});
socket.on('new message', function(data){
$chat.append('<span class="msg"><b>' + data.nick + ': </b>' + data.msg + "</span><br/>");
});
socket.on('whisper', function(data){
$chat.append('<span class="whisper"><b>' + data.nick + ': </b>' + data.msg + "</span><br/>");
});
});
When I load up the website at first it's OK. But as soon as I click the chat box, in inspect elements/console I get to see "GET http://'js':5001/socket.io/1/?.." error appear. Please advise.

In server code, change this:
server = require('http').createServer(app).listen(port),
io = require('socket.io').listen(server),
to
var server = http.createServer(app).listen(app.get('port'), function(){
console.log('Express server listening on port ' + app.get('port'));
});
var io = require('socket.io').listen(server);
If you are in http server not express.io server, use this:
app.listen(app.get('port'), function(){
console.log("Express server listening on port " + app.get('port'));
});

Your main problem is client Jquery script.
var socket = io.connect('js/node_modules/socket.io/node_modules/socket.io-client/dist/socket.io.js');
It is not the proper way to server socket.io.js to client.
You need to change it to http://<ip/domain>:<port>/socket.io/socket.io.js
If Socket.io Server listens properly to your HTTP server, it will automatically serve the client file to via http://<ip/domain>:<port>/socket.io/socket.io.js. You don't need to find it or copy in public accessible folder as resources/js/socket.io.js & serve it manually.
Code sample Express 3.x -
Express 3 requires that you instantiate a http.Server to attach socket.io to first
var express = require('express')
, http = require('http');
//make sure you keep this order
var app = express();
var server = http.createServer(app);
var io = require('socket.io').listen(server);
//...
server.listen(8000);
Happy Coding :)