I am new in Node.js . All I did in here is get file list in folder... Everything in Node.js seem to async but my function I want it in synchronous way.
So, I do as follow.
function getFiles() {
var file = [];
var walker = walk.walk('./files');
walker.on('file', function (root, stat, next) {
file.push(root + '/' + stat.name);
next();
})
walker.on('end', function () {
console.log(JSON.stringify(file));
})
return file;}
It worked as I expected :
["./files/1.mp3","./files/2.mp3","./files/3.mp3","./files/4.mp3","./files/5.mp3","./files/6.mp3","./files/7.mp3","./files/8.mp3"]
but when I assigned that function to variable
var list = getFiles();
response.writeHead(200, {'Content-type':'application/json'});
response.end(JSON.stringify(list));
It always returned nothing, I think that getFiles() run in another thread so can not receive value of data. Thank for your reading.
I can shed some light on the behavior you are experiencing by outlining the flow of the application as it is run:
call to getFiles
declare files array and walker
bind walker event "file" and "end" to callbacks
return files array
walker file event fires
walker end event fires
as you can see the events are firing out of band with the method call. To deal with this the common node.js approach is to setup your code something like the following:
function getFiles(callback) {
var file = [];
var walker = walk.walk('./files');
walker.on('file', function (root, stat, next) {
file.push(root + '/' + stat.name);
next();
})
walker.on('end', function () {
callback(file);
})
}
now when you go to execute this method you would do something like this:
getFiles(function(list){
response.writeHead(200, {'Content-type':'application/json'});
response.end(JSON.stringify(list));
});
obviously this is a little unsightly since the controller now has to create a callback scenario and the getFiles method need to execute that callback in course. Another approach is to use the Promises concept, which I will leave to the discovery of the reader with the following link: https://github.com/kriskowal/q
Async functions return before they are ready
You can't return data thats not there in your own code
Async functions often take a callback argument that is executed when they are ready
You own code could ask for its own callbacks
`
function getFiles(callBack) {
var file = [];
var walker = walk.walk('./files');
walker.on('file', function (root, stat, next) {
file.push(root + '/' + stat.name);
next();
})
walker.on('end', function () {
console.log(JSON.stringify(file));
callBack(file);
})
}
// assuming server environment, express, connect, etc...
app.get('/list', function(req, res){
getFiles(function(D){ res.json(D) });
});
`
Related
I have an express.js app that needs to run a script on the server in order to derive some values using functions later. Here's the gist of it:
shell.exec(commandString);
readFolder();
renderPage();
Essentially, I need to run a script on the server, then run the second function, then run the third function. These need to happen subsequently, but it seems that javascript moves on ahead with the the second and third function no matter what I do. I've tried promises, async, callbacks. All of which I only partially understand and seem to get zero progress.
I will admit that I am a javascript novice. I am working on a project with others and this task fell to me. I doubt this is the best way to accomplish our ultimate goals, but I am left with little choice. please help.
I'll put the entire post here for reference:
//Run script when post is rec'd from root and send to results page
app.post("/", (req, res) => {
var commandString;
//take values and create complete command for Astrum script
commandString = 'bash /home/astrum/Main/Astrum.sh -s ' + req.body.speed + ' -h ' + req.body.host + ' -u ' + req.body.username + ' -p ' + req.body.password;
//execute command in shell
shell.exec(commandString);
readFolder();
renderPage();
//Iterate thru filenames to create arrays for links and link labels
function readFolder() {
fs.readdir('./reports/html/', (err, files) => {
//variable & method for links to html records pages
ipAddressesLink = files; //this is initialized earlier, globally
//variable and method to remove file extension for link labels in pug
ipAddresses = files.map(removeExtension); //this is initialized earlier, globally
});
}
//function to remove last five characters of each element
function removeExtension(value) {
return value.substring(0, value.length - 5);
};
//function to render the page
function renderPage() {
res.render("results", {ipAddressesLink, ipAddresses, title: 'Results'});
}
res.end();
});
You could write it this way:
shell.exec(commandString, (error, stdout, stderr) => {
// Calling the 1st function after shell command is executed
readFolder();
});
function readFolder() {
fs.readdir('./reports/html/', (err, files) => {
// Some stuff
...
// Calls the 2nd function after fs is done reading files
renderPage();
});
}
function renderPage() {
const options = { ... }; // IP addresses etc.
res.render(
"results",
options,
// Calls the final function after render is finished
sendResponse
);
}
function sendResponse(err, html) {
// Sends the response. It’s possible that res.send() is the better solution here
res.end();
}
It’s just the general structure of the callback chain, definitely not the cleanest one. If you want better code structure and readability try switching to async / await syntax.
Is shell here the child_process module? If it is then you can pass an optional callback argument and call your functions from there.
shell.exec(commandString, (error, stdout, stderr) => {
const files = readFolder();
renderPage(files);
});
function readFolder() {
...
return fs.readdirSync(files);
}
function renderPage(files) {
...
}
In my controller called MapController I'm doing a function to do a parse of remote json files, and from an if-else structure add some values in an array called "parsewebservice", apparently everything is working fine but console.log ( parsewebservice); is not returning the values that were passed to the array "parsewebservice" in the place where it is returning it empty. But when I put it inside the forEach it returns, but everything cluttered and repeated then is not the right way.
I wanted to know why the values that were passed to the array "parsewebservice" are not going along with the variable after populada and what would be the correct way to do it?
Here is my code below:
/**
* MapController
*
* #description :: Server-side logic for managing Maps
* #help :: See http://sailsjs.org/#!/documentation/concepts/Controllers
*/
module.exports = {
index: function(req, res, next) {
Data.find(function foundData(err, datas) {
if (err) return next(err);
var parsewebservice = [];
datas.forEach(function(data, index) {
var req = require("request");
var url = data.address + "?f=pjson";
req(url, function(err, res, retorno) {
if (err) {
console.log(err);
} else {
var camadas = JSON.parse(retorno);
if (camadas.mapName) {
camadas.layers.forEach(function(campo, i) {
if (campo.subLayerIds != null) {
} else if (campo.subLayerIds == null) {
parsewebservice.push([i, "dynamicMapLayer", campo.name, data.address]);
}
});
} else if (camadas.serviceDataType) {
parsewebservice.push([null, "imageMapLayer", camadas.name, data.address]);
} else if (camadas.type) {
parsewebservice.push([null, "featureLayer", camadas.name, data.address]);
}
}
});
});
console.log(parsewebservice);
});
},
};
My first comment has to be that you should not combine function(req, res) with var req = require('request')... you lose your access to the original req object!
So, you need to run a list of async tasks, and do something when they are all complete. That will never be entirely easy, and no matter what, you will have to get used to the idea that your code does not run from top to bottom as you've written it. Your console.log at the bottom runs before any of the callbacks (functions you pass in) you pass to your external requests.
The right way to do this is to use promises. It looks like you are using this request library, whose returned requests can only accept callbacks, not be returned as promises. You can create your own promise wrapper for them, or use an alternative library (several are recommended on the page).
I don't want to write a whole intro-to-promises right here, so what I will do is give you a less pretty, but maybe more understandable way to run some code at the completion of all your requests.
Data.find(function foundData(err, datas) {
if (err) return next(err);
var parsewebservice = [];
// here we will write some code that we will run once per returned data
var processResponse = function(resp) {
parsewebservice.push(resp);
if(parsewebservice.length >= datas.length) {
// we are done, that was the final request
console.log(parsewebservice);
return res.send({data: parsewebservice)}); // or whatever
}
};
datas.forEach(function(data, index) {
var request = require("request");
var url = data.address + "?f=pjson";
request(url, function(err, res, retorno) {
// do some processing of retorno...
// call our function to handle the result
processResponse(retorno);
});
});
console.log(parsewebservice); // still an empty array here
});
I solved the problem.
the "request" module is asynchronous so we need to wait for it to respond and then send the response to the view.
To do this we created a function called "foo" to contain the foreach and the request, we made a callback of that function and finally we made the response (res.view) within that function, so that the controller response would only be sent after the response of the "foo" function to the callback. So we were able to parse.json the data from the "data" collection using foreach and the "request" module and send the objects to the view.
Many thanks to all who have helped me, my sincere thanks.
I'm a Node.js newbie trying to send back data to the browser by accessing a function load_blocks() in an external file I wrote and call it with res.send().
I have two simple files:
The first one is the typical app.js file:
const express = require('express');
const app = express();
//import my file
const blockchain = require('./load_blockchain.js');
app.get('/', function(req, res){
res.send('<h3>' + blockchain.load_blocks() + '</h3>');
});
app.listen(3000, function(){
console.log('Example app listening on port 3000!');
});
load_blockchain.js
This file has a function load_blocks() which is meant to read the lines of a file and then return them as text. But when I load my page, the <h3> tag reads undefined
var fs = require('fs');
var readline = require('readline');
module.exports = {
load_blocks: function (){
//Load The File
var return_string = "";
var rd = readline.createInterface({
input:fs.createReadStream('/home/blockchain_data/1.dat$
output: process.stdout,
console: true
});
rd.on('line', function(line){
console.log(line);
return_string += line;
});
return return_string;
}
};
Shouldn't the h3 tag read the data that is in the file I passed now? What is going wrong?
load_blocks() is asynchronous. It does not return the finished string because the readline code inside that function is asynchronous and has not yet finished when load_blocks() returns so thus return_string does not yet have the value you want when you return.
This is a classic asynchronous programming problem in Javascript. Your load_blocks function needs to either return a promise or call a callback when it finishes its work and the caller has to use that returned promise or callback to obtain the final result.
For a lot more info on returning an asynchronous value, see the various options offered here: How do I return the response from an asynchronous call?
You could change the code to something like this:
app.get('/', function(req, res){
blockchain.load_blocks().then(val => {
res.send('<h3>' + val + '</h3>');
}).catch(err => {
console.log(err);
res.status(500).end();
});
});
And, then change blockchain.load_blocks() to return a promise that resolves when you have the final value.
The code you show for load_blocks appears to just be reading an entire file. I don't understand any reason that you are reading it line by line. If you just want to read a whole file, you can just use fs.readFile() to read the whole file.
module.exports = {
load_blocks: function() {
return new Promise((resolve, reject) => {
fs.readFile('/home/blockchain_data/1.dat$', function(err, data) {
if (err) {
reject(err);
} else {
resolve(data);
}
});
});
}
};
Hi there i am having awful trouble with a function I am trying to execute.
I am trying to concat, compile and minify some js and coffeescript which i had working fine when i had it hard coded. My issue is i now created a function that search for the files and returns an array with the file paths. But i cant get the the function to in a synchronous fashion. I have tried so many thing just cant figure it out.
At this stage this is what i have,
This function is used to compile the coffee script this is also called by a async.series which works fine. You can see it calls getFiles which should return the array. But the callback seem to fire before the data s retured
function compileCoffee() {
async.series([
function(callback) {
data = getFiles("coffee");
callback(null, data)
},
function(data, callback) {
console.log(data)
shell.echo(grey("Compiling CoffeeScript files .... "));
if (shell.cat(data).exec('coffee -sc').exec('uglifyjs --compress', {
silent: true
}).to('./public/assets/js/app.min.js').code !== 0) {
shell.echo(red('Coffee script compile error'));
shell.exit(1);
}
shell.echo(green("CoffeeScript files compiled succesfully"));
},
], () => {});
}
Then this is the get files function. I am probly going abut this all wrong, if i am please let me know a better way. If not and you know how to get it working would be amazing.
function getFiles(filetype, callback) {
var files = [];
// Walker options
var walker = walk.walkSync('./frontend/js', {
followLinks: false
});
walker.on('file', function(root, stat, next) {
// Add this file to the list of files
var ext = stat.name.substr(stat.name.lastIndexOf('.') + 1);
if (ext == filetype) {
files.push(root + '/' + stat.name);
}
next();
});
walker.on('end', function() {
console.lof(files)
return files
})
}
Please some one help :D Thanks
getFiles doesn't actually return anything, but it does take a callback that should be called when getFiles completes. Pass the async callback to getFiles (or create another function that calls the callback):
function(callback) {
data = getFiles("coffee", callback);
},
You need to actually call this callback in your getFiles function:
walker.on('error', function(error) {
console.log(error)
callback(error);
});
walker.on('end', function() {
console.log(files)
callback(null, files);
});
I need help.
I've been trying to wrap my head around async programming with node.js and socket.io for a day now. I understand that I need some flow control but I don't seem to understand how to properly implement it.
I have a redis datastore that has modules stored in a set let's say 'moda','modb'
instances of those modules are 'moda:instances' and in 'modb:instances' the properties of those instances are stored in 'moda:instancea' and 'modb:instanceb' as a hash.
I am trying to get the following json:
"moda": {"instancea": {"property1": "value1", "property2", "value2"}}, "modb": {"instanceb": {"property1": "value1"}}
Could someone give me a little push in the right direction?
Here is my current code:
var io = require('socket.io').listen(2000);
var redis = require('redis').createClient();
var http = require('http');
var async = require('async');
var step = require('step');
io.sockets.on('connection', function (socket) {
var notifications = require('redis').createClient();
notifications.subscribe("notification");
notifications.on("message", function (channel, message) {
socket.send(message);
console.log(channel + ':' + message);
});
socket.on('modules', function(params, callback) {
var response = {};
async.series([
function (callback) {
console.log('1>');
redis.smembers('modules', function (err, modules) {
async.forEachSeries(modules, function(module, moduleCallback) {
response[module] = {}
redis.smembers(module + ':instances', function(err, instances) {
async.forEachSeries(instances, function(instance, instanceCallback) {
response[module][instance] = {}
console.log('2>' + module + ':' +instance);
instanceCallback();
});
moduleCallback();
});
});
callback();
});
},
function (callback) {
console.log('3');
callback();
}
], function() {
console.log(JSON.stringify(response));
});
});
});
The output from this code is:
info - socket.io started
debug - client authorized
info - handshake authorized JMMn1I8aiOMGCMPOhC11
debug - setting request GET /socket.io/1/websocket/JMMn1I8aiOMGCMPOhC11
debug - set heartbeat interval for client JMMn1I8aiOMGCMPOhC11
debug - client authorized for
debug - websocket writing 1::
1>
3
{"moda":{}}
2>moda:instancea
2>moda:instanceb
2>modb:instancea
The problem comes from the fact forEachSeries requires an additional callback as a third parameter (to be called when the whole processing is complete). You cannot just put some code after forEachSeries hoping it will be called once it is complete.
Here is your code modified:
var response = {};
async.series([
function (callback) {
console.log('1>');
redis.smembers('modules', function (err, modules) {
async.forEachSeries(modules, function(module, moduleCallback) {
response[module] = {}
redis.smembers(module + ':instances', function(err, instances) {
async.forEachSeries(instances, function(instance, instanceCallback) {
response[module][instance] = {}
console.log('2>' + module + ':' +instance);
instanceCallback();
}, moduleCallback );
});
}, callback );
});
},
function (callback) {
console.log('3');
callback();
}],
function() {
console.log(JSON.stringify(response));
});
Note how the callback, and moduleCallback are used as a third parameter. The output is:
1>
2>moda:instancea
2>moda:instanceb
2>modb:instancea
3
{"moda":{"instancea":{},"instanceb":{}},"modb":{"instancea":{}}}
which I guess is what you expected.
Additional remark: forEachSeries will process everything in sequence, the next operation waiting for the previous one to complete. This will generate plenty of roundtrips to Redis. forEach should be much more efficient here to leverage pipelining.
Take a look also at promise concept, with promises you configure the flow much more readable way.
First you need to prepare promise versions of redis methods that you're using:
var promisify = require('deferred').promisify;
var RedisClient = require('redis').RedisClient;
RedisClient.prototype.psmembers = promisify(RedisClient.prototype.smembers);
Then you can construct your flow as:
console.log('1>');
redis.psmembers('modules').map(function (module) {
response[module] = {};
return redis.psmembers(module + ':instances').map(function (instance) {
response[module][instance] = {};
console.log('2>' + module + ':' +instance);
});
}).end(function () {
console.log('3');
console.log(JSON.stringify(response));
});
Check: https://github.com/medikoo/deferred