Develop Reference

Having difficulty with javascript(node.js) that needs to be synchronous

I have an express.js app that needs to run a script on the server in order to derive some values using functions later. Here's the gist of it:
shell.exec(commandString);
readFolder();
renderPage();
Essentially, I need to run a script on the server, then run the second function, then run the third function. These need to happen subsequently, but it seems that javascript moves on ahead with the the second and third function no matter what I do. I've tried promises, async, callbacks. All of which I only partially understand and seem to get zero progress.
I will admit that I am a javascript novice. I am working on a project with others and this task fell to me. I doubt this is the best way to accomplish our ultimate goals, but I am left with little choice. please help.
I'll put the entire post here for reference:
//Run script when post is rec'd from root and send to results page
app.post("/", (req, res) => {
var commandString;
//take values and create complete command for Astrum script
commandString = 'bash /home/astrum/Main/Astrum.sh -s ' + req.body.speed + ' -h ' + req.body.host + ' -u ' + req.body.username + ' -p ' + req.body.password;
//execute command in shell
shell.exec(commandString);
readFolder();
renderPage();
//Iterate thru filenames to create arrays for links and link labels
function readFolder() {
fs.readdir('./reports/html/', (err, files) => {
//variable & method for links to html records pages
ipAddressesLink = files; //this is initialized earlier, globally
//variable and method to remove file extension for link labels in pug
ipAddresses = files.map(removeExtension); //this is initialized earlier, globally
});
}
//function to remove last five characters of each element
function removeExtension(value) {
return value.substring(0, value.length - 5);
};
//function to render the page
function renderPage() {
res.render("results", {ipAddressesLink, ipAddresses, title: 'Results'});
}
res.end();
});

You could write it this way:
shell.exec(commandString, (error, stdout, stderr) => {
// Calling the 1st function after shell command is executed
readFolder();
});
function readFolder() {
fs.readdir('./reports/html/', (err, files) => {
// Some stuff
...
// Calls the 2nd function after fs is done reading files
renderPage();
});
}
function renderPage() {
const options = { ... }; // IP addresses etc.
res.render(
"results",
options,
// Calls the final function after render is finished
sendResponse
);
}
function sendResponse(err, html) {
// Sends the response. It’s possible that res.send() is the better solution here
res.end();
}
It’s just the general structure of the callback chain, definitely not the cleanest one. If you want better code structure and readability try switching to async / await syntax.

Is shell here the child_process module? If it is then you can pass an optional callback argument and call your functions from there.
shell.exec(commandString, (error, stdout, stderr) => {
const files = readFolder();
renderPage(files);
});
function readFolder() {
...
return fs.readdirSync(files);
}
function renderPage(files) {
...
}

Node Synchronous functions

Hi there i am having awful trouble with a function I am trying to execute.
I am trying to concat, compile and minify some js and coffeescript which i had working fine when i had it hard coded. My issue is i now created a function that search for the files and returns an array with the file paths. But i cant get the the function to in a synchronous fashion. I have tried so many thing just cant figure it out.
At this stage this is what i have,
This function is used to compile the coffee script this is also called by a async.series which works fine. You can see it calls getFiles which should return the array. But the callback seem to fire before the data s retured
function compileCoffee() {
async.series([
function(callback) {
data = getFiles("coffee");
callback(null, data)
},
function(data, callback) {
console.log(data)
shell.echo(grey("Compiling CoffeeScript files .... "));
if (shell.cat(data).exec('coffee -sc').exec('uglifyjs --compress', {
silent: true
}).to('./public/assets/js/app.min.js').code !== 0) {
shell.echo(red('Coffee script compile error'));
shell.exit(1);
}
shell.echo(green("CoffeeScript files compiled succesfully"));
},
], () => {});
}
Then this is the get files function. I am probly going abut this all wrong, if i am please let me know a better way. If not and you know how to get it working would be amazing.
function getFiles(filetype, callback) {
var files = [];
// Walker options
var walker = walk.walkSync('./frontend/js', {
followLinks: false
});
walker.on('file', function(root, stat, next) {
// Add this file to the list of files
var ext = stat.name.substr(stat.name.lastIndexOf('.') + 1);
if (ext == filetype) {
files.push(root + '/' + stat.name);
}
next();
});
walker.on('end', function() {
console.lof(files)
return files
})
}
Please some one help :D Thanks

getFiles doesn't actually return anything, but it does take a callback that should be called when getFiles completes. Pass the async callback to getFiles (or create another function that calls the callback):
function(callback) {
data = getFiles("coffee", callback);
},
You need to actually call this callback in your getFiles function:
walker.on('error', function(error) {
console.log(error)
callback(error);
});
walker.on('end', function() {
console.log(files)
callback(null, files);
});

Incorrect header check zlib

Running the code below to to download and unzip files. It works as intended when I try with one but when I do multiple at the same time I get the following error:
Error: incorrect header check at Zlib._handle.onerror
var downloadUnzipFile = function (mID) {
try {
// Read File
console.log("Started download/unzip of merchant: " + mID + " # " + new Date().format('H:i:s').toString());
request(linkConst(mID))
// Un-Gzip
.pipe(zlib.createGunzip())
// Write File
.pipe(fs.createWriteStream(fileName(mID)))
.on('error', function (err) {
console.error(err);
})
.on('finish', function() {
console.log("CSV created: " + fileName(mID));
console.log("Completed merchant: " + mID + " # " + new Date().format('H:i:s').toString());
//console.log("Parsing CSV...");
//csvReader(fileName);
});
} catch (e) {
console.error(e);
}
}
module.exports = function(sMerchants) {
var oMerchants = JSON.parse(JSON.stringify(sMerchants));
oMerchants.forEach(function eachMerchant(merchant) {
downloadUnzipFile(merchant.merchant_aw_id);
})
};
Any ideas?
Thanks
EDIT:
To clarify, i'd like to run through each item (merchant) in the array (merchants) and download a file + unzip it. The way I currently do it means it this downloading/zipping occurs at the sametime (which I think might be causing the error). When i remove the foreach loop and just try to download/zip one merchant the code works.

Yeah, as you suggest, it's likely that if you try to unzip too many files concurrently, you will run out of memory. Because you are handling streams, the unzip operations are asynchronous, meaning your forEach loop will continue to be called before each unzip operation completes. There are plenty of node packages that allow you to handle async operations so you can run the unzip function sequentially, but the simplest approach might just be to use a recursive function call. E.g.:
var downloadUnzipFile = function (mID) {
try {
// Read File
console.log("Started download/unzip of merchant: " + mID + " # " + new Date().format('H:i:s').toString());
return request(linkConst(mID))
// Un-Gzip
.pipe(zlib.createGunzip())
// Write File
.pipe(fs.createWriteStream(fileName(mID)))
} catch (e) {
console.log(e);
return false;
}
}
module.exports = function(sMerchants) {
var merchants = JSON.parse(JSON.stringify(sMerchants)),
count = 0;
downloadUnzipFile(merchants[count][merchant_aw_id])
.on('error', function(err){
console.log(err);
// continue unzipping files, even if you encounter an error. You can also remove these lines if you want the script to exit.
if(merchants[++count]){
downloadUnzipFile(merchants[count][merchant_aw_id]);
}
})
.on('finish', function() {
if(merchants[++count]){
downloadUnzipFile(merchants[count][merchant_aw_id]);
}
});
};
Haven't tested, of course. The main idea should work thought: call downloadUnzipFile recursively whenever the previous call errors out or finishes, as long as there are still items in the merchants array.

How do I get a value nested inside two assign functions and a forEach loop?

I'm writing a NodeJS module that copies a bunch of folders over from Dropbox, and creates a directory based on the folder structure. The part that is giving me a headache is that I need to get the names of all the folders in the main directory, then the names of all the files within a folder before moving on to the next function.
Here is my process right now:
Get the list of folders in main directory using dropboxClient.readdir()
Iterate through the folders and get the names sub-folders (again with dropboxClient.readdir())
Iterate through these sub-folders and get the names of the files.
If the file is a markdown file, add the name to a list
Return the list of all markdown files in the sub-directories
and some pseudocode:
function getListOfFiles() {
var subfolders = [];
var fileNames = [];
dbClient.readdir('', function(error, folders) {
folders.forEach(function(folder, index) {
subfolders.push(folder);
dbClient.readdir('/'+folder, function(error, subfolders) {
subfolders.forEach(function(subfolder, index) {
dbClient.readdir('/'+folder+'/'+subfolder, function(error, files) {
files.forEach(function(file, index) {
if (isMarkdownFile) {
fileNames.push(file)
}
});
});
});
});
}
});
return fileNames;
}
I've looked into a handful of packages that seem like they are supposed to solve this scenario, as well as JS Generators, but I'm not sure what the simplest solution should be. My code runs fine on Node 0.11.3, so generators are an options, but that's a new concept for me, and I can't seem to find examples that match up to mine.

utilize the async package. Specifically, the each, eachSeries, or eachLimit for the loops, as well as waterfall and series for control flow.
I'd recommend reading up on... each... of the each functions to figure out which is efficient and consistent/reliable for your situation.
function getListOfFiles(callback) {
async.waterfall([
// get a list of the top level folders
function (cb) {
dbClient.readdir('', function (error, topLevelFolders) {
if (error) return cb(error);
cb(null, topLevelFolders); // pass the folders to the next function (this is the "waterfall")
});
},
// get an array of all topLevel/subFolders combos
function (topLevelFolders, cb) {
var everySubFolder = [];
async.each(topLevelFolders, function (folder, subFolderCallback) {
dbClient.readdir(folder, function (error, subFolders) {
if (error) return subFolderCallback(error);
everySubFolder = everySubFolder.concat(subFolders);
});
}, function (error) {
if (error) return cb(error);
cb(null, everySubFolder); // pass all the folder/subfolder combos to the next function
});
},
// get an array of all the files in each folder/subfolder
function (everySubfolder, cb) {
var fileNames = [];
async.each(everySubFolder, function (folder, fileNameCallback) {
dbClient.readdir(folder, function (error, files) {
if (error) return fileNameCallback(error);
fileNames = fileNames.concat(files);
fileNameCallback();
});
}, function (error) {
if (error) return cb(error);
cb(null, fileNames); // pass every file combo to the waterfall callback function
});
}
], function (error, fileNames) {
if (error) return callback(error);
callback(null, fileNames); // all done! Every file combo goes the function's callback!
});
}
When you use it, you'll do:
getListOfFiles(function (err, files) {
// Voila! here are all your files
});
DEFINITELY add the .each error handling. If it bumps into an error during the loops, it will continue looping without it. Which, # of files dependent, could be a little while.

Walking a directory with Node.js [duplicate]

This question already has answers here:
What is the purpose of the var keyword and when should I use it (or omit it)?
(19 answers)
JavaScript closure inside loops – simple practical example
(44 answers)
Closed last year.
I've got a problem with this code in node.js. I want to recursively walk through a directory tree and apply the callback action to every file in the tree. This is my code at the moment:
var fs = require("fs");
// General function
var dive = function (dir, action) {
// Assert that it's a function
if (typeof action !== "function")
action = function (error, file) { };
// Read the directory
fs.readdir(dir, function (err, list) {
// Return the error if something went wrong
if (err)
return action(err);
// For every file in the list
list.forEach(function (file) {
// Full path of that file
path = dir + "/" + file;
// Get the file's stats
fs.stat(path, function (err, stat) {
console.log(stat);
// If the file is a directory
if (stat && stat.isDirectory())
// Dive into the directory
dive(path, action);
else
// Call the action
action(null, path);
});
});
});
};
The problem is that in the for each loop stat is called for every file via the variable path. When the callback is called, path already has another value and so it dives into the wrong directories or calls the action for the wrong files.
Probably this problem could easily get solved by using fs.statSync, but this is not the solution I would prefer, since it is blocking the process.

var path = dir + "/" + file;
You forgot to make path a local variable. Now it won't be changed behind your back in the loop.

Use node-dir for this. Because you need a separate action for directories and files, I'll give you 2 simple iterators using node-dir.
Asynchronously iterate the files of a directory and its subdirectories and pass an array of file paths to a callback.
var dir = require('node-dir');
dir.files(__dirname, function(err, files) {
if (err) throw err;
console.log(files);
//we have an array of files now, so now we'll iterate that array
files.forEach(function(filepath) {
actionOnFile(null, filepath);
})
});
Asynchronously iterate the subdirectories of a directory and its subdirectories and pass an array of directory paths to a callback.
var dir = require('node-dir');
dir.subdirs(__dirname, function(err, subdirs) {
if (err) throw err;
console.log(subdirs);
//we have an array of subdirs now, so now we'll iterate that array
subdirs.forEach(function(filepath) {
actionOnDir(null, filepath);
})
});

Another suitable library is filehound. It supports file filtering (if required), callbacks and promises.
For example:
const Filehound = require('filehound');
function action(file) {
console.log(`process ${file}`)
}
Filehound.create()
.find((err, files) => {
if (err) {
return console.error(`error: ${err}`);
}
files.forEach(action);
});
The library is well documented and provides numerous examples of common use cases.
https://github.com/nspragg/filehound
Disclaimer: I'm the author.

Not sure if I should really post this as an answer, but for your convenience and other users, here is a rewritten version of OP's which might prove useful. It provides:
Better error management support
A global completion callback which is called when the exploration is complete
The code:
/**
* dir: path to the directory to explore
* action(file, stat): called on each file or until an error occurs. file: path to the file. stat: stat of the file (retrived by fs.stat)
* done(err): called one time when the process is complete. err is undifined is everything was ok. the error that stopped the process otherwise
*/
var walk = function(dir, action, done) {
// this flag will indicate if an error occured (in this case we don't want to go on walking the tree)
var dead = false;
// this flag will store the number of pending async operations
var pending = 0;
var fail = function(err) {
if(!dead) {
dead = true;
done(err);
}
};
var checkSuccess = function() {
if(!dead && pending == 0) {
done();
}
};
var performAction = function(file, stat) {
if(!dead) {
try {
action(file, stat);
}
catch(error) {
fail(error);
}
}
};
// this function will recursively explore one directory in the context defined by the variables above
var dive = function(dir) {
pending++; // async operation starting after this line
fs.readdir(dir, function(err, list) {
if(!dead) { // if we are already dead, we don't do anything
if (err) {
fail(err); // if an error occured, let's fail
}
else { // iterate over the files
list.forEach(function(file) {
if(!dead) { // if we are already dead, we don't do anything
var path = dir + "/" + file;
pending++; // async operation starting after this line
fs.stat(path, function(err, stat) {
if(!dead) { // if we are already dead, we don't do anything
if (err) {
fail(err); // if an error occured, let's fail
}
else {
if (stat && stat.isDirectory()) {
dive(path); // it's a directory, let's explore recursively
}
else {
performAction(path, stat); // it's not a directory, just perform the action
}
pending--; checkSuccess(); // async operation complete
}
}
});
}
});
pending--; checkSuccess(); // async operation complete
}
}
});
};
// start exploration
dive(dir);
};

Don't reinvent the wheel - use and contribute to open source instead. Try one of the following:
https://github.com/pvorb/node-dive
https://github.com/coolaj86/node-walk

There is an NPM module for this:
npm dree
Example:
const dree = require('dree');
const options = {
depth: 5, // To stop after 5 directory levels
exclude: /dir_to_exclude/, // To exclude some pahts with a regexp
extensions: [ 'txt', 'jpg' ] // To include only some extensions
};
const fileCallback = function (file) {
action(file.path);
};
let tree;
// Doing it synchronously
tree = dree.scan('./dir', options, fileCallback);
// Doing it asynchronously (returns promise)
tree = await dree.scanAsync('./dir', options, fileCallback);
// Here tree contains an object representing the whole directory tree (filtered with options)

function loop( ) {
var item = list.shift( );
if ( item ) {
// content of the loop
functionWithCallback( loop );
} else {
// after the loop has ended
whatever( );
}
}