I'm comparing the output of childprocess.exec to a string, but I must have overlooked something since I didn't get the expected result.
function download_all(list, callback){
var i=0, cmd="";
function afterDownload(){...}
while(i<list.length)
{
cmd="[ -f ./downloads/"+list[i]+" ] && echo \"E\" || echo\""+list[i]+"\"";
exec(cmd, function(error, stdout, stderr){
if(stdout=="E")
{
console.log("Already Exist");
}else{
console.log("download "+LINK+""+stdout);
download(LINK+stdout, afterDownload());
}
});
i=i+1;
}
Basically, I check if a file exist, look at the output of the command, and if it is not E (which sign the file exist), download it.
The problem is, even when the file exist, the app try to download LINK+E, which doesn't exist and of course fail.
I've tried with === instead of ==, and " instead of ', but it didn't changed anything.
Is there some character in stdout other than E?
NodeJS has the "fs" module which takes care of that for you. The documentation is at http://nodejs.org/api/fs.html
You can do this:
fs.exists(list[i], function (exists) {
console.log("exists = ", exists);
});
Related
I have an express.js app that needs to run a script on the server in order to derive some values using functions later. Here's the gist of it:
shell.exec(commandString);
readFolder();
renderPage();
Essentially, I need to run a script on the server, then run the second function, then run the third function. These need to happen subsequently, but it seems that javascript moves on ahead with the the second and third function no matter what I do. I've tried promises, async, callbacks. All of which I only partially understand and seem to get zero progress.
I will admit that I am a javascript novice. I am working on a project with others and this task fell to me. I doubt this is the best way to accomplish our ultimate goals, but I am left with little choice. please help.
I'll put the entire post here for reference:
//Run script when post is rec'd from root and send to results page
app.post("/", (req, res) => {
var commandString;
//take values and create complete command for Astrum script
commandString = 'bash /home/astrum/Main/Astrum.sh -s ' + req.body.speed + ' -h ' + req.body.host + ' -u ' + req.body.username + ' -p ' + req.body.password;
//execute command in shell
shell.exec(commandString);
readFolder();
renderPage();
//Iterate thru filenames to create arrays for links and link labels
function readFolder() {
fs.readdir('./reports/html/', (err, files) => {
//variable & method for links to html records pages
ipAddressesLink = files; //this is initialized earlier, globally
//variable and method to remove file extension for link labels in pug
ipAddresses = files.map(removeExtension); //this is initialized earlier, globally
});
}
//function to remove last five characters of each element
function removeExtension(value) {
return value.substring(0, value.length - 5);
};
//function to render the page
function renderPage() {
res.render("results", {ipAddressesLink, ipAddresses, title: 'Results'});
}
res.end();
});
You could write it this way:
shell.exec(commandString, (error, stdout, stderr) => {
// Calling the 1st function after shell command is executed
readFolder();
});
function readFolder() {
fs.readdir('./reports/html/', (err, files) => {
// Some stuff
...
// Calls the 2nd function after fs is done reading files
renderPage();
});
}
function renderPage() {
const options = { ... }; // IP addresses etc.
res.render(
"results",
options,
// Calls the final function after render is finished
sendResponse
);
}
function sendResponse(err, html) {
// Sends the response. It’s possible that res.send() is the better solution here
res.end();
}
It’s just the general structure of the callback chain, definitely not the cleanest one. If you want better code structure and readability try switching to async / await syntax.
Is shell here the child_process module? If it is then you can pass an optional callback argument and call your functions from there.
shell.exec(commandString, (error, stdout, stderr) => {
const files = readFolder();
renderPage(files);
});
function readFolder() {
...
return fs.readdirSync(files);
}
function renderPage(files) {
...
}
I have some code that takes an uploaded file, executes it and gets the output. It then compares this output against expected output to check if the script did as expected.
I am now trying to improve this functionality so that an uploaded file will be run several times, each time being checked against a different expected output, or "test case". I then want to push "correct" or "incorrect" onto a results array, so that I can go through that array at the end and check whether there are any "incorrect" (whether the file failed any test case).
I have tried just callbacks within each function.
I have tried using await and async on the getArray as seen below
Using both callbacks and async together.
This is the parent function code that calls for the array to be created, and wants to iterate through it after it has been created.
var resultsArr = await getResults(file.name, files.length, markerDir);
//file.name is the name from the uploaded file object
//files.length is the number of subdirectories (number of testcases to run against)
//markerDir is the str path to where these testcases are stored
if (resultsArr){
for(var i=0;i<resultsArr.length;i++) {
if (resultsArr[i] == "incorrect"){
checkForMarkerCb('incorrect'); //Calls back to frontend to
break; //display result on web app
}
else if (i+1 == resultsArr.length) {
checkForMarkerCb('correct');
}
}
}
The following is inside the getResults function that is called above
for(var i=1; i<=fileLength; i++) {
var sampleOut = markerDir + '/test' + i + '/stdout.txt';
//Grab expected stdout.txt
var markerOut = fs.readFileSync(sampleOut, 'utf-8', function(err){
if (err){
throw err;
};
});
//Run the file and grab the output
executeFile(filename, function(fileOut){
//Compare output with sample stdout
if (markerOut == fileOut){
resultsArr.push('correct');
}
else {
resultsArr.push('incorrect');
}
});
}
//If results array has a response for each testcase
if (resultsArr.length == fileLength) {
return resultsArr;
}
Implementation of executeFile() as requested:
function executeFile(filename, execFileCb){
//pathToUpload is a str path to where the upload is stored
const child = execFile('python', [pathToUpload], (err,stdout,stderr) => {
if (err) {
throw err;
}
execFileCb(stdout); //Callback with output of file
});
}
function executeFileAsync(filename) {
return new Promise(function(resolve,reject){
executeFile(filename, function(err, data){
if (err !== null) reject(err);
else resolve(data);
});
});
}
which was called inside getResults() using
var fileOut = await executeFileAsync(filename)
The initial function that calls getResults().
getResults(): which gets the path to each directory and calls pushes the results of comparing outputs onto a results array.
executeFile(): uses 'child_process' to run a file and calls back with the output.
I expect the code to wait for getResults to return with the resultsArr so that the for loop can iterate through and check for any "incorrect". Instead, getResults returns before resultsArr is populated.
Using some logging, I see that the code for checking markerOut == fileOut is executed at the end after the getResults() for loop has already completed. I tried setting up the call to executeFile() to also be an async/await similar to how getResults() is called but still no change.
I may not be using async/callbacks correctly, any help is greatly appreciated.
Your executeFileAsync function currently calls executeFile with a callback that is expecting two arguments, but executeFile then does call this execFileCb always with only one argument which is interpreted as an error. It also should not use throw in an asynchronous callback.
Instead, merge them into one function:
function executeFile(filename) {
return new Promise(function(resolve,reject){
//pathToUpload is a str path to where the upload is stored
const child = execFile('python', [pathToUpload], (err,stdout,stderr) => {
if (err) reject(err);
else resolve(stdout); //Callback with output of file
});
});
}
I have a function that tests whether a file exists or not before editing it. I use fs.stat.
fs.stat('../fill/bower.json', function (err, stats) {
if (err) {
console.log('You don\'t have a ' + clc.red('bower.json') + ' file! Type ' + clc.bgBlack.white('touch bower.json') + ' to get started.');
return;
} if (stats.isFile()) {
var json = JSON.parse(fs.readFileSync('../bower.json', 'utf8')),
string = '\n Dependencies: ' + json;
fs.writeFile('../fill/README.md,', string, 'utf8');
console.log('it\'s saved!');
}
})
However, every time I run it (bower.json doesn't exist on purpose), it returns undefined before You don't have a bower.json file!. Why does this happen and how can I stop the function printing undefined?
Edit: for reference, here's my terminal window after running the command:
Why is undefined printed, and what do I do to have that not be displayed?
You're returning nothing or undefined from your reading function.
Gist for posterity
I have a file path list file_paths, and I want to detect which file exists.
If any file exists, I want to read that one. Otherwise call another function,
not_found for example.
I wish to use async.detect but I found no way to add a 'Not Found' callback
when all the iterators return false.
I've tried this one, but no work. Returned undefined and nothing outputted.
async = require 'async'
async.detect [1,2,3], (item, callback) ->
callback true if item == 4
, (result) ->
console.log result ? result : 'Not Found'
If there's another way to do it, please add it to the answer.
from the documentation you mentioned.
in case of detect(arr, iterator, callback)
callback(result) - A callback which is called as soon as any iterator
returns true, or after all the iterator functions have finished.
Result will be the first item in the array that passes the truth test (iterator) or the value undefined if none passed.
from your question you want to find a way to detect if no file in list is found, which could be done by comparing the result with undefined and checking whether this condition is true.
like
async.detect(['file1','file2','file3'], fs.exists, function(result){
if(typeof(result)=="undefined") {
//none of the files where found so undefined
}
});
I would use async.each and use fs.exists to detect if the file exists. If it exists, then read the file otherwise, call the not found function then proceed to the next item. See sample code I have written on top of my head below.
async.each(file_paths, processItem, function(err) {
if(err) {
console.log(err);
throw err;
return;
}
console.log('done reading file paths..');
});
function notFound(file_path) {
console.log(file_path + ' not found..');
}
function processItem(file_path, next) {
fs.exists(file_path, function(exists) {
if(exists) {
//file exists
//do some processing
//call next when done
fs.readFile(file_path, function (err, data) {
if (err) throw err;
//do what you want here
//then call next
next();
});
}
else {
//file does not exist
notFound(file_path);
//proceed to next item
next();
}
});
}
This question already has answers here:
What is the purpose of the var keyword and when should I use it (or omit it)?
(19 answers)
JavaScript closure inside loops – simple practical example
(44 answers)
Closed last year.
I've got a problem with this code in node.js. I want to recursively walk through a directory tree and apply the callback action to every file in the tree. This is my code at the moment:
var fs = require("fs");
// General function
var dive = function (dir, action) {
// Assert that it's a function
if (typeof action !== "function")
action = function (error, file) { };
// Read the directory
fs.readdir(dir, function (err, list) {
// Return the error if something went wrong
if (err)
return action(err);
// For every file in the list
list.forEach(function (file) {
// Full path of that file
path = dir + "/" + file;
// Get the file's stats
fs.stat(path, function (err, stat) {
console.log(stat);
// If the file is a directory
if (stat && stat.isDirectory())
// Dive into the directory
dive(path, action);
else
// Call the action
action(null, path);
});
});
});
};
The problem is that in the for each loop stat is called for every file via the variable path. When the callback is called, path already has another value and so it dives into the wrong directories or calls the action for the wrong files.
Probably this problem could easily get solved by using fs.statSync, but this is not the solution I would prefer, since it is blocking the process.
var path = dir + "/" + file;
You forgot to make path a local variable. Now it won't be changed behind your back in the loop.
Use node-dir for this. Because you need a separate action for directories and files, I'll give you 2 simple iterators using node-dir.
Asynchronously iterate the files of a directory and its subdirectories and pass an array of file paths to a callback.
var dir = require('node-dir');
dir.files(__dirname, function(err, files) {
if (err) throw err;
console.log(files);
//we have an array of files now, so now we'll iterate that array
files.forEach(function(filepath) {
actionOnFile(null, filepath);
})
});
Asynchronously iterate the subdirectories of a directory and its subdirectories and pass an array of directory paths to a callback.
var dir = require('node-dir');
dir.subdirs(__dirname, function(err, subdirs) {
if (err) throw err;
console.log(subdirs);
//we have an array of subdirs now, so now we'll iterate that array
subdirs.forEach(function(filepath) {
actionOnDir(null, filepath);
})
});
Another suitable library is filehound. It supports file filtering (if required), callbacks and promises.
For example:
const Filehound = require('filehound');
function action(file) {
console.log(`process ${file}`)
}
Filehound.create()
.find((err, files) => {
if (err) {
return console.error(`error: ${err}`);
}
files.forEach(action);
});
The library is well documented and provides numerous examples of common use cases.
https://github.com/nspragg/filehound
Disclaimer: I'm the author.
Not sure if I should really post this as an answer, but for your convenience and other users, here is a rewritten version of OP's which might prove useful. It provides:
Better error management support
A global completion callback which is called when the exploration is complete
The code:
/**
* dir: path to the directory to explore
* action(file, stat): called on each file or until an error occurs. file: path to the file. stat: stat of the file (retrived by fs.stat)
* done(err): called one time when the process is complete. err is undifined is everything was ok. the error that stopped the process otherwise
*/
var walk = function(dir, action, done) {
// this flag will indicate if an error occured (in this case we don't want to go on walking the tree)
var dead = false;
// this flag will store the number of pending async operations
var pending = 0;
var fail = function(err) {
if(!dead) {
dead = true;
done(err);
}
};
var checkSuccess = function() {
if(!dead && pending == 0) {
done();
}
};
var performAction = function(file, stat) {
if(!dead) {
try {
action(file, stat);
}
catch(error) {
fail(error);
}
}
};
// this function will recursively explore one directory in the context defined by the variables above
var dive = function(dir) {
pending++; // async operation starting after this line
fs.readdir(dir, function(err, list) {
if(!dead) { // if we are already dead, we don't do anything
if (err) {
fail(err); // if an error occured, let's fail
}
else { // iterate over the files
list.forEach(function(file) {
if(!dead) { // if we are already dead, we don't do anything
var path = dir + "/" + file;
pending++; // async operation starting after this line
fs.stat(path, function(err, stat) {
if(!dead) { // if we are already dead, we don't do anything
if (err) {
fail(err); // if an error occured, let's fail
}
else {
if (stat && stat.isDirectory()) {
dive(path); // it's a directory, let's explore recursively
}
else {
performAction(path, stat); // it's not a directory, just perform the action
}
pending--; checkSuccess(); // async operation complete
}
}
});
}
});
pending--; checkSuccess(); // async operation complete
}
}
});
};
// start exploration
dive(dir);
};
Don't reinvent the wheel - use and contribute to open source instead. Try one of the following:
https://github.com/pvorb/node-dive
https://github.com/coolaj86/node-walk
There is an NPM module for this:
npm dree
Example:
const dree = require('dree');
const options = {
depth: 5, // To stop after 5 directory levels
exclude: /dir_to_exclude/, // To exclude some pahts with a regexp
extensions: [ 'txt', 'jpg' ] // To include only some extensions
};
const fileCallback = function (file) {
action(file.path);
};
let tree;
// Doing it synchronously
tree = dree.scan('./dir', options, fileCallback);
// Doing it asynchronously (returns promise)
tree = await dree.scanAsync('./dir', options, fileCallback);
// Here tree contains an object representing the whole directory tree (filtered with options)
function loop( ) {
var item = list.shift( );
if ( item ) {
// content of the loop
functionWithCallback( loop );
} else {
// after the loop has ended
whatever( );
}
}