I have some code that loops through a directory and retrieves the file name of each file. The code then retrieves the contents of each file (usually a number or a short text).
var config = {};
config.liveProcValues = {};
var path = require('path');
walk = function(dir, done) {
var results = {};
fs.readdir(dir, function(err, list) {
if (err) return done(err);
var pending = list.length;
if (!pending) return done(null, results);
list.forEach(function(file) {
file = path.resolve(dir, file);
fs.stat(file, function(err, stat) {
if (stat && stat.isDirectory()) {
walk(file, function(err, res) {
results = results.concat(res);
if (!--pending) done(null, results);
});
} else {
fs.readFileSync(file, 'utf8', function(err, data) {
if (err) {
contents = err;
} else {
contents = data;
}
console.log(filename + " - " + contents);
filename = file.replace(/^.*[\\\/]/, '');
config.liveProcValues[filename] = contents;
});
The console.log line successfully outputs the right information, however when trying to store it into JSON:
config.liveProcValues[filename] = contents;
It simply does remember the information.
walk("testdirectory", function(err, results) {
if (err) throw err;
});
// Output the configuration
console.log(JSON.stringify(config, null, 2));
You have to make sure that you are accessing the data after the filesystem was traversed. In order to do that you have to move the console.log into the walk callback:
walk("testdirectory", function(err, results) {
if (err) throw err;
// Output the configuration
console.log(JSON.stringify(results, null, 2));
});
See Why is my variable unaltered after I modify it inside of a function? - Asynchronous code reference for more info.
That alone won't solve the issue though, since you have a couple of logic errors in your code. You are trying to treat an object as an array (results.concat) and you are not always calling done when you are done (in particular, you are not calling done after you finished reading the files in a directory).
Here is a version that should come closer do what you want.
This uses Object.assign to merge two objects, which is not available in Node yet, but you can find modules that provide the same functionality.
Note thats I also removed the whole config object. It's cleaner if you work with results.
var path = require('path');
function walk(dir, done) {
var results = {};
fs.readdir(dir, function(err, list) {
if (err) return done(err);
if (!list.length) return done(null, results);
var pending = list.length;
list.forEach(function(file) {
file = path.resolve(dir, file);
fs.stat(file, function(err, stat) {
if (stat && stat.isDirectory()) {
walk(file, function(err, res) {
if (!err) {
// Merge recursive results
Object.assign(results, res);
}
if (!--pending) done(null, results);
});
} else {
fs.readFile(file, 'utf8', function(err, data) {
var contents = err || data;
console.log(file + " - " + contents);
file = file.replace(/^.*[\\\/]/, '');
// Assign the result to `results` instead of a shared variable
results[file] = contents;
// Need to call `done` if there are no more files to read
if (!--pending) done(null, results);
});
}
});
});
});
}
But instead of writing your own walk implementation, you could also use an existing package.
Related
I have a problem with my html-pdf document creation. The problem is that often the code runs to fast to complete the process of pdf-docutment creation. The Processes consists out of building an HTML-String by replacing placeholders in an Html file. Below you see the code what happens afterwards.
Object.keys(setter).forEach(function(element, key, _array) {
var regex = new RegExp(element, "g");
data = data.replace(regex, setter[element])
})
var result = data;
fs.writeFile(mergeFileRes, result, 'utf8', function (err) {
if(err) {
console.log(err);
return;
} else {
let html2 = fs.readFileSync(mergeFileRes, 'utf8');
let options = {
format: 'a4' ,
"directory" : "/tmp",
};
if(html2){
pdf.create(html2, options).toStream(function(err, stream2){
if(err) console.log(err);
stream2.pipe(res);
stream2.on('end', function () {
try{
fs.unlink(mergeFileRes)
console.log(3090, "deleted file");
}
catch (err){
console.log(3090, "Did not delete file");
}
});
});
} else {
}
}
});
My problem is that in many cases the html2 variable is not yet created before the pdf.create process starts. This is probably because the readFileSync takes too long to finish.
I was wondering, how can I fix this. How can I make the pdf.create wait for the readFileSync to finish and the html2 variable to be filled.
You can use fs.readFile to read the file asynchronously and html2 will be available within the callback function.
Object.keys(setter).forEach(function(element, key, _array) {
var regex = new RegExp(element, "g");
data = data.replace(regex, setter[element])
})
var result = data;
fs.writeFile(mergeFileRes, result, 'utf8', function (err) {
if(err) {
console.log(err);
return;
} else {
fs.readFile(mergeFileRes, 'utf8', function(err, html2){
if (err) throw err;
let options = {
format: 'a4' ,
"directory" : "/tmp",
};
pdf.create(html2, options).toStream(function(err, stream2){
if(err) console.log(err);
stream2.pipe(res);
stream2.on('end', function () {
try{
fs.unlink(mergeFileRes)
console.log(3090, "deleted file");
}
catch (err){
console.log(3090, "Did not delete file");
}
});
});
});
}
});
Below program is to read all the files under a folder and check if all the files contains email information.
The below test should be failed(i.e.., promise to be rejected and status should be failed) when the condition in the second "if" statement is true. Execution is getting stopped, but the test is getting passed instead of getting failed.
So, How do i make the below test fail.
Thanks in Advance.
var checkFileContent = function(directory) {
var results = [];
fs.readdir(directory, function(err, list) {
// if (err) return done(err);
console.log("The folder or list of file names : " + list);
var i = 0;
(function next()
{
var file = list[i++];
// if (!file) return done(null, results);
file = directory + '/' + file;
fs.stat(file, function(err, stat)
{
if (stat && stat.isDirectory())
{
checkFileContent(file, function(err, res)
{
results = results.concat(res);
next();
});
} else
{
fs.readFile(file, "utf8", function(err, data)
{
// if ( err )
// { throw err;}
console.log(file +" file content is");
console.log(data);
console.log( data.toLowerCase().indexOf('email'));
return new Promise((resolve, reject) => {
if(data.toLowerCase().indexOf('email') != -1)
{
return reject(file + 'contains email ');
}
else
{
return new Promise((resolve, reject) =>
{
fs.readFile(file, "utf8", function(err, data)
{
// if ( err )
// { throw err;}
console.log(file +" file content is");
console.log(data);
console.log( data.toLowerCase().indexOf('email'));
//return newfunc(file,data);
if(data.toLowerCase().indexOf('email') != -1)
{
reject(file + 'contains email ');
}
else
{
console.log(file + 'doesnt contain email');
resolve(true);
}
}).catch ((err) =>
{
//Execution is getting stopped here, but the test is getting passed instead of getting failed.
return reject(err);
});
});
results.push(file);
//console.log("List of files under the current Log folder are : " + results);
next();
} // else closure
}); // fs.stat() closure
})(); });
}
The above function is being called from another JS File using Mocha as shown below :
it('should read Log files', function () {
return ----
.then((abc) => {
------
}).then(()=>
{
return JSFileName.checkFileContent(directory);
}).catch((err) => {
return Promise.reject(err);
})
})
I'm beginner with node.js and I have a problem. I want to display the filename and last modification date in a list with a view ejs.
But, my problem is to pass the variable to my view, I want to fill in an arraylist with filename and one with date but nothing appears..
here is the code :
app.get('/', function (req, res) {
res.setHeader('Content-Type', 'text/html');
var filenameArray = [];
var datefileArray = [];
fs.readdir('./PDF/', function (err, files) {
if (err) {
throw err;
}
files.forEach(function (file) {
fs.stat('./PDF/'+file, function (err, stats) {
if (err) {
throw err;
}
// Fill in the array with filename and last date modification
filenameArray.push(file);
datefileArray.push(stats.mtime);
});
});
});
filenameArray.push("test");
datefileArray.push("pouet");
res.render('files.ejs', { filename: filenameArray, dateModification: datefileArray, index: filenameArray.length });
});
and here is my view :
<p> <%= filename.length %></p>
<ul><%
for(var i = 0 ; i <= index; i++) {
%>
<li><%= filename[i] + " - " + dateModification[i] %></li>
<% } %></ul>
I have only the test item in my array..
Thank you.
Remember: node.js is asynchronous, so when you call render, the fs.readdir and the fs.stat inside it have not returned yet.
You can use the async module to help you with that:
var async = require('async');
app.get('/', function (req, res) {
res.setHeader('Content-Type', 'text/html');
var filenameArray = [];
var datefileArray = [];
fs.readdir('./PDF/', function (err, files) {
if (err) {
throw err;
}
async.each(files, function (file, callback) {
fs.stat('./PDF/'+file, function (err, stats) {
if (err) {
throw err;
}
// Fill in the array with filename and last date modification
filenameArray.push(file);
datefileArray.push(stats.mtime);
callback();
});
}, function (error) {
if (error) return res.status(500).end();
res.render('files.ejs', { filename: filenameArray, dateModification: datefileArray, index: filenameArray.length });
});
});
});
The each function will execute the iterator callback for each item in the array, and at the end, when all iterator functions have finished (or an error occurs), it calls the last callback.
I'm trying to understand the asynchronous programming Node.js but stalled on this code.
This function in their callback returns an array of files in a directory:
function openDir(path, callback) {
path = __dirname + path;
fs.exists(path, function (exists) {
if (exists) {
fs.readdir(path, function (err, files) {
if (err) {
throw err;
}
var result = [];
files.forEach(function (filename, index) {
result[index] = filename;
});
return callback(result);
});
}
});
}
But when I use asynchronous code inside.forEach, it returns nothing:
function openDir(path, callback) {
path = __dirname + path;
fs.exists(path, function (exists) {
if (exists) {
fs.readdir(path, function (err, files) {
if (err) {
throw err;
}
var result = [];
files.forEach(function (filename, index) {
fs.stat(path + filename, function (err, stats) {
if (err) {
throw err;
}
result[index] = filename;
});
});
return callback(result);
});
}
});
}
I understand why it happens, but don't understand how to write correct code.
The issue is that fs.stat is also async, but you could probably do something like:
var result = [],
expectedLoadCount = files.length,
loadCount = 0;
files.forEach(function (filename, index) {
fs.stat(path + filename, function (err, stats) {
if (err) {
throw err;
}
result[index] = filename;
if (++loadCount === expectedLoadCount) callback(result);
});
});
The other answers may work well, but they are currently quite different semantically from the original code: they both execute stats in parallel, rather than sequentially. The forEach will initiate as many asynchronous stats operation as there are files in the list of files. The completion order of those operations may quite well be different from the original order of the list. This may substantially affect the error handling logic.
The following approach implements a state machine, which is aimed to executes stats asynchronously, yet sequentially (untested):
function openDir(path, callback) {
path = __dirname + path;
fs.exists(path, function (exists) {
if (!exists)
callback(null, null); // node (err, result) convention
else {
fs.readdir(path, function (err, files) {
if (err)
callback(err, null); // node (err, result) convention
else {
var results = [];
var i = 0;
nextStep(); // process the first file (the first step)
function nextStep() {
if (i >= files.length) // no more files?
callback(null, result); // node (err, result) convention
else {
fs.stat(path + files[i], function (err, stats) {
if (err)
callback(err, null); // node (err, result) convention
else {
results[i++] = stats;
// proceed to the next file
nextStep();
}
});
}
}
}
}
}
});
});
Promises may help to reduce the nesting level of the famous "Pyramid of Doom" like above.
try this:
function openDir(path, callback) {
path = __dirname + path;
fs.exists(path, function (exists) {
var totalFiles = 0;;
if (exists) {
fs.readdir(path, function (err, files) {
if (err) {
throw err;
}
var result = [];
files.forEach(function (filename, index) {
fs.stat(path + filename, function (err, stats) {
if (err) {
throw err;
}
result[index] = filename;
totalFiles++;
if(totalFiles === files.length){
callback(result);
}
});
});
});
}
});
}
you can also use the Async module, to help on these kinds of situations
I have an array of files, and I am trying to get just the basename of the files, without their long extensions. Here is an example of what the array looks like:
[
'/public/uploads/contentitems/.DS_Store',
'/public/uploads/contentitems/063012A5-60BC-4A4C-AEC2-56B0D5D99EF0/063012A5-60BC-4A4C-AEC2-56B0D5D99EF0.png',
'/public/uploads/contentitems/063012A5-60BC-4A4C-AEC2-56B0D5D99EF0/063012A5-60BC-4A4C-AEC2-56B0D5D99EF0_1.png',
'/public/uploads/contentitems/2A431412-A776-4D11-841A-B640DF37C9E2/2A431412-A776-4D11-841A-B640DF37C9E2_2.png'
]
I want to get:
[
'.DS_Store',
'063012A5-60BC-4A4C-AEC2-56B0D5D99EF0/063012A5-60BC-4A4C-AEC2-56B0D5D99EF0.png',
'063012A5-60BC-4A4C-AEC2-56B0D5D99EF0/063012A5-60BC-4A4C-AEC2-56B0D5D99EF0_1.png',
'2A431412-A776-4D11-841A-B640DF37C9E2/2A431412-A776-4D11-841A-B640DF37C9E2_2.png'
]
According to the docs, the path.basename function should return me the file without the path.
But what I get is the following error:
TypeError: Object /public/uploads/contentitems has no method 'basename'
Here is the code I am using right now:
var walk = function(dir, done) {
var results = [];
fs.readdir(dir, function(err, list) {
if (err) return done(err);
var pending = list.length;
if (!pending) return done(null, results);
list.forEach(function(file) {
file = dir + '/' + file;
fs.stat(file, function(err, stat) {
if (stat && stat.isDirectory()) {
walk(file, function(err, res) {
results = results.concat(res);
if (!--pending) done(null, results);
});
} else {
var suffix = getSuffix(file);
if (!verObj[suffix]) results.push(file);
if (!--pending) done(null, results);
}
});
});
});
};
walk(path, function(err, results) {
if (err) throw err;
results.forEach(function(file) {
console.log(path.basename(file));
});
self.respond({files: results}, {format: 'json'});
});
I am using path = require('path'); at the top of my file as well.
You have a variable name conflict with path.
Use some something else other than path to represent the directory you want to walk.
walk(path, function(err, results) {
You are passing a string named path into walk().
console.log(path.basename(file)); // <-- path is a string here