I am a node.js noob and am trying to do some file processing. I'm using async to process an array of files but the callback function is never called. I believe this is due to calling the next() function twice but I can't see where I'm doing this. If I comment out the last "return next()" I finish with no errors but the final callback doesn't execute. If I uncomment out this line I get the error message "callback was already called". Any help would be greatly appreciated. Here is the code:
/*jslint node: true */
"use strict";
var fs = require('fs'),
dive = require('dive'),
subdirs = require('subdirs'),
async = require('async'),
currentYear = new Date().getFullYear(),
cpFile = __dirname + "/" + "header.txt",
noCopy = __dirname + "/" + "noCopyright.txt",
currentHeads = __dirname + "/" + "currentHeaders.txt",
reYear = /\s(\d{4})[-\s]/i, // matches first 4 digit year
reComment = /(\/\*(?:(?!\*\/).|[\n\r])*\*\/)/, // matches first multi-line comment
allHeaders = {},
stringObj,
year,
top;
function needsHeader (file) {
if ((file.match(/.*\.js$/) || file.match(/.*\.less$/) || file.match(/.*\.groovy$/) || file.match(/.*\.java$/) || file.match(/.*\.template$/) || file.match(/.*\.html$/))) {
fs.appendFile(noCopy, file + "\n", function (err) {
if (err) {
return console.log(err);
}
});
}
}
fs.readFile(cpFile, 'utf8', function (err, copyRight) {
if (err) {
return console.log(err);
}
subdirs(__dirname, 4, function (err, dirs) {
if (err) {
return console.log(err);
}
async.each(dirs, function (dir, next) {
if (! dir.match(/.*\/src$/)) {
return next();
} else {
dive(dir, {all: false}, function (err, file) {
if (err) {
return next(err);
} else {
fs.readFile(file, 'utf8', function (err, data) {
if (err) {
return next(err);
} else {
if (data.match(reComment) && (file.match(/.*\.js$/) || file.match(/.*\.less$/) || file.match(/.*\.groovy$/) || file.match(/.*\.java$/) || file.match(/.*\.template$/))) {
top = data.match(reComment)[0];
if (top.match(reYear)) {
year = top.match(reYear)[1];
if (allHeaders[year]) {
allHeaders[year].push(file);
} else {
allHeaders[year] = [file];
}
} else {
needsHeader(file);
}
} else {
needsHeader(file);
}
return next();
}
});
}
});
}
}, function (err) {
if (err) {
console.log(err);
}
stringObj = JSON.stringify(allHeaders, null, 4);
fs.writeFile(currentHeads, stringObj, function (err) {
if (err) {
return console.log(err);
}
});
});
});
});
It expects you to call next() for each directory, and you are calling it for each file found in the directory. So as soon as some directory contains 2 or more files, you get the error.
To fix it, try call next() on dive complete. See the dive documentation:
complete [optional] may define a second callback, that is called,
when all files have been processed. It takes no arguments.
dive(dir, {all: false}, function (err, file) {
if (err) {
return next(err);
} else {
// your file handling code here
}
}, function complete() {
next();
});
Related
I am using async parallel to get data from db in parallel.
When every task is returning the data I am storing it in local object.
From index.js I am calling cacheService.js .
In cacheService.js I am loading data from mysql database and mongo database in to cache object.
Whenever I am doing npm run local run. I am getting following error.
UNHANDLEDREJECTION Error: Callback was already called.
This error is coming from loadMongoData method of cacheService.js.
I have followed other answers on stackoverflow like adding else part
Here is code for cacheService.js.
'use strict';
var cache = [];
class cacheService {
async init() {
await this.loadMongoData();
}
loadMongoData(env, callback1) {
const _this = this;
console.log('Inside loadMongoData')
async.parallel(
{
task1: function (callback) {
CriteriaDef.find({})
.lean()
.exec(function (err, criteriaDefs) {
if (err) {
console.log('Inside err 1')
logger.error('Error fetching CriteriaDef: ', util.inspect(err));
callback(err, null);
} else if (criteriaDefs) {
console.log('Inside criteriaDefs')
if (criteriaDefs.length && criteriaDefs.length > 0) {
console.log('Inside criteriaDefs 1')
global.CRITERIA_DEFS = criteriaDefs;
cache['criteria_defs'] = criteriaDefs;
}
callback(null, null);
}
});
},
task2: function (callback) {
groupDef
.find({})
.lean()
.exec(function (err, groupDefs) {
if (err) {
console.log('Inside err2')
logger.error('Error fetching groupDefs: ', util.inspect(err));
callback(null, err);
} else if (groupDefs) {
console.log('Inside ')
global.groupDefsWithRoles = groupDefs;
let _groupDefs = [];
_.each(groupDefs, function (groupDef) {
var data = {
value: groupDef._id,
label: `${groupDef._id}: ${groupDef.description}`
};
_groupDefs.push(data);
});
global.groupDefs = _groupDefs;
cache['groupDefs'] = _groupDefs;
logger.info('Loaded groupDefs: ', global.groupDefs.length);
callback(null, null);
}
});
},
task3: function (callback) {
jiraProjects.find({$or: [{archived: {$ne: true}}, {archived: {$exists: false}}]}).exec(function (err, jiraProjects) {
if (err) {
console.log('Inside error 3')
logger.error('Error fetching jiraProjects: ', err);
callback(null, err);
} else if (jiraProjects) {
console.log('Inside jira project')
let _jiraProjects = [];
_.each(jiraProjects, function (jiraProject) {
var data = {
value: jiraProject.key,
label: jiraProject.key,
issueType: jiraProject.issueType ? jiraProject.issueType : 'Bug'
};
_jiraProjects.push(data);
});
global.jiraProjectsList = _jiraProjects;
cache['jiraProjects'] = _jiraProjects;
logger.info('Loaded jira projects: ', global.jiraProjectsList.length);
callback(null, null);
}
});
},
task4: function (callback) {
console.log('Inside task4')
callback(null, null);
},
task5: function (callback) {
inputElements
.find({})
.lean()
.exec(function (err, inputElements) {
if (err) {
console.log('Inside error5')
logger.error('Error fetching inputElements: ', util.inspect(err));
callback(null, err);
} else if (inputElements) {
console.log('Inside inputelements')
global.INPUT_ELEMENTS_DEF = inputElements;
cache['INPUT_ELEMENTS_DEF'] = inputElements;
callback(null, null);
}
});
},
task6: function (callback) {
console.log('Inside task6')
sisp.loadProducts('', callback);
}
},
function (err, results) {
if (err) {
console.log('Inside final error')
logger.error("Something went wrong can't start the app: ", util.inspect(err));
callback1(null, err);
} else {
console.log('Inside final else')
logger.info('loaded all globals properly :)');
callback1(null, null);
}
}
)
}
}
export default new cacheService();
I think your problem cuz you use callback inside of promise.
Please change your code like this:
class cacheService {
async init() {
await new Promise((resolve, reject) => {
this.loadMongoData(env, (ignoreArg, error) => {
if (error) {
return reject(error);
}
resolve();
});
});
}
...
}
Tip: in the each task add statement else for call callback, because maybe your statement not handled and your code can't execute prefect
task1: function (callback) {
CriteriaDef.find({})
.lean()
.exec(function (err, criteriaDefs) {
if (err) {
console.log('Inside err 1')
logger.error('Error fetching CriteriaDef: ', util.inspect(err));
callback(err, null);
} else if (criteriaDefs) {
console.log('Inside criteriaDefs')
if (criteriaDefs.length && criteriaDefs.length > 0) {
console.log('Inside criteriaDefs 1')
global.CRITERIA_DEFS = criteriaDefs;
cache['criteria_defs'] = criteriaDefs;
}
callback(null, null);
} else {
callback(null, null);
}
});
},
Below program is to read all the files under a folder and check if all the files contains email information.
The below test should be failed(i.e.., promise to be rejected and status should be failed) when the condition in the second "if" statement is true. Execution is getting stopped, but the test is getting passed instead of getting failed.
So, How do i make the below test fail.
Thanks in Advance.
var checkFileContent = function(directory) {
var results = [];
fs.readdir(directory, function(err, list) {
// if (err) return done(err);
console.log("The folder or list of file names : " + list);
var i = 0;
(function next()
{
var file = list[i++];
// if (!file) return done(null, results);
file = directory + '/' + file;
fs.stat(file, function(err, stat)
{
if (stat && stat.isDirectory())
{
checkFileContent(file, function(err, res)
{
results = results.concat(res);
next();
});
} else
{
fs.readFile(file, "utf8", function(err, data)
{
// if ( err )
// { throw err;}
console.log(file +" file content is");
console.log(data);
console.log( data.toLowerCase().indexOf('email'));
return new Promise((resolve, reject) => {
if(data.toLowerCase().indexOf('email') != -1)
{
return reject(file + 'contains email ');
}
else
{
return new Promise((resolve, reject) =>
{
fs.readFile(file, "utf8", function(err, data)
{
// if ( err )
// { throw err;}
console.log(file +" file content is");
console.log(data);
console.log( data.toLowerCase().indexOf('email'));
//return newfunc(file,data);
if(data.toLowerCase().indexOf('email') != -1)
{
reject(file + 'contains email ');
}
else
{
console.log(file + 'doesnt contain email');
resolve(true);
}
}).catch ((err) =>
{
//Execution is getting stopped here, but the test is getting passed instead of getting failed.
return reject(err);
});
});
results.push(file);
//console.log("List of files under the current Log folder are : " + results);
next();
} // else closure
}); // fs.stat() closure
})(); });
}
The above function is being called from another JS File using Mocha as shown below :
it('should read Log files', function () {
return ----
.then((abc) => {
------
}).then(()=>
{
return JSFileName.checkFileContent(directory);
}).catch((err) => {
return Promise.reject(err);
})
})
I have a program where user first create a file once file is created i am appending data to the file that is coming from client consistently.The below code is working as expected. I am new to nodejs so just want to get an expert opinion in case when multiple users creating and recording files on their machines at same time, will it work asynchronously or do i need to make some changes to the code ?
io.js
socket.on('createlogfile', function() {
logsRecording.userLogs(function(filename) {
socket.emit('filename', filename);
});
});
socket.on('startrecording', function(obj) {
logsRecording.recordLogs(obj);
});
server.js
userLogs: function (callback) {
var filename = uuid.v4() + '.log';
var file = filePath + '/' + filename;
fs.openSync(file, 'a',function () {
console.log('file created');
});
console.log('userLogs');
callback(filename);
},
recordLogs: function (obj) {
var dir = './app/records/templogs'
var fileAppend = dir + '/'+ obj.file;
console.log('data from recording', obj.data);
fs.readdir(dir, function(err, items) {
items.forEach(function(file){
if(obj.file === file){
fs.appendFile(fileAppend, obj.data+ "\r\n", null, 'utf8', function (err) {
if (err) throw err;
});
console.log('filename in records',obj.file);
}
});
});
}
You are using fs.openSync, which is synchronous and as such can hang the event loop.
You should be using fs.open and callback inside it:
userLogs: function (callback) {
var filename = uuid.v4() + '.log';
var file = filePath + '/' + filename;
fs.open(file, 'a', function (err) {
console.log('file created');
console.log('userLogs');
callback(err, filename);
});
},
And you can flatten recordLogs using async.
Also, it is bad practice to throw error in synchronous function, you should be passing the error in the callback.
As a last tip, Array.forEach is synchronous, and can hang the process, you should be using async.each
recordLogs: function (obj, callback) {
var dir = './app/records/templogs'
var fileAppend = dir + '/'+ obj.file;
console.log('data from recording', obj.data);
async.waterfall([
(callback) => {
fs.readdir(dir, (err, items) => {
callback(err, items);
});
},
(items, callback) => {
async.each(items, (file, callback) => {
if(obj.file === file) {
fs.appendFile(fileAppend, obj.data+ "\r\n", null, 'utf8', function (err) {
callback(err);
});
console.log('filename in records',obj.file);
} else {
callback();
}
}, (err) => {
callback(err);
});
}
], (err, file) => {
if(callback) {
callback(err);
}
});
}
function load_file_contents(path, callback) {
fs.open(path, 'r', function (err, f) {
if (err) {
callback(err);
return;
} else if (!f) {
callback(make_error("invalid_handle",
"bad file handle from fs.open"));
return;
}
fs.fstat(f, function (err, stats) {
if (err) {
callback(err);
return;
}
if (stats.isFile()) {
var b = new Buffer(10000);
fs.read(f, b, 0, 10000, null, function (err, br, buf) {
if (err) {
callback(err);
return;
}
fs.close(f, function (err) {
if (err) {
callback(err);
return;
}
callback(null, b.toString('utf8', 0, br));
});
});
} else {
calback(make_error("not_file", "Can't load directory"));
return;
}
});
});
}
load_file_contents(
"test.txt",
function (err, contents) {
if (err)
console.log(err);
else
console.log(contents);
}
);
In this code, I don't quite understand where does this "f" come from?
after "fs.open()", there is a line
" } else if (!f) {"
what does this mean, where does this f comefrom?
f is the file descriptor that is passed to fs.open()'s callback if the file was able to be opened.
else if (!f) {
callback(make_error("invalid_handle",
"bad file handle from fs.open"));
return;
}
is checking if the file descriptor is falsy (presumably it's checking if it's null or undefined) and calling the function's own callback with an error.
I'm trying to understand the asynchronous programming Node.js but stalled on this code.
This function in their callback returns an array of files in a directory:
function openDir(path, callback) {
path = __dirname + path;
fs.exists(path, function (exists) {
if (exists) {
fs.readdir(path, function (err, files) {
if (err) {
throw err;
}
var result = [];
files.forEach(function (filename, index) {
result[index] = filename;
});
return callback(result);
});
}
});
}
But when I use asynchronous code inside.forEach, it returns nothing:
function openDir(path, callback) {
path = __dirname + path;
fs.exists(path, function (exists) {
if (exists) {
fs.readdir(path, function (err, files) {
if (err) {
throw err;
}
var result = [];
files.forEach(function (filename, index) {
fs.stat(path + filename, function (err, stats) {
if (err) {
throw err;
}
result[index] = filename;
});
});
return callback(result);
});
}
});
}
I understand why it happens, but don't understand how to write correct code.
The issue is that fs.stat is also async, but you could probably do something like:
var result = [],
expectedLoadCount = files.length,
loadCount = 0;
files.forEach(function (filename, index) {
fs.stat(path + filename, function (err, stats) {
if (err) {
throw err;
}
result[index] = filename;
if (++loadCount === expectedLoadCount) callback(result);
});
});
The other answers may work well, but they are currently quite different semantically from the original code: they both execute stats in parallel, rather than sequentially. The forEach will initiate as many asynchronous stats operation as there are files in the list of files. The completion order of those operations may quite well be different from the original order of the list. This may substantially affect the error handling logic.
The following approach implements a state machine, which is aimed to executes stats asynchronously, yet sequentially (untested):
function openDir(path, callback) {
path = __dirname + path;
fs.exists(path, function (exists) {
if (!exists)
callback(null, null); // node (err, result) convention
else {
fs.readdir(path, function (err, files) {
if (err)
callback(err, null); // node (err, result) convention
else {
var results = [];
var i = 0;
nextStep(); // process the first file (the first step)
function nextStep() {
if (i >= files.length) // no more files?
callback(null, result); // node (err, result) convention
else {
fs.stat(path + files[i], function (err, stats) {
if (err)
callback(err, null); // node (err, result) convention
else {
results[i++] = stats;
// proceed to the next file
nextStep();
}
});
}
}
}
}
}
});
});
Promises may help to reduce the nesting level of the famous "Pyramid of Doom" like above.
try this:
function openDir(path, callback) {
path = __dirname + path;
fs.exists(path, function (exists) {
var totalFiles = 0;;
if (exists) {
fs.readdir(path, function (err, files) {
if (err) {
throw err;
}
var result = [];
files.forEach(function (filename, index) {
fs.stat(path + filename, function (err, stats) {
if (err) {
throw err;
}
result[index] = filename;
totalFiles++;
if(totalFiles === files.length){
callback(result);
}
});
});
});
}
});
}
you can also use the Async module, to help on these kinds of situations