I'm trying to use async.series to execute 2 functions in different files in sequential mode.
The use case is to download log files to the local and then move them into a folder.
The functions work fine individually.
Now my next step is to require these 2 files in index.js and call both the functions in sequence.
This is the code to write the files from webdav into local machine:
const fs = require('fs');
const stream = require('stream');
const util = require('util');
const pipeline = util.promisify(stream.pipeline);
const { createClient } = require("webdav");
const path = require("path");
const client = createClient(
"https:/*/webdav/Sites/Logs", {
username: "*",
password: "*"
}
);
async function writeFiles(){
console.log('inside write functions');
let directoryData = await client.getDirectoryContents("/");
console.log('after directorydata' +directoryData);
await Promise.all(directoryData.map(start));
}
async function start(dir) {
let fileName = dir.filename;
console.log('inside start function');
var directory = path.join(__dirname,'/logs/')
if(fileName.includes('.log')){
try {
let fileName = dir.filename;
//console.log(fileName);
await pipeline(
client.createReadStream(fileName),
fs.createWriteStream(directory + fileName)
);
return directory;
console.log('done');
} catch (e) {
console.log(e);
}
}
}
module.exports = writeFiles;
This is the code to create folders and move the log files into them according to the name:
const testFolder = './logs/';
const fs = require('fs');
const path = require("path");
var directory = path.join(__dirname,'/logs/');
async function sort (){
console.log('inside sort function sorting file');
await fs.readdir(directory, (err, files) => {
//console.log('sdcfe');
if(files){
console.log('filesss')
} else {
console.log('no filesss')
}
files.forEach(file => {
let fileFolder = file.split('-')[0];
var folder = fileFolder.replace('.', "");
var dir = directory+folder;
if (!fs.existsSync(dir)){
//console.log('inside if' + dir);
fs.mkdirSync(dir);
}
const currentPath = path.join(directory, file);
const destinationPath = path.join(directory, folder, file);
fs.rename(currentPath, destinationPath, function (err) {
if (err) {
throw err
} else {
return destinationPath;
console.log("Successfully moved the file!");
}
});
});
});
}
module.exports = sort;
This is the index.js file to require both the above files and call them in sequence (once the write function is done, I need to move the logs files into folders)
'use strict';
// Initialize constants
const async = require('async');
const writeFiles = require('./webdavConnect2');
const sort = require('./sortingFiles');
// Initialize the program
let program = require('commander');
// Initialize the version
program.version('0.5.0');
program
.command('say')
.description('Batch command that can be used to zip, upload, import, and re-index for a given site import.')
.action(function () {
function webdavConnect() {
console.log('inside webdav function');
// write the files into local machine
writeFiles();
}
// move the files into folders
function sortingFiles() {
console.log('inside sort function');
sort();
}
async.series([
function(callback) {
webdavConnect(function(directory){
callback(null, directory);
})
},
function(callback) { sortingFiles(function(destinationPath){
callback(null,destinationPath);
})
}
], function(asyncErrorObj) {
// Was an error defined? If so, then throw it
if (asyncErrorObj !== null) { throw asyncErrorObj; }
console.log('END');
});
});
// Parse the command-line arguments
program.parse();
The issue with this is that currently the first function executes and not the second one.
Posting a solution a friend suggested:
I pinpointed some issues regarding the usage of async.series . Try to replace your index.js with the following code. Note the async functions that are now being passed to async.series, and note how those functions now accept the callback that Async.js is supposed to pass to them which are missing in the current code. I couldn't test this because the project is not reproducable (at least without a major prep). If anything breaks you can share another message and/or tag me directly! Hope it helps.
"use strict";
const async = require("async");
const writeFiles = require("./webdavConnect2");
const sort = require("./sortingFiles");
let program = require("commander");
program.version("0.5.0");
program
.command("say")
.description(
"Batch command that can be used to zip, upload, import, and re-index for a given site import."
)
.action(function () {
async function webdavConnect(callback) {
callback(await writeFiles());
}
async function sortingFiles(callback) {
callback(await sort());
}
async.series(
[
function (callback) {
webdavConnect(function (directory) {
callback(null, directory);
});
},
function (callback) {
sortingFiles(function (destinationPath) {
callback(null, destinationPath);
});
}
],
function (asyncErrorObj, results) {
if (asyncErrorObj !== null) {
throw asyncErrorObj;
}
console.log(results);
}
);
});
program.parse();
It seems that both functions are async and they should be called in async function with await. First function did not completed the execution and second started its execution but as it did not find the files so you can't see the output for second function. This is what i would do to fix execution issue:
"use strict";
const writeFiles = require("./webdavConnect2");
const sort = require("./sortingFiles");
// Initialize the program
let program = require("commander");
// Initialize the version
program.version("0.5.0");
program
.command("say")
.description(
"Batch command that can be used to zip, upload, import, and re-index for a given site import."
)
.action(async function() {
await writeFiles();
await sort();
});
// Parse the command-line arguments
program.parse();
Related
I am at a loss as to why I cannot read files inside a directory. I have tried every implementation of fs (both standard and promise-based). With the code below, the main function is called, logs "Main function started..." and both oldDir and newDir, then exits with no error.
import { readdir, rename } from "fs/promises";
import * as path from "path";
const oldDir = path.join(process.cwd(), "../Old Backgrounds");
const newDir = process.cwd();
async function main() {
console.log("Main function started...");
console.log(oldDir);
console.log(newDir);
const oldFiles = await readdir(oldDir);
const newFiles = await readdir(newDir);
console.log(`File names to use: ${oldFiles}`);
console.log(`Files to be renamed: ${newFiles}`);
for (let i = 0; i < oldFiles.length; i++) {
// Take new files and rename them to match oldFiles
const currentName = newFiles[i];
const newName = oldFiles[i];
const currentPath = path.join(newDir, currentName);
const renamePath = path.join(newDir, newName);
await rename(currentPath, renamePath);
}
}
main()
.then(process.exit(0))
.catch((e) => {
console.error(e);
process.exit(1);
});
I am running this code on windows if that matters at all. When I use promises implementation with await readdir, the code exits with no error.
When I use a standard implementation with readdir and a callback function, it returns "undefined".
Both directories exist and have files in them.
Solved per jonrsharpe's comment. The issue was not with my main function at all. I was passing process.exit(0) directly to then when it was looking for a callback function.
Revised the main function call to:
main()
.then(() => process.exit(0))
.catch((e) => {
console.error(e);
process.exit(1);
});
And the issue disappeared.
I am archiving a certain text files from multiple directories. So firstly, I am iterating inside a folder which gives me folderPath and this folderPath can contain many files(txt,pdf,etc.) and there are multiple folder paths, I am using a async readdir for folderPath and appending those individual files in the archiver and then finally closing. If I am doing archive.finalize before the folder loop ends, it isn't generating required number of txt files in the zip just the initial one's which is obvious. And if I keep archive.finalize on line 2 it's throwing me an error as stated below the directory structure. Can someone please help in this regard ?
Directory structure is like:
mainfolder/folder1 mainfolder/folder2
mainfolder/folder1/sometext.txt mainfolder/folder1/someanothertext.txt
mainfolder/folder2/sometext.txt mainfolder/folder2/someanothertext.txt
Now I want to zip it as:
Outuput.zip which contains -> folder1 and folder2 with respective txt
files. I was able to achieve it when using sync function to readdir(readdirsync),
but with async, I am facing some callback issue.
Error :
ArchiverError: queue closed
at Archiver.file (C:\Users\Workspace\code\node_modules\archiver\lib\core.js:692:24)
at C:\Users\Workspace\current_code_snippet
at Array.forEach (<anonymous>)
at C:\Users\current_code_snippet_ line 3 as specified in code snippet
at FSReqCallback.oncomplete (fs.js:156:23) {
code: 'QUEUECLOSED',
data: undefined
}
Code :
this.children.forEach((value, _index, _array) => {
const folderPath = path.join(basePath, value);
fs.readdir(folderPath, (err, fileNames) => {
if (err){
throw err;
}
else {
fileNames.forEach(file => { // line 3
if (outputType === ".txt") {
const filePath = path.join(basePath, value, file);
archive.file(filePath, { name: `${value}/${file}` }); // file is saved as value/file inside parent zip
}
})
}
})
archive.finalize(); // line 1
});
archive.finalize(); // line 2
I would wrap the fs call into a Promise. This makes it possible to just await the operations and it takes some complexity out of the code.
Be aware that forEach loops don't work with async, await
const children = ["path_0", "path_1", "path_2", "path_3"];
// mock fs / async action
const fs = {
readdir: (path, error, success) => {
setTimeout(() => {
success(["file_0", "file_1", "file_2"]);
}, Math.random() * 1000);
}
}
// function I would add to make the process simpler
const readFiles = (path) => {
return new Promise(res => {
fs.readdir(path, () => {}, (s) => {
res(s);
});
});
}
// start helper as async
const start = async() => {
// first await all files to be added to the archive
await Promise.all(children.map(async child => {
const files = await readFiles();
// loop to add all files to the zip
// archive.file(filePath, { name: `${value}/${file}` });
console.log(child, files);
}));
// then archive all files
// archive.finalize();
console.log("finalize archive");
}
start();
I have to write a code which takes one parameter i.e. path to directory, fetch files from the given directory and again does the same for the directories inside the given directory.The whole search should be wrapped in a promise.
But the depth of recursive search is 1.
Final array should look like: [file1, file2, file3, [file1inDir1, file2inDir1, Dir1inDir1, file3inDir1, Dir2inDir1], file4, file5]
My code is:
const fs = require("fs");
const path = require("path");
function checkfile(files){
let result = [];
for(let i=0 ; i<files.length ;i++){
let newpath = path.join(__dirname,files[i]);
fs.stat(newpath, (err,stats)=>{
if(stats.isDirectory()){
fs.readdir(newpath, (error,files)=>{result.push(files)})
}
else{result.push(files[i])}
})
}
return result;
}
let test = (filepath) => {
return new Promise((resolve, reject) =>{
fs.readdir(filepath, (error,files) => {
if (error) {
reject("Error occured while reading directory");
} else {
resolve(checkfile(files));
}
});
}
)}
test(__dirname)
.then(result => {
console.log(result);
})
.catch(er => {
console.log(er);
});
When I run it I get the following output: []
How do I correct this?
test correctly returns a promise, but checkfile does not, thus all the async operations happen after the yet empty result array was synchronously returned.
Fortunately NodeJS already provides utilities that return promises instead of taking callbacks docs, with them writing that code without callbacks going wrong is easy:
async function checkfile(files){
const result = [];
for(let i=0 ; i<files.length ;i++){
let newpath = path.join(__dirname,files[i]);
const stats = await fs.promises.stat(newpath);
if(stats.isDirectory()){
const files = await fs.promises.readdir(newpath);
result.push(files);
} else result.push(files[i]);
}
return result;
}
async function test(filepath) {
const files = await fs.promises.readdir(filepath);
return checkfile(files);
}
I'm looping through files in a directory and storing the file details to an array data. The following code populates the array if I don't attempt to run fs.stat to get things like the file create/edit date:
fs.readdir('../src/templates', function (err, files) {
if (err) {
throw err;
}
var data = [];
files
.forEach(function (file) {
try {
fs.stat('../src/templates/'+file,(error,stats) => {
data.push({ Name : file,Path : path.join(query, file) });
});
} catch(e) {
console.log(e);
}
});
res.json(data);
});
});
If I move the data.push(...) outside the fs.stat the array returns with the file data. Inside the fs.stat it returns empty. I assume this is an asynchronous issue in that the for loop is running and finishing before fs.stat runs.
I'm thinking I need to use a promise here but unsure.
If you want or need to be asynchronous:
const fs = require("fs");
const path = require("path");
const { promisify } = require("util");
const asyncStat = promisify(fs.stat);
fs.readdir('../src/templates', async function(err, files) {
if (err) {
throw err;
}
const data = await Promise.all(files.map(async function(file) {
try {
const stats = await asyncStat('../src/templates/' + file);
return { Name: file, Path: path.join(query, file), stats };
} catch (e) {
console.log(e);
}
}));
res.json(data);
});
Note that I used map instead of forEach and then awaited all Promises (async makes function return a promise).
I also needed to change fs.stat to use promise with util.promisify.
You're right about the issue being in the asynchronous call. You could use a promise, or you could use fs.statSync(...), which returns a fs.Stats object and operates synchonously.
files.forEach(function (file) {
try {
var fileStats = fs.statSync('../src/templates/' + file);
data.push({
Name : file,
Path : path.join(query, file)
});
} catch(e) {
console.log(e);
}
});
Hi I am writing a function in Node JS for which I have to return a filepath. My problem is in that function I am writing to a file and I want after writing to a file is finished then my return should work. Before looking into code, I know this can be duplicate and I have really did a research on this but I am not just being able to get there. I have tried using callback but the problem is I want to return a value which is already defined. So, before making any judgement calls for duplicate or lack of research, please read the code.
Also, tried to return value in fs.append callback but still did not solved.
My function:
const fs = require('fs');
const path = require('path');
module.exports.createDownloadFile = (request) => {
let filePath;
if (request) {
const userID = xyz;
filePath = path.join(__dirname, userID.concat('.txt'));
fs.open(filePath, 'w', (err) => {
if (err) throw new Error('FILE_NOT_PRESENT');
fs.appendFile(filePath, 'content to write');
});
}
return filePath;
};
I am getting the filePath where I am calling function, it's just at that time file is empty that is why I want to return after file is written completely.
Promises allow you to structure code and return values more like traditional synchronous code. util.promisify can help promisify regular node callback functions.
const fs = require('fs')
const path = require('path')
const fsAppendFileAsync = util.promisify(fs.appendFile)
const fsOpenAsync = util.promisify(fs.open)
module.exports.createDownloadFile = async (request) => {
if (!request) throw new Error('nope')
const userID = xyz
let filePath = path.join(__dirname, userID.concat('.txt'))
let fd = await fsOpenAsync(filePath, 'w')
await fsAppendFileAsync(fd, 'content to write')
return filePath
};
Note that async/await are ES2017 and require Node.js 7.6+ or Babel.
Opening a file with w creates or truncates the file and promises will reject on errors that are thrown so I've left the error handler out. You can use try {} catch (e) {} blocks to handle specific errors.
The Bluebird promise library is helpful too, especially Promise.promisifyAll which creates the promisified Async methods for you:
const Promise = require('bluebird')
const fs = Promise.promisifyAll(require('fs'))
fs.appendFileAsync('file', 'content to write')
use promises like this :
const fs = require('fs');
const path = require('path');
module.exports.createDownloadFile = (request) => {
return new Promise((resolve, reject) => {
let filePath;
if (request) {
const userID = xyz;
filePath = path.join(__dirname, userID.concat('.txt'));
fs.open(filePath, 'w', (err) => {
if (err) reject(err);
else
fs.appendFile(filePath, 'content to write', (err) => {
if (err)
reject(err)
else
resolve(filePath)
});
});
}
});
};
and call it like this :
createDownloadFile(requeset).then(filePath => {
console.log(filePath)
})
or use sync functions without Promises:
module.exports.createDownloadFile = (request) => {
let filePath;
if (request) {
const userID = xyz;
filePath = path.join(__dirname, userID.concat('.txt'));
fs.openSync(filePath,"w");
fs.appendFileSync(filePath, 'content to write');
}
return filePath;
};