I've written a script that recursively asynchronously modifies js files in a directory. It's made up of a search(dirname) function which searches a directory for js files, and a modify(filename) which does the modifying.
let totalFilesSearched = 0;
const search = (dir) => {
fs.readdir(dir, (err, list) => {
if (err) return;
list.forEach((filename) => {
const filepath = path.join(dir, filename);
if (filename.endsWith('.js')) {
modify(filepath);
} else if (fs.lstatSync(filepath).isDirectory()) {
search(filepath);
}
})
});
}
const modify = (filename) => {
fs.readFile(filename, 'utf8', (err, data) => {
if (err) console.log(err);
// ... my modification code ...
totalFilesSearched++;
});
}
search(args[0])
console.log(`Total files searched: ${totalFilesSearched}`);
I want to print out the totalFilesSearched at the end of my script but because my code is asynchronous, it just prints Total files searched: 0 right away.
Does anyone know how I'd wait until the script is about to end to print this out? I'm having trouble because both my search() and modify() functions are asynchronous.
Use Promises instead, and then call console.log when everything is resolved. Use promisify to turn the callbacks into promises:
const { promisify } = require('util');
const readFile = promisify(fs.readFile);
const readDir = util.promisify(fs.readdir);
const search = (dir) => (
readDir(dir).then((list) => (
Promise.all(list.map((filename) => {
const filepath = path.join(dir, filename);
if (filename.endsWith('.js')) {
return modify(filepath); // recursively return the promise
} else if (fs.lstatSync(filepath).isDirectory()) {
return search(filepath); // recursively return the promise
}
}))
))
.catch(err => void 0)
);
const modify = (filename) => (
readFile(filename, 'utf8')
.then((data) => {
// other code
totalFilesSearched++;
}).catch(err => console.log(err))
)
search(args[0])
.then(() => {
console.log(`Total files searched: ${totalFilesSearched}`);
});
Self answer:
Just use process.on('exit', callback_function_to_execute_at_end)
Its built into node, your callback will get executed right before the process exits.
Related
I have this code:
const fs = require("fs");
const saveFile = (fileName, data) => {
return new Promise((resolve) => {
fs.writeFile(fileName, data, (err) => {
resolve(true);
});
});
};
const readFile = (fileName) => {
return new Promise((resolve) => {
fs.readFile(fileName, "utf8", (err, data) => {
resolve(data);
});
});
};
const filename = "test.txt";
saveFile(filename, "first");
readFile(filename).then((contents) => {
saveFile(filename, contents + " second");
});
readFile(filename).then((contents) => {
saveFile(filename, contents + " third");
});
I'm hoping to obtain in 'test.txt'
first second third
but instead, I get
first thirdd
The idea is that every time I receive a certain post request. I have to add more text to the file
Does someone have any solution for this?
Thank you so much!
Edit:
The problem of using async await or a chain of .then( ) is that I have to add more text to the file every time I receive a certain post request. So I don't have control over what is written or when. The Idea is that everything is written and nothing is overwritten even if two post requests are received at the same time.
I'm going to share the solution with a linked list I came up with yesterday. But I still want to know if someone has a better solution.
const saveFile = (fileName, data) => {
return new Promise((resolve) => {
fs.writeFile(fileName, data, (err) => {
resolve(true);
});
});
};
const readFile = (fileName) => {
return new Promise((resolve) => {
fs.readFile(fileName, "utf8", (err, data) => {
resolve(data);
});
});
};
class LinkedCommands {
constructor(head = null) {
this.head = head;
}
getLast() {
let lastNode = this.head;
if (lastNode) {
while (lastNode.next) {
lastNode = lastNode.next;
}
}
return lastNode;
}
addCommand(command, description) {
let lastNode = this.getLast();
const newNode = new CommandNode(command, description);
if (lastNode) {
return (lastNode.next = newNode);
}
this.head = newNode;
this.startCommandChain();
}
startCommandChain() {
if (!this.head) return;
this.head
.command()
.then(() => {
this.pop();
this.startCommandChain();
})
.catch((e) => {
console.log("Error in linked command\n", e);
console.log("command description:", this.head.description);
throw e;
});
}
pop() {
if (!this.head) return;
this.head = this.head.next;
}
}
class CommandNode {
constructor(command, description = null) {
this.command = command;
this.description = description;
this.next = null;
}
}
const linkedCommands = new LinkedCommands();
const filename = "test.txt";
linkedCommands.addCommand(() => saveFile(filename, "first"));
linkedCommands.addCommand(() =>
readFile(filename).then((contents) =>
saveFile(filename, contents + " second")
)
);
linkedCommands.addCommand(() =>
readFile(filename).then((contents) => saveFile(filename, contents + " third"))
);
Because these are async functions they notify you that the work is completed in the then function.
That means you want to use a then chain (or an async function) like so:
readFile(filename).then((contents) => {
return saveFile(filename, contents + " second");
}).then(() => {
return readFile(filename)
}).then((contents) => {
saveFile(filename, contents + " third");
});
You can use a FIFO queue of functions that return promises for this.
const { readFile, writeFile } = require("fs/promises");
let queue = [];
let lock = false;
async function flush() {
lock = true;
let promise;
do {
promise = queue.shift();
if (promise) await promise();
} while (promise);
lock = false;
}
function createAppendPromise(filename, segment) {
return async function append() {
const contents = await readFile(filename, "utf-8");
await writeFile(
filename,
[contents.toString("utf-8"), segment].filter((s) => s).join(" ")
);
};
}
async function sequentialWrite(filename, segment) {
queue.push(createAppendPromise(filename, segment));
if (!lock) await flush();
}
async function start() {
const filename = "test.txt";
// Create all three promises right away
await Promise.all(
["first", "second", "third"].map((segment) =>
sequentialWrite(filename, segment)
)
);
}
start();
So how's this work? Well, we use a FIFO queue of promise functions. As requests come in we create them and add them to the queue.
Every time we add to the array we attempt to flush it. If there's a lock in place, we know we're already flushing, so just leave.
The flushing mechanism will grab the first function in the queue, delete it from the queue, invoke it, and await on the promise that returns. It will continue to do this until the queue is empty. Because all of this is happening asynchronously, the queue can continue to get populated while flushing.
Please keep in mind that if this file is shared on a server somewhere and you have multiple processes reading from this file (such as with horizontal scaling) you will lose data. You should instead use some kind of distributed mutex somewhere. A popular way of doing this is using Redis and redlock.
Hope this helps!
Edit: by the way, if you want to prove that this indeed works, you can add a completely random setTimeout to the createAppendPromise function.
function createAppendPromise(filename, segment) {
const randomTime = () =>
new Promise((resolve) => setTimeout(resolve, Math.random() * 1000));
return async function append() {
await randomTime();
const contents = await readFile(filename, "utf-8");
await writeFile(
filename,
[contents.toString("utf-8"), segment].filter((s) => s).join(" ")
);
};
}
Chaining is fine, even if you don't know in advance when or how many promises will be created. Just keep the end of the chain handy, and chain to it whenever you create a promise...
// this context must persist across posts
let appendChain = Promise.resolve();
const filename = "test.txt";
// assuming op's readFile and saveFile work...
const appendToFile = (filename, data) =>
return readFile(filename).then(contents => {
return saveFile(filename, contents + data);
});
}
function processNewPost(data) {
return appendChain = appendChain.then(() => {
return appendToFile(filename, data);
});
}
Here's a demonstration. The async functions are pretend read, write and append. The <p> tag is the simulated contents of a file. Press the button to add new data to the pretend file.
The button is for you to simulate the external event that triggers the need to append. The append function has a 1s delay, so, if you want, you can get in several button clicks before all the appends on the promise chain are write is done.
function pretendReadFile() {
return new Promise(resolve => {
const theFile = document.getElementById('the-file');
resolve(theFile.innerText);
})
}
function pretendWriteFile(data) {
return new Promise(resolve => {
const theFile = document.getElementById('the-file');
theFile.innerText = data;
resolve();
})
}
function pretendDelay(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
function appendFile(data) {
return pretendDelay(1000).then(() => {
return pretendReadFile()
}).then(result => {
return pretendWriteFile(result + data);
});
}
document.getElementById("my-button").addEventListener("click", () => click());
let chain = Promise.resolve();
let count = 0
function click() {
chain = chain.then(() => appendFile(` ${count++}`));
}
<button id="my-button">Click Fast and At Random Intervals</button>
<h3>The contents of the pretend file:</h3>
<p id="the-file">empty</p>
I'm trying to develop a simple app that if you pass a parameter in command line the application will search inside a directory and if the text match in some of the files the file should be save in a list, but when I put the console.log the value is not updated
here is my code:
const folder = "./movies/data";
const fs = require("fs");
var args = process.argv.slice(2);
console.log("myArgs: ", args);
var count = 0;
var list = [];
fs.readdir(folder, (err, files) => {
files.forEach((file) => {
fs.readFile(`movies/data/${file}`, "utf8", function (err, data) {
if (err) console.log(err);
if (data.includes("walt disney")) {
count++;
list.push(data);
console.log("Found in: ", data);
}
});
});
console.log(`Foram encontradas ${count} ocorrências pelo termo ${args}.`);
});
any suggestions about what i'm doing wrong?
For your program to work, you will have to add some Promise / async/await logic. On the moment you try to read from the files, the files are still undefined so the fs.readDir() function will not provide the wanted result.
This should work:
const { resolve } = require('path');
const { readdir } = require('fs').promises;
const fs = require("fs");
var args = process.argv.slice(2);
const pathToFiles = "./movies/";
async function getFiles(dir) {
const dirents = await readdir(dir, { withFileTypes: true });
const files = await Promise.all(dirents.map((dirent) => {
const res = resolve(dir, dirent.name);
return dirent.isDirectory() ? getFiles(res) : res;
}));
return Array.prototype.concat(...files);
}
getFiles(pathToFiles)
.then(files => {
console.log(files)
files.forEach((file) => {
fs.readFile(file, 'utf8', (err, data) => {
if (err) console.log(err);
if (data.includes(args)) {
console.log(`${args} found in ${file}.`);
} else {
console.log(`${args} not found.`);
}
});
})
})
.catch (e => console.error(e));
I am create a simple NODE-JS function that Converts PDF to Image > Crops Image > Merge Them back with ImageMagick.
and this is the complete code i am using :
var os = require('os');
var fs = require('fs');
var path = require('path');
var gs = require('node-gs');
var sharp = require('sharp');
var areaMap = require('./areaMap');
const { performance } = require('perf_hooks');
var spawn = require('child_process').spawnSync;
var pExcep = 'someException';
var gsPath = 'Ghostscript/gs26';
var src = path.join(os.tmpdir(), '/');
var Files = {
file1: path.join(src, 'out1.jpeg'),
file2: path.join(src, 'out2.jpeg'),
OutImg: path.join(src, 'out.jpeg')
}
var crop = function (s, sFile) {
return new Promise((res, rej) => {
s = areaMap[s];
sharp(Files.OutImg).extract(s)
.toFile(sFile)
.then(()=> res())
.catch((err) => rej(err));
});
};
var getBaseCard = function (s) {
if (RegExp('^([0-9]{8})$').test(s)) { return 'SOMETHINGHERE' } else { return 'inception'; }
//This can be done on client side.
}
var GetCardType = function (base, sInfo) {
return new Promise((res, rej) => {
if (base === 'SOEMTHINGHERE') {
if (sInfo.includes('SOMETHINGHERE2')) {
if (sInfo.includes(pExcep)) {
res('PA_S_')
} else {
res('PA_S2')
}
} else {
res('PA_ST')
}
} else {
res('SA_')
}
})
}
var PdfToText = function (file, pass) {
return new Promise((res, rej) => {
gs()
.batch().safer().nopause().res(2).option('-dDEVICEWIDTHPOINTS=20').option('-dDEVICEHEIGHTPOINTS=20').option('-dFIXEDMEDIA').option('-sPDFPassword=' + pass).device('txtwrite').output('-').input(file).executablePath(gsPath)
.exec((err, stdout, stderr) => {
if (!err) {
res(stdout);
} else {
console.log(stdout);
console.log(err);
console.log(stderr);
}
})
});
}
var getBaseImage = function (file, pass, quality) {
return new Promise((res, rej) => {
gs()
.batch().nopause().safer().res(300 * quality).option('-dTextAlphaBits=4').option('-dGraphicsAlphaBits=4').option('-sPDFPassword=' + pass)
.executablePath(gsPath).device('jpeg').output(Files.OutImg).input(file)
.exec((err, stdout, stderr) => {
if (!err) { res(); } else { rej(stdout) };
})
})
}
exports.processCard = function (file, password, quality) {
return new Promise((resolve, reject) => {
getBaseImage(file, password, quality) // Convert PDF to Image
.then(() => {
PdfToText(file, password) // Extract Text from pdf
.then((res) => {
GetCardType(getBaseCard(password), res) // finally get PDF Type
.then((ct) => {
// crop image here using Sharp
Promise.all([
crop(ct + 'A_' + quality, Files.file1),
crop(ct + 'B_' + quality, Files.file2)])
.then(() => {
// Merge Above two image into one using ImageMagick convert
spawn('convert', [Files.file1, Files.file2, '+append', 'files/out1.jpg']);
fs.unlinkSync(Files.OutImg); // Unlink tmp folders
fs.unlinkSync(Files.file1);
fs.unlinkSync(Files.file2);
resolve(); // finally resolve
}).catch((err) => reject(err));
}).catch((err) => reject(err))
}).catch((err) => reject(err))
}).catch((err) => reject(err))
})
}
and now these are the problem i am facing:
1. ImageMagick isn't creating the output file.
2. fs.unlinksysnc throws ENOENT: no such file or directory, unlink '/tmp/out1.jpeg'
on average every second execution.
3. Using above code increases execution time.
For Example: getBaseImage should complete in 600ms but it takes 1400 using above code.
About speed in General it (The Complete Function not just getBaseImage) should finish in 1100-1500ms(*) on average but the time taken is ~2500ms.
*1100-1500ms time is achievable by using function chaining but that is hard to read and maintaine for me.
I am going to use this function in Firebase Functions.
How to properly chain these functions ?
EDIT
exports.processCard = function (file, password, quality) {
return new Promise((resolve, reject) => {
console.log(performance.now());
getBaseImage(file, password, quality) //Convert PDF TO IMAGE
.then(() => { return PdfToText(file, password) })
.then((res) => {return GetCardType(getBaseCard(password), res) })
.then((ct) => {
return Promise.all([
crop(ct + 'A_' + quality, Files.file1),
crop(ct + 'B_' + quality, Files.file2)])
})
.then(() => {
spawn('convert', [Files.file1, Files.file2, '+append', 'files/out1.jpg']);
fs.unlinkSync(Files.OutImg); // Unlink tmp folders
fs.unlinkSync(Files.file1);
fs.unlinkSync(Files.file2);
resolve();
})
.catch((err) => { console.log(err) });
Using above pattern didn't solved my issues here.
There's a good chance this weirdness is caused by using the file system. If I understand it correctly, the fs in cloud functions is in memory, so when you write to it, read from it, and remove from it, you're using more and less os memory. That can get weird if a function is called repeatedly and re uses the loaded module.
One thing to try to keep the state clean for each invocation is to put everything (including the requires) inside the scope of the handler. That way you instantiate everything freshly on each invocation.
Finally, you don't seem to be waiting for the spawned convert command to run, you'll need to wait for it to complete:
const convertProc = spawn('convert', [Files.file1, Files.file2, '+append', 'files/out1.jpg']);
convertProc.on('close', function() {
fs.unlinkSync(Files.OutImg); // Unlink tmp folders
fs.unlinkSync(Files.file1);
fs.unlinkSync(Files.file2);
resolve();
})
convertProc.on('close', function(error) {
reject(error);
});
Then you wait for it to complete before you resolve.
I wrote a function running recursively to find out files whose name include given world. I do not understand how promises works and cannot find a way to write this function with promises despite trying hard.
I tried returning a promise inside findPath function but I couldn't use it since extractFiles calls findPath. I tried to create a list of promises and return all but couldn't succeed neither.
So how could I write these functions with promises?
const fs = require('fs');
const path = require('path');
function findPath(targetPath, targetWord, done) {
if (!fs.existsSync(targetPath)) return;
fs.readdir(targetPath, (err, allPaths) => {
if (err) done(err, null);
for (aPath of allPaths) {
aPath = path.join(targetPath, aPath);
extractFiles(aPath, targetWord, done);
}
});
function extractFiles(aPath, targetWord, done) {
fs.lstat(aPath, (err, stat) => {
if (err) done(err, null);
if (stat.isDirectory()) {
findPath(aPath, targetWord, done);
}
else if (aPath.indexOf(targetWord) >= 0) {
let fileName = aPath.split('.')[0];
done(null, fileName);
}
});
}
}
findPath('../modules', 'routes', file => {
console.log(file);
});
Firstly, to make the "core" code more readable, I'd promisify the fs functions
const promisify1p = fn => p1 => new Promise((resolve, reject) => {
fn(p1, (err, result) => {
if(err) {
reject(err);
} else {
resolve(result);
}
});
});
const readdirAsync = promisify1p(fs.readdir);
const lstatAsync = promisify1p(fs.lstat);
Then, just chain the promises as you would with any other promises
const fs = require('fs');
const path = require('path');
function findPath(targetPath, targetWord) {
const readPath = target =>
readdirAsync(target)
.then(allPaths =>
Promise.all(allPaths.map(aPath => extractFiles(path.join(target, aPath))))
.then(x => x.filter(x=>x)) // remove all the "false" entries - i.e. that don't match targetWord
.then(x => [].concat.apply([], x)) // flatten the result
);
const extractFiles = aPath =>
lstatAsync(aPath).then(stat => {
if (stat.isDirectory()) {
return readPath(aPath);
} else if (aPath.includes(targetWord)) {
return aPath.split('.')[0];
}
return false;
});
return readPath(targetPath);
}
findPath('../modules', 'routes')
.then(results => {
// do things with the results - which is an array of files that contain the targetWord
})
.catch(err => console.error(err));
Not much to it at all.
In node.js, I have a module that loops through a folder of files. That actually has a function callback that triggers when it finishes reading from directory. However for each file it finds, I run a readFile command which is async function, to read the file, and that has a callback function too. The question is, how can I set it up so that there is a callback once the directory looping function finishes and also each of the readFile functions?
var klaw = require('klaw');
var fse = require('fs-extra');
var items = [];
klaw("items").on('data', function (item) {
var dir = item.path.indexOf(".") == -1;
// if its a file
if (!dir) {
var filename = item.path;
if (filename.toLowerCase().endsWith(".json")) {
fse.readFile(filename, function(err, data) {
if (err) return console.error(err);
items.push(JSON.parse(data.toString()));
});
}
}
}).on('end', function () {
});
try something like this
import Promise from 'Bluebird';
const processing = []
const items = [];
klaw("items")
.on('data', item => processing.push(
Promise.promisify(fs.readFile))(item.path)
.then(content => items.push(JSON.parse(content.toString())))
.catch(err => null)
)
.on('end', () => {
Promise.all(processing)
.then(nothing => console.log(items))
})
or like
const processing = []
klaw("items")
.on(
'data',
item => processing.push(Promise.promisify(fs.readFile)(item.path))
)
.on(
'end',
() => {
Promise.all(processing)
.then(contents => (
contents.map(content =>(JSON.parse(content.toString())))
)
.then(items => console.log(items))
})