I'm a beginner in non-blocking environment, such NodeJS. Below is my simple code, which list all files in directory :
var readline = require('readline');
var rl = readline.createInterface(process.stdin, process.stdout);
var fs = require('fs');
var datafolder = './datafolder';
var datafoldername = 'datafolder';
rl.setPrompt('Option> ');
rl.prompt();
rl.on('line', function(line) {
if (line === "right") rl.close();
if (line == '1') {
listFile();
}
rl.prompt();
}).on('close', function() {
process.exit(0);
});
function listFile() {
console.log(`File(s) on ${datafolder}`);
fs.readdirSync(datafolder, (err, files) => {
if (err) {
console.log(err);
} else {
files.forEach(filename => {
console.log(filename);
});
}
});
}
If user press 1, it's suppose to execute method listFile and show all files inside.
My question is, why fs.readdirSync not executed? The program works if I do it with readdir(), but it'll mess the output to user.
You are passing a callback to fs.readdirSync() but *Sync() functions don't take callbacks. The callback is never run (because the function does not take a callback), so you see no output. But fs.readdirSync() does in fact execute.
fs.readdirSync() simply returns it's value (which may make the program easier to read, but also means the call will block, which may be OK depending on what your program does and how it is used.)
var resultsArray = fs.readdirSync(datafolder);
(You may want to wrap it in a try/catch for error handling.)
Related
I already asked the question but I fele like I didn't asked it properly.
I'm trying to make a little encryption module in NodeJS but I have a very hard time with the asynchronous nature of it.
First of, the result variable in my main file is undefined a milisecond after the script is called, I was expecting that.
Also, the file is processed while the encryption occur, not before.
Note that I do not wish to encrypt the file itself.
The big question:
How can I make everything run smoothly and in order? :D
www.js
var mod = require('./mymodule.js')
var result = mod.doencrypt();
mymodule.js
module.exports.doencrypt = function() {
var content = processFile(); //Open a file, increment counter
var key = generateKey();
var iv = generateIV();
var encrypt = doEncryption(content);
return encrypt;
}
//File manipulation
async function openFile() {
return new Buffer(await readFile('monolitic.txt', "binary"));
}
async function saveFile(bin) {
await fs.writeFile("monolitic.txt", bin, "binary", function(err) {
if(err) {
console.log(err);
} else {
console.log("The monolitic file was saved!");
return bin;
}
});
}
function processFile() {
console.log("Reading buffer")
openFile().then(function (bin) {
monoCounter = bin;
//Increment
inc(monoCounter);
console.log(monoCounter);
monoCounter = saveMonoCounter(monoCounter);
return monoCounter;
}).catch((err) => {
monoCounter = Buffer.alloc(128);
saveMonoCounter(Buffer.alloc(128));
});
}
I am having a slight issue when using async.queue with a filestream
I have a scenario where my filestream will finish
I set fileRead to true
however the queue will be empty and already have called drain
this then leads my "done" to never be called
what is the proper way to say "end the queue" after my filestream is "end" and the queue is empty?
var fs = require('fs')
, util = require('util')
, stream = require('stream')
, es = require('event-stream');
var async = require('async');
var fileRead = false;
var lineNr = 0;
var q = async.queue(function(task, callback) {
task(function(err, lineData){
responseLines.push(lineData);
callback();
});
}, 5);
var q.drain = function() {
if(fileRead){
done(null, responseLines);
}
}
var s = fs.createReadStream('very-large-file.csv')
.pipe(es.split())
.pipe(es.mapSync(function(line){
s.pause();
q.push(async.apply(insertIntoDb, line))
s.resume();
})
.on('error', function(err){
done(err);
})
.on('end', function(){
fileRead = true;
})
);
or is there a better use of async which would allow me to do this?
async process line by line with the ability to exit early if one of the lines has errors
Firstly, I'm not sure how much of your snippet is pseudo code but var q.drain = ... is not valid javascript and should error. It should just be q.drain = as you're defining a property on an existing object not declaring a new variable. This could be why your drain function isn't firing if it isn't pseudo code.
There are a few ways you could achieve what I think you're trying to do. One would be to check the length of the queue in your end handler and set the drain function if there are still items to process.
.on('end', function(){
if(!q.length){
callDone();
}
else {
q.drain = callDone;
}
});
function callDone(){
done(null, responseLines);
}
This is effectively saying "if the queue's been processed call done, if not, call done when it has!" I'm certain there are lots of ways to tidy up your code but hopefully this provides a solution to your specific problem.
My problem is that the code does not seem to be running in order, as seen below.
This code is for my discord.js bot that I am creating.
var Discord = require("discord.js");
var bot = new Discord.Client();
var yt = require("C:/Users/username/Documents/Coding/Discord/youtubetest.js");
var youtubetest = new yt();
var fs = require('fs');
var youtubedl = require('youtube-dl');
var prefix = "!";
var vidid;
var commands = {
play: {
name: "!play ",
fnc: "Gets a Youtube video matching given tags.",
process: function(msg, query) {
youtubetest.respond(query, msg);
var vidid = youtubetest.vidid;
console.log(typeof(vidid) + " + " + vidid);
console.log("3");
}
}
};
bot.on('ready', () => {
console.log('I am ready!');
});
bot.on("message", msg => {
if(!msg.content.startsWith(prefix) || msg.author.bot || (msg.author.id === bot.user.id)) return;
var cmdraw = msg.content.split(" ")[0].substring(1).toLowerCase();
var query = msg.content.split("!")[1];
var cmd = commands[cmdraw];
if (cmd) {
var res = cmd.process(msg, query, bot);
if (res) {
msg.channel.sendMessage(res);
}
} else {
let msgs = [];
msgs.push(msg.content + " is not a valid command.");
msgs.push(" ");
msgs.push("Available commands:");
msgs.push(" ");
msg.channel.sendMessage(msgs);
msg.channel.sendMessage(commands.help.process(msg));
}
});
bot.on('error', e => { console.error(e); });
bot.login("mytoken");
The youtubetest.js file:
var youtube_node = require('youtube-node');
var ConfigFile = require("C:/Users/username/Documents/Coding/Discord/json_config.json");
var mybot = require("C:/Users/username/Documents/Coding/Discord/mybot.js");
function myyt () {
this.youtube = new youtube_node();
this.youtube.setKey(ConfigFile.youtube_api_key);
this.vidid = "";
}
myyt.prototype.respond = function(query, msg) {
this.youtube.search(query, 1, function(error, result) {
if (error) {
msg.channel.sendMessage("There was an error finding requested video.");
} else {
vidid = 'http://www.youtube.com/watch?v=' + result.items[0].id.videoId;
myyt.vidid = vidid;
console.log("1");
}
});
console.log("2");
};
module.exports = myyt;
As the code shows, i have an object for the commands that the bot will be able to process, and I have a function to run said commands when a message is received.
Throughout the code you can see that I have put three console.logs with 1, 2 and 3 showing in which order I expect the parts of the code to run. When the code is run and a query is found the output is this:
I am ready!
string +
2
3
1
This shows that the code is running in the wrong order that I expect it to.
All help is very highly appreciated :)
*Update! Thank you all very much to understand why it isn't working. I found a solution where in the main file at vidid = youtubetest.respond(query, msg) when it does that the variable is not assigned until the function is done so it goes onto the rest of my code without the variable. To fix I simply put an if statement checking if the variable if undefined and waiting until it is defined.*
Like is mentioned before, a lot of stuff in javascript runs in async, hence the callback handlers. The reason it runs in async, is to avoid the rest of your code being "blocked" by remote calls. To avoid ending up in callback hell, most of us Javascript developers are moving more and more over to Promises. So your code could then look more like this:
myyt.prototype.respond = function(query, msg) {
return new Promise(function(resolve, reject) {
this.youtube.search(query, 1, function(error, result) {
if (error) {
reject("There was an error finding requested video."); // passed down to the ".catch" statement below
} else {
vidid = 'http://www.youtube.com/watch?v=' + result.items[0].id.videoId;
myyt.vidid = vidid;
console.log("1");
resolve(2); // Resolve marks the promises as successfully completed, and passes along to the ".then" method
}
});
}).then(function(two) {
// video is now the same as myyt.vidid as above.
console.log(two);
}).catch(function(err) {
// err contains the error object from above
msg.channel.sendMessage(err);
})
};
This would naturally require a change in anything that uses this process, but creating your own prototypes seems.. odd.
This promise returns the vidid, so you'd then set vidid = youtubetest.response(query, msg);, and whenever that function gets called, you do:
vidid.then(function(id) {
// id is now the vidid.
});
Javascript runs async by design, and trying to hack your way around that leads you to dark places fast. As far as I can tell, you're also targetting nodeJS, which means that once you start running something synchronously, you'll kill off performance for other users, as everyone has to wait for that sync call to finish.
Some suggested reading:
http://callbackhell.com/
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise
https://stackoverflow.com/a/11233849/3646975
I'd also suggest looking up ES6 syntax, as it shortens your code and makes life a hellofalot easier (native promises were only introduced in ES6, which NodeJS 4 and above supports (more or less))
In javascript, please remember that any callback function you pass to some other function is called asynchronously. I.e. the calls to callback function may not happen "in order". "In order" in this case means the order they appear on the source file.
The callback function is simply called on certain event:
When there is data to be processed
on error
in your case for example when the youtube search results are ready,
'ready' event is received or 'message' is received.
etc.
I am hosting Node as a server locally to interact with hardware.
My web application then makes a request to Node to execute 2 executable, whichever executable returns a data first will Respond it back to my Web Application.
By doing so, it causes the other executable to still be running in the background waiting for response from the hardware.
I am unable to kill off that process either, I have to either manually stop Node and run it again or task kill that executable.
My code are below:
Node.JS
var exec = require('child_process').exec;
app.get('/Page', function (req, res) {
var Page = function () {
var a = exec('F:/Example1.exe', function (err, data) {
console.log(err);
console.log(data);
b.kill();
if (!res.headersSent) {
res.send(data);
}
});
var b = exec('F:/Example2.exe', function (err, data) {
console.log(err);
console.log(data);
a.kill();
if (!res.headersSent) {
res.send(data);
}
});
}
Page();
});
Apparently, even with the kill command, I am still unable to terminate the process.
I should let you guys know, I am also using AngularJS for my front-end.
I have sourced online for solution, however Google's results are all slowly turning purple.
Thank you so much for those who post their solution, please explain to me the details of the solution as well. I would really want to learn more.
Thank you so much.
The problem with exec is it will wait until the program has executed for it's callback to run.
You can use spawn instead, then you have control over the process as it's running.
var spawn = require('child_process').spawn;
app.get('/Page', function(req, res) {
var Page = function() {
var a = spawn('F:/Example1.exe');
var b = spawn('F:/Example2.exe');
var aData = '',
bData = '';
a.stdout.on('data', function(data) {
aData += data.toString();
});
b.stdout.on('data', function(data) {
bData += data.toString();
});
a.on('close', function() {
b.kill();
if (!res.headersSent) {
res.send(aData);
}
});
b.on('close', function() {
a.kill();
if (!res.headersSent) {
res.send(bData);
}
});
}
Page();
});
I have never used exec in nodejs but javascript scoping I think Page is executed every request, so the a and b processes from previous requests are no longer around.
You could store references to the processes globally so that each request has access to the processes, (real incomplete rough example):
var exec = require('child_process').exec;
var a = null;
var b = null;
app.get('/Page', function (req, res) {
var Page = function () {
if (a) {
// a is already a running process? Do something?
} else {
// start a ?
a = exec('your command');
}
if (b) {
// b is already running? Do something?
}
}
Page();
});
I'm not quite grasping how exactly node works regarding async and loops.
What I want to achieve here is have the console print out "Command: " and await for the user's input. But while it's waiting I want it to run "someRandomFunction()" endlessly until the user inputs "exit" onto the terminal.
Would appreciate all the help - and possibly an explanation so I can understand!
Thank you! :)
var rl = readline.createInterface({
input: process.stdin,
output: process.stdout
});
rl.question("Command: ", function(answer) {
if (answer == "exit"){
rl.close();
} else {
// If not "exit", How do I recall the function again?
}
});
someRandomFunction();
I would suggest making the function repeatable like so.
var rl = readline.createInterface({
input: process.stdin,
output: process.stdout
});
var waitForUserInput = function() {
rl.question("Command: ", function(answer) {
if (answer == "exit"){
rl.close();
} else {
waitForUserInput();
}
});
}
Then call
waitForUserInput();
someRandomFunction();
I'm unsure if the syntax you are using for .question is correct though, does that part of the code work?
You may also write this in the following manner.
var rl = readline.createInterface({
input: process.stdin,
output: process.stdout
});
function waitForUserInput() {
rl.question("Command: ", function(answer) {
if (answer == "exit"){
rl.close();
} else {
waitForUserInput();
}
});
}
The important lesson here is that to re-use a function it has to be named and be available in scope. If you have any more questions about this please ask.
The other answer is fine but uses recursion unnecessarily.
The key is to solving this is to separate, in your mind, the simple loop-based approach that is used in other languages from the asynchronous approach of Node.
In other languages, you might use a loop like this:
while not finished:
line = readline.read()
if line == 'woof':
print('BARK')
elif line == 'exit':
finished = True
... # etc
Node, at least with Readline, doesn't work this way.
In Node, you fire up Readline, give it event handlers, then return, and handle the completion of the readline loop later.
Consider this code, which you can copy-paste-run:
const readline = require('readline');
function replDemo() {
return new Promise(function(resolve, reject) {
let rl = readline.createInterface(process.stdin, process.stdout)
rl.setPrompt('ready> ')
rl.prompt();
rl.on('line', function(line) {
if (line === "exit" || line === "quit" || line == 'q') {
rl.close()
return // bail here, so rl.prompt() isn't called again
}
if (line === "help" || line === '?') {
console.log(`commands:\n woof\n exit|quit\n`)
} else if (line === "woof") {
console.log('BARK!')
} else if (line === "hello") {
console.log('Hi there')
} else {
console.log(`unknown command: "${line}"`)
}
rl.prompt()
}).on('close',function(){
console.log('bye')
resolve(42) // this is the final result of the function
});
})
}
async function run() {
try {
let replResult = await replDemo()
console.log('repl result:', replResult)
} catch(e) {
console.log('failed:', e)
}
}
run()
Run this and you'll get output like this:
$ node src/repl-demo.js
ready> hello
Hi there
ready> boo
unknown command: "boo"
ready> woof
BARK!
ready> exit
bye
repl result: 42
Note that the run function calls replDemo and "awaits" the result of the promise.
If you're unfamiliar with async/await, here's the same logic written it the "traditional" Promise style:
function run2() {
replDemo().then(result => {
console.log('repl result:', result)
}).catch(e => {
console.log('failed:', e)
})
console.log('replDemo has been called')
}
Note that I added output "replDemo has been called" for a reason - Running the above shows output like this:
$ node src/repl-demo.js
ready> replDemo has been called
woof
BARK!
ready> hello
Hi there
ready> bye
repl result: 42
Note how "replDemo has been called" appears immediately after the first "ready>" prompt. That's because the replDemo() function returns immediately, then run2() exits immediately, and the main is all done - yet the readline is still executing!
That's hard to grasp if you come from an imperative programming background like me. The async event-driven loop at the core of nodejs keeps running until all the work is done, and that occurs when the last promise is resolved, which happens when the readline instance is "closed", which happens when "exit" is entered by the user (or EOF is received, which is CTRL+D on most systems, and CTRL+Z on Windows).