My code as following:
var data_array = [];
readData('file.txt');
console.log(data_array[0]);
console.log(data_array[1]);
console.log(data_array.length.toString());
console.log(data_array[data_array.length-1]);
//reads file into array
function readData(filepath) {
var fs = require('fs');
if (fs.existsSync(filepath)) {
var array = fs.readFileSync(filepath).toString().split("\n");
var data_array = array.slice(7,array.length - 2);
} else {
process.exit();
}
}
When I run this, I got following
undefined
undefined
0
undefined
See Data_array is used within the if statement.
I think the array did not receive anything, that is why it is printing nothing but undefined and its length is 0.
How can I enforce it execute step by step these lines in the written order
var data_array = [];
readData('file.txt');
console.log(data_array[0]);
console.log(data_array[1]);
....
Your code is failing, because you are creating local scope variable and it shadows global. To fix, replace var data_array = array... with data_array = array....
Also, keep in mind that you are using few antipatterns:
First, don't check for file existence before reading it — because it's possible for someone to delete file between your checks. Instead, just read it and handle exception.
Second, read file with readFileSync(filepath, { encoding: 'utf8' }) — this will return the string right away, so you won't need toString().
Third — array.slice() supports negative indexes (they count from the end of array), so you can literally have array.slice(7, -2).
And in general, unless this is a single use throw-away code, I'd suggest you to use asynchronous function counterparts:
const Promise = require('bluebird');
const fs = Promise.promisifyAll(require('fs'));
// reads file into array
const readData = Promise.coroutine(function *(filepath) {
try {
const array = (yield fs.readFileAsync(filepath, { encoding: 'utf8' })
).split("\n");
const data_array = array.slice(7, -2);
return data_array;
} catch(e) {
console.error(e);
process.exit();
}
});
Promise.coroutine(function *() {
const data_array = yield readData('file.txt');
console.log(data_array[0]);
console.log(data_array[1]);
console.log(data_array.length.toString());
console.log(data_array[data_array.length-1]);
})();
Related
I am having issues saving the results from a spawned python process. After converting data into json, I push the data to an array defined within the function before the spawn process is called, but the array keeps returning undefined. I can console.log and show the data correctly, but the array that is returned from the function is undefined. Any input would be greatly appreciated. Thanks in advance.
function sonar_projects(){
const projects = [];
let obj;
let str = '';
const projects_py = spawn('python', ['sonar.py', 'projects']);
let test = projects_py.stdout.on('data', function(data){
let projects = [];
let json = Buffer.from(data).toString()
str += json
let json2 = json.replace(/'/g, '"')
obj = JSON.parse(json2)
console.log(json2)
for(var dat in obj){
var project = new all_sonar_projects(obj[dat].key, obj[dat].name, obj[dat].qualifier, obj[dat].visibility, obj[dat].lastAnalysisDate);
projects.push(project);
}
for (var i= 0; i < projects.length; i++){
console.log(projects[i].key + ' ' + projects[i].name + ' ' + projects[i].qualifier + ' ' + projects[i].visibility + ' ' + projects[i].lastAnalysisDate)
}
console.log(projects)
return projects;
});
}
First of all, going through the NodeJS documentation, we have
Child Process
[child_process.spawn(command, args][, options])
Child Process Class
Stream
Stream Readable Event "data"
Even though projects_py.stdout.on(event_name, callback) accepts an callback, it returns either the EventEmitter-like object, where the events are registered (in this case, stdout that had it's method on called), or the parent element (the ChildProcess named projects_py).
It's because the callback function will be called every time the "data" event occurs. So, if the assign of the event returned the same as the callback function, it'd return only one single time, and then every next happening of the "data" event would be processed by the function, but not would be done.
In this kind of situation, we need a way to collect and compile the data of the projects_py.stdout.on("data", callback) event after it's done.
You already have the collecting part. Now see the other:
Right before you create the on "data" event, we create a promise to encapsulate the process:
// A promise says "we promise" to have something in the future,
// but it can end not happening
var promise = new Promise((resolve, reject) => {
// First of all, we collect only the string data
// as things can come in parts
projects_py.stdout.on('data', function(data){
let json = Buffer.from(data).toString()
str += json
});
// When the stream data is all read,
// we say we get what "we promised", and give it to "be resolved"
projects_py.stdout.on("end", () => resolve(str));
// When something bad occurs,
// we say what went wrong
projects_py.stdout.on("error", e => reject(e));
// With every data collected,
// we parse it (it's most your code now)
}).then(str => {
let json2 = str.replace(/'/g, '"')
// I changed obj to arr 'cause it seems to be an array
let arr = JSON.parse(json2)
//console.log(json2)
const projects = []
// With for-of, it's easier to get elements of
// an object / an array / any iterable
for(var dat of arr){
var project = new all_sonar_projects(
dat.key, dat.name, dat.qualifier,
dat.visibility, dat.lastAnalysisDate
);
projects.push(project);
}
// Template strings `a${variable or expression}-/b`
// are easier to compile things into a big string, yet still fast
for(var i = 0; i < projects.length; i++)
console.log(
`${projects[i].key} ${projects[i].name} ` +
`${projects[i].qualifier} ${projects[i].visibility} ` +
projects[i].lastAnalysisDate
)
console.log(projects)
// Your projects array, now full of data
return projects;
// Finally, we catch any error that might have happened,
// and show it on the console
}).catch(e => console.error(e));
}
Now, if you want to do anything with your array of projects, there are two main options:
Promise (then / catch) way
// Your function
function sonar_projects(){
// The new promise
var promise = ...
// As the working to get the projects array
// is already all set up, you just use it, but in an inner scope
promise.then(projects => {
...
});
}
Also, you can just return the promise variable and do the promise-things with it out of sonar_projects (with then / catch and callbacks).
async / await way
// First of all, you need to convert your function into an async one:
async function sonar_projects(){
// As before, we get the promise
var promise = ...
// We tell the function to 'wait' for it's data
var projects = await promise;
// Do whatever you would do with the projects array
...
}
I am writing code that generates a very large JSON object, saves it to a file, then loads the file and inserts the data into a Mongo collection. I want to pass a string from the command line when calling the script that I use to set the file name, as well as the collection name. I call it like so: node --max-old-space-size=8192 data_generator.js foo 1000000.
The code fails with error ENOENT: no such file or directory, open 'foo.json' on the third line of the function gen_collection() where I set the variable data. This error does not appear when a file foo.json already exists, even if it is empty. Before it fails, the code successfully creates a file foo.json but it contains only an empty array [].
The code fails with this same exact error when I include any reference to process.argv. This includes when I try to set any variable to a value from the process.argv array. The code works when I set the variables fname as const fname = "foo" and size as const size = 0. However, even if the only reference I have to process.argv is in a console.log i.e. adding console.log(process.argv[2] to main(), it fails with the exact same error as above.
Here is the code I am trying to run:
const { MongoClient } = require("mongodb");
const fs = require('fs');
const bjson = require('big-json');
async function main() {
const uri = "my db uri";
const client = new MongoClient(uri);
const fname = process.argv[2];
const size = parseInt(process.argv[3]);
// const fname = 'small'
// const size = 1
try {
await client.connect({ useUnifiedTopology: true });
await write_data_to_disk(fname, size);
await gen_collection(client, fname);
} catch (e) {
console.error(e);
} finally {
await client.close();
}
};
// generate data as json aray and write to local file
async function write_data_to_disk(fname, size) {
let arr = [];
for (let i = 0; i < size; i++) {
let doc = gen_document();
arr.push(doc);
}
const strStream = bjson.createStringifyStream({
body: arr
})
let logger = fs.createWriteStream(`${fname}.json`);
strStream.on('data', (d) => {
logger.write(d);
})
};
async function gen_collection(client, fname) {
let db = client.db('test');
let collection = db.collection(fname);
let data = JSON.parse(fs.readFileSync(`${fname}.json`, 'utf8')); // ERROR APPEARS ON THIS LINE
bulkUpdateOps = [];
data.forEach((doc) => {
bulkUpdateOps.push({"insertOne": {"document": doc}});
if (bulkUpdateOps.length === 1000) {
collection.bulkWrite(bulkUpdateOps);
bulkUpdateOps = [];
}
})
if (bulkUpdateOps.length > 0) {
collection.bulkWrite(bulkUpdateOps);
}
};
function gen_document() {
// returns json object
};
You're doing
await write_data_to_disk(...)
but that function doesn't return a promise that is connected to when it's done. So, you're trying to read the resulting file BEFORE it has been created or before it has valid content in it and thus the ENOENT error as the file doesn't yet exist when you're trying to read from it in the following function.
Writestreams do not play nicely with promises unless you wrap them in your own promise that resolves when you are completely done writing to the stream and the file has been closed.
Also, you probably want to just .pipe() strStream to the logger stream. Much easier and you can then just monitor when that pipe() operation is done to resolve the promise you wrap around that operation.
You can promisify write_data_to_disk() like this:
// generate data as json aray and write to local file
function write_data_to_disk(fname, size) {
return new Promise((resolve, reject) => {
const arr = [];
for (let i = 0; i < size; i++) {
let doc = gen_document();
arr.push(doc);
}
const strStream = bjson.createStringifyStream({ body: arr });
const dest = fs.createWriteStream(`${fname}.json`, {emitClose: true});
// monitor for completion and errors
dest.on('error', reject).on('close', resolve);
strStream.on('error', reject);
// pipe all the content from strStream to the dest writeStream
strStream.pipe(dest);
});
}
Since this returns a promise that is truly tied to when the write operation is done, you can then use await write_data_to_disk(...).
Is there a simple way in node.js to do something like this, i.e. a function, e.g., readline reading exact one line from stdin and returning a string?
let a = parseInt(readline(stdin));
let b = parseFloat(readline(stdin));
I don't wish to read a whole block of lines and parse it line by line, like using process.stdin.on("data") or rl.on("line").
In the answer provided in http://stackoverflow.com/questions/20086849/how-to-read-from-stdin-line-by-line-in-node, every line is processed by the same function block, and I still can not assign each line to a variable on reading a line.
var readline = require('readline');
var rl = readline.createInterface({
input: process.stdin,
output: process.stdout,
terminal: false
});
rl.on('line', function(line){
console.log(line);
})
It's pretty evident that reading a line from a stream is going to be an async operation anyway. (as you don't know when the line actually appears in the stream). So you should deal either with callbacks/promises or with generators. And when the read happens you get the line (in a callback, as a value returned in .then or simply by assign it to a variable as you want if you use generators).
So if it's an option for you to use ES6 and comething like 'co' to run the generator here is what you can try.
const co = require('co');
//allows to wait until EventEmitter such as a steam emits a particular event
const eventToPromise = require('event-to-promise');
//allows to actually read line by line
const byline = require('byline');
const Promise = require('bluebird');
class LineReader {
constructor (readableStream) {
let self = this;
//wrap to receive lines on .read()
this.lineStream = byline(readableStream);
this.lineStream.on('end', () => {
self.isEnded = true;
});
}
* getLine () {
let self = this;
if (self.isEnded) {
return;
}
//If we recieve null we have to wait until next 'readable' event
let line = self.lineStream.read();
if (line) {
return line;
}
yield eventToPromise(this.lineStream, 'readable');
//'readable' fired - proceed reading
return self.lineStream.read();
}
}
I used this to run it for test purposes.
co(function *() {
let reader = new LineReader(process.stdin);
for (let i = 0; i < 100; i++) {
//You might need to call .toString as it's a buffer.
let line = yield reader.getLine();
if (!line) {break;}
console.log(`Received the line from stdin: ${line}`);
}
});
Definitely it's going to work out of the box if you're using koa.js (generator based Express-like framework)
If you do not want ES6 you can do the same on bare Promises. It is going to be something like this. http://jsbin.com/qodedosige/1/edit?js
My use case is this: I am looking to read a CSV file in Node and get only the headers. I don't want to write the results of a read stream to a file, rather push the headers to an array once the file is read, so I can take that array and do something to it later on. OR, better yet, take the stream and as it is being read, transform it, then send it to an array. File is a contrived value. I am stuck at this point, where the current output of datafile is an empty array:
const fs = require('fs');
const parse = require('csv-parse');
const file = "my file path";
let dataFile = [];
rs = fs.createReadStream(file);
parser = parse({columns: true}, function(err, data){
return getHeaders(data)
})
function getHeaders(file){
return file.map(function(header){
return dataFile.push(Object.keys(header))
})
}
What do I need to do in order to get the results I need? I am expecting the headers to be found in an array as the end result.
Ok, so there is some confusing things in your code, and one mistake : you didn't actually call your code :)
First, a solution, add this line, after parser :
rs.pipe(parser).on('end', function(){
console.log(dataFile);
});
And magic, dataFile is not empty.
You stream the file from disk, pass it to the parser, then at the end, call a callback.
For the confusing parts :
parser = parse({columns: true}, function(err, data){
// You don't need to return anything from the callback, you give the impression that parser will be the result of getHeaders, it's not, it's a stream.
return getHeaders(data)
})
function getHeaders(file){
// change map to each, with no return, map returns an array of the return of the callback, you return an array with the result of each push (wich is the index of the new object).
return file.map(function(header){
return dataFile.push(Object.keys(header))
})
}
And finaly :
Please choose with ending line with ; or not, but not a mix ;)
You should end with something like :
const fs = require('fs');
const parse = require('csv-parse');
const file = "./test.csv";
var dataFile = [];
rs = fs.createReadStream(file);
parser = parse({columns: true}, function(err, data){
getHeaders(data);
});
rs.pipe(parser).on('end', function(){
console.log(dataFile);
});
function getHeaders(file){
file.each(function(header){
dataFile.push(Object.keys(header));
});
}
I have a NodeJS readstream listener within a function and I am trying to get my function to return a value (contents of the file) which is derived from within the listener.
e.g. :
MyObj.prototype.read = function(){
var file3 = fs.createReadStream('test.txt', {encoding: 'utf8'});
var contentRead = '';
file3.addListener('data', function(data) {
contentRead += data.toString('utf-8');
return contentRead;
});
}
I would like to do something like var contents = myDerivedObj.read() to return the contents of the file.
However, the return from inside the listener is not getting returned correctly - getting 'undefined'. And, returning from outside the listener just returns an empty string.
I cannot change the signature of read(), so I cannot add a callback as an argument.
In general, this is a poor pattern: NodeJS REALLY REALLY REALLY doesn't like it when you do things like this 'cause you block the main thread. You will discover that your performance TRULY sucks. So DON'T DO THIS. But if you MUST then you can try this:
MyObj.prototype.read = function(){
var file3 = fs.createReadStream('test.txt', {encoding: 'utf8'});
var contentRead = '';
var done = false;
file3.addListener('data', function(data) {
contentRead += data.toString('utf-8');
});
file3.addListener("end", function () {
done = true;
});
while(!done);
return contentRead;
}
EDIT: #Brandon is right, and I'm wrong. I just tested this and although I'd thought the callbacks would work the entire node process locks up. Do this:
MyObj.prototype.read = function(){
return fs.readFileSync('test.txt', 'utf-8');
}