I have a txt file contains:
{"date":"2013/06/26","statement":"insert","nombre":1}
{"date":"2013/06/26","statement":"insert","nombre":1}
{"date":"2013/06/26","statement":"select","nombre":4}
how I can convert the contents of the text file as array such as:
statement = [
{"date":"2013/06/26","statement":"insert","nombre":1},
{"date":"2013/06/26","statement":"insert","nombre":1},
{"date":"2013/06/26","statement":"select","nombre":4}, ];
I use the fs module node js. Thanks
Sorry
I will explain more detailed:
I have an array :
st = [
{"date":"2013/06/26","statement":"insert","nombre":1},
{"date":"2013/06/26","statement":"insert","nombre":5},
{"date":"2013/06/26","statement":"select","nombre":4},
];
if I use this code :
var arr = new LINQ(st)
.OrderBy(function(x) {return x.nombre;})
.Select(function(x) {return x.statement;})
.ToArray();
I get the result I want.
insert select insert
but the problem my data is in a text file.
any suggestion and thanks again.
There is no reason for not to do your file parser yourself. This will work on any size of a file:
var fs = require('fs');
var fileStream = fs.createReadStream('file.txt');
var data = "";
fileStream.on('readable', function() {
//this functions reads chunks of data and emits newLine event when \n is found
data += fileStream.read();
while( data.indexOf('\n') >= 0 ){
fileStream.emit('newLine', data.substring(0,data.indexOf('\n')));
data = data.substring(data.indexOf('\n')+1);
}
});
fileStream.on('end', function() {
//this functions sends to newLine event the last chunk of data and tells it
//that the file has ended
fileStream.emit('newLine', data , true);
});
var statement = [];
fileStream.on('newLine',function(line_of_text, end_of_file){
//this is the code where you handle each line
// line_of_text = string which contains one line
// end_of_file = true if the end of file has been reached
statement.push( JSON.parse(line_of_text) );
if(end_of_file){
console.dir(statement);
//here you have your statement object ready
}
});
If it's a small file, you might get away with something like this:
// specifying the encoding means you don't have to do `.toString()`
var arrayOfThings = fs.readFileSync("./file", "utf8").trim().split(/[\r\n]+/g).map(function(line) {
// this try/catch will make it so we just return null
// for any lines that don't parse successfully, instead
// of throwing an error.
try {
return JSON.parse(line);
} catch (e) {
return null;
}
// this .filter() removes anything that didn't parse correctly
}).filter(function(object) {
return !!object;
});
If it's larger, you might want to consider reading it in line-by-line using any one of the many modules on npm for consuming lines from a stream.
Wanna see how to do it with streams? Let's see how we do it with streams. This isn't a practical example, but it's fun anyway!
var stream = require("stream"),
fs = require("fs");
var LineReader = function LineReader(options) {
options = options || {};
options.objectMode = true;
stream.Transform.call(this, options);
this._buffer = "";
};
LineReader.prototype = Object.create(stream.Transform.prototype, {constructor: {value: LineReader}});
LineReader.prototype._transform = function _transform(input, encoding, done) {
if (Buffer.isBuffer(input)) {
input = input.toString("utf8");
}
this._buffer += input;
var lines = this._buffer.split(/[\r\n]+/);
this._buffer = lines.pop();
for (var i=0;i<lines.length;++i) {
this.push(lines[i]);
}
return done();
};
LineReader.prototype._flush = function _flush(done) {
if (this._buffer.length) {
this.push(this._buffer);
}
return done();
};
var JSONParser = function JSONParser(options) {
options = options || {};
options.objectMode = true;
stream.Transform.call(this, options);
};
JSONParser.prototype = Object.create(stream.Transform.prototype, {constructor: {value: JSONParser}});
JSONParser.prototype._transform = function _transform(input, encoding, done) {
try {
input = JSON.parse(input);
} catch (e) {
return done(e);
}
this.push(input);
return done();
};
var Collector = function Collector(options) {
options = options || {};
options.objectMode = true;
stream.Transform.call(this, options);
this._entries = [];
};
Collector.prototype = Object.create(stream.Transform.prototype, {constructor: {value: Collector}});
Collector.prototype._transform = function _transform(input, encoding, done) {
this._entries.push(input);
return done();
};
Collector.prototype._flush = function _flush(done) {
this.push(this._entries);
return done();
};
fs.createReadStream("./file").pipe(new LineReader()).pipe(new JSONParser()).pipe(new Collector()).on("readable", function() {
var results = this.read();
console.log(results);
});
fs.readFileSync("myfile.txt").toString().split(/[\r\n]/)
This gets your each line as a string
You can then use UnderscoreJS or your own for loop to apply the JSON.parse("your json string") method to each element of the array.
var arr = fs.readFileSync('mytxtfile', 'utf-8').split('\n')
I think this is the simplest way of creating an array from your text file
Related
How to combine these two codes, so it doesn't just covert csv to Json (first code), but also save this as an json array in an extra file?(second code)
this (first) code converts csv file to json array:
const fs = require("fs");
let fileReadStream = fs.createReadStream("myCsvFile.csv");
let invalidLineCount = 0;
const csvtojson = require("csvtojson");
csvtojson({ "delimiter": ";", "fork": true })
.preFileLine((fileLineString, lineIdx)=> {
let invalidLinePattern = /^['"].*[^"'];/;
if (invalidLinePattern.test(fileLineString)) {
console.log(`Line #${lineIdx + 1} is invalid, skipping:`, fileLineString);
fileLineString = "";
invalidLineCount++;
}
return fileLineString
})
.fromStream(fileReadStream)
.subscribe((dataObj) => {
console.log(dataObj);
// I added the second code hier, but it wirtes the last object of the array (because of the loop?)
}
});
and this (second) code saves the json array to an external file:
fs.writeFile('example.json', JSON.stringify(dataObj, null, 4);
The quistion is how to put the second codes into the first code (combine them)?
You can use .on('done',(error)=>{ ... }) method. (csvtojson). Push the data into a variable in subscribe method and write the data as JSON in .on('done'). (test was successful).
Check it out:
let fileReadStream = fs.createReadStream("username-password.csv");
let invalidLineCount = 0;
let data = []
csvtojson({ "delimiter": ";", "fork": true })
.preFileLine((fileLineString, lineIdx)=> {
let invalidLinePattern = /^['"].*[^"'];/;
if (invalidLinePattern.test(fileLineString)) {
console.log(`Line #${lineIdx + 1} is invalid, skipping:`, fileLineString);
fileLineString = "";
invalidLineCount++;
}
return fileLineString
})
.fromStream(fileReadStream)
.subscribe((dataObj) => {
// console.log(dataObj)
data.push(dataObj)
})
.on('done',(error)=>{
fs.writeFileSync('example.json', JSON.stringify(data, null, 4))
})
Not sure if you are able to change the library but I would definitely recommend Papaparse for this - https://www.npmjs.com/package/papaparse
Your code would then look something like this:
const fs = require('fs'), papa = require('papaparse');
var readFile = fs.createReadStream(file);
papa.parse(readFile, {
complete: function (results, file) {
fs.writeFile('example.json', JSON.stringifiy(results.data), function (err) {
if(err) console.log(err);
// callback etc
})
}
});
I have a "clients.txt" file where I have a list of emails. I try to run a program for sending emails where I chose a number of emails to use from the file, in that case the number is 2. After I use the two emails I want to overwrite "clients.txt" without them. The problem is when I try to run the code just for one single time every thing is working! but if I make a loop something is wrong. Looking forward to see any help from you guys. Thanks! I add the code bellow. PS: Sorry for my bad english!
function readEmails(){
const fs = require('fs');
clients_list = fs.readFileSync('clients.txt', 'utf8').split('\n');
let filtered = clients_list.filter(function (el) {
return el != null && el != '';
});
return filtered
}
function dump_array(arr, file){
let fs = require('fs');
let file = fs.createWriteStream(file);
file.on('error', function(err) { /* error handling */ });
arr.forEach(function(v) { file.write(v + '\n'); });
file.end();
}
while_var = 0;
while (while_var < 2){
while_var ++;
let all_clients = readEmails();
let selected_clients = [];
if (all_clients.length > 0){
selected_clients = all_clients.splice(0,2);
dump_array(all_clients, 'clients.txt');
console.log(selected_clients);
}else{
console.log('No more clients')
}
}
const fs = require('fs');
function readEmails(){
const clients_list = fs.readFileSync('clients.txt', 'utf8').split('\n');
const filtered = clients_list
// clear false, 0 and undefined too
.filter(el => !!el)
// remove extra spaces and \r symbols
.map(el => el.trim());
return filtered;
}
function dump_array(arr, file){
// Here you need sync method.
fs.writeFileSync(file, arr.join('\n'));
// And here was 'already declared' error in orginal code
}
let while_var = 0;
while (while_var++ < 2){
let all_clients = readEmails();
let selected_clients = [];
if (all_clients.length > 0){
selected_clients = all_clients.splice(0,2);
dump_array(all_clients, 'clients.txt');
console.log(selected_clients);
}else{
console.log('No more clients')
}
}
I have more than 6 environments against which i have to run the same set of rest api scripts. For that reason i have stored all that test data and the end points/resource paths in a json file. I then try to read this json file into my karate-config.js file, this is because i want to fetch the data corresponding to the environment that is being passed from the command line (karate.env), which am reading into my karate-config.js file
Below is my json file sample
[
{
"qa":{
"username_cm_on":"test_cm_on_qa",
"password_cm_on":"Test123$",
"nonadmin_username_cm_on":"test_non_admin_cm_on_qa",
"nonadmin_password_cm_on":"Test123$",
"username_cm_off":"test_cm_off_qa",
"password_cm_off":"Test123$",
"nonadmin_username_cm_off":"test_non_admin_cm_off_qa",
"nonadmin_password_cm_off":"Test123$",
"zuul_urls":{
"home-sec-uri":"https://qa.abc.com/qa/home-sec-uri",
"home-res-uri":"https://qa.abc.com/qa/home-res-uri"
}
}
},
{
"uat":{
"username_cm_on":"test_cm_on_uat",
"password_cm_on":"Test123$",
"nonadmin_username_cm_on":"test_non_admin_cm_on_uat",
"nonadmin_password_cm_on":"Test123$",
"username_cm_off":"test_cm_off_uat",
"password_cm_off":"Test123$",
"nonadmin_username_cm_off":"test_non_admin_cm_off_uat",
"nonadmin_password_cm_off":"Test123$",
"zuul_urls":{
"home-sec-uri":"https://uat.abc.com/qa/home-sec-uri",
"home-res-uri":"https://uat.abc.com/qa/home-res-uri"
}
}
}
]
and below is my karate-config.js file
function() {
// var env = karate.env; // get system property 'karate.env'
var env = 'qa';
var cm = 'ON';
var envData = call read('classpath:env_data.json'); //require("./env_data.json");
// write logic to read data from the json file _ Done, need testing
karate.log('karate.env system property was:', env);
switch(env) {
case "qa":
if(cm === 'ON'){
config.adminusername_cm_on = getData().username_cm_on;
config.adminpassword_cm_on = "";
config.nonadminusername_cm_on = getData().nonadmin_username_cm_on;
config.nonadminpassword_cm_on = "";
}else if(cm === "OFF") {
config.adminusername_cm_off = getData().username_cm_off;
config.adminpassword_cm_off = "";
config.nonadminusername_cm_off = getData().nonadmin_username_cm_off;
config.nonadminpassword_cm_off = "";
}
break;
case "uat":
break;
default:
break;
}
// This method will return the data from the env_data.json file
var getData = function() {
for(var i = 0; i < obj.length; i++) {
for(var e in obj[i]){
var username_cm_on = obj[i][e]['username_cm_on'];
var nonadmin_username_cm_on = obj[i][e]['nonadmin_username_cm_on'];
var username_cm_off = obj[i][e]['username_cm_off'];
var nonadmin_username_cm_off = obj[i][e]['nonadmin_username_cm_off'];
return {
username_cm_on: username_cm_on,
nonadmin_username_cm_on: nonadmin_username_cm_on,
username_cm_off: username_cm_off,
nonadmin_username_cm_off: nonadmin_username_cm_off
}
}
}
}
var config = {
env: env,
data: getData(),
}
return config;
}
I tried several ways to load the env-data.json file into karate-config.js as below
var envData = call read('classpath:env_data.json');
I know the above is not valid from this stackover flow answer Karate - How to import json data by Peter Thomas
So,tried with the below ones
var envData = read('classpath:env_data.json');
var envData = require("./env_data.json");
var envData = require('./env_data.json');
But, still facing issues with reading the json file. Appreciate help on this.
I think you over-complicated your JSON. You just need one object and no top-level array. Just use this as env_data.json:
{
"qa":{
"username_cm_on":"test_cm_on_qa",
"password_cm_on":"Test123$",
"nonadmin_username_cm_on":"test_non_admin_cm_on_qa",
"nonadmin_password_cm_on":"Test123$",
"username_cm_off":"test_cm_off_qa",
"password_cm_off":"Test123$",
"nonadmin_username_cm_off":"test_non_admin_cm_off_qa",
"nonadmin_password_cm_off":"Test123$",
"zuul_urls":{
"home-sec-uri":"https://qa.abc.com/qa/home-sec-uri",
"home-res-uri":"https://qa.abc.com/qa/home-res-uri"
}
},
"uat":{
"username_cm_on":"test_cm_on_uat",
"password_cm_on":"Test123$",
"nonadmin_username_cm_on":"test_non_admin_cm_on_uat",
"nonadmin_password_cm_on":"Test123$",
"username_cm_off":"test_cm_off_uat",
"password_cm_off":"Test123$",
"nonadmin_username_cm_off":"test_non_admin_cm_off_uat",
"nonadmin_password_cm_off":"Test123$",
"zuul_urls":{
"home-sec-uri":"https://uat.abc.com/qa/home-sec-uri",
"home-res-uri":"https://uat.abc.com/qa/home-res-uri"
}
}
}
And then this karate-config.js will work:
function() {
var env = 'qa'; // karate.env
var temp = read('classpath:env_data.json');
return temp[env];
}
And your tests can be more readable:
Given url zuul_urls['home-sec-uri']
If you have trouble understanding how this works, refer to this answer: https://stackoverflow.com/a/59162760/143475
I have a node/express server and I'm trying to get a function from a helper file to my app.js for use. Here is the function in the helper file:
CC.CURRENT.unpack = function(value)
{
var valuesArray = value.split("~");
var valuesArrayLenght = valuesArray.length;
var mask = valuesArray[valuesArrayLenght-1];
var maskInt = parseInt(mask,16);
var unpackedCurrent = {};
var currentField = 0;
for(var property in this.FIELDS)
{
if(this.FIELDS[property] === 0)
{
unpackedCurrent[property] = valuesArray[currentField];
currentField++;
}
else if(maskInt&this.FIELDS[property])
{
//i know this is a hack, for cccagg, future code please don't hate me:(, i did this to avoid
//subscribing to trades as well in order to show the last market
if(property === 'LASTMARKET'){
unpackedCurrent[property] = valuesArray[currentField];
}else{
unpackedCurrent[property] = parseFloat(valuesArray[currentField]);
}
currentField++;
}
}
return unpackedCurrent;
};
At the bottom of that helper file I did a module.export (The helper file is 400 lines long and I don't want to export every function in it):
module.exports = {
unpackMessage: function(value) {
CCC.CURRENT.unpack(value);
}
}
Then in my app.js I called
var helperUtil = require('./helpers/ccc-streamer-utilities.js');
and finally, I called that function in app.js and console.log it:
res = helperUtil.unpackMessage(message);
console.log(res);
The problem is that the console.log gives off an undefined every time, but in this example: https://github.com/cryptoqween/cryptoqween.github.io/tree/master/streamer/current (which is not node.js) it works perfectly. So I think I am importing wrong. All I want to do is use that utility function in my app.js
The unPackMessage(val) call doesn't return anything:
module.exports = {
unpackMessage: function(value) {
CCC.CURRENT.unpack(value);
}
}
you need to return CCC.CURRENT.UNPACK(value);
module.exports = {
unpackMessage: function(value) {
return CCC.CURRENT.unpack(value);
}
}
I have a Node.js module which exports two functions init(data) , where data is Buffer ,and test(word) , where word is a string.
I would like to read lines from data Buffer instance line by line inside test() function .
I have no experience in Node.js, only JS. All I come to know from this stack is how to export multiple function from Node.js module.
Here is function declaration so far:
module.exports = {
init: function(data) {
},
test: function(word) {
}
}
according to your comment, data is instanceof Buffer, and it contains a dictionary with one english word per line. So, now you can convert data to array of string, splitting by new line characters. with module format:
module.exports.init = function (data) {
if (!(data instanceof Buffer)) {
throw new Error('not a instanceof Buffer');
}
this.currentData = data.toString().split(/(?:\r\n|\r|\n)/g);
};
module.exports.test = function (word) {
// for example
var yourTestMethod = function (lineNumber, lineContent, testWord) {
return true;
};
if (this.currentData && this.currentData.length) {
for (var line = 0; line < this.currentData.length; line++) {
if (yourTestMethod(line, this.currentData[line], word)) {
return true;
}
}
}
return false;
};
if you save this code as testModule.js, you can use this module in main code like:
// load module
var testModule = require('./testModule.js');
// init
var buf = new Buffer(/* load dictionaly */);
testModule.init(buf);
// test
console.log(testModule.test('foo'));
I think it is more simple. thanks.
(old answer)
I think you can use readline module.
But readline accepts a stream, not a buffer.
So it needs to convert. for example.
var readline = require('readline');
var stream = require('stream');
// string to buffer
var baseText = 'this is a sample text\n(empty lines ...)\n\n\n\nend line:)';
var buf = new Buffer(baseText);
// http://stackoverflow.com/questions/16038705/how-to-wrap-a-buffer-as-a-stream2-readable-stream
var bufferStream = new stream.PassThrough();
bufferStream.end(buf);
var rl = readline.createInterface({
input: bufferStream,
});
var count = 0;
rl.on('line', function (line) {
console.log('this is ' + (++count) + ' line, content = ' + line);
});
then output is:
> node test.js
this is 1 line, content = this is a sample text
this is 2 line, content = (empty lines ...)
this is 3 line, content =
this is 4 line, content =
this is 5 line, content =
this is 6 line, content = end line:)
how is that?