csvtojson conversion using Nodejs gives gibberish data - javascript

I am trying to convert a csv file to json using csvtojson converter in nodejs.My code is as shown below.
I get an output of data as shown below. I am not sure why this is happening and how to prevent it.
var Converter = require("csvtojson").Converter;
var fileStream = fs.createReadStream("input.csv");
var converter = new Converter({constructResult:false});
converter.on("end_parsed", function (jsonObj) {
var jsonfile = require('jsonfile');
var file = 'output.json';
jsonfile.writeFile(file, jsonObj, function (err) { console.error(err); });});
fileStream.pipe(converter);
{"��P\u0000a\u0000c\u0000k\u0000a\u0000g\u0000e\u0000 \u0000N\u0000a\u0000m\u0000e\u0000":"\u0000c\u0000o\u0000m\u0000.\u0000t\u0000r\u0000i\u0000n\u0000e\u0000t\u0000.\u0000h\u0000r\u0000p\u0000m\u0000o\u0000b\u0000i\u0000l\u0000e\u0000","\u0000A\u0000p\u0000p\u0000 \u0000V\u0000e\u0000r\u0000s\u0000i\u0000o\u0000n\u0000 \u0000C\u0000o\u0000d\u0000e\u0000":"\u00006\u00002\u0000","\u0000R\u0000e\u0000v\u0000i\u0000e\u0000w\u0000e\u0000r\u0000 \u0000L\u0000a\u0000n\u0000g\u0000u\u0000a\u0000g\u0000e\u0000":"\u0000e\u0000n\u0000","\u0000D\u0000e\u0000v\u0000i\u0000c\u0000e\u0000":"\u0000","\u0000R\u0000e\u0000v\u0000i\u0000e\u0000w\u0000 \u0000S\u0000u\u0000b\u0000m\u0000i\u0000t\u0000 \u0000D\u0000a\u0000t\u0000e\u0000 \u0000a\u0000n\u0000d\u0000 \u0000T\u0000i\u0000m\u0000e\u0000":"\u00002\u00000\u00001\u00005\u0000-\u00001\u00002\u0000-\u00002\u00002\u0000T\u00000\u00003\u0000:\u00003\u00002\u0000:\u00003\u00008\u0000Z\u0000","\u0000R\u0000e\u0000v\u0000i\u0000e\u0000w\u0000 \u0000S\u0000u\u0000b\u0000m\u0000i\u0000t\u0000 \u0000M\u0000i\u0000l\u0000l\u0000i\u0000s\u0000 \u0000S\u0000i\u0000n\u0000c\u0000e\u0000 \u0000E\u0000p\u0000o\u0000c\u0000h\u0000":"\u00001\u00004\u00005\u00000\u00007\u00005\u00005\u00001\u00005\u00008\u00002\u00006\u00002\u0000","\u0000R\u0000e\u0000v\u0000i\u0000e\u0000w\u0000 \u0000L\u0000a\u0000s\u0000t\u0000 \u0000U\u0000p\u0000d\u0000a\u0000t\u0000e\u0000 \u0000D\u0000a\u0000t\u0000e\u0000 \u0000a\u0000n\u0000d\u0000 \u0000T\u0000i\u0000m\u0000e\u0000":"\u00002\u00000\u00001\u00005\u0000-\u00001\u00002\u0000-\u00002\u00002\u0000T\u00000\u00003\u0000:\u00003\u00002\u0000:\u00003\u00008\u0000Z\u0000","\u0000R\u0000e\u0000v\u0000i\u0000e\u0000w\u0000 \u0000L\u0000a\u0000s\u0000t\u0000 \u0000U\u0000p\u0000d\u0000a\u0000t\u0000e\u0000 \u0000M\u0000i\u0000l\u0000l\u0000i\u0000s\u0000 \u0000S\u0000i\u0000n\u0000c\u0000e\u0000 \u0000E\u0000p\u0000o\u0000c\u0000h\u0000":"\u00001\u00004\u00005\u00000\u00007\u00005\u00005\u00001\u00005\u00008\u00002\u00006\u00002\u0000","\u0000S\u0000t\u0000a\u0000r\u0000 \u0000R\u0000a\u0000t\u0000i\u0000n\u0000g\u0000":"\u00005\u0000","\u0000R\u0000e\u0000v\u0000i\u0000e\u0000w\u0000 \u0000T\u0000i\u0000t\u0000l\u0000e\u0000":"\u0000","\u0000R\u0000e\u0000v\u0000i\u0000e\u0000w\u0000 \u0000T\u0000e\u0000x\u0000t\u0000":"\u0000","\u0000D\u0000e\u0000v\u0000e\u0000l\u0000o\u0000p\u0000e\u0000r\u0000 \u0000R\u0000e\u0000p\u0000l\u0000y\u0000 \u0000D\u0000a\u0000t\u0000e\u0000 \u0000a\u0000n\u0000d\u0000 \u0000T\u0000i\u0000m\u0000e\u0000":"\u0000","\u0000D\u0000e\u0000v\u0000e\u0000l\u0000o\u0000p\u0000e\u0000r\u0000 \u0000R\u0000e\u0000p\u0000l\u0000y\u0000 \u0000M\u0000i\u0000l\u0000l\u0000i\u0000s\u0000 \u0000S\u0000i\u0000n\u0000c\u0000e\u0000 \u0000E\u0000p\u0000o\u0000c\u0000h\u0000":"\u0000","\u0000D\u0000e\u0000v\u0000e\u0000l\u0000o\u0000p\u0000e\u0000r\u0000 \u0000R\u0000e\u0000p\u0000l\u0000y\u0000 \u0000T\u0000e\u0000x\u0000t\u0000":"\u0000","\u0000R\u0000e\u0000v\u0000i\u0000e\u0000w\u0000 \u0000L\u0000i\u0000n\u0000k\u0000":"\u0000"},
{"��P\u0000a\u0000c\u0000k\u0000a\u0000g\u0000e\u0000 \u0000N\u0000a\u0000m\u0000e\u0000":"\u0000"}

I resolved this issue: It was an encoding issue: correct code was "utf16 "
var csvEncoding = { encoding: 'utf16le' };
var csvString = fs.readFileSync(csvfile, csvEncoding).toString();
converter.fromString(csvString, function(err,result){
//your code here console.log(err);
console.log(result);
});

I had a very similar issue to OP but was using csv-parse with a file coming out of S3.
Thanks to OP I got on the right path with an encoding issue, I was able to resolve my issue by using utf16le in my stream coupled with iconv-lite like so:
s3
.getObject(getObjectParams)
.createReadStream()
.on('end', () => cb(null))
.pipe(iconv.decodeStream('utf16le'))
.pipe(parse({ delimiter: '\t', columns: true }))
.pipe(transformer);
Hopefully this helps others in the same boat!

var Converter = require("csvtojson").Converter;
var fs = require('fs');
var fileStream = fs.createReadStream("input.csv");
var converter = new Converter({constructResult:true});
converter.on("end_parsed", function (jsonObj) {
var jsonfile = require('jsonfile');
var file = 'output.json';
console.log(jsonObj);
jsonfile.writeFile(file, jsonObj, function (err,result) {
console.error(err);
console.log(result) ;
});
});
fileStream.pipe(converter);

Related

Save js variable data in txt

I have the following code and I need to save allProducts in a .txt file with json format. How can I achieve that?
var fruit = [{"name: x","id: y",..}, {...},...]
var driedFruitsNut = [{"name: x","id: y",..}, {...},...]
fruit.forEach((item) =>{
item.category = "Fruits & Vegetables";
item.subtCategory = "Fruit";
});
driedFruitsNut.forEach((item) => {
item.category = "Fruits & Vegetables";
item.subtCategory = "Dried Fruits & Nuts";
});
var allProducts = fruit.concat(driedFruitsNut);
I tried something like this; but I have no idea if this is even a correct code:
var fs = require('fs');
fs.writeFile("test.txt", allProducts, function(err) {
if (err) {
console.log(err);
}
});
but when I run node fetchFruitVeg.js in my terminal. I get a text file looking like this:
You can try something like this:
var fs = require('fs');
fs.writeFile("test.txt", JSON.stringify(allProducts), function(err) {
if (err) {
console.log(err);
}
});
since allProducts is a javascript object, first you need to convert it into normal string data to write it in a .txt file or .json file

Write CSV file column title in javascript

This is my code. I am trying to output my data as CSV and I am being successful but I can't add the title of the columns. Although I saved the CSV file but the column titles are missing. Can anyone help in this?
var d= new Date();
var t=d.getHours()+":"+d.getMinutes()+":"+d.getSeconds();
var data = dataAWS
var result = data.map(function(val) {
return t+'","'+val.payload.test_data;
}).join('"\n"');
var csvData = '"'+ result +'"';
var fs = require('fs');
fs.writeFile("tempN.csv", csvData, 'utf8', function (err) {
if (err) {
console.log("An error occured while saving CSV.");
return console.log(err);
}
console.log("CSV file has been saved.");
});
Without knowing more I would suggest something like:
data.unshift(column_headers);
var result = data.map(function(val) {
return t+'","'+val.payload.test_data;
}).join("\n")
etc, etc

how to write the xpath (output) into a json file

I have xpath example :element(by.xpath("xxx")) which will get me an output that needs to write into json file ,can someone help me on how to proceed in protractor.
var fs = require('fs'); // node.js file system
var filename = 'object.json';
element(by.xpath('xxx')).getText().then(function(text) {
if (text) {
var textAsJSON = JSON.parse(text);
return fs.writeFileSync(fileName, textAsJSON);
// or return fs.appendFileSync(filename, text);
}
});

MongoDB : Read a file stored using GridFS from MongoDB in Javascript(using Node.JS)

In a mongodb from javascript code I am able to insert a file(of any extension) using gridfs, It's stored as fs.chunks & fs.files. Now i want to read this file's stream, but on any of the read opration calls on gridStore its giving error : "this.data.buffer.slice is not a function". I have used mongojs library from Node.JS.
// Write a file into MongoDB - Working fine.
// Our file ID
var fileId = new ObjectID();
// Open a new file
var gridStore;
// Create a file and open it
gridStore = new GridStore(db, fileId, "D:/Files/Sofa 2 N050210.3DS", "w+");
var data = fs.readFileSync('D:/Files/Sofa 2 N050210.3DS');
gridStore.open(function (err, gridStore) {
gridStore.write(data, function (err, gridStore) {
// Flush the file to GridFS
gridStore.close(function (err, fileData) {
// Write successfull
});
});
});
Now problem is when reading, After Closing gridStore, I tried to open file for reading & it is giving error at read call i.e on line no-4 in below code.
gridStore.close(function (result) {
var gridStore = new GridStore(db, "test_gs_seek_with_buffer", "r");
gridStore.open(function (err, gridStore) {
gridStore.read(5, function (err, data) {
if(err){
console.log("Error"); return;
}
console.log(data.toString());
});
});
});
Please help me find solution or the way to read back the file stored in GridFS (From the javascript code not from the command prompt).
This code is working fine for me. Try with this. Hope it works.
var mongoose = require('mongoose');
var Grid = require('gridfs-stream');
mongoose.connection.once('open', function () {
gfs = new Grid(mongoose.connection.db, mongoose.mongo);//If you are using mongoose
});
var db = new mongo.Db('yourDatabaseName', new mongo.Server("127.0.0.1", 27017)); //if you are using mongoDb directily
var gfs = Grid(db, mongo);
var rstream = gfs.createReadStream(filename);
var bufs = [];
rstream.on('data', function (chunk) {
bufs.push(chunk);
}).on('error', function () {
res.send();
})
.on('end', function () { // done
var fbuf = Buffer.concat(bufs);
var File = (fbuf.toString('base64'));
res.send(File);
});

Node.js node-csv module - working with a local CSV file

I'm trying to use the new version of the node-csv node module to do some CSV manipulation.
node-csv
I've used perl in the past, but would like to try JavaScript this time. I'm having trouble figuring out how to import a local CSV file instead of using the built in generator. The documentation for node-csv doesn't show how to do this as far as I can tell(although it does provide an example for the previous version).
Here is the example code, which works as expected.
var csv = require('csv');
var generator = csv.generate({seed: 1, columns: 2, length: 20});
var parser = csv.parse();
var transformer = csv.transform(function(data){
return data.map(function(value){return value.toUpperCase()});
});
var stringifier = csv.stringify();
generator.on('readable', function(){
while(data = generator.read()){
parser.write(data);
}
});
parser.on('readable', function(){
while(data = parser.read()){
transformer.write(data);
}
});
transformer.on('readable', function(){
while(data = transformer.read()){
stringifier.write(data);
}
});
stringifier.on('readable', function(){
while(data = stringifier.read()){
process.stdout.write(data);
}
});
I plan on using the FS module, but am not sure how to pass the local file into the node-csv functions.
var fs = require('fs');
Here is an example for the PREVIOUS version, which uses completely different syntax:
// node samples/sample.js
var csv = require('csv');
var fs = require('fs');
csv()
.from.stream(fs.createReadStream(__dirname+'/sample.in'))
.to.path(__dirname+'/sample.out')
.transform( function(row){
row.unshift(row.pop());
return row;
})
.on('record', function(row,index){
console.log('#'+index+' '+JSON.stringify(row));
})
.on('end', function(count){
console.log('Number of lines: '+count);
})
.on('error', function(error){
console.log(error.message);
});
Any suggestions?

Categories

Resources