How to convert CSV to JSON in Node.js - javascript

I am trying to convert csv file to json. I am using .
Example CSV:
a,b,c,d
1,2,3,4
5,6,7,8
...
Desired JSON:
{"a": 1,"b": 2,"c": 3,"d": 4},
{"a": 5,"b": 6,"c": 7,"d": 8},
...
I tried node-csv parser library.But the output is like array not like I expected.
I'm using Node 0.8 and express.js and would like a recommendation on how to easily accomplish this.

Node.js csvtojson module is a comprehensive nodejs csv parser. It can be used as node.js app library / a command line tool / or browser with help of browserify or webpack.
the source code can be found at: https://github.com/Keyang/node-csvtojson
It is fast with low memory consumption yet powerful to support any of parsing needs with abundant API and easy to read documentation.
The detailed documentation can be found here
Here are some code examples:
Use it as a library in your Node.js application (csvtojson#2.0.0 +):
Install it through npm
npm install --save csvtojson#latest
Use it in your node.js app:
// require csvtojson
var csv = require("csvtojson");
// Convert a csv file with csvtojson
csv()
.fromFile(csvFilePath)
.then(function(jsonArrayObj){ //when parse finished, result will be emitted here.
console.log(jsonArrayObj);
})
// Parse large csv with stream / pipe (low mem consumption)
csv()
.fromStream(readableStream)
.subscribe(function(jsonObj){ //single json object will be emitted for each csv line
// parse each json asynchronousely
return new Promise(function(resolve,reject){
asyncStoreToDb(json,function(){resolve()})
})
})
//Use async / await
const jsonArray=await csv().fromFile(filePath);
Use it as a command-line tool:
sh# npm install csvtojson
sh# ./node_modules/csvtojson/bin/csvtojson ./youCsvFile.csv
-or-
sh# npm install -g csvtojson
sh# csvtojson ./yourCsvFile.csv
For advanced usage:
sh# csvtojson --help
You can find more details from the github page above.

You can try to use underscore.js
First convert the lines in arrays using the toArray function :
var letters = _.toArray(a,b,c,d);
var numbers = _.toArray(1,2,3,4);
Then object the arrays together using the object function :
var json = _.object(letters, numbers);
By then, the json var should contain something like :
{"a": 1,"b": 2,"c": 3,"d": 4}

Had to do something similar, hope this helps.
// Node packages for file system
var fs = require('fs');
var path = require('path');
var filePath = path.join(__dirname, 'PATH_TO_CSV');
// Read CSV
var f = fs.readFileSync(filePath, {encoding: 'utf-8'},
function(err){console.log(err);});
// Split on row
f = f.split("\n");
// Get first row for column headers
headers = f.shift().split(",");
var json = [];
f.forEach(function(d){
// Loop through each row
tmp = {}
row = d.split(",")
for(var i = 0; i < headers.length; i++){
tmp[headers[i]] = row[i];
}
// Add object to list
json.push(tmp);
});
var outPath = path.join(__dirname, 'PATH_TO_JSON');
// Convert object to string, write json to file
fs.writeFileSync(outPath, JSON.stringify(json), 'utf8',
function(err){console.log(err);});

Here is a solution that does not require a separate module. However, it is very crude, and does not implement much error handling. It could also use more tests, but it will get you going. If you are parsing very large files, you may want to seek an alternative. Also, see this solution from Ben Nadel.
Node Module Code, csv2json.js:
/*
* Convert a CSV String to JSON
*/
exports.convert = function(csvString) {
var json = [];
var csvArray = csvString.split("\n");
// Remove the column names from csvArray into csvColumns.
// Also replace single quote with double quote (JSON needs double).
var csvColumns = JSON
.parse("[" + csvArray.shift().replace(/'/g, '"') + "]");
csvArray.forEach(function(csvRowString) {
var csvRow = csvRowString.split(",");
// Here we work on a single row.
// Create an object with all of the csvColumns as keys.
jsonRow = new Object();
for ( var colNum = 0; colNum < csvRow.length; colNum++) {
// Remove beginning and ending quotes since stringify will add them.
var colData = csvRow[colNum].replace(/^['"]|['"]$/g, "");
jsonRow[csvColumns[colNum]] = colData;
}
json.push(jsonRow);
});
return JSON.stringify(json);
};
Jasmine Test, csv2jsonSpec.js:
var csv2json = require('csv2json.js');
var CSV_STRING = "'col1','col2','col3'\n'1','2','3'\n'4','5','6'";
var JSON_STRING = '[{"col1":"1","col2":"2","col3":"3"},{"col1":"4","col2":"5","col3":"6"}]';
/* jasmine specs for csv2json */
describe('csv2json', function() {
it('should convert a csv string to a json string.', function() {
expect(csv2json.convert(CSV_STRING)).toEqual(
JSON_STRING);
});
});

If you want just a command line converter, the quickest and most clean solution for me is to use csvtojson via npx (included by default in node.js)
$ npx csvtojson ./data.csv > data.json

Using ES6
const toJSON = csv => {
const lines = csv.split('\n')
const result = []
const headers = lines[0].split(',')
lines.map(l => {
const obj = {}
const line = l.split(',')
headers.map((h, i) => {
obj[h] = line[i]
})
result.push(obj)
})
return JSON.stringify(result)
}
const csv = `name,email,age
francis,francis#gmail.com,33
matty,mm#gmail.com,29`
const data = toJSON(csv)
console.log(data)
Output
// [{"name":"name","email":"email","age":"age"},{"name":"francis","email":"francis#gmail.com","age":"33"},{"name":"matty","email":"mm#gmail.com","age":"29"}]

Using lodash:
function csvToJson(csv) {
const content = csv.split('\n');
const header = content[0].split(',');
return _.tail(content).map((row) => {
return _.zipObject(header, row.split(','));
});
}

I haven't tried csv package https://npmjs.org/package/csv but according to documentation it looks quality implementation http://www.adaltas.com/projects/node-csv/

I started with node-csvtojson, but it brought too many dependencies for my linking.
Building on your question and the answer by brnd, I used node-csv and underscore.js.
var attribs;
var json:
csv()
.from.string(csvString)
.transform(function(row) {
if (!attribs) {
attribs = row;
return null;
}
return row;
})
.to.array(function(rows) {
json = _.map(rows, function(row) {
return _.object(attribs, row);
});
});

I have a very simple solution to just print json from csv on console using csvtojson module.
// require csvtojson
var csv = require("csvtojson");
const csvFilePath='customer-data.csv' //file path of csv
csv()
.fromFile(csvFilePath)``
.then((jsonObj)=>{
console.log(jsonObj);
})

I have used csvtojson library for converting csv string to json array.
It has variety of function which can help you to convert to JSON.
It also supports reading from file and file streaming.
Be careful while parsing the csv which can contain the comma(,) or any other delimiter .
For removing the delimiter please see my answer here.

Step 1:
Install node module:
npm install csvtojson --save
Step 2:
var Converter = require("csvtojson").Converter;
var converter = new Converter({});
converter.fromFile("./path-to-your-file.csv",function(err,result){
if(err){
console.log("Error");
console.log(err);
}
var data = result;
//to check json
console.log(data);
});

Node-ETL package is enough for all BI processing.
npm install node-etl;
Then :
var ETL=require('node-etl');
var output=ETL.extract('./data.csv',{
headers:["a","b","c","d"],
ignore:(line,index)=>index!==0, //ignore first line
});

Me and my buddy created a web service to handle this kind of thing.
Check out Modifly.co for instructions on how to transform CSV to JSON with a single RESTful call.

Use csv parser library, I'm explaining in more details how to use it here .
var csv = require('csv');
csv.parse(csvText, {columns: true}, function(err, data){
console.log(JSON.stringify(data, null, 2));
});

npm install csvjson --save
In you Node JS File
const csvjson = require('csvjson');
convertCSVToJSON(*.csv);
convertCSVToJSON = (file) => {
const convertedObj = csvjson.toObject(file);
}

csvtojson module is a comprehensive nodejs csv parser to convert csv to json or column arrays. It can be used as node.js library / command line tool / or in browser. Below are some features:
/** csv file
a,b,c
1,2,3
4,5,6
*/
const csvFilePath='<path to csv file>'
const csv=require('csvtojson')
csv()
.fromFile(csvFilePath)
.then((jsonObj)=>{
console.log(jsonObj);
/**
* [
* {a:"1", b:"2", c:"3"},
* {a:"4", b:"5". c:"6"}
* ]
*/
})
// Async / await usage
const jsonArray=await csv().fromFile(csvFilePath);

I converted a large (315 MB) csv file to json by installing the csvtojson module and then using the below code:
const fs = require('fs')
const Converter = require('csvtojson').Converter
const csvConverter = new Converter({
constructResult:false,
downstreamFormat:"array",
})
csvConverter.subscribe=function(json,row,index){
json["rowIndex"]=index
};
const readStream = fs.createReadStream('./data.csv') // my csv file
const writeStream = fs.createWriteStream('./data.json') // my new json file
readStream.pipe(csvConverter).pipe(writeStream)
The resulting json file is in the desired format:
[
{"a": 1,"b": 2,"c": 3,"d": 4},
{"a": 5,"b": 6,"c": 7,"d": 8},
]

Once figured out how to csv data into two dimention array:
[['header1','header2'],['data1','data2']]
Convert to json is simply map and reduce:
const keys = input[0]
const jsonOutput = input.slice(1)
.map(arr2 => keys.reduce((accumulator, element, index) => {
return { ...accumulator,
[element]: arr2[index]
};
}, {}))

In my case JSON.stringify didn't help as the files where too big.
This solved my needs:
let csvFile = fs.readFileSync(
csvFilePath,
{ encoding: "utf-8" },
function (err) {
console.log(err);
}
);
csvFile = csvFile.split("\n");
let strFile = "export default [";
csvFile.forEach(function (d) {
let row = d.split(",");
strFile += `[${row}],`;
});
strFile += "]";

Related

How can I parse a YAML file to read the comments as well using node.js?

I am trying to parse a YAML file. I was able to parse the file properly but the comments in the YAML file are not getting read. Is there any way to do it? Attaching the parser code and config.json. Also attaching the screenshot of the file and output for reference.
var fs= require('fs');
var path= require('path');
var yaml = require('js-yaml')
var fname= "config.json"
var jPath= path.join(__dirname,"..","ConfigGen","Config",fname);
var jsString= fs.readFileSync(jPath, 'utf8')
// Get path for files from Config file
var tType= "cto" //Get this from input
var pth= JSON.parse(jsString)[tType] //perform error handling
var cType = "jbod" //Get this from input
//Use that path
fs.readdir(pth, function(err,files) {
files.forEach(function(file){
fName= cType+"_"+tType+"_uut.yaml-example";
if(file==fName){
var flContent= fs.readFileSync(path.join(pth,file),"utf8")
// return path.join from here and use the next part in a separate function
var data= yaml.safeLoad(flContent)[0][0]
console.log(data)
for (var index in data){
var prefix = index
for (idx in data[index]){
//console.log(prefix, idx ,data[prefix][idx])
}
}
}
})
})
Reiterating flyx's comment, according to the YAML spec on comments:
Comments are a presentation detail and must not be used to convey content information.
So assuming you're not going to be able to correlate the comments to any adjacent fields, you can just read the whole file as a string and match against any characters after a #
You can read the file and parse with this regex like this:
var { promises: fs } = require('fs');
(async() => {
let path = "./config.yaml"
let file = await fs.readFile(path, "utf8")
let matches = file.matchAll(/#.*/g)
let comments = [...matches].map(m => m[0])
console.log(comments)
})()
If you have a yaml file that looks like this:
# block comment
env: dev
prop: value # inline comment
It will log the following:
[ '# block comment', '# inline comment' ]

How to get excel sheet names in JavaScript

I am trying to get the list of the sheets from an excel file in order to display it in a combobox. The idea is that the user can select the file that he want to import from an excel file instead of typing and avoid typo errors
function getSheets(file)
{
var reader;
reader = new FileReader();
reader.onload = function(e)
{
var data;
var workbook;
var sheets;
data = e.target.result();
workbook = XLSX.read(data, {type: 'binary'});
sheet = workbook.SheetNames;
for (var i in sheets.items)
{
console.log(sheets.items[i].name);
}
}
reader.onerror = function(ex)
{
console.log(ex);
}
reader.readAsBinaryString(file);
}
The following error is displayed when the function is executed:
Function expected on line: data = e.target.result();
Any idea of how should I face it?
Detail explaination can be found Here
sheet.forEach(function (y) { /*Iterate through all sheets*/
/*Convert the cell value to Json*/
var exceljson = XLSX.utils.sheet_to_json(workbook.Sheets[y]);
});
In NodeJS you can use the Node XLSX (NodeJS excel file parser & builder).
There is a parse method that reads the Excel file and gives you an array of all it's sheets.
Then you could loop the array and get the name of each sheet and all the data.
To Install the package use:
npm install node-xlsx --save
This code will print all the sheet names in an Excel file:
var xlsx = require('node-xlsx');
const worksheetsArray = xlsx.parse('example.xlsx'); // parses a file
worksheetsArray.forEach(sheet => {
console.log(sheet.name);
})
I like to use the following lib: npm install xlsx
const xlsx = require('xlsx')
const workbook = xlsx.readFile(file); //Here you should pass the file path
const sheetList = workbook.SheetNames; //Array of sheet names.
console.log(sheetList)

How can I render a static HTML file with Handlebars on a Nodejs server?

I have come across plenty of resources online for this but haven't been able to find one that is straight forward enough for me to understand.
At the moment, I have multiple massive <script> tags in an HTML document that has handlebars content. The server sends this HTML document to the client where the client then renders the page with data from an AJAX call. I'd like to move this entire process server-side so that all the server has to do is send a static file and re-render the page when data is updated. Data changes a few times per day - which is why it isn't hard coded in and I would like to run the handlebars compiler on the HTML document when data is updated.
Is it possible to simply put the HTML document with handlebars templating in <script> tags through a function to generate a new HTML file with data filled in?
Here is the code I have within my app.js file that is runned the Node server that does not do what I want it to:
function registerHelpers(callback){
Handlebars.registerHelper('equal', function(lvalue, rvalue, options) {
if (arguments.length < 3)
throw new Error("Handlebars Helper equal needs 2 parameters");
if( lvalue!=rvalue ) {
return options.inverse(this);
} else {
return options.fn(this);
}
});
Handlebars.registerHelper('trim', function(text) {
text = text.replace(/ /g, '');
return new Handlebars.SafeString(text);
});
callback();
}
function buildHomePage() {
var source = require(__dirname + '/public/home.handlebars');
var template = Handlebars.precompile(source);
var collection = db.get('datalist'); //Monk call to MongoDB
collection.find({}, function (err, docs){
var result = template(docs);
console.log(result)
var fs = require('fs');
fs.writeFile("test.html", result, function(err) {
if(err) {
console.log(err);
}
});
});
};
registerHelpers(buildHomePage);
The following can render handlebars to static html. Run node example.js. You may need to run npm install --save handlebars prior.
var fs = require('fs');
var Handlebars = require('handlebars');
function render(filename, data)
{
var source = fs.readFileSync(filename,'utf8').toString();
var template = Handlebars.compile(source);
var output = template(data);
return output;
}
var data = JSON.parse(fs.readFileSync("./data/strings.json", 'utf8'));
var result = render('./templates/somefile.html', data);
console.log(result);
If your handlebars templates are simple, with only string replacement, you can do this with underscore.js. Assume this example is named 'generate.js'
var fs = require('fs');
var _ = require('underscore');
_.templateSettings.interpolate = /\{\{(.+?)\}\}/g;
function render(filename, data)
{
var source = fs.readFileSync(filename,'utf8').toString();
var compiled = _.template(source);
return compiled(data);
}
var data = JSON.parse(fs.readFileSync("./data/strings.json", 'utf8'));
var result = render('./templates/somefile.html', data);
console.log(result);
Then run node generate.js to output the rendered template to the console. You may need to do npm install --save underscore prior.

Node.js node-csv module - working with a local CSV file

I'm trying to use the new version of the node-csv node module to do some CSV manipulation.
node-csv
I've used perl in the past, but would like to try JavaScript this time. I'm having trouble figuring out how to import a local CSV file instead of using the built in generator. The documentation for node-csv doesn't show how to do this as far as I can tell(although it does provide an example for the previous version).
Here is the example code, which works as expected.
var csv = require('csv');
var generator = csv.generate({seed: 1, columns: 2, length: 20});
var parser = csv.parse();
var transformer = csv.transform(function(data){
return data.map(function(value){return value.toUpperCase()});
});
var stringifier = csv.stringify();
generator.on('readable', function(){
while(data = generator.read()){
parser.write(data);
}
});
parser.on('readable', function(){
while(data = parser.read()){
transformer.write(data);
}
});
transformer.on('readable', function(){
while(data = transformer.read()){
stringifier.write(data);
}
});
stringifier.on('readable', function(){
while(data = stringifier.read()){
process.stdout.write(data);
}
});
I plan on using the FS module, but am not sure how to pass the local file into the node-csv functions.
var fs = require('fs');
Here is an example for the PREVIOUS version, which uses completely different syntax:
// node samples/sample.js
var csv = require('csv');
var fs = require('fs');
csv()
.from.stream(fs.createReadStream(__dirname+'/sample.in'))
.to.path(__dirname+'/sample.out')
.transform( function(row){
row.unshift(row.pop());
return row;
})
.on('record', function(row,index){
console.log('#'+index+' '+JSON.stringify(row));
})
.on('end', function(count){
console.log('Number of lines: '+count);
})
.on('error', function(error){
console.log(error.message);
});
Any suggestions?

How to read data From *.CSV file using JavaScript?

My CSV data looks like this:
heading1,heading2,heading3,heading4,heading5
value1_1,value2_1,value3_1,value4_1,value5_1
value1_2,value2_2,value3_2,value4_2,value5_2
...
How do you read this data and convert to an array like this using JavaScript?:
[
heading1: value1_1,
heading2: value2_1,
heading3: value3_1,
heading4: value4_1
heading5: value5_1
],[
heading1: value1_2,
heading2: value2_2,
heading3: value3_2,
heading4: value4_2,
heading5: value5_2
]
....
I've tried this code but no luck!:
<script type="text/javascript">
var allText =[];
var allTextLines = [];
var Lines = [];
var txtFile = new XMLHttpRequest();
txtFile.open("GET", "file://d:/data.txt", true);
txtFile.onreadystatechange = function()
{
allText = txtFile.responseText;
allTextLines = allText.split(/\r\n|\n/);
};
document.write(allTextLines);
document.write(allText);
document.write(txtFile);
</script>
No need to write your own...
The jQuery-CSV library has a function called $.csv.toObjects(csv) that does the mapping automatically.
Note: The library is designed to handle any CSV data that is RFC 4180 compliant, including all of the nasty edge cases that most 'simple' solutions overlook.
Like #Blazemonger already stated, first you need to add line breaks to make the data valid CSV.
Using the following dataset:
heading1,heading2,heading3,heading4,heading5
value1_1,value2_1,value3_1,value4_1,value5_1
value1_2,value2_2,value3_2,value4_2,value5_2
Use the code:
var data = $.csv.toObjects(csv):
The output saved in 'data' will be:
[
{ heading1:"value1_1",heading2:"value2_1",heading3:"value3_1",heading4:"value4_1",heading5:"value5_1" }
{ heading1:"value1_2",heading2:"value2_2",heading3:"value3_2",heading4:"value4_2",heading5:"value5_2" }
]
Note: Technically, the way you wrote the key-value mapping is invalid JavaScript. The objects containing the key-value pairs should be wrapped in brackets.
If you want to try it out for yourself, I suggest you take a look at the Basic Usage Demonstration under the 'toObjects()' tab.
Disclaimer: I'm the original author of jQuery-CSV.
Update:
Edited to use the dataset that the op provided and included a link to the demo where the data can be tested for validity.
Update2:
Due to the shuttering of Google Code. jquery-csv has moved to GitHub
NOTE: I concocted this solution before I was reminded about all the "special cases" that can occur in a valid CSV file, like escaped quotes. I'm leaving my answer for those who want something quick and dirty, but I recommend Evan's answer for accuracy.
This code will work when your data.txt file is one long string of comma-separated entries, with no newlines:
data.txt:
heading1,heading2,heading3,heading4,heading5,value1_1,...,value5_2
javascript:
$(document).ready(function() {
$.ajax({
type: "GET",
url: "data.txt",
dataType: "text",
success: function(data) {processData(data);}
});
});
function processData(allText) {
var record_num = 5; // or however many elements there are in each row
var allTextLines = allText.split(/\r\n|\n/);
var entries = allTextLines[0].split(',');
var lines = [];
var headings = entries.splice(0,record_num);
while (entries.length>0) {
var tarr = [];
for (var j=0; j<record_num; j++) {
tarr.push(headings[j]+":"+entries.shift());
}
lines.push(tarr);
}
// alert(lines);
}
The following code will work on a "true" CSV file with linebreaks between each set of records:
data.txt:
heading1,heading2,heading3,heading4,heading5
value1_1,value2_1,value3_1,value4_1,value5_1
value1_2,value2_2,value3_2,value4_2,value5_2
javascript:
$(document).ready(function() {
$.ajax({
type: "GET",
url: "data.txt",
dataType: "text",
success: function(data) {processData(data);}
});
});
function processData(allText) {
var allTextLines = allText.split(/\r\n|\n/);
var headers = allTextLines[0].split(',');
var lines = [];
for (var i=1; i<allTextLines.length; i++) {
var data = allTextLines[i].split(',');
if (data.length == headers.length) {
var tarr = [];
for (var j=0; j<headers.length; j++) {
tarr.push(headers[j]+":"+data[j]);
}
lines.push(tarr);
}
}
// alert(lines);
}
http://jsfiddle.net/mblase75/dcqxr/
Don't split on commas -- it won't work for most CSV files, and this question has wayyyy too many views for the asker's kind of input data to apply to everyone. Parsing CSV is kind of scary since there's no truly official standard, and lots of delimited text writers don't consider edge cases.
This question is old, but I believe there's a better solution now that Papa Parse is available. It's a library I wrote, with help from contributors, that parses CSV text or files. It's the only JS library I know of that supports files gigabytes in size. It also handles malformed input gracefully.
1 GB file parsed in 1 minute:
(Update: With Papa Parse 4, the same file took only about 30 seconds in Firefox. Papa Parse 4 is now the fastest known CSV parser for the browser.)
Parsing text is very easy:
var data = Papa.parse(csvString);
Parsing files is also easy:
Papa.parse(file, {
complete: function(results) {
console.log(results);
}
});
Streaming files is similar (here's an example that streams a remote file):
Papa.parse("http://example.com/bigfoo.csv", {
download: true,
step: function(row) {
console.log("Row:", row.data);
},
complete: function() {
console.log("All done!");
}
});
If your web page locks up during parsing, Papa can use web workers to keep your web site reactive.
Papa can auto-detect delimiters and match values up with header columns, if a header row is present. It can also turn numeric values into actual number types. It appropriately parses line breaks and quotes and other weird situations, and even handles malformed input as robustly as possible. I've drawn on inspiration from existing libraries to make Papa, so props to other JS implementations.
I am using d3.js for parsing csv file. Very easy to use.
Here is the docs.
Steps:
npm install d3-request
Using Es6;
import { csv } from 'd3-request';
import url from 'path/to/data.csv';
csv(url, function(err, data) {
console.log(data);
})
Please see docs for more.
Update -
d3-request is deprecated. you can use d3-fetch
Here's a JavaScript function that parses CSV data, accounting for commas found inside quotes.
// Parse a CSV row, accounting for commas inside quotes
function parse(row){
var insideQuote = false,
entries = [],
entry = [];
row.split('').forEach(function (character) {
if(character === '"') {
insideQuote = !insideQuote;
} else {
if(character == "," && !insideQuote) {
entries.push(entry.join(''));
entry = [];
} else {
entry.push(character);
}
}
});
entries.push(entry.join(''));
return entries;
}
Example use of the function to parse a CSV file that looks like this:
"foo, the column",bar
2,3
"4, the value",5
into arrays:
// csv could contain the content read from a csv file
var csv = '"foo, the column",bar\n2,3\n"4, the value",5',
// Split the input into lines
lines = csv.split('\n'),
// Extract column names from the first line
columnNamesLine = lines[0],
columnNames = parse(columnNamesLine),
// Extract data from subsequent lines
dataLines = lines.slice(1),
data = dataLines.map(parse);
// Prints ["foo, the column","bar"]
console.log(JSON.stringify(columnNames));
// Prints [["2","3"],["4, the value","5"]]
console.log(JSON.stringify(data));
Here's how you can transform the data into objects, like D3's csv parser (which is a solid third party solution):
var dataObjects = data.map(function (arr) {
var dataObject = {};
columnNames.forEach(function(columnName, i){
dataObject[columnName] = arr[i];
});
return dataObject;
});
// Prints [{"foo":"2","bar":"3"},{"foo":"4","bar":"5"}]
console.log(JSON.stringify(dataObjects));
Here's a working fiddle of this code.
Enjoy! --Curran
You can use PapaParse to help.
https://www.papaparse.com/
Here is a CodePen.
https://codepen.io/sandro-wiggers/pen/VxrxNJ
Papa.parse(e, {
header:true,
before: function(file, inputElem){ console.log('Attempting to Parse...')},
error: function(err, file, inputElem, reason){ console.log(err); },
complete: function(results, file){ $.PAYLOAD = results; }
});
If you want to solve this without using Ajax, use the FileReader() Web API.
Example implementation:
Select .csv file
See output
function readSingleFile(e) {
var file = e.target.files[0];
if (!file) {
return;
}
var reader = new FileReader();
reader.onload = function(e) {
var contents = e.target.result;
displayContents(contents);
displayParsed(contents);
};
reader.readAsText(file);
}
function displayContents(contents) {
var element = document.getElementById('file-content');
element.textContent = contents;
}
function displayParsed(contents) {
const element = document.getElementById('file-parsed');
const json = contents.split(',');
element.textContent = JSON.stringify(json);
}
document.getElementById('file-input').addEventListener('change', readSingleFile, false);
<input type="file" id="file-input" />
<h3>Raw contents of the file:</h3>
<pre id="file-content">No data yet.</pre>
<h3>Parsed file contents:</h3>
<pre id="file-parsed">No data yet.</pre>
function CSVParse(csvFile)
{
this.rows = [];
var fieldRegEx = new RegExp('(?:\s*"((?:""|[^"])*)"\s*|\s*((?:""|[^",\r\n])*(?:""|[^"\s,\r\n]))?\s*)(,|[\r\n]+|$)', "g");
var row = [];
var currMatch = null;
while (currMatch = fieldRegEx.exec(this.csvFile))
{
row.push([currMatch[1], currMatch[2]].join('')); // concatenate with potential nulls
if (currMatch[3] != ',')
{
this.rows.push(row);
row = [];
}
if (currMatch[3].length == 0)
break;
}
}
I like to have the regex do as much as possible. This regex treats all items as either quoted or unquoted, followed by either a column delimiter, or a row delimiter. Or the end of text.
Which is why that last condition -- without it it would be an infinite loop since the pattern can match a zero length field (totally valid in csv). But since $ is a zero length assertion, it won't progress to a non match and end the loop.
And FYI, I had to make the second alternative exclude quotes surrounding the value; seems like it was executing before the first alternative on my javascript engine and considering the quotes as part of the unquoted value. I won't ask -- just got it to work.
Per the accepted answer,
I got this to work by changing the 1 to a 0 here:
for (var i=1; i<allTextLines.length; i++) {
changed to
for (var i=0; i<allTextLines.length; i++) {
It will compute the a file with one continuous line as having an allTextLines.length of 1. So if the loop starts at 1 and runs as long as it's less than 1, it never runs. Hence the blank alert box.
$(function() {
$("#upload").bind("click", function() {
var regex = /^([a-zA-Z0-9\s_\\.\-:])+(.csv|.xlsx)$/;
if (regex.test($("#fileUpload").val().toLowerCase())) {
if (typeof(FileReader) != "undefined") {
var reader = new FileReader();
reader.onload = function(e) {
var customers = new Array();
var rows = e.target.result.split("\r\n");
for (var i = 0; i < rows.length - 1; i++) {
var cells = rows[i].split(",");
if (cells[0] == "" || cells[0] == undefined) {
var s = customers[customers.length - 1];
s.Ord.push(cells[2]);
} else {
var dt = customers.find(x => x.Number === cells[0]);
if (dt == undefined) {
if (cells.length > 1) {
var customer = {};
customer.Number = cells[0];
customer.Name = cells[1];
customer.Ord = new Array();
customer.Ord.push(cells[2]);
customer.Point_ID = cells[3];
customer.Point_Name = cells[4];
customer.Point_Type = cells[5];
customer.Set_ORD = cells[6];
customers.push(customer);
}
} else {
var dtt = dt;
dtt.Ord.push(cells[2]);
}
}
}
Actually you can use a light-weight library called any-text.
install dependencies
npm i -D any-text
use custom command to read files
var reader = require('any-text');
reader.getText(`path-to-file`).then(function (data) {
console.log(data);
});
or use async-await :
var reader = require('any-text');
const chai = require('chai');
const expect = chai.expect;
describe('file reader checks', () => {
it('check csv file content', async () => {
expect(
await reader.getText(`${process.cwd()}/test/files/dummy.csv`)
).to.contains('Lorem ipsum');
});
});
This is an old question and in 2022 there are many ways to achieve this. First, I think D3 is one of the best alternatives for data manipulation. It's open sourced and free to use, but also it's modular so we can import just the fetch module.
Here is a basic example. We will use the legacy mode so I will import the entire D3 library. Now, let's call d3.csv function and it's done. This function internally calls the fetch method therefore, it can open dataURL, url, files, blob, and so on.
const fileInput = document.getElementById('csv')
const outElement = document.getElementById('out')
const previewCSVData = async dataurl => {
const d = await d3.csv(dataurl)
console.log({
d
})
outElement.textContent = d.columns
}
const readFile = e => {
const file = fileInput.files[0]
const reader = new FileReader()
reader.onload = () => {
const dataUrl = reader.result;
previewCSVData(dataUrl)
}
reader.readAsDataURL(file)
}
fileInput.onchange = readFile
<script type="text/javascript" src="https://unpkg.com/d3#7.6.1/dist/d3.min.js"></script>
<div>
<p>Select local CSV File:</p>
<input id="csv" type="file" accept=".csv">
</div>
<pre id="out"><p>File headers will appear here</p></pre>
If we don't want to use any library and we just want to use pain JavaScrip (Vanilla JS) and we managed to get the text content of a file as data and we don't want to use d3 we can implement a simple function that will split the data into a text array then we will extract the first line and split into a headers array and the rest of the text will be the lines we will process. After, we map each line and extract its values and create a row object from an array created from mapping each header to its correspondent value from values[index].
NOTE:
We also we going to use a little trick array objects in JavaScript can also have attributes. Yes so we will define an attribute rows.headers and assign the headers to it.
const data = `heading_1,heading_2,heading_3,heading_4,heading_5
value_1_1,value_2_1,value_3_1,value_4_1,value_5_1
value_1_2,value_2_2,value_3_2,value_4_2,value_5_2
value_1_3,value_2_3,value_3_3,value_4_3,value_5_3`
const csvParser = data => {
const text = data.split(/\r\n|\n/)
const [first, ...lines] = text
const headers = first.split(',')
const rows = []
rows.headers = headers
lines.map(line => {
const values = line.split(',')
const row = Object.fromEntries(headers.map((header, i) => [header, values[i]]))
rows.push(row)
})
return rows
}
const d = csvParser(data)
// Accessing to the theaders attribute
const headers = d.headers
console.log({headers})
console.log({d})
Finally, let's implement a vanilla JS file loader using fetch and parsing the csv file.
const fetchFile = async dataURL => {
return await fetch(dataURL).then(response => response.text())
}
const csvParser = data => {
const text = data.split(/\r\n|\n/)
const [first, ...lines] = text
const headers = first.split(',')
const rows = []
rows.headers = headers
lines.map(line => {
const values = line.split(',')
const row = Object.fromEntries(headers.map((header, i) => [header, values[i]]))
rows.push(row)
})
return rows
}
const fileInput = document.getElementById('csv')
const outElement = document.getElementById('out')
const previewCSVData = async dataURL => {
const data = await fetchFile(dataURL)
const d = csvParser(data)
console.log({ d })
outElement.textContent = d.headers
}
const readFile = e => {
const file = fileInput.files[0]
const reader = new FileReader()
reader.onload = () => {
const dataURL = reader.result;
previewCSVData(dataURL)
}
reader.readAsDataURL(file)
}
fileInput.onchange = readFile
<script type="text/javascript" src="https://unpkg.com/d3#7.6.1/dist/d3.min.js"></script>
<div>
<p>Select local CSV File:</p>
<input id="csv" type="file" accept=".csv">
</div>
<pre id="out"><p>File contents will appear here</p></pre>
I used this file to test it
Here is another way to read an external CSV into Javascript (using jQuery).
It's a little bit more long winded, but I feel by reading the data into arrays you can exactly follow the process and makes for easy troubleshooting.
Might help someone else.
The data file example:
Time,data1,data2,data2
08/11/2015 07:30:16,602,0.009,321
And here is the code:
$(document).ready(function() {
// AJAX in the data file
$.ajax({
type: "GET",
url: "data.csv",
dataType: "text",
success: function(data) {processData(data);}
});
// Let's process the data from the data file
function processData(data) {
var lines = data.split(/\r\n|\n/);
//Set up the data arrays
var time = [];
var data1 = [];
var data2 = [];
var data3 = [];
var headings = lines[0].split(','); // Splice up the first row to get the headings
for (var j=1; j<lines.length; j++) {
var values = lines[j].split(','); // Split up the comma seperated values
// We read the key,1st, 2nd and 3rd rows
time.push(values[0]); // Read in as string
// Recommended to read in as float, since we'll be doing some operations on this later.
data1.push(parseFloat(values[1]));
data2.push(parseFloat(values[2]));
data3.push(parseFloat(values[3]));
}
// For display
var x= 0;
console.log(headings[0]+" : "+time[x]+headings[1]+" : "+data1[x]+headings[2]+" : "+data2[x]+headings[4]+" : "+data2[x]);
}
})
Hope this helps someone in the future!
A bit late but I hope it helps someone.
Some time ago even I faced a problem where the string data contained \n in between and while reading the file it used to read as different lines.
Eg.
"Harry\nPotter","21","Gryffindor"
While-Reading:
Harry
Potter,21,Gryffindor
I had used a library csvtojson in my angular project to solve this problem.
You can read the CSV file as a string using the following code and then pass that string to the csvtojson library and it will give you a list of JSON.
Sample Code:
const csv = require('csvtojson');
if (files && files.length > 0) {
const file: File = files.item(0);
const reader: FileReader = new FileReader();
reader.readAsText(file);
reader.onload = (e) => {
const csvs: string = reader.result as string;
csv({
output: "json",
noheader: false
}).fromString(csvs)
.preFileLine((fileLine, idx) => {
//Convert csv header row to lowercase before parse csv file to json
if (idx === 0) { return fileLine.toLowerCase() }
return fileLine;
})
.then((result) => {
// list of json in result
});
}
}
I use the jquery-csv to do this.
and I provide two examples as below
async function ReadFile(file) {
return await file.text()
}
function removeExtraSpace(stringData) {
stringData = stringData.replace(/,( *)/gm, ",") // remove extra space
stringData = stringData.replace(/^ *| *$/gm, "") // remove space on the beginning and end.
return stringData
}
function simpleTest() {
let data = `Name, Age, msg
foo, 25, hello world
bar, 18, "!! 🐬 !!"
`
data = removeExtraSpace(data)
console.log(data)
const options = {
separator: ",", // default "," . (You may want to Tab "\t" or somethings.
delimiter: '"', // default "
headers: true // default true
}
// const myObj = $.csv.toObjects(data, options)
const myObj = $.csv.toObjects(data) // If you want to use default options, then you can omit them.
console.log(myObj)
}
window.onload = () => {
const inputFile = document.getElementById("uploadFile")
inputFile.onchange = () => {
const inputValue = inputFile.value
if (inputValue === "") {
return
}
const selectedFile = document.getElementById('uploadFile').files[0]
const promise = new Promise(resolve => {
const fileContent = ReadFile(selectedFile)
resolve(fileContent)
})
promise.then(fileContent => {
// Use promise to wait for the file reading to finish.
console.log(fileContent)
fileContent = removeExtraSpace(fileContent)
const myObj = $.csv.toObjects(fileContent)
console.log(myObj)
})
}
}
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.6.0/jquery.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery-csv/1.0.11/jquery.csv.min.js"></script>
<label for="uploadFile">Demo 1</label>
<input type="file" id="uploadFile" accept=".csv"/>
<button onclick="simpleTest()">Demo 2</button>
With this function csvToObjs you can transform data-entries from format CSV to an array of objects.
function csvToObjs(string) {
const lines = data.split(/\r\n|\n/);
let [headings, ...entries] = lines;
headings = headings.split(',');
const objs = [];
entries.map(entry=>{
obj = entry.split(',');
objs.push(Object.fromEntries(headings.map((head, i)=>[head, obj[i]])));
})
return objs;
}
data = `heading1,heading2,heading3,heading4,heading5
value1_1,value2_1,value3_1,value4_1,value5_1
value1_2,value2_2,value3_2,value4_2,value5_2`
console.log(csvToObjs(data));

Categories

Resources