Develop Reference

Converting CSV to JSON using JavaScript

I'm learning JavaScript and i'm trying to convert a CSV file to a JSON.
The structure of my csv is :
Name
Class
region_count
Coordinates
foto_4.jpeg
soccer
1
"all_points_x":[90,80,77,74,82,89,113,142,146,153,162,174,184,199,220,235,261,280,289,298,298,287,279,276,271,268,265,266,270,270,262,249,236,222,213,188,170,151,114,92],"all_points_y":[145,171,192,211,241,263,291,308,310,301,288,275,265,257,246,244,241,238,241,243,235,223,208,196,176,165,148,134,119,114,109,99,96,93,90,89,89,94,116,146]
foto_4.jpeg
tennis
2
"all_points_x":[394,418,445,456,467,472,469,461,448,436,425,412,402,394,384,383,392],"all_points_y":[276,259,260,266,279,296,313,327,335,341,342,338,334,326,313,294,279]
foto_5.jpeg
basket
1
"all_points_x":[345,373,427,479,509,540,553,549,541,526,490,467,455,447,430,421,411,406,400,394,391,381,368,349,337,327,320,308,301],"all_points_y":[23,11,7,22,44,89,140,182,207,230,261,274,263,260,255,255,255,261,268,273,278,286,279,273,268,258,250,242,237]
the desired json structure is:
{"foto_1jpg.jpg121349":{"filename":"foto_1jpg.jpg","regions":[{"shape_attributes":{"all_points_x":[154,157,230,275,278,218,160,112,113,154],"all_points_y":[461,461,455,495,576,625,625,563,505,463]},"region_attributes":{"name":"tennis"}},{"shape_attributes":{"all_points_x":[446,557,685,795,826,815,738,628,505,422,346,331,354,443],"all_points_y":[230,186,212,321,411,538,641,687,684,632,525,426,331,224]},"region_attributes":{"name":"soccer"}}],"file_attributes":{}},"foto_2.jpg325912":{"filename":"foto_2.jpg","regions":[{"shape_attributes":{"all_points_x":[331,403,518,626,688,734,758,681,594,484,369,314,282,274,329],"all_points_y":[399,340,316,342,380,463,607,736,787,796,745,683,592,503,405]},"region_attributes":{"name":"soccer"}},{"shape_attributes":{"name":"polygon","all_points_x":[972,887,830,802,789,804,857,963,1050,1135,1220,1284,1314,1307,1263,1178,1116,1057,955],"all_points_y":[144,195,261,327,397,484,579,639,660,647,603,524,424,335,238,174,146,140,157]}],"file_attributes":{}},"foto_3.jpg196633":{"filename":"foto_3.jpg","regions":[{"shape_attributes":{"name":"polygon","all_points_x":[65,43,49,107,160,215,290,351,406,431,409,373,334,274,203,150,107,70],"all_points_y":[349,421,503,586,622,630,629,593,537,465,356,313,283,264,256,278,303,346]},"region_attributes":{"name":"soccer"}}],"file_attributes":{}}}
I tried to convert this CSV to JSON with this code
var csv = `,Name,Class,Region_count,Coordinates
0,foto_1jpg.jpg,tennis,1,"""all_points_x"":[154,157,230,275,278,218,160,112,113,154],""all_points_y"":[461,461,455,495,576,625,625,563,505,463]"
1,foto_1jpg.jpg,soccer,2,"""all_points_x"":[446,557,685,795,826,815,738,628,505,422,346,331,354,443],""all_points_y"":[230,186,212,321,411,538,641,687,684,632,525,426,331,224]"
2,foto_1jpg.jpg,basket,3,"""all_points_x"":[941,1065,1161,1310,1438,1497,1509,1471,1382,1279,1124,998,916,874,847,874,938],""all_points_y"":[132,44,26,48,144,266,396,514,628,673,687,631,560,479,328,233,135]"
3,foto_2jpg.jpg,soccer,1,"all_points_x:[331,403,518,626,688,734,758,681,594,484,369,314,282,274,329],""all_points_y"":[399,340,316,342,380,463,607,736,787,796,745,683,592,503,405]"
4,foto_2jpg.jpg,basket,2,"""all_points_x"":[972,887,830,802,789,804,857,963,1050,1135,1220,1284,1314,1307,1263,1178,1116,1057,955],""all_points_y"":[144,195,261,327,397,484,579,639,660,647,603,524,424,335,238,174,146,140,157]"
5,foto_2jpg.jpg,tennis,3,"all_points_x:[1186,1233,1273,1282,1267,1231,1178,1154,1135,1131,1142,1182],""all_points_y"":[921,921,891,845,806,777,775,789,819,859,895,919]"
6,foto_3jpg.jpg,soccer,1,"""all_points_x"":[65,43,49,107,160,215,290,351,406,431,409,373,334,274,203,150,107,70],""all_points_y"":[349,421,503,586,622,630,629,593,537,465,356,313,283,264,256,278,303,346]"
7,foto_3jpg.jpg,basket,2,"""all_points_x"":[523,588,647,739,809,854,871,877,860,845,830,823,816,811,804,802,796,774,765,753,726,712,699,685,682,671,670,666,664,632,601,583,549,515,496,469,446,448,467,518],""all_points_y"":[242,203,196,206,247,307,361,436,491,509,515,521,530,537,549,561,572,583,583,562,550,547,554,557,571,578,591,601,620,615,612,608,588,552,532,496,428,370,319,244]"
8,foto_3jpg.jpg,tennis,3,"all_points_x:[838,881,901,917,912,888,869,845,821,804,792,791,813],""all_points_y"":[544,544,569,600,627,646,654,654,651,634,601,578,552]"`;
var map = {};
var rows = csv.split(/\n/g);
var keys = rows.shift().split(",");
rows.forEach(raw_row=>{
var row = {};
var row_key;
var columns = raw_row.split(/,(?=(?:(?:[^"]*"){2})*[^"]*$)/);
columns.forEach((column, index)=>{
var key = keys[index];
if(!key) return;
if(key === 'Name'){
row_key = column;
return;
}
if(key === "Coordinates"){
column = column.replace(/""/g, '"');
column = column.substring(1, column.length-1);
column = column.replace(/([a-zA-Z_]+):/g, `"$1":`);
try{ column = JSON.parse(`{${column}}`); }catch(e){}
}
row[key] = column;
});
map[row_key] = row;
});
console.log(map);
the JSON create is this:
{"foto_1jpg.jpg":{"Class":"basket","region_count":"3","Coordinates":{"all_points_x":[941,1065,1161,1310,1438,1497,1509,1471,1382,1279,1124,998,916,874,847,874,938],"all_points_y":[132,44,26,48,144,266,396,514,628,673,687,631,560,479,328,233,135]}},"foto_2jpg.jpg":{"Class":"tennis","region_count":"3","Coordinates":{"all_points_x":[1186,1233,1273,1282,1267,1231,1178,1154,1135,1131,1142,1182],"all_points_y":[921,921,891,845,806,777,775,789,819,859,895,919]}},"foto_3jpg.jpg":{"Class":"tennis","region_count":"3","Coordinates":{"all_points_x":[838,881,901,917,912,888,869,845,821,804,792,791,813],"all_points_y":[544,544,569,600,627,646,654,654,651,634,601,578,552]}}}
With my code I can't iterate for all the polygons contained in one picture and I can't add region key (example region) that contains other keys
How can I reach my desired output?

Your initial approach looks pretty good. We just need some more modifications to the data you generated. Instead of directly mapping each row to JSON, first keep data in the array as a line item, and then build the JSON data as follows.
var csv = `,Name,Class,Region_count,Coordinates
0,foto_1jpg.jpg,tennis,1,"""all_points_x"":[154,157,230,275,278,218,160,112,113,154],""all_points_y"":[461,461,455,495,576,625,625,563,505,463]"
1,foto_1jpg.jpg,soccer,2,"""all_points_x"":[446,557,685,795,826,815,738,628,505,422,346,331,354,443],""all_points_y"":[230,186,212,321,411,538,641,687,684,632,525,426,331,224]"
2,foto_1jpg.jpg,basket,3,"""all_points_x"":[941,1065,1161,1310,1438,1497,1509,1471,1382,1279,1124,998,916,874,847,874,938],""all_points_y"":[132,44,26,48,144,266,396,514,628,673,687,631,560,479,328,233,135]"
3,foto_2jpg.jpg,soccer,1,"all_points_x:[331,403,518,626,688,734,758,681,594,484,369,314,282,274,329],""all_points_y"":[399,340,316,342,380,463,607,736,787,796,745,683,592,503,405]"
4,foto_2jpg.jpg,basket,2,"""all_points_x"":[972,887,830,802,789,804,857,963,1050,1135,1220,1284,1314,1307,1263,1178,1116,1057,955],""all_points_y"":[144,195,261,327,397,484,579,639,660,647,603,524,424,335,238,174,146,140,157]"
5,foto_2jpg.jpg,tennis,3,"all_points_x:[1186,1233,1273,1282,1267,1231,1178,1154,1135,1131,1142,1182],""all_points_y"":[921,921,891,845,806,777,775,789,819,859,895,919]"
6,foto_3jpg.jpg,soccer,1,"""all_points_x"":[65,43,49,107,160,215,290,351,406,431,409,373,334,274,203,150,107,70],""all_points_y"":[349,421,503,586,622,630,629,593,537,465,356,313,283,264,256,278,303,346]"
7,foto_3jpg.jpg,basket,2,"""all_points_x"":[523,588,647,739,809,854,871,877,860,845,830,823,816,811,804,802,796,774,765,753,726,712,699,685,682,671,670,666,664,632,601,583,549,515,496,469,446,448,467,518],""all_points_y"":[242,203,196,206,247,307,361,436,491,509,515,521,530,537,549,561,572,583,583,562,550,547,554,557,571,578,591,601,620,615,612,608,588,552,532,496,428,370,319,244]"
8,foto_3jpg.jpg,tennis,3,"all_points_x:[838,881,901,917,912,888,869,845,821,804,792,791,813],""all_points_y"":[544,544,569,600,627,646,654,654,651,634,601,578,552]"`;
var items = []
var rows = csv.split(/\n/g);
var keys = rows.shift().split(",");
rows.forEach(raw_row => {
var row = {};
var columns = raw_row.split(/,(?=(?:(?:[^"]*"){2})*[^"]*$)/);
columns.forEach((column, index)=>{
var key = keys[index];
if(!key) return;
if(key === "Coordinates"){
column = column.replace(/""/g, '"');
column = column.substring(1, column.length-1);
column = column.replace(/([a-zA-Z_]+):/g, `"$1":`);
try{ column = JSON.parse(`{${column}}`); }catch(e){}
}
row[key] = column;
});
items.push(row);
});
const map = {}
for (const item of items) {
if (!map[item['Name']]) {
map[item['Name']] = {
'filename': item['Name'],
'regions': [],
'file_attributes': {}
}
}
map[item['Name']].regions.push(
{
"shape_attributes": item['Coordinates'],
"region_attributes": { "name": item['Class'] }
}
)
}
console.log(map)

Little Code Review | From Excel to Json Node JS

Intro -> I have to convert an excel file, which comes usually in this way:
to a Json file that must looks like this:
"DE": {
"Title":"Title_DE",
"Subtitle":"Subtitle_DE",
"Title_2":"Title_2_DE",
"Subtitle_2":"Subtitle_2_DE"
}
"GR": {
"Title":"Title_GR",
"Subtitle":"Subtitle_GR",
"Title_2":"Title_2_GR",
"Subtitle_2":"Subtitle_2_GR"
}
"EN": {
"Title":"Title_EN",
"Subtitle":"Subtitle_EN",
"Title_2":"Title_2_EN",
"Subtitle_2":"Subtitle_2_EN"
}
What I've done -> I have used this node module xlsx to parse the excel file. The code currently works but something in my mind tells me that was not the best way to do it or at least there was a clearest or a smartest way.
const XLSX = require('xlsx');
const fs = require('fs');
let workbook = XLSX.readFile('./src/xls/test.xlsx');
let sheet_name_list = workbook.SheetNames; //--> sheet names
let isocodes = [];
let labels = [];
let copy = [];
let obj = {};
let j = 0;
sheet_name_list.map(sheet => { //--> sheet names loop
let worksheet = workbook.Sheets[sheet];
for (index in worksheet) {
let idCol = index.substring(1, 3);
let idRow = index.substring(0, 1);
if (idCol === "1") {
isocodes.push((worksheet[index].v).trim()); //--> isocodes
}
if (idRow === "A") {
labels.push((worksheet[index].v).trim()); //--> labels
}
if (idRow != "A" && idCol != "1") {
copy.push(worksheet[index].v); //--> copy
}
}
});
isocodes.shift(); //--> delete undefined
labels.shift();
copy.shift();
isocodes.map(iso => { //--> create object to convert
obj[iso] = {};
for (let i = 0; i < labels.length; i++) {
obj[iso][labels[i]] = copy[(i * isocodes.length) + j];
}
j++;
});
//--> write json files in their folder
fs.writeFile('src/lang/desktop.json', JSON.stringify(obj).replace(/\\n|\\r/g, ''), 'utf8', console.log("desktop json file created"));
fs.writeFile('src/lang/mobile.json', JSON.stringify(obj).replace(/\\n|\\r/g, ''), 'utf8', console.log("mobile json file created"));
Please share your thoughts!!!

Filereader read file using correct encoding when read as readAsArrayBuffer

I am working on reading .csv /xlsx file uploaded using javaScript and get the result as array containing each row . I was able to read the file and get data using FileReader and SheetJs with following code.
// code for the new excel reader
$scope.do_file = function(files)
{
$scope.fileContent = [];
var X = XLSX;
var global_wb;
var f = files[0];
var reader = new FileReader();
reader.onload = function(e)
{
var data = e.target.result;console.log(data);
global_wb = X.read(data, {type: 'array'});
var output = "";
var result = {};
global_wb.SheetNames.forEach(function(sheetName) {
var roa = X.utils.sheet_to_json(global_wb.Sheets[sheetName], {header:1});
if(roa.length) result[sheetName] = roa;
});
$scope.fileContent = result["Sheet1"];
if(!result["Sheet1"])
{
$scope.fileContent = result["contacts"].filter(function(el) { return typeof el != "object" || Array.isArray(el) || Object.keys(el).length > 0; });
}
};
reader.readAsArrayBuffer(f);
};
For reading most of the files the code works , but when file containing Hebrew text with Windows-1255 encoding i get gibberish data.
Looking in for more options i tried to read the file as text using reader.readAsText and change the encoding as necessary , check the following code:
function is_Hebrew(data)
{
var position = data.search(/[\u0590-\u05FF]/);
return position >= 0;
}
$scope.do_file = function(files)
{
var fullResult = [];
var file =files[0];
var reader = new FileReader();
reader.onload = function(e){
var data = e.target.result;
if(!is_Hebrew(data.toString()))
{
reader.readAsText(file,'ISO-8859-8');
}
};
reader.readAsText(file);
reader.onloadend = function(){
var lines = reader.result.split('\r\n');
console.log(lines);
lines.forEach(element => {
var cell = element.split(',');
fullResult.push(cell);
});
console.log(reader);
};
};
but the above code is not suitable as it does not read the file as each row identifying each cell. if any one of cell contains string with coma separated value (for example if a cell contains a string value such as "25,28,29" ) the array output gives wrong data as it considers each values as each cell.
So i decided to stick with first method but i am not able to change the encoding .Is there a possible way to change encoding in the first code where i have used the readAsArrayBuffer to read the file data ?

After going through lot of possible solutions i found that answer to the above question was to combine the above two methods. The first method for reading the xlsx files and second method for reading csv files. Also i have used an additional javaScript library called papaparse in the second method to solve the problem of reading data in each cell
$scope.is_Hebrew = function($data){
var position = $data.search(/[\u0590-\u05FF]/);
return position >= 0;
}
// code for the new excel reader
$scope.do_file = function(files)
{
var config = {
delimiter: "", // auto-detect
newline: "", // auto-detect
quoteChar: '"',
escapeChar: '"',
header: false,
trimHeader: false,
dynamicTyping: false,
preview: 0,
encoding: "",
worker: false,
comments: false,
step: undefined,
complete: undefined,
error: undefined,
download: false,
skipEmptyLines: false,
chunk: undefined,
fastMode: undefined,
beforeFirstChunk: undefined,
withCredentials: undefined
};
$scope.fileContent = [];
var f = files[0];
var fileExtension = f.name.replace(/^.*\./, '');
if(fileExtension == 'xlsx')
{
var X = XLSX;
var global_wb;
var reader = new FileReader();
reader.onload = function(e)
{
var data = e.target.result;
global_wb = X.read(data, {type: 'array'});
var result = {};
global_wb.SheetNames.forEach(function(sheetName) {
var roa = X.utils.sheet_to_json(global_wb.Sheets[sheetName], {header:1});
if(roa.length) result[sheetName] = roa;
});
$scope.fileContent = result["Sheet1"];
if(!result["Sheet1"])
{
$scope.fileContent = result["contacts"].filter(function(el) { return typeof el != "object" || Array.isArray(el) || Object.keys(el).length > 0; });
}
};
reader.readAsArrayBuffer(f);
}
else if(fileExtension == 'csv')
{
var reader = new FileReader();
reader.onload = function(e)
{
var data = e.target.result;
console.log(f);
console.log($scope.is_Hebrew(data.toString()));
if(!$scope.is_Hebrew(data.toString()))
{
reader.readAsText(f,'ISO-8859-8');
}
};
reader.readAsText(f);
reader.onloadend = function(e){
var c = Papa.parse(reader.result,[ config])
console.log(c);
$scope.fileContent = c["data"].filter(function(el) { return typeof el != "object" || Array.isArray(el) || Object.keys(el).length > 0; });
};
}
else
{
alert("File Not supported!");
}
$scope.fileContent.push([]);
};

How to parse, and iterate through, all rows from an Excel sheet file using JavaScript

I am trying to parse and read every cell from an Excel sheet; it seems that I am parsing the file but I just can't go through each cell and display them.
I am using the following code:
var workbook = XLSX.read('datasets/persons.xlsx', { type: 'binary' });
var sheet_name_list = workbook.SheetNames;
// console.log(sheet_name_list);
sheet_name_list.forEach(function (y) { /* iterate through sheets */
//Convert the cell value to Json
console.log(y);
var roa = XLSX.utils.sheet_to_json(workbook.Sheets[y]);
console.log(roa);
if (roa.length > 0) {
result = roa;
}
});
I am getting an empty array when I try to print console.log(roa), any idea how I should iterate through each cell from the file?

you can use
XLSX.utils.sheet_to_row_object_array(workbook.Sheets[y])
to parse each cells in excel file.
Here is full code use to display Excel sheet file using JavaScript and JQuery.
function handleFile(e) {
//alert(e);
var exceljsonObj = [];
var files = e.target.files;
var i, f;
for (i = 0, f = files[i]; i != files.length; ++i) {
var reader = new FileReader();
var name = f.name;
reader.onload = function (e) {
var data = e.target.result;
var result;
var workbook = XLSX.read(data, { type: 'binary' });
var sheet_name_list = workbook.SheetNames;
sheet_name_list.forEach(function (y) { /* iterate through sheets */
//var exceljsonObj = [];
var rowObject = XLSX.utils.sheet_to_row_object_array(workbook.Sheets[y]);
exceljsonObj = rowObject;
for(var i=0;i<exceljsonObj.length;i++){
//var recordcount = exceljsonObj.length;
var data = exceljsonObj[i];
$('#myTable tbody:last-child').append("<tr><td>"+data.ID+"</td><td>"+data.Name+"</td><td>"+data.Months+"</td></tr>");
}
//alert(exceljsonObj.length);
$('#alermessage').each(function() {
//this points to item
alert('Record Count is '+exceljsonObj.length);
});
});
};
reader.readAsArrayBuffer(f);
}
}
$(document).ready(function(){
$('#files').change(handleFile);
});

XLSX parser for parsing excel

I am trying to parse xlsx file and save as a table (Along the lines of Excel to JSON javascript code? which is working fine for xls files). However, I am unable to convert to json and display as a table. Please find below the code snippet. Can anyone please guide how to close on this.
function filePicked(oEvent) {
// Get The File From The Input
var oFile = oEvent.target.files[0];
var sFilename = oFile.name;
// Create A File Reader HTML5
var reader = new FileReader();
// Ready The Event For When A File Gets Selected
reader.onload = function(e) {
var data = e.target.result;
var cfb = XLSX.read(data, {type: 'binary'});
var wb = XLSX.parse_xlscfb(cfb);
wb.SheetNames.forEach(function(sheetName) {
// Obtain The Current Row As CSV
//var sCSV = XLSX.utils.make_csv(wb.Sheets[sheetName]);
var data = XLSX.utils.make_json(wb.Sheets[sheetName], {header:1});
alert(data.length);
//var columns = data[0].split(",");
$.each(data, function( indexR, valueR ) {
var sRow = "<tr>";
$.each(data[indexR], function( indexC, valueC ) {
sRow = sRow + "<td>" + valueC + "</td>";
});
sRow = sRow + "</tr>";
$("#my_file_output").append(sRow);
});
});
};
// Tell JS To Start Reading The File.. You could delay this if desired
reader.readAsBinaryString(oFile);
}

use this code :
note: use jszip.js and xlsx.js library
reader.onload = function(evt) {
debugger;
var data = evt.target.result;
//var xlsx = XLSX.read(data, {type: 'binary'});
var arr = String.fromCharCode.apply(null, new Uint8Array(data));
var xlsx = XLSX.read(btoa(arr), {
type: 'base64'
});
result = xlsx.Strings;
result = {};
xlsx.SheetNames.forEach(function(sheetName) {
var rObjArr = XLSX.utils.sheet_to_row_object_array(xlsx.Sheets[sheetName]);
if (rObjArr.length > 0) {
result[sheetName] = rObjArr;
}
});
return result;
// that.b64toBlob(xlsx, "binary");
};
reader.readAsArrayBuffer(file);
use readAsArrayBuffer method which will support on all browser.