I have a huge .csv file. It has about 130,000 rows and 13 columns. I need to create a bunch of graphs using that data in a web app. I am very new to JavaScript. Can anyone help me reading this huge file and store the data into a 2D array?
I think you can use this lib
http://c2fo.io/fast-csv
or
https://www.papaparse.com/
this libs use streams, and you can use huge file and modify each line
If you are working on a node app, try looking for packages on npm,
https://www.npmjs.com/search?q=csv
Else, you might want to take a look at the following link for core javascript code:
How to read data From *.CSV file using javascript?
This code will help you:
<head>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
// Reading csv data from file using get ou post or whateven protocol
$(document).ready(function() {
$.ajax({
type: "GET",
url: "data.txt",
dataType: "text",
success: function(data) {
var lines = processData(data);
console.log(lines);
}
});
});
// Passing csv data direct from text string
csv = "heading1,heading2,heading3,heading4,heading5\nvalue1_1,value2_1,value3_1,value4_1,value5_1\nvalue1_2,value2_2,value3_2,value4_2,value5_2";
var lines = processData(data);
console.log(lines);
function processData(allText) {
var allTextLines = allText.split(/\r\n|\n/);
var headers = allTextLines[0].split(',');
var lines = [];
for (var i=1; i<allTextLines.length; i++) {
var data = allTextLines[i].split(',');
if (data.length == headers.length) {
var tarr = [];
for (var j=0; j<headers.length; j++) {
tarr.push(data[j]);
}
lines.push(tarr);
}
}
return lines;
}
</head>
Related
I have a getJSON jquery function that uses data from an external file eg
$.getJSON( "data/dataset.json", function( data ) {
var mygeojson = {
type: "FeatureCollection",
features: [],
};
var waypoints = []
var all_points=[]
for (i = 0; i < data.length; i++) {
var lng = data[i].lng;
.....
I now have a local variable that contains the same data as the external data file.
eg
const dataset = [{..}, {..},......]
The data is in the exact same format.
How can I use a pure js function that references the dataset variable rather than the jquery getjson function?
Lets say you have a file with the following code as the data,
datasetFile.js
export const dataset = [{..}, {..},......];
and you have another file where you want to use this data
useDataset.js
import { dataset } from './datasetFile.js
console.log(dataset);
You should be good to go.
I need to display the images on my site from a JSON request.
I have the JSON:
https://api.flickr.com/services/rest/?method=flickr.photos.search&api_key=6a970fbb976a06193676f88ef2722cc8&text=sampletext&sort=relevance&privacy_filter=1&safe_search=1&per_page=5&page=1&format=json&nojsoncallback=1
And I have the format I need to put the photo URL in:
https://www.flickr.com/services/api/misc.urls.html
But I don't know how I would loop through that, I found some examples similar, but I am still having trouble seeing what I need.
I am using JavaScript/jQuery to pull the info.
I figure I would have this in a loop.
CurrentPhotoUrl = 'https://farm'+CurrentPhotoFarm+'.staticflickr.com/'+CurrentPhotoServer+'/'+CurrentPhotoId+'_'+CurrentPhotoSecret+'_n.jpg'
But each of those variables would need to be populated with an value from the element. I would need to loop through all 5 elements that are in the JSON.
Any help on how to create this loop would be greatly appreciated.
Try this code
var n = JSON.parse(x) //x is the json returned from the url.
var _s = n.photos.photo;
for(var z = 0 ; z < n.photos.photo.length ; z++)
{
var CurrentPhotoUrl = 'https://farm'+_s[z]['farm']+'.staticflickr.com/'+_s[z]['server']+'/'+_s[z]['id']+'_'+_s[z]['secret']+'_n.jpg'
console.log(CurrentPhotoUrl);
}
Edit ( With actual JQUERY AJAX call )
var n ='';
$.ajax({url: "https://api.flickr.com/services/rest/?method=flickr.photos.search&api_key=6a970fbb976a06193676f88ef2722cc8&text=sampletext&sort=relevance&privacy_filter=1&safe_search=1&per_page=5&page=1&format=json&nojsoncallback=1", success: function(result){
console.log(result);
n = result;
var _s = n.photos.photo;
for(var z = 0 ; z < n.photos.photo.length ; z++)
{
var CurrentPhotoUrl = 'https://farm'+_s[z]['farm']+'.staticflickr.com/'+_s[z]['server']+'/'+_s[z]['id']+'_'+_s[z]['secret']+'_n.jpg'
console.log(CurrentPhotoUrl);
}
}});
Output:
https://farm8.staticflickr.com/7198/6847644027_ed69abc879_n.jpg
https://farm3.staticflickr.com/2517/3905485164_84cb437a29_n.jpg
https://farm1.staticflickr.com/292/32625991395_58d1f16cea_n.jpg
https://farm9.staticflickr.com/8181/7909857670_a64e1dd2b2_n.jpg
https://farm9.staticflickr.com/8143/7682898986_ec78701508_n.jpg
This answer assumes your json data will not change. So inside a .js file, set your json to a variable.
var json = <paste json here>;
// the photos are inside an array, so use forEach to iterate through them
json.photos.photo.forEach((photoObj) => {
// the photo will render via an img dom node
var img = document.createElement('img');
var farmId = photoObj.farm;
// you can fill out the rest of the variables ${} in the url
// using the farm-id as an example
img.src = `https://farm${farmId}.staticflickr.com/${serverId}/${id}_${secret}.jpg`
// add the image to the dom
document.body.appendChild(img);
}
Inside your html file that contains a basic html template, load this javascript file via a script tag, or just paste it inside a script tag.
If you want to get the json from the web page and assuming you have the jquery script loaded...
$.ajax({
type: 'GET',
url: <flicker_url_for_json>,
success: (response) => {
// iterate through json here
},
error: (error) => {
console.log(error);
}
});
I'm not sure if this is the best solution but its is something someone suggested and it worked.
const requestURL = 'https://api.flickr.com/services/rest/?method=flickr.photos.search&api_key=6a970fbb976a06193676f88ef2722cc8&text=sampletext&sort=relevance&privacy_filter=1&safe_search=1&per_page=5&page=1&format=json&nojsoncallback=1'
$.ajax(requestURL)
.done(function (data) {
data.photos.photo.forEach(function (currentPhoto) {
console.log('https://farm'+currentPhoto.farm+'.staticflickr.com/'+currentPhoto.server+'/'+currentPhoto.id+'_'+currentPhoto.secret+'_n.jpg')
})
})
Varun's solution worked for me as well. I don't know which one is better but I thought I would post this as well since it looks like they were done fairly differently.
I have two local JSON files that I'm trying to access in a function in my main.js file. Everything works fine with just one JSON file, but I'm not sure how to incorporate the second. Ideally, something like data_set1=$.getJSON("file1.json") would work perfectly, but I see similar questions have been repeatedly asked and because of asynchronous calls, that's not necessarily possible (I don't completely understand all the answers to those questions).
This works as it is:
$.getJSON("data.json", function(json){
var data_points = [];
for (i = 0; i < json.length; i++){
data_points.push([json[i].name, json[i].age]);
}
$(function () {
//do stuff with data_points
but I don't know how to incorporate the second JSON call to make another list for the function at the end to use.
You can use jQuery deferred loading.
var xFile, yFile;
var requestX = $.getJSON("data1.json", function(json){
xFile = json;
});
var requestY = $.getJSON("data2.json", function(json){
yFile = json;
});
$.when(requestX, requestY).then(function(){
// do something;
// this function only gets called when both requestX & requestY complete.
});
Check out JQuery WHEN
An ajax request is ONE request for ONE resource. You can't fetch two different resources with the same request. You'd either need TWO requests:
var completed = 0;
$.getJSON('file1.json', function(json) {
completed++;
if (completed == 2) { all_done(); }
}
$.getJSON('file2.json', function(json) {
completed++;
if (completed == 2) { all_done(); }
}
function all_done(...) { ... }
Or simply combine your two json files into a single one on the server:
{"file1":{data from file one here}, "file2":{data from file two here}}
and access them as data.file1 and data.file2 in your code.
You could just set a var to indicate the file is loaded and call the same function while checking the status. This might be a bad idea depending on the amount of data in the JSON.
var xFileLoaded = false, yFileLoaded = false;
var xFile, yFile;
$.getJSON("data.json", function(json){
xFile = json;
xFileLoaded = true;
doSomething();
});
$.getJSON("data.json", function(json){
yFile = json;
yFileLoaded = true;
doSomething();
});
function doSomething() {
if(xFileLoaded && yFileLoaded) {
// good to go
}
}
Using something very similar to #MarcB's solution, I would keep the files in an array and generalize things slightly:
var files = ["file1.json", "file2.json"];
var results = {};
var completed = 0;
function afterEach(fileName, json) {
results[fileName] = json;
if (++completed >= files.length) {
afterAll();
}
}
files.forEach(function (file) {
$.getJSON(file, afterEach.bind(this, file));
});
This should request all files in parallel, allowing the browser to choose how many connections to actually open and download. After each finishes, the results (json) are put in a hash for storage, under the filename. After all files have completed (assuming files doesn't change), then afterAll function will be called.
It exists with jquery an elegant way to do this :
$.when(
$.ajax({
url: 'file1.json/',
success: function(data) {
// Treatement
}
}),
$.ajax({
url: 'file2.json',
success: function(data) {
// Treatement
}
})
).then( function(){
// Two calls are completed
});
Once the two calls are completed, you can do any manipulation in the block "then".
I need some help with "synchronizing" my AJAX calls. I have a function scripted that takes in a file with certain test parameters and uses those parameters to kick off test via an AJAX call/s. The way the code is suppose to work is thatonce the test run is complete, another AJAX call is suppose to update the eventLog with the test results for that run and then move to the next iteration of the for loop.
The ajax calls can be seen in the for loop towards the bottom of the code. I looked into some documentation on using the jQuery Deffered class, but I am pretty new to JavaScript in general and I'm having trouble understanding how that code works. Thanks in advance for any help.
function runTest(modelName, serialNum, passArea) {
//Pull in sequence file
var str = "";
var table = document.getElementById('taskTable');
var output = document.getElementById('outputStrArea');
var passFail = document.getElementById(passArea).innerHTML;
var rowCount = table.rows.length;
for(var i=1; i<rowCount; i++) {
var row = table.rows[i];
str+= row.innerHTML+"\n";
}
str = str.split("<td>");
delete str[0];
delete str[-1];
//Create a list of tests that can be read out
var testList = [];
for (var i=1; i <str.length; i++){
str[i]= str[i].replace("</td>", "");
str[i]= str[i].replace(" ", "");
if (str[i].search("checkbox") < 0){
testList.push(str[i]);
}
}
var model = document.getElementById(modelName).innerHTML;
model = model.replace("<b>", "");
model = model.replace("</b>", "");
model = model.replace(" ", "");
var serial = document.getElementById(serialNum).innerHTML;
var info = model+" "+serial+" ";
for(var k=0; k<testList.length; k+=5){
info+= testList[k]+" "+testList[k+1]+" "+testList[k+2]+" "+testList[k+3]+" "+testList[k+4]+" ";
ajax("loadTestSequence?info="+info, [], passArea);
ajax("loadEventLog", [], 'eventLog');
}
}
The ajax calls have a success callback in which you can invoke another ajax call id you want. Syntax is something like this:
$.ajax({
url: url,
data: data,
success: function(){
$.ajax({
url: url,
data: data,
success: success,
dataType: dataType
});
},
dataType: dataType
});
So now you can chain your respective ajax calls similarly. Hope that helps.
I am developing a web page in Wordpress. The webpage needs to have a combobox with all counties. I have a dataset in csv format which has some 10k rows for all these counties.
When the user selects a county in the dropdown, I want only the selected county data displayed in the web page. This is my requirement.
In wordpress, my web page I am adding the js file using
<script type="text/javascript" src="http://xxx/wp content/uploads/2014/05/countyList1.js"></script>
and the code for webpage dropdown is
<select name="county" id="county" onload="setCounties();" onchange="getSelectedCountyData();"></select>
In countyList1.js file I have the setCounties() and getSelectedCountyData() functions.
So far I can see the dropdown with counties list. I don't know how to read the CSV file and apply the selected county filter to this list.
I tried the FileReader object and I can load the CSV contents on the web page but I don't want the user to select the CSV. I have the dataset already.
I am trying to use this jquery.csv-0.71 library from this SO post How to read data From *.CSV file using javascript? but I need help.
Here's the code which gets called when a county is selected
function getSelectedCountyData() {
cntrySel = document.getElementById('county');
//selCty = countyList[cntrySel.value];
handleFiles();
}
function handleFiles() {
$(document).ready(function () {
$.ajax({
type: "GET",
url: "D:\Docs\Desktop\csvfile.csv",
dataType: "csv",
success: function (data) { processData(data); }
});
});
}
function processData(allText) {
var allTextLines = allText.split(/\r\n|\n/);
var headers = allTextLines[0].split(',');
var lines = [];
for (var i = 1; i < allTextLines.length; i++) {
var data = allTextLines[i].split(',');
if (data.length == headers.length) {
var tarr = [];
for (var j = 0; j < headers.length; j++) {
tarr.push(headers[j] + ":" + data[j]);
}
lines.push(tarr);
}
}
console.log(lines);
drawOutput(lines);
}
function drawOutput(lines) {
//Clear previous data
document.getElementById("output").innerHTML = "";
var table = document.createElement("table");
for (var i = 0; i < lines.length; i++) {
var row = table.insertRow(-1);
for (var j = 0; j < lines[i].length; j++) {
var firstNameCell = row.insertCell(-1);
firstNameCell.appendChild(document.createTextNode(lines[i][j]));
}
}
document.getElementById("output").appendChild(table);
}
I highly recommend looking into this plugin:
http://github.com/evanplaice/jquery-csv/
I used this for a project handling large CSV files and it handles parsing a CSV into an array quite well. You can use this to call a local file that you specify in your code, also, so you are not dependent on a file upload.
Once you include the plugin above, you can essentially parse the CSV using the following:
$.ajax({
url: "pathto/filename.csv",
async: false,
success: function (csvd) {
data = $.csv.toArrays(csvd);
},
dataType: "text",
complete: function () {
// call a function on complete
}
});
Everything will then live in the array data for you to manipulate as you need. I can provide further examples for handling the array data if you need.
There are a lot of great examples available on the plugin page to do a variety of things, too.
You can't use AJAX to fetch files from the user machine. This is absolutely the wrong way to go about it.
Use the FileReader API:
<input type="file" id="file input">
js:
console.log(document.getElementById("file input").files); // list of File objects
var file = document.getElementById("file input").files[0];
var reader = new FileReader();
content = reader.readAsText(file);
console.log(content);
Then parse content as CSV. Keep in mind that your parser currently does not deal with escaped values in CSV like: value1,value2,"value 3","value ""4"""
If your not overly worried about the size of the file then it may be easier for you to store the data as a JS object in another file and import it in your . Either synchronously or asynchronously using the syntax <script src="countries.js" async></script>. Saves on you needing to import the file and parse it.
However, i can see why you wouldnt want to rewrite 10000 entries so here's a basic object orientated csv parser i wrote.
function requestCSV(f,c){return new CSVAJAX(f,c);};
function CSVAJAX(filepath,callback)
{
this.request = new XMLHttpRequest();
this.request.timeout = 10000;
this.request.open("GET", filepath, true);
this.request.parent = this;
this.callback = callback;
this.request.onload = function()
{
var d = this.response.split('\n'); /*1st separator*/
var i = d.length;
while(i--)
{
if(d[i] !== "")
d[i] = d[i].split(','); /*2nd separator*/
else
d.splice(i,1);
}
this.parent.response = d;
if(typeof this.parent.callback !== "undefined")
this.parent.callback(d);
};
this.request.send();
};
Which can be used like this;
var foo = requestCSV("csvfile.csv",drawlines(lines));
The first parameter is the file, relative to the position of your html file in this case.
The second parameter is an optional callback function the runs when the file has been completely loaded.
If your file has non-separating commmas then it wont get on with this, as it just creates 2d arrays by chopping at returns and commas. You might want to look into regexp if you need that functionality.
//THIS works
"1234","ABCD" \n
"!#£$" \n
//Gives you
[
[
1234,
'ABCD'
],
[
'!#£$'
]
]
//This DOESN'T!
"12,34","AB,CD" \n
"!#,£$" \n
//Gives you
[
[
'"12',
'34"',
'"AB',
'CD'
]
[
'"!#',
'£$'
]
]
If your not used to the OO methods; they create a new object (like a number, string, array) with their own local functions and variables via a 'constructor' function. Very handy in certain situations. This function could be used to load 10 different files with different callbacks all at the same time(depending on your level of csv love! )
This is what I used to use a csv file into an array. Couldn't get the above answers to work, but this worked for me.
$(document).ready(function() {
"use strict";
$.ajax({
type: "GET",
url: "../files/icd10List.csv",
dataType: "text",
success: function(data) {processData(data);}
});
});
function processData(icd10Codes) {
"use strict";
var input = $.csv.toArrays(icd10Codes);
$("#test").append(input);
}
Used the jQuery-CSV Plug-in linked above.
The original code works fine for reading and separating the csv file data but you need to change the data type from csv to text.