Split A large JSON into Smaller parts in Javascript/jQuery - javascript

I have a file upload functionality in my application which can not upload JSON files which are more than 10MB in size. If user uploads a file >= 10 MB , My app should split it into smaller JSON files each less than 10MB. Also, the Proper JSON objects needs to be maintained in the new low-sized files.
Is there a way to do this in Javascript or jQuery?

I propose a solution like this without any specific library. It does use a bit of modern techniques but maybe useful to you:
var openFile = function(event, callback) {
// get target input
var input = event.target;
// create an instance of filereader
var reader = new FileReader();
// define handler to get results
reader.onload = function(e){
var contents = e.target.result;
// use a promise maybe to make this neater
callback(contents);
};
// make sure you tell it to read as text
// also maybe add some validation on your input
// for correct types
reader.readAsText(input.files[0]);
};
var getChunks = function(str){
var chunks = [];
// not best at these things but this should be
// around 1mb max
var chunkSize = 1000000;
// while the chunk is less than the size indicated it goes
// into the same item of array
while (str) {
if (str.length < chunkSize) {
chunks.push(str);
break;
}
else {
chunks.push(str.substr(0, chunkSize));
str = str.substr(chunkSize);
}
}
return chunks;
}
var fileInput = document.querySelector('#jsonUpload');
fileInput.addEventListener('change', function(event){
openFile(event, function(str){
console.log(getChunks(str));
});
});
Then it would read the json file from:
<input type='file' accept='*' id="jsonUpload">
Link to the fiddle

Related

how to read a static file from url in javascript to create an array

I searched but I don't find, I am coding a simulator and I want to do the calculus using javascript, the simulator takes 2 kind of entries. The first entries are given by user, this part is done. The second part is a lot of coefficient which are stored in csv/tsv file, the file is uploaded on the server. And I am not able to read this file, I found a lot of code on how to convert csv to array and I think that I will be able to do it alone. For now I am doing step by step so I just want to read the csv file to put it inside a table, when I use the code shown it works if I use an < input type="file" > but I am not able to make it works with a static url. Can You help me?
function myprocessFile()
{
var fileSize = 0;
var theFile = document.getElementById("myFile").files[0];
document.getElementById("toto").innerHTML = blob;
if (theFile)
{
var table = document.getElementById("myTable");
var headerLine = "";
var myReader = new FileReader();
myReader.onload = function(e)
{
// CREATE TABLE
}
myReader.readAsText(theFile);
}
return false;
}
You could use the fetch API :
fetch('url/to/your/csv/file')
.then(function(response) {
return response.text()
})
.then(function(csv) {
// convert your csv to an array
});

Loop multiple files from input, save each file readAsDataURL data to array

I need your help with following problem:
I have HTML input which supports multiple files;
I upload let's say 5 files;
Each file is processed: it is readAsDataURL by FileReader and data of file is saved to object(there will be other params saved too, that is why object), which is pushed to array.
After I run flow I described, length of final array is NOT changed.
I believe problem is in async behaviour, but I cannot understand how should I change code to make it work, that is why I ask you for a help. Please find code below:
var controls = document.getElementById('controls');
function processUploadedFilesData(files) {
if (!files[0]) {
return;
};
var uploads = [];
for (var i = 0, length = files.length; i < length; i++) {
(function(file) {
var reader = new FileReader();
//I need object, as other params will be saved too in future;
var newFile = {};
reader.readAsDataURL(file);
reader.onloadend = function(e) {
newFile.data = e.target.result;
uploads.push(newFile);
}
})(files[i]);
}
return uploads;
}
controls.addEventListener('change', function(e) {
var uploadedFilesOfUser = processUploadedFilesData(e.target.files);
alert(uploadedFilesOfUser.length);
});
Codepen example - https://codepen.io/yodeco/pen/xWevRy

I want to load images withen the folder and want to get all images in base64 in my js for future use

i am facing the issue i always get the last image in my image array due to kind of Filereader library function onloadend.
how can i get base64 for all images in my folder.
<input id="file-input" multiple webkitdirectory type="file" />
var input = document.getElementById('file-input');
var file_names = "";
var entries_length = 0;
var entries_count = 0;
var image = new Array();
var obj = {};
var j = 0;
input.onchange = function(e) {
var files = e.target.files; // FileList
entries_length = files.length;
console.log(files);
for (var i = 0, f; f = files[i]; ++i){
console.log("i:"+i);
entries_count = entries_count + 1;
//console.debug(files[i].webkitRelativePath);
if(files[i].type=="image/jpeg")
{
var string = files[i].webkitRelativePath;
var name = string.split("/")[3]; //this is because my image in 3rd dir in the folder
var reader = new FileReader();
reader.onloadend = function() {
obj.name = string.split("/")[3];
obj.image = reader.result;
image[j] = obj;
j = j+1;
}
reader.readAsDataURL(files[i]);
}
}
console.log(image);
}
The issue is caused by the asynchronous loading of files. You iterate over the array and set the onloadend handler for the reader each time, then start loading by calling readAsDataURL.
One problem is that by the time your first image loads, it is possible the for loop has completed, and i is already at the last index of the array.
At this point, obtaining the path from files[i].webkitRelativePath will give you the last filename, and not the one you are expecting.
Check the example for readAsDataURL on MDN to see one possible solution - each load is performed in a separate function, which preserves its scope, along with file.name. Do not be put off by the construction they are using: [].forEach.call(files, readAndPreview). This is a way to map over the files, which are a FileList and not a regular array (so the list does not have a forEach method of its own).
So, it should be sufficient to wrap the loading logic in a function which takes the file object as a parameter:
var images = [];
function loadFile(f) {
var reader = new FileReader();
reader.onloadend = function () {
images.push({
name : f.name, // use whatever naming magic you prefer here
image : reader.result
});
};
reader.readAsDataURL(f);
}
for (var i=0; i<files.length; i++) {
loadFile(files[i]);
}
Each call of the function 'remembers' the file object it was called with, and prevents the filename from getting messed up. If you are interested, read up on closures.
This also has the nice effect of isolating your reader objects, because I have a sneaking suspicion that, although you create a new 'local' reader each iteration, javascript scoping rules are weird and readers could also be interfering with each other (what happens if one reader is loading, but in the same scope you create a new reader with the same variable name? Not sure).
Now, you do not know how long it would take for all images to be loaded, so if you want to take an action right after that, you would have to perform a check each time an onloadend gets called. This is the essence of asynchronous behavior.
As an aside, I should note that it is pointless to manually keep track of the last index of images, which is j. You should just use images.push({ name: "bla", image: "base64..." }). Keeping indices manually opens up possibilities for bugs.

How to get objects from txt file to use in javascript array?

I am very new to coding and javascript; just a few days in. I was wondering if there was a way to import objects from a text file(separated by lines) to use in my array: replyText. Here is what I'm working with:
// Variables
var theButton = document.getElementById("theButton");
var mainText = document.getElementById("mainText");
var replyText = [...,...,...,...,];
var i = 0;
// Functions
function nextText() {
mainText.innerHTML = replyText[i++ % replyText.length];
}
// MAIN SCRIPT
theButton.onclick = function() {
nextText();
};
You can use XMLHttpRequest to get the .txt file just pass the path of it.
var file = new XMLHttpRequest();
file.open("GET", "file:/../file.txt", false);
file.onreadystatechange = function () {
if (file.readyState === 4) {
if (file.status === 200 || file.status == 0) {
var text = file.responseText;
alert(text);
}
}
}
EDIT: you must pass the absolute path file:///C:/your/path/to/file.txt
For client/browser-side file reading:
You cannot easily read a file on the client-side as you are not allowed direct access to the client's file system. However, you can place a input element of file type in your HTML markup via which the client can load a file for your program to process. For example:
<input type="file" id="file" onchange="readFile()" />
Now when the client selects a file for use, the readFile() function will be called which will read and process the file. Here's an example:
function readFile() {
var file = document.getElementById('file').files[0]; // select the input element from the DOM
var fileReader = new FileReader(); // initialize a new File Reader object
fileReader.onload(function() { // call this function when file is loaded
console.log(this.result); // <--- You can access the file data from this variable
// Do necessary processing on the file
});
fileReader.readAsText(file); // Read the file as text
}
For more information on File Reader, check out the docs.
To add on to Paulo's solution, read below for splitting string by line breaks (new line character)
var replyText = text.split("\n"); // "\n" is new line character

Get Base64 encode file-data from Input Form

I've got a basic HTML form from which I can grab a bit of information that I'm examining in Firebug.
My only issues is that I'm trying to base64 encode the file data before it's sent to the server where it's required to be in that form to be saved to the database.
<input type="file" id="fileupload" />
And in Javascript+jQuery:
var file = $('#fileupload').attr("files")[0];
I have some operations based on available javascript: .getAsBinary(), .getAsText(), .getAsTextURL
However none of these return usable text that can be inserted as they contain unusable 'characters' - I don't want to have a 'postback' occur in my file uploaded, and I need to have multiple forms targeting specific objects so it's important I get the file and use Javascript this way.
How should I get the file in such a way that I can use one of the Javascript base64 encoders that are widely available!?
Thanks
Update - Starting bounty here, need cross-browser support!!!
Here is where I'm at:
<input type="file" id="fileuploadform" />
<script type="text/javascript">
var uploadformid = 'fileuploadform';
var uploadform = document.getElementById(uploadformid);
/* method to fetch and encode specific file here based on different browsers */
</script>
Couple of issues with cross browser support:
var file = $j(fileUpload.toString()).attr('files')[0];
fileBody = file.getAsDataURL(); // only would works in Firefox
Also, IE doesn't support:
var file = $j(fileUpload.toString()).attr('files')[0];
So I have to replace with:
var element = 'id';
var element = document.getElementById(id);
For IE Support.
This works in Firefox, Chrome and, Safari (but doesn't properly encode the file, or at least after it's been posted the file doesn't come out right)
var file = $j(fileUpload.toString()).attr('files')[0];
var encoded = Btoa(file);
Also,
file.readAsArrayBuffer()
Seems to be only supported in HTML5?
Lots of people suggested: http://www.webtoolkit.info/javascript-base64.html
But this only returns an error on the UTF_8 method before it base64 encodes? (or an empty string)
var encoded = Base64.encode(file);
It's entirely possible in browser-side javascript.
The easy way:
The readAsDataURL() method might already encode it as base64 for you. You'll probably need to strip out the beginning stuff (up to the first ,), but that's no biggie. This would take all the fun out though.
The hard way:
If you want to try it the hard way (or it doesn't work), look at readAsArrayBuffer(). This will give you a Uint8Array and you can use the method specified. This is probably only useful if you want to mess with the data itself, such as manipulating image data or doing other voodoo magic before you upload.
There are two methods:
Convert to string and use the built-in btoa or similar
I haven't tested all cases, but works for me- just get the char-codes
Convert directly from a Uint8Array to base64
I recently implemented tar in the browser. As part of that process, I made my own direct Uint8Array->base64 implementation. I don't think you'll need that, but it's here if you want to take a look; it's pretty neat.
What I do now:
The code for converting to string from a Uint8Array is pretty simple (where buf is a Uint8Array):
function uint8ToString(buf) {
var i, length, out = '';
for (i = 0, length = buf.length; i < length; i += 1) {
out += String.fromCharCode(buf[i]);
}
return out;
}
From there, just do:
var base64 = btoa(uint8ToString(yourUint8Array));
Base64 will now be a base64-encoded string, and it should upload just peachy. Try this if you want to double check before pushing:
window.open("data:application/octet-stream;base64," + base64);
This will download it as a file.
Other info:
To get the data as a Uint8Array, look at the MDN docs:
https://developer.mozilla.org/en/DOM/FileReader
My solution was use readAsBinaryString() and btoa() on its result.
uploadFileToServer(event) {
var file = event.srcElement.files[0];
console.log(file);
var reader = new FileReader();
reader.readAsBinaryString(file);
reader.onload = function() {
console.log(btoa(reader.result));
};
reader.onerror = function() {
console.log('there are some problems');
};
}
I used FileReader to display image on click of the file upload button not using any Ajax requests. Following is the code hope it might help some one.
$(document).ready(function($) {
$.extend( true, jQuery.fn, {
imagePreview: function( options ){
var defaults = {};
if( options ){
$.extend( true, defaults, options );
}
$.each( this, function(){
var $this = $( this );
$this.bind( 'change', function( evt ){
var files = evt.target.files; // FileList object
// Loop through the FileList and render image files as thumbnails.
for (var i = 0, f; f = files[i]; i++) {
// Only process image files.
if (!f.type.match('image.*')) {
continue;
}
var reader = new FileReader();
// Closure to capture the file information.
reader.onload = (function(theFile) {
return function(e) {
// Render thumbnail.
$('#imageURL').attr('src',e.target.result);
};
})(f);
// Read in the image file as a data URL.
reader.readAsDataURL(f);
}
});
});
}
});
$( '#fileinput' ).imagePreview();
});
Inspired by #Josef's answer:
const fileToBase64 = async (file) =>
new Promise((resolve, reject) => {
const reader = new FileReader()
reader.readAsDataURL(file)
reader.onload = () => resolve(reader.result)
reader.onerror = (e) => reject(e)
})
const file = event.srcElement.files[0];
const imageStr = await fileToBase64(file)
Complete example
Html file input
<style>
.upload-button {
background-color: grey;
}
.upload-button input{
display:none;
}
</style>
<label for="upload-photo" class="upload-button">
Upload file
<input
type="file"
id="upload-photo"
</input>
</label>
JS Handler
document.getElementById("upload-photo").addEventListener("change", function({target}){
if (target.files && target.files.length) {
try {
const uploadedImageBase64 = await convertFileToBase64(target.files[0]);
//do something with above data string
} catch() {
//handle error
}
}
})
function convertFileToBase64(file) {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.readAsDataURL(file);
reader.onload = () => resolve(reader.result);
// Typescript users: use following line
// reader.onload = () => resolve(reader.result as string);
reader.onerror = reject;
});
}
After struggling with this myself, I've come to implement FileReader for browsers that support it (Chrome, Firefox and the as-yet unreleased Safari 6), and a PHP script that echos back POSTed file data as Base64-encoded data for the other browsers.
So why dont you agree with user of the system to select an image from a known folder? Or they can set their choice folder for images.
Most browsers wont support full path but you can get the filename eg "image.png"
Using PHP inbuilt function to encode:
#$picture_base64 = base64_encode( file_get_contents($image_file_name) );
The sign # will suppress error if path is not found but the result will be a null for variable $picture_base64 so i guess youre ok with null like i am else do a check for null before proceeding.
In html you can select an image filename to the input e.g. "image.png" ( but not the full path)
<input type="file" name="image" id="image" >
Then in PHP you can do:
$path = "C:\\users\\john\\Desktop\\images\\"
#$picture_base64 = base64_encode( file_get_contents( $path. $_POST['image']);
Then $picture_base64 will be something like
"AQAAAAMAAAAHAAAADwAAAB8AAAA/AAAAfwAAAP8AAAD/AQAA/w"
I've started to think that using the 'iframe' for Ajax style upload might be a much better choice for my situation until HTML5 comes full circle and I don't have to support legacy browsers in my app!

Categories

Resources