I am currently working on a piece of code that uploads mp4 videos from my localhost onto a server. The basic thing is that if the video is a .mp4 it is directly uploaded, else it is converted to .mp4 and then uploaded. I'm using the video converter using handbrake-js.
All works fine except for a tiny part. When the file isn't that huge, say less than 70-80 Mb, it works like a charm. But the problem is with larger files. Even though I am explicitly calling the res.end / res.send in the .on(end) callback, I receive some blank response in my angular controller, even before the conversion has finished. I have noticed it happens at around 30 to 40% of the conversion. It has a readystate which is equal to XMLHttpRequest.DONE and also status = 200.
Here is the Node side code:
try {
if (fs.existsSync(uploadPath + filename.substring(0, filename.lastIndexOf('.')) + '.mp4')) {
res.end('<b><i>' + filename + '</i></b> already exists in the directory.');
}
else {
const fstream = fs.createWriteStream(path.join(cfg.tempStoragePath, filename));
file.pipe(fstream);
console.log("\nfile_type: " + file_type);
filename = filename.substring(0, filename.lastIndexOf('.'));
// On finish of the copying file to temp location
fstream.on('close', () => {
hbjs.spawn({
input: cfg.tempStoragePath + filename + '.' + file_type,
output: uploadPath + filename + '.mp4'
})
.on('error', err => {
// invalid user input, no video found etc
console.log('error! No input video found at: \n: ' + cfg.tempStoragePath + filename + '.' + file_type);
res.send('Conversion of the file, <b><i>' + filename + '</i></b>, from <b>.' + file_type + '</b>' + ' to <b>.mp4</b> failed because no input video was found at: \n: ' + cfg.tempStoragePath + filename + '.' + file_type);
})
.on('progress', progress => {
progress_percent = (Number(progress.percentComplete) * 2 <= 100) ? Number(progress.percentComplete) * 2 : 100;
eta = progress.eta.split(/[a-zA-Z]/);
minutes = ((+eta[0]) * 60 + (+eta[1])) / 2;
console.log('Percent complete: %d, ETA: %d ///// %s ==> mp4', progress_percent, minutes, file_type);
})
.on('end', end => {
console.log('Conversion from .' + file_type + ' to .mp4 complete.');
//delete the temp file
fs.unlink(cfg.tempStoragePath + filename + '.' + file_type);
let new_path = uploadPath + filename + '.mp4';
let stat = fs.statSync(new_path);
console.log(`Upload of '${filename}' finished`);
if(Number(progress_percent) === Number(100))
res.send('The file, <b><i>' + filename + '</i></b>, has been converted from <b>.' + file_type + '</b>' + ' to <b>.mp4</b> complete.');
})
});
}
}
catch (err) {
res.end(err);
}
Following is part of my angular controller:
request = new XMLHttpRequest();
request.onreadystatechange = function () {
if (request.readyState === XMLHttpRequest.DONE && request.status === 200) {
showConversionModal('<p>' + request.responseText + '</p>', 'done');
}
};
showSomeModal('something');
request.open("POST", client.clientHost + ":" + client.clientPort + "/uploadVideoService");
formData = new FormData();
formData.append("file", files[0], files[0].name);
request.send(formData);
NOTE : All the data which is do a console.log() inside the node is as expected. And even the res.end/send works fine for smaller files which take lesser time. But the problem arises only for those which conversion takes longer than the smaller files.
Also, the if loop which checks for existing file scenario doesn't work as expected for these larger files. I thought at least that should work, because it doesn't even get into the handbrake part. But that is not the case.
And in the browser I get this error:
Failed to load resource: net::ERR_CONNECTION_RESET
which points to the request.send(formData); line, and I have also tried almost all solution from this SO article, with no good effect. But still, the conversion happens fine.
P.S.: Also note that the conversion and upload happens without a problem even for the larger files, it's just the response I'm receiving on the client side that has been my headache.
UPDATE: I tried using the debugger in VS Code, and saw that the breakpoints are rightly hitting the res.end() inside the if loop which checks for existing file scenario, but for some strange reason, the angular controller isn't reacting to it. And this happens only for the larger files.
I figured it out myself after a long time. Turns out there's something like a buffer in Busboy.
Here, I used to link the busboy and express like this:
app.use(busboy());
Just setting the field called highWaterMark to an arbitrary size which would be greater than the file size did the trick.
app.use(busboy({
highWaterMark: 2048 * 1024 * 1024, // Set buffer size in MB
}));
I don't know what might have caused this problem, or what the new field does that solves it, but, it just works is all that I know. It would be helpful if someone could elaborate on this a bit.
Related
Similar to this question,
I have a script that downloads a file to a given url via http.get.
How can I make sure the pipe is finished before continuing to the next iteration with just the http/https module??
//nodejs default libs
var fs = require("fs");
var http = require('https');
function dlFile(fullFilePath, dlUrl, fsize, fname){
var file = fs.createWriteStream(fullFilePath); //fullFilePath will dictate where we will save the file + filename.
var rsult ='';
var downloadedFsize;
var stats; //stats of the file will be included here
var request = http.get( dlUrl, function(response) {
let rsult = response.statusCode;
//will respond with a 200 if the file is present
//404 if file is missing
response.pipe(file);
/*pipe writes the file...
how do we stop the iteration while it is not yet finished writing?
*/
console.log(" \n FILE : " + fname);
console.log("File analysis finished : statusCode: " + rsult + " || Saved on " + fullFilePath);
console.log(' \n Downloaded from :' + dlUrl);
console.log(' \n SQL File size is : ' + fsize);
//identify filesize
stats = fs.statSync(fullFilePath);
downloadedFsize = stats["size"]; //0 because the pipe isn't finished yet...
console.log(' actual file size is : ' + downloadedFsize);
}).on('error', function(e) {
console.error(e);
//log that an error happened to the file
}).on('end', function(e){
//tried putting the above script here but nothing happens
});
return rsult;
}
Is there a cleaner approach similar to what I have in mind above? or should I approach this differently? I tried putting the code on .on('end' but it does nothing
The end event is not triggered on the request, instead it is triggered on the response (docs):
response.on("end", function() {
console.log("done");
});
As #Jonas Wilms says, the trigger was indeed on response.
//nodejs default libs
var fs = require("fs");
var http = require('https');
function dlFile(fullFilePath, dlUrl, fsize, fname){
var file = fs.createWriteStream(fullFilePath); //fullFilePath will dictate where we will save the file + filename.
var rsult ='';
var downloadedFsize;
var stats; //stats of the file will be included here
var request = http.get( dlUrl, function(response) {
let rsult = response.statusCode;
//will respond with a 200 if the file is present
//404 if file is missing
response.pipe(file).on('finish', function(e){
console.log(" \n FILE : " + fname);
console.log("File analysis finished : statusCode: " + rsult + " || Saved on " + fullFilePath);
console.log(' \n Downloaded from :' + dlUrl);
console.log(' \n SQL File size is : ' + fsize);
//identify filesize
stats = fs.statSync(fullFilePath);
downloadedFsize = stats["size"];
console.log(' actual file size is : ' + downloadedFsize);
});
/*pipe writes the file above, and output the results once it's done */
}).on('error', function(e) {
console.error(e);
//log that an error happened to the file
}).on('end', function(e){
//tried putting the above script here but nothing happens
});
return rsult;
}
I am converting all .png files to .jpg files in a directory and then running some manipulations on them which can be only to to jpeg files. But node.js doesn't seem to notice the converted files and the deleted png files until I run the same script again.
const fs = require('fs')
const pngToJpeg = require('png-to-jpeg');
let dirrCont = fs.readdirSync( dir );
files = dirrCont.filter( ( elm ) => /.*\.(png|jpg)/gi.test(elm) );
for (i in files)
{
let file = files[i]
let file_name_without_ext = file.replace(/\.[^/.]+$/, "")
let extension = file.match(/\.[^/.]+$/)
if (extension[0] == '.png')
{
console.log('found')
let buffer = fs.readFileSync(dir+file);
pngToJpeg({quality: 100})(buffer)
.then(output => fs.writeFileSync(dir+file_name_without_ext+'.jpg', output));
fs.unlinkSync(dir+file)
extension = '.jpg'
}
let target_file = target + file_name_without_ext + '.' + suffix + extension
// do some manipulations on dir+file_name_without_ext+extension
I always receive the error that the new jpg files are not found thus the manipulations don't work although the png files get converted to jpg files. When I run the same script again since now all the files are jpeg the file manipulations run this time.
EDIT
as suggested in one of the answers by #CertainPerformance
I changed the code to do most of my stuff inside the then block but again hit the same error
for (i in files)
{
let file = files[i]
let file_name_without_ext = file.replace(/\.[^/.]+$/, "")
let extension = file.match(/\.[^/.]+$/)
if (extension[0] == '.png')
{
console.log('found')
let buffer = fs.readFileSync(dir+file);
pngToJpeg({quality: 100})(buffer)
.then(output => {
fs.writeFileSync(dir+file_name_without_ext+'.jpg', output);
extension = '.jpg'
//
let target_file = target + file_name_without_ext + '.' + suffix + extension
// Do some manipulations
// I am done with the manipulations and I now want to delete
// the jpg file I just created
fs.unlinkSync(dir+file_name_without_ext+'.jpg') // Gives me back the same error
});
}
NOTE: There is a little bit of change up in the edit and I am deleting the jpg file instead of the png file (which I was doing originally)
As you can see by the .then, pngToJpeg is asynchronous - if you want to do work on dir+file_name_without_ext, you have to wait for the initial .then and the writeFileSync to resolve first. Put everything that depends on the asynchronous operations inside the then. For example:
if (extension[0] == '.png') {
console.log('found');
const buffer = fs.readFileSync(dir + file);
pngToJpeg({ quality: 100 })(buffer)
.then(output => {
fs.unlinkSync(dir + file);
fs.writeFileSync(dir + file_name_without_ext + '.jpg', output);
extension = '.jpg';
const target_file = target + file_name_without_ext + '.' + suffix + extension;
// do some manipulations on dir+file_name_without_ext+extension
});
}
(You should also take care not to implicitly create global variables. For example, use for (const i in files) instead, or perhaps for (const file of files) to avoid having to fiddle with the unimportant indicies)
A bit of a pickle here. I've got a Google Cloud Storage bucket set up, and a webpage that, probably about 60% of the time, DOES upload a file to the bucket. However, sometimes it doesn't and I don't really know why.
I'm trying this from a mobile handset so it could be the signal. As such, I guess one thing I need to think about is making it resumable (the files themselves aren't big - less than 2mb, but the service we're getting isn't great).
So, how could I make this into a resumable (rather than multipart) upload? And do you think this would solve the issue?
Code below:
function insertObject(event) {
try {
var fileData = event.target.files[0];
}
catch(e) {
return;
}
var boundary = '-------314159265358979323846';
var delimiter = "\r\n--" + boundary + "\r\n";
var close_delim = "\r\n--" + boundary + "--";
var reader = new FileReader();
reader.readAsBinaryString(fileData);
reader.onload = function(e) {
var contentType = fileData.type || 'application/octet-stream';
var metadata = {
'name': fileData.name,
'mimeType': contentType
};
var base64Data = btoa(reader.result);
var multipartRequestBody =
delimiter +
'Content-Type: application/json\r\n\r\n' +
JSON.stringify(metadata) +
delimiter +
'Content-Type: ' + contentType + '\r\n' +
'Content-Transfer-Encoding: base64\r\n' +
'\r\n' +
base64Data +
close_delim;
//Note: gapi.client.storage.objects.insert() can only insert
//small objects (under 64k) so to support larger file sizes
//we're using the generic HTTP request method gapi.client.request()
var request = gapi.client.request({
'path': '/upload/storage/' + API_VERSION + '/b/' + BUCKET + '/o',
'method': 'POST',
'params': {'uploadType': 'multipart'},
'headers': {
'Content-Type': 'multipart/mixed; boundary="' + boundary + '"'
},
'body': multipartRequestBody});
//Remove the current API result entry in the main-content div
//listChildren = document.getElementById('main-content').childNodes;
//if (listChildren.length > 1) {
// listChildren[1].parentNode.removeChild(listChildren[1]);
//}
//look at http://stackoverflow.com/questions/30317797/uploading-additional-metadata-as-part-of-file-upload-request-to-google-cloud-sto
try{
//Execute the insert object request
//document.getElementById("authorize-button").click();
executeRequest(request, 'insertObject');
//Store the name of the inserted object
document.getElementById("btnUpload").click();
object = fileData.name;
}
catch(e) {
alert('An error has occurred: ' + e.message);
}
}
}
EDIT - just tried it from my PC and it's uploaded 17 files with no problems - normally it just starts dropping them straight away from the mobile handset. So I think it is a signal issue and making it resumable would do the job.
Well, this is embarrassing.
I decided to check that the code was uploading the file correctly, and got rid of the
document.getElementById("btnUpload").click();
object = fileData.name;
in the final element of the code (the btnUpload does some C# stuff that I do need, but I don't need the object at all). I tried 20 pictures from my mobile handset and, when it would have usually only done about 12, it did all 20 - and a lot more effectively.
So I think the issue is the btnUpload.click that is taking control of the process away from the file transfer before it's finished. I've just got to find a way to incorporate that back into the code without interfering with things.
Github link
And just moved the btnUpload.click() from being within the insertobject to the executeRequest (just after where it calls the log). There just seems to be a small problem now, but not with the file upload. The btnUpload stuff gets called where it should, and the rest is simply sorting out the webpage.
Edit - as an aside, in order to cope with the request length being long (for files which are 4mb or more), I added this to the web.config (in the system.web):
<httpRuntime maxRequestLength="214748364"/>
The script below contains some URLs in "links" array. The function gatherLinks() is used to gather more URLs from sitemap.xml of the URLs in "links" array. Once the "links" array has enough URLs (decided by variable "limit"), function request() is called for each URL in "links" array to send a request to the server and fetch the response. Time taken for each response is reported. Total time taken by the program is reported when the program ends.
I wrote a PhantomJS program (source below) to send some requests and calculate the time taken (in order to compare the performance of 2.0.0 and 1.9.8). I get links using sitemap.xml file of the sites I hardcode in "links" array.
When run using PhantomJS 2.0.0, after some 65 requests the program (method page.open() of request function) starts outputting the following:
select: Invalid argument
select: Invalid argument
select: Invalid argument
select: Invalid argument
select: Invalid argument
.
.
.
.
When run using PhantomJS 1.9.8, it crashes after about 200 requests with the following error.
"PhantomJS has crashed. Please read the crash reporting guide at https://github.com/ariya/phantomjs/wiki/Crash-Reporting and file a bug report at https://github.com/ariya/phantomjs/issues/new with the crash dump file attached: /tmp/2A011800-3367-4B4A-A945-3B532B4D9B0F.dmp"
I tried to send the crash report but their guide is not very useful for me.
It's not the urls that I use, I have tried using other urls but same results.
Is there something wrong with my program? I am using OSX.
var system = require('system');
var fs = require('fs');
var links = [];
links = [
"http://somesite.com",
"http://someothersite.com",
.
.
.
];
var index = 0, fail = 0, limit = 300;
finalTime = Date.now();
var gatherLinks = function(link){
var page = require('webpage').create();
link = link + "/sitemap.xml";
console.log("Fetching links from " + link);
page.open(link, function(status){
if(status != "success"){
console.log("Sitemap Request FAILED, status: " + status);
fail++;
return;
}
var content = page.content;
parser = new DOMParser();
xmlDoc = parser.parseFromString(content, 'text/xml');
var loc = xmlDoc.getElementsByTagName('loc');
for(var i = 0; i < loc.length; i++){
if(links.length < limit){
links[links.length] = loc[i].textContent;
} else{
console.log(links.length + " Links prepared. Starting requests.\n");
index = 0;
request();
return;
}
}
if(index >= links.length){
index = 0;
console.log(links.length + " Links prepared\n\n");
request();
}
gatherLinks(links[index++]);
});
};
var request = function(){
t = Date.now();
var page = require('webpage').create();
page.open(links[index], function(status) {
console.log('Loading link #' + (index + 1) + ': ' + links[index]);
console.log("Time taken: " + (Date.now() - t) + " msecs");
if(status != "success"){
console.log("Request FAILED, status: " + status);
fail++;
}
if(index >= links.length-1){
console.log("\n\nAll links done, final time taken: " + (Date.now() - finalTime) + " msecs");
console.log("Requests sent: " + links.length + ", Failures: " + fail);
console.log("Success ratio: " + ((links.length - fail)/links.length)*100 + "%");
phantom.exit();
}
index++;
request();
});
}
gatherLinks(links[0]);
After playing around with the program, I couldn't find any particular pattern to the problems I mention below. For 2.0.0, I could only once succeed in sending 300 requests without an error. I have tried all different combinations of URLs, program usually fails between request 50-80. I maintain a log of urls that failed, all of them run fine when I send a single request using another PhantomJS program. For 1.9.8, it's much more stable and the crash I mention below is not very frequent. But again, I couldn't find any pattern to the crashing, it still crashes once in a while.
There are lots of problems with your code. The main one is probably that you're creating a new page for every single request and never close it afterwards. I think you're running out of memory.
I don't see a reason to create a new page for every request, so you can easily reuse a single page for all requests. Simply move the line var page = require('webpage').create(); to the global scope out of gatherLinks() and request(). If you don't want to do that, then you can call page.close() after you're done with it, but keep the asynchronous nature of PhantomJS in mind.
If the reason to use multiple page objects was to prevent cache re-use for later requests, then I have to tell you that this doesn't solve that problem. page objects in a single PhantomJS process can be regarded as tabs or windows and they share cookies and cache. If you want to isolate every request, then you will need to run every request in its own process for example through the use of the Child Process Module.
There is another problem with your code. You probably wanted to write the following in gatherLinks():
if(index >= links.length){
index = 0;
console.log(links.length + " Links prepared\n\n");
request();
return; // ##### THIS #####
}
gatherLinks(links[index++]);
I have the following nodejs code which uploads a file by calling a server-side API (written by me) and passing the file content as a multi-part request. The problem is that my code works perfectly with small files but it fails with large files (1 MB or above). I'm pretty sure it's a problem in my code but I'm not able to find out what it is.
// assume file content have been read into post_data array
//Make post
var google = http.createClient(443, host, secure = true);
var filepath = '/v2_0/put_file/';
var GMTdate = (new Date()).toGMTString();
var fileName = encodeURIComponent(destination);
console.log("fileName : " + fileName);
console.log("Path : " + filepath);
var header = {
'Host': host,
'Authorization': 'Basic ' + authStr,
'Content-Type': 'multipart/form-data; boundary=0xLhTaLbOkNdArZ',
'Last-Modified': GMTdate,
'Filename': fileName,
'Last-Access-By': username
};
var request = google.request('POST', filepath, header);
for (var i = 0; i < post_data.length; i++) {
request.write(post_data[i]);
}
request.end();
request.addListener('response', function(response){
var noBytest = 0;
response.setEncoding('utf8');
console.log('STATUS: ' + response);
console.log('STATUS: ' + response.statusCode);
console.log('HEADERS: ' + JSON.stringify(response.headers));
console.log('File Size: ' + response.headers['content-length'] + " bytes.");
From the logs, I see that control comes to request.end(); but I do not see the last few logs written after request.addListener() block.
I've been pulling my hair off for last couple of days trying to understand why it works for small files but not for larger files. I don't see any timeouts and the code just seems to be hung till I kill it off.
Can anyone guide me as to what am I doing wrong?
UPDATE:
post_data is an array, here is what I'm doing
post_data = [];
console.log('ContentType =' + ContentType + "\n\nEncoding Style =" + encodingStyle);
post_data.push(new Buffer(EncodeFilePart(boundary, ContentType, 'theFile', FileNameOnly), 'ascii'));
var file_contents = '';
var file_reader = fs.createReadStream(filename, {
encoding: encodingStyle
});
file_reader.on('data', function(data){
console.log('in data');
file_contents += data;
});
file_reader.on('end', function(){
post_data.push(new Buffer(file_contents, encodingStyle))
post_data.push(new Buffer("\r\n--" + boundary + "--\r\n", 'ascii'));
...
var request = google.request('POST', filepath, header);
for (var i = 0; i < post_data.length; i++) {
request.write(post_data[i]);
}
I look forward to your suggestions.
You should be passing either an array or a string to request.write . Is post_data an array of strings, or an array of arrays?
Also, you are posting it as multipart/form-data, so that means you have to modify your data to that format. Have you done so, or is post_data just the raw data from a file?
checkout node-formidable and this post http://debuggable.com/posts/parsing-file-uploads-at-500-mb-s-with-node-js:4c03862e-351c-4faa-bb67-4365cbdd56cb