Node.js formidable file upload working slow on server - javascript

I am trying to send files as form data along with some fields using http post request in angular.js and receiving file in app.post in node.js. The file sending works fine on localhost. As they say formidable uploads files at 500 mb/sec speed but on server when I am trying to send a file of 5 to 10 mb it takes 40 to 80 seconds. Please check is there any problem in my implementation.
I am using nginx and pm2 on server.
Node.js code:
// route for uploading audio asynchronously
app.post('/v1/uploadAudio', function(req, res) {
var userName, useravatar, hasfile, ismusicfile, isType, showMe, DWimgsrc, DWid, msgtime;
var imgdatetimenow = Date.now();
var form = new formidable.IncomingForm({
uploadDir: __dirname + '/public/app/upload/music',
keepExtensions: true
});
form.on('end', function() {
res.end();
});
form.parse(req, function(err, fields, files) {
console.log("files : ", files);
console.log("fields : ", fields);
var data = {
username: fields.username,
userAvatar: fields.userAvatar,
repeatMsg: true,
hasFile: fields.hasFile,
isMusicFile: fields.isMusicFile,
istype: fields.istype,
showme: fields.showme,
dwimgsrc: fields.dwimgsrc,
dwid: fields.dwid,
serverfilename: baseName(files.file.path),
msgTime: fields.msgTime,
filename: files.file.name,
size: bytesToSize(files.file.size)
};
var audio_file = {
dwid: fields.dwid,
filename: files.file.name,
filetype: fields.istype,
serverfilename: baseName(files.file.path),
serverfilepath: files.file.path,
expirytime: imgdatetimenow + (120000)
};
files_array.push(audio_file);
ios.sockets.emit('new message music', data);
});
});
AngularJS code:
// =========================================== Audio Sending Code =====================
$scope.$watch('musicFiles', function() {
$scope.sendAudio($scope.musicFiles);
});
// opens the sent music file on music_icon click on new window
$scope.openClickMusic = function(msg) {
$http.post($rootScope.baseUrl + "/v1/getfile", msg).success(function(response) {
if (!response.isExpired) {
window.open($rootScope.baseUrl + '/' + response.serverfilename, "_blank");
} else {
var html = '<p id="alert">' + response.expmsg + '</p>';
if ($(".chat-box").has("p").length < 1) {
$(html).hide().prependTo(".chat-box").fadeIn(1500);
$('#alert').delay(1000).fadeOut('slow', function() {
$('#alert').remove();
});
}
}
});
}
// recieving new music message
$socket.on("new message music", function(data) {
if (data.username == $rootScope.username) {
data.ownMsg = true;
data.dwimgsrc = "app/images/spin.gif";
} else {
data.ownMsg = false;
}
if ((data.username == $rootScope.username) && data.repeatMsg) {
checkMessegesMusic(data);
} else {
$scope.messeges.push(data);
}
});
// replacing spinning wheel in sender message after music message delivered to everyone.
function checkMessegesMusic(msg) {
for (var i = ($scope.messeges.length - 1); i >= 0; i--) {
if ($scope.messeges[i].hasFile) {
if ($scope.messeges[i].istype === "music") {
if ($scope.messeges[i].dwid === msg.dwid) {
$scope.messeges[i].showme = true;
$scope.messeges[i].serverfilename = msg.serverfilename;
$scope.messeges[i].filename = msg.filename;
$scope.messeges[i].size = msg.size;
$scope.messeges[i].dwimgsrc = "app/images/musicplay_icon.png";
break;
}
}
}
};
}
// download music file if it exists on server else return error message
$scope.downloadMusic = function(ev, elem) {
var search_id = elem.id;
for (var i = ($scope.messeges.length - 1); i >= 0; i--) {
if ($scope.messeges[i].hasFile) {
if ($scope.messeges[i].istype === "music") {
if ($scope.messeges[i].dwid === search_id) {
$http.post($rootScope.baseUrl + "/v1/getfile", $scope.messeges[i]).success(function(response) {
if (!response.isExpired) {
var linkID = "#" + search_id + "A";
$(linkID).find('i').click();
return true;
} else {
var html = '<p id="alert">' + response.expmsg + '</p>';
if ($(".chat-box").has("p").length < 1) {
$(html).hide().prependTo(".chat-box").fadeIn(1500);
$('#alert').delay(1000).fadeOut('slow', function() {
$('#alert').remove();
});
}
return false;
}
});
break;
}
}
}
};
}
// validate file type to 'music file' function
$scope.validateMP3 = function(file) {
if (file.type == "audio/mp3" || file.type == "audio/mpeg") {
return true;
} else {
var html = '<p id="alert">Select MP3.</p>';
if ($(".chat-box").has("p").length < 1) {
$(html).hide().prependTo(".chat-box").fadeIn(1500);
$('#alert').delay(1000).fadeOut('slow', function() {
$('#alert').remove();
});
}
return false;
}
}
// sending new 'music file' function
$scope.sendAudio = function(files) {
if (files && files.length) {
$scope.isFileSelected = true;
for (var i = 0; i < files.length; i++) {
var file = files[i];
var dateString = formatAMPM(new Date());
var DWid = $rootScope.username + "dwid" + Date.now();
var audio = {
username: $rootScope.username,
userAvatar: $rootScope.userAvatar,
hasFile: $scope.isFileSelected,
isMusicFile: true,
istype: "music",
showme: false,
dwimgsrc: "app/images/musicplay_icon.png",
dwid: DWid,
msgTime: dateString
}
$socket.emit('send-message', audio, function(data) { // sending new image message via socket
});
var fd = new FormData();
fd.append('file', file);
fd.append('username', $rootScope.username);
fd.append('userAvatar', $rootScope.userAvatar);
fd.append('hasFile', $scope.isFileSelected);
fd.append('isMusicFile', true);
fd.append('istype', "music");
fd.append('showme', false);
fd.append('dwimgsrc', "app/images/musicplay_icon.png");
fd.append('dwid', DWid);
fd.append('msgTime', dateString);
fd.append('filename', file.name);
$http.post('/v1/uploadAudio', fd, {
transformRequest: angular.identity,
headers: {
'Content-Type': undefined
}
}).then(function(response) {
// console.log(response);
});
}
}
};

I've used Formidable on a couple of side projects, and when uploading to localhost, I do see the 500mb/sec quoted capability (depending on physical hardware of the computer).
However, when uploading a file over the internet, you are subject to the bandwidth limitations of your ISP upload speed as well as the download speed of your server.
You report that a 10MB file takes ~80 seconds to upload to the server. That's about 125KBps (or around 1 megabit/second) which seems fairly reasonable for a home/office ISP upload speed (depending on region of the world).
A good way to eliminate your home/office network performance from the troubleshooting equation would be to write a node.js script that uploads a file several times and calculates an average speed. Run that test file on your local computer, then try again from a different server in the cloud.

Related

XMPP File Transfer via JavaScript Strophe.js in Openfire

I am trying to get Strophe.js based XMPP file transfer to work. I can get logged in to work on my openfire server. I can send messages and receive messages fine but I am having trouble with file transfer.
HTML:
<form name='file_form' class="panel-body">
<input type="file" id="file" name="file[]" />
<input type='button' id='btnSendFile' value='sendFile' />
<output id="list"></output>
</form>
Javascript file:
// file
var sid = null;
var chunksize;
var data;
var file = null;
var aFileParts, mimeFile, fileName;
function sendFileClick() {
file =$("#file")[0].files[0];
sendFile(file);
readAll(file, function(data) {
log("handleFileSelect:");
log(" >data="+data);
log(" >data.len="+data.length);
});
}
function sendFile(file) {
var to = $('#to').get(0).value;
var filename = file.name;
var filesize = file.size;
var mime = file.type;
chunksize = filesize;
sid = connection._proto.sid;
log('sendFile: to=' + to);
// send a stream initiation
connection.si_filetransfer.send(to, sid, filename, filesize, mime, function(err) {
fileTransferHandler(file, err);
});
}
function fileTransferHandler(file, err) {
log("fileTransferHandler: err=" + err);
if (err) {
return console.log(err);
}
var to = $('#to').get(0).value;
chunksize = file.size;
chunksize = 20 * 1024;
// successfully initiated the transfer, now open the band
connection.ibb.open(to, sid, chunksize, function(err) {
log("ibb.open: err=" + err);
if (err) {
return console.log(err);
}
readChunks(file, function(data, seq) {
sendData(to, seq, data);
});
});
}
function readAll(file, cb) {
var reader = new FileReader();
// If we use onloadend, we need to check the readyState.
reader.onloadend = function(evt) {
if (evt.target.readyState == FileReader.DONE) { // DONE == 2
cb(evt.target.result);
}
};
reader.readAsDataURL(file);
}
function readChunks(file, callback) {
var fileSize = file.size;
var chunkSize = 20 * 1024; // bytes
var offset = 0;
var block = null;
var seq = 0;
var foo = function(evt) {
if (evt.target.error === null) {
offset += chunkSize; //evt.target.result.length;
seq++;
callback(evt.target.result, seq); // callback for handling read chunk
} else {
console.log("Read error: " + evt.target.error);
return;
}
if (offset >= fileSize) {
console.log("Done reading file");
return;
}
block(offset, chunkSize, file);
}
block = function(_offset, length, _file) {
log("_block: length=" + length + ", _offset=" + _offset);
var r = new FileReader();
var blob = _file.slice(_offset, length + _offset);
r.onload = foo;
r.readAsDataURL(blob);
}
block(offset, chunkSize, file);
}
function sendData(to, seq, data) {
// stream is open, start sending chunks of data
connection.ibb.data(to, sid, seq, data, function(err) {
log("ibb.data: err=" + err);
if (err) {
return console.log(err);
}
// ... repeat calling data
// keep sending until you're ready you've reached the end of the file
connection.ibb.close(to, sid, function(err) {
log("ibb.close: err=" + err);
if (err) {
return console.log(err);
}
// done
});
});
}
$('#btnSendFile').bind('click', function() {
console.log('File clicked:');
sendFileClick();
});
Full code is based on:
Complete example of Strophe.js file transfer
http://plnkr.co/edit/fYpXo1mFRWPxrLlgr123 (source can be download here: has errors). I changed the sendFileClick function.
I am getting:
ibb.open: err=Error: feature-not-implemented? Why is this error I am getting?

video tag makes call for one byte of data only on mobile (Node.js)

I have created a node server that streams mp4 videos from a Mongo database using gridfs. I have no issue when streaming the video to a desktop browser but when I try to stream to any mobile device I only see the video player but it will not play. When I check my server logs I see that the desktop version makes a call for 0-total bytes of the video but on mobile it only makes a call for bytes 0-1 and then stops, no errors given.
These are the two server side functions which work great when streaming to a pc browser.
function StreamGridFile(req, res, GridFile) {
console.log("Request: " + req);
if (req.headers['range']) {
// Range request, partialle stream the file
console.log('Range Request');
var parts = req.headers['range'].replace(/bytes=/, "").split("-");
var partialstart = parts[0];
var partialend = parts[1];
var start = parseInt(partialstart, 10);
var end = partialend ? parseInt(partialend, 10) : GridFile.length - 1;
var chunksize = (end - start) + 1;
console.log('Range ', start, '-', end);
res.writeHead(206, {
'Content-Range': 'bytes ' + start + '-' + end + '/' + GridFile.length,
'Accept-Ranges': 'bytes',
'Content-Length': chunksize,
'Content-Type': GridFile.contentType
});
// Set filepointer
GridFile.seek(start, function() {
// get GridFile stream
var stream = GridFile.stream();
// write to response
stream.on('data', function(buff) {
// count data to abort streaming if range-end is reached
// perhaps theres a better way?
start += buff.length;
if (start >= end) {
// enough data send, abort
GridFile.close();
res.end();
} else {
res.write(buff);
}
});
stream.on("end", function() {
// Record the end was called
GridFile.close()
db.close(function() {
counter--;
console.log("closed db, counter: " + counter);
});
});
});
} else {
// stream back whole file
console.log('No Range Request');
var stream = GridFile.stream();
stream.pipe(res);
stream.on("end", function() {
// Record the end was called
gridfile.close();
db.close(function() {
counter--;
//console.log("closed db, counter: " + counter);
});
});
}
}
and here is the second
exports.streamVideo = function(req, res, id) {
console.log("entered the stream video function");
if (counter == 0) {
console.log("counter = 0");
db.open(function(err, db) {
console.log("db open for video stream");
counter++;
console.log(counter);
if (!err) {
var gs = GridStore(db, ObjectID(id));
gs.open(function(err, GridFile) {
console.log("gridstore open");
if (err) {
res.writeHead(404, {
'Content-Type': 'text/javascript'
});
res.end("Video not found");
gs.close();
db.close();
return;
}
console.log("no error, entering StreamGridFile function");
StreamGridFile(req, res, GridFile);
});
} else {
res.writeHead(404, {
'Content-Type': 'text/javascript'
});
res.end("database error");
gs.close();
db.close();
}
})
}
}
I am pretty sure the error falls into this code because I had initially streamed the video from a static file but later converted to using mongo and that was when it stopped working.
Give this a shot:
function StreamGridFile (req, res, GridFile) {
console.log("Request: " + req);
if (req.headers['range']) {
// Range request, partialle stream the file
console.log('Range Request');
var parts = req.headers['range'].replace(/bytes=/, "").split("-");
var maxIdx = GridFile.length - 1;
var start = parseInt(parts[0], 10);
var end = parseInt(parts[1], 10);
if (isInvalidRange(start, end, maxIdx)) {
return res.writeHead(416, {
'Accept-Ranges': 'bytes',
'Content-Type': GridFile.contentType,
'Content-Range': 'bytes */' + GridFile.length,
}).end();
}
if (end > maxIdx || end !== end) {
end = maxIdx
}
var bytesToSend = end - start + 1;
res.writeHead(206, {
'Accept-Ranges': 'bytes',
'Content-Type': GridFile.contentType,
'Content-Range': 'bytes ' + start + '-' + end + '/' + GridFile.length,
'Content-Length': bytesToSend
});
// Set filepointer
GridFile.seek(start, function() {
// get GridFile stream
var stream = GridFile.stream(true);
// write to response
stream.on('data', function(buff) {
if (buff.length > bytesToSend) {
buff = buff.slice(0, bytesToSend)
}
res.write(buff);
bytesToSend -= buff.length;
if (bytesToSend <= 0) {
// enough data send, abort
GridFile.close();
db.close(function() {
counter--;
console.log("closed db, counter: " + counter);
});
res.end();
}
});
});
} else {
// stream back whole file
console.log('No Range Request');
res.header('Accept-Range', 'bytes');
res.header('Content-Type', GridFile.contentType);
var stream = GridFile.stream(true);
stream.pipe(res);
stream.on('end', function(){
GridFile.close();
db.close(function() {
counter--;
console.log("closed db, counter: " + counter);
});
});
}
}
function isInvalidRange (start, end, maxIdx) {
return start !== start // NaN
|| start < 0
|| end < start
|| start > maxIdx
}
This obviously is not able to handle a multi-byte-range request (where the Range header looks like bytes=2048-4095,4096-6143), but it should be able to handle normal byte-range requests.

Uploading video without 777 does not work

I am locally testing my node video upload. my upload class looks like this:
var videoExtensions = ['mp4', 'webm', 'mov'];
var audioExtensions = [];
//Media object
function Media(file, targetDirectory) {
this.file = file;
this.targetDir = targetDirectory;
}
Media.prototype.isVideo = function () {
return this.file.mimetype.indexOf('video') >= 0;
};
Media.prototype.isAudio = function () {
return this.file.mimetype.indexOf('audio') >= 0;
};
Media.prototype.getName = function () {
return this.file.originalname.substr(0, this.file.originalname.indexOf('.'))
};
router.route('/moduleUpload')
.post(function (request, response) {
var media = new Media(request.files.file, '../user_resources/module/' + request.body.module_id + '/');
if (!fs.existsSync(media.targetDir)) {
fs.mkdirSync(media.targetDir, 0777, function (err) {
if (err) {
console.log(err);
response.send("ERROR! Can't make the directory! \n"); // echo the result back
}
});
}
if (media.isVideo()) {
convertVideos(media);
}
else if (media.isAudio()) {
convertAudio(media);
}
else {
moveFile(media);
}
response.status(200).json('user_resources/module/' + request.body.module_id + '/' + media.getName());
});
router.route('/retrieveFile')
.post(function (request, response) {
var path = '../' + request.body.data;
var file = fs.createReadStream(path);
file.pipe(response);
});
function convertVideos(media) {
var ffmpeg = require('fluent-ffmpeg');
videoExtensions.forEach(function (extension) {
var proc = new ffmpeg({source: media.file.path, nolog: false})
.withVideoCodec('libx264')
.withVideoBitrate(800)
.withAudioCodec('libvo_aacenc')
.withAudioBitrate('128k')
.withAudioChannels(2)
.toFormat(extension)
.saveToFile(media.targetDir + media.getName() + '.' + extension,
function (retcode, error) {
console.log('file has been converted succesfully');
});
});
}
function convertAudio(media) {
var ffmpeg = require('fluent-ffmpeg');
audioExtensions.forEach(function (extension) {
var proc = new ffmpeg({source: media.file.path, nolog: false})
.withVideoCodec('libx264')
.withVideoBitrate(800)
.withAudioCodec('libvo_aacenc')
.withAudioBitrate('128k')
.withAudioChannels(2)
.toFormat(extension)
.saveToFile(media.targetDir + media.getName() + '.' + extension,
function (retcode, error) {
console.log('file has been converted succesfully');
});
});
}
When a video file is uploaded it is convert into 3 different files.
Now the file i wish to upload is in my /Video folder at first this did not have any permissions. Which resulted in the upload could not play. However as soon as i changed the permission of the file to 777 the video plays without a problem.
My question is why? am i missing something in my upload and is chmod 777 wise?
also note im using ubuntu 14.04

Downloading files stuck android application: worklight

i implemented a application on work-light in which I need to download files from the server, its works well on Iphone5 means files are downloading smoothly without stuck the application flow.however when i run the application on my Samsung galaxy s2 (V 4.1) and start downloading files using for loop my application get stuck till the downloading completed. however its works fine when i have only one file to dowload but when the count is above 3 or 4 application get stuck.
if(networkInfo.networkConnectionType=='WIFI'){
$(brandClassDis).addClass('ui-disabled'); // Disabling the Brand.
$(".lms_loadernew").css("display", "block");
var syncProgBar = "#syncProgressBar"+result[0].json.BrandID;
var syncProgLabel = "#syncLoadingLabel"+result[0].json.BrandID;
$(syncProgBar).progressbar({
value: 0,
}).show();
$(syncProgLabel).text(parseInt(0, 10)+"%").show();
localStorage.setItem("download"+result[0].json.BrandID,0);
localStorage.setItem("downloadSucc"+result[0].json.BrandID,0);
for(var i=0; i<result.length; i++){
var obj = {VideoID:result[i].json.VideoID,BrandID:result[i].json.BrandID,CourseID:result[i].json.CourseID,LoadingStatus:"0"};
VideosList.add(obj,{push:false});
result[i].json.IsDownload = 2;
Videos.replace(result);
**downloadFolder(result[i],result.length)**
}
}
function downloadFolder(result,numVideoBrand){
try{
var loaderPer = 0;
var courseId ="#sync_"+result.json.CourseID.replace(/ /g,'');
var courseLabelId ="#loadingLabel"+result.json.CourseID.replace(/ /g,'');
var syncProgBar = "#syncProgressBar"+result.json.BrandID.replace(/ /g,'');
var syncProgLabel = "#syncLoadingLabel"+result.json.BrandID.replace(/ /g,'');
var serverLoc = encodeURI(result.json.DownloadName);
window.requestFileSystem(LocalFileSystem.PERSISTENT, 0, function(fileSystem) {
fileSystem.root.getDirectory("LMS_APP", {create: true, exclusive: false}, function(directory){
var localPath = directory.fullPath+"/"+"Videos"+"/"+zipFileName;
var ft = new FileTransfer();
ft.download(serverLoc,localPath, function(entry) {
entry.file(function(file) {
WL.Logger.debug("File size: " + file.size);
if(file.size<=854){
downloadFail("",result,numVideoBrand);
entry.remove(successRemove, failRemove);
}else{
if(localStorage.getItem("download"+result.json.BrandID) == null || localStorage.getItem("download"+result.json.BrandID) =="" || localStorage.getItem("download"+result.json.BrandID) == undefined){
localStorage.setItem("download"+result.json.BrandID,1);
}else{
localStorage.setItem("download"+result.json.BrandID,(localStorage.getItem("download"+result.json.BrandID)-(-1)));
}
localStorage.setItem("downloadSucc"+result.json.BrandID,(localStorage.getItem("downloadSucc"+result.json.BrandID)-(-1)));
WL.Logger.debug("Folder is:---->"+directory.fullPath+"/"+zipFileName);
WL.Logger.debug("download"+localStorage.getItem("download"+result.json.BrandID)+"..."+numVideoBrand+"........"+ localStorage.getItem("downloadSucc"+result.json.BrandID));
var loadedVideoPer = (( localStorage.getItem("download"+result.json.BrandID)/numVideoBrand)* 100);
$(syncProgBar).progressbar({
value: loadedVideoPer,
});
$(syncProgLabel).text(parseInt(loadedVideoPer, 10)+"%");
$(courseId).hide();
$(courseLabelId).hide();
}
}, function(error){downloadFail(error,result,numVideoBrand);});
}, function(error) {
});
$(courseId).progressbar({
value: loaderPer,
}).show();
$(courseLabelId).text(parseInt(loaderPer, 10)+"%").show();
ft.onprogress = function(progressEvent) {
if (progressEvent.lengthComputable) {
loaderPer = ((progressEvent.loaded / progressEvent.total)*100);
$(courseId).progressbar({
value: loaderPer,
});
$(courseLabelId).text(parseInt(loaderPer, 10)+"%");
//courseLabelId.text(parseInt(loaderPer, 10) + "%" );
loadingStatus(result);
}
};
},function(error){
downloadFail(error,result,numVideoBrand);
});
}, function(error){
downloadFail(error,result,numVideoBrand);
});
}catch(e){
WL.Logger.debug("exp in downloadFile: "+e);
//alert("exp "+videoLoader);
}
}
From the comments, the solution was to:
download file in queue, coz downloading all file at once cause the
application stuck...

How to get all images from document and store to local

My goal is to get all images from document, then download all images bigger than 150x150px to local.
I'm stucked on retrieving files from URL i got on previous steps. Here is the buggy code line (full code - at the end):
...
var copyResult = fs.copy(imagesURLs[i], destFile);
...
When i run from console it just hangs up on fs.copy(), without any errors.
As i can understand, fs.copy() doesn't work with remote URLs, even if you set all proper args (--load-images=yes, --local-to-remote-url-access=yes). Am i right or there's something i did wrong with copy()? And are there any methods to get files directly from webkit's cache?
Got latest phantomjs version and ubuntu server.
I would be appreciate for any kind of help.
Full script code:
if (phantom.args.length < 1 || phantom.args.length > 2)
{
console.log('Usage: phantomjs ' + phantom.scriptName + ' <URL>');
phantom.exit();
}
else
{
var page = new WebPage(),
address = phantom.args[0];
page.viewportSize = { width: 1200, height: 4000 };
page.open(address, function (status)
{
if (status === 'success')
{
var imagesURLs = page.evaluate(function ()
{
var documentImages = [], imagesCount = document.images.length, index = 0;
while (index < imagesCount)
{
if ((document.images[index].width >= 150) && (document.images[index].height >= 150))
{
documentImages.push(document.images[index].src);
}
index++;
}
return documentImages;
});
var fs = require('fs');
for (var i in imagesURLs)
{
var fileName = imagesURLs[i].replace(/^.*[\\\/]/, '');
var destFile = '' + fs.workingDirectory + '/www/images/' + fileName;
console.log(destFile);
var copyResult = fs.copy(imagesURLs[i], destFile);
console.log(copyResult);
}
}
else
{
console.log('status: ' + status);
}
phantom.exit();
});
}
man try this.
function SaveAs(imgURL)
{
var oPop = window.open(imgURL,"","width=1, height=1, top=5000, left=5000");
for(;oPop.document.readyState != "complete"; )
{
if (oPop.document.readyState == "complete")break;
}
oPop.document.execCommand("SaveAs");
oPop.close();
}

Categories

Resources