I'm writing this web-page in angularJS where I want people to edit and store text and images. I've created a file uploading function that let's you upload files from the users computer. The problem is getting this file stored into mongoDB. I've read alot of examples on gridFS but none of them quite matched what I'm trying to do.
Here's my code:
web-server.js:
app.post('/uploadFile', function(req,res){
console.log("Retrieved:");
console.log(req.files);
var Grid = require('gridfs-stream');
var gfs = Grid(DB, mongoose.mongo);
// streaming to gridfs
var writestream = gfs.createWriteStream(req.files.file);
fs.createReadStream(req.files.file.path).pipe(writestream);
services.js:
function uploadFilesToServer(file){
var fd = new FormData();
fd.append("file", file);
var deferred = $q.defer();
console.log("trying to save:");
console.log(file);
$http({
method:"POST",
url: "uploadFile",
data: fd,
withCredentials: true,
headers: {'Content-Type': undefined },
transformRequest: angular.identity
}).success(function(data){
var returnValue = [true, file, data];
deferred.resolve(returnValue);
}).error(function(data){
var returnValue = [false, file, data];
deferred.resolve(returnValue);
});
return deferred.promise;
}
At the moment I'm not getting any error messages when I run the code, but neither is the images stored in the db.files or db.chunks. Any help is appreciated.
GridFS-stream usually stores it's data in db.fs.files/db.fs.chunks if not set by the user.
To change this, you'll have to add:
{
....
root: 'my_collection'
....
}
to gridfs-stream options.
From NPM docs.:
createWriteStream
To stream data to GridFS we call createWriteStream passing any options.
var writestream = gfs.createWriteStream([options]);
fs.createReadStream('/some/path').pipe(writestream);
Options may contain zero or more of the following options...
{
_id: '50e03d29edfdc00d34000001', // a MongoDb ObjectId
filename: 'my_file.txt', // a filename
mode: 'w', // default value: w+, possible options: w, w+ or r,
see [GridStore]
(http://mongodb.github.com/node-mongodb-native/api-generated/gridstore.html)
//any other options from the GridStore may be passed too, e.g.:
chunkSize: 1024,
content_type: 'plain/text',
// For content_type to work properly, set "mode"-option to "w" too!
root: 'my_collection',
metadata: {
...
}
}
See https://www.npmjs.org/package/gridfs-stream for more.
Related
I'm trying to sent an audio blob on some Google drive folder. To succeed I translate blob in file before sending it.
I received since the starting an error :
Error: File not found.
code: 404, errors: [ { domain: 'global',
reason: 'notFound',
message: 'File not found: 1aazd544z3FOCAsdOA5E7XcOaS3297sU.',
locationType: 'parameter',
location: 'fileId' } ] }
progressive edit : So far I have converted my audio blob in base64 string in order to ease the processing of my blob.
But, I fail always to write a file with my base 64 audio blob :
Here my driveApi.js :
// request data from req.body
var data = req.body.data ; // data variable is presented in form of base64 string
var name = req.body.word ;
(...)
// WRITE FILE AND STORE IT IN BODY HEADER PROPERTY
body: fs.writeFile((name + ".mp3"), data.substr(data.indexOf(',')+1), {encoding: 'base64'}, function(err) {
console.log('File created')
})
Three steps: create a temporary file with your base64 data out of the drive.files.create function, then give this file a specific name -e.g. tempFile, also you can customize this name with a time value. After that, pass this file on a "fs.createReadStream" method to upload it on Google drive.
Some hints:
Firstly - use path.join(__dirname, name + "-" + Date.now() +".ext" ) to create to file name
Secondly - make this process asynchronously to avoid data flow conflict (trying to create file before file is created), so call the drive.files.create after having setting a fs.writeFile function.
Thirdly - Destroy the tempFile after the operation has been done. It allows you to automatize the process.
I let you dive in the methods you need. But basically fs should do the job.
Again, be careful on the data flow and use callback to control it. Your code can crash just because the function gone up in a no-operational way.
Some links :
https://nodejs.org/api/path.html
https://nodejs.org/api/fs.html#fs_fs_writefile_file_data_options_callback
here an instance :
// datavalue = some time value
fs.writeFile(
path.join(__dirname, name + "-" + datevalues +".mp3" ),
data.substr(data.indexOf(',')+1),
{encoding: 'base64'},
// callback
function(err) {
if(err){ console.log("error writting file : " + err)}
console.log('File created')
console.log("WRITTING") // control data flow
fileCreate(name)
})
function fileCreate (name){
// upload file in specific folder
var folderId = "someID";
var fileMetadata = {
'name': name + ".mp3" ,
parents: [folderId]
}; console.log("MEDIA") // control data flow
var media = {
mimeType: 'audio/mp3',
body: fs.createReadStream(path.join(__dirname, name + "-" + datevalues +".mp3" ))
};
drive.files.create({
auth: jwToken,
resource: fileMetadata,
media: media,
fields: 'id'
}, function (err, file) {
if (err) {
// Handle error
console.error(err);
} else {
console.log('File Id: ', file.data.id);
}
// make a callback to a deleteFile() function // I let you search for it
});
}
How about this modification? I'm not sure the condition of blob from reactApp.js. So could you please try to use this modification? In this modification, file or blob from reactApp.js are used.
Modified script :
var stream = require('stream'); // Added
module.exports.uploadFile = function(req){
var file ;
console.log("driveApi upload reached")
function blobToFile(req){
file = req.body.blob
//A Blob() is almost a File() - it's just missing the two properties below which we will add
file.lastModifiedDate = new Date();
file.name = req.body.word;
return file;
}
var bufStream = new stream.PassThrough(); // Added
bufStream.end(file); // Or bufStream.end(### blob from reactApp.js ###) Added
console.log(typeof 42);
// upload file in specific folder
var folderId = "1aa1DD993FOCADXUDNJKLfzfXcOaS3297sU";
var fileMetadata = {
"name": req.body.word,
parents: [folderId]
}
var media = {
mimeType: "audio/mp3",
body: bufStream // Modified
}
drive.files.create({
auth: jwToken,
resource: fileMetadata,
media: media,
fields: "id"
}, function (err, file) {
if (err) {
// Handle error
console.error(err);
} else {
console.log("File Id: ", file.id);
}
console.log("driveApi upload accomplished")
});
}
If this didn't work, I'm sorry.
I have a jenkins job that takes a file as an input. The job reads the input file and then processes the job.
I am trying to run a jenkins job from node js script. I am able to run the particular job using the jenkins api for npm.
My problem is that I am not able to run the job by passing the file required.
In Jenkins, I have to find the particular job and then migrate to it and then click "Build with parameters" and then select the file and build.
I am trying to uncomplicate this by having a react UI that takes a file as input and submits it to the node js script. Now that script has to upload the file to Jenkins and then build the job.
Node js code is,
var file0 = '/Users/m0a00pf/Documents/react-js/asda/src/files/APA.csv';
exports.buildJenkinsJob = function buildJenkinsJob(){
jenkins.job.build({"name":"Create a job",
"parameters":
{
"name": "\\src\\main\\resources\\com\\asda\\qa\\data\\APA\\APA.csv", "file": file0
}
}
,function(err, data){
if(err)
throw err;
else
console.log(data);
});
}
The parameters part is not working. when I run.
jenkins.job.build({"Create a job"});
this works fine.
Changed the options as,
var options = {
method: 'POST',
url: 'http://localhost/job/JobName/buildWithParameters?delay=0sec&Jenkins-Crumb=asdf345672das',
auth : {
username : jenkins.username,
password : jenkins.password
},
headers:
{
'content-type': 'multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW'
},
body:
{
'fileParameterName' :
{ value: fs.createReadStream(absoluteFilePath), options: { filename: FileName, contentType: null } },
'param': 'value'
}
};
There is a plain JS way of doing this, with just popular libraries such as Axios and form-data
var axios = require('axios');
var FormData = require('form-data');
var fs = require('fs');
var data = new FormData();
const jenkinsUrl = 'http://example_jenkins_server_url.com/job/<JOB_NAME>/build';
// notice the URL has build not buildWithParameters if the job has file parameters. ^
const userName = 'example_user';
const password = 'example_pass'
const params = {"parameter": [
// file0 here is the field name we are appending to data object,
// this informs Jenkins which file maps to which job parameter, so even multiple file uploads can be done using this approach!
{"name":"<Name Of file parameter in Job>", "file":"file0"}
// incase you have additional string parameters you need to pass add it here.
{"name": "StringParam1", "value": "value"}
]}
data.append('file0', fs.createReadStream(<full file path>));
data.append('json', JSON.stringify(params));
var config = {
method: 'post',
url: jenkinsUrl,
headers: {
Authorization: `Basic ${Buffer.from(`${userName}:${password}`).toString('base64')}`,
...data.getHeaders()
},
data : data
};
axios(config)
.then(function (response) {
console.log(JSON.stringify(response.data));
})
I am trying to upload a sound file from ngCordova's $cordovaCapture service to UploadCare. The uploadcare.fileFrom('object') keeps failing with an'upload' error. I have the public key set. I am able to upload the file by sending it through and tag and accessing document.getElementById('fileTag').files[0].
$cordovaCapture.captureAudio()
.then(function (audioData) {
return uploadcare.fileFrom('object', audioData[0])
.done(function (fileInfo) {
console.log(fileInfo);
}).fail(function (err) {
console.log(err);
})
})
the audioData[0] object looks like this
MediaFile {
end:0
fullPath:"file:/storage/emulated/0/Sounds/Voice%20002.m4a"
lastModified:null
lastModifiedDate:1481324751000
localURL:"cdvfile://localhost/sdcard/Sounds/Voice%20002.m4a"
name:"Voice 002.m4a"
size:49227
start:0
type:"audio/mpeg"
} __proto__:File
I thought the problem might be that the object is a MediaFile rather than a File but I could use some help casting one to the other.
FileEntry
filesystem:FileSystem
fullPath:"/Sounds/Voice 002.m4a"
isDirectory:false
isFile:true
name:"Voice 002.m4a"
nativeURL:"file:///storage/emulated/0/Sounds/Voice%20002.m4a"
__proto__:Entry
File
end:49227
lastModified:1481324751000
lastModifiedDate:1481324751000
localURL:"cdvfile://localhost/sdcard/Sounds/Voice%20002.m4a"
name:"Voice 002.m4a"
size:49227
start:0
type:"audio/mpeg"
__proto__:Object
using window.resolveLocalFileSystemUrl() you end up with the above FileEntry object that give the above File object but uploadcare still fails with an "upload" error.
Using ngCordova $cordovaFileTransfer() you can send audio files to uploadcare.
var fileName = filePath.split('/').pop();
var uploadcareOptions = {
fileKey: "file",
fileName: fileName,
chunkedMode: false,
mimeType: 'audio/mp4',
params: {
"UPLOADCARE_PUB_KEY": "upload-care-public-key",
"UPLOADCARE_STORE": 'auto',
fileName: fileName
}
};
return $cordovaFileTransfer.upload('https://upload.uploadcare.com/base/', filePath, uploadcareOptions)
The important part is to specify the mime type when sending files as uploadcare will assume it's a image otherwise.
uploadcare.fileFrom uploads a file from a native file object. Try this:
window.resolveLocalFileSystemURL(audioData[0].localURL,function(fileEntry){
fileEntry.file(function(file) {
uploadcare.fileFrom('object', file);
...
});
});
I have a javascript code that gets the xml list http://BUCKETNAME.s3.REGION.amazonaws.com/ of s3 bucket and uses it as a playlist:
AWS.config=
{ "accessKeyId": "ACCESS KEY",
"secretAccessKey": "SECRET KEY",
"region": "REGION" };
// Create S3 service object
s3 = new AWS.S3();
var params = {
Bucket: 'BUCKET NAME', /* required */
Delimiter: '',
EncodingType: 'url',
Marker: '',
MaxKeys: 0,
Prefix: '',
RequestPayer: 'requester'
};
s3.listObjects(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else
{
console.log('the list is approved '); // successful response
// Here is the function that convert the file list in the xml to an array
var b = document.documentElement;
b.setAttribute('data-useragent', navigator.userAgent);
b.setAttribute('data-platform', navigator.platform);
var radioName;
var radioTitle;
var tracklength= 0;
// setupPlayer function
function setupPlayer(href,name){
radioName= href;
radioTitle= name;
$.ajax({
type: "GET",
url: "http://BUCKETNAME.s3.REGION.amazonaws.com/?prefix=radio/"+radioName+"/",
dataType: "xml",
success: function(xml){
//tracklength=0;
tracks =[];
$(xml).find('Contents').each(function(){
tracklength=tracklength+1;
tracks.push({
"track": tracklength,
"file" : $(this).find('Key').text()
});
});
radio(tracks);
},
error: function() {
alert("An error occurred while processing XML file.");
}
});
}
}
As you can see, in this code I am taking the XML file and add a radio name (which is the folder name) , after that the ajax will save all the file names in this folder to an array tracks.
This code works perfectly if there is a list grantee permission for Everyone. So there is no need for aws config here. I can run the code inside else statement in listObjects function and it will give me the same response.
What I do want is to give the grant access to this key only, to make this function not work without the access key and secret key.
So no one can access the xml list except those who have the access and secret keys.
Is that possible ?
(This is not the full code, but you got the Idea, accessing the XML file of the bucket and getting the keys an saving them to an array).
You should use s3.getObject (http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#getObject-property) to get your xml files instead of $.ajax call.
I have an implementation that uploads an audio/video blob upload to S3 using putObject.
Let's assume obj contains the following data:
object {extension: "webm", type: "video/webm",
contents: "data:video/webm;base64,GkXfo0AgQoaBAUL3gQFC8oEEQvO…"}
where contents is the video data URI (a base64 string).
The server side code is:
var s3 = new AWS.S3();
obj.contents = obj.contents.split(',').pop();
buffer = new Buffer(obj.contents, "base64");
var params = {
Bucket: S3_BUCKET + "/videos/",
Key: fname, // add new name
Body: buffer,
ACL: 'private',
ContentType: obj.type
};
s3.putObject(params, handler);
If I want to do this on the client-side, how do I go about passing the base64 string
to putObject? There is no such thing such as Buffer in javascript.
I see that putObject expects either a new Buffer('...') || streamObject || 'STRING_VALUE',
but is the STRING_VALUE and option for audio/video files, or it is only for text?
First of all, I'd like to thank the AWS team for helping responding this question. It seems that, though not well documented, putObject() does accept Blobs or any native typed arrays (https://forums.aws.amazon.com/message.jspa?messageID=539115). This means that you can directly take any Blob in your browser (e.g. audio or video) and upload it directly to an S3 file.
audioBlob = this.GetAudioBlob()
var params = {
Bucket: "MYBUCKET",
Key: "audio_file_test.wav",
Body: audioBlob,//this hast to be a string
ACL: 'private',
ContentType: 'audio/wav',
};
s3.putObject(params, function(err,data){ console.log(err); } );
audioBlob in exactly a Blob object. Blob {type: "audio/wav", size: 262188, slice: function}