I'm uploading a data stream to Azure Storage,
I would get the link to the blob file.
let insertFile = async function (blobName,stream){
const containerName= 'texttospeechudio';
try{
await blobService.createContainerIfNotExists(containerName, {
publicAccessLevel: 'blob'},(err,result, response) => {
if(!err) {
console.log(result);
}
});
let resultstream = blobService.createWriteStreamToBlockBlob(containerName, blobName,(err,result, response)=>{
console.log(res)
});
stream.pipe(resultstream);
stream.on('error', function (error) {
console.log(error);
});
stream.once('end', function (end) {
console.log(end)
//OK
});
}
catch(err) {
console.log(err);
}
}
I added createWriteStreamToBlockBlob callback , but I'm not getting inside it.
I would find a way to get uploaded file url.
There is no file URL returned in the response according to put-blob's rest spec.
And Azure storage's resource URL can be commonly composed with following pattern:
https://{myaccount}.blob.core.windows.net/{mycontainer}/{myblob}
Related
I just want to delete zip folder after sent response so I am looking for any alternative solution
Here is my code / it is get request
exports.Download = async (req, res) => {
try {
var zp = new admz();
zp.addLocalFolder(`./download/${folder}`);
const file_after_download = 'downloaded.zip';
const data = zp.toBuffer();
res.set('Content-Type', 'application/octet-stream');
res.set('Content-Disposition', `attachment; filename=${file_after_download}`);
res.set('Content-Length', data.length);
return res.send(data);
// HERE I want execute this code
let dir = `./download/${folder}`;
if (fse.existsSync(dir)) {
fse.rmdirSync(dir, { recursive: true })
}
} catch (err) {
console.log(err)
return res.render('pages/404');
}
}
Update
If send code without return ( res.send(data);)
Im getting this error //Error [ERR_HTTP_HEADERS_SENT]: Cannot set headers after they are sent to the client //
If I put return res.send(data); at the end of block , then downloaded zip file will be empty - because its deleted already
From the docs of Express, you can use res.download() function which has a callback parameters to be executed once download is done.
res.download(filePath, 'yourFileName', function(err) {
if (err) {
next(err)
} else {
console.log('Delete:', filePath);
}
})
I am trying to build a web app to stream music. I use MongoDB to store the audio, a Node API to connect to the database and a Vuejs frontend. Below is the endpoint which streams the music, based on this article: https://medium.com/#richard534/uploading-streaming-audio-using-nodejs-express-mongodb-gridfs-b031a0bcb20f
trackRoute.get('/:trackID', (req, res) => {
try {
var trackID = new ObjectID(req.params.trackID);
} catch (err) {
return res.status(400).json({ message: "Invalid trackID in URL parameter. Must be a single String of 12 bytes or a string of 24 hex characters" });
}
res.set('content-type', 'audio/mp3');
res.set('accept-ranges', 'bytes');
let bucket = new mongodb.GridFSBucket(db, {
bucketName: 'tracks'
});
let downloadStream = bucket.openDownloadStream(trackID);
downloadStream.on('data', (chunk) => {
res.write(chunk);
});
downloadStream.on('error', () => {
res.sendStatus(404);
});
downloadStream.on('end', () => {
res.end();
});
});
I tested it with Postman and it works there. I am trying to read the stream in my Vuejs application. I'm just not sure how to do it. I tried the following to test it:
const url = 'http://localhost:4343/api/track/6061c90b2658b9001e65311d';
http.get(url, function (res) {
res.on('data', function (buf) {
console.log(buf);
});
res.on('end', function () {
console.log('ended');
});
})
This does not work however. How should I go about reading it in the frontend?
My apollo-server is using graphql-upload package which includes file upload support for GraphQL endpoints. But they only documented about uploading single files. But we need multiple file upload support. Well, I get the streams as an Array. But whenever I createReadStream for each streams & pipe them to cloudinary uploader var, it just uploads the last created stream rather then uploading the each stream.
Code
// graphql reolver
const post = async (_, { post }, { isAuthenticated, user }) => {
if (!isAuthenticated) throw new AuthenticationError("User unauthorized");
const files = await Promise.all(post.files);
let file_urls = [];
const _uploadableFiles = cloudinary.uploader.upload_stream({ folder: "post_files" },
(err, result) => {
console.log("err:", err);
console.log("result:", result);
if (err) throw err;
file_urls.push({
url: result.secure_url,
public_id: result.public_id,
file_type: result.metadata,
});
return result;
}
);
files.forEach(async (file) => await file.createReadStream().pipe(_uploadableFiles));
.... other db related stuff
}
After that, I get the Secure_URL from uploaded files which is returned by cloudinary upload_stream functions callback. But it only gives me the properties of one stream which was the last of the all streams. Please help me in this case. Is there any way to pipe multiple streams?
Instead of making one const upload stream you make it into a factory function that returns an upload stream on each call for pipe'ing
Use array map so that you get an array that you can use in Promise.all
One by one each file should get uploaded to their own respective upload stream, appending the generated file url info to file_urls(on success callback), when all are done Promise.all would resolve and the code can resume to do other db related stuff
const post = async (_, { post }, { isAuthenticated, user }) => {
if (!isAuthenticated) throw new AuthenticationError("User unauthorized");
const files = await Promise.all(post.files);
let file_urls = [];
function createUploader(){
return cloudinary.uploader.upload_stream({ folder: "post_files" },
(err, result) => {
console.log("err:", err);
console.log("result:", result);
if (err) throw err;
file_urls.push({
url: result.secure_url,
public_id: result.public_id,
file_type: result.metadata,
});
return result;
}
);
}
await Promise.all( files.map(async (file) => await file.createReadStream().pipe(createUploader())) ); //map instead of forEach
//.... other db related stuff
}
I have two functions in separate files to split up the workflow.
const download = function(url){
const file = fs.createWriteStream("./test.png");
const request = https.get(url, function(response) {
response.pipe(file);
});
}
This function in my fileHelper.js is supposed to take a URL with an image in it and then save it locally to test.png
function uploadFile(filePath) {
fs.readFile('credentials.json', (err, content) => {
if (err) return console.log('Error loading client secret file:', err);
// Authorize a client with credentials, then call the Google Drive API.
authorize(JSON.parse(content), function (auth) {
const drive = google.drive({version: 'v3', auth});
const fileMetadata = {
'name': 'testphoto.png'
};
const media = {
mimeType: 'image/png',
body: fs.createReadStream(filePath)
};
drive.files.create({
resource: fileMetadata,
media: media,
fields: 'id'
}, (err, file) => {
if (err) {
// Handle error
console.error(err);
} else {
console.log('File Id: ', file.id);
}
});
});
});
}
This function in my googleDriveHelper.js is supposed to take the filePath of call and then upload that stream into my google drive. These two functions work on their own but it seems that the https.get works asynchronously and if I try to call the googleDriveHelper.uploadFile(filePath) function after the download, it doesn't have time to get the full file to upload so instead a blank file will be uploaded to my drive.
I want to find a way so that when the fileHelper.download(url) is called, it automatically uploads into my drive.
I also don't know if there is a way to create a readStream directly from the download function to the upload function, so I can avoid having to save the file locally to upload it.
I believe your goal as follows.
You want to upload a file retrieving from an URL to Google Drive.
When you download the file from the URL, you want to upload it to Google Drive without creating the file.
You want to achieve this using googleapis with Node.js.
You have already been able to upload a file using Drive API.
For this, how about this answer?
Modification points:
At download function, the retrieved buffer is converted to the stream type, and the stream data is returned.
At uploadFile function, the retrieved stream data is used for uploading.
When the file ID is retrieved from the response value of Drive API, please use file.data.id instead of file.id.
By above modification, the file downloaded from the URL can be uploaded to Google Drive without creating a file.
Modified script:
When your script is modified, please modify as follows.
download()
const download = function (url) {
return new Promise(function (resolve, reject) {
request(
{
method: "GET",
url: url,
encoding: null,
},
(err, res, body) => {
if (err && res.statusCode != 200) {
reject(err);
return;
}
const stream = require("stream");
const bs = new stream.PassThrough();
bs.end(body);
resolve(bs);
}
);
});
};
uploadFile()
function uploadFile(data) { // <--- Modified
fs.readFile("drive_credentials.json", (err, content) => {
if (err) return console.log("Error loading client secret file:", err);
authorize(JSON.parse(content), function (auth) {
const drive = google.drive({ version: "v3", auth });
const fileMetadata = {
name: "testphoto.png",
};
const media = {
mimeType: "image/png",
body: data, // <--- Modified
};
drive.files.create(
{
resource: fileMetadata,
media: media,
fields: "id",
},
(err, file) => {
if (err) {
console.error(err);
} else {
console.log("File Id: ", file.data.id); // <--- Modified
}
}
);
});
});
}
For testing
For example, when above scripts are tested, how about the following script?
async function run() {
const url = "###";
const data = await fileHelper.download(url);
googleDriveHelper.uploadFile(data);
}
References:
Class: stream.PassThrough
google-api-nodejs-client
I'm doing an application with react-native. Now I'm trying to send an image from the mobile to the server (Node Js). For this I'm using react-native-image-picker. And the problem is that when I send the image it save a file but it's empty not contain the photo. I think that the problem probably is that the server can't access to the path of the image because is in a different device. But I don't know how I can do it.
React-Native:
openImagePicker(){
const options = {
title: 'Select Avatar',
storageOptions: {
skipBackup: true,
path: 'images'
}
}
ImagePicker.showImagePicker(options, (imagen) =>{
if (imagen.didCancel) {
console.log('User cancelled image picker');
}
else if (imagen.error) {
console.log('ImagePicker Error: ', imagen.error);
}
else if (imagen.customButton) {
console.log('User tapped custom button: ', imagen.customButton);
}
else {
let formdata = new FormData();
formdata.append("file[name]", imagen.fileName);
formdata.append("file[path]", imagen.path);
formdata.append("file[type]", imagen.type);
fetch('http://X/user/photo/58e137dd5d45090d0b000006', {
method: 'PUT',
headers: {
'Content-Type': 'multipart/form-data'
},
body: formdata
})
.then(response => {
console.log("ok");
})
.catch(function(err) {
console.log(err);
})
}})}
Node Js:
addPhotoUser = function (req, res) {
User.findById(req.params.id, function(err, user) {
fs.readFile(req.body.file.path, function (err, data) {
var pwd = 'home/ubuntu/.../';
var newPath = pwd + req.body.file.name;
fs.writeFile(newPath, data, function (err) {
imageUrl: URL + req.body.file.name;
user.save(function(err) {
if(!err) {
console.log('Updated');
} else {
console.log('ERROR: ' + err);
}
res.send(user);
});
});
});
});
};
Yes, the problem is that the filepath is on the local device and not the server. You want to send the actual data returned to you by react-native-image-picker not the uri. It looks like that library encodes the data with base64 so you're going to want send that to your server, not the uri returned from the library because it won't be accessible on a remote server.
What this means is that you won't be reading any files on your server but instead just decoding a base64 string in the response body and writing that to your filesystem.
For the client side:
let formdata = new FormData();
formdata.append("file[name]", imagen.fileName);
formdata.append("file[data]", imagen.data); // this is base64 encoded!
formdata.append("file[type]", imagen.type);
fetch('http://X/user/photo/58e137dd5d45090d0b000006', {
method: 'PUT',
headers: {
'Content-Type': 'multipart/form-data'
},
body: formdata
})
On the server side atob to decode from base64 before writing to the filesystem:
let decoded = atob(req.body.data)
// now this is binary and can written to the filesystem
From there:
fs.writeFile(newPath, decoded, function (err) {
imageUrl: newPath;
user.save(function(err) {
if(!err) {
console.log('Updated');
} else {
console.log('ERROR: ' + err);
}
res.send(user);
});
});
Note, you don't need the filesystem write that's in your code because you're decoding the image that was sent as a b64 string in your request.
There also seems to be some oddities with how you're using that user object. You seem to be only passing a function that handles errors and not any actual data. I don't know what ORM you're using so it's hard to say how it should work. Maybe something like this?
user.save({imageUrl:uriReturnedByFsWrite}, (err, data)=>{...})
Good luck :)
Make an object then send that object to the server. The object will consist of name,path and type, like this:
var imageData = {name: 'image1', path: uri, type: 'image/jpeg'}
Above is a one way to send the image data. The other way is to convert it into BLOB so that server side programmer doesn't have to do this task on their end. You can make BLOB by use of react-native-fetch-blob.
One more way is to directly upload the images to the amazon server(s3) and send the link to the backend..
Function that returns base64 string:
var RNFetchBlob = require('react-native-fetch-blob').default;
getImageAttachment: function(uri_attachment, mimetype_attachment) {
return new Promise((RESOLVE, REJECT) => {
// Fetch attachment
RNFetchBlob.fetch('GET', config.apiRoot+'/app/'+uri_attachment)
.then((response) => {
let base64Str = response.data;
var imageBase64 = 'data:'+mimetype_attachment+';base64,'+base64Str;
// Return base64 image
RESOLVE(imageBase64)
})
}).catch((error) => {
// error handling
console.log("Error: ", error)
});
},
Cheers :)