I have two functions in separate files to split up the workflow.
const download = function(url){
const file = fs.createWriteStream("./test.png");
const request = https.get(url, function(response) {
response.pipe(file);
});
}
This function in my fileHelper.js is supposed to take a URL with an image in it and then save it locally to test.png
function uploadFile(filePath) {
fs.readFile('credentials.json', (err, content) => {
if (err) return console.log('Error loading client secret file:', err);
// Authorize a client with credentials, then call the Google Drive API.
authorize(JSON.parse(content), function (auth) {
const drive = google.drive({version: 'v3', auth});
const fileMetadata = {
'name': 'testphoto.png'
};
const media = {
mimeType: 'image/png',
body: fs.createReadStream(filePath)
};
drive.files.create({
resource: fileMetadata,
media: media,
fields: 'id'
}, (err, file) => {
if (err) {
// Handle error
console.error(err);
} else {
console.log('File Id: ', file.id);
}
});
});
});
}
This function in my googleDriveHelper.js is supposed to take the filePath of call and then upload that stream into my google drive. These two functions work on their own but it seems that the https.get works asynchronously and if I try to call the googleDriveHelper.uploadFile(filePath) function after the download, it doesn't have time to get the full file to upload so instead a blank file will be uploaded to my drive.
I want to find a way so that when the fileHelper.download(url) is called, it automatically uploads into my drive.
I also don't know if there is a way to create a readStream directly from the download function to the upload function, so I can avoid having to save the file locally to upload it.
I believe your goal as follows.
You want to upload a file retrieving from an URL to Google Drive.
When you download the file from the URL, you want to upload it to Google Drive without creating the file.
You want to achieve this using googleapis with Node.js.
You have already been able to upload a file using Drive API.
For this, how about this answer?
Modification points:
At download function, the retrieved buffer is converted to the stream type, and the stream data is returned.
At uploadFile function, the retrieved stream data is used for uploading.
When the file ID is retrieved from the response value of Drive API, please use file.data.id instead of file.id.
By above modification, the file downloaded from the URL can be uploaded to Google Drive without creating a file.
Modified script:
When your script is modified, please modify as follows.
download()
const download = function (url) {
return new Promise(function (resolve, reject) {
request(
{
method: "GET",
url: url,
encoding: null,
},
(err, res, body) => {
if (err && res.statusCode != 200) {
reject(err);
return;
}
const stream = require("stream");
const bs = new stream.PassThrough();
bs.end(body);
resolve(bs);
}
);
});
};
uploadFile()
function uploadFile(data) { // <--- Modified
fs.readFile("drive_credentials.json", (err, content) => {
if (err) return console.log("Error loading client secret file:", err);
authorize(JSON.parse(content), function (auth) {
const drive = google.drive({ version: "v3", auth });
const fileMetadata = {
name: "testphoto.png",
};
const media = {
mimeType: "image/png",
body: data, // <--- Modified
};
drive.files.create(
{
resource: fileMetadata,
media: media,
fields: "id",
},
(err, file) => {
if (err) {
console.error(err);
} else {
console.log("File Id: ", file.data.id); // <--- Modified
}
}
);
});
});
}
For testing
For example, when above scripts are tested, how about the following script?
async function run() {
const url = "###";
const data = await fileHelper.download(url);
googleDriveHelper.uploadFile(data);
}
References:
Class: stream.PassThrough
google-api-nodejs-client
Related
noob question, I'm just getting started with Google Drive API v3. How can I download dynamic file from google drive when I only have fileId. file can be, image, pdf, or docs.
I tried searching but I couldn't found any reference or example related to this.
This what I have so far but it only download specific file extension.
downloadFile(req, res) {
const auth = new google.auth.JWT(
client_email,
null,
private_key,
SCOPES,
);
const { fileId } = req.params;
const drive = google.drive({ version: 'v3', auth});
var dest = fs.createWriteStream('./tmp/downloads/dummy.pdf')
drive.files.get({
fileId,
alt: 'media',
}, {
responseType: 'stream'
}).then((driveResponse) => {
driveResponse.data.on('end', () => {
console.log(`downloading fileID ${fileId}`);
})
.on('error', (err) => {
console.log(err);
})
.on('data', (d) => {
console.log(d);
})
.pipe(dest)
})
.catch((err) => {
console.log(err);
})
}
Is there way to download dynamic files from google drive?
I believe your goal as follows.
You want to download the files from Google Drive using the service account and the file ID.
The files include both Google Docs files and the files except for Google Docs files.
You want to achieve this using googleapis for Node.js.
Modification points:
Unfortunately, from it only download specific file extension., I cannot understand about the detail of your situation. But I guess that the reason of your issue might be due to downloading both Google Docs files and the files except for Google Docs files.
When Google Docs files are downloaded, the files are required to be downloaded using the method of "Files: export" in Drive API.
When the files except for Google Docs files are downloaded, the files are required to be downloaded using the method of "Files: get" in Drive API.
I thought that above situation might be the reason of your issue.
In order to download both Google Docs files and the files except for Google Docs files, I propose the following flow.
Check the mimeType of the file ID.
Download the file using each method by the mimeType.
When above points are reflected to your script, it becomes as follows.
Modified script:
From:
var dest = fs.createWriteStream('./tmp/downloads/dummy.pdf')
drive.files.get({
fileId,
alt: 'media',
}, {
responseType: 'stream'
}).then((driveResponse) => {
driveResponse.data.on('end', () => {
console.log(`downloading fileID ${fileId}`);
})
.on('error', (err) => {
console.log(err);
})
.on('data', (d) => {
console.log(d);
})
.pipe(dest)
})
.catch((err) => {
console.log(err);
})
To:
drive.files.get({ fileId, fields: "*" }, async (err, { data }) => {
if (err) {
console.log(err);
return;
}
let filename = data.name;
const mimeType = data.mimeType;
let res;
if (mimeType.includes("application/vnd.google-apps")) {
const convertMimeTypes = {
"application/vnd.google-apps.document": {
type:
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
ext: ".docx",
},
"application/vnd.google-apps.spreadsheet": {
type:
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
ext: ".xlsx",
},
"application/vnd.google-apps.presentation": {
type:
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
ext: ".pptx",
},
};
filename += convertMimeTypes[mimeType].ext;
res = await drive.files.export(
{
fileId,
mimeType: convertMimeTypes[mimeType].type,
},
{ responseType: "stream" }
);
} else {
res = await drive.files.get(
{
fileId,
alt: "media",
},
{ responseType: "stream" }
);
}
const dest = fs.createWriteStream(filename);
res.data
.on("end", () => console.log("Done."))
.on("error", (err) => {
console.log(err);
return process.exit();
})
.pipe(dest);
});
Note:
In this modification, I prepared 3 types of Google Docs files at convertMimeTypes. When you want to download other mimeTypes, please modify convertMimeTypes. In this case, for example, Google Docs files are downloaded as Microsoft Docs files.
References:
Download files
Files: get
Files: export
enter code here**Hi All,
I have root folder in google drive named sample and within sample there are multiple folder such as s1,s2,s3 etc. and within that folder again there are some files and folder I'm trying to download the same folder structure with files within them to my local drive and then upload to google storage bucket simultaneously in NODE JS using google drive API's (drive.files.list) In below code I'm calling the same function recursively with it is uploading all file in root folder level
anybody's help will be highly grateful
Thank u
sample code for reference
async function listDriveFiles(folder){
console.log("In listDriveFiles");
const drive = google.drive({ version: 'v3', auth: oauth2Client });
const drivefordownload = google.drive({ version: 'v3', auth: oauth2Client });
drive.files.list({
q: `'${folder.id}' in parents`,
fields: 'nextPageToken, files(*)',
}, async (err, response) => {
if (err) {
console.log('3', err)
//done({ errors: err, statusCode: 500 })
}
if (response != undefined || response != null) {
const files = response.data.files;
if (files.length) {
console.log("files length:" + files.length)
files.map(async (file) => {
if (validMimeTypeList.indexOf(file.mimeType) > -1) {
try {
console.log("before download")
await downLoadRawSalesFiles(drivefordownload, folder.name, file);
}
catch (err) {
console.log("Errors from downloading",err);
}
}else if(file.mimeType == 'application/vnd.google-apps.folder'){
await listDriveFiles(file);
}
});
}
}
});
}
I have created a web service to connect with google drive API. Also there is an another system which is frontend developed using react js. I need to send a file from react system to google drive through the web service which I developed. For that I used file stream. When I send it, I got an error which says "part.body.pipe is not a function". This comes from the google drive api.
Below code is sample code for POST request which I sent.
const axios = require('axios');
const fs = require('fs');
const stream = fs.createReadStream('./download.jpg');
axios.post('http://localhost:3008/google-drive/upload-file', {
stream,
name: 'add.jpg',
mimeType: 'image/jpeg',
}
)
.then((res) => {
console.log(`statusCode: ${res.statusCode}`)
console.log(res)
})
.catch((error) => {
console.error(error)
})
Below is the google drive integration for file upload.
function uploadFiles(auth, mainRequest, mainResponse) {
const drive = google.drive({version: 'v3', auth});
const {
stream,
name,
mimeType,
} = mainRequest.body;
let fileMetaData = {
name,
};
let media = {
mimeType,
body: stream,
};
drive.files.create({
media,
resource: fileMetaData,
fields: 'id',
}, (error, file) => {
if (error) {
console.log(error);
return mainResponse.status(200).json({
message: 'The API returned an error: ' + error
});
}
return mainResponse.status(200).json({
fileId: file.id
});
});
}
I think the problem is in createReadStream. I'm not sure I used it correctly or not.
Thanks in advance. :)
You made a mistake in the fs
Rather try using this synchronous func
fs.readFileSync('filename.jpg','utf8')
I´m trying to use puppeteer to log in a website and "download" a pdf directly to my drive. I've managed to reach the pdf page with puppeteer and I tried (between other tries) to get the blob using fetch with the cookies to send to drive. I can´t post the login information here, but if you could help me looking for an error (or more) in the code it would be great! For now, it goes to the page before pdf, gets the link, fetch with cookies and insert a pdf in drive, but the pdf is corrupted with 0 kb.
I tried setRequestInterception, getPdf (from puppeteer) and using buffer with some stuff I found on my research.
//Page before pdfPage. Here I got the link: urlPdf
//await page.goto(urlPdf);
//await page.waitForNavigation();
//const htmlPdf = await page.content();
const cookies = await page.cookies()
const opts = {
headers: {
cookie: cookies
}
};
let blob = await fetch(urlPdf,opts).then(r => r.blob());
console.log("pegou o blob")
// upload file in specific folder
var file ;
console.log("driveApi upload reached")
function blobToFile(req){
file = req.body.blob
//A Blob() is almost a File() - it's just missing the two properties below which we will add
file.lastModifiedDate = new Date();
file.name = teste.pdf;//req.body.word;
return file;
}
var folderId = myFolderId;
var fileMetadata = {
'name': 'teste.pdf',
parents: [folderId]
};
var media = {
mimeType: 'application/pdf',
body: file
};
drive.files.create({
auth: jwToken,
resource: fileMetadata,
media: media,
fields: 'id'
}, function(err, file) {
if (err) {
// Handle error
console.error(err);
} else {
console.log('File Id: ', file.data.id);
}
});
I tried many things, but the final solution I came with is posted here:
Puppeteer - How can I get the current page (application/pdf) as a buffer or file?
await page.setRequestInterception(true);
page.on('request', async request => {
if (request.url().indexOf('exibirFat.do')>0) { //This condition is true only in pdf page (in my case of course)
const options = {
encoding: null,
method: request._method,
uri: request._url,
body: request._postData,
headers: request._headers
}
/* add the cookies */
const cookies = await page.cookies();
options.headers.Cookie = cookies.map(ck => ck.name + '=' + ck.value).join(';');
/* resend the request */
const response = await request_client(options);
//console.log(response); // PDF Buffer
buffer = response;
let filename = 'file.pdf';
fs.writeFileSync(filename, buffer); //Save file
} else {
request.continue();
}
});
This solution needs: const request_client = require('request-promise-native');
I'm doing an application with react-native. Now I'm trying to send an image from the mobile to the server (Node Js). For this I'm using react-native-image-picker. And the problem is that when I send the image it save a file but it's empty not contain the photo. I think that the problem probably is that the server can't access to the path of the image because is in a different device. But I don't know how I can do it.
React-Native:
openImagePicker(){
const options = {
title: 'Select Avatar',
storageOptions: {
skipBackup: true,
path: 'images'
}
}
ImagePicker.showImagePicker(options, (imagen) =>{
if (imagen.didCancel) {
console.log('User cancelled image picker');
}
else if (imagen.error) {
console.log('ImagePicker Error: ', imagen.error);
}
else if (imagen.customButton) {
console.log('User tapped custom button: ', imagen.customButton);
}
else {
let formdata = new FormData();
formdata.append("file[name]", imagen.fileName);
formdata.append("file[path]", imagen.path);
formdata.append("file[type]", imagen.type);
fetch('http://X/user/photo/58e137dd5d45090d0b000006', {
method: 'PUT',
headers: {
'Content-Type': 'multipart/form-data'
},
body: formdata
})
.then(response => {
console.log("ok");
})
.catch(function(err) {
console.log(err);
})
}})}
Node Js:
addPhotoUser = function (req, res) {
User.findById(req.params.id, function(err, user) {
fs.readFile(req.body.file.path, function (err, data) {
var pwd = 'home/ubuntu/.../';
var newPath = pwd + req.body.file.name;
fs.writeFile(newPath, data, function (err) {
imageUrl: URL + req.body.file.name;
user.save(function(err) {
if(!err) {
console.log('Updated');
} else {
console.log('ERROR: ' + err);
}
res.send(user);
});
});
});
});
};
Yes, the problem is that the filepath is on the local device and not the server. You want to send the actual data returned to you by react-native-image-picker not the uri. It looks like that library encodes the data with base64 so you're going to want send that to your server, not the uri returned from the library because it won't be accessible on a remote server.
What this means is that you won't be reading any files on your server but instead just decoding a base64 string in the response body and writing that to your filesystem.
For the client side:
let formdata = new FormData();
formdata.append("file[name]", imagen.fileName);
formdata.append("file[data]", imagen.data); // this is base64 encoded!
formdata.append("file[type]", imagen.type);
fetch('http://X/user/photo/58e137dd5d45090d0b000006', {
method: 'PUT',
headers: {
'Content-Type': 'multipart/form-data'
},
body: formdata
})
On the server side atob to decode from base64 before writing to the filesystem:
let decoded = atob(req.body.data)
// now this is binary and can written to the filesystem
From there:
fs.writeFile(newPath, decoded, function (err) {
imageUrl: newPath;
user.save(function(err) {
if(!err) {
console.log('Updated');
} else {
console.log('ERROR: ' + err);
}
res.send(user);
});
});
Note, you don't need the filesystem write that's in your code because you're decoding the image that was sent as a b64 string in your request.
There also seems to be some oddities with how you're using that user object. You seem to be only passing a function that handles errors and not any actual data. I don't know what ORM you're using so it's hard to say how it should work. Maybe something like this?
user.save({imageUrl:uriReturnedByFsWrite}, (err, data)=>{...})
Good luck :)
Make an object then send that object to the server. The object will consist of name,path and type, like this:
var imageData = {name: 'image1', path: uri, type: 'image/jpeg'}
Above is a one way to send the image data. The other way is to convert it into BLOB so that server side programmer doesn't have to do this task on their end. You can make BLOB by use of react-native-fetch-blob.
One more way is to directly upload the images to the amazon server(s3) and send the link to the backend..
Function that returns base64 string:
var RNFetchBlob = require('react-native-fetch-blob').default;
getImageAttachment: function(uri_attachment, mimetype_attachment) {
return new Promise((RESOLVE, REJECT) => {
// Fetch attachment
RNFetchBlob.fetch('GET', config.apiRoot+'/app/'+uri_attachment)
.then((response) => {
let base64Str = response.data;
var imageBase64 = 'data:'+mimetype_attachment+';base64,'+base64Str;
// Return base64 image
RESOLVE(imageBase64)
})
}).catch((error) => {
// error handling
console.log("Error: ", error)
});
},
Cheers :)