How to download audio file from URL and store it in local directory?
I'm using Node.js and I tried the following code:
var http = require('http');
var fs = require('fs');
var dest = 'C./test'
var url= 'http://static1.grsites.com/archive/sounds/comic/comic002.wav'
function download(url, dest, callback) {
var file = fs.createWriteStream(dest);
var request = http.get(url, function (response) {
response.pipe(file);
file.on('finish', function () {
file.close(callback); // close() is async, call callback after close completes.
});
file.on('error', function (err) {
fs.unlink(dest); // Delete the file async. (But we don't check the result)
if (callback)
callback(err.message);
});
});
}
No error occured but the file has not been found.
Duplicate of How to download a file with Node.js (without using third-party libraries)?, but here is the code specific to your question:
var http = require('http');
var fs = require('fs');
var file = fs.createWriteStream("file.wav");
var request = http.get("http://static1.grsites.com/archive/sounds/comic/comic002.wav", function(response) {
response.pipe(file);
});
Your code is actually fine, you just don't call the download function. Try adding this to the end :
download(url, dest, function(err){
if(err){
console.error(err);
}else{
console.log("Download complete");
}
});
Also, change the value of dest to something else, like just "test.wav" or something. 'C./test' is a bad path.
I tried it on my machine and your code works fine just adding the call and changing dest.
Here is an example using Axios with an API that may require authorization
const Fs = require('fs');
const Path = require('path');
const Axios = require('axios');
async function download(url) {
let filename = "filename";
const username = "user";
const password = "password"
const key = Buffer.from(username + ':' + password).toString("base64");
const path = Path.resolve(__dirname, "audio", filename)
const response = await Axios({
method: 'GET',
url: url,
responseType: 'stream',
headers: { 'Authorization': 'Basic ' + key }
})
response.data.pipe(Fs.createWriteStream(path))
return new Promise((resolve, reject) => {
response.data.on('end', () => {
resolve();
})
response.data.on('error', () => {
reject(err);
})
})
}
Related
Here is how my API works:
You can find SeaweedFS here on GitHub.
And the code here:
// /drivers/seaweedfs.js Defines how API interacts with SeaweedFS
const { error } = require("console");
const http = require("http");
module.exports = class Weed {
constructor(mserver) {
this.mserver = new URL(mserver);
}
get(fileId) {
return new Promise((resolve, reject) => {
let options = {
hostname: this.mserver.hostname,
port: this.mserver.port,
method: "GET",
path: `/${fileId}`,
timeout: 6000,
};
let data;
const fileReq = http.request(options, (res) => {
console.log(`Statuscode ${res.statusCode}`);
res.on("data", (response) => {
data += response;
});
res.on("end", () => {
resolve(data);
});
});
fileReq.on("error", () => {
console.error(error);
reject();
});
fileReq.end();
});
}
};
// /routes/file.js An Express router
const express = require("express");
const router = express.Router();
const Weed = require("../drivers/seaweedfs");
let weedClient = new Weed("http://localhost:60002");
router.get("/:fileId", (req, res) => {
weedClient.get(req.params.fileId)
.then(data=>{
res.write(data)
res.end()
})
}
)
module.exports = router;
MongoDB driver not yet implemented.
When I try to GET a file(using Firefox, Hoppscotch says Could not send request: Unable to reach the API endpoint. Check your network connection and try again.), I get something whose MIME type is application/octet-stream for some reason. It's bigger than the original file. I know there must be some problems with my code, but I don't know where and how to fix it.
Closed. This question needs to be more focused. It is not currently accepting answers.
Want to improve this question? Update the question so it focuses on one problem only by editing this post.
Closed 4 years ago.
Improve this question
I'm trying to write a script to download images using node.js. This is what I have so far:
var maxLength = 10 // 10mb
var download = function(uri, callback) {
http.request(uri)
.on('response', function(res) {
if (res.headers['content-length'] > maxLength*1024*1024) {
callback(new Error('Image too large.'))
} else if (!~[200, 304].indexOf(res.statusCode)) {
callback(new Error('Received an invalid status code.'))
} else if (!res.headers['content-type'].match(/image/)) {
callback(new Error('Not an image.'))
} else {
var body = ''
res.setEncoding('binary')
res
.on('error', function(err) {
callback(err)
})
.on('data', function(chunk) {
body += chunk
})
.on('end', function() {
// What about Windows?!
var path = '/tmp/' + Math.random().toString().split('.').pop()
fs.writeFile(path, body, 'binary', function(err) {
callback(err, path)
})
})
}
})
.on('error', function(err) {
callback(err)
})
.end();
}
I, however, want to make this more robust:
Are there libraries that do this and do this better?
Is there a chance that response headers lie (about length, about content type)?
Are there any other status codes I should care about? Should I bother with redirects?
I think I read somewhere that binary encoding is going to be deprecated. What do I do then?
How can I get this to work on windows?
Any other ways you can make this script better?
Why: for a feature similar to imgur where users can give me a URL, I download that image, and rehost the image in multiple sizes.
I'd suggest using the request module. Downloading a file is as simple as the following code:
var fs = require('fs'),
request = require('request');
var download = function(uri, filename, callback){
request.head(uri, function(err, res, body){
console.log('content-type:', res.headers['content-type']);
console.log('content-length:', res.headers['content-length']);
request(uri).pipe(fs.createWriteStream(filename)).on('close', callback);
});
};
download('https://www.google.com/images/srpr/logo3w.png', 'google.png', function(){
console.log('done');
});
I ran into this problem some days ago, for a pure NodeJS answer I would suggest using Stream to merge the chunks together.
var http = require('http'),
Stream = require('stream').Transform,
fs = require('fs');
var url = 'http://www.google.com/images/srpr/logo11w.png';
http.request(url, function(response) {
var data = new Stream();
response.on('data', function(chunk) {
data.push(chunk);
});
response.on('end', function() {
fs.writeFileSync('image.png', data.read());
});
}).end();
The newest Node versions won't work well with binary strings, so merging chunks with strings is not a good idea when working with binary data.
*Just be careful when using 'data.read()', it will empty the stream for the next 'read()' operation. If you want to use it more than once, store it somewhere.
You can use Axios (a promise-based HTTP client for Node.js) to download images in the order of your choosing in an asynchronous environment:
npm i axios
Then, you can use the following basic example to begin downloading images:
const fs = require('fs');
const axios = require('axios');
/* ============================================================
Function: Download Image
============================================================ */
const download_image = (url, image_path) =>
axios({
url,
responseType: 'stream',
}).then(
response =>
new Promise((resolve, reject) => {
response.data
.pipe(fs.createWriteStream(image_path))
.on('finish', () => resolve())
.on('error', e => reject(e));
}),
);
/* ============================================================
Download Images in Order
============================================================ */
(async () => {
let example_image_1 = await download_image('https://example.com/test-1.png', 'example-1.png');
console.log(example_image_1.status); // true
console.log(example_image_1.error); // ''
let example_image_2 = await download_image('https://example.com/does-not-exist.png', 'example-2.png');
console.log(example_image_2.status); // false
console.log(example_image_2.error); // 'Error: Request failed with status code 404'
let example_image_3 = await download_image('https://example.com/test-3.png', 'example-3.png');
console.log(example_image_3.status); // true
console.log(example_image_3.error); // ''
})();
var fs = require('fs'),
http = require('http'),
https = require('https');
var Stream = require('stream').Transform;
var downloadImageToUrl = (url, filename, callback) => {
var client = http;
if (url.toString().indexOf("https") === 0){
client = https;
}
client.request(url, function(response) {
var data = new Stream();
response.on('data', function(chunk) {
data.push(chunk);
});
response.on('end', function() {
fs.writeFileSync(filename, data.read());
});
}).end();
};
downloadImageToUrl('https://www.google.com/images/srpr/logo11w.png', 'public/uploads/users/abc.jpg');
if you want progress download try this:
var fs = require('fs');
var request = require('request');
var progress = require('request-progress');
module.exports = function (uri, path, onProgress, onResponse, onError, onEnd) {
progress(request(uri))
.on('progress', onProgress)
.on('response', onResponse)
.on('error', onError)
.on('end', onEnd)
.pipe(fs.createWriteStream(path))
};
how to use:
var download = require('../lib/download');
download("https://www.google.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png", "~/download/logo.png", function (state) {
console.log("progress", state);
}, function (response) {
console.log("status code", response.statusCode);
}, function (error) {
console.log("error", error);
}, function () {
console.log("done");
});
note: you should install both request & request-progress modules using:
npm install request request-progress --save
This is an extension to Cezary's answer. If you want to download it to a specific directory, use this. Also, use const instead of var. Its safe this way.
const fs = require('fs');
const request = require('request');
var download = function(uri, filename, callback){
request.head(uri, function(err, res, body){
request(uri).pipe(fs.createWriteStream(filename)).on('close', callback);
});
};
download('https://www.google.com/images/srpr/logo3w.png', './images/google.png', function(){
console.log('done');
});
Building on the above, if anyone needs to handle errors in the write/read streams, I used this version. Note the stream.read() in case of a write error, it's required so we can finish reading and trigger close on the read stream.
var download = function(uri, filename, callback){
request.head(uri, function(err, res, body){
if (err) callback(err, filename);
else {
var stream = request(uri);
stream.pipe(
fs.createWriteStream(filename)
.on('error', function(err){
callback(error, filename);
stream.read();
})
)
.on('close', function() {
callback(null, filename);
});
}
});
};
I got a large list of URLs (8000+) I want to download the images from.
I created a script that will download the files, as long as I'm setting the limit to about a 100.
If I try to download more I'm getting errors like
(node:6740) UnhandledPromiseRejectionWarning: Error: read ECONNRESET
(node:3808) UnhandledPromiseRejectionWarning: Error: connect ETIMEDOUT
(node:7052) UnhandledPromiseRejectionWarning: Error: Client network socket disconnected before secure TLS connection was established
This is the code that reads my CSV with URL's:
const fotoDownload = require('./async-foto');
const csv = require('csv-parser');
const fs = require('fs');
const results = [];
fs.createReadStream('\\\\hk-nas02\\import\\Partij\\Files\\partij.csv')
.pipe(csv( { separator: ';'}))
.on('data', (data) => results.push(data))
.on('end', () => {
console.log(results.length)
let NoPartijen = results.length;
for(i = 0;i < 50; i++){
//console.log(results[i]);
itemno = i
path = '..\\files\\images\\'+results[i]['partij.VPARTIJNR']+'_'+results[i]['partij.PARTIJNR']+'_H.jpg';
console.log(path)
fotoDownload.fotoDownload(results[i]['partij.EXFOTOURL'], path, itemno)
}
console.log('Test');
});
and calls the following code to download:
const util = require('util')
const fs = require('fs')
const axios = require("axios").default;
module.exports = {
fotoDownload: async (url, path, itemno) => {
try {
const response = await axios({
method: "GET",
url: url,
responseType: "stream",})
await response.data.pipe(fs.createWriteStream(path));
console.log('Start foto download' + itemno);
return;
} catch (err) {
throw new Error(err)
}
}
}
I'm assuming I need the chop the data into chunks or something, but I'm a bit lost here. Can someone put me in the right direction?
You need to set a timeout for downloading the images & add connection keep-alive
something like this
axios.defaults.timeout = 30000; //or whateve your desired timeout
axios.defaults.httpsAgent = new https.Agent({ keepAlive: true });
module.exports = {
fotoDownload: async (url, path, itemno) => {
try {
const response = await axios({
method: "GET",
url: url,
responseType: "stream",})
await response.data.pipe(fs.createWriteStream(path));
console.log('Start foto download' + itemno);
return;
} catch (err) {
throw new Error(err)
}
}
}
Note: make sure u have https installed
run npm i http or npm i https
I'm trying to upload an external url to my server. Here's what I got so far
var fs = require('fs');
var request = require('request');
var path = require('path');
const imagesFolder = 'downloadedAssets/imgs';
function download(url, dest, filename, cb) {
var file = fs.createWriteStream(dest + "/" + filename + path.extname(url));
request( {url: url}, function(err, response) {
if(err) {
console.log(err.message);
return;
}
response.pipe(file);
file.on('error', function(err) {
console.log(err.message);
file.end();
});
file.on('finish', function() {
file.close(cb);
});
});
}
and then executing the function...
var url = 'http://pngimg.com/uploads/spongebob/spongebob_PNG44.png';
download(url, imagesFolder, 'sponge', function onComplete(err) {
if (err) {
console.log(err.message);
} else {
console.log('image uploaded to server');
}
});
This doesn't throw any errors, and it creates a file name sponge.png, but the file is empty. Any idea why?
You might have mixed up the examples on the official website
Try using pipe() like below.
function download(url, dest, filename, cb) {
var file = fs.createWriteStream(dest + "/" + filename + path.extname(url));
request( {url: url}).pipe(file);
}
I am trying to download a File from google drive using Google SDK API using node.js.
But I am unable to write/save file at server side - node.js
Code:-
var GoogleTokenProvider = require("refresh-token").GoogleTokenProvider,
async = require('async'),
fs = require("fs"),
request = require('request'),
_accessToken;
var _XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest;
var https = require('https');
const CLIENT_ID = "";
const CLIENT_SECRET = "";
const REFRESH_TOKEN = '';
const ENDPOINT_OF_GDRIVE = 'https://www.googleapis.com/drive/v2';
async.waterfall([
//-----------------------------
// Obtain a new access token
//-----------------------------
function(callback) {
var tokenProvider = new GoogleTokenProvider({
'refresh_token': REFRESH_TOKEN,
'client_id': CLIENT_ID,
'client_secret': CLIENT_SECRET
});
tokenProvider.getToken(callback);
},
//--------------------------------------------
// Retrieve the children in a specified folder
//
// ref: https://developers.google.com/drive/v2/reference/files/children/list
//-------------------------------------------
function(accessToken, callback) {
_accessToken = accessToken;
request.get({
'url': ENDPOINT_OF_GDRIVE + '/files?' + "q='root' in parents and (mimeType = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document')",
'qs': {
'access_token': accessToken
}
}, callback);
},
//----------------------------
// Parse the response
//----------------------------
function(response, body, callback) {
var list = JSON.parse(body);
if (list.error) {
return callback(list.error);
}
callback(null, list.items[0]);
},
//-------------------------------------------
// Get the file information of the children.
//
// ref: https://developers.google.com/drive/v2/reference/files/get
//-------------------------------------------
function(children, callback) {
var xhr = new _XMLHttpRequest();
xhr.open('GET', children.downloadUrl);
xhr.setRequestHeader('Authorization', 'Bearer ' + _accessToken);
xhr.onload = function() {
console.log("xhr.responseText", xhr.responseText)
fs.writeFile("download.docx", xhr.responseText)
callback(xhr.responseText);
};
xhr.onerror = function() {
callback(null);
};
xhr.send();
}
],
function(err, results) {
if (!err) {
console.log(results);
}
});
I am getting this in console:-
Content of xhr.responseText is something like that
��▬h��↕E6M��~��3�3∟�9�� � �►��/2�:���♂�4��]�♀I�R���►
$SB6Q���c↔��H�=;+
���►q�3Tdכ��#!T��hEl_�|�I�↨��h(�^:▬�[h̓D♠��f���♠*���ݾ��M→
�1⌂♦"N�↑�o�]�7U$��A6����♠�W��k`�f▬♫��K�Z�^‼�0{<Z�▼�]F�����
���J♥A♀��♣�a�}7�
"���H�w"�♥���☺w♫̤ھ�� �P�^����O֛���;�<♠�aYՠ؛`G�kxm��PY�[��g
Gΰino�/<���<�1��ⳆA$>"f3��\�ȾT��∟I S�������W♥����Y
Please help me to know what is the format of the data I am getting from Drive Api and write it in which format so that I got a complete .docx file
Edit
I am open to use any method other than xmlRequest if it helps me downloading the file(.docx).
node-XMLHttpRequest, it seems, does not support binary downloads - see this issue. What you are seeing is the file's binary contents converted into String which, in JavaScript, is an irreversible and destructive process for binary data (which means you cannot convert the string back to buffer and get the same data as the original contents).
Using request, you can download a binary file this way:
var request = require('request')
, fs = require('fs')
request.get(
{ url: 'your-file-url'
, encoding: null // Force Request to return the data as Buffer
, headers:
{ Authorization: 'Bearer ' + accessTokenHere
}
}
, function done (err, res) {
// If all is well, the file will be at res.body (buffer)
fs.writeFile('./myfile.docx', res.body, function (err) {
// Handle err somehow
// Do other work necessary to finish the request
})
}
)
Note: This will buffer the whole file into memory before it can be saved to disk. For small files, this is fine, but for larger files, you might want to look into implementing this as a streamed download. This SO question already answers that, I recommend you have a look.
More information about how to authorize your requests can be found on Google Developers docs.
Complete Working example: Downloading file from GoogleDrive - Node.js API
var GoogleTokenProvider = require("refresh-token").GoogleTokenProvider,
async = require('async'),
fs = require("fs"),
request = require('request'),
_accessToken;
const CLIENT_ID = "";
const CLIENT_SECRET = "";
const REFRESH_TOKEN = '';
const ENDPOINT_OF_GDRIVE = 'https://www.googleapis.com/drive/v2';
async.waterfall([
//-----------------------------
// Obtain a new access token
//-----------------------------
function(callback) {
var tokenProvider = new GoogleTokenProvider({
'refresh_token': REFRESH_TOKEN,
'client_id': CLIENT_ID,
'client_secret': CLIENT_SECRET
});
tokenProvider.getToken(callback);
},
//--------------------------------------------
// Retrieve the children in a specified folder
//
// ref: https://developers.google.com/drive/v2/reference/files/children/list
//-------------------------------------------
function(accessToken, callback) {
_accessToken = accessToken;
request.get({
'url': ENDPOINT_OF_GDRIVE + '/files?' + "q='root' in parents and (mimeType = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document')",
'qs': {
'access_token': accessToken
}
}, callback);
},
//----------------------------
// Parse the response
//----------------------------
function(response, body, callback) {
var list = JSON.parse(body);
if (list.error) {
return callback(list.error);
}
callback(null, list.items);
},
//-------------------------------------------
// Get the file information of the children.
//
// ref: https://developers.google.com/drive/v2/reference/files/get
//-------------------------------------------
function(children, callback) {
for(var i=0;i<children.length;i++) {
var file = fs.createWriteStream(children[i].title);
// Downnload and write file from google drive
(function(child) {
request.get(
{ url: child.downloadUrl
, encoding: null // Force Request to return the data as Buffer
, headers:
{ Authorization: 'Bearer ' + _accessToken
}
}
, function done (err, res) {
res.pipe(file)
// If all is well, the file will be at res.body (buffer)
fs.writeFile('./' + child.title, res.body, function (err) {
if(!err) {
console.log('done')
} else {
console.log(err)
}
// Handle err somehow
// Do other work necessary to finish the request
})
}
)
})(children[i])
}
}
],
function(err, results) {
if (!err) {
console.log(results);
}
});
I was just having issues with this, I've included an example of how I managed to get this working using the Google API Node.js library: https://gist.github.com/davestevens/6f376f220cc31b4a25cd