Related
I am generating a file from url using fs module to my local system. and it is hopefully success. but when I try to post that file into group using createReadStream() function it is sending me a corrupt file.
Currently I tried with excel file.
function saveFileToLocal(urlData,fileName){
return new Promise ((res,rej) => {
https.get(urlData, function(response) {
if (response.statusCode === 200) {
var file = fs.createWriteStream(path+`/controller/localmaping/files/${fileName}`)
response.pipe(file);
file.on('finish', function() {
file.close(); // close() is async
});
res('done')
}
else{
rej('failed')
}
})
})
}
async function postAttachmentToGroup(groupid, senderId, text, urlData) {
let fileName = urlData.substring(urlData.lastIndexOf('/') + 1).split("?")[0] // File name
await saveFileToLocal(urlData,fileName)
console.log(fileName)
console.log(path)
request({
url: "https://graph.facebook.com/group_file_revisions",
headers: { "Content-Type": "form-data","Authorization":"Bearer "+config.ACCESS_TOKEN },
method: "POST",
formData:{"file":fs.createReadStream(path+`/controller/localmaping/files/${fileName}`)}
}, (err, res, body) => {
if (!err) {
let id = JSON.parse(body).id;
let request_body = {
"message": text + "\nPost by #[" + senderId + "]",
"formatting":"MARKDOWN",
"files": [ id ]
}
request({
"uri": `https://graph.workplace.com/${groupid}/feed`,
"qs": { "access_token": config.ACCESS_TOKEN },
"method": "POST",
"json": request_body
}, (err, res, body) => {
if (!err) {
console.log(body)
console.log('Post Success in feed!')
}
else {
console.error("Unable to Post in feed" + err);
}
});
}
else {
console.error("Unable to Post in feed" + err);
}
})
}
I solved this problem by adding callback function. Also added a functionality to delete file from the local storage after successful file sending process.
function postAttachmentToGroup(groupid, senderId, text, urlData) {
let fileName = urlData.substring(urlData.lastIndexOf('/') + 1).split("?")[0]
https.get(urlData, function(response) {
if (response.statusCode === 200) {
var file = fs.createWriteStream(path+`/controller/localmaping/files/${fileName}`)
response.pipe(file);
file.on('finish', function() {
request({
url: "https://graph.facebook.com/group_file_revisions",
headers: { "Content-Type": "multipart/form-data","Authorization":"Bearer "+config.ACCESS_TOKEN },
method: "POST",
formData:{"file":fs.createReadStream(path+`/controller/localmaping/files/${fileName}`)}
}, (err, res, body) => {
if (!err) {
let id = JSON.parse(body).id;
console.log(id)
let request_body = {
"message": text + "\nPost by #[" + senderId + "]",
"formatting":"MARKDOWN",
"files": [ id ]
}
request({
"uri": `https://graph.workplace.com/${groupid}/feed`,
"qs": { "access_token": config.ACCESS_TOKEN },
"method": "POST",
"json": request_body
}, (err, res, body) => {
if (!err) {
console.log(body)
console.log('Post Success in feed!')
fs.unlink(path+`/controller/localmaping/files/${fileName}`,function (err){
if(err){
console.log("error in deleting file")
}
else{
console.log("file deleted successfully")
}
})
}
else {
console.error("Unable to Post in feed" + err);
}
});
}
else {
console.error("Unable to Post in feed" + err);
}
})
});
}
else{
console.log('failed')
}
})
}
I am working on MERN Stack Application(Mean,Express,ReactJS,NodeJS). I have one issue is that I have many more methods in mlcontroller.js page and I call some methods on REST API so I call that methods under that API from mlrouter.js but all that API is Async so currently API takes data slots vise means I give u an example that in one time take 100 data from first method and then pass to another method and pass from all methods again come to first method and take next 100 data and repeat same process again but I need to take all data in one time means untill one method will not complete not move on another method how's that possible with node js?
I place my code here :
mlrouter.js
ensureApiAuthenticated,
authController.checkReadOnlyUser,
mlController.getAPIData,
mlController.getData,
mlController.getCRCDetails,
mlController.getDetails,
mlController.uploadData
)
MlController.js
async function getAPIData(req, res, next) {
try {
let loanboardapi = " ", dealersocket = " ";
loanboardapi = {
url: "https://loanboard.houstondirectauto.com/api/User/GetAuthorizationToken?username=amin#houstondirectauto.com&password=test#123",
method: "GET"
};
dealersocket = {
url: 'https://idms.dealersocket.com/api/authenticate/GetUserAuthorizationToken?username=ankur#houstondirectauto.com&password=H5d465#!ddfdd45dsfd688&InstitutionID=105815',
method: 'GET'
};
request(loanboardapi,
(err, res, body) => {
console.log("res = ", res);
console.log("body =", body);
loantoken = JSON.parse(body).token;
console.log(loantoken);
});
request(dealersocket,
(err, res, body) => {
console.log("res = ", res);
console.log("body =", body);
dealertoken = JSON.parse(body).Token;
console.log(dealertoken);
next();
});
}
catch (e) {
req.error = e;
next();
}
}
function getData(req, res, next) {
try {
let result;
request.get('https://idms.dealersocket.com/api/account/getaccountlist?token=' + dealertoken + '&LayoutID=2002313&PageNumber=1&accounttype=i&accountstatus=a,c,b,o,r,s,x',
(err, res, body) => {
console.log("res = ", res);
console.log("body =", body);
result = JSON.parse(body);
console.log(result);
totalpage = parseInt(result.TotalPages);
let resultdata = Object.assign({}, result.Data);
console.log(resultdata);
//getSSN(totalpage, dealertoken, next);
next();
})
}
catch (e) {
req.error = e;
next();
}
}
async function getCRCDetails(req,res,next) {
async.eachSeries(ssn, async (item) => {
let CBCOptions = {
method: "POST",
url: "https://loanboard.houstondirectauto.com/api/Report",
headers: {
"Content-Type": "application/json",
Cookie: "ci_session=udmojmlc5tfl3epbrmtvgu6nao2f031p",
},
body: JSON.stringify({
token: loantoken,
action: "CBCReport",
variables: {
ssn: item,
},
}),
};
let EMpInfoption = {
method: "POST",
url: "https://loanboard.houstondirectauto.com/api/Report",
headers: {
"Content-Type": "application/json",
Cookie: "ci_session=udmojmlc5tfl3epbrmtvgu6nao2f031p",
},
body: JSON.stringify({
token: loantoken,
action: "getEmployerInfo",
variables: {
ssn: item,
},
}),
};
try {
let resultCBCOptions = await requestpromise(CBCOptions);
let EmployerInfoOptions = await requestpromise(EMpInfoption);
console.log(resultCBCOptions)
console.log(EmployerInfoOptions)
CRCOptions.push(resultCBCOptions);
EmpOption.push(EmployerInfoOptions);
} catch (error) {
console.log(error);
}
},
() => {
next();
}
);
}
async function getDetails(req,res,next) {
for(let i =0;i<CRCOptions.length;i++){
for(let j=0;j<EmpOption.length;j++){
let resdata = JSON.parse(CRCOptions[i]);
console.log(resdata);
result = resdata.data.DigifiResponse;
console.log(result);
let bodydata = JSON.parse(EmpOption[i]).data;
let crcssn = JSON.parse(CRCOptions[i]).ssn;
let empssn = JSON.parse(EmpOption[i]).ssn;
console.log("CRCSSN=",crcssn);
console.log("EMPSSN=",empssn);
if(crcssn == empssn)
{
for(let r=0;r<result.length;r++){
let crcdata = result[r];
console.log(crcdata);
for(let b=0;b<bodydata.length;b++) {
let annual_income;
console.log(bodydata[b]);
let mergedata = Object.assign(crcdata, bodydata[b]);
console.log("merge", mergedata);
if (mergedata["IncomeFrequency"] == "Monthly") {
annual_income = (parseInt(mergedata["Income"]) * 12).toString();
console.log(annual_income);
}
else {
annual_income = mergedata["Income"];
}
let binary = {
"accounts_opened": mergedata["total_number_of_open_accounts"],
"bankruptcies": mergedata["total_number_of_bankruptcies"],
"collections": mergedata["total_number_of_collections"],
"credit_inquiries_last_6_months": mergedata["total_number_of_inquires_in_the_last_6_months"],
"past_due_accounts": mergedata["total_number_of_accounts_currently_past_due"],
"open_accounts": mergedata["total_number_of_open_accounts"],
"high_credit_limit": mergedata["total_credit_limit_amount"],
"annual_income": annual_income
}
console.log(binary);
let arraybinary = Object.assign({},binary);
console.log(arraybinary);
binarydata.push(arraybinary);
console.log(binarydata);
let categorical = {
"bankruptcies_last_18_months": mergedata["count_of_bankruptcies_last_24_months"],
"credit_inquiries_last_6_months": mergedata["count_of_auto_loan_inquiries_last_9_months"],
"months_since_most_recent_inquiry": mergedata["total_number_of_inquires_in_the_last_6_months"],
"ninety_plus_delinquencies_last_18_months": mergedata["total_number_of_accounts_with_90180_day_delinquencies"],
"number_of_accounts_currently_30dpd": mergedata["total_number_of_accounts_with_3059_day_delinquencies"],
"open_credit_accounts": mergedata["total_number_of_open_auto_accounts"],
"pre_loan_debt_to_income": mergedata["total_amount_of_credit_debt"],
"total_current_balance": mergedata["total_account_balance"],
"total_high_credit_limit": mergedata["total_credit_limit_amount"],
"annual_income": annual_income
}
console.log(categorical);
let arraycategory = Object.assign({},categorical);
console.log(arraycategory);
categoricaldata.push(arraycategory);
let Linear = {
"bankruptcies_last_18_months": mergedata["count_of_bankruptcies_last_24_months"],
"credit_inquiries_last_6_months": mergedata["count_of_auto_loan_inquiries_last_9_months"],
"months_since_most_recent_inquiry": mergedata["total_number_of_inquires_in_the_last_6_months"],
"ninety_plus_delinquencies_last_18_months": mergedata["total_number_of_accounts_with_90180_day_delinquencies"],
"number_of_accounts_currently_30dpd": mergedata["total_number_of_accounts_with_3059_day_delinquencies"],
"open_credit_accounts": mergedata["total_number_of_open_auto_accounts"],
"pre_loan_debt_to_income": mergedata["total_amount_of_credit_debt"],
"total_current_balance": mergedata["total_account_balance"],
"total_high_credit_limit": mergedata["total_credit_limit_amount"],
"annual_income": annual_income
}
console.log(Linear);
let arraylinear = Object.assign({},Linear);
console.log(arraylinear);
Lineardata.push(arraylinear);
}
}
}
break;
}
}
console.log(binarydata.length);
console.log(binarydata);
converter.json2csv(binarydata,(err,csv) => {
if(err)throw err;
console.log(csv);
file.writeFileSync('/home/rita_gatistavam/Downloads/CSV/binarydata.csv',csv);
console.log('File Written');
})
converter.json2csv(Lineardata,(err,csv) => {
if(err)throw err;
console.log(csv);
file.writeFileSync('/home/rita_gatistavam/Downloads/CSV/lineardata.csv',csv);
console.log('File Written');
})
converter.json2csv(categoricaldata,(err,csv) => {
if(err)throw err;
console.log(csv);
file.writeFileSync('/home/rita_gatistavam/Downloads/CSV/categorydata.csv',csv);
console.log('File Written');
})
next();
}
async function uploadData(req,res,next){
let moduletype = sessionStorage.getItem('moduletype');
console.log(moduletype);
req.params.id = sessionStorage.getItem('modelid');
console.log(req.params.id);
try {
res.status(200).send({
status: 200,
timeout: 10000,
type: 'success',
text: 'Changes saved successfully!',
successProps: {
successCallback: 'func:window.createNotification',
},
responseCallback: 'func:this.props.reduxRouter.push',
pathname: `/ml/models/${req.params.id}/training/historical_data_${moduletype}`,
});
} catch (e) {
periodic.logger.warn(e.message);
res.status(500).send({ message: 'Error updating model type.', });
}
}```
Cannot understand your question, but I assume you want to get all the async request in one go.
You can achieve this with Promise.all, all the results will be returned as an array, and all the request will run at the same time.
const results = await Promise.all([asyncRequest1, asyncRequest2, asyncRequest3])
getting resulsts sequentially.
await asyncRequest1();
await asyncRequest2();
await asyncRequest3();
I am a beginner to callback concept and looking for a solution to my problem.
I calling third party API using request package in node.js here is the code:
In reusable library file: auth.js
let getAuthToken = () => {
let authToken;
var options = {
'method': 'GET',
'url': 'https://<apiURL>/V1/auth_token',
'headers': {
'Authorization': 'Basic <token>'
}
};
request(options, (error, response) => {
if (error) {
throw new Error(error);
} else {
authToken = JSON.parse(response.body);
}
});
return authToken;
}
on my route: http://127.0.0.1:3000/api/v1/musics/authorize-account, I am calling my controller function named "getAuthorizationToken()"
controllerfile: music.controller.js
const auth = require('../middleware/auth');
let getAuthorizationToken = async (req, res, next) => {
let token = await auth.getAuthToken();
console.log(auth.getAuthToken());
res.send(token);
}
Problem is the controller function is getting executed completely and then the third party API is being called event I have added await to the function.
Do explain to me the problem I am facing and any workaround solution will be heartily helpful.
You have to return a promise to be able to await something and have it work as expected:
let getAuthToken = () => {
let authToken;
var options = {
'method': 'GET',
'url': 'https://<apiURL>/V1/auth_token',
'headers': {
'Authorization': 'Basic <token>'
}
};
return new Promise((resolve, reject) => {
request(options, (error, response) => {
if (error) {
reject(error);
} else {
authToken = JSON.parse(response.body);
resolve(authToken);
}
});
})
}
await is only useful on promises. In your case your getAuthToken does not return an promise. But you can change it.
let getAuthToken = () => {
return new Promise((res, rej) => {
let authToken;
var options = {
method: "GET",
url: "https://<apiURL>/V1/auth_token",
headers: {
Authorization: "Basic <token>"
}
};
request(options, (error, response) => {
if (error) {
rej(error);
} else {
authToken = JSON.parse(response.body);
res(authToken);
}
});
});
};
In addition you should also wrap your await in a try / catch
let getAuthorizationToken = async (req, res, next) => {
try {
let token = await auth.getAuthToken();
console.log(token);
return res.send(token);
} catch(err) {
console.log(err);
return res.status(500).send(err);
}
}
Instead of an 500 error you should send an different error code like:
400 Bad request: If there are some missing credentials like the token is missing
401 Unauthorized: If the token is wrong
I want to return the value that has been declared in the first Function CreateTag and using it as variable in the second Function CreateStream, but it won't work..
I'm working with nodejs Express.
I try to use RETURN but it won't work..
I have tried it in differance ways, but still not work..
Can you someone help me, please?
'use strict';
var express = require('express');
var router = express.Router();
/* GET home page. */
//Function 1: createTag
var createTag = function hi (TanentValue) {
var https = require('https');
var data = JSON.stringify({
name: TanentValue,
schemaPath: "Tag"
});
var options = {
hostname: 'qlik_dev.be',
path: '/meteor/qrs/tag?xrfkey=1234567890123456',
method: 'POST',
headers: {
'x-qlik-xrfkey': '1234567890123456',
'hdr-usr': 'gak\\gaka',
'Content-Type': 'application/json'
},
};
var req = https.request(options, (res) => {
//console.log(res)
res.on('data', (d) => {
console.log("hi tag")
var getResult = "GaLvAnI"; // ----> return this and use it into the function createStream
return getResult;
})
})
;
req.on('error', (error) => {
console.error(error)
});
req.write(data);
req.end();
}
//Function 2: createStream
var createStream = function (TanentValue) {
var https = require('https');
var galvani = hi(); // --------> here I made a variable to call return value
var data = JSON.stringify({
name: TanentValue,
});
var options = {
hostname: 'qlik_dev.be',
path: '/meteor/qrs/stream?xrfkey=1234567890123456',
method: 'POST',
headers: {
'x-qlik-xrfkey': '1234567890123456',
'hdr-usr': 'gak\\gaka',
'Content-Type': 'application/json'
},
};
var req = https.request(options, (res) => {
res.on('data', (d) => {
console.log(galvani); // -----> use the variable here
})
})
;
req.on('error', (error) => {
console.error(error)
});
req.write(data);
req.end();
}
//homepage
router.get('/', function (req, res) {
res.render('index', { title: 'MCS Test' });
});
//create
router.post('/create', function (req, res) {
//create tag
console.log('POST / Call Create Tag');
createTag(req.body.TanentValue);
//create stream
console.log('POST / Call Create Stream');
createStream(req.body.TanentValue);
res.send('Stream and Tag has been created');
});
module.exports = router;
you can not directly return value from async function. you have to use promise. something like this:
'use strict';
var express = require('express');
var router = express.Router();
/* GET home page. */
//Function 1: createTag
var createTag = function (TanentValue) { // function should be anonymouse
return new Promise((resolve, reject) => {
var https = require('https');
var data = JSON.stringify({
name: TanentValue,
schemaPath: "Tag"
});
var options = {
hostname: 'qlik_dev.be',
path: '/meteor/qrs/tag?xrfkey=1234567890123456',
method: 'POST',
headers: {
'x-qlik-xrfkey': '1234567890123456',
'hdr-usr': 'gak\\gaka',
'Content-Type': 'application/json'
},
};
var req = https.request(options, (res) => {
//console.log(res)
res.on('data', (d) => {
console.log("hi tag")
var getResult = "GaLvAnI"; // ----> return this and use it into the function createStream
resolve(getResult); // success call
})
})
;
req.on('error', (error) => {
reject(error); // error call
});
req.write(data);
req.end();
});
}
//Function 2: createStream
var createStream = function (TanentValue) {
createTag().then((val) => {
var https = require('https');
var galvani = val; // use that value from sucess call
var data = JSON.stringify({
name: TanentValue,
});
var options = {
hostname: 'qlik_dev.be',
path: '/meteor/qrs/stream?xrfkey=1234567890123456',
method: 'POST',
headers: {
'x-qlik-xrfkey': '1234567890123456',
'hdr-usr': 'gak\\gaka',
'Content-Type': 'application/json'
},
};
var req = https.request(options, (res) => {
res.on('data', (d) => {
console.log(galvani); // -----> use the variable here
})
})
;
req.on('error', (error) => {
console.error(error)
});
req.write(data);
req.end();
})
.catch((error) => {
// handle error from createTag function here
});
}
//homepage
router.get('/', function (req, res) {
res.render('index', { title: 'MCS Test' });
});
//create
router.post('/create', function (req, res) {
//create tag
console.log('POST / Call Create Tag');
createTag(req.body.TanentValue);
//create stream
console.log('POST / Call Create Stream');
createStream(req.body.TanentValue);
res.send('Stream and Tag has been created');
});
module.exports = router;
You can solve it using just callback function or the promise.
Using callbacks.
'use strict';
var express = require('express');
var router = express.Router();
/* GET home page. */
//Function 1: createTag
var createTag = (TanentValue, callback) => {
var https = require('https');
var data = JSON.stringify({
name: TanentValue,
schemaPath: "Tag"
});
var options = {
hostname: 'qlik_dev.be',
path: '/meteor/qrs/tag?xrfkey=1234567890123456',
method: 'POST',
headers: {
'x-qlik-xrfkey': '1234567890123456',
'hdr-usr': 'gak\\gaka',
'Content-Type': 'application/json'
},
};
var req = https.request(options, (res) => {
res.on('data', (d) => {
console.log("hi tag")
var getResult = "GaLvAnI"; // ----> return this and use it into the function createStream
callback(false, getResult);
})
});
req.on('error', (error) => {
//console.error(error)
callback(true, error);
});
req.write(data);
req.end();
}
//Function 2: createStream
var createStream = (TanentValue, callback) => {
var https = require('https');
var data = JSON.stringify({
name: TanentValue,
});
var options = {
hostname: 'qlik_dev.be',
path: '/meteor/qrs/stream?xrfkey=1234567890123456',
method: 'POST',
headers: {
'x-qlik-xrfkey': '1234567890123456',
'hdr-usr': 'gak\\gaka',
'Content-Type': 'application/json'
},
};
createTag(TanentValue, (is_error, galvani) => {
if(err || !data){
// do error handling...
callback(true); // true for there was an error
}else{
var req = https.request(options, (res) => {
res.on('data', (d) => {
callback(false);
console.log(galvani); // -----> use the variable here
})
});
req.on('error', (error) => {
callback(true);
console.error(error)
});
req.write(data);
req.end();
}
})
}
//homepage
router.get('/', function (req, res) {
res.render('index', { title: 'MCS Test' });
});
//create
router.post('/create', function (req, res) {
/*
// Since the stream seems to depend on the tag created,
// you don't need to call createTag explicitly because
// it is always/already called from createStream.
//create tag
console.log('POST / Call Create Tag');
createTag(req.body.TanentValue, function(is_error, data){
if(!is_error){
// do something
}else{
// do error handling
console.error(error);
res.send('Tag could not be created, please try later again..');
}
});
*/
//create stream
console.log('POST / Call Create Stream');
createStream(req.body.TanentValue, is_error => {
if(!is_error){
res.send('Stream and Tag has been created');
}else{
res.send('Stream could not be created, please try later again..');
}
});
});
module.exports = router;
Using Promise
'use strict';
var express = require('express');
var router = express.Router();
/* GET home page. */
//Function 1: createTag
var createTag = TanentValue => {
var https = require('https');
var data = JSON.stringify({
name: TanentValue,
schemaPath: "Tag"
});
var options = {
hostname: 'qlik_dev.be',
path: '/meteor/qrs/tag?xrfkey=1234567890123456',
method: 'POST',
headers: {
'x-qlik-xrfkey': '1234567890123456',
'hdr-usr': 'gak\\gaka',
'Content-Type': 'application/json'
},
};
return new Promise((resolve, reject) => {
var req = https.request(options, (res) => {
res.on('data', (d) => {
console.log("hi tag")
var getResult = "GaLvAnI"; // ----> return this and use it into the function createStream
resolve(getResult);
})
});
req.on('error', (error) => {
//console.error(error)
reject(error);
});
req.write(data);
req.end();
})
}
//Function 2: createStream
var createStream = TanentValue => {
var https = require('https');
var data = JSON.stringify({
name: TanentValue,
});
var options = {
hostname: 'qlik_dev.be',
path: '/meteor/qrs/stream?xrfkey=1234567890123456',
method: 'POST',
headers: {
'x-qlik-xrfkey': '1234567890123456',
'hdr-usr': 'gak\\gaka',
'Content-Type': 'application/json'
},
};
createTag(TanentValue).then( galvani => {
return new Promise((resolve, reject) => {
var req = https.request(options, (res) => {
res.on('data', (d) => {
console.log(galvani); // -----> use the variable here
resolve(d);
})
});
req.on('error', (error) => {
console.error(error)
reject({ msg: 'request error while creating the stream', error: error})
});
req.write(data);
req.end();
})
}).catch( error => {
// do error handling...
reject({msg: 'Error while creating a tag', error: error}); // true for there was an error
});
}
//homepage
router.get('/', function (req, res) {
res.render('index', { title: 'MCS Test' });
});
//create
router.post('/create', function (req, res) {
/*
// Since the stream seems to depend on the tag created,
// you don't need to call createTag explicitly because
// it is always/already called from createStream.
//create tag
console.log('POST / Call Create Tag');
createTag(req.body.TanentValue).then( data => {
// do something
}).catch( error => {
// do error handling
});
*/
//create stream
console.log('POST / Call Create Stream');
createStream(req.body.TanentValue).then( data => {
res.send('Stream and Tag has been created');
}).catch(error => {
// 'Stream could not be created, please try later again..'
res.send(error.msg);
});
});
module.exports = router;
Very handfull!! thank you it works!
But while passing the data (Json) from function 1 to function 2 with Promise, the data (json) is undefine in function 2. If I pass a data (String) from function 1 to function 2 than it works..
Why does it give me 'undefine' when it is a json?
//var id;
var req = https.request(options, (res) => {
//console.log(res)
res.setEncoding('utf8');
res.on('data', function (data) {
var json = JSON.parse(data);
var TagId = JSON.stringify(json[0]);
console.log("2 hi getTap");
console.log(TagId); // -------> here it works well
resolve(TagId);
});
});
var req = https.request(options, (res) => {
res.on('data', (d) => {
console.log("3 hi createStream");
console.log(galvani); // -------> here it doesn't work.. it gives me undefine
})
});
here a printscreen of response
I am trying to write a function that would:
Take a remote URL as a parameter,
Get the file using axios
Upload the stream to amazon s3
And finally, return the uploaded url
I found help here on stackoverflow. So far, I have this:
/*
* Method to pipe the stream
*/
const uploadFromStream = (file_name, content_type) => {
const pass = new stream.PassThrough();
const obj_key = generateObjKey(file_name);
const params = { Bucket: config.bucket, ACL: config.acl, Key: obj_key, ContentType: content_type, Body: pass };
s3.upload(params, function(err, data) {
if(!err){
return data.Location;
} else {
console.log(err, data);
}
});
return pass;
}
/*
* Method to upload remote file to s3
*/
const uploadRemoteFileToS3 = async (remoteAddr) => {
axios({
method: 'get',
url: remoteAddr,
responseType: 'stream'
}).then( (response) => {
if(response.status===200){
const file_name = remoteAddr.substring(remoteAddr.lastIndexOf('/')+1);
const content_type = response.headers['content-type'];
response.data.pipe(uploadFromStream(file_name, content_type));
}
});
}
But uploadRemoteFileToS3 does not return anything (because it's a asynchronous function). How can I get the uploaded url?
UPDATE
I have further improved upon the code and wrote a class. Here is what I have now:
const config = require('../config.json');
const stream = require('stream');
const axios = require('axios');
const AWS = require('aws-sdk');
class S3RemoteUploader {
constructor(remoteAddr){
this.remoteAddr = remoteAddr;
this.stream = stream;
this.axios = axios;
this.config = config;
this.AWS = AWS;
this.AWS.config.update({
accessKeyId: this.config.api_key,
secretAccessKey: this.config.api_secret
});
this.spacesEndpoint = new this.AWS.Endpoint(this.config.endpoint);
this.s3 = new this.AWS.S3({endpoint: this.spacesEndpoint});
this.file_name = this.remoteAddr.substring(this.remoteAddr.lastIndexOf('/')+1);
this.obj_key = this.config.subfolder+'/'+this.file_name;
this.content_type = 'application/octet-stream';
this.uploadStream();
}
uploadStream(){
const pass = new this.stream.PassThrough();
this.promise = this.s3.upload({
Bucket: this.config.bucket,
Key: this.obj_key,
ACL: this.config.acl,
Body: pass,
ContentType: this.content_type
}).promise();
return pass;
}
initiateAxiosCall() {
axios({
method: 'get',
url: this.remoteAddr,
responseType: 'stream'
}).then( (response) => {
if(response.status===200){
this.content_type = response.headers['content-type'];
response.data.pipe(this.uploadStream());
}
});
}
dispatch() {
this.initiateAxiosCall();
}
async finish(){
//console.log(this.promise); /* return Promise { Pending } */
return this.promise.then( (r) => {
console.log(r.Location);
return r.Location;
}).catch( (e)=>{
console.log(e);
});
}
run() {
this.dispatch();
this.finish();
}
}
But still have no clue how to catch the result when the promise is resolved. So far, I tried these:
testUpload = new S3RemoteUploader('https://avatars2.githubusercontent.com/u/41177');
testUpload.run();
//console.log(testUpload.promise); /* Returns Promise { Pending } */
testUpload.promise.then(r => console.log); // does nothing
But none of the above works. I have a feeling I am missing something very subtle. Any clue, anyone?
After an upload you can call the getsignedurl function in s3 sdk to get the url where you can also specify the expiry of the url as well. You need to pass the key for that function. Now travelling will update with example later.
To generate a simple pre-signed URL that allows any user to view the
contents of a private object in a bucket you own, you can use the
following call to getSignedUrl():
var s3 = new AWS.S3();
var params = {Bucket: 'myBucket', Key: 'myKey'};
s3.getSignedUrl('getObject', params, function (err, url) {
console.log("The URL is", url);
});
Official documentation link
http://docs.amazonaws.cn/en_us/AWSJavaScriptSDK/guide/node-examples.html
Code must be something like this
function uploadFileToS3AndGenerateUrl(cb) {
const pass = new stream.PassThrough();//I have generated streams from file. Using this since this is what you have used. Must be a valid one.
var params = {
Bucket: "your-bucket", // required
Key: key , // required
Body: pass,
ContentType: 'your content type',
};
s3.upload(params, function(s3Err, data) {
if (s3Err) {
cb(s3Err)
}
console.log(`File uploaded successfully at ${data.Location}`)
const params = {
Bucket: 'your-bucket',
Key: data.key,
Expires: 180
};
s3.getSignedUrl('getObject', params, (urlErr, urlData) => {
if (urlErr) {
console.log('There was an error getting your files: ' + urlErr);
cb(urlErr);
} else {
console.log(`url: ${urlData}`);
cb(null, urlData);
}
})
})
}
Please check i have update your code might its help you.
/*
* Method to upload remote file to s3
*/
const uploadRemoteFileToS3 = async (remoteAddr) => {
const response = await axios({
method: 'get',
url: remoteAddr,
responseType: 'stream'
})
if(response.status===200){
const file_name = remoteAddr.substring(remoteAddr.lastIndexOf('/')+1);
const content_type = response.headers['content-type'];
response.data.pipe(uploadFromStream(file_name, content_type));
}
return new Promise((resolve, reject) => {
response.data.on('end', (response) => {
console.log(response)
resolve(response)
})
response.data.on('error', () => {
console.log(response);
reject(response)
})
})
};
*
* Method to pipe the stream
*/
const uploadFromStream = (file_name, content_type) => {
return new Promise((resolve, reject) => {
const pass = new stream.PassThrough();
const obj_key = generateObjKey(file_name);
const params = { Bucket: config.bucket, ACL: config.acl, Key: obj_key, ContentType: content_type, Body: pass };
s3.upload(params, function(err, data) {
if(!err){
console.log(data)
return resolve(data.Location);
} else {
console.log(err)
return reject(err);
}
});
});
}
//call uploadRemoteFileToS3
uploadRemoteFileToS3(remoteAddr)
.then((finalResponse) => {
console.log(finalResponse)
})
.catch((err) => {
console.log(err);
});