Develop Reference

Node js use variable outside main function and set order of functions

Introduction
I have a three functions, each one would feed data into then next. The objective is first to retrieve data then authenticate a API key then finally using the generated API key and data retrieve from the first function post to the API in the third function.
Order
First function function to retrieve data from a post.
Second function gets API key requested from a API.
Third function posts data to the API.
Needed functionality
I need the variables retried in the first function and the API key generated in the second function to be available for use in the third function.
Problems and questions
emailUser is not being found to use in the third function
api_key is not being found in the third function
also the functions need to run in order first, second then third
This all works if I was to insert the data manual but when input the variables it dose not work, I understand that it is because the variables being within the function but how do I fix this, also how do I set the order of the functions ?
Full code
// Grab the packages needs and sets server
//---------------------------------- Grab the packages we need and set variables --------------------------------------------------
// --------------------------------------------------------------------------------------------------------------------------------
var express = require('express');
var request = require('request');
var nodePardot = require('node-pardot');
var bodyParser = require('body-parser');
var app = express();
var port = process.env.PORT || 8080;
// Varibles to use in second and third function
var password = 'password';
var userkey = '6767712';
var emailAdmin = 'admin#admin.com';
// start the server
app.listen(port);
app.use(bodyParser.json()); // support json encoded bodies
app.use(bodyParser.urlencoded({extended: true})); // support encoded bodies
console.log('Server started! At http://localhost:' + port);
// First Retrieve posted data from Front-End
//---------------------------------- Retrieve posted data from Front-End -----------------------------------------------------
// ---------------------------------------------------------------------------------------------------------------------------
// POST http://localhost:8080/api/index
app.post('/api/data', function (req, res) {
console.log(req.body);
var Fname = req.body.fname;
var Lname = req.body.lname;
var emailUser = req.body.email;
res.send(Fname + ' ' + Lname + ' ' + emailUser);
});
app.get('/', function (req, res) {
res.send('hello world, Nothing to see here...');
});
// Second Get Posted variables
//---------------------------------- Now authenticate the api and get api_key -----------------------------------------------------
// --------------------------------------------------------------------------------------------------------------------------------
nodePardot.PardotAPI({
userKey: userkey,
email: emailAdmin,
password: password,
// turn off when live
DEBUG: true
}, function (err, client) {
if (err) {
// Authentication failed
// handle error
console.error("Authentication Failed", err)
} else {
// Authentication successful
// gets api key
var api_key = client.apiKey;
console.log("Authentication successful !", api_key);
}
});
// Third Retrieve posted data from Front-End
//---------------------------------- Send all data to API -----------------------------------------------------
// ------------------------------------------------------------------------------------------------------------
// Set the headers
var headers = {
'User-Agent': 'Super Agent/0.0.1',
'Content-Type': 'application/x-www-form-urlencoded'
};
// Configure the request
var options = {
url: 'https://pi.pardot.com/api/prospect/version/4/do/create/email',
method: 'POST',
headers: headers,
form: {
'email': emailUser,
'user_key': userkey,
'api_key': api_key
}
};
// Start the request
request(options, function (error, response, body) {
if (!error && response.statusCode == 200) {
// Print out the response body
console.log("error",body)
}
else {
console.log("Sent Data",body);
}
});

best way is using async package (install with npm install async) that is very famous and useful package in npm your functions will be something like this:
var async=require('async');
var handler = function (req,res) {
async.auto
(
{
getBody: function (cb, results) {
var body=req.body;
//prepare body here then send it to next function
cb(null, body)
},
getApi: ['getBody', function (results, cb) {
var preparedBody=results.getBody;
// get the api here and send it to next function
var apiKey=getApi()
cb(null, {apiKey:apiKey,preparedBody:preparedBody})
}],
third: ['getApi', function (results, cb) {
var preparedBody=results.getApi.preparedBody;
var apiKey=results.getApi.apiKey;
// now data are here
cb(null,true)
}]
},
function (err, allResult) {
// the result of executing all functions goes here
}
)
}

Another way to handle this problem is by allowing the express middleware flow to do those things for you on a separate Router.
I have setup a sample Glitch for your reference using stand in functions to simulate network calls HERE.
In your case, you would have to do something like:
//API route
var express = require('express');
var router = express.Router();
router.post('/data', function (req, res, next) {
console.log(req.body);
req.bundledData = {};
req.bundledData.fname = req.body.fname;
req.bundledData.lname = req.body.lname;
req.bundledData.emailUser = req.body.email;
next();
});
router.use(function(req, res, next){
nodePardot.PardotAPI({
userKey: userkey,
email: emailAdmin,
password: password,
// turn off when live
DEBUG: true
}, function (err, client) {
if (err) {
// Authentication failed
// handle error
console.error("Authentication Failed", err)
} else {
// Authentication successful
// gets api key
req.bundledData.api_key = client.apiKey;
console.log("Authentication successful !", api_key);
next();
}
});
});
router.use(function(req, res, next){
// Set the headers
var headers = {
'User-Agent': 'Super Agent/0.0.1',
'Content-Type': 'application/x-www-form-urlencoded'
};
// Configure the request
var options = {
url: 'https://pi.pardot.com/api/prospect/version/4/do/create/email',
method: 'POST',
headers: headers,
form: {
'email': emailUser,
'user_key': userkey,
'api_key': api_key
}
};
// Start the request
request(options, function (error, response, body) {
if (!error && response.statusCode == 200) {
// Print out the response body
console.log("error",body)
}
else {
console.log("Sent Data",body);
//Processing is complete
res.json({
success:true,
body:body
});
}
});
});

How to download audio file from URL in Node.js?

How to download audio file from URL and store it in local directory?
I'm using Node.js and I tried the following code:
var http = require('http');
var fs = require('fs');
var dest = 'C./test'
var url= 'http://static1.grsites.com/archive/sounds/comic/comic002.wav'
function download(url, dest, callback) {
var file = fs.createWriteStream(dest);
var request = http.get(url, function (response) {
response.pipe(file);
file.on('finish', function () {
file.close(callback); // close() is async, call callback after close completes.
});
file.on('error', function (err) {
fs.unlink(dest); // Delete the file async. (But we don't check the result)
if (callback)
callback(err.message);
});
});
}
No error occured but the file has not been found.

Duplicate of How to download a file with Node.js (without using third-party libraries)?, but here is the code specific to your question:
var http = require('http');
var fs = require('fs');
var file = fs.createWriteStream("file.wav");
var request = http.get("http://static1.grsites.com/archive/sounds/comic/comic002.wav", function(response) {
response.pipe(file);
});

Your code is actually fine, you just don't call the download function. Try adding this to the end :
download(url, dest, function(err){
if(err){
console.error(err);
}else{
console.log("Download complete");
}
});
Also, change the value of dest to something else, like just "test.wav" or something. 'C./test' is a bad path.
I tried it on my machine and your code works fine just adding the call and changing dest.

Here is an example using Axios with an API that may require authorization
const Fs = require('fs');
const Path = require('path');
const Axios = require('axios');
async function download(url) {
let filename = "filename";
const username = "user";
const password = "password"
const key = Buffer.from(username + ':' + password).toString("base64");
const path = Path.resolve(__dirname, "audio", filename)
const response = await Axios({
method: 'GET',
url: url,
responseType: 'stream',
headers: { 'Authorization': 'Basic ' + key }
})
response.data.pipe(Fs.createWriteStream(path))
return new Promise((resolve, reject) => {
response.data.on('end', () => {
resolve();
})
response.data.on('error', () => {
reject(err);
})
})
}

Receiving "Uploading file too fast!" error from Imgur API

I created a node.js server that uses busboy to take requests, and pipe the files to Imgur for upload. However, I keep getting an "Uploading file too fast!" response from Imgur, and I'm not sure exactly what the problem is. Here is the code snippet involving busboy:
var express = require('express');
var Busboy = require('busboy');
var fs = require('fs');
var request = require('request-promise');
var router = express.Router();
router.post('/u', function(req, res, next) {
var busboy = new Busboy({headers: req.headers});
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
if(fieldname == 'image') {
var options = {
uri: 'https://api.imgur.com/3/image',
method: 'POST',
headers: {
'Authorization': 'Client-ID ' + clientID // put client id here
},
form: {
image: file,
type: 'file'
}
};
request(options)
.then(function(parsedBody) {
console.log(parsedBody);
})
.catch(function(err) {
console.log(err);
});
}
});
busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated) {
console.log('field');
});
busboy.on('finish', function() {
res.status(200).end();
});
req.pipe(busboy);
});
As you can see I'm piping the request file directly into my request for imgur. Providing a ReadStream by simply saving the file to disc and then using fs.createReadStream() works perfectly, so I'm not really sure why trying to pipe directly from request to request gives me the error. The exact response I'm getting from Imgur is:
StatusCodeError: 400 - {"data":{"error":"Uploading file too fast!","request":"\/3\/image","method":"POST"},"success":false,"status":400}
If anyone has encountered this before, it would be helpful...

The first issue is that you should be using formData instead of form for file uploads. Otherwise, the request library won't send the correct HTTP request.
The second issue is that the stream object won't have the correct content length until it's fully processed. We can buffer the data ourselves and pass it after the initial file stream from busboy has processed.*
This gives us something that looks like
var express = require('express');
var Busboy = require('busboy');
var fs = require('fs');
var request = require('request-promise');
var router = express.Router();
router.post('/u', function(req, res, next) {
var busboy = new Busboy({headers: req.headers});
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
if(fieldname == 'image') {
// the buffer
file.fileRead = [];
file.on('data', function(data) {
// add to the buffer as data comes in
this.fileRead.push(data);
});
file.on('end', function() {
// create a new stream with our buffered data
var finalBuffer = Buffer.concat(this.fileRead);
var options = {
uri: 'https://api.imgur.com/3/image',
method: 'POST',
headers: {
'Authorization': 'Client-ID ' + clientID // put client id here
},
formData: {
image: finalBuffer,
type: 'file'
}
};
request(options)
.then(function(parsedBody) {
console.log(parsedBody);
})
.catch(function(err) {
console.log(err);
});
});
}
});
busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated) {
console.log('field');
});
busboy.on('finish', function() {
res.status(200).end();
});
req.pipe(busboy);
});
Code for the buffering is from http://thau.me/2014/02/nodejs-streaming-files-to-amazons3/
Lastly, you may want to consider using the request library, as the request-promise library discourages the use of streams. See the github repo for more details: https://github.com/request/request-promise

https request basic authentication node.js

I am really getting crazy looking for this over the web and stackoverflow.
Other posts about this topic talk of http request, not httpS.
I'm coding server side with node.js and I need to make an https request to another website to login
If I use postman tool in chrome trying with https://user:pass#webstudenti.unica.it/esse3/auth/Logon.do everything works fine and I log in.
If I use request library in node I can't login and I get a page with a custom error message about an error in my getting/sending data.
Maybe I am wrong setting the options to pass to request.
var request = require('request');
var cheerio = require('cheerio');
var user = 'xxx';
var pass = 'yyy';
var options = {
url : 'https://webstudenti.unica.it',
path : '/esse3/auth/Logon.do',
method : 'GET',
port: 443,
authorization : {
username: user,
password: pass
}
}
request( options, function(err, res, html){
if(err){
console.log(err)
return
}
console.log(html)
var $ = cheerio.load(html)
var c = $('head title').text();
console.log(c);
})
http://jsfiddle.net/985bs0sc/1/

You're not setting your http auth options correctly (namely authorization should instead be auth). It should look like:
var options = {
url: 'https://webstudenti.unica.it',
path: '/esse3/auth/Logon.do',
method: 'GET',
port: 443,
auth: {
user: user,
pass: pass
}
}

http/https should make no difference in the authentication. Most likely your user/pass needs to be base64 encoded. Try
var user = new Buffer('xxx').toString('base64');
var pass = new Buffer('yyy').toString('base64');
See: https://security.stackexchange.com/questions/29916/why-does-http-basic-authentication-encode-the-username-and-password-with-base64

Don't use npm package request because it is deprecated, use Node native https instead
const https = require('https')
var options = {
host: 'test.example.com',
port: 443,
path: '/api/service/'+servicename,
// authentication headers
headers: {
'Authorization': 'Basic ' + new Buffer(username + ':' + passw).toString('base64')
}
};
//this is the call
request = https.get(options, function(res){
console.log(`statusCode: ${res.statusCode}`)
res.on('data', d => {
process.stdout.write(d)
})
})
req.on('error', error => {
console.error(error)
})
req.end()

With the updated version, I am able to make https call with basic auth.
var request = require('request');
request.get('https://localhost:15672/api/vhosts', {
'auth': {
'user': 'guest',
'pass': 'guest',
'sendImmediately': false
}
},function(error, response, body){
if(error){
console.log(error)
console.log("failed to get vhosts");
res.status(500).send('health check failed');
}
else{
res.status(200).send('rabbit mq is running');
}
})

Use Node.js' URL module to build your URL object. The httpsAgent module is required if you are calling servers w self-signed certificates.
const https = require('https');
const httpsAgent = new https.Agent({
rejectUnauthorized: false,
});
// Allow SELF_SIGNED_CERT, aka set rejectUnauthorized: false
let options = {
agent: httpsAgent
}
let address = "10.10.10.1";
let path = "/api/v1/foo";
let url = new URL(`https://${address}${path}`);
url.username = "joe";
url.password = "password123";
url.agent = httpsAgent
let apiCall = new Promise(function (resolve, reject) {
var data = '';
https.get(url, options, res => {
res.on('data', function (chunk){ data += chunk })
res.on('end', function () {
resolve(data);
})
}).on('error', function (e) {
reject(e);
});
});
try {
let result = await apiCall;
} catch (e) {
console.error(e);
} finally {
console.log('We do cleanup here');
}

HTTP GET Request in Node.js Express

How can I make an HTTP request from within Node.js or Express.js? I need to connect to another service. I am hoping the call is asynchronous and that the callback contains the remote server's response.

Here is a snippet of some code from a sample of mine. It's asynchronous and returns a JSON object. It can do any form of GET request.
Note that there are more optimal ways (just a sample) - for example, instead of concatenating the chunks you put into an array and join it etc... Hopefully, it gets you started in the right direction:
const http = require('http');
const https = require('https');
/**
* getJSON: RESTful GET request returning JSON object(s)
* #param options: http options object
* #param callback: callback to pass the results JSON object(s) back
*/
module.exports.getJSON = (options, onResult) => {
console.log('rest::getJSON');
const port = options.port == 443 ? https : http;
let output = '';
const req = port.request(options, (res) => {
console.log(`${options.host} : ${res.statusCode}`);
res.setEncoding('utf8');
res.on('data', (chunk) => {
output += chunk;
});
res.on('end', () => {
let obj = JSON.parse(output);
onResult(res.statusCode, obj);
});
});
req.on('error', (err) => {
// res.send('error: ' + err.message);
});
req.end();
};
It's called by creating an options object like:
const options = {
host: 'somesite.com',
port: 443,
path: '/some/path',
method: 'GET',
headers: {
'Content-Type': 'application/json'
}
};
And providing a callback function.
For example, in a service, I require the REST module above and then do this:
rest.getJSON(options, (statusCode, result) => {
// I could work with the resulting HTML/JSON here. I could also just return it
console.log(`onResult: (${statusCode})\n\n${JSON.stringify(result)}`);
res.statusCode = statusCode;
res.send(result);
});
UPDATE
If you're looking for async/await (linear, no callback), promises, compile time support and intellisense, we created a lightweight HTTP and REST client that fits that bill:
Microsoft typed-rest-client

Try using the simple http.get(options, callback) function in node.js:
var http = require('http');
var options = {
host: 'www.google.com',
path: '/index.html'
};
var req = http.get(options, function(res) {
console.log('STATUS: ' + res.statusCode);
console.log('HEADERS: ' + JSON.stringify(res.headers));
// Buffer the body entirely for processing as a whole.
var bodyChunks = [];
res.on('data', function(chunk) {
// You can process streamed parts here...
bodyChunks.push(chunk);
}).on('end', function() {
var body = Buffer.concat(bodyChunks);
console.log('BODY: ' + body);
// ...and/or process the entire body here.
})
});
req.on('error', function(e) {
console.log('ERROR: ' + e.message);
});
There is also a general http.request(options, callback) function which allows you to specify the request method and other request details.

Request and Superagent are pretty good libraries to use.
note: request is deprecated, use at your risk!
Using request:
var request=require('request');
request.get('https://someplace',options,function(err,res,body){
if(err) //TODO: handle err
if(res.statusCode === 200 ) //etc
//TODO Do something with response
});

You can also use Requestify, a really cool and very simple HTTP client I wrote for nodeJS + it supports caching.
Just do the following for GET method request:
var requestify = require('requestify');
requestify.get('http://example.com/api/resource')
.then(function(response) {
// Get the response body (JSON parsed or jQuery object for XMLs)
response.getBody();
}
);

This version is based on the initially proposed by bryanmac function which uses promises, better error handling, and is rewritten in ES6.
let http = require("http"),
https = require("https");
/**
* getJSON: REST get request returning JSON object(s)
* #param options: http options object
*/
exports.getJSON = function (options) {
console.log('rest::getJSON');
let reqHandler = +options.port === 443 ? https : http;
return new Promise((resolve, reject) => {
let req = reqHandler.request(options, (res) => {
let output = '';
console.log('rest::', options.host + ':' + res.statusCode);
res.setEncoding('utf8');
res.on('data', function (chunk) {
output += chunk;
});
res.on('end', () => {
try {
let obj = JSON.parse(output);
// console.log('rest::', obj);
resolve({
statusCode: res.statusCode,
data: obj
});
}
catch (err) {
console.error('rest::end', err);
reject(err);
}
});
});
req.on('error', (err) => {
console.error('rest::request', err);
reject(err);
});
req.end();
});
};
As a result you don't have to pass in a callback function, instead getJSON() returns a promise. In the following example the function is used inside of an ExpressJS route handler
router.get('/:id', (req, res, next) => {
rest.getJSON({
host: host,
path: `/posts/${req.params.id}`,
method: 'GET'
}).then(({ statusCode, data }) => {
res.json(data);
}, (error) => {
next(error);
});
});
On error it delegates the error to the server error handling middleware.

Unirest is the best library I've come across for making HTTP requests from Node. It's aiming at being a multiplatform framework, so learning how it works on Node will serve you well if you need to use an HTTP client on Ruby, PHP, Java, Python, Objective C, .Net or Windows 8 as well. As far as I can tell the unirest libraries are mostly backed by existing HTTP clients (e.g. on Java, the Apache HTTP client, on Node, Mikeal's Request libary) - Unirest just puts a nicer API on top.
Here are a couple of code examples for Node.js:
var unirest = require('unirest')
// GET a resource
unirest.get('http://httpbin.org/get')
.query({'foo': 'bar'})
.query({'stack': 'overflow'})
.end(function(res) {
if (res.error) {
console.log('GET error', res.error)
} else {
console.log('GET response', res.body)
}
})
// POST a form with an attached file
unirest.post('http://httpbin.org/post')
.field('foo', 'bar')
.field('stack', 'overflow')
.attach('myfile', 'examples.js')
.end(function(res) {
if (res.error) {
console.log('POST error', res.error)
} else {
console.log('POST response', res.body)
}
})
You can jump straight to the Node docs here

Check out shred. It's a node HTTP client created and maintained by spire.io that handles redirects, sessions, and JSON responses. It's great for interacting with rest APIs. See this blog post for more details.

Check out httpreq: it's a node library I created because I was frustrated there was no simple http GET or POST module out there ;-)

For anyone who looking for a library to send HTTP requests in NodeJS, axios is also a good choice. It supports Promises :)
Install (npm): npm install axios
Example GET request:
const axios = require('axios');
axios.get('https://google.com')
.then(function (response) {
// handle success
console.log(response);
})
.catch(function (error) {
// handle error
console.log(error);
})
Github page
Update 10/02/2022
Node.js integrates fetch in v17.5.0 in experimental mode. Now, you can use fetch to send requests just like you do on the client-side. For now, it is an experimental feature so be careful.

If you just need to make simple get requests and don't need support for any other HTTP methods take a look at: simple-get:
var get = require('simple-get');
get('http://example.com', function (err, res) {
if (err) throw err;
console.log(res.statusCode); // 200
res.pipe(process.stdout); // `res` is a stream
});

Use reqclient: not designed for scripting purpose
like request or many other libraries. Reqclient allows in the constructor
specify many configurations useful when you need to reuse the same
configuration again and again: base URL, headers, auth options,
logging options, caching, etc. Also has useful features like
query and URL parsing, automatic query encoding and JSON parsing, etc.
The best way to use the library is create a module to export the object
pointing to the API and the necessary configurations to connect with:
Module client.js:
let RequestClient = require("reqclient").RequestClient
let client = new RequestClient({
baseUrl: "https://myapp.com/api/v1",
cache: true,
auth: {user: "admin", pass: "secret"}
})
module.exports = client
And in the controllers where you need to consume the API use like this:
let client = require('client')
//let router = ...
router.get('/dashboard', (req, res) => {
// Simple GET with Promise handling to https://myapp.com/api/v1/reports/clients
client.get("reports/clients")
.then(response => {
console.log("Report for client", response.userId) // REST responses are parsed as JSON objects
res.render('clients/dashboard', {title: 'Customer Report', report: response})
})
.catch(err => {
console.error("Ups!", err)
res.status(400).render('error', {error: err})
})
})
router.get('/orders', (req, res, next) => {
// GET with query (https://myapp.com/api/v1/orders?state=open&limit=10)
client.get({"uri": "orders", "query": {"state": "open", "limit": 10}})
.then(orders => {
res.render('clients/orders', {title: 'Customer Orders', orders: orders})
})
.catch(err => someErrorHandler(req, res, next))
})
router.delete('/orders', (req, res, next) => {
// DELETE with params (https://myapp.com/api/v1/orders/1234/A987)
client.delete({
"uri": "orders/{client}/{id}",
"params": {"client": "A987", "id": 1234}
})
.then(resp => res.status(204))
.catch(err => someErrorHandler(req, res, next))
})
reqclient supports many features, but it has some that are not supported by other
libraries: OAuth2 integration and logger integration
with cURL syntax, and always returns native Promise objects.

If you ever need to send GET request to an IP as well as a Domain (Other answers did not mention you can specify a port variable), you can make use of this function:
function getCode(host, port, path, queryString) {
console.log("(" + host + ":" + port + path + ")" + "Running httpHelper.getCode()")
// Construct url and query string
const requestUrl = url.parse(url.format({
protocol: 'http',
hostname: host,
pathname: path,
port: port,
query: queryString
}));
console.log("(" + host + path + ")" + "Sending GET request")
// Send request
console.log(url.format(requestUrl))
http.get(url.format(requestUrl), (resp) => {
let data = '';
// A chunk of data has been received.
resp.on('data', (chunk) => {
console.log("GET chunk: " + chunk);
data += chunk;
});
// The whole response has been received. Print out the result.
resp.on('end', () => {
console.log("GET end of response: " + data);
});
}).on("error", (err) => {
console.log("GET Error: " + err);
});
}
Don't miss requiring modules at the top of your file:
http = require("http");
url = require('url')
Also bare in mind that you may use https module for communicating over secured network. so these two lines would change:
https = require("https");
...
https.get(url.format(requestUrl), (resp) => { ......

## you can use request module and promise in express to make any request ##
const promise = require('promise');
const requestModule = require('request');
const curlRequest =(requestOption) =>{
return new Promise((resolve, reject)=> {
requestModule(requestOption, (error, response, body) => {
try {
if (error) {
throw error;
}
if (body) {
try {
body = (body) ? JSON.parse(body) : body;
resolve(body);
}catch(error){
resolve(body);
}
} else {
throw new Error('something wrong');
}
} catch (error) {
reject(error);
}
})
})
};
const option = {
url : uri,
method : "GET",
headers : {
}
};
curlRequest(option).then((data)=>{
}).catch((err)=>{
})