node.js express, a very strange behaviour - javascript

I have an Express server running.
What i do is upload an Excel File from HTML and then Express parse that file and do some calculating.
In the Excel file, each row has information about address of a User.
For each Address, our Express server will use google map geocoding API to calculate the latitude & longitude.
But, because Google doesn't accept more than 50 request per second to their geocoding API, so i have to use settimeout to delay the calculating.
For example if the Excel file has 50 addresses, so i have to use settimeout on each address to avoid that rate limit.
Here is my code to use settimeout and calculate latitude, longitude
createFromFile(req, res) {
var form = new formidable.IncomingForm();
return form.parse(req, function (err, fields, files) {
if (err)
return res.status(500).json({error: err.message})
var workbook = new exceljs.Workbook();
return workbook.xlsx.readFile(files.excelfile.path).then(function() {
// use workbook
var worksheet = workbook.getWorksheet(1)
var data = []
for (var i=2; i<=worksheet.rowCount; i++) {
data.push({
from_name: worksheet.getCell('A'+i).value + '',
from_address: worksheet.getCell('B'+i).value + '',
from_phone: worksheet.getCell('C'+i).value + '',
receiver_name: worksheet.getCell('D'+i).value + '',
receiver_address: worksheet.getCell('E'+i).value + '',
receiver_phone: worksheet.getCell('F'+i).value + '',
note: worksheet.getCell('H'+i).value + ''
})
}
var delay = function(t) {
return new Promise(function(resolve) {
setTimeout(resolve, t)
})
}
return Promise.all(data.map(function(item, i) {
return function() {
return delay(750*i).then(function() {
winston.log('debug', 'process for item '+i)
return geocoder.geocode(item.from_address).then(function(geo_data) {
data[i].from_coord = {
lat: geo_data[0].latitude,
lng: geo_data[0].longitude
}
return geocoder.geocode(item.receiver_address).then(function(geo_data) {
data[i].receiver_coord = {
lat: geo_data[0].latitude,
lng: geo_data[0].longitude
}
})
})
.catch(function(geo_error) {
winston.log('error', 'geo_error', {error: geo_error})
throw new Error('Address in line ' + i + ' is not valid')
})
})
}()
}))
.then(function() {
winston.log('debug', 'we are done calculating location')
return res.status(201).json(data)
})
})
.catch(function(e) {
winston.log('error', 'an error occurred')
return res.status(500).json({error: e.message})
})
})
}
And below is my codes to call that Express API, i used React to do frontend job & used javascript fetch api to request to server.
startUploadFile() {
this.props.showLoader()
let data = new FormData()
data.append('excelfile', this.state.selectedFile)
data.append('name', 'excelfile')
var me = this
fetch(Const.restServer + '/create-from-file', {
headers: {
'Access-Token': this.props.access_token,
},
method: 'POST',
body: data
})
.then(function(r) {
return r.json()
})
.then(function(r) {
if (r.hasOwnProperty('error'))
throw new Error(r.error)
me.props.hideLoader()
me.props.showDialog('success', 'Process Complete')
})
.catch(function(e) {
console.log(e)
me.props.hideLoader()
me.props.showDialog('error', e.message)
})
}
My problem is when i used above codes to uploading a file on browser, i see two request in express log file. Something like this:
I will provide my codes to log info of every request here too
app.use(function(req, res, next) {
winston.log('debug', 'call api:', {
api: req.url, requestMethod: req.method
})
res.header("Access-Control-Allow-Origin", "*");
res.header(
"Access-Control-Allow-Headers",
"Origin, X-Requested-With, Content-Type, Accept, Access-Token, Web-Admin-Request"
);
next()
});
function isAuthenticated(req, res, next) {
/**
** Rest API Middleware check if access_token is valid
*/
let accessToken = req.body.accessToken || req.get('Access-Token')
// bunch of codes to check accessToken
next()
}
app.post('/order/create-from-file', isAuthenticated, orderController.createFromFile);
I don't understand why this happen. If I use Postman & select a File & upload, it works fine - just only One Request in logs.
Can anyone tell me what is the reason. I feel like this is a Bug of Express. My Express version is 4.15.2

The Access-Token request header your code adds to the request triggers browsers to send a CORS preflight OPTIONS request before trying the POST request.
https://developer.mozilla.org/en-US/docs/Web/HTTP/Access_control_CORS#Preflighted_requests gives more detail but the gist of it is, as long as your code’s adding that Access-Token header to the request, there’s no way you can prevent browsers from making the extra preflight OPTIONS request—because it’s something browsers do automatically as part of the CORS protocol.
The reason you don’t see the extra request when you use Postman is that it doesn’t do the CORS preflight OPTIONS request—only browsers send it, and only for requests made by XHR/Fetch from frontend JavaScript code running in a browser at a specific origin (which Postman isn’t).

Well, finally i can fix this problem by passing 'Access-Token' into request body instead of request header (now, my server always receive only one request). Thank to #sideshowbarker because your comment trigger me to do this method.
I still think this Problem is a bug in Express, but because it didn't occur on my local development environment, so i will not send a report to them.

Related

How do I connect service to a Cloud Foundry app?

I'm new to IBM cloud and I'm trying to build an application where I can write a text, press a button and that text is analysed by the service tone analyser returning a JSON so I can show it.
I have created an instance of said service and I have connected it to my application (a toolchain) using the 'connections' tab on the service.
I also have this code on the app.js file on my app:
const ToneAnalyzerV3 = require('ibm-watson/tone-analyzer/v3');
const { IamAuthenticator } = require('ibm-watson/auth');
const toneAnalyzer = new ToneAnalyzerV3({
version: '2019-10-10',
authenticator: new IamAuthenticator({
apikey: [API key found in service credentials],
}),
url: [API url found in service credentials],
});
app.get('/', function(req, res) {
res.render('index');
});
app.post('/api/tone', async function(req, res, next) {
try {
const { result } = await toneAnalyzer.tone(req.body);
res.json(result);
} catch (error) {
next(error);
}
});
The problem is that when I make the following call on my javascript:
$.post( "/api/tone", {text: textInput}, function(data){
console.log(data);
});
I get the error: 500 (Internal Server Error).
Does anybody know what I am doing wrong?
The issue is that you are sending req.body to be analysed for tone. If you take a look at the API Docs - https://cloud.ibm.com/apidocs/tone-analyzer?code=node#tone - you will see that you only need to send
const toneParams = {
toneInput: { 'text': text },
contentType: 'application/json',
};
I doubt very much that req.body has a toneInput field, and if it does have contentType it may not be set to one of the allowable values.

Proper way to send JSON data along with sendFile() with Node.js and Express

I've setup a Node.js server with express and I've setup routing. When navigating to a specific location: "/users/id", I'm currently using sendFile(), but I'd also like to pass JSON data into the page.
I know I can serve the page and then make an ajax request on page load, but it seems like I would want to minimize server side calls as much as possible.
I would like to do something like below, but even if this works I'm not sure how to grab the data from the web page side. (If I'm not making a separate ajax call)
app.get('/'+element+'/:id', (request, response) => {
const id = request.params.id;
pool.query('SELECT * FROM '+element+' WHERE id = ?', id,(error, result)=>{
if(error) throw error;
response.sendFile(path.resolve(__dirname+'/../html/index.html'));
response.json({message: result.name});
});
});
I'd like to return the data and the file and I'd like to know how to grab the data if I send it at the same time as the file.
Since you only need to to accommodate a simple string for the example you've given, you might consider accomplishing this through a response header. While it's not the most conventional approach, it might be the easiest in your case.
app.get('/' + element + '/:id', function(request, response, next) {
let id = request.params.id;
pool.query('SELECT * FROM ' + element + ' WHERE id = ?', id, (error, result) => {
if (error) throw error;
let options = {
dotfiles: 'deny',
headers: {
'x-timestamp': Date.now(),
'x-sent': true,
'x-result-message': result.name // your custom header here
}
}
let fileName = path.resolve(__dirname + '/../html/index.html')
response.sendFile(fileName, options, function(err) {
if (err) {
next(err)
} else {
console.log('Sent:', fileName)
}
});
});
});

Download a PDF file in Node and redirect into front end React

Can you help me to download a file in Node.js and redirect page into the front end? I am using MERN stack (Mongo, Express, React, Node).
After authenticating with the Google Auth, I want to download a file in Node, then I want to redirect the page.
router.get(
'/auth/google/callback',
passportGoogle.authenticate('google', {
failureRedirect: '/',
}),
(req, res) => {
try {
var file = 'resume.pdf';
res.download(file, (err => {
if (err) {
console.log(err)
} else {
window.location.href = '/';
}
}));
}
);
I tried this code, but after downloading it's not redirecting page to front end.
(req, res) => {
try {
var file = 'resume.pdf';
res.download(file, (error => {
if (!error) {
window.location.replace("http://stackoverflow.com");
} else {
console.log(error)
}
}));
}
catch {
console.log(error)
}
Since the headers are already sent with the download response, you'll have to go different route than this.
You'll need to change the response yourself.
var data = //filedata
res.set({
Content-Type: 'text/plain',
Location: '/'
});
res.end(data);
Utilize the Location header accordingly for your redirect.
On the client, you'll want to use:
window.location.replace("/headerLocation");
You want to use this on the client after the success of your callback to your download pdf method.
The reason your getting window undefined is because you're attempting to execute this on your nodejs server. The window object exists on the client.

Node-LinkedIn module returns "Unknown Authentication Scheme"

I'm using the node-linkedin npm package to authenticate and read information about from other users (name, job title, company name, profile pic, shared connections). I can correctly receive and store the access token (verified in my own LinkedIn profile's approved apps & console logging the token), but I am unable to return any of the requested information. My calls are copied & pasted from the package docs, but it returns the following:
2018-02-28T03:46:53.459839+00:00 app[web.1]: { errorCode: 0,
2018-02-28T03:46:53.459843+00:00 app[web.1]: message: 'Unknown authentication scheme',
2018-02-28T03:46:53.459845+00:00 app[web.1]: requestId: '3B55EVY7XQ',
2018-02-28T03:46:53.459847+00:00 app[web.1]: status: 401,
2018-02-28T03:46:53.459848+00:00 app[web.1]: timestamp: 1519789613443 }
I have included my routes below. Solely for the purpose of testing, myToken and linkedin are server-side global variables to the linkedin-controller scope. (I understand this will need to change for the final product, which is a student project.)
app.get('/companies', function (req, res) {
console.log(linkedin.connections.config.accessToken);
linkedin.companies_search.name('facebook', 1, function(err, company) {
console.log('Merpy merpy mc merpers'
,company);
// name = company.companies.values[0].name;
// desc = company.companies.values[0].description;
// industry = company.companies.values[0].industries.values[0].name;
// city = company.companies.values[0].locations.values[0].address.city;
// websiteUrl = company.companies.values[0].websiteUrl;
res.redirect("/");
});
});
app.get('/companies2', function (req, res) {
linkedin.companies.company('162479', function(err, company) {
console.log(company);
res.redirect("/");
});
});
app.get('/connections', function (req, res) {
linkedin.connections.retrieve(function(err, connections) {
console.log(connections);
res.redirect("/");
});
});
This is my authorization code, which appears to work:
app.get('/auth', function (req, res) {
// This is the redirect URI which linkedin will call to and provide state and code to verify
/**
*
* Attached to the redirect_uri will be two important URL arguments that you need to read from the request:
code — The OAuth 2.0 authorization code.
state — A value used to test for possible CSRF attacks.
*/
//TODO: validate state here to secure against CSRF
var error = req.query.error;
var error_description = req.query.error_description;
var state = req.query.state;
var code = req.query.code;
if (error) {
next(new Error(error));
}
/**
*
* The code is a value that you will exchange with LinkedIn for an actual OAuth 2.0 access
* token in the next step of the authentcation process. For security reasons, the authorization code
* has a very short lifespan and must be used within moments of receiving it - before it expires and
* you need to repeat all of the previous steps to request another.
*/
//once the code is received handshake back with linkedin to send over the secret key
handshake(req.query.code, res);
});
function handshake(code, ores) {
//set all required post parameters
var data = querystring.stringify({
grant_type: "authorization_code",
code: code,
redirect_uri: OauthParams.redirect_uri,//should match as in Linkedin application setup
client_id: OauthParams.client_id,
client_secret: OauthParams.client_secret// the secret
});
var options = {
host: 'www.linkedin.com',
path: '/oauth/v2/accessToken',
protocol: 'https:',
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': Buffer.byteLength(data)
}
};
var req = http.request(options, function (res) {
var data = '';
res.setEncoding('utf8');
res.on('data', function (chunk) {
data += chunk;
});
res.on('end', function () {
//once the access token is received store it
myToken = JSON.parse(data);
linkedin = Linkedin.init(myToken);
ores.redirect("/");
});
req.on('error', function (e) {
console.log("problem with request: " + e.message);
});
});
req.write(data);
req.end();
}
In my troubleshooting research, it seems I need to pass the token into the request; however, I can't find anywhere or any way to do so in the package. And with as many daily downloads as the package has, I can't possibly be the only one to experience this error. The author's Issues section of GitHub were unhelpful, as were other searches for this package's error.
My deployment: https://linkedin-api-test.herokuapp.com/
(When visiting the deployment, you must click the blue "Want to
connect to LinkedIn?" link prior to manually changing the uri
according to the routes. The results will also only display in the
Heroku logs, which is most likely largely unhelpful to you. It was
supposed to be a simple test, so I simply stole the front end from my
prior project.)
My Repo: https://github.com/SteveSonoa/LinkedIn-Test
node-linkedin Docs: https://github.com/ArkeologeN/node-linkedin/blob/master/README.md
This is my first question I haven't been able to find the answer to; I apologize if I left out anything important while asking. Thank you in advance for any help!
The solution was to pass the following token code into the linkedin variable instead of simply passing myToken:
linkedin = Linkedin.init(myToken.access_token || myToken.accessToken);
I don't understand the downvote, as no comments were left; I apologize if I left out important or generally expected information, as this was the first question I've asked. I want to make sure the solution is posted for anyone coming after me with the same issue. This issue is now solved.

HTTP GET Request in Node.js Express

How can I make an HTTP request from within Node.js or Express.js? I need to connect to another service. I am hoping the call is asynchronous and that the callback contains the remote server's response.
Here is a snippet of some code from a sample of mine. It's asynchronous and returns a JSON object. It can do any form of GET request.
Note that there are more optimal ways (just a sample) - for example, instead of concatenating the chunks you put into an array and join it etc... Hopefully, it gets you started in the right direction:
const http = require('http');
const https = require('https');
/**
* getJSON: RESTful GET request returning JSON object(s)
* #param options: http options object
* #param callback: callback to pass the results JSON object(s) back
*/
module.exports.getJSON = (options, onResult) => {
console.log('rest::getJSON');
const port = options.port == 443 ? https : http;
let output = '';
const req = port.request(options, (res) => {
console.log(`${options.host} : ${res.statusCode}`);
res.setEncoding('utf8');
res.on('data', (chunk) => {
output += chunk;
});
res.on('end', () => {
let obj = JSON.parse(output);
onResult(res.statusCode, obj);
});
});
req.on('error', (err) => {
// res.send('error: ' + err.message);
});
req.end();
};
It's called by creating an options object like:
const options = {
host: 'somesite.com',
port: 443,
path: '/some/path',
method: 'GET',
headers: {
'Content-Type': 'application/json'
}
};
And providing a callback function.
For example, in a service, I require the REST module above and then do this:
rest.getJSON(options, (statusCode, result) => {
// I could work with the resulting HTML/JSON here. I could also just return it
console.log(`onResult: (${statusCode})\n\n${JSON.stringify(result)}`);
res.statusCode = statusCode;
res.send(result);
});
UPDATE
If you're looking for async/await (linear, no callback), promises, compile time support and intellisense, we created a lightweight HTTP and REST client that fits that bill:
Microsoft typed-rest-client
Try using the simple http.get(options, callback) function in node.js:
var http = require('http');
var options = {
host: 'www.google.com',
path: '/index.html'
};
var req = http.get(options, function(res) {
console.log('STATUS: ' + res.statusCode);
console.log('HEADERS: ' + JSON.stringify(res.headers));
// Buffer the body entirely for processing as a whole.
var bodyChunks = [];
res.on('data', function(chunk) {
// You can process streamed parts here...
bodyChunks.push(chunk);
}).on('end', function() {
var body = Buffer.concat(bodyChunks);
console.log('BODY: ' + body);
// ...and/or process the entire body here.
})
});
req.on('error', function(e) {
console.log('ERROR: ' + e.message);
});
There is also a general http.request(options, callback) function which allows you to specify the request method and other request details.
Request and Superagent are pretty good libraries to use.
note: request is deprecated, use at your risk!
Using request:
var request=require('request');
request.get('https://someplace',options,function(err,res,body){
if(err) //TODO: handle err
if(res.statusCode === 200 ) //etc
//TODO Do something with response
});
You can also use Requestify, a really cool and very simple HTTP client I wrote for nodeJS + it supports caching.
Just do the following for GET method request:
var requestify = require('requestify');
requestify.get('http://example.com/api/resource')
.then(function(response) {
// Get the response body (JSON parsed or jQuery object for XMLs)
response.getBody();
}
);
This version is based on the initially proposed by bryanmac function which uses promises, better error handling, and is rewritten in ES6.
let http = require("http"),
https = require("https");
/**
* getJSON: REST get request returning JSON object(s)
* #param options: http options object
*/
exports.getJSON = function (options) {
console.log('rest::getJSON');
let reqHandler = +options.port === 443 ? https : http;
return new Promise((resolve, reject) => {
let req = reqHandler.request(options, (res) => {
let output = '';
console.log('rest::', options.host + ':' + res.statusCode);
res.setEncoding('utf8');
res.on('data', function (chunk) {
output += chunk;
});
res.on('end', () => {
try {
let obj = JSON.parse(output);
// console.log('rest::', obj);
resolve({
statusCode: res.statusCode,
data: obj
});
}
catch (err) {
console.error('rest::end', err);
reject(err);
}
});
});
req.on('error', (err) => {
console.error('rest::request', err);
reject(err);
});
req.end();
});
};
As a result you don't have to pass in a callback function, instead getJSON() returns a promise. In the following example the function is used inside of an ExpressJS route handler
router.get('/:id', (req, res, next) => {
rest.getJSON({
host: host,
path: `/posts/${req.params.id}`,
method: 'GET'
}).then(({ statusCode, data }) => {
res.json(data);
}, (error) => {
next(error);
});
});
On error it delegates the error to the server error handling middleware.
Unirest is the best library I've come across for making HTTP requests from Node. It's aiming at being a multiplatform framework, so learning how it works on Node will serve you well if you need to use an HTTP client on Ruby, PHP, Java, Python, Objective C, .Net or Windows 8 as well. As far as I can tell the unirest libraries are mostly backed by existing HTTP clients (e.g. on Java, the Apache HTTP client, on Node, Mikeal's Request libary) - Unirest just puts a nicer API on top.
Here are a couple of code examples for Node.js:
var unirest = require('unirest')
// GET a resource
unirest.get('http://httpbin.org/get')
.query({'foo': 'bar'})
.query({'stack': 'overflow'})
.end(function(res) {
if (res.error) {
console.log('GET error', res.error)
} else {
console.log('GET response', res.body)
}
})
// POST a form with an attached file
unirest.post('http://httpbin.org/post')
.field('foo', 'bar')
.field('stack', 'overflow')
.attach('myfile', 'examples.js')
.end(function(res) {
if (res.error) {
console.log('POST error', res.error)
} else {
console.log('POST response', res.body)
}
})
You can jump straight to the Node docs here
Check out shred. It's a node HTTP client created and maintained by spire.io that handles redirects, sessions, and JSON responses. It's great for interacting with rest APIs. See this blog post for more details.
Check out httpreq: it's a node library I created because I was frustrated there was no simple http GET or POST module out there ;-)
For anyone who looking for a library to send HTTP requests in NodeJS, axios is also a good choice. It supports Promises :)
Install (npm): npm install axios
Example GET request:
const axios = require('axios');
axios.get('https://google.com')
.then(function (response) {
// handle success
console.log(response);
})
.catch(function (error) {
// handle error
console.log(error);
})
Github page
Update 10/02/2022
Node.js integrates fetch in v17.5.0 in experimental mode. Now, you can use fetch to send requests just like you do on the client-side. For now, it is an experimental feature so be careful.
If you just need to make simple get requests and don't need support for any other HTTP methods take a look at: simple-get:
var get = require('simple-get');
get('http://example.com', function (err, res) {
if (err) throw err;
console.log(res.statusCode); // 200
res.pipe(process.stdout); // `res` is a stream
});
Use reqclient: not designed for scripting purpose
like request or many other libraries. Reqclient allows in the constructor
specify many configurations useful when you need to reuse the same
configuration again and again: base URL, headers, auth options,
logging options, caching, etc. Also has useful features like
query and URL parsing, automatic query encoding and JSON parsing, etc.
The best way to use the library is create a module to export the object
pointing to the API and the necessary configurations to connect with:
Module client.js:
let RequestClient = require("reqclient").RequestClient
let client = new RequestClient({
baseUrl: "https://myapp.com/api/v1",
cache: true,
auth: {user: "admin", pass: "secret"}
})
module.exports = client
And in the controllers where you need to consume the API use like this:
let client = require('client')
//let router = ...
router.get('/dashboard', (req, res) => {
// Simple GET with Promise handling to https://myapp.com/api/v1/reports/clients
client.get("reports/clients")
.then(response => {
console.log("Report for client", response.userId) // REST responses are parsed as JSON objects
res.render('clients/dashboard', {title: 'Customer Report', report: response})
})
.catch(err => {
console.error("Ups!", err)
res.status(400).render('error', {error: err})
})
})
router.get('/orders', (req, res, next) => {
// GET with query (https://myapp.com/api/v1/orders?state=open&limit=10)
client.get({"uri": "orders", "query": {"state": "open", "limit": 10}})
.then(orders => {
res.render('clients/orders', {title: 'Customer Orders', orders: orders})
})
.catch(err => someErrorHandler(req, res, next))
})
router.delete('/orders', (req, res, next) => {
// DELETE with params (https://myapp.com/api/v1/orders/1234/A987)
client.delete({
"uri": "orders/{client}/{id}",
"params": {"client": "A987", "id": 1234}
})
.then(resp => res.status(204))
.catch(err => someErrorHandler(req, res, next))
})
reqclient supports many features, but it has some that are not supported by other
libraries: OAuth2 integration and logger integration
with cURL syntax, and always returns native Promise objects.
If you ever need to send GET request to an IP as well as a Domain (Other answers did not mention you can specify a port variable), you can make use of this function:
function getCode(host, port, path, queryString) {
console.log("(" + host + ":" + port + path + ")" + "Running httpHelper.getCode()")
// Construct url and query string
const requestUrl = url.parse(url.format({
protocol: 'http',
hostname: host,
pathname: path,
port: port,
query: queryString
}));
console.log("(" + host + path + ")" + "Sending GET request")
// Send request
console.log(url.format(requestUrl))
http.get(url.format(requestUrl), (resp) => {
let data = '';
// A chunk of data has been received.
resp.on('data', (chunk) => {
console.log("GET chunk: " + chunk);
data += chunk;
});
// The whole response has been received. Print out the result.
resp.on('end', () => {
console.log("GET end of response: " + data);
});
}).on("error", (err) => {
console.log("GET Error: " + err);
});
}
Don't miss requiring modules at the top of your file:
http = require("http");
url = require('url')
Also bare in mind that you may use https module for communicating over secured network. so these two lines would change:
https = require("https");
...
https.get(url.format(requestUrl), (resp) => { ......
## you can use request module and promise in express to make any request ##
const promise = require('promise');
const requestModule = require('request');
const curlRequest =(requestOption) =>{
return new Promise((resolve, reject)=> {
requestModule(requestOption, (error, response, body) => {
try {
if (error) {
throw error;
}
if (body) {
try {
body = (body) ? JSON.parse(body) : body;
resolve(body);
}catch(error){
resolve(body);
}
} else {
throw new Error('something wrong');
}
} catch (error) {
reject(error);
}
})
})
};
const option = {
url : uri,
method : "GET",
headers : {
}
};
curlRequest(option).then((data)=>{
}).catch((err)=>{
})

Categories

Resources