Develop Reference

Accessing http request as json format in javascript [Node JS] [duplicate]

This question already has answers here:
How do I return the response from an asynchronous call?
(41 answers)
Closed 1 year ago.
I have the following snippet that gets data in json format but I want to return the data rater than just printing it out in console.
var request = require('request');
var headers = {
'Connection': 'keep-alive',
'Accept': 'application/json, text/plain, /',
'User-Agent': 'Mozilla/5.0 (Macintosh; Ontel Mac OS X 11_2_3) AppleWebKit/537/36 (KHTML, like Gecko) Chrome/89.0.4389.128 Safari/537.36',
'Content-Type': 'application/json',
'Accept-Language': 'en, zh-TW;q=0.8,en-US;q=0.7'
};
const PublicKeyBase58Check = "";
const Username = "elonmusk";
var dataString = '{"PublicKeyBase58Check":"' + PublicKeyBase58Check + '","Username":"' + Username + '"}';
var options = {
url: 'https://api.bitclout.com/get-single-profile',
method: 'POST',
headers: headers,
body: dataString
};
function callback(error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body);
var profile = JSON.parse(body);
return profile;
}
};
request(options, callback);
What change do I need to make in request function() so that I can access the json object profile from the callback function and use it for later purposes?

You need to write asynchronous code using async/await or Promises or Observables or any npm package that handles asynchronous requests. The npm package that you used (request) is deprecated. In case you want to continue using the same, the rest of the execution can be added inside the function. No need to return the value.
function callback(error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body);
var profile = JSON.parse(body);
/*
Write rest of the code here
*/
}
};
request(options, callback);`

if i understood correctly, you are getting your data in JSON notation from server side and you want to store it for further use?
Your callback already has a profile:
make a global variable var incoming_data =[];
Every time your callback returns a result store it to array:
function callback(error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body);
var profile = JSON.parse(body);
data.push[{incomingData:profile}];
return profile;
}
};
Once it is globally assigned you will have an array with all results for further usage.
Keep in mind that all setup is asynchronous so maybe is a good practice to use promises.
function callback(error, response, body) {
var promise = new Promise(function(resolve, reject) {
if (!error && response.statusCode == 200) {
console.log(body);
var profile = JSON.parse(body);
data.push[{incomingData:profile}];
resolve(profile);
return profile;
// you can also do: return resolve(profile);
}
});// END PROMISE
return promise;
}
then your call will look like this: let resolvedData = request(options, callback).then(function(done){console.log(done);});
The only thing i'm not aware is the fact that is kind deprecated method. So maybe it will cause you timeouts. (browser side...)
Also check this source tutorial for alternative ways of HTTP requests, source

async.js Ordering of functions

So I'm having trouble getting one javascript function to finish before the next one starting. I've spent quite a lot of time trying to use callback methods described on other stackoverflow posts. I could get simple examples that used timeouts to work but couldn't get it to work with my API request. I stumbled upon async.js and thought that perhaps using async.series would be a good idea to get my two functions to perform one after another. So I tried this approach, however I still seem to be having the problem where the first function takes a bit longer to execute (which is fine) but the execution process moves past this function instead of waiting for it to end. I feel I have a misconception of some sort since I have tried several methods but to no avail.
What is strange is, is that that when running server.js, it goes into the first function but then it leaves the async.series() function even before the request is finished. When I print inside of tokenReq(), I can see that the request was successful as a token code is returned successfully however this happens to late as execution has moved on. The output is shown below.
server.js:
var access_code;
async.series([
function() {
access_code = queries.data.tokenReq(code);
console.log("Finished inside function 1");
},
function() {
console.log("\n Starting function 2 \n");
if (access_code === "error") {
res.json("An error has occured");
} else {
var response = queries.data.messagesReq(access_code);
res.json(response);
}
}
],
function(err, access_code) {
});
console.log("Outside");
queries.js:
tokenReq: function(code) {
var tokenUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/token";
var form = {
code: code,
client_id: "__ID__",
redirect_uri: "__Site__/",
grant_type: "authorization_code",
client_secret: "__Secret__",
};
var formData = querystring.stringify(form);
var contentLength = formData.length;
request({
headers: {
'Content-Length': contentLength,
'Content-Type': 'application/x-www-form-urlencoded'
},
uri: tokenUrl,
body: formData,
method: 'POST'
}, function (error, response, body) {
if (error != "null") {
var access_token = JSON.parse(body).access_token;
console.log("\n INSIDE FUNCTION REQUEST, Token: " + access_token + " \n");
return access_token;
} else {
console.log('error:', error); // Print the error if one occurred
console.log('statusCode:', response && response.statusCode); // Print the response status code if a response was received
// console.log('body:', body); // Print the HTML for the Google homepage.
return "error";
}
});
},
Output:
Finished inside function 1
Outside
INSIDE FUNCTION REQUEST, Token: 8Swhd.......

You missed a major point here. Since node.js is asynchronous there should not be a way to know when a function completes its execution. That is why we specify callbacks so that the invoking function knows whom to call when it finishes its execution. Once you have functions with callbacks, you can enforce series/parallel/waterfall behavior with async module.
tokenReq: function(code, cb) {
var tokenUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/token";
var form = {
code: code,
client_id: "__ID__",
redirect_uri: "__Site__/",
grant_type: "authorization_code",
client_secret: "__Secret__",
};
var formData = querystring.stringify(form);
var contentLength = formData.length;
request({
headers: {
'Content-Length': contentLength,
'Content-Type': 'application/x-www-form-urlencoded'
},
uri: tokenUrl,
body: formData,
method: 'POST'
}, function (error, response, body) {
if (error != "null") {
var access_token = JSON.parse(body).access_token;
console.log("\n INSIDE FUNCTION REQUEST, Token: " + access_token + " \n");
return cb(null, access_token);
} else {
console.log('error:', error); // Print the error if one occurred
console.log('statusCode:', response && response.statusCode); // Print the response status code if a response was received
// console.log('body:', body); // Print the HTML for the Google homepage.
return cb(new Error("whatever"));
}
});
},
Now, you can use the callback inside server.js
var access_code;
async.series([
function(cb) {
return queries.data.tokenReq(code, cb);
},
function(access_code, cb) {
console.log("\n Starting function 2 \n");
if (access_code === "error") {
res.json("An error has occured");
} else {
var response = queries.data.messagesReq(access_code);
res.json(response);
}
// do whatever you want after this
return cb();
}
],
function(err, access_code) {
if (err) {
console.log(err);
}
// wrap your logic around a function and call the correspoding callback here
});

Redirect client while performing a function with Express

Right now, I have an Express route that is posted to by a form, below is a truncated example. The initial form is inside an iframe, so after I receive a response from http://example.com/endpoint, I send a response back to the iframe with a link going to a "signing" page, targeting the parent frame.
Unfortunately, the response from http://example.com/endpoint can take pretty long, which causes the iframe to timeout and never receive a response. What I'd like to do is send some type of response back to the iframe immediately and redirect the page to some sort of "loading page" – this would be shown while the router waits for a response from http://example.com/endpoint.
I'm using Express to serve the page that contains the iframe to the user right now – all the views are controlled on the server side.
I'm wondering if there's any resources somebody could point me towards, or something to nudge me in the right direction.
router.post('/api/orders', function(req, res) {
var order = {
'model': req.body.model,
'options': optionsArray
}
request.post({
url: 'http://example.com/endpoint,
body: order,
json: true
}, function(err, response, body) {
if (!error && response.statusCode === 200) {
if (!body.isCustom) {
hellosign.embedded.getSignUrl(body.signatureId)
.then(function(response) {
var signatureUrl = response.embedded.sign_url;
var resSignatureUrl = encodeURIComponent(signatureUrl);
res.send('Click to sign');
})
.catch(function(err) {
console.log(err);
})
} else {
res.send('You selected custom options.');
}
}
if (error || response.statusCode === 403) {
res.json({
message: 'something went wrong with your order',
errorCode: response.statusCode,
errorMessage: body.message
});
}
});
});

I would put the hellosign/whatever long running API call in its own module. Test that separately to make sure its working.
Then your iframe or whatever (do you really need an iframe?) sends the request which is just a 'start order' request which gets an order id from the hellosign module. And then use a setInterval or something to check a new endpoint which is an 'orderstatus' endpoint.
Your orderstatus endpoint accesses your new hellosign module to check the status of the order.
So it could be something like:
post('/start', function(req,res) {
var id = hellosign.startorder(req.body.model);
res.send(id);
});
get('/status', function(req,res) {
res.send(hellosign.checkstatus(req.body.id));
}
// hellosign.js
var status = {};
exports.startorder = function(model) {
var id = uuid.v4(); // some unique id;
status[id] = 'started';
request.post(api, /* ... */ ).then(function(signurl) { status[id] = signurl });
return id;
}
exports.checkstatus = function(id) {
return status[id];
}

Node JS, make HTTPS request synchronously from two links

I want to make a HTTPS request to an external link through Node JS. On my first call, I need to fetch user id by looping through several users. On my second call, I need to input that user id in the URL link and fetch user properties. Keep repeating this process till I go through all users. The end goal is to store data of every user in a JSON format. There is no front-end involved. Any direction/advice is much appreciated.
I can't share the actual link due to api keys. But here is the hypothetical scenario. I only show 2 users here. I have about 10,000 users in my actual data set.
Link 1
https://www.google.com/all_users
JSON Output
{
"name": "joe",
"uri": "/id/UserObject/User/1234-1234",
},
{
"name": "matt",
"uri": "/id/UserObject/User/5678-5678",
}
Link 2
https://www.google.com//id/UserObject/User/1234-1234
JSON Output
{
"name": "joe",
"uri": "/id/UserObject/User/1234-1234",
"Property Values": {
"height": "2",
"location": "canada"
},
"Other Values": {
"work": "google",
"occupation": "developer"
}
}
Nested JSON
{
"PropertySetClassChildrenResponse": {
"PropertySetClassChildren": {
"PropertySetInstances": {
"totalCount": "1",
"Elements": [
{
"name": "SystemObject",
"uri": "/type/PropertySetClasses/SystemObject"
}
]
}
}
}
}

Not tested, but this should point you in the right direction. It uses Promises and assumes that run in an ES6 environment:
const rp = require('request-promise');
const Promise = require('bluebird');
fetchAllUsers()
.then(extractUserUris)
.then(extractUserIds)
.then(buildUserDetailRequests)
.then(Promise.all) // run all the user detail requests in parallel
.then(allUserData => {
// allUserData is an array of all users' data
});
function fetchAllUsers() {
return rp('https://api.whatever.com/all_users');
}
function extractUserUris(users) {
return users.map(user => user.uri);
}
function extractUserIds(userUris) {
return userUris.map(userUri => userUri.split('/').pop());
}
function buildUserDetailRequests(userIds) {
return userIds.map(userId => rp("https://api.whatever.com/user/" + userId));
}

I'd suggest using the request package to make your HTTP requests easier.
> npm install request
Then you would obtain a list of all users with something like this:
var request = require('request');
request.get({url: "https://example.org/all_users"}, handleUsersResponse);
You'd handle the request response like this:
function(err, response, body) {
if (!err && response.statusCode == 200) {
// parse json (assuming array of users)
var users = JSON.parse(body);
// iterate through each user and obtain user info
for(var i = 0; i < users.length; i++) {
var userUri = users[i].uri;
obtainUserInfo(userUri)
}
}
}
obtainUserInfo function would be similar to the above code.
One important thing to keep in mind is that since the HTTP requests are being made asynchronously, when you make the requests in a loop, the next iteration of the loop does not wait until the work is finished before moving to the next iteration and starting the next request. So in effect, your loop would start all the HTTP requests nearly in parallel. This can easily overwhelm both your client and the server. One way to get around this is to use a worker queue to enqueue the work and ensure that only a maximum number of HTTP requests are being executed at any given time.

You don't want to do synchronous calls, it defeats the purpose of using Node. So by the Node powers invested in me by the State of Texas I hereby cast that synchronous way I thinking out of you!
Just kidding :), but let's do this the Node way.
Install these two libraries:
sudo npm install Promise
sudo npm install request
And set your code to look like:
var Promise = require('promise');
var request = require('request');
//Get your user data, and print the data in JSON:
getUserData()
.then(function(userData) {
console.log(JSON.stringify(userData));
}).catch(function(err) {
console.log('Error: ' +err);
});
/**
* Prepares an Object containing data for all users.
* #return Promise - Contains object with all user data.
*/
function getUserData() {
return new Promise(function(fulfill, reject) {
// Make the first request to get the user IDs:
var url1 = 'https://www.google.com/all_users';
get(url1)
.then(function(res) {
res = JSON.parse(res);
// Loop through the object to get what you need:
// Set a counter though so we know once we are done.
var counter = 0;
for (x=0; x<res.users.length; x++) {
var url2 = 'https://www.google.com//id/UserObject/User/';
url2 = url2 + res.users.id; //Wherever the individual ID is stored.
var returnDataArr = [];
get(url2)
.then(function(res2) {
// Get what you need from the response from the 2nd URL.
returnDataArr.push(res2);
counter++;
if (counter === res.users.length) {
fulfill({data: returnDataArr}); //Return/Fulfill an object containing an array of the user data.
}
}).catch(function(err) {
// Catch any errors from the 2nd HTTPS request:
reject('Error: ' +err);
});
}).catch(function(err) {
// Catch any errors from the 1st HTTPS request:
reject('Error: ' +err);
});
}
/**
* Your HTTPS GET Request Function
* #param url - The url to GET
* #return Promise - Promise containing the JSON response.
*/
function get(url) {
return new Promise(function(fulfill, reject) {
var options = {
url: url,
headers: {
'Header Name': 'Header Value',
'Accept': 'application/json',
'Content-Type': 'application/json'
};
request(options, function(err, res, body) {
if (err) {
reject(err);
} else {
fulfill(body);
}
});
});
}
So what this Promise does, is that it returns the value once we actually have it. In the code above, we are first getting that list of users, and then as we parse through it, we are making a new asynchronous HTTP request to get the additional data on it. Once we get the user data, we push it to an array.
Finally, once our counter hits its endpoint, we know that we have gotten all the user data, and so we call fulfill which essentially means return, and it returns an object containing an array of the user data.
Let me know if this makes sense.

The answers above helped me go further with my solution and get the desired outcome. However, I spent a lot of time trying to understand node, promises in node, making an API call, etc. Hopefully, this will help to a beginner level node developer.
NODE
Node.js® is a JavaScript runtime built on Chrome's V8 JavaScript engine. Node.js uses an event-driven, non-blocking I/O model that makes it lightweight and efficient. Node.js' package ecosystem, npm, is the largest ecosystem of open source libraries in the world.
If you are a JavaScript developer, you would prefer to use Node as you wouldn't have to spend time learning a new language like Java or Python.
GOAL
Make a HTTPS call to an external link to fetch all server URIs. Pass in the URI as a param to create a second link to fetch all server properties. Loop through to all server uris and properties. Refer the original post on the top for the data structure. The external link also required basic auth and headers.
CODE
Install NPM modules request (https call), bluebird (promises) and lodash(utility) and express(node framework).
/
********************** MODULES/DEPENDENCIES **********************/
var express = require('express');
var request = require('request');
var Promise = require('bluebird');
var _ = require("lodash");
/********************** INITIATE APP **********************/
var app = express();
console.log("Starting node server...");
/**
* Your HTTPS GET Request Function
* #param url - The url to GET
* #return Promise - Promise containing the JSON response.
*/
function get(url) {
return new Promise(function(resolve, reject) {
// var auth = "Basic " + new Buffer(username + ':' + password).toString("base64");
var options = {
url: url,
headers: {
// 'Authorization': auth,
'Content-Type': 'application/json',
'Accept': 'application/json'
}
};
console.log("Calling GET: ", url);
if ('development' == app.get('env')) {
console.log("Rejecting node tls");
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
}
request(options, function(error, response, body) {
if (error) {
reject(error);
} else {
// console.log("THIS IS BODY: ", body);
resolve(body);
}
});
});
}
/********************** GET DATA FUNCTION **********************/
function getServerData() {
/********************** URI VARIABLES **********************/
var username = 'username',
password = 'password',
role = 'Read-Only',
url_host = 'https://link.com:10843';
/********************** URL 1 **********************/
var url1 = url_host + '/type/PropertySetClasses/SystemObject/Server/?maxResults=1000&username=' + username + '&password=' + password + '&role=' + role;
console.log("Getting server data...", url1);
/********************** GET REQUEST 1 **********************/
return get(url1)
.then(function(res) {
console.log("Got response!");
res = JSON.parse(res);
res = res.PropertySetClassChildrenResponse.PropertySetClassChildren.PropertySetInstances.Elements;
// console.log("THIS IS RES: ", res);
/********************** FETCH URI FROM RES NESTED OBJECT **********************/
var server_ids = _.map(res, function(server) {
return server.uri;
});
console.log("Calling server urls", server_ids);
// Loop through the object to get what you need:
// Set a counter though so we know once we are done.
return Promise.map(server_ids, function (id) {
var url2 = url_host + id + '?username=' + username + '&password=' + password + '&role=' + role;
console.log("Calling URL", url2);
return get(url2)
.then(function(res2) {
res2 = JSON.parse(res2);
var elements = res2.PropertySetInstanceResponse.PropertySetInstance.PropertyValues.Elements;
console.log("Got second response", res2, elements);
return elements;
});
})
.then(function (allUrls) {
console.log("Got all URLS", allUrls);
return allUrls;
});
})
.catch(function(err) {
console.error(err);
throw err;
});
};
app.listen(8080, function() {
console.log("Server listening and booted on: " + 8080);
app.get("/serverInfo", function (req, res) {
console.log("Calling server info");
return getServerData()
.then(function(userData) {
var userData = JSON.stringify(userData, null, "\t");
console.log("This is USERDATA Data: ", userData);
res.send(userData);
})
.catch(function(err) {
console.error(err);
res.send({
__error: err,
message: err.message
});
});
});
});

NodeJS return JSONP NOT using express

Im trying to call my nodejs server and show the results from an angularjs app. ive been following an example, but when i change the example code for my code, it always calls the error callback. i have a feeling its something wrong with my server, but i cant make it work, its nodejs without express or other libraries, and all other answers involve using express. how can i make it work without using express?
the code in angularjs, where i call the server:
app.controller('Main', function ($scope, $http) {
$scope.init = function() {
//var api = 'http://api.trakt.tv/calendar/premieres.json/7c989229f2fa626832e7576eb59820e9/20131103/30/?callback=JSON_CALLBACK';
$http.jsonp('http://localhost:8888/selectAllLocations?callback=JSON_CALLBACK').success(function(data) {
console.log(data);
}).error(function(error) {
console.log('My error: ' + error);
});
};
});
if i use the value in the var api, the result is ok and success is called, not with mine.
This is the code in my nodejs server.
var result = sequelize.query(query, null, options, parameters)
.success(function (rows) {
if (type == 'select') {
response.writeHead(200, { "Content-Type": "application/json" });
response.write(JSON.stringify(rows));
response.end();
} else if (type == 'modify') {
response.writeHead(200, { "Content-Type": "text/html" });
response.write("Query was successful");
response.end();
}
}
).error(function (e) {
console.log("An error occured: ", e);
response.writeHead(404, { "Content-Type": "text/html" });
response.write("There was an error man, shits on fire yo ---> " + e);
response.end();
}
);
Im pretty sure i have to wrap the json result in a function, but i dont know how, and i cant even know the function name, as this is all i get in my server:
path: /selectAllLocations
params: {"callback":"angular.callbacks._0"}
I understand im returning the pure json, and i can see it in chrome in my response, so, there is something preventing it from calling the success function. Can somebody point me in the right direction?

The function name is specified as the callback parameter.
response.write(params.callback + '(' + JSON.stringify(rows) + ')');