I want to make a HTTPS request to an external link through Node JS. On my first call, I need to fetch user id by looping through several users. On my second call, I need to input that user id in the URL link and fetch user properties. Keep repeating this process till I go through all users. The end goal is to store data of every user in a JSON format. There is no front-end involved. Any direction/advice is much appreciated.
I can't share the actual link due to api keys. But here is the hypothetical scenario. I only show 2 users here. I have about 10,000 users in my actual data set.
Link 1
https://www.google.com/all_users
JSON Output
{
"name": "joe",
"uri": "/id/UserObject/User/1234-1234",
},
{
"name": "matt",
"uri": "/id/UserObject/User/5678-5678",
}
Link 2
https://www.google.com//id/UserObject/User/1234-1234
JSON Output
{
"name": "joe",
"uri": "/id/UserObject/User/1234-1234",
"Property Values": {
"height": "2",
"location": "canada"
},
"Other Values": {
"work": "google",
"occupation": "developer"
}
}
Nested JSON
{
"PropertySetClassChildrenResponse": {
"PropertySetClassChildren": {
"PropertySetInstances": {
"totalCount": "1",
"Elements": [
{
"name": "SystemObject",
"uri": "/type/PropertySetClasses/SystemObject"
}
]
}
}
}
}
Not tested, but this should point you in the right direction. It uses Promises and assumes that run in an ES6 environment:
const rp = require('request-promise');
const Promise = require('bluebird');
fetchAllUsers()
.then(extractUserUris)
.then(extractUserIds)
.then(buildUserDetailRequests)
.then(Promise.all) // run all the user detail requests in parallel
.then(allUserData => {
// allUserData is an array of all users' data
});
function fetchAllUsers() {
return rp('https://api.whatever.com/all_users');
}
function extractUserUris(users) {
return users.map(user => user.uri);
}
function extractUserIds(userUris) {
return userUris.map(userUri => userUri.split('/').pop());
}
function buildUserDetailRequests(userIds) {
return userIds.map(userId => rp("https://api.whatever.com/user/" + userId));
}
I'd suggest using the request package to make your HTTP requests easier.
> npm install request
Then you would obtain a list of all users with something like this:
var request = require('request');
request.get({url: "https://example.org/all_users"}, handleUsersResponse);
You'd handle the request response like this:
function(err, response, body) {
if (!err && response.statusCode == 200) {
// parse json (assuming array of users)
var users = JSON.parse(body);
// iterate through each user and obtain user info
for(var i = 0; i < users.length; i++) {
var userUri = users[i].uri;
obtainUserInfo(userUri)
}
}
}
obtainUserInfo function would be similar to the above code.
One important thing to keep in mind is that since the HTTP requests are being made asynchronously, when you make the requests in a loop, the next iteration of the loop does not wait until the work is finished before moving to the next iteration and starting the next request. So in effect, your loop would start all the HTTP requests nearly in parallel. This can easily overwhelm both your client and the server. One way to get around this is to use a worker queue to enqueue the work and ensure that only a maximum number of HTTP requests are being executed at any given time.
You don't want to do synchronous calls, it defeats the purpose of using Node. So by the Node powers invested in me by the State of Texas I hereby cast that synchronous way I thinking out of you!
Just kidding :), but let's do this the Node way.
Install these two libraries:
sudo npm install Promise
sudo npm install request
And set your code to look like:
var Promise = require('promise');
var request = require('request');
//Get your user data, and print the data in JSON:
getUserData()
.then(function(userData) {
console.log(JSON.stringify(userData));
}).catch(function(err) {
console.log('Error: ' +err);
});
/**
* Prepares an Object containing data for all users.
* #return Promise - Contains object with all user data.
*/
function getUserData() {
return new Promise(function(fulfill, reject) {
// Make the first request to get the user IDs:
var url1 = 'https://www.google.com/all_users';
get(url1)
.then(function(res) {
res = JSON.parse(res);
// Loop through the object to get what you need:
// Set a counter though so we know once we are done.
var counter = 0;
for (x=0; x<res.users.length; x++) {
var url2 = 'https://www.google.com//id/UserObject/User/';
url2 = url2 + res.users.id; //Wherever the individual ID is stored.
var returnDataArr = [];
get(url2)
.then(function(res2) {
// Get what you need from the response from the 2nd URL.
returnDataArr.push(res2);
counter++;
if (counter === res.users.length) {
fulfill({data: returnDataArr}); //Return/Fulfill an object containing an array of the user data.
}
}).catch(function(err) {
// Catch any errors from the 2nd HTTPS request:
reject('Error: ' +err);
});
}).catch(function(err) {
// Catch any errors from the 1st HTTPS request:
reject('Error: ' +err);
});
}
/**
* Your HTTPS GET Request Function
* #param url - The url to GET
* #return Promise - Promise containing the JSON response.
*/
function get(url) {
return new Promise(function(fulfill, reject) {
var options = {
url: url,
headers: {
'Header Name': 'Header Value',
'Accept': 'application/json',
'Content-Type': 'application/json'
};
request(options, function(err, res, body) {
if (err) {
reject(err);
} else {
fulfill(body);
}
});
});
}
So what this Promise does, is that it returns the value once we actually have it. In the code above, we are first getting that list of users, and then as we parse through it, we are making a new asynchronous HTTP request to get the additional data on it. Once we get the user data, we push it to an array.
Finally, once our counter hits its endpoint, we know that we have gotten all the user data, and so we call fulfill which essentially means return, and it returns an object containing an array of the user data.
Let me know if this makes sense.
The answers above helped me go further with my solution and get the desired outcome. However, I spent a lot of time trying to understand node, promises in node, making an API call, etc. Hopefully, this will help to a beginner level node developer.
NODE
Node.js® is a JavaScript runtime built on Chrome's V8 JavaScript engine. Node.js uses an event-driven, non-blocking I/O model that makes it lightweight and efficient. Node.js' package ecosystem, npm, is the largest ecosystem of open source libraries in the world.
If you are a JavaScript developer, you would prefer to use Node as you wouldn't have to spend time learning a new language like Java or Python.
GOAL
Make a HTTPS call to an external link to fetch all server URIs. Pass in the URI as a param to create a second link to fetch all server properties. Loop through to all server uris and properties. Refer the original post on the top for the data structure. The external link also required basic auth and headers.
CODE
Install NPM modules request (https call), bluebird (promises) and lodash(utility) and express(node framework).
/
********************** MODULES/DEPENDENCIES **********************/
var express = require('express');
var request = require('request');
var Promise = require('bluebird');
var _ = require("lodash");
/********************** INITIATE APP **********************/
var app = express();
console.log("Starting node server...");
/**
* Your HTTPS GET Request Function
* #param url - The url to GET
* #return Promise - Promise containing the JSON response.
*/
function get(url) {
return new Promise(function(resolve, reject) {
// var auth = "Basic " + new Buffer(username + ':' + password).toString("base64");
var options = {
url: url,
headers: {
// 'Authorization': auth,
'Content-Type': 'application/json',
'Accept': 'application/json'
}
};
console.log("Calling GET: ", url);
if ('development' == app.get('env')) {
console.log("Rejecting node tls");
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
}
request(options, function(error, response, body) {
if (error) {
reject(error);
} else {
// console.log("THIS IS BODY: ", body);
resolve(body);
}
});
});
}
/********************** GET DATA FUNCTION **********************/
function getServerData() {
/********************** URI VARIABLES **********************/
var username = 'username',
password = 'password',
role = 'Read-Only',
url_host = 'https://link.com:10843';
/********************** URL 1 **********************/
var url1 = url_host + '/type/PropertySetClasses/SystemObject/Server/?maxResults=1000&username=' + username + '&password=' + password + '&role=' + role;
console.log("Getting server data...", url1);
/********************** GET REQUEST 1 **********************/
return get(url1)
.then(function(res) {
console.log("Got response!");
res = JSON.parse(res);
res = res.PropertySetClassChildrenResponse.PropertySetClassChildren.PropertySetInstances.Elements;
// console.log("THIS IS RES: ", res);
/********************** FETCH URI FROM RES NESTED OBJECT **********************/
var server_ids = _.map(res, function(server) {
return server.uri;
});
console.log("Calling server urls", server_ids);
// Loop through the object to get what you need:
// Set a counter though so we know once we are done.
return Promise.map(server_ids, function (id) {
var url2 = url_host + id + '?username=' + username + '&password=' + password + '&role=' + role;
console.log("Calling URL", url2);
return get(url2)
.then(function(res2) {
res2 = JSON.parse(res2);
var elements = res2.PropertySetInstanceResponse.PropertySetInstance.PropertyValues.Elements;
console.log("Got second response", res2, elements);
return elements;
});
})
.then(function (allUrls) {
console.log("Got all URLS", allUrls);
return allUrls;
});
})
.catch(function(err) {
console.error(err);
throw err;
});
};
app.listen(8080, function() {
console.log("Server listening and booted on: " + 8080);
app.get("/serverInfo", function (req, res) {
console.log("Calling server info");
return getServerData()
.then(function(userData) {
var userData = JSON.stringify(userData, null, "\t");
console.log("This is USERDATA Data: ", userData);
res.send(userData);
})
.catch(function(err) {
console.error(err);
res.send({
__error: err,
message: err.message
});
});
});
});
Related
I have this front-end function:
var finalSelValue = ""
function changeFunc() {
var selectBox = document.getElementById("selectBox");
var selectedValue = selectBox.options[selectBox.selectedIndex].value;
finalSelValue = selectedValue
fetch('/adminFilter', {
method: 'POST',
headers: {'content-type': 'application/json'},
body: JSON.stringify({
finalSelValue
})
})
}
and then this on the backend:
router.post('/adminFilter', auth.ensureAuthenticated, auth.roleCheck('ADMIN'), (req, res) => {
console.log("filter " + req.body.finalSelValue)
var query = "select * from tkwdottawa where STATUS = '" + req.body.finalSelValue + "'"
ibmdb.open(DBCredentials.getDBCredentials(), function(err, conn) {
if (err) return console.log(err);
conn.query(query, function(err, rows) {
if (err) {
res.writeHead(404);
}
for (var i = 0; i < rows.length; i++) {
console.log(rows[i])
}
res.render('AdminDash', {
page_title: "AdminDash",
data: rows
});
conn.close(function() {
console.log('done adminFilter');
});
});
});
})
it is logging the new values properly, where the loop is, but it is not re-rendering the page with the new data! How do I fix this!
The issue might be because you have not fully configured your nodejs application. You should be aware that the render() method must have a templating engine that handles views to be available. You can just install one of the below templating engine i.e
Pug
EJS
Handlebars
from Npm repository.
Just use npm install [pug or ejs] i suggest this two because they have very easy learning curve
Another solution is to just use send() or sendFile() methods. This will send back the data to the client in this case the fetch request, then you can get the response from the .then callback.
and update the view from front end script.
Just a note
Render method is used with a templating engine and mostly is a server side rendering oriented approach
I want to create a uptime monitor using NodeJS and MongoDB. I want to run a cron job in NodeJS and store the data into MongoDB. If the website response status code is not equal to 200 then it will be saved in the database. I want to make a database entry like this,
url : http://www.google.com
status_code : 500
start_time :- start time
end_time :- end time
I can run the cron job but not sure how to save the downtime in the database. As, I don't want to store every response into the database. Only when response status code is other than 200 , then it will start tracking (start_time) the URL and it keeps the time when website is back to 200 as end_time.
cron.js :-
var async=require('async');
const Entry = require('../models/health.model.js');
var https = require('https');
var request = require('request');
module.exports = function getHttpsRequests () {
Entry.find({},function(err,entrys){
console.log(err);
if(!err && entrys){
async.each(entrys,function(entry,callback){
request(entry.url, function (error, response, body) {
entry.statuscheck=response.statusCode;
entry.save();
callback();
});
},function (error) {
});
}
});
}
health.model.js :-
const mongoose = require('mongoose');
const EntrySchema = mongoose.Schema({
url: String,
statuscheck: String
}, {
timestamps: true
});
module.exports = mongoose.model('Entry', EntrySchema);
I would do something like this to handle updating the database. I went ahead and put standard arrow functions in, because it was easier for me that way. I put some comments in so that should clear most questions up. It may not be the most elegant solution because I wrote it in 5 minutes, but if you follow this general logic flow, you should be much closer to your solution (its completely untested mind you.)
var async=require('async');
const Entry = require('../models/health.model.js');
var https = require('https');
var request = require('request');
module.exports = function getHttpsRequests () {
Entry.find({}, (err,entrys) => {
console.log(err);
if (!err && entrys) {
async.each(entrys, (entry,callback) => {
request(entry.url, (error, response, body) => {
//first check if the url has a document in the db.
Entry.find({ url: entry.url }, (err, entry) => {
if(!entry) {
//since the document does not exist, check the statusCode.
if(response.statusCode===200) { //if the statusCode is 200, continue the loop.
callback();
} else { //if the status code is not 200, lets save this to the db.
console.log("Saving object: " + entry)
entry.status_code = response.statusCode;
entry.start_time = new Date();
entry.save();
callback();
}
} else if (entry) {
//since the document exists, lets check the statusCode.
if(response.statusCode===200) { //if the statusCode is 200, update the stop_time.
entry.end_time = new Date();
Entry.findOneAndUpdate({ url: entry.url }, entry, (err, object) => { //this returns the entry after update, so we can put that in the console for easy debug.
if (err) {
console.log(err);
callback();
} else {
console.log("Object saved: " + object);
callback();
}
});
}
} else { //there was an error finding the document in the db, just go to the next one.
callback();
});
});
});
}
});
}
What is the best way to have a single list of users which is shared between several processes?. The processes are initiated using PM2.
The processes will have access to the list in order to add, remove, and check if a user exist in the list already.
The easiest way is to use redis(or memocache, even mongodb) to store those user list.
Or you will have to handle very complex IPC in your case, since pm2 uses node cluter, based on child_process.
You can use an in-memory data store like Redis.
Redis runs as a separate process and serves requests on a TCP port(by default 6379). Redis is a key-value data store and can be used by all your node processes.
Here's how you can do it:
List item
Install redis. (https://redis.io/)
Install node client for redis:
npm install --save redis
Now you can use redis to store your application state data and share it accross processes.
Refer this link for code example.
i just wrote a Job tracking logger for large web crawler system up to 1200 instances using redis.
Ok! Let's do it!
First you will need define it:
const redis = require("redis");
const client_redis = redis.createClient({
retry_strategy: function(options) {
if (options.error && options.error.code === "ECONNREFUSED") {
// End reconnecting on a specific error and flush all commands with
// a individual error
return new Error("The server refused the connection");
}
if (options.total_retry_time > 1000 * 60 * 60) {
// End reconnecting after a specific timeout and flush all commands
// with a individual error
return new Error("Retry time exhausted");
}
if (options.attempt > 10) {
// End reconnecting with built in error
return undefined;
}
// reconnect after
return Math.min(options.attempt * 100, 3000);
},
});
This function for update and create log.
function create_and_update_log(productName2, url2, proc, msg) {
var data_value = {
id: 'BESTBUY::DATA_LOG::'+md5(productName2 + url2),
totalrv: 'WAIT',
product: productName2,
url: url2,
process: proc,
task: msg,
timestamp: moment().format('DD/MM/YYYY HH:mm:ss')
};
client_redis.set('BESTBUY::DATA_LOG::'+md5(productName2 + url2), JSON.stringify(data_value));
}
This function for query all data
async function get_log_redis() {
return new Promise(function(resolve, reject) {
try {
var logger_data = {
logger: []
};
client_redis.multi()
.keys('BESTBUY::DATA_LOG::*', function(err, replies) {
replies.forEach(function(reply, index) {
client_redis.get(reply, function(err, data) {
if (!data.includes("Total reviews left: 0")) {
logger_data.logger.push(JSON.parse(data));
}
if (index == replies.length - 1) {
resolve(logger_data);
}
});
});
})
.exec(function(err, replies) {});
} catch (err) {
console.log(err);
}
});
}
Remember to replace :
BESTBUY::DATA_LOG::
... with what you want to define.
And the final is how to fetch all log belong to my key name begin with "BESTBUY::DATA_LOG::"
var log_obj_data = "";
(async () => {
var log_obj_data = await get_log_redis();
response.writeHead(200, {
"Content-Type": "application/json"
});
response.end(JSON.stringify(log_obj_data));
})();
I'm developing an application using Sails JS and Braintree. I'm trying to send the all past transaction details that the customer has made.
Here is my getTransaction action
getTransaction: function(req, res) {
var customerId = req.param('customerId');
var gateway = setUpGateway();
var stream = gateway.transaction.search(function(search) {
search.customerId().is(customerId);
}, function(err, response) {
if (err) {
return res.serverError(err);
}
res.send(response);
});
},
But the problem is, if I directly send the response which I got from braintree server, it throws the circular dependency error. So, to overcome that error I'm fetching only those details that I need from response like this
getTransaction: function(req, res) {
var customerId = req.param('customerId');
var gateway = setUpGateway();
var stream = gateway.transaction.search(function(search) {
search.customerId().is(customerId);
}, function(err, response) {
if (err) {
return res.serverError(err);
}
var transactions = [];
response.each(function(err, transaction) {
var temp = [];
temp.push(transaction.id);
temp.push(transaction.amount);
temp.push(transaction.createdAt);
transactions.push(temp);
});
res.send(transactions);
});
},
But here the .each function is getting executed asynchronously and hence res.send returns the empty array. So what should I do to return all the transaction that the user has made?
Full disclosure: I work at Braintree. If you have any further questions, feel free to contact our support team.
You are correct that the iterator executes asynchronously. You should use Node's stream semantics to process the request
getTransaction: function(req, res) {
var customerId = req.param('customerId');
var gateway = setUpGateway();
var transactions = [];
var stream = gateway.transaction.search(function(search) {
search.customerId().is(customerId);
});
stream.on('data', function (transaction) {
transactions.push(transaction);
});
stream.on('end', function () {
res.send(transactions);
});
},
This will wait until all transactions have been processed before sending the result.
This page provides more information about searching using our Node client library and Node's Stream API.
I'm utilizing a MEAN stack and Socket.io to pull images from the real-time Instagram API. Everything is working great, but I now want to begin saving image data to a MongoDB database so I have a "history" of images from locations (rather than simply the most recent photos).
Below is the relevant (working) code I have so far:
Node server-side code to handle new photo updates from Instagram API and emit event to Angular controller:
// for each new post Instagram sends the data
app.post('/callback', function(req, res) {
var data = req.body;
// grab the object_id (as geo_id) of the subscription and send as an argument to the client side
data.forEach(function(data) {
var geo_id = data.object_id;
sendUpdate(geo_id);
});
res.end();
});
// send the url with the geo_id to the client side
// to do the ajax call
function sendUpdate(geo_id) {
io.sockets.emit('newImage', { geo_id: geo_id });
}
Angular controller code when 'newImage' event is received:
socket.on('newImage', function(geo_id) {
// pass geo_id into Instagram API call
Instagram.get(geo_id).success(function(response) {
instagramSuccess(response.geo_id, response);
});
// Instagram API callback
var instagramSuccess = function(scope,res) {
if (res.meta.code !== 200) {
scope.error = res.meta.error_type + ' | ' + res.meta.error_message;
return;
}
if (res.data.length > 0) {
$scope.items = res.data;
} else {
scope.error = "This location has returned no results";
}
};
});
Angular factory to handle calls to Instagram API:
angular.module('InstaFactory', []).factory('Instagram', function($http) {
var base = "https://api.instagram.com/v1";
var client_id = 'MY-CLIENT-ID';
return {
'get': function(geo_id) {
var request = '/geographies/' + geo_id.geo_id + '/media/recent?client_id=' + client_id;
var url = base + request;
var config = {
'params': {
'callback': 'JSON_CALLBACK'
}
};
return $http.jsonp(url, config);
}
};
});
I also have the following Angular Controller which currently GETS details of each location from my Stadia mongoDB model. This model also contains an (empty for now) 'photos' array that I want to PUSH photo details (url, username, user profile url, etc.) onto each time I receive them from Instagram:
angular.module('StadiaFactory', []).factory('Stadia', function($http) {
var base = "http://localhost:6060/api/stadia/";
return {
'get': function(id) {
var request = id;
var url = base + request;
var config = {
'params': {
'callback': 'JSON_CALLBACK'
}
};
return $http.jsonp(url, config);
}
};
});
This is where I get confused. Where do I fire off the PUT request to my Stadia API and does this Node route for my Stadia API look reasonable? Note: I omitted my GET route which works perfectly. PUT is just throwing me for a loop:
// add photos to stadium photos array
app.put('/api/stadia/:stadium_id', function(req, res) {
// use mongoose to get and update stadium
Stadium.findByIdAndUpdate(req.params.stadium_id,
{$push: {"photos": {img: ?, link: ?, username: ?, profile_picture: ?}}},
{safe: true, upsert: true},
function(err, stadium) {
// if there is an error retrieving, send the error. nothing after res.send(err) will execute
if (err)
res.send(err)
res.jsonp(stadium); // return stadium in JSON format
});
});
Well there are a few problems with your current structure.
When your callback route is called, with a possibility of N objects in it, you're triggering your socket event and retrieving all the last photos of your geography each time. So let's say you will have 3 new objects, you will call 3 times the same thing to get the same data, which is a bit loss when you have the power of the sockets.
You can also have problems if you try to get the object data from the client-side and PUTing it to your server, since all your clients may receive the socket and you could end-up with duplicates, not to mention that this is a lot of traffic for not much, and this will burn your quota API limit, which is also not safe on the client-side since everyone can see your key.
To me, a good way to get something working (even if I don't really know what your :stadium_id param stands for) is to get the info you want directly on the server side in your callback using the request module.
You should only get the pictures, because you can retrieve a lot of things like users, tags or videos that you may don't want to get. So you will have to listen for the image objects, and nothing else.
You could have something like this:
var request = require('request');
var CLIENT_ID = 'yourId';
function newImage(data) {
io.sockets.emit('newImage', data);
}
app.post('/callback', function (req, res) {
//loop in all the new objects
req.body.forEach(function (data) {
if (data.type !== 'image') { return ; }
//BTW I think you should try with the id property instead of object_id
request('https://api.instagram.com/v1/media/' + data.object_id + '?access_token=' + CLIENT_ID,
function (error, response, body) {
if (error) { return ; }
//Here we have one JSON object with all the info about the image
var image = JSON.parse(body);
//Save the new object to your DB. (replace the STADIUM_ID)
Stadium.findByIdAndUpdate(STADIUM_ID, { $push: {'photos':
{ img: image.images.standard_resolution.url,
link: image.link,
username: image.user.username,
profile_picture: image.user.profile_picture
}}},
{ safe: true, upsert: true });
//Send a socket to your client with the new image
newImage({
id: image.id,
img: image.images.standard_resolution.url,
link: image.link,
username: image.user.username,
profile: image.user.profile_picture
});
}
});
res.end();
});
And then in your client, you will only have to push the new images received in the newImage socket event in the $scope.items.