The API call only allows me to get only 1000 users at a time. However, I know I have 9800 users. I can pass in a parameter in the url where to start and maximum results(limited to 1000). But I need to fetch them all and don't know how to go about it.
I am using Node. And Bluebird and Request NPM modules.
Here is my code
/********************** MODULES/DEPENDENCIES **********************/
var express = require('express');
var request = require('request');
var Promise = require('bluebird');
var _ = require("lodash");
/********************** INITIATE APP **********************/
var app = express();
console.log("Starting node server...");
/**
* #param url - The url to GET
* HTTPS GET Request Function
* #return Promise - Promise containing the JSON response.
*/
/********************** URL GET FUNCTION **********************/
function get(url) {
return new Promise(function(resolve, reject) {
// var auth = "Basic " + new Buffer(username + ':' + password).toString("base64");
var options = {
url: url,
headers: {
// 'Authorization': auth,
'Content-Type': 'application/json',
'Accept': 'application/json'
}
};
console.log("Calling GET: ", url);
if ('development' == app.get('env')) {
console.log("Rejecting node tls");
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
}
request(options, function(error, response, body) {
if (error) {
reject(error);
} else {
// console.log("This is body: ", body.length);
resolve(body);
}
});
});
};
/********************** HTTPS GET SRERVER DATA **********************/
function getServerData() {
/********************** URL VARIABLES **********************/
var username = 'username',
password = 'password',
role = 'Read-Only',
url_host = 'https://link.com:1000';
/********************** URL 1 **********************/
var url1 = url_host + '/type/PropertySetClasses/SystemObject/Server/?username=' + username + '&password=' + password + '&role=' + role;
// firstResult=9500&maxResults=1&
console.log("Getting server data...", url1);
/********************** GET REQUEST 1 **********************/
return get(url1)
.then(function(res) {
console.log("Got response!");
/********************** FETCH URI FROM RES NESTED OBJECT **********************/
res = JSON.parse(res);
res = res.PropertySetClassChildrenResponse.PropertySetClassChildren.PropertySetInstances.Elements;
// console.log("This is res 1: ", res);
var server_ids = _.map(res, function(server) {
return server.uri;
});
console.log("Calling server urls", server_ids);
/********************** RETURN URL WITH SERVER URI **********************/
return Promise.map(server_ids, function (id) {
var url2 = url_host + id + '?username=' + username + '&password=' + password + '&role=' + role;
console.log("Calling URL", url2);
/********************** RETURN SERVER PROPERTIES **********************/
return get(url2)
.then(function(res2) {
res2 = JSON.parse(res2);
var elements = res2.PropertySetInstanceResponse.PropertySetInstance.PropertyValues.Elements;
console.log("Got second response", res2, elements);
return elements;
});
})
.then(function (allUrls) {
console.log("Got all URLS", allUrls);
return allUrls;
});
})
.catch(function(err) {
console.error(err);
throw err;
});
};
/********************** PORT HANDLER **********************/
app.listen(8080, function() {
console.log("Server listening and booted on: " + 8080);
/********************** ROUTER **********************/
app.get("/serverInfo", function (req, res) {
console.log("Calling server info");
/********************** RETURN PROMISE **********************/
return getServerData()
.then(function(userData) {
var userData = JSON.stringify(userData, null, "\t");
console.log("This is USERDATA Data: ", userData);
res.send(userData);
})
.catch(function(err) {
console.error(err);
res.send({
__error: err,
message: err.message
});
});
});
});
Related
Have a Node webserver(webserver.js) which serves file to clients browser. This webserver uses my file and api require("google-calendar.js");.
What is best practise to serve the google calendar result from Node to the webbrowser?
The webbserver wants to send pages syncronized but the google-api runs asynchronized which currently means result is showing up once the page is reloaded.
Below is the current code for doing this. All kind of feedback is appreciated.
www/index.html:
<html>
<head>
</head>
<body>
<iframe type="text/html" src="calendar.html"></iframe>
</body>
</html>
www/calendar.html:
CONTENT GENERATED BY webserver.js
webserver.js:
var port = 9800;
var serverUrl = "127.0.0.1";
var XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest;
var http = require("http");
var path = require("path");
var fs = require("fs");
var url = require('url');
var checkMimeType = false;
var calendar = require('./google-calendar');
console.log("Starting web server at " + serverUrl + ":" + port);
var calPage = "";
function setCalendarPage(content) {
calPage = content;
console.log("Setting calendar content: " + calPage);
}
http.createServer(function(req, res) {
calendar.getEvents(setCalendarPage);
var request = req;
var filename = req.url;
console.log("====================");
console.log(filename);
if (filename == '/') {
filename = "/index.html";
}
sendFile(filename, res);
}).listen(port, serverUrl);
function getMimeType(filename) {
var ext = path.extname(filename);
var validExtensions = {
".html": "text/html",
".js": "application/javascript",
".css": "text/css",
".txt": "text/plain",
".jpg": "image/jpeg",
".gif": "image/gif",
".png": "image/png",
".woff": "application/font-woff",
".woff2": "application/font-woff2"
};
return validExtensions[ext];
}
function sendFile(filename, res) {
var localPath = __dirname;
localPath += ("/www" + filename);
fs.exists(localPath, function(exists) {
if (exists) {
let mimeType = getMimeType(filename);
getFile(localPath, res, mimeType);
} else {
console.log("File not found: " + localPath);
res.writeHead(404);
res.end();
}
});
}
function getFile(localPath, res, mimeType) {
fs.readFile(localPath, function(err, contents) {
if (!err) {
if (mimeType != undefined) {
res.setHeader("Content-Type", mimeType);
}
res.statusCode = 200;
if (localPath.includes("calendar")) {
var calContent = getHtmlPage(calPage);
res.setHeader("Content-Length", calContent.length);
res.end(calContent);
} else {
res.setHeader("Content-Length", contents.length);
res.end(contents);
}
} else {
res.writeHead(500);
res.end();
}
});
function getHtmlPage(text) {
var html = '<html><head>';
html += '<meta charset="UTF-8">';
html += '<title></title>';
html += '</head>';
html += '<body>';
html += text;
html += '</body>';
html += '</html>';
return html;
}
}
google-calendar.js:
const fs = require('fs');
const mkdirp = require('mkdirp');
const readline = require('readline');
const {google} = require('googleapis');
const OAuth2Client = google.auth.OAuth2;
const SCOPES = ['https://www.googleapis.com/auth/calendar.readonly'];
const TOKEN_PATH = 'credentials.json';
var result = "";
// Load client secrets from a local file.
module.exports = {
getEvents : function (callback){
if(result != ""){
return;
}
fs.readFile('client_secret.json', (err, content) => {
if (err) return console.log('Error loading client secret file:', err);
// Authorize a client with credentials, then call the Google Drive API.
return authorize(JSON.parse(content),callback);
});
}
};
/**
* Create an OAuth2 client with the given credentials, and then execute the
* given callback function.
* #param {Object} credentials The authorization client credentials.
* #param {function} callback The callback to call with the authorized client.
*/
function authorize(credentials,callback) {
const {client_secret, client_id, redirect_uris} = credentials.installed;
const oAuth2Client = new OAuth2Client(client_id, client_secret, redirect_uris[0]);
// Check if we have previously stored a token.
fs.readFile(TOKEN_PATH, (err, token) => {
if (err) return getAccessToken(oAuth2Client, callback);
oAuth2Client.setCredentials(JSON.parse(token));
return listEvents(oAuth2Client, callback);
});
}
/**
* Get and store new token after prompting for user authorization, and then
* execute the given callback with the authorized OAuth2 client.
* #param {google.auth.OAuth2} oAuth2Client The OAuth2 client to get token for.
* #param {getEventsCallback} callback The callback for the authorized client.
*/
function getAccessToken(oAuth2Client, callback) {
const authUrl = oAuth2Client.generateAuthUrl({
access_type: 'offline',
scope: SCOPES,
});
console.log('Authorize this app by visiting this url:', authUrl);
const rl = readline.createInterface({
input: process.stdin,
output: process.stdout,
});
rl.question('Enter the code from that page here: ', (code) => {
rl.close();
oAuth2Client.getToken(code, (err, token) => {
if (err) return callback(err);
oAuth2Client.setCredentials(token);
// Store the token to disk for later program executions
fs.writeFile(TOKEN_PATH, JSON.stringify(token), (err) => {
if (err) console.error(err);
console.log('Token stored to', TOKEN_PATH);
});
callback(oAuth2Client);
});
});
}
/**
* Lists the next 10 events on the user's primary calendar.
* #param {google.auth.OAuth2} auth An authorized OAuth2 client.
*/
function listEvents(auth, callback) {
result = "";
const calendar = google.calendar({version: 'v3', auth});
calendar.events.list({
calendarId: 'primary',
timeMin: (new Date()).toISOString(),
maxResults: 10,
singleEvents: true,
orderBy: 'startTime',
}, (err, {data}) => {
if (err) return console.log('The API returned an error: ' + err);
const events = data.items;
if (events.length) {
console.log('Upcoming 10 events:');
events.map((event, i) => {
const start = event.start.dateTime || event.start.date;
console.log(`${start} - ${event.summary}`);
result += `${start} - ${event.summary} <br>`;
});
callback(result);
} else {
console.log('No upcoming events found.');
}
});
}
I have recently started using Node.js and jQuery and I can't figure out what's wrong in my project.
I have a client application that sends a post request to a node js server; this server gets the datas from the post and performs a query; finally it should send back to the client a json containing the informations retrieved from the previous query. The problem is that the response sent by node js is an empty object( {} ) and I don't know why.
Here my source code:
Node JS:
var http = require("http");
var url = require('url');
var querystring = require('querystring');
var express = require('express');
var cookieParser = require('cookie-parser');
var mysql = require('mysql');
var con = mysql.createConnection({
host: "",
user: "",
password: "",
database : ''
});
var app = express();
app.use(express.static(__dirname + '/public'))
.use(cookieParser());
app.post('/search',function(req, res) {
if (req.method == 'POST') { //PUT Only
var body = '';
req.on('data', function (data){body += data;});
req.on('end', function () {
var post = querystring.parse(body);
console.log("Dati in ingresso > " + JSON.stringify(post));
//res.writeHead(200, {'Content-Type': 'text/plain'})
query("SELECT Nome FROM Mood WHERE Nome LIKE \'" + post.data + "%\'",res);
});
}else{
console.log("Not POST request")
res.end();
}
});
var query = function(q,res) {
var results = {};
con.query(q, function (err, result, fields) {
if (err) throw err;
results = JSON.stringify(JSON.parse(JSON.stringify(result))[0].Nome);
console.log("Risultati query: ");
console.log(results);
});
res.json(results);
res.end();
}
app.listen(8888);
Client:
$("document").ready(function () {
$('#search-box').on('input propertychange', function(e){
var input = $(this).val();
var array = ['prova1','prova2','prova3'];
var ul = $('#results-options ul');
$.ajax({
type: "POST",
url: "/search",
data: {data : input},
contentType: "application/json",
success: function(d) {
console.log(d);
console.log(JSON.stringify(d));
console.log(JSON.parse(d));
},
error: function(d) {
console.log("Error");
}
});
$('#results-options').show();
});
$("#results-options").on('click',function (e) {
$('this').show();
e.stopPropagation();
})
$(document).on('click', function(){
$('#results-options').hide();
})
});
As I stated above, your query function is sending back a response before the query to the database has finished. That's why it is coming back empty. I moved the res.json(results); res.end(); code inside the callback of the DB query.
var query = function(q,res) {
var results = {};
con.query(q, function (err, result, fields) {
if (err) throw err;
results = JSON.stringify(JSON.parse(JSON.stringify(result))[0].Nome);
console.log("Risultati query: ");
console.log(results);
// I moved the code here
res.json(results);
res.end();
});
// Your old code
// res.json(results);
// res.end();
};
I have a file, controller.js in trying to import functionality into app.js.
I keep getting syntax errors:
, expected
statement expected
Simple to fix I though however when I fix one 10 more pop up, So can some one look at my code and see what doing wrong ?
app.js
Promise.all([controller.firstFunction(), controller.secondFunction()]) .then(controller.thirdFunction);
controller.js
module.exports = {
var express = require('express');
// var rp = require('request-promise');
var app = express();
// var request = require('request');
var nodePardot = require('node-pardot');
// Credential's for pardot API
var password = ';lu.88';
var userkey = 'kol;';
var emailAdmin = 'j.j#jj.co.uk';
//
// // Start the server
// app.listen(port);
// app.use(bodyParser.json()); // support json encoded bodies
// app.use(bodyParser.urlencoded({extended: true})); // support encoded bodies
// console.log('Test server started! At http://localhost:' + port); // Confirms server start
//
// app.use('/', router);
var firstFunction = function () {
return new Promise(function (resolve) {
setTimeout(function () {
app.post('/back-end/test', function (req, res) {
console.log(req.body);
var login = req.body.LoginEmail;
res.send(login);
resolve({
data_login_email: login
});
});
console.error("First done");
}, 2000);
});
};
var secondFunction = function () {
return new Promise(function (resolve) {
setTimeout(function () {
nodePardot.PardotAPI({
userKey: userkey,
email: emailAdmin,
password: password,
DEBUG: false
}, function (err, client) {
if (err) {
// Authentication failed
console.error("Authentication Failed", err);
} else {
// Authentication successful
var api_key = client.apiKey;
console.log("Authentication successful !", api_key);
resolve({data_api: api_key});
}
});
console.error("Second done");
}, 2000);
});
};
function thirdFunction(result) {
return new Promise(function () {
setTimeout(function () {
var headers = {
'User-Agent': 'Super Agent/0.0.1',
'Content-Type': 'application/x-www-form-urlencoded'
};
// Configure the request
var api = result[1].data_api;
var login_email = result[0].data_login_email;
var options = {
url: 'https://pi.pardot.com/api/prospect/version/4/do/read',
method: 'POST',
headers: headers,
form: {
'email': login_email,
'user_key': userkey,
'api_key': api
},
json: true // Automatically stringifies the body to JSON
};
// Start the request
rp(options)
.then(function (parsedBody) {
console.error(login_email, "Is a user, login pass!");
})
.catch(function (err) {
console.error("fail no such user");
// res.status(400).send()
});
console.error("Third done");
}, 3000);
}
);
}
};
This is because you wrapped your code inside an object, {} tags.
You have a couple of options, my suggestion is to use Prototypes like so
var express = require('express');
// var rp = require('request-promise');
var app = express();
// var request = require('request');
var nodePardot = require('node-pardot');
// Credential's for pardot API
var password = ';lu.88';
var userkey = 'kol;';
var emailAdmin = 'j.j#jj.co.uk';
//
// // Start the server
// app.listen(port);
// app.use(bodyParser.json()); // support json encoded bodies
// app.use(bodyParser.urlencoded({extended: true})); // support encoded bodies
// console.log('Test server started! At http://localhost:' + port); // Confirms server start
//
// app.use('/', router);
function Functions(){};
Functions.prototype.firstFunction = function () {
return new Promise(function (resolve) {
setTimeout(function () {
app.post('/back-end/test', function (req, res) {
console.log(req.body);
var login = req.body.LoginEmail;
res.send(login);
resolve({
data_login_email: login
});
});
console.error("First done");
}, 2000);
});
};
Functions.prototype.secondFunction = function () {
return new Promise(function (resolve) {
setTimeout(function () {
nodePardot.PardotAPI({
userKey: userkey,
email: emailAdmin,
password: password,
DEBUG: false
}, function (err, client) {
if (err) {
// Authentication failed
console.error("Authentication Failed", err);
} else {
// Authentication successful
var api_key = client.apiKey;
console.log("Authentication successful !", api_key);
resolve({data_api: api_key});
}
});
console.error("Second done");
}, 2000);
});
};
Functions.prototype.thirdFunction(result) {
return new Promise(function () {
setTimeout(function () {
var headers = {
'User-Agent': 'Super Agent/0.0.1',
'Content-Type': 'application/x-www-form-urlencoded'
};
// Configure the request
var api = result[1].data_api;
var login_email = result[0].data_login_email;
var options = {
url: 'https://pi.pardot.com/api/prospect/version/4/do/read',
method: 'POST',
headers: headers,
form: {
'email': login_email,
'user_key': userkey,
'api_key': api
},
json: true // Automatically stringifies the body to JSON
};
// Start the request
rp(options)
.then(function (parsedBody) {
console.error(login_email, "Is a user, login pass!");
})
.catch(function (err) {
console.error("fail no such user");
// res.status(400).send()
});
console.error("Third done");
}, 3000);
}
);
}
module.exports = Functions;
Then you would create a instance of the class within the file you require it (in this case app.js)
var myFunctions = new Functions();
From there you can access your methods using:
myFunctions.firstFunction();
If you however wanted to go on about the way you have done so already, you should use object structure like so
module.exports = {
firstFunction : function()
{
//Function Body
},
secondFunction : function()
{
//Function Body
}
}
Issue with you code is :
you were using var inside module.export and that means you are declaring var inside export that is not valid,
module.export should be in json format.
Try this code :
var express = require('express');
// var rp = require('request-promise');
var app = express();
// var request = require('request');
var nodePardot = require('node-pardot');
// Credential's for pardot API
var password = ';lu.88';
var userkey = 'kol;';
var emailAdmin = 'j.j#jj.co.uk';
//
// // Start the server
// app.listen(port);
// app.use(bodyParser.json()); // support json encoded bodies
// app.use(bodyParser.urlencoded({extended: true})); // support encoded bodies
// console.log('Test server started! At http://localhost:' + port); // Confirms server start
//
// app.use('/', router);
module.exports = {
firstFunction : function () {
return new Promise(function (resolve) {
setTimeout(function () {
app.post('/back-end/test', function (req, res) {
console.log(req.body);
var login = req.body.LoginEmail;
res.send(login);
resolve({
data_login_email: login
});
});
console.error("First done");
}, 2000);
});
},
secondFunction : function () {
return new Promise(function (resolve) {
setTimeout(function () {
nodePardot.PardotAPI({
userKey: userkey,
email: emailAdmin,
password: password,
DEBUG: false
}, function (err, client) {
if (err) {
// Authentication failed
console.error("Authentication Failed", err);
} else {
// Authentication successful
var api_key = client.apiKey;
console.log("Authentication successful !", api_key);
resolve({data_api: api_key});
}
});
console.error("Second done");
}, 2000);
});
},
thirdFunction : function(result) {
return new Promise(function () {
setTimeout(function () {
var headers = {
'User-Agent': 'Super Agent/0.0.1',
'Content-Type': 'application/x-www-form-urlencoded'
};
// Configure the request
var api = result[1].data_api;
var login_email = result[0].data_login_email;
var options = {
url: 'https://pi.pardot.com/api/prospect/version/4/do/read',
method: 'POST',
headers: headers,
form: {
'email': login_email,
'user_key': userkey,
'api_key': api
},
json: true // Automatically stringifies the body to JSON
};
// Start the request
rp(options)
.then(function (parsedBody) {
console.error(login_email, "Is a user, login pass!");
})
.catch(function (err) {
console.error("fail no such user");
// res.status(400).send()
});
console.error("Third done");
}, 3000);
}
);
}
};
You need to use object notation inside of an object ( module.exports):
var express = require('express');
// var rp = require('request-promise');
var app = express();
// var request = require('request');
var nodePardot = require('node-pardot');
// Credential's for pardot API
var password = ';lu.88';
var userkey = 'kol;';
var emailAdmin = 'j.j#jj.co.uk';
module.exports = {
firstFunction() {
return new Promise(function(){
...
});
},
secondFunction(){},
thirdFunction(){}
};
and exporting dependencies and passwords is not really useful...
I just started working with Nodejs.
I am using Restify to get data from: http://api.geonames.org/citiesJSON?north=44.1&south=-9.9&east=-22.4&west=55.2&lang=de&username=demo'.
My code below gives me an error: {"code":"ResourceNotFound","message":"/ does not exist"}
var restify =require("restify");
var server = restify.createServer();
server.use(restify.acceptParser(server.acceptable));
server.use(restify.queryParser());
server.use(restify.bodyParser());
server.get('http://api.geonames.org/citiesJSON?north=44.1&south=-9.9&east=-22.4&west=55.2&lang=de&username=demo', function (req, res) {
console.log(req.body);
res.send(200,req.body);
});
server.listen(7000, function () {
console.log('listening at 7000');
});
That's because Restify is for creating REST endpoints, not consuming them. You should check out this SO post for help consuming data from an API.
e.g. create test.js with the following:
var http = require('http');
var options = {
host: 'api.geonames.org',
path: '/citiesJSON?north=44.1&south=-9.9&east=-22.4&west=55.2&lang=de&username=demo'
};
var req = http.get(options, function(res) {
console.log('STATUS: ' + res.statusCode);
console.log('HEADERS: ' + JSON.stringify(res.headers));
// Buffer the body entirely for processing as a whole.
var bodyChunks = [];
res.on('data', function(chunk) {
// You can process streamed parts here...
bodyChunks.push(chunk);
}).on('end', function() {
var body = Buffer.concat(bodyChunks);
console.log('BODY: ' + body);
// ...and/or process the entire body here.
})
});
req.on('error', function(e) {
console.log('ERROR: ' + e.message);
});
then run node test.js.
I found what I was looking for. You can use restify client to get JSON data:
Here is my solution:
var restify = require("restify");
function getJSONDataFromUrl(){
var query = "?north=44.1&south=-9.9&east=-22.4&west=55.2&lang=de&username=demo";
var options = {};
options.url = "http://api.geonames.org";
options.type = options.type || "json";
options.path = "/citiesJSON" + query;
options.headers = {Accept: "application/json"};
var client = restify.createClient(options);
client.get(options, function(err, req, res, data) {
if (err) {
console.log(err);
return;
}
client.close();
console.log(JSON.stringify(data));
return JSON.stringify(data);
});
}
getJSONDataFromUrl();
I'm writing a pretty basic HTTP request using SailsJS. I'm getting the data I want from the api that I'm calling, but I can't figure out how to pass it from my service back to the controller and eventually respond to the request with the data in the body.
Here's my controller:
module.exports = {
retrieve: function(req, res) {
var output = AccountService.retrieveAccountInfo();
console.log(output);
return res.send(output);
}
}
And here is my service that I'm calling.
module.exports = {
retrieveAccountInfo: function() {
var http = require('http');
var options = {
host: 'localhost',
port: 8280,
path: '/sample/account?id=1',
method: 'GET',
headers: {
'sample': 'header'
}
};
var req = http.request(options, function(res) {
var data = '';
console.log('STATUS: ' + res.statusCode);
console.log('HEADERS: ' + JSON.stringify(res.headers) + '\n\n');
res.setEncoding('utf8');
res.on('data', function(chunk) {
console.log('BODY: ' + chunk);
data += chunk;
});
res.on('end', function() {
console.log('hit the end');
return JSON.stringify(data);
});
});
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
});
req.end();
}
}
I'm probably missing some basic JavaScript stuff here. I can get to the end callback and print "hit the end" as expected, but I can't get the data returned to the controller.
the method you're using in your service is an asynchronous method, you can modify them like following
module.exports = {
retrieve: function(req, res) {
AccountService.retrieveAccountInfo(function(error, output) {
console.log(output);
return res.send(output);
});
}
}
service - use callback method
module.exports = {
retrieveAccountInfo: function(callback) {
var http = require('http');
//.....
res.on('end', function() {
console.log('hit the end');
callback(null, JSON.stringify(data));
});
//.....
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
callback(e); //error
});
req.end();
}
}