I want to parse body outside of request. But i can not figure out a way to get it outside of the request function. Can you please show me how? Or at least give me an example. I do not understand it.
"use strict"
var robotsParser = require('robots-parser');
var request = require('request');
var fs = require('fs')
let url = 'http://google.de/robots.txt'
request(url, function(error, response, body) {
//console.log(body)
robots = robotsParser(url, body)
var reserveisDisallowed = robots.isDisallowed('http://google.de/maps/reserve/api/', '*')
console.log(reserveisDisallowed)
})
Use jQuery deferred, if you want to execute this instruction after success callback as:
var $deferred = $.Deferred(),
robots,
globalBody;
$deferred.done(function(body){
robots = robotsParser(url, body);
globalBody = body; //After here globalBody object has body available. Do whatever you want to do now.
var reserveisDisallowed = robots.isDisallowed('http://google.de/maps/reserve/api/', '*')
console.log(reserveisDisallowed)
});
request(url, function(error, response, body) {
//console.log(body)
$deferred.resolve(body);
})
or if U just want body available outside, simply use 'globalBody' variable in your code
As:
request(url, function(error, response, body) {
//console.log(body)
robots = robotsParser(url, body);
globalBody = body;
var reserveisDisallowed = robots.isDisallowed('http://google.de/maps/reserve/api/', '*')
console.log(reserveisDisallowed)
})
Related
I have been researching and testing how to do API calls in node js. I'm currently trying to use part of a JSON object that gets returned from an API call in a module to return a Token.
var request = require("request");
var timestamp = require("unix-timestamp");
var jwt = require("jsonwebtoken");
var EventEmitter = require("events").EventEmitter;
timestamp.round = true;
//create current unix timestamp
var current = timestamp.now();
//create unix experation time
var experation = timestamp.add(current, "+5m");
//create header
var header = {"header"}
//create payload
var payload = {
"iss": process.env.CKEY,
"aud": "https://iformbuilder.com/exzact/api/oauth/token",
"exp": experation,
"iat": current
};
var signature = process.env.SKEY;
//Create assertion
var assert = jwt.sign(payload, signature);
var grant = 'urn:ietf:params:oauth:grant-type:jwt-bearer';
//set the options
var options = { method: 'POST',
url: 'https://iformbuilder.com/exzact/api/oauth/token',
qs: {
grant_type: grant,
assertion: assert
},
headers: {
'content-type': 'application/x-www-form-urlencoded',
'cache-control': 'no-cache'
}
};
var data = {};
var tkn = new EventEmitter();
module.exports = {
token: function() {
request(options, function (error, response, body) {
if (error) throw new Error(error);
console.log(body);
tkn.body = body;
tkn.emit('update');
});
tkn.on('update', function(){
data = JSON.parse(tkn.body);
return data.access_token;
});
}
}
The problem is that I can only use the returned item within the scope of tkn.on. I can nesting another API call within to use the token. However, I would like to use it without having to use the same code over again. The only solution that I can get to work is writing to a file. I'm wondering if I'm even going about this the right way. I can't seem to find any good source online to help me with this and maybe I'm asking the wrong question.
You have to use a callback (or promise) to get the value. Below is an example with callbacks I'll let you research promises. I would say, understand it with callbacks first then move on to promises.
someOtherModule.js
var tokenGetter = require('./tokenGetter');
function doSomethingWithToken(accessToken){
//Do what you want with the token here
}
tokenGetter.token(doSomethingWithToken);
tokenGetter.js
module.exports = {
token: function(callback) {
request(options, function (error, response, body) {
if (error) throw new Error(error);
console.log(body);
tkn.body = body;
tkn.emit('update');
});
tkn.on('update', function(){
data = JSON.parse(tkn.body);
callback(data.access_token);
});
}
}
The reason you have to do this is because with asynchronous operations, you do not know when you will get a response. The code is no longer linear in the same way it was before.
You have to adjust your code to continue its operations WHEN you get a response instead of AFTER you do the operation.
i'm new in Node.js, I'm trying to use a variable declared within a Request, but I need to use it outside the Request.Is this possible?.
Sample code:
//stackExample
var request = require('request')
cheerio = require('cheerio')
jsonArr = []
request ({
url: 'http://youtube.com',
encoding: 'utf8'
},
function (err, resp, body){
if(!err && resp.statusCode == 200){
var $ = cheerio.load(body);
$('.yt-lockup-title').each(function(){
var title = $(this).find('a').text();
jsonArr.push({
titulo: title,
});
});
}
console.log(jsonArr) //Here Works!
}
);
console.log(jsonArr) //Here not :(, I need works here :'(
Your jsonArr variable only works inside your request function because this wait for the url response, while your outside console.log its executed line by line. You can create a function that will be called inside your anonymous function to use it outside the request function.
var request = require('request')
cheerio = require('cheerio')
jsonArr = []
request ({
url: 'http://youtube.com',
encoding: 'utf8'
},
function (err, resp, body){
if(!err && resp.statusCode == 200){
var $ = cheerio.load(body);
$('.yt-lockup-title').each(function(){
var title = $(this).find('a').text();
jsonArr.push({
titulo: title,
});
updated()
});
}
console.log(jsonArr)
}
);
function updated(){
console.log(jsonArr)
}
The issue with above code is that the value in jsonArr is pushed when the request returns from the http call and then injects the value into jsonArr.
You would want to access the value of jsonArr after that call returns.Which you can do by using promises.
Or just a hack using setTimeout.
setTimeout(function(){
console.log(jsonArr); // use value after time interval
},1000); // you can set this value depending on approx time it takes for your request to complete.
Or use the defered function libraries refer this.
var sequence = Futures.sequence();
sequence
.then(function(next) {
http.get({}, next);
})
.then(function(next, res) {
res.on("data", next);
})
.then(function(next, d) {
http.get({}, next);
})
.then(function(next, res) {
...
})
You need to wrap the console.log in a callback as follows.
The following code will log [5]
var jsonArry = [];
var test = function(num, callback){
jsonArry.push(num);
callback(jsonArry);
};
test(5, function(data){
console.log(data);
});
Goal: (solved: updated working code below)
Create an array of urls
Create a function that accepts a single url, makes a request to that url, and writes some data to file
Run above function in series multiple times for each url in the array.
Node Dependencies: Request, Cheerio, async, fs
Here is what I have so far:
//load dependencies
var request = require("request"),
cheerio = require("cheerio"),
fs = require('fs'),
async = require('async'),
wstream = fs.createWriteStream('./model/data.txt');
//create url array
var arr = [];
for (var i = 0; i < 10; i++){
arr.push('http://www.test.com/' + i)
}
//function that gets data from url and creates write stream
function getUrlData(url, callback){
request(url, function (error, response, body) {
if (!error && response.statusCode == 200) {
var $ = cheerio.load(body);
var data = $('h1').html();
wstream.write(data);
callback();
}
});
}
//close write stream
function endStream(){
wstream.end();
}
//updated working version
async.eachSeries(arr,getUrlData,endStream);
Since you want getUrlData to be executed sequentually, you'll need to add a callback to getUrlData:
function getUrlData(url, callback){
request(url, function (error, response, body) {
if (!error && response.statusCode == 200) {
var $ = cheerio.load(body);
var data = $('h1').html();
wstream.write(data);
callback();
}
callback(error);
});
}
Then use eachSeries:
async.eachSeries(arr, getUrlData, endStream);
I apologize if this is a stupid question, but I am new to Javascript and Node.js really hurts my head because it is asynchronous.
My goal is to query for a JSON object from an API and be able to work with it. I have tried to look for questions and answers on what I should be doing but none of it really makes sense to me, so I am hoping to learn by just seeing proper code.
var request = require('request');
var url = 'url here';
var temp;
var done = false;
request(url, function (error, response, body) {
if (!error) {
temp = body;
done = true;
console.log(temp);
} else {
console.log(error);
}
});
if (done){
console.log(temp);
}
Can someone please walk me through the proper way to restructure my code?
The function you are creating with the line
request(url, function (error, response, body) {
is not executed until the response is received. The rest of your code continues to run. Think of the flow something like this:
var request = require('request');
var url = 'url here';
var temp;
var done = false;
request(url, XXX);
if (done){
console.log(temp);
then when the response is received (perhaps much later on) the function XXX is executed.
As you can see, done will always be false when the line
if (done){
is executed.
I assign an empty array to the global variable artistURLs. I then push strings (the local variable artistURL) into the artistURLs array with in the Cheerio .each() iterator method.
var request = require('request'),
cheerio = require('cheerio'),
url = 'http://www.primarytalent.com/',
artistURLs = [];
request(url, function(err, resp, body){
if(err)
throw err;
$ = cheerio.load(body);
$('#rosterlists div li a').each(function(){
var urlCap = this.attr('href').slice(1);
var artistURL = url.concat(urlCap);
artistURLs.push(artistURL);
});
console.log(artistURLs);
});
I know that artistURL is successfully being pushed into artistURLs because
console.log(artistURLs);
will display the populated array in my terminal. The problem is if I try running console.log(artistURLs); outside of the callback function in the global scope. For example
var request = require('request'),
cheerio = require('cheerio'),
url = 'http://www.primarytalent.com/',
artistURLs = [];
request(url, function(err, resp, body){
if(err)
throw err;
$ = cheerio.load(body);
$('#rosterlists div li a').each(function(){
var urlCap = this.attr('href').slice(1);
var artistURL = url.concat(urlCap);
artistURLs.push(artistURL);
});
});
console.log(artistURLs);
So you can see that I have moved console.log(artistURLs); outside of request(). For some reason trying to access artistURLs in the global scope returns me an empty array as if all the processing that happened with in `request()~ never even happened.
How come this is happening and how can ensure that all the urls that are being pushed into artistURLs remain in artistURLs?
Thanks
The request() module is asynchronous, so console.log() is executing before the HTTP call completes when you use console.log(). For example, take this code:
var request = require('request');
var cheerio = require('cheerio');
var url = 'http://www.primarytalent.com/';
var artistURLs = [];
request(url, function(err, resp, body){
if (err)
throw err;
$ = cheerio.load(body);
$('#rosterlists div li a').each(function(){
var urlCap = this.attr('href').slice(1);
var artistURL = url.concat(urlCap);
artistURLs.push(artistURL);
console.log('push');
});
});
console.log(artistURLs);
You will see this result:
[]
push
push
...
push
push
push
To prevent this from happening, only use the variable artistURLs inside of the HTTP request callback.