Node.js https.get return invalid json - javascript

Any help is welcome, I have been struggling for many hours now...
I have a somewhat straight forward code in which I ping the GitHub API to retrieve a JSON. If I execute this code synchronously it works very well. However, when I receiving multiple call at the same time (or if I run it in async.parallel), sometimes the result of aa is an invalid JSON (I can see it in my logs) and JSON.parse crash. It seems like there's a missing chunk in the final version of aa
app.get('/update-tests', function(req, res) {
const getOptionsGitHub = {
...
};
var req = https.get(getOptionsGitHub, function(res) {
aa = "";
res.on('data', function(chunk) { aa += chunk; });
res.on('end', function() {
try {
console.dir(aa);
callback(JSON.parse(aa));
} catch (e) {
console.dir(e);
}
});
});
res.send(200);
});
Any idea why I would have some missing chunk sometimes?

Related

GET call, get query params from callback url

I am making a GET call with the following URL
https://auth.ebay.com/oauth2/authorize?client_id=CLIENT_ID&response_type=code&redirect_uri=REDIRECT_URI&scope=https://api.ebay.com/oauth/api_scope
This URL will redirect me to a "success.php" website from my server. With that redirection, it adds in params to the URL. For example: https://www.example.com/success.php?code=12345.
I need to get that code param from this redirection. How can I do that?
I tried to do a basic .get() call, but it doesnt seem to work..
https.get(url, (resp) => {
let data = '';
// A chunk of data has been received.
resp.on('data', (chunk) => {
data += chunk;
});
// The whole response has been received. Print out the result.
resp.on('end', () => {
console.log(JSON.parse(data).explanation);
});
}).on("error", (err) => {
console.log("Error: " + err.message);
});
I have tried other ways that I thought would work from research on here, like waiting for the redirect, etc. Nothing seems to work.
It is a query param ( seems to me you are working with an oAuth flow, reading about how these flows work would also help you work out an approach to this)
So I would capture it the following way
app.get('/callback', function (req, res) {
var code = req.query.code || null;
console.log(code);
}
where /callback is the URL you are redirected to and where you can capture the code to request the authorization token
Based on the code you already have it seems you might want the following.
var https = require('https');
var url = 'https://auth.ebay.com/oauth2/authorize?client_id=CLIENT_ID&response_type=code&redirect_uri=REDIRECT_URI&scope=https://api.ebay.com/oauth/api_scope';
https.get(url, (resp) => {
var location = new URL(resp.headers.location);
var code = location.searchParams.get('code');
console.log(code);
}).on("error", (err) => {
console.log("Error: " + err.message);
});
My answer assumes you are writing the code that is making the request while #Jordi Riera assumes you are writing code to process the request. Might you tell us which it is?

Node writeFile not writing the whole file

I am trying to read a csv file and write the contents to a new csv file but am having a few issues in doing so.
My code seems to be reading the file ok and if i log the results of body in the console it shows the complete file. However when I try and write to the file, it does not seem to write all the contents.
The code I am using is as follows...
var http = require('http');
var fs = require('fs');
var readfile = "http://datafeed.api.productserve.com/datafeed.csv";
var writefile = "app/data/file.csv";
http.get(readfile, function(response) {
var body = '';
response.on("data", function(chunk) {
body += chunk;
});
response.on('end', function(){
fs.writeFile(writefile, body, function(error) {
if(error) {
console.error("write error: " + error.message);
} else {
console.log('Successful');
}
});
});
});
Am I doing something incorrect? Or is there some sort of limit on how much you can write?
Any help is much appreciated :)

Node.js cheerio weird characters (cyrillic letters)

I get weird characters when i am trying to parse a page.
Here is my code:
var getPageContent = function getPageContent(url, callback) {
https.get(url, function (res) {
var data = "";
res.on('data', function (chunk) {
data += chunk;
});
res.on("end", function () {
callback(data));
});
}).on("error", function () {
callback(null);
});
};
getPageContent(url, function (response) {
var $ = cheerio.load(response, { decodeEntities: false });
$("div.details-info").each(function() {
console.log($(this).html());
});
});
My result is:
<span>Ст��атегии</span>
<span>Стратег��и</span>
<span>Стра��егии</span>
<span>Стратегии</span>
<span>Стратегии</span>
...
The strangest thing is that from same url, sometimes i get this strange characters, sometimes i don't. And also when i am running this from my computer it's working fine. I get this characters on server only.
You will probably need to manually convert the charset of response to UTF-8. You can do this using the iconv or iconv-lite modules. cheerio itself does not automatically handle charset conversions.

Is there a way to check the body of a Node.js https request before I send it?

I'm trying to debug some API calls, and I want to confirm that I'm sending what I think I'm sending.
var req = https.request(request, function(res) {
var body;
res.setEncoding('utf8');
body = "";
res.on('data', function(chunk) {
return body += chunk;
});
res.on('end', function() {
if (res.statusCode === 200) {
return settings.success(null, JSON.parse(body), res);
} else {
return settings.error(body, null, res);
}
});
return res.on('error', function() {
return settings.error(null, Array.prototype.slice.apply(arguments), res);
});
});
if (settings.data != null) {
req.write(settings.data);
}
// check body of req here
settings.data is a protobuf buffer. I want to check the final written body because there is a bug somewhere in the call and I would like to see exactly what I am sending. Can anyone help?
EDIT: I misunderstood the original question. To check what you're sending, you can use util.inspect(settings.data) to return a string that displays the contents of that variable or use console.dir(settings.data) to display that string implicitly to stdout (console.dir() uses util.inspect() behind the scenes).
Original answer:
I believe that protocol buffers are binary, however you're converting all of the data to a utf8 string. You might try keeping it all as binary:
var body = [], nb = 0;
res.on('data', function(chunk) {
body.push(chunk);
nb += chunk.length;
});
res.on('end', function() {
body = Buffer.concat(body, nb);
// now `body` contains all protocol buffer data in the original binary format
});
If you then want to get a textual representation of what's in the buffer, you could use util.inspect(). For example:
var inspect = require('util').inspect;
// ...
res.on('end', function() {
body = Buffer.concat(body, nb);
if (res.statusCode === 200)
settings.success(null, inspect(body), res);
else
settings.error(body, null, res);
});

Unable to get data from REST Calls using Node.js

I am making a REST API call from my php and Node.js application to a particular URL provided by the client which returns a Json object. It works fine from with the PHP. However, I am unable to receive data from my node application? What might be the possible reason can someone help me ?
Note: I have pasted a dummy REST URI for security reasons
It works fine with PHP infact i get the json formatted data in like couple of seconds.
$response =
file_get_contents('http://xyz.net/v2_resmgr/providers/pools'); echo
$response;
I try the same url using node.js i get a TimeOut error. I also tried setting the timeout but it would still not work.
var job = new CronJob({
cronTime: '0 */3 * * * *',
onTick: function () {
url= "http://xyznet/v2_resmgr/providers/pools";
var request = http.get(url, function (response) {
var buffer = "",
data,
route;
response.on("data", function (chunk) {
buffer += chunk;
});
response.on("end", function (err) {
console.log(buffer);
});
request.setTimeout( 200000, function( ) {
// handle timeout here
console.log("Time Out call to the Rest API");
});
});
},
start: true
});
job.start();
I don't know if this is the answer you are looking for, but life gets easier when you use the 'request' package (https://www.npmjs.org/package/request)
Here is what the code would look like using the request module:
var request = require('request');
request('http://xyznet/v2_resmgr/providers/pools', function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body) // Print the body of the response.
}
})
Update: I coded something a little closer to your post. The code below does not use the "request" module and it contacts the server every 3 seconds.
setInterval(function () {
http.get('http://echo.jsontest.com/key/value', function (response) {
var responseBody = '';
response.on('data', function (chunk) {
responseBody += chunk;
});
response.on('end', function () {
console.log(responseBody);
var object = JSON.parse(responseBody)
});
});
}, 3000);

Categories

Resources