Develop Reference

GET call, get query params from callback url

I am making a GET call with the following URL
https://auth.ebay.com/oauth2/authorize?client_id=CLIENT_ID&response_type=code&redirect_uri=REDIRECT_URI&scope=https://api.ebay.com/oauth/api_scope
This URL will redirect me to a "success.php" website from my server. With that redirection, it adds in params to the URL. For example: https://www.example.com/success.php?code=12345.
I need to get that code param from this redirection. How can I do that?
I tried to do a basic .get() call, but it doesnt seem to work..
https.get(url, (resp) => {
let data = '';
// A chunk of data has been received.
resp.on('data', (chunk) => {
data += chunk;
});
// The whole response has been received. Print out the result.
resp.on('end', () => {
console.log(JSON.parse(data).explanation);
});
}).on("error", (err) => {
console.log("Error: " + err.message);
});
I have tried other ways that I thought would work from research on here, like waiting for the redirect, etc. Nothing seems to work.

It is a query param ( seems to me you are working with an oAuth flow, reading about how these flows work would also help you work out an approach to this)
So I would capture it the following way
app.get('/callback', function (req, res) {
var code = req.query.code || null;
console.log(code);
}
where /callback is the URL you are redirected to and where you can capture the code to request the authorization token

Based on the code you already have it seems you might want the following.
var https = require('https');
var url = 'https://auth.ebay.com/oauth2/authorize?client_id=CLIENT_ID&response_type=code&redirect_uri=REDIRECT_URI&scope=https://api.ebay.com/oauth/api_scope';
https.get(url, (resp) => {
var location = new URL(resp.headers.location);
var code = location.searchParams.get('code');
console.log(code);
}).on("error", (err) => {
console.log("Error: " + err.message);
});
My answer assumes you are writing the code that is making the request while #Jordi Riera assumes you are writing code to process the request. Might you tell us which it is?

How to stop CasperJS execution and let the user input some value and then continue to execute?

I'm using PhantomJS and CasperJS to automate some of my tasks. In one of the task, I need to manually provide captcha strings before I can actually work on the task. For this problem, what I can think of is to capture a screenshot of the web page, then manually check the captured image and save the captcha string into a text file. After that I can use the file system module in CasperJS to read that value and continue to do the process. I want to know what's the best way to do this kind of tasks.

Because of the stuctured manner/control flow of CasperJS compared to PhantomJS, such a task is not easy.
1. Pull approach (file polling)
Let's say there is a secondary program (type 1) which handles showing the CAPTCHA, receiving the input and writing a text file with the CAPTCHA input. All that CasperJS can handle is to write the CAPTCHA screenshot to disk and wait for the file with the "parsed" text.
var fs = require("fs"),
captchaFile = "cfile.png",
parsedFile = "pfile.txt";
casper.waitForCaptcha = function(captchaFile, parsedFile){
casper.then(function(){
this.captureSelector(captchaFile, "someSelectorOfTheCaptcha");
});
casper.waitFor(function check(){
return fs.exists(parsedFile);
}, function then(){
// do something on time
// check if correct...
if (!correct) {
fs.remove(captchaFile);
fs.remove(parsedFile);
this.waitForCaptcha(captchaFile, parsedFile);
// Problem: the secondary process needs to sense that a new CAPTCHA is presented
}
}, function onTimeout(){
// do something when failed
}, 60000); // 1min should suffice as a timeout
return this;
};
casper.start(url).waitForCaptcha(captchaFile, parsedFile).run();
This code assumes that you want to retry when the CAPTCHA was wrong, but not if the minute deliberately passed without the decoded file. This is a pull process by polling if files are already there.
2. Push approach
A push process is also possible where the secondary program (type 2) sends requests to the CasperJS process by utilizing the PhantomJS webserver module. Because there will be two concurrent control flows, the CasperJS part needs to wait a long time, but as soon as a request is received with the decoded words the waiting can be broken with unwait.
var server = require('webserver').create(),
fs = require("fs"),
captchaFile = "cfile.png";
function neverendingWait(){
this.wait(5000, neverendingWait);
}
casper.checkCaptcha = function(captchaFile, phantomPort, secondaryPort){
// here the CAPTCHA is saved to disk but it can also be set directly if captured through casper.captureBase64
this.captureSelector(captchaFile, "someSelectorOfTheCaptcha");
// send request to the secondary program from the page context
this.evaluate(function(file){
__utils__.sendAJAX("http://localhost:"+secondaryPort+"/", "POST", {file: file}, true);
}, captchaFile);
// start the server to receive solved CAPTCHAs
server.listen(phantomPort, {
'keepAlive': true
}, function (request, response) {
console.log('Request received at ' + new Date());
if (request.post) { // is there a response?
this.then(function(){
// check if it is correct by reading request.post ...
if (!correct){
response.statusCode = 404;
response.headers = {
'Cache': 'no-cache',
'Content-Type': 'text/plain;charset=utf-8'
};
response.close();
server.close();
this.checkCaptcha(captchaFile, phantomPort, secondaryPort);
} else {
response.statusCode = 200;
response.headers = {
'Cache': 'no-cache',
'Content-Type': 'text/plain;charset=utf-8'
};
response.close();
server.close();
this.unwait(); // abort the neverendingWait
}
});
} else {
response.statusCode = 404;
response.headers = {
'Cache': 'no-cache',
'Content-Type': 'text/plain;charset=utf-8'
};
response.close();
server.close();
this.checkCaptcha(captchaFile, phantomPort, secondaryPort);
}
});
return this;
};
casper.start(url).then(function(){
this.checkCaptcha(captchaFile, 8080, 8081);
}).then(neverendingWait).then(function(){
// Do something here when the captcha is successful
}).run();

Node.js sending http request in a loop

I'm actually facing a problem with my javascript code executed with node.js
i need to send http requests in a loop to a distant server (i set www.google.ca in the code).
Here is my code :
var http = require('http');
var options = {
hostname: 'www.google.ca',
port: 80,
path: '/',
method: 'GET'
};
function sendRequest(options){
console.log('hello');
var start = new Date();
var req = http.request(options,function(res) {
console.log('Request took:', new Date() - start, 'ms');
});
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
});
req.end();
};
for(var i=0;i<10;i++){
sendRequest(options);
}
The problem I have is that, no matter how many times i go through my loop, i get a response for only the 5 first of them. For the rest of the requests, the function sendRequest() is called but I don't get any responses, neither error message. And then the program terminates.
However it works fine when I set localhost as a host.
Is anyone would have a solution to this problem ?
Thanks in advance !

perhaps either your machine or the remote machine is getting overwhelmed by the 10 simultaneous requests you make. try sending them one at a time, you will have to wait until the first request completes before continuing. one easy way to do so is with async.timesSeries
var http = require('http');
var async = require('async');
var options = {
hostname: 'www.google.ca',
port: 80,
path: '/',
method: 'GET'
};
function sendRequestWrapper(n, done){
console.log('Calling sendRequest', n);
sendRequest(options, function(err){
done(err);
});
};
function sendRequest(options, callback){
//console.log('hello');
var start = new Date();
var req = http.request(options,function(res) {
// I don't know if this callback is called for error responses
// I have only used the `request` library which slightly simplifies this
// Under some circumstances you can accidentally cause problems by calling
// your callback function more than once (e.g. both here and on('error')
console.log('Request took:', new Date() - start, 'ms');
callback(null);
});
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
callback(err);
});
req.end();
};
async.timesSeries(10, sendRequestWrapper);

Nodejs http request returning json not in json format

I'm trying to make a request to here, if you click on the link you should see a JSON response (as expected). I've, tried https and http, it doesnt matter (at least I don't think so).
Anyways the problem when I try to get a response from the command line, I get non UTF-8 characters like �������B��������E��9 as a response, even when I specify utf-8 encoding. Ive tried the npm module request and doing node http/https requests.
All i need is to just return a JSON response.
I've also tried JSON.parse() but to no avail.
Here's the code I've tried
var request = require("request")
var url = endpoint;
request({
url: url,
json: true
}, function (error, response, body) {
if (!error && response.statusCode === 200) {
console.log(body); // Print the json response
}
})
and the basic http request
var endpoint = 'http://api.stackexchange.com/2.1/search/advanced?order=desc&sort=relevance&q=jsonp&site=stackoverflow';
var req = http.request(endpoint, function(res) {
res.setEncoding('utf8');
res.on('data', function (chunk) {
console.log('BODY: ' + chunk);
});
});
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
});
// write data to request body
req.write('data\n');
req.write('data\n');
req.end();

Stackoverflow servers are misconfigured, so they return gzip-encoded body even though you didn't ask for it.
Looks like you have to gunzip your response after receiving it.

Function to check if externally hosted textfile contains string acting slow- Node

I have the following code, which will retrieve a text file from an external server, and search the file for a specific string.
The function:
function checkStringExistsInFile(String, cb) {
var opt = {
host: 'site.com',
port: 80,
path: '/directory/data.txt',
method: 'GET',
};
var request = http.request(opt, function(response){
response
.on('data', function(data){
var string = data+"";
var result = ((string.indexOf(" "+ String +" ")!=-1)?true:false);
cb(null, result);
})
.on('error', function(e){
console.log("-ERR: Can't get file. "+JSON.stringify(e));
if(cb) cb(e);
})
.on('end', function(){
console.log("+INF: End of request");
});
});
request.end();
}
And this is where I call the function, and do something with the results.
checkStringExistsInFile(String, function(err, result){
if(!err) {
if(result) {
//there was a result
} else {
//string not present in file
}
} else {
// error occured
}
});
This worked great in the beginning (small text file), but my textfile is getting larger (4000 characters+) and this is not working anymore.
What can I do to solve this? Should I safe the temporary save the file on my server first, should I open the file as a stream?
It would be appreciated if you can support your answer with a relevant example. Thanks in advance!

Documentation :
If you attach a data event listener, then it will switch the stream into flowing mode, and data will be passed to your handler as soon as it is available.
If you just want to get all the data out of the stream as fast as possible, this is the best way to do so.
http://nodejs.org/api/stream.html#stream_event_data
Data event is emitted as soon as there are data, even if the stream is not completely loaded. So your code just look for your string in the first lines of your stream, then callbacks.
What to do ?
Your function should only call callback on the end() event, or as soon as it finds something.
function checkStringExistsInFile(String, cb) {
var opt = {
host: 'site.com',
port: 80,
path: '/directory/data.txt',
method: 'GET',
};
var request = http.request(opt, function(response){
var result = false;
response
.on('data', function(data){
if(!result) {
var string = data+"";
result = (string.indexOf(" "+ String +" ")!=-1);
if(result) cb(null, result);
}
})
.on('error', function(e){
console.log("-ERR: Can't get file. "+JSON.stringify(e));
if(cb) cb(e);
})
.on('end', function(){
console.log("+INF: End of request");
cb(null, result)
});
});
request.end();
}