Develop Reference

How to get JSON objects embedded in HTML page result of JS running by PhantomJS and pass them to java code?

I use JS script code that described in this answer, but I don't want to save html result page in html file. I want to extract Json object from <div class="rg_meta"> and pass them to Java code.
In searching, I find using "document", but I get undefined error. I am newbie in PhantomJS and working with JSON in Java.
var page = require('webpage').create();
var fs = require('fs');
var system = require('system');
var url = "";
var searchParameter = "";
var count=0;
if (system.args.length === 4) {
url=system.args[1];
searchParameter=system.args[2];
count=system.args[3];
}
if(url==="" || searchParameter===""){
phantom.exit();
}
page.settings.userAgent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36';
page.zoomFactor = 0.1;
page.viewportSize = {
width: 1920,
height: 1080
};
var divCount="-1";
var topPosition=0;
var unchangedCounter=0;
page.open(url, function(status) {
console.log("Status: " + status);
if(status === "success") {
window.setInterval(function() {
var newDivCount = page.evaluate(function() {
var divs = document.querySelectorAll(".rg_di.rg_bx.rg_el.ivg-i");
return divs[divs.length-1].getAttribute("data-ri");
});
topPosition = topPosition + 1080;
page.scrollPosition = {
top: topPosition,
left: 0
};
if(newDivCount===divCount){
page.evaluate(function() {
var elems=document.getElementByClassName("rg_meta");
console.log(elems.length);
var button = document.querySelector("#smb");
if(!(typeof button === "undefined")) {
button.click();
console.log('Clicked');
return true;
}else{
return false;
}
});
if(parseInt(unchangedCounter,10) === parseInt(count,10)){
/* var path = searchParameter+'.html';
fs.write('seedHtml/'+path, page.content, 'w');
console.log('printing html');*/
phantom.exit();
}else{
unchangedCounter=unchangedCounter+1;
}
}else{
unchangedCounter=0;
}
divCount = newDivCount;
}, 500);
}else{
phantom.exit();
}
});

HTML5 data Attributes
Fortunately, HTML5 introduces custom data attributes.
<div id="msglist" data-user="bob" data-list-size="5" data-maxage="180"></div>
Custom data attributes:
are strings — you can store anything which can be string encoded, such as JSON. Type conversion must be handled in JavaScript.
should only be used when no suitable HTML5 element or attribute exists.
JavaScript Parsing #1:
Every browser will let you fetch and modify data- attributes using the getAttribute and setAttribute methods, e.g.
var msglist = document.getElementById("msglist");
var show = msglist.getAttribute("data-list-size");
msglist.setAttribute("data-list-size", show+3);
It works, but should only be used as a fallback for older browsers.
JavaScript Parsing #2:
Since version 1.4.3, jQuery’s data() method has parsed HTML5 data attributes. You don’t need to specify the data- prefix so the equivalent code can be written:
var msglist = $("#msglist");
var show = msglist.data("list-size");
msglist.data("list-size", show+3);
Hope it helps!!!

How to click a button ajax via casperjs?

This is my code. It run ok, it access site, and fill zipcode ok. But i dont know why it cant click button "GO" . What wrong in my code ? Thank you
Site is: https://www.az.aaa.com/membership/gift-membership-form?promocode=zumz0
Thank you !
var casper = require("casper").create();
var mouse = require("mouse").create(casper);
var casper = require('casper').create({
verbose: true,
logLevel: 'debug', // debug, info, warning, error
clientScripts: ["includes/jquery.min.js"],
pageSettings:{
loadImages: true,
loadPlugins: false,
userAgent: 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'
}
}
);
var fs = require('fs');
//var password = casper.cli.get(1);
function strip_tags(str) {
str = str.toString();
return str.replace(/<\/?[^>]+>/gi, '');
}
casper.start().thenOpen("https://www.az.aaa.com/membership/gift-membership-form?promocode=zumz0", function(){
console.log("1. Access Site");
});
casper.waitForSelector('#edit-zipcode', function(){
console.log("2. Box Zipcode Found ! ");
this.capture('site_box_zipcode.png');
});
casper.then(function(){
fs.write("debug_zipcode.html", this.getPageContent(), 'w');
});
casper.then(function(){
console.log("3. Filling Form Zipcode")
this.evaluate(function(){
$('#edit-zipcode').attr('value', '86322');
$('#edit-tqs1-submit').click();
//document.getElementById("edit-zipcode").value='86322';
//document.getElementById("edit-tqs1-submit").click();
});
//this.mouse.rightclick('button.col-xs-12 col-md-1 product-button js-product-button pull-right');
})
//
casper.then(function(){
this.wait(7000);//Wait a bit so page loads (there are a lot of ajax calls and that is why we are waiting 6 seconds)
this.capture('Afterpostzipcode.png');
console.log("4. Finish Capture Picture");
});
casper.then(function(){
fs.write("debug_fl.html", this.get, 'w');
});
casper.run();

Phantomjs check for response headers and then execute something

I have the following RequestURL.js file.
var webPage = require('webpage');
var system = require('system');
var page = webPage.create();
page.customHeaders = {"pragma": "akamai-x-feo-trace"};
page.settings.userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36"
if (system.args.length === 1) {
console.log('Try to pass some args when invoking this script!');
} else {
page.open(system.args[1], function (status) {
var content = page.content;
console.log(content);
phantom.exit();
});
}
Now I execute this as phantomjs --ignore-ssl-errors=yes --ssl-protocol=any RequestURL.js #my_url_here > body.html
Now I have a parser written in python that takes body.html and executes it. Now before that I want the page source to get generated only if the response contains the following header.
X-Akamai-FEO-State:TRANSFORMING
Is there a way to modify my RequestURL.js to get there.

It is expected that page.onResourceReceived is triggered before the page.onLoadFinished callback of page.open().
var transforming = false;
page.onResourceReceived = function(response){
if (response.url === system.args[1]) { // TODO handle redirects if necessary
response.headers.forEach(function(header){
if(header.name === 'X-Akamai-FEO-State') {
transforming = header.value === 'TRANSFORMING';
}
});
}
};
page.open(system.args[1], function (status) {
if (transforming) {
console.log(page.content);
}
phantom.exit();
});

Scrape chained selects with updated data using CasperJS

There are 2 selects with IDs. The 2nd select box data is linked based on what you select on the first select box. In other words, if you select "BMW" in the second select box should appear 316,318,320 ... you got the point.
The first select has listener
('#brand').change(function(){
call ajax and fill the data for the second select box
}
What I want to do at the end is to get all the options for models, for all the brands :-)
What I got so far is:
var casper = require('casper').create({
loadImages:false,
verbose: true,
logLevel: 'debug',
clientScripts: ["includes/jquery.min.js"]
});
casper.userAgent('Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36');
casper.on('remote.message', function(msg) {
this.echo('remote message caught: ' + msg);
});
casper.on( 'page.error', function (msg, trace) {
this.echo( 'Error: ' + msg, 'ERROR' );
});
casper.start('http://www.mywebsite.kitchen/');
casper.then(function(){
this.evaluate(function(valueOptionSelect){
$('#brand').val(6).trigger('change');
},optionFirstSelect);
this.waitFor(function check() {
return this.evaluate(function(casper) {
var len = $('#model1 option').length;
console.log('length of options is ->', len);
return $('#model1 option').length > 1;
});
}, function then() {
//well i still haven't reach that point
}, function timeOut(){
casper.echo(arguments)
});
});
casper.run(function() {
//finish execution script
this.exit();
});
now what I get for console log is:
//EDIT - the length is 1 not 0
length of options is 1
When I execute $('#brand').val(6).trigger('change'); $('#model1 option').length in my browser console I get correct results.

fastest way to scrape text node with casperjs

I have this structure and I need get text from plain text node like this
<strong><font color="#666666">Phones:</font></strong>
<br>
<br>
<img src="/image/fgh.jpg" title="Velcom" alt="Velcom" style="margin: 2 5 -3 5;">
"+375 29" //get this
<b>611 77 83</b> //and this
I try to use XPath copied from chrome console
casper.thenOpen('url', function() {
result = this.getElementInfo(x('//*[#id="main_content"]/table[2]/tbody/tr[17]/td/table/tbody/tr/td[1]/p[1]/text()[3]'));
});
casper.then(function() {
this.echo(result.text);
});
but it is not working. Also when I try result.data
console.log(this.getElementInfo(x('//*[#id="main_content"]/table[2]/tbody/tr[17]/td/table/tbody/tr/td[1]/p[1]/text()[3]')));
returns null, but this element exists in the page, I checked it out

Make sure you have included:
var x = require('casper').selectXPath;
If that is still not working the following will retrieve all text from page then you can parse. This is not recommended for performance but does work if you have anchor text to parse on. You will need to slightly modify.
var casper = require("casper").create ({
waitTimeout: 15000,
stepTimeout: 15000,
verbose: true,
viewportSize: {
width: 1400,
height: 768
},
onWaitTimeout: function() {
logConsole('Wait TimeOut Occured');
this.capture('xWait_timeout.png');
this.exit();
},
onStepTimeout: function() {
logConsole('Step TimeOut Occured');
this.capture('xStepTimeout.png');
this.exit();
}
});
casper.on('remote.message', function(msg) {
logConsole('***remote message caught***: ' + msg);
});
casper.userAgent('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.4');
// vars
var gUrl = 'WebAddy'; //+++ Update URL
casper.start(gUrl, function() {
var tPlainText = this.evaluate(function() {
var bodyText = document.body;
var textContent = bodyText.textContent || bodyText.innerText;
var tCheck = textContent.indexOf("Phones:");
if (tCheck === -1) {
tPlainText = 'Phone Text Not Found';
return tPlainText;
} else {
// parse text
var tSplit = textContent.split('Phones:');
var tStr = (tSplit[1]) ? tSplit[1] : '';
var tPos1 = tStr.indexOf(''); //+++ insert text to stop parse
var tDesiredText = (tPos1 !== -1) ? tStr.substring(0, tPos1) : null;
return tDesiredText;
}
});
console.log(tPlainText);
});
casper.run();

An old question but I had the same issue. I need to get the following text, so here is how I did it.
__utils__.getElementByXPath("//bla...bla/following-sibling::node()").textContent;