CasperJS Screenshotting Different URLs - javascript

I've just started using CasperJS so I'm pretty stuck on an issue.
What I want to do is load a URL (Login protected), find a list of links (Which change the theme of the forum, specifically this part: ".styleChooser .overlayScroll li a"), click each of them and screenshot the result of the page after the click at two resolutions.
My code is currently just a grouping of other suggestions around the net trying to get this working, however I believe all the necessary code is basically there I just can't get it working. Any help would be very much appreciated!
var casper = require("casper").create();
var screenshotUrl = "URL Here";
var screenshotPaths = "rivals";
function getLinks() {
var links = document.querySelectorAll('.styleChooser .overlayScroll li a');
links = Array.prototype.map.call(links,function(link){
return link.getAttribute('href');
});
return links;
}
screenshotNow = new Date(),
screenshotDateTime = screenshotNow.getFullYear() + pad(screenshotNow.getMonth() + 1) + pad(screenshotNow.getDate()),
viewports = [
{
'name': 'smartphone-portrait',
'viewport': {width: 320, height: 480}
},
{
'name': 'desktop-standard',
'viewport': {width: 1280, height: 1024}
}
];
i = -1;
casper.start();
casper.setHttpAuth('Username', 'Password');
casper.thenOpen(screenshotUrl, function(response) {
var linksArray = this.evaluate(getLinks);
this.eachThen(linksArray, function(response) {
var url = response.data;
this.each(viewports, function(casper, viewport) {
this.then(function() {
this.viewport(viewport.viewport.width, viewport.viewport.height);
});
this.thenOpen(url, function() {
this.wait(5000);
});
casper.then(function(){
casper.echo('Screenshot for '+ screenshotPaths + '/' + "homepage " + viewport.name + ' (' + viewport.viewport.width + 'x' + viewport.viewport.height + ')', 'info');
casper.capture('screenshots/' + screenshotPaths + '/' + screenshotDateTime + '/' + "homepage" + '/' + viewport.name + '-' + viewport.viewport.width + 'x' + viewport.viewport.height + '.png', {
top: 0,
left: 0,
width: viewport.viewport.width,
height: viewport.viewport.height
});
});
});
});
++i;
}); // error is here
casper.run();
function pad(number) {
var r = String(number);
if ( r.length === 1 ) {
r = '0' + r;
}
return r;
}
And this is the error:
C:\xampp\htdocs\caspertest>casperjs newestcasper.js
CasperError: You can only define a step as a function
C:/casperjs/modules/casper.js:1755 in then
C:/xampp/htdocs/caspertest/newestcasper.js:52
Unsafe JavaScript attempt to access frame with URL about:blank from frame with U
RL file:///C:/casperjs/bin/bootstrap.js. Domains, protocols and ports must match
.

If you want to open the page that you want to take a screenshot of, you should use thenOpen. Right now, you use then which is only a step function without opening anything.
casper.thenOpen(screenshotUrl, function(response) {

Related

batch rest call in sharepoint online

I am trying to understand how the batch rest calls work.
I could not find any simple example on the internet. I have found the examples from https://github.com/andrewconnell/sp-o365-rest but can't run these examples or I have no idea how yet. I am guessing you have to deploy the app to a sharepoint site.
Given that, I am just looking for the simplest example of a add list item and update list item in bulk/batch. Also if anyone knows how I can make the app from that git to run will be really appreciated.
Thanks.
The github project is a add-in project so you need deploy the add-in project, then you can use it.
You could check below script from here.
My test result in this thread
(function () {
jQuery(document).ready(function () {
jQuery("#btnFetchEmployees").click(function () {
addEmployees();
});
});
})();
function addEmployees() {
var employeesAsJson = undefined;
employeesAsJson = [
{
__metadata: {
type: 'SP.Data.EmployeeInfoListItem'
},
Title: 'Geetanjali',
LastName: 'Arora',
Technology: 'SharePoint'
},
{
__metadata: {
type: 'SP.Data.EmployeeInfoListItem'
},
Title: 'Geetika',
LastName: 'Arora',
Technology: 'Graphics'
},
{
__metadata: {
type: 'SP.Data.EmployeeInfoListItem'
},
Title: 'Ashish',
LastName: 'Brajesh',
Technology: 'Oracle'
}
];
addEmployeeInfoBatchRequest(employeesAsJson);
}
function generateUUID() {
var d = new Date().getTime();
var uuid = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) {
var r = (d + Math.random() * 16) % 16 | 0;
d = Math.floor(d / 16);
return (c == 'x' ? r : (r & 0x7 | 0x8)).toString(16);
});
return uuid;
};
function addEmployeeInfoBatchRequest(employeesAsJson) {
// generate a batch boundary
var batchGuid = generateUUID();
// creating the body
var batchContents = new Array();
var changeSetId = generateUUID();
// get current host
var temp = document.createElement('a');
temp.href = _spPageContextInfo.webAbsoluteUrl;
var host = temp.hostname;
// iterate through each employee
for (var employeeIndex = 0; employeeIndex < employeesAsJson.length; employeeIndex++) {
var employee = employeesAsJson[employeeIndex];
// create the request endpoint
var endpoint = _spPageContextInfo.webAbsoluteUrl
+ '/_api/web/lists/getbytitle(\'EmployeeInfo\')'
+ '/items';
// create the changeset
batchContents.push('--changeset_' + changeSetId);
batchContents.push('Content-Type: application/http');
batchContents.push('Content-Transfer-Encoding: binary');
batchContents.push('');
batchContents.push('POST ' + endpoint + ' HTTP/1.1');
batchContents.push('Content-Type: application/json;odata=verbose');
batchContents.push('');
batchContents.push(JSON.stringify(employee));
batchContents.push('');
}
// END changeset to create data
batchContents.push('--changeset_' + changeSetId + '--');
// batch body
var batchBody = batchContents.join('\r\n');
batchContents = new Array();
// create batch for creating items
batchContents.push('--batch_' + batchGuid);
batchContents.push('Content-Type: multipart/mixed; boundary="changeset_' + changeSetId + '"');
batchContents.push('Content-Length: ' + batchBody.length);
batchContents.push('Content-Transfer-Encoding: binary');
batchContents.push('');
batchContents.push(batchBody);
batchContents.push('');
// create request in batch to get all items after all are created
endpoint = _spPageContextInfo.webAbsoluteUrl
+ '/_api/web/lists/getbytitle(\'EmployeeInfo\')'
+ '/items?$orderby=Title';
batchContents.push('--batch_' + batchGuid);
batchContents.push('Content-Type: application/http');
batchContents.push('Content-Transfer-Encoding: binary');
batchContents.push('');
batchContents.push('GET ' + endpoint + ' HTTP/1.1');
batchContents.push('Accept: application/json;odata=verbose');
batchContents.push('');
batchContents.push('--batch_' + batchGuid + '--');
batchBody = batchContents.join('\r\n');
// create the request endpoint
var endpoint = _spPageContextInfo.webAbsoluteUrl + '/_api/$batch';
var batchRequestHeader = {
'X-RequestDigest': jQuery("#__REQUESTDIGEST").val(),
'Content-Type': 'multipart/mixed; boundary="batch_' + batchGuid + '"'
};
// create request
jQuery.ajax({
url: endpoint,
type: 'POST',
headers: batchRequestHeader,
data: batchBody,
success: function (response) {
var responseInLines = response.split('\n');
$("#tHead").append("<tr><th>First Name</th><th>Last Name</th><th>Technology</th></tr>");
for (var currentLine = 0; currentLine < responseInLines.length; currentLine++) {
try {
var tryParseJson = JSON.parse(responseInLines[currentLine]);
$.each(tryParseJson.d.results, function (index, item) {
$("#tBody").append("<tr><td>" + item.Title + "</td><td>" + item.LastName + "</td><td>" + item.Technology + "</td></tr>");
});
} catch (e) {
}
}
},
fail: function (error) {
}
});
}

PhantomJS Button Clicking and Form Submission

I'm pretty new to PhantomJS, so I'm just trying to stumble through some practical examples (the documentation feels a little light on those).
One thing I was attempting was to submit votes to a phony PollDaddy poll, but I'm struggling. Using the script below, I can tell from the first screenshot that my option button is being clicked/selected. But why does my answer not get submitted? The second screenshot looks identical to the first, despite my code executing a click event on the VOTE button in the second evaluate. Anyone know why? Know how to make it work?
var webPage = require('webpage');
var page = webPage.create();
//******* BEGIN LOGGING METHODS *******
// http://phantomjs.org/api/webpage/handler/on-url-changed.html
page.onUrlChanged = function(targetUrl) {
console.log('New URL: ' + targetUrl);
};
// http://phantomjs.org/api/webpage/handler/on-console-message.html
page.onConsoleMessage = function(msg, lineNum, sourceId) {
console.log('CONSOLE: ' + msg + ' (from line #' + lineNum + ' in "' + sourceId + '")');
};
// http://phantomjs.org/api/webpage/handler/on-error.html
page.onError = function(msg, trace) {
var msgStack = ['ERROR: ' + msg];
if (trace && trace.length) {
msgStack.push('TRACE:');
trace.forEach(function(t) {
msgStack.push(' -> ' + t.file + ': ' + t.line + (t.function ? ' (in function "' + t.function +'")' : ''));
});
}
console.error(msgStack.join('\n'));
};
// http://phantomjs.org/api/webpage/handler/on-resource-error.html
page.onResourceError = function(resourceError) {
console.log('Unable to load resource (#' + resourceError.id + ' URL:' + resourceError.url + ')');
console.log('Error code: ' + resourceError.errorCode + '. Description: ' + resourceError.errorString);
};
// http://phantomjs.org/api/webpage/handler/on-resource-timeout.html
page.onResourceTimeout = function(request) {
console.log('Response Timeout (#' + request.id + '): ' + JSON.stringify(request));
};
//******* END LOGGING METHODS *******
page.open('https://polldaddy.com/poll/9424638/', function(status) {
console.log('Status: ' + status);
//make selection
var myselection = page.evaluate(function() {
var ev = document.createEvent("MouseEvent");
ev.initMouseEvent("click", true, true, window, null,0, 0, 0, 0, false, false, false, false, 0, null);
//radio button for Curly has id=PDI_answer42988707
var myselection = document.querySelector("#PDI_answer42988707");
(myselection).dispatchEvent(ev);
return document.title;
});
//screen capture the selection
page.render('selection.png');
console.log(myselection);
//click the Vote button
var dovote = page.evaluate(function() {
var ev = document.createEvent("MouseEvent");
ev.initMouseEvent("click", true, true, window, null,0, 0, 0, 0, false, false, false, false, 0, null);
//get a handle to the vote button
var votebutton = document.querySelector("a.vote-button");
//click the vote button
(votebutton).dispatchEvent(ev);
return document.title;
});
//delay, then take screenshot...
setTimeout(
function(){
page.render('voted.png');
console.log(dovote);
}
,2000
);
});
I'm executing the script using PhantomJS 1.9.0 on Linux Mint. The execution parameters are like this to overcome any SSL Handshake failures:
phantomjs --ignore-ssl-errors=true --ssl-protocol=any myscript.js
I've also tried setting --cookies-file and --local-storage-path on the command line, but that didn't help either.
The output I get from all of the console logging above is this:
New URL: https://polldaddy.com/poll/9424638/ <-- from urlChange
CONSOLE: JQMIGRATE: Logging is active (from line # in "") <-- from onConsoleMessage
Status: success
Who is your favorite Stooge (poll 9424638) | Polldaddy.com
Who is your favorite Stooge (poll 9424638) | Polldaddy.com
My useless poll is here: https://polldaddy.com/poll/9424638/, and you'll see from my script that I'm trying to stack the results for Curly.
If you look at the javascript that is being included with the polldaddy page, you'll find a vote function with the following check:
if (event.pageX) {
eventX = event.pageX;
eventY = event.pageY;
} else {
eventX = event.clientX;
eventY = event.clientY;
}
...
if (eventX == 0 && eventY == 0) {
return false
}
Basically they are looking at the event location and short-circuiting if the x and y location are both equal to 0.
If you change the mouse event that you create so that the clientX or clientY value is not 0, then your script should work.

Jquery mobile open individual feed items on separate page

I have this code which is supposed to be an RSS reader using JQuery, HTML5 and CSS.
The challenge is that, I'm trying to open each individual feed item on a separate page after clicking on the title.
I'm not sure how to go about it.
Here's my code:
function getRSSFeed(feed) {
var qryRSS = 'select * from rss where url=' + '"' + feed + '"';
$.getJSON("http://query.yahooapis.com/v1/public/yql", {
q: qryRSS,
format: "json"
},
function (data) {
for (i = 0; i < 10; i++)
{
var con = data.query.results.item[i].link;
$('#body-content').append(' <div id="rss-bdy">' + data.query.results.item[i].encoded + '</div>');
$('#body-content').append('<hr>')
$('#body-content').append('<div id="rss-ttle"><h4>' + data.query.results.item[i].title + '</h4></div>');
$('#body-content').append('<p id="rss-dte">' + data.query.results.item[i].pubDate + '</p><br>');
$('#body-content2').append('<div id="full-article">' + con + '</div>');
}
$(function () {
$("#rssDivBody").rssDivBody();
});
});
};
getRSSFeed('meps4wildlife.eu/feed');

How can I make my page.open() requests sequentially?

I coded this, but it's still non-sequential. I hoped with the functions it would wait until the actual request finishes until a new one is called .... but that doesn't work.
Problem 1: The page.open() calls are not sequential as you can see here:
6 protocol: https: type: Content
7 protocol: https: type: Content
8 protocol: https: type: Content
9 protocol: https: type: Content
LINE: https://www.roller.de/einrichten/
10 protocol: https: type: Content
11 protocol: https: type: Content
12 protocol: https: type: Content
LINE: https://www.roller.de/einrichten/anrichte/
LINE: https://www.roller.de/einrichten/arbeitsstuhl/
LINE: https://www.roller.de/einrichten/arbeitstisch/
LINE: https://www.roller.de/einrichten/armlehnstuehle/
LINE: https://www.roller.de/einrichten/badezimmermoebel
LINE: https://www.roller.de/einrichten/bistrostuehle/
LINE: https://www.roller.de/einrichten/buecherregal/
13 protocol: https: type: Content
14 protocol: https: type: Content
15 protocol: https: type: Content
16 protocol: https: type: Content
LINE: https://www.roller.de/einrichten/buerocontainer/
LINE: https://www.roller.de/einrichten/bueroregale/
17 protocol: https: type: Content
18 protocol: https: type: Content
The LINE: should only be printed once per request, but it appears several time without the page.open result, resulting in an early stream.atEnd() = true. That should be impossible if it's sequential.
Problem 2: The last line is not taken, when I have a .txt file with 100 links (1 per line), 99 are printed, one is not
Problem 3: it crashes when I give it a list with 1000 urls
Problem 4: 10 links = 10 prints, 100 links = 98 prints and stream.atEnd() does appear several times, 500 links = 497-498 prints + stream.atEnd() problem, 1000 links = Crash
console.log('Hello, world!');
var fs = require('fs');
var stream = fs.open('100sitemap.txt', 'r');
var webPage = require('webpage');
var i = 1;
function nextPage() {
if (stream.atEnd()) {
//stream.close();
console.log("STREAM END: " + stream.atEnd());
console.log("FILE ENDS HERE");
//phantom.exit();
}
if (!stream.atEnd()) {
var line = stream.readLine();
console.log("LINE: " + line);
getRequest(line);
}
}
function getRequest(line2) {
//console.log(line);
var page = webPage.create();
page.settings.loadImages = false;
page.open(line2, function() {});
//console.log("page.open() " + line2);
//console.log("opened " + line2);
page.onResourceRequested = function(requestData, request) {
//console.log("BEFORE: " + requestData.url);
var match = requestData.url.match(/example.com\/ca/g)
//console.log("Match: " + match);
//console.log(request.url);
if (match != null) {
hasFound = true;
var targetString = decodeURI(JSON.stringify(requestData.url));
var klammerauf = targetString.indexOf("{");
var jsonobjekt = targetString.substr(klammerauf, (targetString.indexOf("}") - klammerauf) + 1);
targetJSON = (decodeURIComponent(jsonobjekt));
var t = JSON.parse(targetJSON);
console.log(i + " " + t['groups'] + " " + t['campID']);
i++;
//console.log(targetJSON);
request.abort;
}
};
page.onLoadFinished = function(status) {
if (!hasFound) {
console.log(i + " :NOT FOUND: " + line2);
i++;
}
//request.abort();
page.close();
nextPage();
}
}
nextPage();
Now it works with this code, iFrames seems to trigger onLoadFinished() twice, so I check that with hasOnLoadFinished to prevent the multiple entries in the function (using multiple page.open() at once is a really bad idea in PhantomJS).
Be aware that 2.0 will crash with too many links / too many urls (in my case 120-180) for unknown reasons (most times no error message, rare times a "QThread::start: Failed to create thread ()".
To prevent that, use the 1.9.8 version instead of 2.0, seems to be a bug there, filled a crash report with dumps on Github.
/edit crashed without an error message after 3836 links with 1.9.8 links ............... PHANTOM.
console.log('Hello, world!');
var fs = require('fs');
var stream = fs.open('linklist.de.txt', 'r');
var webPage = require('webpage');
var i = 1;
var hasFound = Array();
var hasonLoadFinished = Array();
function handle_page(link) {
var page = webPage.create();
page.settings.loadImages = false;
page.open(link, function() {});
page.onResourceRequested = function(requestData, request) {
var match = requestData.url.match(/example.com\/searchmeI'maString/g)
if (match != null) {
hasFound[link] = true;
var targetString = decodeURI(JSON.stringify(requestData.url));
var klammerauf = targetString.indexOf("{");
var jsonobjekt = targetString.substr(klammerauf, (targetString.indexOf("}") - klammerauf) + 1);
targetJSON = (decodeURIComponent(jsonobjekt));
var t = JSON.parse(targetJSON);
console.log(i + " " + t + " " + t['id']);
//console.log(targetJSON);
//console.log("");
request.abort;
} else {
request.abort;
return;
}
};
page.onLoadFinished = function(status) {
if (!hasonLoadFinished[link]) {
hasonLoadFinished[link] = true;
//console.log(" " + status + " " + link);
//console.log("onLoadFinished()")
//setTimeout(function(){/* Look mah! No name! */},1000);
if (!hasFound[link]) {
console.log(i + " :NOT FOUND: " + link);
console.log("");
}
i++;
page.close();
nextPage();
}
}
};
function nextPage() {
var link = stream.readLine();
if (!link) {
end = Date.now();
console.log("");
console.log(((end - start) / 1000) + " Sekunden");
console.log("FILE ENDS HERE!!!");
phantom.exit(0);
}
hasFound[link] = false;
hasonLoadFinished[link] = false;
handle_page(link);
}
start = Date.now();
nextPage();

Merging JSON api Response using Javascript

I am trying get paged json responses from Topsy (http://code.google.com/p/otterapi/) and am having problems merging the objects. I want to do this in browser as the api rate limit is per ip/user and to low to do things server side.
Here is my code. Is there a better way? Of course there is because this doesn't work. I guess I want to get this working, but also to understand if there is a safer, and/or more efficient way.
The error message I get is ...
TypeError: Result of expression 'window.holdtweetslist.prototype' [undefined] is not an object.
Thanks in advance.
Cheers
Stephen
$("#gettweets").live('click', function(event){
event.preventDefault();
getTweets('stephenbaugh');
});
function getTweets(name) {
var MAX_TWEETS = 500;
var TWEETSPERPAGE = 50;
var BASE = 'http://otter.topsy.com/search.json?type=tweet&perpage=' + TWEETSPERPAGE + '&window=a&nohidden=0&q=#' + name + '&page=1';
var currentpage = 1;
alert(BASE);
$.ajax({
dataType: "json",
url: BASE,
success: function(data) {
window.responcesreceived = 1;
var response=data.response;
alert(response.total);
window.totalweets = response.total;
window.pagestoget = Math.ceil(window.totalweets/window.TWEETSPERPAGE);
window.holdtweetslist = response.list;
window.holdtweetslist.prototype.Merge = (function (ob) {var o = this;var i = 0;for (var z in ob) {if (ob.hasOwnProperty(z)) {o[z] = ob[z];}}return o;});
// alert(data);
;; gotTweets(data);
var loopcounter = 1;
do
{
currentpage = currentpage + 1;
pausecomp(1500);
var BASE = 'http://otter.topsy.com/search.json?type=tweet&perpage=' + TWEETSPERPAGE + '&window=a&nohidden=0&q=#' + name + '&page=' + currentpage;
alert(BASE);
$.ajax({dataType: "json", url: BASE, success: gotTweets(data)});
}
while (currentpage<pagestoget);
}
});
};
function gotTweets(data)
{
window.responcesreceived = window.responcesreceived + 1;
var response = data.response;
alert(response.total);
window.holdtweetslist.Merge(response.list);
window.tweetsfound = window.tweetsfound + response.total;
if (window.responcesreceived == window.pagestoget) {
// sendforprocessingsendtweetlist();
alert(window.tweetsfound);
}
}
You are calling Merge as an static method, but declared it as an "instance" method (for the prototype reserved word).
Remove prototype from Merge declaration, so you'll have:
window.holdtweetslist.Merge = (function(ob)...
This will fix the javascript error.
This is Vipul from Topsy. Would you share the literal JSON you are receiving? I want to ensure you are not receiving a broken response.
THanks to Edgar and Vipul for there help. Unfortunately they were able to answer my questions. I have managed to work out that the issue was a combination of jquery not parsing the json properly and needing to use jsonp with topsy.
Here is a little test I created that works.
Create a doc with this object on it ....
RUN TEST
You will need JQUERY
<script src="http://ajax.googleapis.com/ajax/libs/jquery/1.5.1/jquery.min.js"></script>
And put the following in a script too. The is cycle through the required number of tweets from Topsy.
Thanks again everyone.
$("#gettweets").live('click', function(event){
event.preventDefault();
getTweets('stephenbaugh');
});
var MAX_TWEETS = 500;
var TWEETSPERPAGE = 50;
var BASE = 'http://otter.topsy.com/search.json';
var currentpage;
var responcesreceived;
var totalweets;
var pagestoget;
var totalweets;
var TWEETSPERPAGE;
var holdtweetslist = [];
var requestssent;
var responcesreceived;
var tweetsfound;
var nametoget;
function getTweets(name) {
nametoget=name;
currentpage = 1;
responcesreceived = 0;
pagestoget = 0;
var BASE = 'http://otter.topsy.com/search.js?type=tweet&perpage=' + TWEETSPERPAGE + '&window=a&nohidden=0&q=#' + nametoget + '&page=1';
$('#gettweets').html(BASE);
$.ajax({url: BASE,
dataType: 'jsonp',
success : function(data) {
getalltweets(data);
}
});
};
function getalltweets(data) {
totalweets = data.response.total;
$('#gettweets').append('<p>'+"total tweets " + totalweets+'</p>');
$('#gettweets').append('<p>'+"max tweets " + MAX_TWEETS+'</p>');
if (MAX_TWEETS < totalweets) {
totalweets = 500
}
$('#gettweets').append('<p>'+"new total tweets " + totalweets+'</p>');
gotTweets(data);
pagestoget = Math.ceil(totalweets/TWEETSPERPAGE);
var getpagesint = self.setInterval(function() {
currentpage = ++currentpage;
var BASE = 'http://otter.topsy.com/search.js?type=tweet&perpage=' + TWEETSPERPAGE + '&window=a&nohidden=0&q=#' + nametoget + '&page=' + currentpage;
$.ajax({url: BASE,
dataType: 'jsonp',
success : function(data) {
gotTweets(data);
}
});
if (currentpage == pagestoget) {
$('#gettweets').append('<p>'+"finished sending " + currentpage+ ' of ' + pagestoget + '</p>');
clearInterval(getpagesint);
};
}, 2000);
};
function gotTweets(data)
{
responcesreceived = responcesreceived + 1;
holdlist = data.response.list;
for (x in holdlist)
{
holdtweetslist.push(holdlist[x]);
}
// var family = parents.concat(children);
$('#gettweets').append('<p>receipt # ' + responcesreceived+' - is page : ' +data.response.page+ ' array length = ' + holdtweetslist.length +'</p>');
// holdtweetslist.Merge(response.list);
tweetsfound = tweetsfound + data.response.total;
if (responcesreceived == pagestoget) {
// sendforprocessingsendtweetlist();
$('#gettweets').append('<p>'+"finished receiving " + responcesreceived + ' of ' + pagestoget + '</p>');
}
}

Categories

Resources