login to a webpage using phantomjs and Jquery - javascript

I am new to phantomjs, Java script and WebScraping in General. What I want to do is basic http authentication and then visit another URL to get some information. Here is what I have till now. Please tell me what I am doing wrong.
var page = require('webpage').create();
var system = require('system');
page.onConsoleMessage = function(msg) {
console.log(msg);
};
page.onAlert = function(msg) {
console.log('alert!!>' + msg);
};
page.settings.userName = "foo";
page.settings.password = "bar";
page.open("http://localhost/login", function(status) {
console.log(status);
var retval = page.evaluate(function() {
return "test";
});
console.log(retval);
page.open("http://localhost/ticket/" + system.args[1], function(status) {
if ( status === "success" ) {
page.injectJs("jquery.min.js");
var k = page.evaluate(function () {
var a = $("div.description > h3 + p");
if (a.length == 2) {
console.log(a.slice(-1).text())
}
else {
console.log(a.slice(-2).text())
}
//return document.getElementById('addfiles');
});
}
});
phantom.exit();
});
I am passing an argument to this file: a ticket number which gets appended to the 2nd URL.

I would recommend CasperJS highly for this.
CasperJS is an open source navigation scripting & testing utility written in Javascript and based on PhantomJS — the scriptable headless WebKit engine. It eases the process of defining a full navigation scenario and provides useful high-level functions, methods & syntactic sugar for doing common tasks such as:
defining & ordering browsing navigation steps
filling & submitting forms
clicking & following links
capturing screenshots of a page (or part of it)
testing remote DOM
logging events
downloading resources, including binary ones
writing functional test suites, saving results as JUnit XML
scraping Web contents
(from the CasperJS website)
I recently spent a day trying to get PhantomJS by itself to do things like fill out a log-in form and navigate to the next page.
CasperJS has a nice API purpose built for forms as well:
http://docs.casperjs.org/en/latest/modules/casper.html#fill
var casper = require('casper').create();
casper.start('http://some.tld/contact.form', function() {
this.fill('form#contact-form', {
'subject': 'I am watching you',
'content': 'So be careful.',
'civility': 'Mr',
'name': 'Chuck Norris',
'email': 'chuck#norris.com',
'cc': true,
'attachment': '/Users/chuck/roundhousekick.doc'
}, true);
});
casper.then(function() {
this.evaluateOrDie(function() {
return /message sent/.test(document.body.innerText);
}, 'sending message failed');
});
casper.run(function() {
this.echo('message sent').exit();
});

Related

Send multiple messages using google chrome console

I want to test message functionality in my browser game. Instead of sending messages manually to 50 test accounts I thought maybe I can do it automatically in Chrome console.
For first I would make a for loop.
for (i=0; i > 50; i++)
{
//to do
}
I would use my games link to fill the sending form
game.php?page=messages&mode=write&id=1&subject=message_test
ID is for user ids and Subject fill the subject area.
In my message send template I use this JavaScript to send a message
function check(){
if($('#text').val().length == 0) {
alert('{$LNG.mg_empty_text}');
return false;
} else {
$('submit').attr('disabled','disabled');
$.post('game.php?page=messages&mode=send&id={$id}&ajax=1', $('#message').serialize(), function(data) {
alert(data);
parent.$.fancybox.close();
return true;
});
}
}
Maybe someone can help me how to execute a function using Chrome console and send out messages? Maybe put me to a proper tutorial.
if you want to "export" your functions to the console into the google chrome or another dev tools, you can export it to a window object, so
// create a namespace to have all functions under your custom namespace
window.MyCustomNS = {};
window.MyCustomNS.yourFunction = function(id, dataObject) {
if($('#text').val().length == 0) {
alert('{$LNG.mg_empty_text}');
return false;
} else {
$('submit').attr('disabled','disabled');
$.post('game.php?page=messages&mode=send&id=' + + '&ajax=1', $.param(dataObject), function(data) {
alert(data);
parent.$.fancybox.close();
return true;
});
}
};
then into the console you try this,
window.MyCustomNS.yourFunction('YOUR_ID', { name: 'some parameter', another: 'another parameter' });

Automating Facebook group post deletion using CasperJS

I'm working on writing a script to delete posts from a Facebook Group, since Facebook's Graph API won't allow a developer to do so unless the posts were made from the developer's account.
So far, I have been able to log into Facebook, then navigate to the desired group page. From there I can get the XPath for each post visible on the page (using the selector a[data-testid='post_chevron_button']). My script fails while trying to call this.click() on each XPath selector.
My current script is as follows:
phantom.casperTest = true;
var x = require('casper').selectXPath;
var casper = require('casper').create({
verbose: true,
pageSettings: {
loadImages: false,
loadPlugins: false,
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.4'
}
});
// print out all the messages in the headless browser context
casper.on('remote.message', function(msg) {
this.echo('remote message caught: ' + msg);
});
// print out all the messages in the headless browser context
casper.on("page.error", function(msg, trace) {
this.echo("Page Error: " + msg, "ERROR");
});
var url = 'http://www.facebook.com/';
casper.start(url, function() {
console.log("page loaded");
this.test.assertExists('form#login_form', 'form is found');
this.fill('form#login_form', {
email: '{email}',
pass: '{password}'
}, true);
this.click('#u_0_q');
this.wait(1000, function() {
this.echo("Capturing image of page after login.");
this.capture('loggedin.png');
});
});
casper.thenOpen('https://www.facebook.com/groups/{group-id}/', function() {
this.echo(this.getTitle());
this.wait(1000, function() {
this.capture('group.png');
});
var elements = casper.getElementsInfo("a[data-testid='post_chevron_button']");
var index = 1;
elements.forEach(function(element){
var xpath = '//*[#id="' + element.attributes["id"] + '"]';
console.log(xpath);
this.click(x(xpath));
this.wait(100, function() {
this.capture('chevronlink' + index + '.png');
});
index++;
});
});
casper.run();
When the script gets to this.click(x(xpath)); I get the error message TypeError: undefined is not a constructor (evaluating 'this.click(x(xpath))'). If I simply replace the last bit of code that creates an array and iterates through it with this.click("a[data-testid='post_chevron_button']");, my script has no problem.
Does anyone know what CasperJS doesn't like about calling click() with the XPath selector? XPath appears to be a valid selector going off of CasperJS's docs.
UPDATE
I've updated the title of the question to more accurately describe the desired result.
Per dasmelch's advice, I've reworked the script a bit and incorporated this bit into the script instead (after the casper.thenOpen portion):
casper.then(function() {
var elements = casper.getElementsAttribute("a[data-
testid='post_chevron_button']", 'id');
while (elements.length > 0) {
// get always the last element with target id
element = elements.pop();
(function(element) {
var xpath = '//*[#id="' + element + '"]';
console.log(xpath);
// do it step by step
casper.then(function() {
this.click(x(xpath));
});
casper.then(function() {
this.capture('chevronlink' + element + '.png');
});
// go back to the page with the links (if necessary)
casper.then(function() {
casper.back();
});
})(element);
};
});
I now get this error: Cannot dispatch mousedown event on nonexistent selector: xpath selector: //*[#id="u_0_47"].
Last night, I decided to go about it a little differently. I got closer to the desired end result, but now CasperJS and/or PhantomJS is having trouble finding the elements that are present in the dropdown after clicking the post_chevron_button. Here is what I ended up with (everything prior to casper.thenOpen remains the same in the script shown originally):
casper.thenOpen('https://www.facebook.com/groups/{group-id}/', function() {
this.echo(this.getTitle());
this.wait(1000, function() {
this.capture('group.png');
});
var elements = casper.getElementsInfo("a[data-
testid='post_chevron_button']");
while (elements.length > 0) {
this.click("a[data-testid='post_chevron_button']");
this.wait(1000, function() {
this.capture('chevron_click.png');
console.log("chevron_click.png saved");
});
var chevronLinks = casper.getElementsInfo("a[ajaxify]")
console.log("Found " + chevronLinks.length + " elements with ajaxify attribute.");
var chevronLinksIndex = 1;
chevronLinks.forEach(function(element){
var ajaxifyValue = element.attributes["ajaxify"];
console.log(ajaxifyValue);
if (ajaxifyValue.indexOf("delete.php?group_id={group-id}") !== -1) {
this.click("a[ajaxify='"+ajaxifyValue+"']");
this.wait(100, function(){
this.capture('deletePost' + chevronLinksIndex);
});
chevronLinksIndex++;
}
});
if (chevronLinksIndex === 1) {
break;
}
elements = casper.getElementsInfo("a[data-testid='post_chevron_button']");
}
});
I know there should be an element that contains an ajaxify attribute with the value I'm searching for (because stepping through it in a browser myself shows the element after the click on a[data-testid='post_chevron_button']), but Casper cannot find it. Not only that, my chevron_click.png image file should be updating on each run of this script, but it's not.
Some of the code execution is not happening in order. For instance, the logging of ajaxify attribute values is happening in the console prior to seeing chevron_click.png saved. This may be expected, but unfortunately I don't have a lot of JS experience. This execution order problem may explain why my search for the necessary element is not returning what I expect.
Here is an example of the element needing clicked for deletion of a post:
<a class="_54nc" href="#" rel="async-post"
ajaxify="/ajax/groups/mall/delete.php?group_id={group-id}&message_id=806608486110204&story_dom_id=mall_post_806608486110204%3A6%3A0&entstory_context=%7B%22last_view_time%22%3A1495072771%2C%22fbfeed_context%22%3Atrue%2C%22location_type%22%3A2%2C%22outer_object_element_id%22%3A%22mall_post_806608486110204%3A6%3A0%22%2C%22object_element_id%22%3A%22mall_post_806608486110204%3A6%3A0%22%2C%22is_ad_preview%22%3Afalse%2C%22is_editable%22%3Afalse%2C%22mall_how_many_post_comments%22%3A2%2C%22bump_reason%22%3A0%2C%22story_width%22%3A502%2C%22shimparams%22%3A%7B%22page_type%22%3A16%2C%22actor_id%22%3A664025626%2C%22story_id%22%3A806608486110204%2C%22ad_id%22%3A0%2C%22_ft_%22%3A%22%22%2C%22location%22%3A%22group%22%7D%2C%22story_id%22%3A%22u_0_21%22%2C%22caret_id%22%3A%22u_0_22%22%7D&surface=group_post_chevron"
role="menuitem"><span><span class="_54nh"><div class="_41t5"><i
class="_41t7 img sp_gJvT8CoKHU- sx_0f12ae"></i><i class="_41t8 img
sp_s36yWP_7MD_ sx_7e9f7d"></i>Delete Post</div></span></span></a>
I was able to accomplish what I was trying to do with the Selenium 2 API for .NET.
The solution code is below:
class Program
{
static void Main(string[] args)
{
var options = new ChromeOptions();
options.AddUserProfilePreference("profile.default_content_setting_values.notifications", 2);
using (IWebDriver driver = new ChromeDriver(options))
{
// Maximize window
driver.Manage().Window.Maximize();
// Log into Facebook
driver.Navigate().GoToUrl("http://www.facebook.com/");
driver.FindElement(By.Id("email")).SendKeys("username");
driver.FindElement(By.Id("pass")).SendKeys("password");
driver.FindElement(By.Id("pass")).SendKeys(Keys.Enter);
driver.Navigate().GoToUrl("https://www.facebook.com/groups/{group-id}/");
var chevronPostLinks = driver.FindElements(By.XPath("//a[#data-testid='post_chevron_button']"));
chevronPostLinks.FirstOrDefault().Click();
Thread.Sleep(1000);
var deletePostElements = driver.FindElements(By.XPath("//a[contains(#ajaxify,'delete.php?group_id={group-id}')]"));
while (deletePostElements.Count > 0 && chevronPostLinks.Count > 0)
{
Thread.Sleep(1000);
deletePostElements.Where(x => x.Displayed == true).FirstOrDefault().Click();
Thread.Sleep(1000);
driver.FindElement(By.ClassName("layerConfirm")).Click();
Thread.Sleep(2000);
chevronPostLinks = driver.FindElements(By.XPath("//a[#data-testid='post_chevron_button']"));
if (chevronPostLinks.Count > 0)
{
chevronPostLinks.FirstOrDefault().Click();
}
else
{
driver.Navigate().GoToUrl("https://www.facebook.com/groups/{group-id}/");
chevronPostLinks = driver.FindElements(By.XPath("//a[#data-testid='post_chevron_button']"));
chevronPostLinks.FirstOrDefault().Click();
}
Thread.Sleep(1000);
deletePostElements = driver.FindElements(By.XPath("//a[contains(#ajaxify,'delete.php?group_id={group-id}')]"));
}
}
}
}
There are some improvements I'd like to make, like using Selenium to wait for elements to be visible instead of using Thread.Sleep(), but it's working just fine for my purpose.
You do the xpath stuff correct, but it seems the method forEach is not working for this.
You can grab directly the id of all these elements with casper.getElementsAttribute an iterate easy with a while loop throw that them more easily like that:
...
casper.thenOpen('https://www.facebook.com/groups/{group-id}/', function() {
this.echo(this.getTitle());
this.wait(1000, function() {
this.capture('group.png');
});
});
// do a while loop with where you can use every single element and jump back
casper.then(function() {
var elements = casper.getElementsAttribute("a[data-testid='post_chevron_button']", 'id');
while (elements.length > 0) {
// get always the last element with target id
element = elements.pop();
(function(element) {
var xpath = '//*[#id="' + element + '"]';
console.log(xpath);
// do it step by step
casper.then(function() {
this.click(x(xpath));
});
casper.then(function() {
this.capture('chevronlink' + element + '.png');
});
// go back to the page with the links (if necessary)
casper.then(function() {
casper.back();
});
})(element);
};
});
...
Without looking at FB, i guess you have to go back (casper.back) to the site where the links (elements) are.

How do you properly split up casperjs tests with a shared browser instance

I'm trying to test different parts of a "mostly" single page application. I'd like to split the tests up, but I really only want to load the page once and then have the tests go through and click the links etc.
Here's my code:
PRE.js
var port = require('system').env.PORT
var tester;
casper.options.viewportSize = {width: 1024, height: 768};
casper.test.begin('Test login', function suite(test) {
var done = false;
casper.on("page.error", function(msg, trace) {
this.echo("Error: " + msg, "ERROR");
this.echo("file: " + trace[0].file, "WARNING");
this.echo("line: " + trace[0].line, "WARNING");
this.echo("function: " + trace[0]["function"], "WARNING");
});
casper.on('remote.message', function(message) {
this.echo('remote message caught: ' + message);
if (message == "done") {
done = true;
}
});
casper.start('http://localhost:' + port, function() {
// Verify that the main menu links are present.
test.assertExists('input[name=username]');
// 10 articles should be listed.
test.assertElementCount('input', 3);
casper.fill("form", {
"username": "username",
"password": "my password goes right here you cant have it"
}, true);
casper.then(function() {
casper.waitFor(function(){
return done;
}, function(){
tester = casper.evaluate(function(){
return tester;
});
test.assert("undefined" != typeof tester);
test.assert(Object.keys(tester).length > 0);
});
});
});
casper.run(function() {
test.done();
});
});
and then I have a second file (and there will be lots more like this):
TEST.js
casper.test.assert(true);
casper.capture('.screenshot.png');
casper.test.done();
I'm hoping to get a screenshot of the browser session from pre.js.
I run it from a specialized program that starts up my program, but in essence it runs:
casperjs test casper_tests --pre=pre.js
casper_tests holds both files above
My Question:
What's the right way to do this? No screenshot is being taken, and perhaps more important (though I haven't tried it yet) I want to be able to click things inside and verify that other pieces are working. The screenshot just verifies that i'm in the right neighborhood.
This will not be easily possible and potentially dangerous. Every action that you do, would need to be reversed to not break the other tests. If you later decide that writing tests in a modular manner is a good thing, you will have a headache writing your tests.
PRE.js will be your start script which you modify to execute your tests in between. In the following fully working example you see how you can schedule multiple test cases for one execution of casper. This is bad, because the canvas test case depends on the proper back execution of the link test case.
casper.start('http://example.com');
casper.then(function() {
this.test.begin("link", function(test){
var url = casper.getCurrentUrl();
test.assertExists("a");
casper.click("a");
casper.then(function(){
test.assert(this.getCurrentUrl() !== url);
this.back(); // this is bad
test.done();
});
});
this.test.begin("canvas", function(test){
test.assertNotExists("canvas");
test.done();
});
});
casper.run();
Of course you can open the root again for the new test case, but then you have the same problem as with your initial code.
var url = 'http://example.com';
casper.start();
casper.thenOpen(url, function() {
this.test.begin("link", function(test){
var url = casper.getCurrentUrl();
test.assertExists("a");
casper.click("a");
casper.then(function(){
test.assert(this.getCurrentUrl() !== url);
test.done();
});
});
});
casper.thenOpen(url, function() {
this.test.begin("canvas", function(test){
test.assertNotExists("canvas");
test.done();
});
});
casper.run();
Now the test cases don't depend on each other, but you also load the page multiple times.
If you need some initial actions for every test case then the PRE.js is not the right place for that.
Create include.js and put the following code there:
function login(suite, username, password){
username = username || "defaultUsername";
password = password || "defaultPassword";
casper.test.begin('Test login', function suite(test) {
var done = false;
// event handlers
casper.start('http://localhost:' + port, function() {
// login if the session expired or it is the first run
if (!loggedIn) {
// login
}
// wait
});
casper.then(function(){
suite.call(casper, test);
});
casper.run(function() {
test.done();
});
});
}
Then you can run it as casperjs test casper_tests --includes=include.js with test files like
login(function(test){
this.click("#something");
this.waitForSelector(".somethingChanged");
this.then(function(){
test.assertExists(".somethingElseAlsoHappened");
});
});
Of course you can have different login functions (with different names) or more lightweight ones.
Building on the previous snippets, you can make a start script and load the test files yourself. Then you have all the flexibility you need to do this.
include.js:
function login(testScript, username, password, next){
// event handlers
casper.start('http://localhost:' + port, function() {
// login if the session expired or it is the first run
// waiting
});
testScript.forEach(function(case){
casper.thenOpen(case.name, function(){
this.test.begin(function suite(test){
case.func.call(casper, test);
casper.then(function(){
test.done();
});
});
});
});
casper.run(next);
}
start.js:
// pass the test file folder to the script and read it with sys.args
// fs.list(path) all files in that path and iterate over them
var filesContents = files.map(function(filename){
return require(filename).testcases;
});
var end = null;
// stack the test cases into the `run` callback of the previous execution
filesContents.forEach(function(case){
var newEnd = end;
var newFunc = function(){ login(case, u, p, newEnd) };
end = newFunc;
});
end(); // run the stack in reverse
each test file would look like this:
exports.testcases = [
{
name: "sometest",
func: function(test){
test.assert(true)
this.echo(this.getCurrenturl());
}
},
{
name: "sometest2",
func: function(test){
test.assert(true)
this.echo(this.getCurrenturl());
}
},
];
This is just a suggestion.

How to Append to a file in a Firefox add-on?

var tabs = require("sdk/tabs");
var iofile = require("sdk/io/file");
var widgets = require("sdk/widget");
var selection = require("sdk/selection");
function console_log(text) {
console.log(selection.text);
}
function print(text) {
console.log(text);
}
function dir_object(object_to_parse) {
var name = '';
for (name in object_to_parse) {
print(name);
}
}
function write_text(filename, text) {
var fh = iofile.open(filename, 'w');
var content = fh.read();
dir_object(fh);
selected_text = text + "\n";
fh.write(selected_text);
fh.flush();
fh.close()
}
function select_text_handler() {
write_text('/tmp/foo', selection.text);
}
var widget = widgets.Widget({
id: "scribus-link",
label: "Scribus website",
contentURL: "http://www.mozilla.org/favicon.ico",
onClick: function() {
}
});
selection.on('select', function () { select_text_handler(); });
'open' the file in 'w' and that truncates my existing file! How do i open in 'append' mode and then 'seek'?? https://addons.mozilla.org/en-US/developers/docs/sdk/latest/modules/sdk/io/file.htm
The file module of the SDK is pretty limited. When opening a file for writing it will always be truncated (code). Also, it is uses entirely synchronous I/O on the main thread, which isn't really a good thing to do, as it will block the entire UI during the I/O.
You should probably use another mechanism via the chrome module. See OS.File and/or the MDN File I/O snippets.

Using the 'webpage' Phantom module in node.js

I am trying to wrap a PhantomJS script in a node.js process. The phantom script grabs a url from the arguments provided on the command line and outputs a pdf (much similar to the rasterize.js example included with the pahntom install).
The phantom script I have works fine, it's just my employer wants a node script if possible. No problem, I can use the node-phantom node module to wrap it.
But now I've hit a stumbling block, my phantom script has:
var page = require('webpage').create();
So, node.js is trying to find a module called 'webpage', the 'webpage' module is built into the phantom install so node can't find it. As far as I can tell, there is no npm module called 'webpage'.
'webpage' is used like this:
page.open(address, function (status) {
if (status !== 'success') {
// --- Error opening the webpage ---
console.log('Unable to load the address!');
} else {
// --- Keep Looping Until Render Completes ---
window.setTimeout(function () {
page.render(output);
phantom.exit();
}, 200);
}
});
where address is the url specified on the command line and output is another argument, the name and type of the file.
Can anyone help me out? This is quite an abstract one so I'm not expecting much if I'm honest, worth a try though.
Thanks.
EDIT - Approx 2hrs later
I now have this which throws out a PDF:
var phanty = require('node-phantom');
var system = require('system');
phanty.create(function(err,phantom) {
//var page = require('webpage').create();
var address;
var output;
var size;
if (system.args.length < 4 || system.args.length > 6) {
// --- Bad Input ---
console.log('Wrong usage, you need to specify the BLAH BLAH BLAH');
phantom.exit(1);
} else {
phantom.createPage(function(err,page){
// --- Set Variables, Web Address, Output ---
address = system.args[2];
output = system.args[3];
page.viewportSize = { width: 600, height: 600 };
// --- Set Variables, Web Address ---
if (system.args.length > 4 && system.args[3].substr(-4) === ".pdf") {
// --- PDF Specific ---
size = system.args[4].split('*');
page.paperSize = size.length === 2 ? { width: size[0], height: size[1], margin: '0px' }
: { format: system.args[4], orientation: 'portrait', margin: '1cm' };
}
// --- Zoom Factor (Should Never Be Set) ---
if (system.args.length > 5) {
page.zoomFactor = system.args[5];
} else {
page.zoomFactor = 1;
}
//----------------------------------------------------
page.open(address ,function(err,status){
if (status !== 'success') {
// --- Error opening the webpage ---
console.log('Unable to load the address!');
} else {
// --- Keep Looping Until Render Completes ---
process.nextTick(function () {
page.render(output);
phantom.exit();
}, 200);
}
});
});
}
});
But! It's not the right size! The page object created using the phantom 'webpage' create() function looks like this before it's passed the URL:
Whereas mine in my node script, looks like this:
Is it possible to hard code the properties to achieve A4 formatting? What properties am I missing?
I'm so close!
It should be something like:
var phantom=require('../node-phantom');
phantom.create(function(error,ph){
ph.createPage(function(err,page){
page.open(url ,function(err,status){
// do something
});
});
});
Your confusion here is because you want to reuse the same concepts and metaphors from your PhantomJS script. It does not work that way. I suggest that you spend some time studying the included tests of node-phantom, see https://github.com/alexscheelmeyer/node-phantom/tree/master/test.
Using https://github.com/sgentle/phantomjs-node I have made an A4 page in nodejs using phantom with the following code:
phantom.create(function(ph){
ph.createPage(function(page) {
page.set("paperSize", { format: "A4", orientation: 'portrait', margin: '1cm' });
page.open("http://www.google.com", function(status) {
page.render("google.pdf", function(){
console.log("page rendered");
ph.exit();
})
})
})
});
Side Note:
the page.set() function takes any variable that you would set in the rasterize.js example. See how paperSize is set above and compare it to the relevant lines in rasterize.js

Categories

Resources