I met some problem when I use CasperJS to scrape a website. The website is dynamically loaded like Twitter, so I want to do infinite scroll,
and thanks to #Artjom B. I found you code to do this.
var tryAndScroll = function (casper) {
try {
casper.echo('SCROLL!!');
casper.scrollToBottom();
if (casper.exists('div.loading')) {
var curItems = casper.evaluate(getCurrentInfosNum);
casper.echo(curItems);
casper.waitFor(function check() {
return curItems != casper.evaluate(getCurrentInfosNum);
}, function then() {
casper.wait(800);
tryAndScroll(casper);
}, function onTimeout() {
casper.emit('scroll.timeout',curItems);
}, 15000);
} else {
casper.echo("No more items");
return true;
}
} catch (err) {
casper.echo(err);
}
} //casper.tryAndScroll
And now, I want to continue to scroll many times when the timeout function invoked so I create my own event listener,‘scroll.timeout’.
var SRCOLL_NUM = 0;
var PreOfLoaded = 0;
casper.on('scroll.timeout', function (NumOfLoaded) {
if (SRCOLL_NUM <= 4) {
if (PreOfLoaded == NumOfLoaded)
SRCOLL_NUM++;
this.echo("Scroll Timeout,reScroll");
PreOfLoaded = NumOfLoaded;
tryAndScroll(casper);
} else {
this.echo("Scroll Timeout,reScroll times maximum");
SRCOLL_NUM = 0;
PreOfLoaded = 0;
}
});
However, when scroll timeout occurred, it printed Scroll Timeout,reScroll on the console. Then it skips tryAndScroll() and go to the next step in the main function. I want to continue to next step after retry scroll many times. What should I do?
I found CasperJS author illustrate :Automatic retry when open fails
var casper = require('casper').create();
casper.tryOpen = function(url, then) {
return this.then(function() {
this.open(url);
this.waitFor(function testStatus() {
return this.getCurrentHTTPStatus === 200;
}, then, function onFail() {
console.log('failed, retrying');
this.tryOpen(url);
}, 2000);
});
};
casper.start().tryOpen('http://failing.url.com/foo.bar', function() {
this.echo('wow, it worked, wtf');
}).run();
unfortunately, it doesn't work for me.
Try this
return this.currentHTTPStatus === 200;
I tested with the newest version of casperjs 1.1.1, it's working fine
Related
I'm using the iteminvokedHandler and was wonder if there is a better way to interact with the listView.
Currently using this:
WinJS.UI.processAll(root).then(function () {
var listview = document.querySelector('#myNotePad').winControl;
listview.addEventListener("iteminvoked", itemInvokedHandler,false);
function itemInvokedHandler(e) {
e.detail.itemPromise.done(function (invokedItem) {
myEdit();
});
};
});
The problem is that everytime I click on the listview myEdit() is run and propagates within the listview. I was wondering how to do it once and stop invoking listview until I am done with myEdit? Is there a simpler way to handle such a situation as this?
Simple yet hard to see when you have a mind block and forget some of the basics (yes yes I'm still learning):
var testtrue = true;
WinJS.UI.processAll(root).then(function () {
var listview = document.querySelector('#myNotePad').winControl;
listview.addEventListener("iteminvoked", itemInvokedHandler,false);
function itemInvokedHandler(e) {
e.detail.itemPromise.done(function (invokedItem) {
if (testtrue === true){
myEdit();
}
});
};
});
In myEdit:
function myEdit() {
var theelem = document.querySelector(".win-selected #myNotes");
var gestureObject = new MSGesture();
gestureObject.target = theelem;
theelem.gestureObject = gestureObject;
theelem.addEventListener("pointerdown", pointerDown, false);
theelem.addEventListener("MSGestureHold", gestureHold, false);
function pointerDown(e) {
e.preventDefault();
e.target.gestureObject.addPointer(e.pointerId);
}
function gestureHold(e) {
if (e.detail === e.MSGESTURE_FLAG_BEGIN && test === true) {
e.preventDefault();
editNotes();
} else {
}
console.log(e);
}
theelem.addEventListener("contextmenu", function (e) {
e.preventDefault();}, false); //Preventing system menu
};
function editNotes() {
//The Code I wish to execute
return test = false;
};
What I needed was a conditional statement so that it would run if true and not if false. That same test needed to be done in the gestureHold otherwise it would continue to fire myEdit on the invoked item because of the way the gesture is attached to the item the first time it is run.
I am currently working with PhantomJS and CasperJS to scrape for links in a website. The site uses javascript to dynamically load results. The below snippet however is not getting me all the results the page contains. What I need is to scroll down to the bottom of the page, see if the spinner shows up (meaning there’s more content still to come), wait until the new content had loaded and then keep scrolling until no more new content was shown. Then store the links with class name .title in an array. Link to the webpage for scraping.
var casper = require('casper').create();
var urls = [];
function tryAndScroll(casper) {
casper.waitFor(function() {
this.page.scrollPosition = { top: this.page.scrollPosition["top"] + 4000, left: 0 };
return true;
}, function() {
var info = this.getElementInfo('.badge-post-grid-load-more');
if (info["visible"] == true) {
this.waitWhileVisible('.badge-post-grid-load-more', function () {
this.emit('results.loaded');
}, function () {
this.echo('next results not loaded');
}, 5000);
}
}, function() {
this.echo("Scrolling failed. Sorry.").exit();
}, 500);
}
casper.on('results.loaded', function () {
tryAndScroll(this);
});
casper.start('http://example.com/', function() {
this.waitUntilVisible('.title', function() {
tryAndScroll(this);
});
});
casper.then(function() {
casper.each(this.getElementsInfo('.title'), function(casper, element, j) {
var url = element["attributes"]["href"];
urls.push(url);
});
});
casper.run(function() {
this.echo(urls.length + ' links found:');
this.echo(urls.join('\n')).exit();
});
I've looked at the page. Your misconception is probably that you think the .badge-post-grid-load-more element vanishes as soon as the next elements are loaded. This is not the case. It doesn't change at all. You have to find another way to test whether new elements were put into the DOM.
You could for example retrieve the current number of elements and use waitFor to detect when the number changes.
function getNumberOfItems(casper) {
return casper.getElementsInfo(".listview .badge-grid-item").length;
}
function tryAndScroll(casper) {
casper.page.scrollPosition = { top: casper.page.scrollPosition["top"] + 4000, left: 0 };
var info = casper.getElementInfo('.badge-post-grid-load-more');
if (info.visible) {
var curItems = getNumberOfItems(casper);
casper.waitFor(function check(){
return curItems != getNumberOfItems(casper);
}, function then(){
tryAndScroll(this);
}, function onTimeout(){
this.echo("Timout reached");
}, 20000);
} else {
casper.echo("no more items");
}
}
I've also streamlined tryAndScroll a little. There were completely unnecessary functions: the first casper.waitFor wasn't waiting at all and because of that the onTimeout callback could never be invoked.
I've run into an issue with jQuery code that I'd like some help on. While the next and previous buttons work correctly, there appears to be an issue correctly stopping a slideshow. This is probably something we're completely overlooking, but I can't see what.
There is no JavaScript errors showing in the console. The interrupt method was being reached - at one point I had console.log calls in there to verify (cleaned up for presentation reasons).
http://jsfiddle.net/eLn83ep0/
Your help is much appreciated.
The below code is the functionality for prev/stop/start/next:
if ($(settings.next).size()) {
$(settings.next).bind('click.scrollface', function (e) {
methods.interrupt.call($this);
methods.next.call($this);
});
}
if ($(settings.pause).size()) {
$(settings.pause).bind('click.scrollface', function (e) {
methods.interrupt.call($this);
});
}
if ($(settings.play).size()) {
$(settings.play).bind('click.scrollface', function (e) {
methods.interrupt.call($this);
methods.start.call($this);
});
}
/*
* Setup up prev bindings
*/
if ($(settings.prev).size()) {
$(settings.prev).bind('click.scrollface', function (e) {
methods.interrupt.call($this);
methods.prev.call($this);
});
}
Here is the method for interrupt:
interrupt: function (time) {
return $(this).each(function () {
var data = $(this).data('scrollface');
console.log(data);
console.log('...');
if (!data) {
return false;
}
var $this = $(this),
period = 0;
/*
* Stop the timer, and wait a period of time before restarting it.
* Period defaults to the timer interval
*/
if (data.timer) {
if (typeof time !== "number") {
period = data.interval;
} else {
period = time;
}
methods.stop.call(this);
setTimeout(function resume_timer () {
clearInterval(data.timer);
data.timer = null;
methods.start.call($this);
}, period);
}
});
},
if ($(settings.pause).size()) {
$(settings.pause).bind('click.scrollface', function (e)
methods.stop.call($this);
});
}
if ($(settings.play).size()) {
$(settings.play).bind('click.scrollface', function (e) {
methods.start.call($this);
});
}
and delete the interrupt call in prev and next
//methods.interrupt.call(this);
because the interrupt method stopping the interval method and after timeout starting again, so most time you pause in this timeout but after this timeout the interrupt method starting the intervall again and dont care if you stoped it manually
see your updated fiddle working here http://jsfiddle.net/7ko4rdda/
Edit:
if you call interrupt with 0 for the buttons prev next click handler then you have the expected behavier that the period for the intervall starts new after next/prev click
if ($(settings.prev).size()) {
$(settings.prev).bind('click.scrollface', function (e) {
methods.interrupt.call($this,0);
methods.prev.call($this);
});
}
if ($(settings.next).size()) {
$(settings.next).bind('click.scrollface', function (e) {
methods.interrupt.call($this,0);
methods.next.call($this);
});
}
http://jsfiddle.net/7ko4rdda/1/
I'm new to JavaScript and I'm having problems with this script.
it's part of a web game and the script is suppose to refresh the page until the player wins or loses.
for some reason it doesn't stop refreshing, I put an alert function to check if the functions works, and i get the alerts but it's still continue refreshing the page.
what am i doing wrong?
var t;
$(document).ready(function () {
intervals();
});
function intervals() {
t = self.setInterval('refreshData()', 10000);
}
function youWin() {
var f = $('#status:contains("YOU ARE THE WINNER!")');
if (f.length > 0) {
alert("YOU ARE THE WINNER!");
t = clearInterval(t);
}
}
function youlose() {
var f = $('#status:contains("You lost!")');
if (f.length > 0) {
alert("You lost!");
t = clearInterval(t);
}
}
function refreshData() {
$('#ajaxGame').load('RefreshCurrentPlayerServlet #ajaxGame');
youWin();
youlose();
}
You need to fix the reference to self and fix the .load() call.
.load() is asynchronous so it does not complete before you call youWin() and youLose() right after it. You need a completion function so you can check winning or losing after the .load() completes successfully.
refreshData() should be structured like this:
function refreshData() {
$('#ajaxGame').load('RefreshCurrentPlayerServlet #ajaxGame', function() {
youWin();
youlose();
});
}
You also should change this:
t= self.setInterval('refreshData()',10000);
to this:
t = window.setInterval(refreshData, 10000);
I don't see that self was even defined so that could have also been causing your problem and you should use the function reference directly rather than put in a string.
And, as a cleanup issue, you should change both occurences of this:
t = clearInterval(t);
to this:
clearInterval(t);
Here's a cleaned up version of the code that also eliminates global variables and unnecessary function definitions:
$(document).ready(function() {
var t = window.setInterval(function() {
$('#ajaxGame').load('RefreshCurrentPlayerServlet #ajaxGame', function() {
youWin();
youlose();
});
}, 10000);
function youWin() {
if ($('#status:contains("YOU ARE THE WINNER!")').length) {
alert("YOU ARE THE WINNER!");
clearInterval(t);
}
}
function youlose() {
if ($('#status:contains("You lost!")').length) {
alert("You lost!");
clearInterval(t);
}
}
});
I'm putting together some code to essentially replace the contents of a div when I mouseover a specific link. I then added the changer function to cycle through the content replacement automatically. I set flags for mouseover and mouseout and I can actually get the changer function to stop on mouseover but I can't quite figure out how to make it start up again on mouseout. Any advice is appreciated.
var pause=false;
$('.banner-nav a').mouseover(function () {
pause=true;
setFeature(this);
return false;
});
$('.banner-nav a').mouseout(function () {
pause=false;
});
changer(0, 5000);
function setFeature(f) {
var m = $(f).attr('rel');
$('.banner-nav a').not(f).removeClass('active');
$(f).addClass('active');
$('#featureContainer').html($(m).html());
}
function changer(index, interval) {
var buttons = $('.trigger'),
buttons_length = buttons.length;
var button = buttons.eq(index % buttons_length);
setFeature($(button));
setTimeout(function() {
if (!pause) {
changer(++index, interval);
}
}, interval)
}
The issue is that changer is responsible for its own delayed execution, but pausing it stops the scheduled execution. Another problem is that the next scheduled execution (if any) still happens after pausing.
Use setInterval instead of setTimeout. Instead of using a flag, clear the interval to pause and start it again to unpause.
(function() {
var index=0;
function changer() {
var buttons = $('.trigger'),
buttons_length = buttons.length;
var button = buttons.eq(index % buttons_length);
setFeature($(button));
++index;
}
var changerInterval,
period = 5000;
function startChanger() {
if (! changerInterval) {
changerInterval = setInterval(changer, interval);
}
}
function stopChanger() {
clearInterval(changerInterval);
changerInterval = 0;
}
$('.banner-nav a').mouseover(function () {
stopChanger();
setFeature(this);
return false;
});
$('.banner-nav a').mouseout(function () {
startChanger();
});
/* could implement other functions to e.g. change the period */
function setChangerPeriod() {
...
}
window.setChangerPeriod = setChangerPeriod;
...
})