Count HTTP only cookies with Mozilla add-on - javascript

I'm trying develop a Firefox add-on. I would like to count the cookies that are marked as HTTP only. When manually checking, I have seen that many websites have more than one HTTP only cookie. But, my result is always 0 or 1. Where is my fault?
Here is my code:
var {Cc, Ci, Cu} = require("chrome");
Cu.import("resource://gre/modules/Services.jsm");
var cookieManager = Cc["#mozilla.org/cookiemanager;1"].getService(Ci.nsICookieManager2);
var tabs = require("sdk/tabs");
tabs.on('load', function(tab) {
var URL = tab.url;
var url = require("sdk/url").URL(URL);
var host = url.host;
function getCookies(host){
var cookies = cookieManager.getCookiesFromHost(host);
var count = 0;
while (cookies.hasMoreElements()){
var cookie = cookies.getNext().QueryInterface(Ci.nsICookie2);
//var count = 0;
var httpCookie = cookie.isHttpOnly;
if(httpCookie){
return count=count+1 ;
}else{
return 0;
}
console.log("Cookie host: " + cookie.host + "; Cookie Name :" + cookie.name
+ " = Cookie value:" + cookie.value + "\n");
dump("\tCookie host: " + cookie.host + " Is Domain cookie: " +cookie.isDomain
+ "; Cookie Name :" + cookie.name +" = Cookie value:" + cookie.value
+ "; Is Session Cookie :" + cookie.isSession
+ "; Expiry time :" + cookie.expiry
+ "; It is an Http only cookie :" + cookie.isHttpOnly + "\n");
}
return count;
}
var getResult = getCookies(host);
console.log("Http Cookies: " + getResult);
});

Within your function getCookies(host) you have a while loop that is intended to loop through all cookies for the specified host. However, the inner portion of that loop is only executed once.
Within that loop you have an if statement:
var httpCookie = cookie.isHttpOnly;
if(httpCookie){
return count=count+1 ;
}else{
return 0;
}
This statement results in the function immediately returning either a 1 or a 0 depending on if the first cookie found has the property cookie.isHttpOnly as true or false. [Note: cookie is always 0 when this if statement is executed for the first and only time.] No other cookies are checked other than the first one because you immediately return the value. Execution of your function ends at either of the two return statements within this if statement. The lines within the function after the if will not be executed.
From what you describe you desire, your if statement would be better as:
if(cookie.isHttpOnly){
count++;
}
Note: Given that you only use cookie.isHttpOnly once, there is no need to assign it to a separate variable.

Related

Cookie is not being read by Edge or Safari

I'm having a problem where my cookies are not being read by Safari or MS Edge. It works fine in all the other browsers just not in these 2.
Has anyone come across a similar issue? If so how did you manage to sort it out? Thank you.
$('.primary-button.eligibility-button').mousedown(function() {
var selectedCard = $('input[name="cardInterestOption"]:checked').next().text();
var balanceTransferOption = $('input[name="balanceTransferOption"]:checked').next().text().replace(/[a-z]/g,'');
var annualIncome = parseFloat($( "input[name='annualIncome']").val());
var otherIncome = parseFloat($( "input[name='otherIncome']").val());
var totalIncome = annualIncome + otherIncome;
document.cookie = "creditCardSelectorEligibility=" + selectedCard + "; path=/;";
document.cookie = "creditCardBalanceTransfer=" + balanceTransferOption + "; path=/;";
document.cookie = "creditCardTotalIncome=" + totalIncome + "; path=/;";
});
(function() {
var page = "";
var percentage = "NA";
var path = location.pathname;
if(/results/.test(path)){
page = "Success";
var results = "";
var percent = $(".results-banner:eq(0)").text();
if(/\b[0-9]{2}/.test(percent)){
results = percent.match(/\b[0-9]{2}/)[0]
}
}
else if(/verify/.test(path)){
page = "Verify";
}
else if(/problem/.test(path)){
page = "Problem";
}
else if(/noteligible/.test(path)){
page = "Not Eligible";
}
else if(/unavailable/.test(path)){
page = "Unavailable";
}
var selectedCard = bt_cookie('yrd_creditCardSelectorEligibility')
var balanceTransferOption = bt_cookie('yrd_creditCardBalanceTransfer')
var totalIncome = bt_cookie('yrd_creditCardTotalIncome')
return page + "|" + selectedCard + "|" + balanceTransferOption + "|" + totalIncome + "|" + results;
})();
What we have found happen sometimes, is that the page loads too quick for the cookie to be created. So we have had to change the way the Cookies get created. For instance you can change the code to create the Cookie as the user interacts with the form. ie: If the select a button or fill in a field. Capture the cookie there, rather than at the end of the page where the customer has to click the submit. Hopefully this makes sense?

Sending multiple HTTP GET requests to api with a loop

I'm looking for a way to send many requests to an api using a different api url each time.
An example url for my project is:
http://api.bandsintown.com/artists/Hippo%20Campus/events.json?lapi_version=2.0&app_id=music_matcher
I'm using an HTTP request to pull the JSON info into my script and works perfectly...the first time. However, I want to be able to call it 50-100 ish times (max) in a loop with different artist names in the url (I'm using the BandsInTown API). For some reason, when I try to use a loop to call the http request multiple times, only one output appears and it is unpredictable which element in the order it will be (it's usually the output associated with the first or second element in the array). This is what my code looks like:
// HTTP GET call to BandsInTown API
function httpGetAsync(theUrl, callback) { //theURL or a path to file
var httpRequest = new XMLHttpRequest();
httpRequest.onreadystatechange = function() {
if (httpRequest.readyState == 4 && httpRequest.status == 200) {
var data = JSON.parse(httpRequest.responseText);
if (callback) {
callback(data);
}
}
else {
alert("error loading JSON doc");
}
};
httpRequest.open('GET', theUrl, true);
httpRequest.send(null);
}
//extracts data from api for each artist
function parseEvent(artist) {
var url = "http://api.bandsintown.com/artists/" + artist + "/events.json?lapi_version=2.0&app_id=music_matcher";
httpGetAsync(url, function(data) {
var numEvents = Object.keys(data).length;
//var events = [];
for (var j = 0; j < numEvents; j++) {
document.write(data[j].venue.name + "-> ");
document.write("LAT:" + data[j].venue.latitude + " " + "LNG:" + data[j].venue.longitude);
document.write("ARTIST: " + data[j].artists[0].name);
document.write("DATE: " + data[j].datetime);
document.write(" " + j + " ");
}
});
}
var artists = ["Drake", "Mac Demarco", "Hippo Campus", "STRFKR"];
for (var i = 0; i < artists.length; i++) {
parseEvent(artists[i]);
document.write(" ---NEXT ARTIST--- ");
}
So I can't tell exactly what's going on but things are acting weird with my current code. I don't have a whole lot of javascript and web development experience yet so any help is appreciated! I was preferably looking for a way to implement this with pure javascript. I have had trouble figureing out how to handle Node.js and/or JQuery in Eclipse Neon (the IDE I am using)
You have implemented closure pretty well so clearly this isn't a problem of success callback of one function overwriting response of all others.But now when you look at document.write() it all gets clear, this function first wipes your whole content clean then it writes whatever you told it to .That's why you hardly see anyone use it
`document.write('a');`
`document.write('b');`
`document.write('c');` // a and b are gone you would see only 'c'
So after loop gets over you would only see the output of the last call.Though it's mostly random as to which call would finish last it mostly biased towards some particular value due to the the way servers are tuned.
So better approach is to use some <div> or something and pour your results into it like this one
<div id="op"></div>
and
function parseEvent(artist) {
var url = "http://api.bandsintown.com/artists/" + artist + "/events.json?lapi_version=2.0&app_id=music_matcher";
httpGetAsync(url, function(data) {
var numEvents = Object.keys(data).length;
var op = document.getElementById('op');
op.innerHTML = op.innerHTML + " <br><br> <h2>---NEXT ARTIST---<h2> <br>";
//var events = [];
for (var j = 0; j < numEvents; j++) {
op.innerHTML = op.innerHTML + "<br>" + data[j].venue.name + "-> ";
op.innerHTML = op.innerHTML + "<br>" + "LAT:" + data[j].venue.latitude + " " + "LNG:" + data[j].venue.longitude ;
op.innerHTML = op.innerHTML + "<br>" +"ARTIST: " + data[j].artists[0].name;
op.innerHTML = op.innerHTML + "<br>" +"DATE: " + data[j].datetime;
op.innerHTML = op.innerHTML + "<br>" + " " + j + " <br>";
}
});
}
var artists = ["Drake", "Hippo Campus", "STRFKR","Mac Demarco"];
for (var i = 0; i < artists.length; i++) {
parseEvent(artists[i]);
}

What will happen if I define more than one parameter but only pass in one parameter?

The function was defined as follow, but I don't know how the function will work if I pass in only one parameter and ignore the order.
function setCookie(name,value,path,expires){
value = escape(value);
if(!expires){
var now = new Date();
now.setMonth(now.getMonth() + 6);
expires = now.toUTCString();
}
if(path){
path = ";path = " + path;
}
document.cookie = name + "=" + value + ";expires = " + expires + path;
}
The rest will be undefined.
This would have been a good thing to try out on your own before asking. You could have written a test program and tested it in probably the same amount of time it took you to write the question.
Ommitting parameters to a JS function implicitly sets them to undefined. Extra parameters are ignored.
This is often used to provide default arguments (prior to ES6), like so:
function foo(bar) {
if (bar === undefined) bar = "default";
console.log(bar);
}
foo("test"); // logs "test"
foo(); // logs "default"
It will work for the most part, but you need to make sure each variable you use is defined. In the line, function setCookie(name,value,path,expires){, name, value, path, expires are all being defined right there. Its pretty much the same as var name =..., getting set to whatever you passed in. But if you don't pass anything in, it leaves out the part that defines name or the other variables, leaving those variables = undefined. When you have a variable equal to undefined, you can't use it in lines like document.cookie = name + "=" + value + ";expires = " + expires + path;. What you need to do is make sure that each one exists, and if it doesn't, define it as, in this case, a blank string.
JavaScript has very loose rules when it comes to arguments. You can make input infinite arguments that the function doesn't list (though they have to be accessed through the built in arguments variable).
function showArguments() {
for(var index = 0; arguments.length >= index; index++) {
var argument = arguments[index];
var p = document.createElement('p');
p.textContent = argument;
document.body.appendChild(p)
}
}
showArguments('foo', 'test')
You can run a function asking for 10 arguments with 0, and only when you try to access a non-existent property will it error.
function foo(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10) {
console.log('no error, until you try to access something like length on the undefined argument');
var length = arg1.length;
}
foo();
But because of this looseness, when you try to access arguments that should be there, undefined is returned. So in your case beside the first possible error of running escape on undefined, there should be no errors, and document.cookie will be equal to name + '=' + undefined + ';expires = ' + now.toUTCString() + undefined.
function setCookie(name,value,path,expires){
value = escape(value);
if(!expires){
var now = new Date();
now.setMonth(now.getMonth() + 6);
expires = now.toUTCString();
}
if(path){
path = ";path = " + path;
}
var cookie = name + "=" + value + ";expires = " + expires + path;
var div = document.createElement('div');
var p = document.createElement('p');
var p2 = document.createElement('p');
p.textContent = 'name + "=" + value + ";expires = " + expires + path: ' + cookie;
p2.textContent = 'name + "=" + undefined + ";expires = " + now.toUTCString() + undefined: ' + name + "=" + undefined + ";expires = " + now.toUTCString() + undefined;
div.appendChild(p);
div.appendChild(p2);
document.body.appendChild(div);
return cookie
}
setCookie('foo');
To set a default for these undefined variables add var foo = foo || 'default'.

Using casperjs and phantomjs to scrape multiple pages

I'm trying to scrape a number of pages that have a standard format. I've been able to use Phantomjs to successfully scrape a single page, but when I try to iterate over multiple ones, the asynchronous processing makes things hang up. What's the proper way to tell Casper/Phantom to wait?
var page = require('webpage').create();
var fs = require('fs');
page.onConsoleMessage = function(msg) {
phantom.outputEncoding = "utf-8";
console.log(msg);
};
// this overwrites the previous output file
f = fs.open("lat_long.txt", "w");
f.write("--");
f.close();
// this is the unique identifier for the locations. For now, I just have three datapoints
var EPAID = ["KYD980501076","ME8170022018", "MEN000103584"];
/// this code will be used to loop through the different locations. For now, set to look at only one.
for (q= 0; q < 1; q++) {
var processing = false;
//we construct the target url
var url = "http://iaspub.epa.gov/enviro/efsystemquery.cerclis?fac_search=site_epa_id&fac_value=" + EPAID[0] + "&fac_search_type=Beginning+With&postal_code=&location_address=&add_search_type=Beginning+With&city_name=&county_name=&state_code=&program_search=1&report=2&page_no=1&output_sql_switch=TRUE&database_type=CERCLIS" ;
page.open(url);
page.onLoadFinished = function(status) {
if ( status === "success" ) {
page.includeJs("http://ajax.googleapis.com/ajax/libs/jquery/1.6.1/jquery.min.js", function() {
var str = page.evaluate(function() {
$value = [];
$Object = $(".result tr");
for (i =0 ; i < 10; i++) {
$value.push($Object.find('td').html(),$Object.find('td').next().next().html() );
$Object = $Object.next();
}
$string = "{ EPAID: "+ $value[0] + ", " +
"Name: "+ $value[1] + ", " +
"City: "+ $value[4] + ", " +
"State: "+ $value[6] + ", " +
"ZipCode: "+ $value[8] + ", " +
"Latitude: "+ $value[14] + ", " +
"Longitude: "+ $value[16] + " }" ;
return $string;
});
f = fs.open("lat_long.txt", "a");
f.write(str);
f.close();
processing = true;
console.log("writing to file");
phantom.exit();
});
}
// right here it should delay until the previous page is completed
// while (!processing) {
// setTimeout(function(){ console.log("waiting....");},1000);
// }
};
}
console.log("finished all pages");
If you switched to using casperJS, it is as simple as changing your page.open() into page.thenOpen(). (This CasperJS - How to open up all links in an array of links question looks very similar to yours?)
If you wanted to stick with PhantomJS you need to start the next page load in the onSuccess callback of the previous load. This is tedious, and needs care to avoid large memory usage. (I did it once or twice, but now simply use CasperJS.)
An alternative approach is to create the page object inside the loop. However that is not quite answering your question, as then they will run in parallel. But you could use setTimeout to stagger each once to avoid a burst of activity if you have hundreds of URLs!
Here is the code that ultimately works (using the timeout approach since I wasn't able to get the success callback to work better).
With casperjs installed, I named this file "process.js" and was able to run it from the command line as "casperjs process.js"
var page = require('webpage').create();
var fs = require('fs');
page.onConsoleMessage = function(msg) {
phantom.outputEncoding = "utf-8";
console.log(msg);
};
// this overwrites the previous output f
// this is the unique identifier for the locations.
var EPAID = ["NED981713837",... , "FLD049985302", "NJD986643153"];
f = fs.open("lat_long.txt", "w");
f.write("-<>-");
f.close();
var count = 0;
var target = 1400;
var written = [];
function yourFunction(){
if (count < target) {
process(count);
count++;
setTimeout(yourFunction, 5000);
} else {
console.log("exiting");
phantom.exit();
return;
}
}
function process(counter){
var processing = false;
console.log("Beginning record #" + counter);
//we construct the target url
var url = "http://iaspub.epa.gov/enviro/efsystemquery.cerclis?fac_search=site_epa_id&fac_value=" + EPAID[counter] + "&fac_search_type=Beginning+With&postal_code=&location_address=&add_search_type=Beginning+With&city_name=&county_name=&state_code=&program_search=1&report=2&page_no=1&output_sql_switch=TRUE&database_type=CERCLIS" ;
page.open(url);
page.onLoadFinished = function(status) {
if ( status === "success" ) {
page.includeJs("http://ajax.googleapis.com/ajax/libs/jquery/1.6.1/jquery.min.js", function() {
var str = page.evaluate(function() {
$value = [];
$Object = $(".result tr");
for (i =0 ; i < 10; i++) {
$value.push($Object.find('td').html(),$Object.find('td').next().next().html() );
$Object = $Object.next();
}
$string = "{ \"EPAID\": \""+ $value[0] + "\", " +
"\"Name\": \""+ $value[1] + "\", " +
"\"City\": \""+ $value[4] + "\", " +
"\"State\": \""+ $value[6] + "\", " +
"\"ZipCode\": \""+ $value[8] + "\", " +
"\"Latitude\": "+ $value[14] + ", " +
"\"Longitude\": "+ $value[16] + " }," ;
return $string;
});
if (written[counter] === undefined) {
f = fs.open("lat_long.txt", "a");
f.write(str);
f.close();
written[counter] = true;
console.log("Writing to file #"+ counter);
}
});
}
};
}
console.log("Start...");
yourFunction();

CasperJS Load next page in loop

I've been working on a script which collates the scores for a list of user from a website. One problem is though, I'm trying to load the next page in the while loop, but the function is not being loaded...
casper.then(function () {
var fs = require('fs');
json = require('usernames.json');
var length = json.username.length;
leaderboard = {};
for (var ii = 0; ii < length; ii++) {
var currentName = json.username[ii];
this.thenOpen("http://www.url.com?ul=" + currentName + "&sortdir=desc&sort=lastfound", function (id) {
return function () {
this.capture("Screenshots/" + json.username[id] + ".png");
if (!casper.exists(x("//*[contains(text(), 'That username does not exist in the system')]"))) {
if (casper.exists(x('//*[#id="ctl00_ContentBody_ResultsPanel"]/table[2]'))) {
this.thenEvaluate(tgsagc.tagNextLink);
tgsagc.cacheCount = 0;
tgsagc.
continue = true;
this.echo("------------ " + json.username[id] + " ------------");
while (tgsagc.
continue) {
this.then(function () {
this.evaluate(tgsagc.tagNextLink);
var findDates, pageNumber;
pageNumber = this.evaluate(tgsagc.pageNumber);
findDates = this.evaluate(tgsagc.getFindDates);
this.echo("Found " + findDates.length + " on page " + pageNumber);
tgsagc.checkFinds(findDates);
this.echo(tgsagc.cacheCount + " Caches for " + json.username[id]);
this.echo("Continue? " + tgsagc["continue"]);
this.click("#tgsagc-link-next");
});
}
leaderboard[json.username[id]] = tgsagc.cacheCount;
console.log("Final Count: " + leaderboard[json.username[id]]);
console.log(JSON.stringify(leaderboard));
} else {
this.echo("------------ " + json.username[id] + " ------------");
this.echo("0 Caches Found");
leaderboard[json.username[id]] = 0;
console.log(JSON.stringify(leaderboard));
}
} else {
this.echo("------------ " + json.username[id] + " ------------");
this.echo("No User found with that Username");
leaderboard[json.username[id]] = null;
console.log(JSON.stringify(leaderboard));
}
});
while (tgsagc.continue) {
this.then(function(){
this.evaluate(tgsagc.tagNextLink);
var findDates, pageNumber;
pageNumber = this.evaluate(tgsagc.pageNumber);
findDates = this.evaluate(tgsagc.getFindDates);
this.echo("Found " + findDates.length + " on page " + pageNumber);
tgsagc.checkFinds(findDates);
this.echo(tgsagc.cacheCount + " Caches for " + json.username[id]);
this.echo("Continue? " + tgsagc["continue"]);
return this.click("#tgsagc-link-next");
});
}
Ok, looking at this code I can suggest a couple of changes you should make:
I don't think you should be calling return from within your function within then(). This maybe terminating the function prematurely. Looking at the casperjs documentation, the examples don't return anything either.
Within your while loop, what sets "tgsagc.continue" to false?
Don't use "continue" as a variable name. It is a reserved word in Javascript used for terminating an iteration of a loop. In your case this shouldn't be a problem, but its bad practice anyhow.
Don't continually re-define the method within your call to the then() function. Refactor your code so that it is defined once elsewhere.
We ended up having to scope the function, so it loads the next page in the loop.
This is mainly because CasperJS is not designed to calculate scores, and it tries to asynchronously do the calculation, missing the required functions

Categories

Resources