Develop Reference

Chrome Extension - read source code with dynamic content

I have written code for a small Chrome Extension that scrapes content based on selectors. Once I have the content I need, I push it to my PHP server in order to store it in a database. The code works well except when content is loaded dynamically, I can only see what is visible the source code from the browser.
I thought I could solve the issue by waiting until the page was fully loaded (setInterval) but even when I see that the content is loaded, the source code I'm getting is still the initial one.
I use the following code to create a tab and retrieve the source code:
function CreateTab (createProperties)
{
chrome.tabs.create(createProperties, tab =>
{
if (chrome.runtime.lastError)
{
console.log(chrome.runtime.lastError);
}
else
{
container = "h2";
var i = 0;
var checkClass = setInterval (function()
{
if (i<=3)
{
console.log('Iteration :'+i);
chrome.tabs.executeScript(tab.id,
{
file: 'GetSource.js',
}, async function(results)
{
let e = chrome.runtime.lastError;
if(e !== undefined)
{
console.log(tab.id, e);
}
// GETTING HTML
parser = new DOMParser();
content = parser.parseFromString(results, "text/html");
console.log(content);
// CAPTURE TITLES
try
{
datatable = content.querySelectorAll(container);
}
catch(err)
{
console.log('Iteration '+i+' table not found');
datatable = "";
}
if (datatable.length>0)
{
clearInterval(checkClass);
removeTab(tab.id,3)
// DATA FOUND!
i=10;
}
});
i++;
}
else
{
clearInterval(checkClass);
console.log('Container not found');
removeTab(tab.id,5)
}
},10000);
}
});
}
The script I'm calling to get the source is the following (GetSource.js)
function DOMtoString(document_root) {
var html = '',
node = document_root.firstChild;
while (node) {
switch (node.nodeType) {
case Node.ELEMENT_NODE:
html += node.outerHTML;
break;
case Node.TEXT_NODE:
html += node.nodeValue;
break;
case Node.CDATA_SECTION_NODE:
html += '<![CDATA[' + node.nodeValue + ']]>';
break;
case Node.COMMENT_NODE:
html += '<!--' + node.nodeValue + '-->';
break;
case Node.DOCUMENT_TYPE_NODE:
// (X)HTML documents are identified by public identifiers
html += "<!DOCTYPE " + node.name + (node.publicId ? ' PUBLIC "' + node.publicId + '"' : '') + (!node.publicId && node.systemId ? ' SYSTEM' : '') + (node.systemId ? ' "' + node.systemId + '"' : '') + '>\n';
break;
}
node = node.nextSibling;
}
return html;
}
sourcecode = DOMtoString (document);
sourcecode
How can I make sure that the GetSource.js script is getting the final rendered source code? There are more and more sites using dynamic content loaded after the initial load and with this script I'm stuck on those sites.
There are ready to use scrapers in the Chrome store that can get dynamic content so there must be a solution for my script too. I don't want to use to solutions as I need to retrieve the selectors I want to catch from my server and once the scraping is done pushing them back to the server.
Do you have any idea of how I could achieve this?
Thanks

What is wrong with fileOpener2?

This question is edit for better undersatnding:
I am trying to use the cordova fileOpener2 plugin to open pdf files in my app's assets.
I obtain the file's uri thanks to lines 14 and onward of code below.
The uri is then stored as a varialbe (nonencodeduri).
However when I try to use the variable in the second part of the code where FileOpener2 needs the path to the file (from line 58), it stalls.
This is surprising because if I write hardcode the path to the same file line 58 (var uri = var uri = encodeURI("path to file in the assets of the app"), it works.
Thanks for helping me resolve this.
Here is the full code (credits: Ghandi)
var entry, documentname, documentid, referenceID, callLogID, filePath, blob,cdr,fileObject;
var filename = "test.pdf";
$(document).ready(function() {
document.addEventListener("deviceready", onDeviceReady, false);
});
var fileURL = "";
var imagePath = "";
function onDeviceReady() {
sessionStorage.platform = device.platform;
var fileTransfer = new FileTransfer();
$('a[href$=\\.pdf]').click(function()
{
try {
alert('Hi boys');
var urinonencoded = this.href;
alert(urinonencoded + ' and voila');
if (sessionStorage.platform.toLowerCase() == "android") {
window.resolveLocalFileSystemURL(cordova.file.externalRootDirectory,onFileSystemSuccess, onError);
}
else {
window.requestFileSystem(LocalFileSystem.PERSISTENT, 0,onFileSystemSuccess, onError);
}
}
catch(err) {
alert("ER - " + err.message);
}
});
function onError(e) {
alert("onError");
};
function onFileSystemSuccess(fileSystem) {
var entry="";
if (sessionStorage.platform.toLowerCase() == "android") {
entry=fileSystem;
}
else {
entry=fileSystem.root;
}
entry.getDirectory("Cordova", {create: true, exclusive: false}, onGetDirectorySuccess, onGetDirectoryFail);
};
function onGetDirectorySuccess(dir) {
dir.getDirectory("Sample_App", {create: true, exclusive: false}, onGetDirectorySuccess1, onGetDirectoryFail);
};
function onGetDirectorySuccess1(dir) {
cdr = dir;
dir.getFile(filename,{create:true, exclusive:false},gotFileEntry, errorHandler);
};
function gotFileEntry(fileEntry) {
/*var uri = encodeURI(myref);*/
var uri = urinonencoded;
alert (uri);
alert("dest - " + cdr.nativeURL+filename);
fileTransfer.download(uri,cdr.nativeURL+filename,
function(entry) {
openFile();
},
function(error) {
alert("download error source " + error.source);
alert("download error target " + error.target);
alert("upload error code" + error.code);
alert("error");
},
true);
};
function openFile() {
alert("URL - " + cdr.nativeURL+filename);
cordova.plugins.fileOpener2.open(
cdr.nativeURL+filename,
'application/pdf',
//'text/plain',
{
error : function(e) {
alert('Error status: ' + e.status + ' - Error message: ' + e.message);
},
success : function () {
}
}
);
};
function onFileSystemSuccessDelete(fileSystem) {
var entry="";
if (sessionStorage.platform.toLowerCase() == "android") {
entry=fileSystem;
}
else {
entry=fileSystem.root;
}
entry.getDirectory("Cordova/Sample_App", {create: true, exclusive: false}, onGetDirectorySuccessDelete, onGetDirectoryFail);
};
function onGetDirectorySuccessDelete(dir) {
dir.getFile(filename,{create: true, exclusive:false},gotFileEntryDelete, fail);
};
function gotFileEntryDelete(fileEntry) {
fileEntry.remove();
var uri = encodeURI("http://SERVER_IP:PORT/test.pdf");
fileTransfer.download(uri,cdr.nativeURL+filename,
function(entry) {
console.log("download complete: " + entry.toURL());
openFile();
},
function(error) {
alert("download error source " + error.source);
alert("download error target " + error.target);
alert("upload error code" + error.code);
alert("error");
},
true);
};
function fail(error){
alert("ec - " + error.code);
};
function errorHandler(e) {
var msg = '';
switch (e.code) {
case FileError.QUOTA_EXCEEDED_ERR:
msg = 'QUOTA_EXCEEDED_ERR';
break;
case FileError.NOT_FOUND_ERR:
msg = 'NOT_FOUND_ERR';
break;
case FileError.SECURITY_ERR:
msg = 'SECURITY_ERR';
break;
case FileError.INVALID_MODIFICATION_ERR:
msg = 'INVALID_MODIFICATION_ERR';
break;
case FileError.INVALID_STATE_ERR:
msg = 'INVALID_STATE_ERR';
break;
default:
msg = e.code;
break;
};
alert("Msg - " + msg);
};
function onGetDirectoryFail(error) {
alert("onGetDirectoryFail");
};
$('#delete').click(ClearDirectory);
function ClearDirectory() {
alert("delete");
if (sessionStorage.platform.toLowerCase() == "android") {
window.resolveLocalFileSystemURL(cordova.file.externalRootDirectory,onFileSystemDirSuccess, fail);
}
else {
window.requestFileSystem(LocalFileSystem.PERSISTENT, 0,onFileSystemDirSuccess, fail);
}
}
function onFileSystemDirSuccess(fileSystem) {
var entry = "";
if (sessionStorage.platform.toLowerCase() == "android") {
entry=fileSystem;
}
else {
entry=fileSystem.root;
}
entry.getDirectory("Cordova",{create : true, exclusive : false},
function(entry) {
entry.removeRecursively(function() {
console.log("Remove Recursively Succeeded");
}, fail);
}, getDirFail);
}
function getDirFail(error){
alert("getDirFail - " + error.code);
};
}
I used:
<script>
$('a[href$=\\.pdf]').click(function() {
var myuri = this.href ;
alert(this.href);
/*alert just to make sure I got the right uri (which works fine)*/
cordova.plugins.fileOpener2.open(
'this.href', // You can also use a Cordova-style file uri: cdvfile://localhost/persistent/Download/starwars.pdf
'application/pdf',
{
error : function(e)
{
alert('Error status: ' + e.status + ' - Error message: ' + e.message);
},
success : function ()
{
alert('file opened successfully');
}
}
);
return false;
});
</script>
and it hangs (i have the plugin declared in the config.xml file and present in the assets.
Can you pinpoint my error(s)?
Many thanks

Basically the issue is with the way you pass the file path paramter you pass to fileopener's open function and nothing wrong with the plugin as such.
If you pass this object inside single quote like 'this.href', it will be treated as a plain string and that's the issue. So you can use the approach as Joerg suggested in his answer.
Do check out this basic working sample app that demonstrates Cordova file operations. It should help you get started.

'this.href' is wrong, try it this way:
$('a[href$=\\.pdf]').click(function () {
var myuri = this.href;
alert(this.href);
/*alert just to make sure I got the right uri (which works fine)*/
cordova.plugins.fileOpener2.open(
myuri, // You can also use a Cordova-style file uri: cdvfile://localhost/persistent/Download/starwars.pdf
'application/pdf',
{
error: function (e) {
alert('Error status: ' + e.status + ' - Error message: ' + e.message);
},
success: function () {
alert('file opened successfully');
}
}
);
return false;
});

To sum up the answers for this complex problem:
first: fileopener2 will not work on its own, it needs other plugins to fit the purpose of opening pdf files on android. The other plugins needed are mentioned in Gandhi's excellent demo app available here.
Second: it is esssentiel to use Gandhi's app.js file that combines the different calls to plugins.
Third there are a number of functions involved in that file. In it the path is hard-coded, so everything seems to work well. But if you must use a dynamic path, then it must be stated globally (in other words, do not use "var" but simply myvar = this.href.
Although I found the last bits on my own, all the credit goes to Gandhi's excellent demo app which is a treasure trove for new plugin users he made available on github. +1 :-)

looking to Work around the 403 error with php script

I have been using JavaScript to retrieve my images from a directory and on local-host this works just fine but now I am running it on a remote sever I get the 403 Forbidden Error, I know why this is but I am looking for a way around it, keeping my Java functioning much the same, so I was thinking if I put an index.php in the gallery folder and called it with a path and have it return a file list back to my JavaScript and a lough it to carry on.
How would I go about this as I am not very good with PHP at the moment? Thanks.
<script type="text/javascript">
$(document).ready(function() {
$("a").click(function() {
var dir_path = $(this).data("albumid");
LoadGallery(dir_path);
return false;
});
});
function LoadGallery(dir_path) {
$.ajax({
url: dir_path,
success: function(data) {
$(".image-container").empty();
$(data).find("a:contains(.jpg), a:contains(.png), a:contains(.jpeg)").each(function() {
this.href.replace(window.location.host, "").replace("http:///", "");
var file = dir_path + $(this).text();
$(".image-container").append($("<a href='javascript:;' class='thumb' data-src='" + file + "'><img src='" + file + "' title='Click to enlarge' alt='#'/></a>"));
if ($(".image-container").children("a").length === 30) {
return false;
}
});
$(".image-container").append("<strong><p>Click on a thumb nail to show a larger image.</p></strong>");
$(".thumb").bind('click', function() {
var Popup = "<div class='bg'></div>" + "<div class='wrapper'><img src='<img src=''/>" + "<label href='javascript:;' class='prev-image'>«</label><label href='javascript:;' class='next-image'>»</label><a href='javascript:;' class='close' title='Close'>Close</a>";
var Img = $(this).attr("data-src");
$("body").prepend(Popup);
$(".bg").height($(window).height() * 4);
$(".wrapper img").attr("src", Img);
$(".prev-image").bind('click', function() {
var prev = $(".image-container").find("img[src='" + Img + "']").parent().prev('a').find("img").attr('src');
if (!prev || prev.length === 0)
return false;
else {
$(".wrapper img").attr("src", prev);
Img = prev;
}
});
$(".next-image").bind('click', function() {
var next = $(".image-container").find("img[src='" + Img + "']").parent().next('a').find("img").attr('src');
if (!next || next.length === 0)
return false;
else {
$(".wrapper img").attr("src", next);
Img = next;
}
});
$(".close").bind('click', function() {
$(this).siblings("img").attr("src", "")
.closest(".wrapper").remove();
$(".bg").remove();
});
});
}
});
}
</script>

Original why are you getting the 403 error? returning a file(name) list from php to your scripts probably isn't going to work around this error, because the javascript would ultimately be making the same request.
on the other hand if you know that you have access to the image files from the server-side there are a few things you could do, like return the binary value of the image (javascript can take this) or generating a close of the file.
Update
if the server allows it, you can use the exec command in php to return a list of files from the directory: eg
\<\
<?php
jpgs = exec('ls /your_directory_here');
echo json_encode(jpgs);
?>
if they don't allow exec(), which they might not, you could try...
http://www.w3schools.com/php/func_directory_readdir.asp
does that help?

Plupload not working in ipad

i am using Plupload to upload images in my site. it work like a charm on all desktop browser but not in i-pad. When i click on upload button, it refreshes the page. I searched and didn't found anything to fix it. So finally, i m here to ask my question. I am using pl upload Core Api.
Here is my code:
var uploader = new plupload.Uploader({
runtimes : 'html5,flash,silverlight,html4',
browse_button : 'local_file_upload', // you can pass an id...
container: document.getElementById('container'), // ... or DOM Element itself
url : "<?php echo WEBSITE_URL . '/includes/upload.php';?>",
unique_names : true,
multi_selection: false,
max_file_count: 1,
flash_swf_url : pl_upload_url+'Moxie.swf',
silverlight_xap_url : pl_upload_url+'/Moxie.xap',
filters : {
max_file_size : '10mb',
mime_types: [
{title : "Image files", extensions : "jpg,gif,png,jpeg"},
]
},
init: {
        PostInit: function() {
            document.getElementById('filelist').innerHTML = '';
 
            document.getElementById('upload_images').onclick = function() {
                uploader.start();
jQuery("#upload_images").css("display","none");
                return false;
            };
        },
FileUploaded: function (up, file) {
if(image_path != "")
{
jQuery("#filelist").css("display","none");
var ajaxURL = "../includes/ajax.php";
jQuery.post(ajaxURL,"image_path="+image_path+"&ajax=true&action=uploadImage",function(data)
{
jQuery("#preview_image").html("<img src='<?php echo WEBSITE_URL.'/uploads/users/'.$user_id.'/thumbnail/'?>"+file.target_name+"' style='width:100%'/>");
});
}
else
{
alert("<?php echo _("Kinldy select image to upload");?>");
}
},
        FilesAdded: function(up, files) {
if (uploader.files.length == 2) {
uploader.removeFile(uploader.files[0]);
}
            plupload.each(files, function(file) {
jQuery("#filelist").css("display","block");
                var img_type = file.name;
var type = img_type.split(".");
image_path = file.id+"."+type[1];
document.getElementById('filelist').innerHTML = '<div id="' + file.id + '" >' + img_type + ' (' + plupload.formatSize(file.size) + ') <b image_path="'+file.id+"."+type[1]+'" style="color:#fff;" onclick="removeImage(this);">x</b></div>';
jQuery("#upload_images").css("display","block");
            });
        },
 
        UploadProgress: function(up, file) {
            jQuery("#"+file.id).find("b").html('<span>' + file.percent + "%</span>");
        },
 
        Error: function(up, err) {
            document.getElementById('console').innerHTML += "\nError #" + err.code + ": " + err.message;
        }
    }
});
uploader.init();
Waiting for some reply. Thanks

I am having a similar problem with plupload on the ipad.
I copied your code and it indeed refreshes the page.
I tried removing the "PostInit"-code and that stopped the refreshing of the page.

How can I return the path of a screenshot capture to a function and return via JSON to javascript?

I have a PHP script that invokes a casperjs script via exec function and this is working fine.
Is it possible to return the path where I saved a screenshot via exec as JSON?
My scripts are below:
PHP code:
// Execute to CasperJS via asynchronous process
<?php
$username = $_POST['username'];
$password = $_POST['password'];
$target = $_POST['target'];
$filename = $_POST['file'];
$retorno = array()
try {
exec("{$casperjs_run} {$script} {$username} {$password} {$filename} 2>&1", $output);
} catch (Exception $e) {
$retorno['error404'] = "Desculpe! Não foi possivel acessar a página solicitada.";
}
// Return Data if success
// Retorna para front-end
if (empty($output)){
$retorno['success'] = $output;
echo json_encode($retorno);
return false;
} else {
$retorno['error'] = $output;
echo json_encode($retorno);
return false;
}
?>
CasperJS code:
casper.thenOpen(minhaoi, function myaccount() {
this.capture('pic2.png');
this.log('Acessando informações da conta, aguarde...');
if (!this.exists(('div.panel-horizontal'))) {
this.log(JSON.stringify("Não foi encontrado um plano colaborador, aguarde..."));
noDetails = this.captureSelector(filename + '.png', 'div.panel-horizontal', {quality: 100});
} else {
casper.waitForResource("Análise de Conta", function orderDetails(details) {
return details;
}, function onReceive() {
this.log('ScreenShot Begin');
myDetails = this.captureSelector(path_images + filename + '.png', '#content', { quality: 100 } );
this.log(' ScreenShot Done'); });
});
}
});
// Logout & Exit
casper.eachThen(oi_out, function () {
this.capture('pic3.png');
if (noDetails != "") {
return noDetails;
} else {
return myDetails;
}).run();
Here my JS code that receive the information from casperjs via JSON.
Javascript Code:
success: function(data) {
if (data.success) {
$('#retorno').html(data.success);
$('#imagem').attr('src', '/details/' + filename);
$('#resultado').show();
}
},
error: function(data) {
// check error
$('#retorno').attr("class='alert alert-danger' role='alert'");
$('#retorno').html(data.error);
}
In my mind filename should be the whole name of the screenshot like this, pi9rxw2fqlh.png plus the complete path too. And display the image in the browser.
What's wrong in my approach?

For this.log to actually print something, you need to set the logLevel to at least debug as it is the default log level. So either increase the log level casper.options.logLevel = 'debug'; or use this.echo instead of this.log.
It looks like you're using waitForResource wrong. Since there can't be resources with spaces in them, you might want to checkout waitForText under the assumption that the loaded resource adds that string to the DOM:
casper.waitForText("Análise de Conta", function onReceive() {
this.log('ScreenShot Begin');
myDetails = this.captureSelector(path_images + filename + '.png', '#content', { quality: 100 } );
this.log(' ScreenShot Done'); });
});
capture as well as captureSelector return the casper instance and not the image details. So you need to pass the filename.
Since you use php's exec with the output array, you can casper.echo the filename in question with a unique beginning string (here #noDetails#):
this.captureSelector(filename + '.png', 'div.panel-horizontal', {quality: 100});
this.echo("#noDetails#" + filename + ".png");
In the client javascript you can then iterate over the data.success or data.error arrays and extract the filename from the match line:
data.success.forEach(function(line){
if (line.indexOf("#noDetails#") === 0) {
var filename = line.split("#noDetails#")[1];
$('#imagem').attr('src', '/details/' + filename);
}
});
With this, you can completely remove the if block from the eachThen callback.
The other option is to set the specific screenshot variable and write the JSON object in the last line.
this.captureSelector(filename + '.png', 'div.panel-horizontal', {quality: 100});
noDetails = filename + ".png";
and at the end:
casper.eachThen(oi_out, function () {
this.capture('pic3.png');
if (noDetails != "") {
this.echo(JSON.stringify({filename:noDetails}));
} else {
this.echo(JSON.stringify({filename:myDetails}));
}
});
On the client side, you would need to only look in the last line of the array:
var obj = JSON.parse(data.success[data.success.length-1]);
$('#imagem').attr('src', '/details/' + obj.filename);