Develop Reference

Modifying a JS browser extension's HTML parser to exclude images

I'll get out of the way that although I have a programming background I have no experience with any of the tools or languages I'm working with here. Sorry if I've made simple misunderstandings or communicate something unclearly.
I'm using an extension to generate EPUBs from website links. The extension has the option to customize the parsing logic from the options dialog. Doing so yields this code for the parser it's currently using:
var parser = new DOMParser();
var dom = parser.parseFromString(source, "text/html");
var new_link = null;
var subchaps = [];
// Wordpress content
var main_cont = dom.querySelector(".entry-content")
if(main_cont != null){
var ancs = main_cont.querySelectorAll("a");
ancs.forEach((element) => {
if(RegExp(/click here to read|read here|continue reading/i).test(element.innerText)){
new_link = helpers["link_fixer"](element.href, url);
} else if (RegExp(/chapter|part/i).test(element.innerText)) {
subchaps.push(helpers["link_fixer"](element.href, url))
}
});
}
if (new_link != null) {
var res = await fetch(new_link);
var r_txt = await res.text();
dom = parser.parseFromString(r_txt, "text/html");
var out = helpers["readability"](dom);
return {title: out.title, html: out.content};
} else if (subchaps.length > 0) {
var html = "";
for(var subc in subchaps){
console.log(subchaps[subc]);
var cres = await fetch(subchaps[subc]);
var c_txt = await cres.text();
var cdom = parser.parseFromString(c_txt, "text/html");
var out = helpers["readability"](cdom);
html += "<h1>"+out.title+"</h1>"+ out.content
}
return {title: title, html: html};
}
var out = helpers["readability"](dom);
return {title: out.title, html: out.content};
I've inspected this code and gathered that it handles three cases: two ways that it needs to follow links deeper before parsing, and the simple case where it is already in the right place. The lions share of the code deals with the first two cases and it's largely the third that I'm interested in. Unfortunately, it appears that the second to last line is where the parsing actually happens:
var out = helpers["readability"](dom);
And this is a magic box to me. I can't for the life of me figure out what this is referencing.
I've searched the full file for a definition of 'helpers' or even 'readability' but come up blank. I was under the impression that the part I was editing was the 'readibility' parser. I thought I'd be able to pop into the parser logic, add a line to exclude nodes with the <img> tag, and live happily ever after. What am I mistaken about? Or is what I want to do impossible, given what the extension is letting me modify?
To be clear, I am not asking for a full guide on how to write parser logic. I considered just parsing and repackaging the document before the line in question, but I didn't want to write the same code 3 times, and I don't like that I can't tell what it's doing. I couldn't even begin to search for documentation, given that I can't find the definition in the first place. Even just explaining what that line does and pointing to any relevant documentation would be a great help.
Thanks in advance.
(And here is the full file, if you feel like verifying that I didn't miss anything.)
main_parser: |
var link = new URL(url);
var parser = new DOMParser();
var dom = parser.parseFromString(source, "text/html");
switch (link.hostname) {
case "www.novelupdates.com":
var paths = link.pathname.split("/");
if (paths.length > 1 && paths[1] == "series") {
return {page_type:"toc", parser:"chaps_nu"};
}
}
// Default to all links
return {page_type:"toc", parser:"chaps_all_links"};
toc_parsers:
chaps_nu:
name: Novel Update
code: |
var parser = new DOMParser();
var dom = parser.parseFromString(source, "text/html");
var chap_popup = dom.querySelector("#my_popupreading");
if (chap_popup == null) {
return []
}
var chap_lis = chap_popup.querySelectorAll("a");
var chaps = [];
chap_lis.forEach((element) => {
if (element.href.includes("extnu")) {
chaps.unshift({
url_title: element.innerText,
url: helpers["link_fixer"](element.href, url),
});
}
});
var tit = dom.querySelector(".seriestitlenu").innerText;
var desc = dom.querySelector("#editdescription").innerHTML;
var auth = dom.querySelector("#authtag").innerText;
var img = dom.querySelector(".serieseditimg > img");
if (img == null){
img = dom.querySelector(".seriesimg > img");
}
return {"chaps":chaps,
meta:{title:tit, description: desc, author: auth, cover: img.src, publisher: "Novel Update"}
};
chaps_name_search:
name: Chapter Links
code: |
var parser = new DOMParser();
var dom = parser.parseFromString(source, "text/html");
var ancs = dom.querySelectorAll("a");
var chaps = []
ancs.forEach((element) => {
if(RegExp(/chap|part/i).test(element.innerText)){
chaps.push({
url_title: element.innerText,
url: helpers["link_fixer"](element.href, url),
});
}
});
return {"chaps":chaps,
meta:{}
};
chaps_all_links:
name: All Links
code: |
var parser = new DOMParser();
var dom = parser.parseFromString(source, "text/html");
var ancs = dom.querySelectorAll("a");
var chaps = []
ancs.forEach((element) => {
chaps.push({
url_title: element.innerText,
url: helpers["link_fixer"](element.href, url),
});
});
return {"chaps":chaps,
meta:{}
};
chap_main_parser: |
var url = new URL(url);
var parser = new DOMParser();
var dom = parser.parseFromString(source, "text/html");
// Generic parser
return {chap_type: "chap", parser:"chaps_readability"};
chap_parsers:
chaps_readability:
name: Readability
code: |
var parser = new DOMParser();
var dom = parser.parseFromString(source, "text/html");
var new_link = null;
var subchaps = [];
// Wordpress content
var main_cont = dom.querySelector(".entry-content")
if(main_cont != null){
var ancs = main_cont.querySelectorAll("a");
ancs.forEach((element) => {
if(RegExp(/click here to read|read here|continue reading/i).test(element.innerText)){
new_link = helpers["link_fixer"](element.href, url);
} else if (RegExp(/chapter|part/i).test(element.innerText)) {
subchaps.push(helpers["link_fixer"](element.href, url))
}
});
}
if (new_link != null) {
var res = await fetch(new_link);
var r_txt = await res.text();
dom = parser.parseFromString(r_txt, "text/html");
var out = helpers["readability"](dom);
return {title: out.title, html: out.content};
} else if (subchaps.length > 0) {
var html = "";
for(var subc in subchaps){
console.log(subchaps[subc]);
var cres = await fetch(subchaps[subc]);
var c_txt = await cres.text();
var cdom = parser.parseFromString(c_txt, "text/html");
var out = helpers["readability"](cdom);
html += "<h1>"+out.title+"</h1>"+ out.content
}
return {title: title, html: html};
}
var out = helpers["readability"](dom);
return {title: out.title, html: out.content};
chaps_raw:
name: No Parse
code: |
return {title: title, html: source}

Why doesn't XMLDocument.evaluate find nodes in this WMS GetCapabilities document

I'm trying to parse information from this XML Document.
The JavaScript below works for simple XML test docs but I cannot find XPath that will return any nodes from the real document.
he idea is just to list all the "Layer" nodes from the WMS GetCapabilities XMl Document.
What am I doing wrong?
Thanks, Code below.
<html>
<body>
<script>
var xhttp = new XMLHttpRequest();
xhttp.onreadystatechange = function() {
if (this.readyState == 4 && this.status == 200) {
parser = new DOMParser();
xmlDoc = parser.parseFromString(xhttp.responseText, "text/xml");
var iterator = xmlDoc.evaluate('Layer', xmlDoc.documentElement, null, XPathResult.UNORDERED_NODE_ITERATOR_TYPE, null);
var thisNode = iterator.iterateNext();
while (thisNode) {
documemnt.console.log(thisNode.textContent);
thisNode = iterator.iterateNext();
}
}
};
xhttp.open("GET", "https://geo.weather.gc.ca/geomet?service=WMS&request=GetCapabilities", true);
xhttp.send();
</script>
</body>
</html>

I think you need to learn how to take a default namespace into account, using the third argument of the evaluate function to map a prefix you can choose freely to the namespace the elements like the Layer or Title elements are in and to use that prefix in your XPath expressions:
var req = new XMLHttpRequest();
req.open('GET', 'https://geo.weather.gc.ca/geomet?service=WMS&request=GetCapabilities');
req.onload = function() {
var doc = this.responseXML;
var namespaces = { wms: 'http://www.opengis.net/wms' };
var result = doc.evaluate(
'/wms:WMS_Capabilities/wms:Capability//wms:Layer/wms:Title',
doc,
function(prefix) { return namespaces[prefix]; },
XPathResult.ORDERED_NODE_SNAPSHOT_TYPE,
null);
var ol = document.createElement('ol');
for (var i = 0; i < result.snapshotLength; i++) {
var li = document.createElement('li');
li.textContent = result.snapshotItem(i).textContent;
ol.appendChild(li);
}
document.body.appendChild(ol);
};
req.send();

Processing Server XML Response

So I'm studying for an exam and I only have a quick question that has been buggin me for a while. I use AJAX to obtain a XML file to parse it and insert its values into a select element. This is the code:
<html>
<head>
</head>
<body>
<button onclick="EnviaPedido()">Submeter</button>
<select id="select"></select>
<script type="text/javascript">
var xmlHttpObj;
function CreateXmlHttpRequestObject() {
if (window.XMLHttpRequest) {
xmlHttpObj = new XMLHttpRequest()
}
else if (window.ActiveXObject) {
xmlHttpObj = new ActiveXObject("Microsoft.XMLHTTP")
}
return xmlHttpObj;
}
function EnviaPedido() {
xmlHttpObj = CreateXmlHttpRequestObject();
xmlHttpObj.open("POST", "agenda.xml", true);
xmlHttpObj.onreadystatechange = ProcessaReposta;
xmlHttpObj.send();
}
function ProcessaReposta() {
if (xmlHttpObj.readyState == 4 && xmlHttpObj.status == 200) {
var response = xmlHttpObj.responseText;
var xmlDoc;
if (window.DOMParser) {
parser = new DOMParser();
xmlDoc = parser.parseFromString(response, "text/xml");
} else {
xmlDoc = new ActiveXObject("Microsoft.XMLDOM");
xmlDoc.async = false;
xmlDoc.loadXML(response);
}
var select = document.getElementById("select");
var centrosInvestigacao = xmlDoc.getElementsByTagName("centro_de_investigacao");
for(i = 0; i < centrosInvestigacao.length; i++) {
var option = document.createElement("option");
option.innerHTML = centrosInvestigacao[i].childNodes[1].textContent;
select.appendChild(option);
}
}
}
</script>
<body>
</html>
And this is the XML that is returned:
<FCT>
<centro_de_investigacao id="1">
<nome>GECAD</nome>
<local>ISEP</local>
<classificao>Muito bom</classificao>
</centro_de_investigacao>
<centro_de_investigacao id="2">
<nome>DEF</nome>
<local>ISEP</local>
<classificao>Bom</classificao>
</centro_de_investigacao>
<centro_de_investigacao id="3">
<nome>ABC</nome>
<local>FEUP</local>
<classificao>Muito mau</classificao>
</centro_de_investigacao>
</FCT>
So when I want to obtain the 'nome' field why do I have to use
option.innerHTML = centrosInvestigacao[i].childNodes[1].textContent;
instead of
option.innerHTML = centrosInvestigacao[i].childNodes[0].textContent;
I know it's probably a stupid question but it's starting to piss me off not knowing if this is the expected behaviour or if I'm somehow screwing this up.
Thanks.

I made a small snipset of the problem that also shows part of the answer. I would think that the encoding of the XML document you are parsing is "off", in that sence that the first element of your childNodes, is actually a textnode between the last quote of centro_de_investigacao> and the start tag of <nome.
If you check the snipset (you can play around with the index parameter for example), you would see that the first button doesn't need the index increase to 1, but can work as expected with the first element, nl the one at index 0.
So remove the whitespaces from your XML document, and you should be fine.
Whitespaces: tab, space, newline
// mocked, no real data
var xmlHttpObj;
function CreateXmlHttpRequestObject() {
function Mock() {
this.callready = function() {
this.readyState = 4;
this.status = 200;
this.statusMsg = "OK";
if (this.onreadystatechange && this.onreadystatechange.call) {
setTimeout(this.onreadystatechange.bind(this), 0);
}
};
this.open = function(methodType, url, async) {
var el = document.getElementById('dataXml-' + url.split('.')[0]),
content = el ? el.innerHTML : '';
if (typeof async === 'undefined' || async) {
// no action till send is executed
this.responseText = content;
this.responseXml = content;
return;
}
return content;
};
this.send = function(data) {
this.callready();
};
}
return new Mock();
}
function EnviaPedido(index, url) {
xmlHttpObj = CreateXmlHttpRequestObject();
xmlHttpObj.open("POST", url, true);
xmlHttpObj.onreadystatechange = ProcessaReposta.bind(this, index);
xmlHttpObj.send();
}
function ProcessaReposta(index, url) {
if (xmlHttpObj.readyState == 4 && xmlHttpObj.status == 200) {
var response = xmlHttpObj.responseText;
var xmlDoc;
if (window.DOMParser) {
parser = new DOMParser();
xmlDoc = parser.parseFromString(response, "text/xml");
} else {
xmlDoc = new ActiveXObject("Microsoft.XMLDOM");
xmlDoc.async = false;
xmlDoc.loadXML(response);
}
var select = document.getElementById("select");
var centrosInvestigacao = xmlDoc.getElementsByTagName("centro_de_investigacao");
select.options = [];
for (i = 0; i < centrosInvestigacao.length; i++) {
var option = new Option();
var item = centrosInvestigacao[i].childNodes[index];
option.text = item.textContent;
select.options[i] = option;
}
}
}
<template id="dataXml-agenda"><FCT><centro_de_investigacao id="1"><nome>GECAD</nome><local>ISEP</local><classificao>Muito bom</classificao></centro_de_investigacao><centro_de_investigacao id="2"><nome>DEF</nome><local>ISEP</local><classificao>Bom</classificao></centro_de_investigacao><centro_de_investigacao id="3"><nome>ABC</nome><local>FEUP</local><classificao>Muito mau</classificao></centro_de_investigacao></FCT></template>
<template id="dataXml-original-agenda"><FCT>
<centro_de_investigacao id="1">
<nome>GECAD</nome>
<local>ISEP</local>
<classificao>Muito bom</classificao>
</centro_de_investigacao>
<centro_de_investigacao id="2">
<nome>DEF</nome>
<local>ISEP</local>
<classificao>Bom</classificao>
</centro_de_investigacao>
<centro_de_investigacao id="3">
<nome>ABC</nome>
<local>FEUP</local>
<classificao>Muito mau</classificao>
</centro_de_investigacao>
</FCT></template>
<select id="select"></select>
<button id="btnGenerate" type="button" onclick="EnviaPedido(0, 'agenda.xml');">Get info</button>
<button id="btnGenerate" type="button" onclick="EnviaPedido(1, 'original-agenda.xml');">Get info false contentType</button>

Adding XML element using Jquery

I have been working on adding XML element to XML structure without success, here is my code.
var parseXml;
if (window.DOMParser) {
parseXml = function(xmlStr) {
return ( new window.DOMParser() ).parseFromString(xmlStr, "text/xml");
};
}
else if (typeof window.ActiveXObject != "undefined" && new window.ActiveXObject("Microsoft.XMLDOM")) {
parseXml = function(xmlStr) {
var xmlDoc = new window.ActiveXObject("Microsoft.XMLDOM");
xmlDoc.async = "false";
xmlDoc.loadXML(xmlStr);
return xmlDoc;
};
}
else {
parseXml = function() { return null; }
}
var xmlTree = parseXml("<root></root>");
function add_children(child_name,parent_name)
{
str = '<'+child_name+'></'+child_name+'>';
//strXML = parseXml(str);
$(xmlTree).find(parent_name).append(str);
var xmlString = (new XMLSerializer()).serializeToString(xmlTree);
console.log(xmlString);
}
add_children("apple","root");
add_children("orange","root");
add_children("lychee","root");
but console is showing <root/> all three times

The problem is that you are creating the node in the Html document, not in the XML one, so you can't append it. Change this line:
str = '<'+child_name+'></'+child_name+'>';
For this one:
str = xmlTree.createElement(child_name);
See the working code in http://jsfiddle.net/W9Wpb/

Reading xml document in firefox

I am trying to read customers.xml using javascript.
My professor has taught us to read xml using `ActiveXObjectand he has given us an assignment to create a sample login page which checks username and password by reading customers.xml.
I am trying to use DOMParser so that it works with firefox.
But when I click on Login button I get this error.
Error: syntax error Source File:
file:///C:/Users/Searock/Desktop/home/project/project/login.html
Line: 1, Column: 1 Source Code:
customers.xml
Here's my code.
login.js
var xmlDoc = 0;
function checkUser()
{
var user = document.login.txtLogin.value;
var pass = document.login.txtPass.value;
//xmlDoc = new ActiveXObject("Microsoft.XMLDOM");
/*
xmlDoc = document.implementation.createDocument("","",null);
xmlDoc.async = "false";
xmlDoc.onreadystatechange = redirectUser;
xmlDoc.load("customers.xml");
*/
var parser = new DOMParser();
xmlDoc = parser.parseFromString("customers.xml", "text/xml");
alert(xmlDoc.documentElement.nodeName);
xmlDoc.async = "false";
xmlDoc.onreadystatechange = redirectUser;
}
function redirectUser()
{
alert('');
var user = document.login.txtLogin.value;
var pass = document.login.txtPass.value;
var log = 0;
if(xmlDoc.readyState == 4)
{
xmlObj = xmlDoc.documentElement;
var len = xmlObj.childNodes.length;
for(i = 0; i < len; i++)
{
var nodeElement = xmlObj.childNodes[i];
var userXml = nodeElement.childNodes[0].firstChild.nodeValue;
var passXml = nodeElement.childNodes[1].firstChild.nodeValue;
var idXML = nodeElement.attributes[0].value
if(userXml == user && passXml == pass)
{
log = 1;
document.cookie = escape(idXML);
document.login.submit();
}
}
}
if(log == 0)
{
var divErr = document.getElementById('Error');
divErr.innerHTML = "<b>Login Failed</b>";
}
}
customers.xml
<?xml version="1.0" encoding="UTF-8"?>
<customers>
<customer custid="CU101">
<user>jack</user>
<pwd>PW101</pwd>
<email>jack#rediff.com</email>
</customer>
<customer custid="CU102">
<user>jill</user>
<pwd>PW102</pwd>
<email>jill#rediff.com</email>
</customer>
<customer custid="CU103">
<user>john</user>
<pwd>PW103</pwd>
<email>john#rediff.com</email>
</customer>
<customer custid="CU104">
<user>jeff</user>
<pwd>PW104</pwd>
<email>jeff#rediff.com</email>
</customer>
</customers>
I get parsererror message on line alert(xmlDoc.documentElement.nodeName);
I don't know what's wrong with my code. Can some one point me in a right direction?
Edit :
Ok, I found a solution.
var xmlDoc = 0;
var xhttp = 0;
function checkUser()
{
var user = document.login.txtLogin.value;
var pass = document.login.txtPass.value;
var err = "";
if(user == "" || pass == "")
{
if(user == "")
{
alert("Enter user name");
}
if(pass == "")
{
alert("Enter Password");
}
return;
}
if (window.XMLHttpRequest)
{
xhttp=new XMLHttpRequest();
}
else // IE 5/6
{
xhttp=new ActiveXObject("Microsoft.XMLHTTP");
}
xhttp.onreadystatechange = redirectUser;
xhttp.open("GET","customers.xml",true);
xhttp.send();
}
function redirectUser()
{
var log = 2;
var user = document.login.txtLogin.value;
var pass = document.login.txtPass.value;
if (xhttp.readyState == 4)
{
log = 0;
xmlDoc = xhttp.responseXML;
var xmlUsers = xmlDoc.getElementsByTagName('user');
var xmlPasswords = xmlDoc.getElementsByTagName('pwd');
var userLen = xmlDoc.getElementsByTagName('customer').length;
var xmlCustomers = xmlDoc.getElementsByTagName('customer');
for (var i = 0; i < userLen; i++)
{
var xmlUser = xmlUsers[i].childNodes[0].nodeValue;
var xmlPass = xmlPasswords[i].childNodes[0].nodeValue;
var xmlId = xmlCustomers.item(i).attributes[0].nodeValue;
if(xmlUser == user && xmlPass == pass)
{
log = 1;
document.cookie = xmlId;
document.login.submit();
break;
}
}
}
if(log == 0)
{
alert("Login failed");
}
}
Thanks.

parseFromString is parsing the string "customer.xml" in your case, because the first argument needs to be a string containing the actual content of the XML document and not its name.
You can use something like this to get the xml file:
if (window.XMLHttpRequest)
{
xhttp=new XMLHttpRequest();
}
else // IE 5/6
{
xhttp=new ActiveXObject("Microsoft.XMLHTTP");
}
xhttp.open("GET","customer.xml",false);
xhttp.send();
xmlDoc=xhttp.responseXML;
Source