Develop Reference

I can't crawl the website quotestoscrape using artoo.js Multi level recursive site crawling

I want to use artoo.js to scrape the website https://quotes.toscrape.com/
For handling pagination, I can do it! But for crawling the website ie for each page scrape the quotes and authors. Then take the link of the author and scrape the dob and pob for example. Finally handle the pagination.
Any help is much appreciated, here's my code:
var base_url = 'https://quotes.toscrape.com';
// empty list init
var my_list = []
// define the logic of the first scraper
var scraper1 = {
iterator: 'div.quote',
data: {
'quotes': {
sel: 'span'
},
'author': {
sel: 'small.author'
},
'link': {
sel: 'a',
attr: 'href'
}
}
};
// define the logic of the second scraper
var scraper2 = {
iterator: 'div.author-details',
data: {
'dob': {
sel: 'span.author-born-date'
},
'pob': {
sel: 'span.author-born-location'
}
}
}
// pagination
function nextUrl($page) {
return $page.find('li.next > a').attr('href');
}
artoo.log.debug('Starting the scraper...');
var frontpage = artoo.scrape(scraper1);
// spider
var my_list = []
// artoo spider
function pagination() {
artoo.ajaxSpider(
function(i, $data) {
//console.log($data.innerHTML);
return nextUrl(!i ? artoo.$(document) : $data);
}, {
limit: 1, // number of pages to scrape
scrape: scraper1,
concat: true,
done: function(data) {
artoo.log.debug('Finished retrieving data. Downloading...');
console.log(data);
for (var i = 0; i < my_list.length; i++) {
my_list.push(base_url + data[i].link)
}
console.log(my_list)
}
})
return my_list;
}
// Append links in a list
//my_list.push(base_url + data[0].link);
function crawl(mylist) {
artoo.ajaxSpider(
my_list, {
limit: 1, // number of pages to scrape
scrape: scraper2,
concat: true,
done: function(data) {
console.log(data);
artoo.log.debug('Finished retrieving data. Downloading...');
}
})
}
//var ll = null;
let links = pagination();
crawl(links)

Chrome extension - button listener on notification executes multiple times

I am writing a chrome extension that makes requests to an API and I have noticed that after I create a notification from background script using chrome's notification API, the listeners on the buttons from the notification are executed multiple times. on the first run only once and then increasing. I figured that the listeners just add up on the page but I couldn't find a way to sort of refresh the background page.
This is the function that creates the notification and it's listeners.
var myNotificationID
const displayNotification=(userEmail, password, website,username) =>{
chrome.notifications.create("", {
type: "basic",
iconUrl: "./icon128.png",
title: "PERMISSION",
requireInteraction: true,
message: "question",
buttons: [{
title: "YES",
}, {
title: "NO",
}]
}, function(id) {
myNotificationID = id;
})
chrome.notifications.onButtonClicked.addListener(function(notifId, btnIdx) {
if (notifId === myNotificationID) {
if (btnIdx === 0) {
console.log('inserting')
try{
fetch (`http://localhost:8080/users/${userEmail}/accounts`,{
})
}catch(err){
}
} else if (btnIdx === 1) {
console.log('clearing')
chrome.notifications.clear(myNotificationID)
}
}
});
}
And this is where the function is called
chrome.runtime.onMessage.addListener((message, sender, response)=>{
if(message.message === 'showNotification'){
console.log('received insert')
displayNotification(message.userEmail,message.password, message.currentSite,message.username)
response({status:"received"})
}
})
the fetch within the listener is executed multiple times but the log from the onMessage listener is only displayed once, so the listener is the problem here.
I tried chrome.notifications.onButtonClicked.removeListener(), but as i mentioned there was no success.
Are there any other ways in which i could clean the listeners from the background script once they are used?

Using a notification store:
const notificationsByID = {};
chrome.notifications.onButtonClicked.addListener((notifId, btnIdx) => {
// Avoid access to the notification if not registered by displayNotification
if (!notificationsByID[ notifId ]) { return null; }
if (btnIdx === 0) {
console.log('inserting')
try{
fetch (`http://localhost:8080/users/${ notificationsByID[ notifId ].userEmail }/accounts`,{ /**/ });
}catch(err){
console.log(err);
}
delete notificationsByID[ notifId ]; // Cleanup
} else if (btnIdx === 1) {
console.log('clearing')
chrome.notifications.clear(myNotificationID);
delete notificationsByID[ notifId ]; // Cleanup
}
});
chrome.notifications.onClosed.addListener((notifId) => {
if (notificationsByID[ notifId ]) { delete notificationsByID[ notifId ]; }
});
const displayNotification=(userEmail, password, website,username) =>{
chrome.notifications.create("", {
type: "basic",
iconUrl: "./icon128.png",
title: "PERMISSION",
requireInteraction: true,
message: "question",
buttons: [{ title: "YES", }, { title: "NO", }]
}, function(id) {
// Insertion
notificationsByID[ id ] = { userEmail, password, website,username };
})
}

StateParams passed through function not working using meteor

Here is my function used to retrieve data from the database depending on the parameter idCats:
this.getSubcat = function(){
//Load products on scroll.
this.subscribe('SubcatIndex', () => [ $stateParams.idCats, self.loaded ], {
onReady: function() {
Meteor.call('allSubcats', $stateParams.idCats, function(err, count) {
self.allsubcats = count;
self.limit = self.loaded;
console.log("Test Log: " + $stateParams.idCats);
self.subcats = Products.find({
catLinkID : $stateParams.idCats
},{
fields: {
_id: true,
name: true,
catLinkID: true,
idCat: true,
image: true,
listingsCount: true,
productOffersCount: true,
productSoldCount: true
}
}).fetch();
window.localStorage.setItem('subcats', JSON.stringify(self.subcats) );
self.contentLoaded = true;
self.noPosts = 'No posts available.';
$ionicLoading.hide();
return;
});
},
onStop: function(err){
if(err){
self.contentLoaded = true;
self.noPosts = "No internet connection.";
console.log(JSON.stringify(err));
return;
}
}
});
}
this.getSubcat();
When i change this line:
self.subcats = Products.find({
catLinkID : $stateParams.idCats
}
To:
self.subcats = Products.find({
catLinkID : 7 // 7 for example
}
It is working well ! But as soon as I replace it with $stateParams.idCats, I receive this message coming from the function: No posts available.
Note that there are products using the idCats: 7.
When I log it:
console.log("Test Log: " + $stateParams.idCats);
This returns the same number: Test Log: 7.
If you have any suggestion or a starting point to solve this issue, it will be welcome !
Notice that there are no error in the Console (Both server and client side).
Thank you.

Meteor Blaze order sub-documents by sub-document property

Profile:
_id: Pe0t3K8GG8,
videos: [
{id:'HdaZ8rDAmy', url:'VIDURL', rank: 2},
{id:'22vZ8mj9my', url:'VIDURL2', rank: 0},
{id:'8hyTlk8H^6', url:'VIDURL3', rank: 1},
]
The profile is displayed together with the list of videos. I have a Drag & Drop which updates the videos rank using a Server Method.
1) the database updates correctly on Drop.
2) To sort the videos Array - I declare a helper on the Profile Template and SORT the videos array based on a custom comparison function.
Template.Profile.helpers({
'videosSorted': function(){
let videos = (this.videos);
let videosSorted = videos.sort(function(a, b) {
return parseFloat(a.rank) - parseFloat(b.rank);
});
return videosSorted;
}
});
Problem:
A) In Blaze the {{#each videosSorted}} does not reactively update.
If I F5 refresh then i can see the new order.
I think the issue is because I am providing videosSorted which does not update on changes to the document in the db.
How can I make videosSorted reactive?
Update:
All related code:
Iron Router Controller - I subscribe and set the data context for the layout
ProfileController = RouteController.extend({
subscriptions: function() {
this.subscribe('profile',this.params.slug).wait();
},
data: function () {
//getting the data from the subscribed collection
return Profiles.findOne({'slug':this.params.slug});
},
})
Publication:
Meteor.publish('profile', function (slug) {
const profile = Profiles.find({"slug":slug});
if(profile){
return profile;
}
this.ready();
});
The Profile HTML template:
<template name="Profile">
<ul class="sortlist">
{{#each videosSorted}}
{{> Video}}
{{/each}}
</ul>
</template>
I am using mrt:jquery-ui - sortable function
Template.Profile.onRendered(function () {
thisTemplate = this;
this.$('.sortlist').sortable({
stop: function(e, ui) {
el = ui.item.get(0);
before = ui.item.prev().get(0);
after = ui.item.next().get(0);
if(!before) {
newRank = Blaze.getData(after).rank - 1
} else if(!after) {
newRank = Blaze.getData(before).rank + 1
}
else {
newRank = (Blaze.getData(after).rank +
Blaze.getData(before).rank) / 2
}
let queryData = {
_id: thisTemplate.data._id, //the id of the profile record
videos_objId: Blaze.getData(el).objId, //the id of the sub document to update
new_rank: newRank //the new rank to give it
};
//Update the sub document using a server side call for validation + security
Meteor.call("updateVideoPosition", queryData, function (error, result) {
if(!result){
console.log("Not updated");
}
else{
console.log("successfully updated Individual's Video Position")
}
});
}
})
});
And finally the Meteor method that does the updating
'updateVideoPosition': function (queryData){
let result = Individuals.update(
{_id: queryData._id, 'videos.objId': queryData.videos_objId },
{ $set:{ 'videos.$.rank' : queryData.new_rank } }
)
return result;
}
Note :
As i mentioned - the database updates correctly - and if i have an Incognito window open to the same page - i see the videos reactivly (magically !) switch to the new order.
The schema
const ProfileSchema = new SimpleSchema({
name:{
type: String,
}
videos: {
type: [Object],
optional:true,
},
'videos.$.url':{
type:String,
},
'videos.$.rank':{
type:Number,
decimal:true,
optional:true,
autoform: {
type: "hidden",
}
},
'videos.$.subCollectionName':{
type:String,
optional:true,
autoform: {
type: "hidden",
}
},
'videos.$.objId':{
type:String,
optional:true,
autoform: {
type: "hidden",
}
}
});

I came up with really crude solution, but I don't see other options right now. The simplest solution I can think of is to rerender template manually:
Template.Profile.onRendered(function () {
var self = this;
var renderedListView;
this.autorun(function () {
var data = Template.currentData(); // depend on tmeplate data
//rerender video list manually
if (renderedListView) {
Blaze.remove(renderedListView);
}
if (data) {
renderedListView = Blaze.renderWithData(Template.VideoList, data, self.$('.videos-container')[0]);
}
});
});
Template.VideoList.onRendered(function () {
var tmpl = this;
tmpl.$('.sortlist').sortable({
stop: function (e, ui) {
var el = ui.item.get(0);
var before = ui.item.prev().get(0);
var after = ui.item.next().get(0);
var newRank;
if (!before) {
newRank = Blaze.getData(after).rank - 1
} else if (!after) {
newRank = Blaze.getData(before).rank + 1
}
else {
newRank = (Blaze.getData(after).rank +
Blaze.getData(before).rank) / 2
}
let queryData = {
_id: tmpl.data._id, //the id of the profile record
videos_objId: Blaze.getData(el).objId, //the id of the sub document to update
new_rank: newRank //the new rank to give it
};
//Update the sub document using a server side call for validation + security
Meteor.call("updateVideoPosition", queryData, function (error, result) {
if (!result) {
console.log("Not updated");
}
else {
console.log("successfully updated Individual's Video Position")
}
});
}
});
});
Template.VideoList.helpers({
videosSorted: function () {
return this.videos.sort(function (a, b) {
return a.rank - b.rank;
});
}
});
And HTML:
<template name="Profile">
<div class="videos-container"></div>
</template>
<template name="VideoList">
<ul class="sortlist">
{{#each videosSorted}}
<li>{{url}}</li>
{{/each}}
</ul>
</template>
Reativeness was lost in your case because of JQuery UI Sortable. It doesn't know anything about Meteor's reactiveness and simply blocks template rerendering.
Probably you should consider using something more adopted for Meteor like this (I am not sure it fits your needs).

How can I remove a whole IndexedDB database from JavaScript?

How can one remove a whole IndexedDB database from JavaScript, as opposed to just an object store? I'm using the IndexedDB shim, which may use WebSQL as its backend.
I'd mainly like to know how to do this for the PhantomJS (headless) browser, although Chrome, Safari (on iPad) and IE10 are other important browsers.

As far as I can tell, one should use indexedDB.deleteDatabase:
var req = indexedDB.deleteDatabase(databaseName);
req.onsuccess = function () {
console.log("Deleted database successfully");
};
req.onerror = function () {
console.log("Couldn't delete database");
};
req.onblocked = function () {
console.log("Couldn't delete database due to the operation being blocked");
};
I can confirm that it works with PhantomJS 1.9.0 and Chrome 26.0.1410.43.

I found that the following code works OK but to see the DB removed in the Chrome Resources Tab I have had to refresh the page.
Also I found I had problems with the Chrome debug tools running while doing transactions. Makes it harder to debug but if you close it while running code the code seems to work OK.
Significant also is to set a reference to the object store when opening the page.
Obviously the delete part of the code is in the deleteTheDB method.
Code derived from example provided by Craig Shoemaker on Pluralsight.
var IndDb = {
name: 'SiteVisitInsp',
version: 1000,
instance: {},
storenames: {
inspRecords: 'inspRecords',
images: 'images'
},
defaultErrorHandler: function (e) {
WriteOutText("Error found : " + e);
},
setDefaultErrorHandler: function (request) {
if ('onerror' in request) {
request.onerror = db.defaultErrorHandler;
}
if ('onblocked' in request) {
request.onblocked = db.defaultErrorHandler;
}
}
};
var dt = new Date();
var oneInspRecord =
{
recordId: 0,
dateCreated: dt,
dateOfInsp: dt,
weatherId: 0,
timeArrived: '',
timeDeparted: '',
projectId: 0,
contractorName: '',
DIWConsultant: '',
SiteForeman: '',
NoOfStaffOnSite: 0,
FileME: '',
ObservationNotes: '',
DiscussionNotes: '',
MachineryEquipment: '',
Materials: ''
};
var oneImage =
{
recordId: '',
imgSequence: 0,
imageStr: '',
dateCreated: dt
}
var SVInsp = {
nameOfDBStore: function () { alert("Indexed DB Store name : " + IndDb.name); },
createDB: function () {
openRequest = window.indexedDB.open(IndDb.name, IndDb.version);
openRequest.onupgradeneeded = function (e) {
var newVersion = e.target.result;
if (!newVersion.objectStoreNames.contains(IndDb.storenames.inspRecords)) {
newVersion.createObjectStore(IndDb.storenames.inspRecords,
{
autoIncrement: true
});
}
if (!newVersion.objectStoreNames.contains(IndDb.storenames.images)) {
newVersion.createObjectStore(IndDb.storenames.images,
{
autoIncrement: true
});
}
};
openRequest.onerror = openRequest.onblocked = 'Error'; //resultText;
openRequest.onsuccess = function (e) {
//WriteOutText("Database open");
IndDb.instance = e.target.result;
};
},
deleteTheDB: function () {
if (typeof IndDb.instance !== 'undefined') {
//WriteOutText("Closing the DB");
IndDb.instance.close();
var deleteRequest = indexedDB.deleteDatabase(IndDb.name)
deleteRequest.onblocked = function () {
console.log("Delete blocked.");
}
deleteRequest.onerror =
function () {
console.log("Error deleting the DB");
//alert("Error deleting the DB");
};
//"Error deleting the DB";
deleteRequest.onsuccess = function () {
console.log("Deleted OK.");
alert("*** NOTE : Requires page refresh to see the DB removed from the Resources IndexedDB tab in Chrome.");
//WriteOutText("Database deleted.");
};
};
}
}