Develop Reference

I can't crawl the website quotestoscrape using artoo.js Multi level recursive site crawling

I want to use artoo.js to scrape the website https://quotes.toscrape.com/
For handling pagination, I can do it! But for crawling the website ie for each page scrape the quotes and authors. Then take the link of the author and scrape the dob and pob for example. Finally handle the pagination.
Any help is much appreciated, here's my code:
var base_url = 'https://quotes.toscrape.com';
// empty list init
var my_list = []
// define the logic of the first scraper
var scraper1 = {
iterator: 'div.quote',
data: {
'quotes': {
sel: 'span'
},
'author': {
sel: 'small.author'
},
'link': {
sel: 'a',
attr: 'href'
}
}
};
// define the logic of the second scraper
var scraper2 = {
iterator: 'div.author-details',
data: {
'dob': {
sel: 'span.author-born-date'
},
'pob': {
sel: 'span.author-born-location'
}
}
}
// pagination
function nextUrl($page) {
return $page.find('li.next > a').attr('href');
}
artoo.log.debug('Starting the scraper...');
var frontpage = artoo.scrape(scraper1);
// spider
var my_list = []
// artoo spider
function pagination() {
artoo.ajaxSpider(
function(i, $data) {
//console.log($data.innerHTML);
return nextUrl(!i ? artoo.$(document) : $data);
}, {
limit: 1, // number of pages to scrape
scrape: scraper1,
concat: true,
done: function(data) {
artoo.log.debug('Finished retrieving data. Downloading...');
console.log(data);
for (var i = 0; i < my_list.length; i++) {
my_list.push(base_url + data[i].link)
}
console.log(my_list)
}
})
return my_list;
}
// Append links in a list
//my_list.push(base_url + data[0].link);
function crawl(mylist) {
artoo.ajaxSpider(
my_list, {
limit: 1, // number of pages to scrape
scrape: scraper2,
concat: true,
done: function(data) {
console.log(data);
artoo.log.debug('Finished retrieving data. Downloading...');
}
})
}
//var ll = null;
let links = pagination();
crawl(links)

Select2 group with Infinite scroll

I am using select2 group option with infinite scroll and data are coming by Ajax calling per page 10. Here is some problem arises, suppose user 1 has 15 data and user 2 has 15 data, at first 10 data are coming from user 1 and in next page 10 (5 data for user1 and 5 data for user2). no problem for data getting but the problem is user 1 group showing double. How can I prevent double display to my select2 options group? Has there any way to make an option group again?
HTML CODE
<div class="container">
<form id="frm">
<h1>Solution 1</h1>
<div class="row">
<div class="col-4">
<div class="form-group">
<label for="tagInput">Get data by ajax calling</label>
<select class="form-control" id="pictures_tag_input">
</select>
<small class="form-text text-muted"><p class="text-info">Infinite Scroll</p></small>
</div>
</div>
</div>
</form>
</div>
JS CODE
$(document).ready(function() {
// solution 1
//example.com/articles?page[number]=3&page[size]=1
http: $("#pictures_tag_input").select2({
placeholder: "Search for options",
ajax: {
url: "https://jsonplaceholder.typicode.com/users/1/todos",
dataType: "json",
global: false,
cache: true,
delay: 250,
minimumInputLength: 2,
data: function(params) {
// console.log(params.page || 1);
return {
q: params.term, // search term
_page: params.page || 1,
_limit: 10 // page size
};
},
processResults: function(data, params) {
params.page = params.page || 1;
var datx = getNestedChildren(data);
// console.log(datx);
return {
results: datx,
pagination: {
more: true
}
};
} //end of process results
} // end of ajax
});
function getNestedChildren(list) {
var roots = [];
for (i = 0; i < list.length; i += 1) {
node = list[i];
if (roots.length === 0) {
var obj = {
text: "User " + node.userId,
children: [{ id: node.id, text: node.title }]
};
roots.push(obj);
} else {
var obj = {
text: "User " + node.userId,
children: [{ id: node.id, text: node.title }]
};
var rootArray = $.map(roots, function(val, i) {
var vl = "User " + node.userId;
if (val.text === vl) return val;
else return undefined;
});
if (rootArray.length > 0) {
var obj1 = {
id: node.id,
text: node.title
};
rootArray[0].children.push(obj1);
} else {
roots.push(obj);
}
}
}
return roots;
}
});
Demo
https://codepen.io/mdshohelrana/pen/MLVZEG

Just try to use the following code
templateResult: function(data) {
if (typeof data.children != 'undefined') {
$(".select2-results__group").each(function() {
if (data.text == $(this).text()) {
return data.text = '';
}
});
}
return data.text;
}
NOTE: Need to group from server side, Other wise you have to make master details from client side.

the accepted answer didn't work for me and I don't see why should that work. A return in the $.each will not return from the templateResult() function.
Here is an approach that worked for me.
It is not necessary to build a nested list by getNestedChildren(list) on javascript side. It is much easier to build it on server side instead.
The appearance of search results in the dropdown (options and the optgroups) can be customized by using the templateResult option. I removed the duplicated optgroups and labels by this option.
check the templateResult: formatOptions, part of the code
$(document).ready(function() {
$("#pictures_tag_input").select2({
placeholder: "Search for options",
templateResult: formatOptions,
ajax: {
url: "https://jsonplaceholder.typicode.com/users/1/todos",
dataType: "json",
global: false,
cache: true,
delay: 250,
minimumInputLength: 2,
data: function(params) {
return {
q: params.term,
_page: params.page || 1,
_limit: 10
};
},
processResults: function(data, params) {
params.page = params.page || 1;
return {
results: data,
pagination: {
more: true
}
};
} //end of process results
} // end of ajax
});
function formatOptions(item, container, $el) {
// optgroups section
if (item.children && item.children.length > 0) {
// don't format the repeated optgroups!
if ($(".select2-results__group").text() === item.text) {
return;
}
if ($('[aria-label="' + item.text + '"]').length > 0) {
return;
}
// the first occasion of the given optgroup
return $el;
}
// options section
// here you can implement your own logic
// if you want to customise the output of the options
$el.addClass('something-special-result result');
return $el;
}
});

maybe the problem is a source of a data
You call user 1 .... server return a 1
You call user 2 .... server return a 1
You call user 3 .... server return a 2
You call user 4 .... server return a 2
You call user 5 .... server return a 3
You call user 6 .... server return a 3
curent_user = 1;
$(document).ready(function() {
http: $("#pictures_tag_input").select2({
placeholder: "Search for options",
ajax: {
url: "https://jsonplaceholder.typicode.com/users/1/todos",
dataType: "json",
global: false,
cache: false,
minimumInputLength: 2,
data: function(params) {
console.log("params",params || 1);
return {
q: params.term, // search term
_page: curent_user,
_limit: 10 // page size
};
},
processResults: function(data, params) {
curent_user += 2;
var datx = getNestedChildren(data);
console.log("data: ", data);
return {
results: datx,
pagination: {
more: true
}
};
} //end of process results
} // end of ajax
});
function getNestedChildren(list) {
var roots = [];
for (i = 0; i < list.length; i += 1) {
node = list[i];
if (roots.length === 0) {
var obj = {
text: "User " + node.userId,
children: [{ id: node.id, text: node.title }]
};
roots.push(obj);
} else {
var obj = {
text: "User " + node.userId,
children: [{ id: node.id, text: node.title }]
};
var rootArray = $.map(roots, function(val, i) {
var vl = "User " + node.userId;
if (val.text === vl) return val;
else return undefined;
});
if (rootArray.length > 0) {
var obj1 = {
id: node.id,
text: node.title
};
rootArray[0].children.push(obj1);
} else {
roots.push(obj);
}
}
}
return roots;
}
});
so if you skip a one step
You call user 1 .... server return a 1
You call user 3 .... server return a 2
You call user 5 .... server return a 3

I just found a better solution which does not result in a (duplicated) optgroup being rendered as an empty option:
processResults: function( json, params ){
setTimeout( function() {
var $prevOptions = false;
var $prevGroup = false;
// loop
$('.select2-results__option[role="group"]').each(function(){
// vars
var $options = $(this).children('ul');
var $group = $(this).children('strong');
// compare to previous
if( $prevGroup && $prevGroup.text() === $group.text() ) {
$prevOptions.append( $options.children() );
$(this).remove();
return;
}
// update vars
$prevOptions = $options;
$prevGroup = $group;
});
}, 1 );
return json;
}
Advanced Custom Fields uses the exact same code for their WordPress plugin in order to fix this issue, ajax-load and group posts from different post-types.

How to dynamically increment the limit on a collection cursor

I'm currently trying to implement a "Load more" button on a simple news page.
I've got the search working, thanks to Meteor Chef, and I followed the same principles to implement a "Load more" button. The relevant code is as follows:
Template:
Template.listAllNews.onCreated(function() {
template.limit = new ReactiveVar(10);
Tracker.autorun( () => {
template.subscribe( 'news.all', template.searchQuery.get(), template.limit.get(), () => {
setTimeout( () => {
template.searching.set( false );
}, 300 );
});
});
Template.listAllNews.events({
'click .load-more-button': function (event, template) {
var limit = template.limit.get();
limit += 10;
template.limit.set( limit );
}
});
Publication:
let query = {},
projection = { limit: limit, sort: { submitted: -1 } };
if ( search ) {
let regex = new RegExp( search, 'i' );
query = {
$or: [
{ title: regex }
]
};
projection.limit = 100;
}
return News.find( query, projection );
This works. The problem is: it refreshes all the data instead of just loading the extra 10 news articles. I want it to only load the extra 10 articles upon click, and not to "refresh" the subscription.
How can I do that?

Upload images associated to a meteor collection

I'm having a hard time understanding the whole process of uploading images to a certain Meteor collection eg.(the belongs_to and has_one association with rails).
I have a portfolioitem collection, this is the file:
PortfolioItems = new Mongo.Collection('portfolioItems');
ownsDocument = function(userId, doc) {
return doc && doc.userId === userId;
}
PortfolioItems.allow({
update: function(userId, portfolioItem) { return ownsDocument(userId, portfolioItem); },
remove: function(userId, portfolioItem) { return ownsDocument(userId, portfolioItem); },
});
Meteor.methods({
portfolioItemInsert: function(portfolioItemAttributes) {
check(Meteor.userId(), String);
check(portfolioItemAttributes, {
title: String
});
var portfolioItemWithSameTitle = PortfolioItems.findOne({ title: portfolioItemAttributes.title});
if (portfolioItemWithSameTitle) {
return {
portfolioItemExists: true,
_id: portfolioItemWithSameTitle._id
}
}
var user = Meteor.user();
var portfolioItem = _.extend(portfolioItemAttributes, {
userId: user._id,
submitted: new Date()
});
var portfolioItemId = PortfolioItems.insert(portfolioItem);
return {
_id: portfolioItemId
};
}
});
This is the submit.js template for submitting portfolio items:
Template.submit.events({
'submit #submit-form': function(e) {
e.preventDefault();
var portfolioItem = {
title: $(e.target).find('#submit-title').val()
};
Meteor.call('portfolioItemInsert', portfolioItem, function(error, result) {
if (error) {
return alert(error.reason);
}
if(result.portfolioItemExists) {
alert('Title already taken!');
pause();
}
Router.go('portfolioItemPage', {_id: result._id});
});
}
});

Did you give a try to FSCollection? if not i think its a good option to accomplish this.
You can just declare the collection.
I Suggest you to use GridFS.
just run this 2 commands
meteor add cfs:standard-packages
meteor add cfs:gridfs
Declare the collections like any others.
Images = new FS.Collection("Images", {
stores: [new FS.Store.GridFS("Images")]
});
And you can associate the Simple collection with the FSCollection using metadata.
Template.exampe.events({
'click #addImage':function(){
var file = $('#inputPng').get(0).files[0],
fsFile = new FS.File(file);
fsFile.metadata = {
ownerId:Meteor.userId(),
title:$(e.target).find('#submit-title').val()
}
Images.insert(fsFile,function(err,result){
if(!err){
console.log(result)
}
})
}
})
At this moment the README on the fsCollection its empty so I made a little DEMO about this.

Jquery Context Menu ajax fetch menu items

I have a jquery context menu on my landing page where I have hardcode menu items. Now I want to get the menu items from server. Basically the idea is to show file names in a specified directory in the context menu list and open that file when user clicks it...
This is so far I have reached..
***UPDATE***
C# code
[HttpPost]
public JsonResult GetHelpFiles()
{
List<Manuals> manuals = null;
var filesPath = Server.MapPath(#"\HelpManuals");
var standardPath = new DirectoryInfo(filesPath);
if (standardPath.GetFiles().Any())
{
manuals = standardPath.GetFiles().Select(x => new Manuals
{
Name = GetFileNamewithoutExtension(x.Name),
Path = x.Name
}).ToList();
}
return Json(manuals, JsonRequestBehavior.AllowGet);
}
private string GetFileNamewithoutExtension(string filename)
{
var extension = Path.GetExtension(filename);
return filename.Substring(0, filename.Length - extension.Length);
}
JavaScript Code
$.post("/Home/GetHelpFiles", function (data) {
$.contextMenu({
selector: '#helpIcon',
trigger: 'hover',
delay: 300,
build: function($trigger, e) {
var options = {
callback: function(key) {
window.open("/HelpManuals/" + key);
},
items: {}
};
$.each(data, function (item, index) {
console.log("display name:" + index.Name);
console.log("File Path:" + index.Path);
options.items[item.Value] = {
name: index.Name,
key: index.Path
}
});
}
});
});
Thanks to Matt. Now, the build function gets fire on hover.. but im getting illegal invocation... and when iterating through json result, index.Name and this.Name gives correct result. But item.Name doesn't give anything..

to add items to the context menu dynamically you need to make a couple changes
$.contextMenu({
selector: '#helpIcon',
trigger: 'hover',
delay: 300,
build: function($trigger, e){
var options = {
callback: function (key) {
var manual;
if (key == "adminComp") {
manual = "AdminCompanion.pdf";
} else {
manual = "TeacherCompanion.pdf";
}
window.open("/HelpManuals/" + manual);
},
items: {}
}
//how to populate from model
#foreach(var temp in Model.FileList){
<text>
options.items[temp.Value] = {
name: temp.Name,
icon: 'open'
}
</text>
}
//should be able to do an ajax call here but I believe this will be called
//every time the context is triggered which may cause performance issues
$.ajax({
url: '#Url.Action("Action", "Controller")',
type: 'get',
cache: false,
async: true,
contentType: "application/json; charset=utf-8",
dataType: "json",
success: function (_result) {
if (_result.Success) {
$.each(_result, function(item, index){
options.items[item.Value] = {
name: item.Name,
icon: 'open'
}
});
}
});
return options;
}
});
so you use build and inside of that define options and put your callback in there. The items defined in there is empty and is populated in the build dynamically. We build our list off of what is passed through the model but I believe you can put the ajax call in the build like I have shown above. Hopefully this will get you on the right track at least.

I solved this problem the following way.
On a user-triggered right-click I return false in the build-function. This will prevent the context-menu from opening. Instead of opeing the context-menu I start an ajax-call to the server to get the contextMenu-entries.
When the ajax-call finishes successfully I create the items and save the items on the $trigger in a data-property.
After saving the menuItems in the data-property I open the context-menu manually.
When the build-function is executed again, I get the items from the data-property.
$.contextMenu({
build: function ($trigger, e)
{
// check if the menu-items have been saved in the previous call
if ($trigger.data("contextMenuItems") != null)
{
// get options from $trigger
var options = $trigger.data("contextMenuItems");
// clear $trigger.data("contextMenuItems"),
// so that menuitems are gotten next time user does a rightclick
// from the server again.
$trigger.data("contextMenuItems", null);
return options;
}
else
{
var options = {
callback: function (key)
{
alert(key);
},
items: {}
};
$.ajax({
url: "GetMenuItemsFromServer",
success: function (response, status, xhr)
{
// for each menu-item returned from the server
for (var i = 0; i < response.length; i++)
{
var ri = response[i];
// save the menu-item from the server in the options.items object
options.items[ri.id] = ri;
}
// save the options on the table-row;
$trigger.data("contextMenuItems", options);
// open the context-menu (reopen)
$trigger.contextMenu();
},
error: function (response, status, xhr)
{
if (xhr instanceof Error)
{
alert(xhr);
}
else
{
alert($($.parseHTML(response.responseText)).find("h2").text());
}
}
});
// This return false here is important
return false;
}
});

I have finally found a better solution after reading jquery context menu documentation, thoroughly..
C# CODE
public JsonResult GetHelpFiles()
{
List<Manuals> manuals = null;
var filesPath = Server.MapPath(#"\HelpManuals");
var standardPath = new DirectoryInfo(filesPath);
if (standardPath.GetFiles().Any())
{
manuals = standardPath.GetFiles().Select(x => new Manuals
{
Name = GetFileNamewithoutExtension(x.Name),
Path = x.Name
}).ToList();
}
return Json(manuals, JsonRequestBehavior.AllowGet);
}
HTML 5
<div id="dynamicMenu">
<menu id="html5menu" type="context" style="display: none"></menu>
</div>
JavaScript Code
$.post("/Home/GetHelpFiles", function (data) {
$.each(data, function (index, item) {
var e = '<command label="' + item.Name + '" id ="' + item.Path + '"></command>';
$("#html5menu").append(e);
});
$.contextMenu({
selector: '#helpIcon',
trigger: 'hover',
delay: 300,
items: $.contextMenu.fromMenu($('#html5menu'))
});
});
$("#dynamicMenu").on("click", "menu command", function () {
var link = $(this).attr('id');
window.open("/HelpManuals/" + link);
});

Here's my solution using deferred, important to know that this feature is supported for sub-menus only
$(function () {
$.contextMenu({
selector: '.SomeClass',
build: function ($trigger, e) {
var options = {
callback: function (key, options) {
// some call back
},
items: JSON.parse($trigger.attr('data-storage')) //this is initial static menu from HTML attribute you can use any static menu here
};
options.items['Reservations'] = {
name: $trigger.attr('data-reservations'),
icon: "checkmark",
items: loadItems($trigger) // this is AJAX loaded submenu
};
return options;
}
});
});
// Now this function loads submenu items in my case server responds with 'Reservations' object
var loadItems = function ($trigger) {
var dfd = jQuery.Deferred();
$.ajax({
type: "post",
url: "/ajax.php",
cache: false,
data: {
// request parameters are not importaint here use whatever you need to get data from your server
},
success: function (data) {
dfd.resolve(data.Reservations);
}
});
return dfd.promise();
};