Develop Reference

I can't crawl the website quotestoscrape using artoo.js Multi level recursive site crawling

I want to use artoo.js to scrape the website https://quotes.toscrape.com/
For handling pagination, I can do it! But for crawling the website ie for each page scrape the quotes and authors. Then take the link of the author and scrape the dob and pob for example. Finally handle the pagination.
Any help is much appreciated, here's my code:
var base_url = 'https://quotes.toscrape.com';
// empty list init
var my_list = []
// define the logic of the first scraper
var scraper1 = {
iterator: 'div.quote',
data: {
'quotes': {
sel: 'span'
},
'author': {
sel: 'small.author'
},
'link': {
sel: 'a',
attr: 'href'
}
}
};
// define the logic of the second scraper
var scraper2 = {
iterator: 'div.author-details',
data: {
'dob': {
sel: 'span.author-born-date'
},
'pob': {
sel: 'span.author-born-location'
}
}
}
// pagination
function nextUrl($page) {
return $page.find('li.next > a').attr('href');
}
artoo.log.debug('Starting the scraper...');
var frontpage = artoo.scrape(scraper1);
// spider
var my_list = []
// artoo spider
function pagination() {
artoo.ajaxSpider(
function(i, $data) {
//console.log($data.innerHTML);
return nextUrl(!i ? artoo.$(document) : $data);
}, {
limit: 1, // number of pages to scrape
scrape: scraper1,
concat: true,
done: function(data) {
artoo.log.debug('Finished retrieving data. Downloading...');
console.log(data);
for (var i = 0; i < my_list.length; i++) {
my_list.push(base_url + data[i].link)
}
console.log(my_list)
}
})
return my_list;
}
// Append links in a list
//my_list.push(base_url + data[0].link);
function crawl(mylist) {
artoo.ajaxSpider(
my_list, {
limit: 1, // number of pages to scrape
scrape: scraper2,
concat: true,
done: function(data) {
console.log(data);
artoo.log.debug('Finished retrieving data. Downloading...');
}
})
}
//var ll = null;
let links = pagination();
crawl(links)

Cannot reload data in Fuelux Datagrid

I have tried to reload the data populated by an ajax call but I cant get it to work, it shows the old data even after using the reload method. The thing is that if I change some variables to populate a different data and try to call the following code without refreshing the page it does not reload the updated data =/ Here is my code:
function populateDataGrid() {
$.ajaxSetup({async: false});
var gridinfo="";
$.post("lib/function.php",{activity: activity, shift: shift, date: date},
function (output){
gridinfo = JSON.parse(output);
});
$.ajaxSetup({async: true});
// INITIALIZING THE DATAGRID
var dataSource = new StaticDataSource({
columns: [
{
property: 'id',
label: '#',
sortable: true
},
{
property: 'date',
label: 'date',
sortable: true
},
....
],
formatter: function (items) {
var c=1;
$.each(items, function (index, item) {
item.select = '<input type="button" id="select'+c+'" class="select btn" value="select" onclick="">';
c=c+1;
});
},
data: gridinfo,
delay:300
});
$('#grid').datagrid({
dataSource: dataSource
});
$('#grid').datagrid('reload');
$('#modal-fast-appointment-results').modal({show:true});
}

I found a solution... I had to create a new DataSource (lets call it "AjaxDataSource") and add the ajax request functionality within the data constructor:
(function (root, factory) {
if (typeof define === 'function' && define.amd) {
define(['underscore'], factory);
} else {
root.AjaxDataSource = factory();
}
}(this, function () {
var AjaxDataSource = function (options) {
this._formatter = options.formatter;
this._columns = options.columns;
this._delay = options.delay || 0;
this._data = options.data;
};
AjaxDataSource.prototype = {
columns: function () {
return this._columns;
},
data: function (options, callback) {
var self = this;
setTimeout(function () {
var data;
$.ajax({
url: 'getdata.php',
type: 'POST',
data: 'param1:param1,param2,param2,...,paramN:paramN', // this is optional in case you have to send some params to getdata.php
dataType: 'json',
async: false,
success: function(result) {
data = result;
},
error: function(data){
//in case we want to debug and catch any possible error
// console.log(data);
}
});
// SEARCHING
if (options.search) {
data = _.filter(data, function (item) {
var match = false;
_.each(item, function (prop) {
if (_.isString(prop) || _.isFinite(prop)) {
if (prop.toString().toLowerCase().indexOf(options.search.toLowerCase()) !== -1) match = true;
}
});
return match;
});
}
var count = data.length;
// SORTING
if (options.sortProperty) {
data = _.sortBy(data, options.sortProperty);
if (options.sortDirection === 'desc') data.reverse();
}
// PAGING
var startIndex = options.pageIndex * options.pageSize;
var endIndex = startIndex + options.pageSize;
var end = (endIndex > count) ? count : endIndex;
var pages = Math.ceil(count / options.pageSize);
var page = options.pageIndex + 1;
var start = startIndex + 1;
data = data.slice(startIndex, endIndex);
if (self._formatter) self._formatter(data);
callback({ data: data, start: start, end: end, count: count, pages: pages, page: page });
}, this._delay)
}
};
return AjaxDataSource;
}));
After defining the new DataSource, we just need to create it and call the datagrid as usual:
function populateDataGrid(){
// INITIALIZING THE DATAGRID
var dataSource = new AjaxDataSource({
columns: [
{
property: 'id',
label: '#',
sortable: true
},
{
property: 'date',
label: 'date',
sortable: true
},
....
],
formatter: function (items) { // in case we want to add customized items, for example a button
var c=1;
$.each(items, function (index, item) {
item.select = '<input type="button" id="select'+c+'" class="select btn" value="select" onclick="">';
c=c+1;
});
},
delay:300
});
$('#grid').datagrid({
dataSource: dataSource
});
$('#grid').datagrid('reload');
$('#modal-results').modal({show:true});
}
So now we have our datagrid with data populated via ajax request with the ability to reload the data without refreshing the page.
Hope it helps someone!

Exporting all data from Kendo Grid datasource

I followed that tutorial about exporting Kendo Grid Data : http://www.kendoui.com/blogs/teamblog/posts/13-03-12/exporting_the_kendo_ui_grid_data_to_excel.aspx
Now I´m trying to export all data (not only the showed page) ... How can I do that?
I tried change the pagezise before get the data:
grid.dataSource.pageSize(grid.dataSource.total());
But with that my actual grid refresh with new pageSize. Is that a way to query kendo datasource without refresh the grid?
Thanks

A better solution is to generate an Excel file from the real data, not from the dataSource.
1]
In the html page, add
$('#export').click(function () {
var title = "EmployeeData";
var id = guid();
var filter = $("#grid").data("kendoGrid").dataSource._filter;
var data = {
filter: filter,
title: title,
guid: id
};
$.ajax({
url: '/Employee/Export',
type: "POST",
dataType: 'json',
data: JSON.stringify(data),
contentType: "application/json; charset=utf-8",
success: function (result) {
window.location = kendo.format("{0}?title={1}&guid={2}", '/Employee/GetGeneratedExcel', title, id);
}
});
});
2]
Add a method "Export" to the controller:
[HttpPost]
public JsonResult Export(KendoGridFilter filter, string guid)
{
var gridRequest = new KendoGridRequest();
if (filter != null)
{
gridRequest.FilterObjectWrapper = filter.Filters != null ? filter.ToFilterObjectWrapper() : null;
gridRequest.Logic = filter.Logic;
}
var query = GetQueryable().AsNoTracking();
var results = query.FilterBy<Employee, EmployeeVM>(gridRequest);
using (var stream = new MemoryStream())
{
using (var excel = new ExcelPackage(stream))
{
excel.Workbook.Worksheets.Add("Employees");
var ws = excel.Workbook.Worksheets[1];
ws.Cells.LoadFromCollection(results);
ws.Cells.AutoFitColumns();
excel.Save();
Session[guid] = stream.ToArray();
return Json(new { success = true });
}
}
}
3]
Also add the method "GetGeneratedExcel" to the controller:
[HttpGet]
public FileResult GetGeneratedExcel(string title, string guid)
{
// Is there a spreadsheet stored in session?
if (Session[guid] == null)
{
throw new Exception(string.Format("{0} not found", title));
}
// Get the spreadsheet from session.
var file = Session[guid] as byte[];
string filename = string.Format("{0}.xlsx", title);
// Remove the spreadsheet from session.
Session.Remove(title);
// Return the spreadsheet.
Response.Buffer = true;
Response.AddHeader("Content-Disposition", string.Format("attachment; filename={0}", filename));
return File(file, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", filename);
}
Also see this project on github.
See this live example project where you can export the Employees to Excel. (Although this returns filtered data, but you can modify the code to ignore the kendo grid filter and always return all data.

Really old question but:
To export all the pages use excel.allPages:
$("#grid").kendoGrid({
toolbar: ["excel"],
excel: {
allPages: true
},
// ....
});
See Example

Grid toolbar
..
.ToolBar(toolbar =>
{
toolbar.Template(
#<text>
#Html.Kendo().Button().Name("grid-export").HtmlAttributes(new { type = "button", data_url = #Url.Action("Export") }).Content("Export").Events(ev => ev.Click("exportGrid"))
</text>);
})
..
Endpoint export function
public FileResult Export([DataSourceRequest]DataSourceRequest request)
{
DemoEntities db = new DemoEntities();
byte[] bytes = WriteExcel(db.Table.ToDataSourceResult(request).Data, new string[] { "Id", "Name" });
return File(bytes,
"application/vnd.ms-excel",
"GridExcelExport.xls");
}
a javascript function to generate grid remote export url with all specified parameters
function exportGrid() {
var toolbar = $(this.element);
var gridSelector = toolbar.closest(".k-grid");
var grid = $(gridSelector).data("kendoGrid");
var url = toolbar.data("url");
var requestObject = (new kendo.data.transports["aspnetmvc-server"]({ prefix: "" }))
.options.parameterMap({
page: grid.dataSource.page(),
sort: grid.dataSource.sort(),
filter: grid.dataSource.filter()
});
url = url + "?" + $.param({
"page": requestObject.page || '~',
"sort": requestObject.sort || '~',
"pageSize": grid.dataSource.pageSize(),
"filter": requestObject.filter || '~',
});
window.open(url, '_blank');
}
For detailed solution see my sample project on Github
where u can export grid server side with current configuration (sorting, filtering, paging) using helper function

Knockout Complex Data Model Binding

Let's start with the code..
Javascript to bind viewmodel and display dialog
$("#add-song").click(function () {
$("<div>")
.attr("data-bind", "template: { name: 'manage-song-template' }")
.dialog({
resizable: false,
modal: true,
title: "Add Song",
width: 960,
height: 490,
buttons: [
{
text: "Create Song",
'data-bind': 'click: savechanges',
click: function () {
}
},
{
text: "Close",
click: function () {
$(this).dialog("close");
}
}
],
close: function () {
$(this).dialog('destroy').remove();
},
open: function () {
ko.applyBindings(new my.managesongviewmodel());
jquerybindings();
}
});
});
Now let's take a look at the view model
my.managesongviewmodel = function () {
var
//Scalar Properties
song = null,
//Functions
loadfromserver = function (Id) {
$.post("/song/getsong", "id=1", function (response) {
if (response.Success) {
var data = response.Data;
var song = response.Data.Song;
var songdata = {
Song: {
Id: song.Id,
Title: song.Title,
Description: song.Description,
Lyrics: song.Lyrics,
isMaster: song.isMaster,
AudioFilePath: song.AudioFilePath,
CoverImageFilePath: song.CoverImageFilePath,
CreatedDate: song.CreatedDate
},
Genres: data.Genres,
SongAlternateTitles: data.SongAlternateTitles,
Exploitations: data.Exploitations,
SongWriterSongs: data.SongWriterSongs
};
song = new my.song(songdata);
alert(song.title());
}
});
},
savechanges = function () {
};
loadfromserver();
return {
song: song,
loadfromserver: loadfromserver
};
};
my.song = function (data) {
var
//Scalar Properties
id = ko.observable(data.Song.Id),
title = ko.observable(data.Song.Title),
description = ko.observable(data.Song.Description),
lyrics = ko.observable(data.Song.Lyrics),
ismaster = ko.observable(data.Song.isMaster),
audiofilepath = ko.observable(data.Song.AudioFilePath),
coverimagefilepath = ko.observable(data.Song.CoverImageFilePath),
createddate = ko.observable(data.Song.CreatedDate),
//Arrays
genres = ko.observableArray(data.Genres),
alttitles = ko.observableArray(data.SongAlternateTitles),
exploitations = ko.observableArray(data.Exploitations),
songwritersongs = ko.observableArray(data.SongWriterSongs);
return {
id: id,
title: title,
description: description,
lyrics: lyrics,
ismaster: ismaster,
audiofilepath: audiofilepath,
coverimagefilepath: coverimagefilepath,
createddate: createddate,
genres: genres,
alttitles: alttitles,
exploitations: exploitations,
songwritersongs: songwritersongs
};
};
Here is my template
<script type="text/html" id="manage-song-template">
<div class="modal-form">
<span data-bind="text: song.title"></span>
</div>
</script>
Chrome is throwing the error "Cannot read property 'title' of null;" when I launch the dialog. That said, the "alert(song.title());" actually shows a the song title.
Any thoughts on where I've gone wrong?
Update
I have created a jsfiddle that replicates the issue. http://jsfiddle.net/mcottingham/H7jqa/28/

It's easy. In the moment when you shows modal window your song var still equals null. So you have to wait while info about song will be loaded from server:
$("<div>").attr("data-bind", "template: { name: 'manage-song-template' }").dialog({
// another options
open: function (event, ui) {
var vm = new my.managesongviewmodel(), domNode = this;
vm.loadfromserver().done(function() {
ko.applyBindings(vm, domNode);
jquerybindings();
});
}
})
// another code
And add return statement at the begining of loadfromserver function:
loadfromserver = function (Id) {
return $.post("/song/getsong", "id=1", function (response) {
// code
});
}

Defining multiple instances of a jquery ui widget on a single page

I am developing a website where I use a custom build jQuery widget to load data into multiple divs.
This is the code for the widget:
(function ($, window, document, undefined) {
$.widget ("my.contentloader", {
options: {
loading_message: true
},
_create: function () {
var that = this;
$.ajax ({
type: "POST",
url: that.options.url,
data: {data: that.options.formdata, limit: that.options.limit, offset: that.options.offset},
beforeSend: function (html) {
if (that.options.loading_message) {
$(that.options.target_area).html ("<div id='loading'>Loading</div>");
}
},
success: function (html) {
if (that.options.loading_message) {
$('#loading').remove ();
}
$(that.options.target_area).html (html);
},
error: function (html) {
$(that.options.error_area).html (html);
}
});
},
_setOption: function (key, value) {
this.options[key] = value;
$.Widget.prototype._setOption.apply (this, arguments);
}
});
})(jQuery, window, document);
I load data using the widget like this:
$('#targetdiv').contentloader ({
url: '<?php echo $action_url; ?>',
target_area: '#popup_box',
formdata: {'username' : username_email, 'password' : password}
});
I am having problems loading multiple instances on the same page.
Is there a way to not instantiate the widget on a specific div like this?
$('#targetdiv').contentloader

I think you need to assign each instance to a variable. That way, you can control each instance, or write a function that iterates over an array of instances.
var contentLoaders = [];
$('.target-div').each(function(i, data) {
contentLoaders[i] = $.widget("my.contentloader", { ... });
});
So then you should be able to operate on each loader independently, like:
for (var i in contentLoaders) {
var contentLoader = contentLoaders[i];
contentLoader.option( ... );
}
Also, you're using the DOM ID $('#loading') for multiple instances of the widget. This is wrong. You need to either use separate loaders for each widget, or else check to see if the ID exists and only insert the new node if it doesn't exist. And same for removing it.
** I've added this example block, hope it helps: **
//
// This is a way to do it if you want to explicitly define each contentloader.
// Below that, I'll write out a way to define the contentloaders in a loop.
//
var contentLoader1 = $('#targetdiv1').contentloader ({
url: '<?php echo $action_url; ?>',
target_area: '#popup_box',
formdata: {'username' : username_email, 'password' : password}
});
contentLoader1.option('url', 'http://google.com');
var contentLoader2 = $('#targetdiv2').contentloader ({
url: '<?php echo $action_url; ?>',
target_area: '#popup_box',
formdata: {'username' : username_email, 'password' : password}
});
contentLoader2.option('url', 'http:/apple.com');
// Push each widget instance into an array of widget objects
var contentLoaders = [];
contentLoaders.push(contentLoader1);
contentLoaders.push(contentLoader2);
for (var i in contentLoaders) {
console.log(i, contentLoaders[i].option('url'));
}
// Should print:
// 0 http://google.com
// 1 http://apple.com
//
//
// How to set a bunch of widgets at once from an array of content loader data
//
//
var contentLoaderData = [
{
divid: '#targetDiv1',
url: 'google.com',
formdata: {
username: 'joeshmo',
password: 'joeshmo1'
}
},
{
divid: '#targetDiv2',
url: 'apple.com',
formdata: {
username: 'plainjane',
password: 'plainjane1'
}
}
];
// Array of widget instances
var contentLoaders = [];
$.each(contentLoaderData, function(index, value) {
var contentLoader = $(this.divid).contentloader({
url: this.url,
target_area: '#popup_box',
formdata: {'username' : this.formdata.username, 'password' : this.formdata.password}
});
// Push each contentLoader instance into the contentLoaders array
contentLoaders.push(contentLoader);
});
for (var i in contentLoaders) {
console.log(i, contentLoaders[i].option('url'));
}
// Should print:
// 0 http://google.com
// 1 http://apple.com