Develop Reference

I can't crawl the website quotestoscrape using artoo.js Multi level recursive site crawling

I want to use artoo.js to scrape the website https://quotes.toscrape.com/
For handling pagination, I can do it! But for crawling the website ie for each page scrape the quotes and authors. Then take the link of the author and scrape the dob and pob for example. Finally handle the pagination.
Any help is much appreciated, here's my code:
var base_url = 'https://quotes.toscrape.com';
// empty list init
var my_list = []
// define the logic of the first scraper
var scraper1 = {
iterator: 'div.quote',
data: {
'quotes': {
sel: 'span'
},
'author': {
sel: 'small.author'
},
'link': {
sel: 'a',
attr: 'href'
}
}
};
// define the logic of the second scraper
var scraper2 = {
iterator: 'div.author-details',
data: {
'dob': {
sel: 'span.author-born-date'
},
'pob': {
sel: 'span.author-born-location'
}
}
}
// pagination
function nextUrl($page) {
return $page.find('li.next > a').attr('href');
}
artoo.log.debug('Starting the scraper...');
var frontpage = artoo.scrape(scraper1);
// spider
var my_list = []
// artoo spider
function pagination() {
artoo.ajaxSpider(
function(i, $data) {
//console.log($data.innerHTML);
return nextUrl(!i ? artoo.$(document) : $data);
}, {
limit: 1, // number of pages to scrape
scrape: scraper1,
concat: true,
done: function(data) {
artoo.log.debug('Finished retrieving data. Downloading...');
console.log(data);
for (var i = 0; i < my_list.length; i++) {
my_list.push(base_url + data[i].link)
}
console.log(my_list)
}
})
return my_list;
}
// Append links in a list
//my_list.push(base_url + data[0].link);
function crawl(mylist) {
artoo.ajaxSpider(
my_list, {
limit: 1, // number of pages to scrape
scrape: scraper2,
concat: true,
done: function(data) {
console.log(data);
artoo.log.debug('Finished retrieving data. Downloading...');
}
})
}
//var ll = null;
let links = pagination();
crawl(links)

Is there a way to use NumberFormat() formatter (Google Charts) in vue-google-charts vue.js wrapper

I have been tasked with formatting some columns in charts using vue-google-charts, a vue.js wrapper for Google Charts and I am not sure that 'NumberFormat()' is even supported in vue-google-charts.
First, if somebody knows if it is or isn't, I would like to know so I don't waste much time pursuing something that isn't possible. But if it is, I sure would love an example of how to do it.
What we are doing is returning our chart data from the database and passing it into this vue.js wrapper. We are creating several charts but there are columns that have commas in them we want to remove.
Please review the existing code. I am trying to implement this using #ready as documented in the docs for vue-google-charts.
vue-google-charts docs -> https://www.npmjs.com/package/vue-google-charts
Here is our existing code with a little framework of the onChartReady method already in place.
<GChart
v-if="chart.data"
id="gchart"
:key="index"
:options="{
pieSliceText: chart.dropDownPie,
allowHtml: true
}"
:type="chart.ChartType"
:data="filtered(chart.data, chart.query, chart.query_type)"
:class="[
{'pieChart': chart.ChartType == 'PieChart'},
{'tableChart': chart.ChartType == 'Table'}
]"
#ready = "onChartReady"
/>
And then ...
<script>
import { GChart } from 'vue-google-charts';
import fuzzy from 'fuzzy';
import 'vue-awesome/icons';
import Icon from 'vue-awesome/components/Icon';
export default {
components: {
GChart,
Icon
},
props: {
},
data() {
return {
charts: window.template_data,
selected: 'null',
selects: [],
chartToSearch: false,
directDownloads: {
'Inactive Phones' : {
'slug' : 'phones_by_status',
'search_by' : 2,
'search' : '/Inactive/'
},
'Active Phones' : {
'slug' : 'phones_by_status',
'search_by' : 2,
'search' : '/Active/'
},
}
}
},
created(){
for (let i in this.charts){
if( !this.charts[i].slug ) continue;
$.post(ajaxurl, {
action: 'insights_' + this.charts[i].slug,
}, (res) => {
console.log(res.data);
if (res.success) {
this.$set(this.charts[i], 'data', res.data);
}
});
}
// console.log(this.charts);
},
methods: {
onChartReady(chart,google) {
let formatter = new.target.visualization.NumberFormat({
pattern: '0'
});
formatter.format(data, 0);
chart.draw(data)
},
toggleChart(chart) {
jQuery.post(ajaxurl, {
'action': 'update_insight_chart_type',
'chartType': chart.ChartType,
'chartSlug': chart.slug
}, (res) => {
chart.ChartType = res.data
})
},
csvHREF(chart) {
return window.location.href + '&rr_download_csv=' + chart.slug + '&rr_download_csv_search_by=' + chart.query_type + '&rr_download_csv_search=' + chart.query.trim()
},
filtered(data, query, column) {
query = query.trim();
if (query){
let localData = JSON.parse(JSON.stringify(data));
let column_Headers = localData.shift();
localData = localData.filter((row)=>{
if( query.endsWith('/') && query.startsWith('/') ){
return new RegExp(query.replace(/\//g, '')).test(String(row[column]));
}
return String(row[column]).toLowerCase().indexOf(query.toLowerCase()) > -1;
});
localData.unshift(column_Headers);
return localData;
}
return data;
},
filterIcon(chart) {
chart.searchVisible = !chart.searchVisible;
chart.query = "";
setTimeout(()=>{
document.querySelector(`#chart-${chart.slug} .insightSearch`).focus();
}, 1);
}
}
}
document.getElementsByClassName('google-visualization-table')
If anybody can help in ANY way, I am all ears.
Thanks!

not familiar with vue or the wrapper,
but in google charts, you can use object notation in your data,
to provide the formatted values.
all chart types will display the formatted values by default.
google's formatters just simply do this for you.
so, in your data, replace your number values with objects,
where v: is the value and f: is the formatted value...
{v: 2000, f: '$2,000.00'}
see following working snippet...
google.charts.load('current', {
packages: ['table']
}).then(function () {
var chartData = [
['col 0', 'col 1'],
['test', {v: 2000, f: '$2,000.00'}],
];
var dataTable = google.visualization.arrayToDataTable(chartData);
var table = new google.visualization.Table(document.getElementById('chart_div'));
table.draw(dataTable);
});
<script src="https://www.gstatic.com/charts/loader.js"></script>
<div id="chart_div"></div>

ZingChart X-axis labels showing as numbers instead of strings

I am using the ZingChart library to graph results from an API call. When I pass in a normal array for the "values" field of the chart data object, everything works fine. However, when I pass in an array made from Object.keys(titleSet) (where titleSet is a normal Javascript object), the graph displays as follows:
Example Chart
As you can see, the x-axis is now labeled with numbers instead of the array of strings. But when I print out the the result of Object.keys(titleSet) vs. passing in a normal array, they both appear to be the same in the console. Can anyone help me figure out what I'm doing wrong?
//List of movies inputted by the user
var movieList = [];
var movieSet = {};
var IMDBData = {
"values": [],
"text": "IMDB",
};
var metascoreData = {
"values": [],
"text": "Metascore"
};
var RTMData = {
"values": [],
"text": "Rotten Tomatoes Meter"
};
var RTUData = {
"values": [],
"text": "Rotten Tomatoes User"
};
var chartData = {
"type":"bar",
"legend":{
"adjust-layout": true
},
"plotarea": {
"adjust-layout":true
},
"plot":{
"stacked": true,
"border-radius": "1px",
"tooltip": {
"text": "Rated %v by %plot-text"
},
"animation":{
"effect":"11",
"method":"3",
"sequence":"ANIMATION_BY_PLOT_AND_NODE",
"speed":10
}
},
"scale-x": {
"label":{ /* Scale Title */
"text":"Movie Title",
},
"values": Object.keys(movieSet) /* Needs to be list of movie titles */
},
"scale-y": {
"label":{ /* Scale Title */
"text":"Total Score",
}
},
"series":[metascoreData, IMDBData, RTUData, RTMData]
};
var callback = function(data)
{
var resp = JSON.parse(data);
movieSet[resp.Title] = true;
//Render
zingchart.render({
id:'chartDiv',
data:chartData,
});
};

Full Disclosure, I'm a member of the ZingChart team.
Thank you for updating your question. The problem is you have defined your variable movieSet before the variablechartData. When parsing the page, top down, it is executing Object.keys({}) on an empty object when creating the variable chartData. You should just directly assign it into your config later on chartData['scale-x']['values'] = Object.keys(moviSet).
var callback = function(data)
{
var resp = JSON.parse(data);
movieSet[resp.Title] = true;
//Render
zingchart.render({
id:'chartDiv',
data:chartData,
});
};
There is a problem with the above code as well. It seems you are calling render on the chart every time you call this API. You should have one initial zingchart.render() and then from there on out use our API. I would suggest setdata method as it replaces a whole new JSON packet or modify method.
I am making some assumptions on how you are handling data. Regardless, check out the following demo
var movieValues = {};
var myConfig = {
type: "bar",
scaleX:{
values:[]
},
series : [
{
values : [35,42,67,89,25,34,67,85]
}
]
};
zingchart.render({
id : 'myChart',
data : myConfig,
height: 300,
width: '100%'
});
var callback = function(data) {
movieValues[data.title] = true;
myConfig.scaleX.values = Object.keys(movieValues);
zingchart.exec('myChart', 'setdata', {
data:myConfig
})
}
var index = 0;
var movieNamesFromDB = ['Sausage Party', 'Animal House', 'Hot Rod', 'Blazing Saddles'];
setInterval(function() {
if (index < 4) {
callback({title:movieNamesFromDB[index++]});
}
},1000)
<!DOCTYPE html>
<html>
<head>
<!--Assets will be injected here on compile. Use the assets button above-->
<script src= "https://cdn.zingchart.com/zingchart.min.js"></script>
<script> zingchart.MODULESDIR = "https://cdn.zingchart.com/modules/";
</script>
<!--Inject End-->
</head>
<body>
<div id='myChart'></div>
</body>
</html>
If you noticed in the demo, the length of scaleX.values determines how many nodes are shown on the graph. If you change values to labels this wont happen.

Accessing object data in datatables through get function

I am trying to create an object that I can modify as per
Datatables - Data
I use the following code to create my object
function projectData(projid,projdesc,descdet){
this._projid = 'HTML1'
this._projdesc = 'HTML2'
for(var i=0;i<=7;i++){
this['_day'+i] = 'HTML3';
}
this.projid = function(){
return this._projid
}
this.projdesc = function(){
return this._projdesc
}
this.day0 = function(){ // More of these for day (0-7)
return this._day0
}
}
Then I use the following table initialization. (prjData is an array of New projectData objects)
var table = $('#table-ProjectHours').DataTable({
data: prjData,
"columns": [
{ data: 'projid',"visible": true},
{ data: 'projdesc',"width": "45%" },
{ data: 'day0',"orderDataType": "dom-text-numeric" },
{ data: 'day1',"orderDataType": "dom-text-numeric" },
{ data: 'day2',"orderDataType": "dom-text-numeric" },
{ data: 'day3',"orderDataType": "dom-text-numeric" },
{ data: 'day4',"orderDataType": "dom-text-numeric" },
{ data: 'day5',"orderDataType": "dom-text-numeric" },
{ data: 'day6',"orderDataType": "dom-text-numeric" },
{ data: 'day7',"orderDataType": "dom-text-numeric" }
]
});
I access my table through
var dto = $('#table-ProjectHours').DataTable().data();
I get my objects as seen here:
What I do not understand is why when I attempt to do dto[0].day0 I do not get _day0 --- I just get the function string.
I can access the data through _day0 but it seems wrong...

Try rows().data() https://datatables.net/reference/api/rows().data()
e.g.
var table = $('#example').DataTable();
var data = table
.rows()
.data();
alert( 'The table has '+data.length+' records' );

In order to set data in jquery-datatables you must use .row(index).data(obj) or .cell(selector).data(obj).
You can also use row(index).day0() and if you write a set portion of your method you can set the data like row(index).day0(thing_to_set) You must call row(index).invalidate after changing the data for it to appear correctly.
In this particular situation, .day0 must be called as .day0() since it is a method.

Trying to iterate an array through mutliple highcharts

So I have a large amount of data that I need to display all stored in separate CSV files. So I created two charts just fine in highcharts, one line, one area, but instead of copying and pasting the function over and over again I was hoping I could just iterate through it like so:
var library = ['data/data.csv', 'data/attendanceGroup.csv'];
var libraryLength = library.length;
var area =['#attendanceRoom','#attendanceGroup'];
var i = 0;
function areaChart(){
$(function () {
$.get(library[i], function(csv) {
$(area[i]).highcharts({
chart: {
type: 'area'
},
data: {
csv: csv
},
title: {
text: 'Attendance by Room'
},
yAxis: {
title: {
text: null
},
minorTickInterval: 'auto'
},
legend:{
align: 'left',
verticalAlign: 'top',
floating: true
},
});
});
});
}
for (i = 0; i < libraryLength; i++){
areaChart();
}
I was looking at this Manage multiple highchart charts in a single webpage using jQuery.extend() or Highcharts.setOptions but that sets options for each individual chart and then you just make them over and over again. I thought a better solution might be to just have the one function and then just re-run it for each individual chart especially since I'm pulling the data from .CSV files.
So is this possible? Or should I go with jQuery.extend()?
Thanks for any help in advance!

Just two things I would improve:
$(function () { }); - I would encapsulate whole JS, not only part with AJAX and Highcharts:
$(function () {
var library = ['data/data.csv', 'data/attendanceGroup.csv'];
...
for (i = 0; i < libraryLength; i++){
areaChart();
}
});
make library[i] and area[i] as arguments for areaChart():
$(function () {
var library = ['data/data.csv', 'data/attendanceGroup.csv'];
...
function areaChart(lib, area){
$.get(lib, function(csv) {
$(area).highcharts({
chart: {
type: 'area'
},
data: {
csv: csv
}
});
});
}
for (i = 0; i < libraryLength; i++){
areaChart(library[i], area[i]);
}
});
Of course, you can add more params to areaChart for example type, and pass on what kind of the chart should be rendered:
$(function () {
var library = ['data/data.csv', 'data/attendanceGroup.csv'];
var types = ['line', 'area'];
...
function areaChart(lib, area, type){
$.get(lib, function(csv) {
$(area).highcharts({
chart: {
type: type
},
data: {
csv: csv
}
});
});
}
for (i = 0; i < libraryLength; i++){
areaChart(library[i], area[i], types[i]);
}
});
Don't overdo with the params, no one likes to read 10params and control order etc. Instead you may consider passing one object param (renamed from areaChart to myChart):
myChart({
lib: library[i],
area: area[i],
type: types[i]
});
And in myChart() method:
function myChart(options) {
$.get(options.lib, function(csv) {
$(options.area).highcharts({
chart: {
type: options.type
},
data: {
csv: csv
}
});
});
}