Develop Reference

Need help scraping image from craigslist

I've tried everything I can think of. I'm able to get postUrl, date, title, price and location. If you go to https://sandiego.craigslist.org/search/sss?query=surfboards and paste the code snippet below into the console it returns all the images. But when I try to access in my code it's returning undefined. Any help on this would be greatly appreciated!
$('#search-results > li').each((index, element) => {
console.log( $(element).children().find('img').attr('src') )
})
import axios from 'axios'
import request from 'request-promise'
import cheerio from 'cheerio'
import express from 'express'
import path from 'path'
const __dirname = path.resolve();
const PORT = process.env.PORT || 8000;
const app = express();
app.get('', (req, res) => {
res.sendFile(__dirname + '/views/index.html')
});
const surfboards = [];
axios("https://sandiego.craigslist.org/search/sss?query=surfboards")
.then(res => {
const htmlData = res.data;
const $ = cheerio.load(htmlData);
$('#search-results > li').each((index, element) => {
const postUrl = $(element).children('a').attr('href');
const date = $(element).children('.result-info').children('.result-date').text();
const title = $(element).children('.result-info').children('.result-heading').text().trim();
const price = $(element).children('.result-info').children('.result-meta').children('.result-price').text();
const location = $(element).children('.result-info').children('.result-meta').children(".result-hood").text().trim();
// Why is this not working?!?!?!?!?!
const img = $(element).children().find('img').attr('src');
surfboards.push({
title,
postUrl,
date,
price,
location,
img
})
})
return surfboards
}).catch(err => console.error(err))
app.get('/api/surfboards', (req, res) => {
const usedboards = surfboards
return res.status(200).json({
results: usedboards
})
})
// Make App listen
app.listen(PORT, () => console.log(`Server is listening to port ${PORT}`))

Looks like the page sets the images with JavaScript. Thus axios gets the HTML without actual links to images.
But there seems to be a workaround here. You can generate links to images by concatenate https://images.craigslist.org and data-ids value from parent a tag.
You can get the data-ids like this:
var data_ids = $(element).children('a').attr('data-ids')
then split it to array by comma, delete first two 3: symbols and concat it like this:
`${img_base_url}/${ids}_${resolution_and_extension}`
But if you need to get URL only for first image then there is no need to create new array each time. Use substring instead (note that sometimes li don't have image at all):
if (data_ids && data_ids.includes(',')) {
data_ids.substring(data_ids.indexOf('3:') + 2, data_ids.indexOf(','))
} else if (data_ids) {
data_ids.substring(data_ids.indexOf('3:') + 2, data_ids.length)
}

POST request for paginating, sorting results

I am currently doing a GET request that uses a bunch of methods that I wrote to query the database and display them. This works great, but I want to make it a POST request method so that the methods do not have to depend on req.query, deal with a json body instead of a string of URL params which would would facilitate the url string not to have anything else except the endpoints and make it as dynamic as possible. Any idea how to do this?
This is my controller method:
exports.getBooks = async (req, res, next) => {
const bookList = new BookList(Book.find(), req.query)
.filter()
.sort()
.paginate();
const books = await bookList.query;
res.status(200)
.json({
books,
});
};
This is the BookList class that has all the methods:
class BookList {
constructor(query, queryString) {
this.query = query;
this.queryString = queryString;
}
filter() {
const queryObj = { ...this.queryString };
const excludedFields = ['page', 'sort', 'limit', 'fields'];
excludedFields.forEach(el => delete queryObj[el]);
let queryStr = JSON.stringify(queryObj);
this.query = this.query.find(JSON.parse(queryStr));
return this;
}
sort() {
if (this.queryString.sort) {
const sortBy = this.queryString.sort.split(',').join(' ');
this.query = this.query.sort(sortBy);
} else {
this.query = this.query.sort('-createdAt');
}
return this;
}
paginate() {
const page = Number(this.queryString.page) || 1;
const limit = Number(this.queryString.limit) || 100;
const skip = (page - 1) * limit;
this.query = this.query.skip(skip).limit(limit);
return this;
}
}
module.exports = BookList;

This is what worked for me:
exports.getBooks = async (req, res, next) => {
let bookBody = req.body
const bookList = new BookList(Book.find(), req.query)
.filter(bookBody, req)
.sort()
.paginate();
const books = await bookList.query;
res.status(200)
.json({
books,
});
};
filter(bookBody, req) {
const filterBooks = bookBody.filter
const bookId = req.params.bookId
let requiredFilter
if (filterBooks) {
requiredFilter = {bookStatus: filterBooks.bookStatus, bookId};
} else {
requiredFilter = { bookId}
}
this.query = this.query.find(requiredFilter)
return this;
}

If you need to convert it to POST with json body then you will use something like below
var bodyParser = require('body-parser')
// parse application/json
app.use(bodyParser.json())
exports.getBooks = async (req, res, next) => {
const bookList = new BookList(Book.find(), req.body)
.filter()
.sort()
.paginate();
const allReports = await bookList.query;
res.status(200)
.json({
books,
});
};
In this I assumed that parameters name will still be the same. Also the two lines for json parser needs to be in the file where you initialise the express server

When using an asynchronous javascript XML (AJAX) request to send and receive back data from the server, it is possible to specify either a "POST" or a "GET" request. Any number of items may be sent or received via a single such request, as is shown in the sample script below. In your case, you would tailor it to have the script send your JSON data.
function AJAXroutine(val1,val2) { /* NAME IT WHATEVER YOU LIKE */
var formAction=document.getElementById("MyForm").getAttribute('action');
function Q(){
var K=new XMLHttpRequest();
var frm=new FormData();
/* CAN HAVE ANY NUMBER OF VALUES SENT BACK TO SERVER */
frm.append('V1',val1);
frm.append('V2',val2);
frm.append('V3',document.getElementById('another_elmt').value);
frm.append('V4',document.getElementById('another_one').value);
K.onreadystatechange=function(){
if(this.readyState==4&&this.status==200){
/* IF MULTIPLE VALUES ARE RETURNED */
var data=this.responseText.split("|");
document.getElementById('my_elmt').innerHTML=data[0];
document.getElementById('my_text').value=data[1];
/* ETC. */
/* IF JUST ONE VALUE IS RETURNED */
document.getElementById('my_elmt').value=this.responseText;
};
};
/* CAN OPTIONALLY CHANGE 'POST' TO 'GET' */
K.open('post',formAction);
K.send(frm)
}
Q();
}
This is a very basic template for an AJAX request and can be easily adapted to suit one's individual requirements.

I cant get data from my firestore db using geofirestore in express

I created a query in express to extract my data
this is my express function for endpoint:
app.get('/locatii', (req, res) => {
const geofirestore = new GeoFirestore(db);
const geocollection = geofirestore.collection('locatii2');
const query = geocollection.near({
center: new firebase.firestore.GeoPoint(45.831922, 27.431149),
radius: 3000
});
query.get().then(snap => {
console.log(snap.docs())
let places = [];
snap.forEach(doc => {
places.push(doc.data());
console.log(doc.data())
});
res.json(places);
})
.catch(err => res.send(err));
});
below it's my firesore structure:
When I acces my endpoint it don't shows in console data or in postman.I don't know what I missed.Thank you!

Your data structure doesn't seem to have a geohash in it. Having a geohash in each document is required for being able to query those documents by their location.
From the example app that come with GeoFirestoreJS, it seems that you can get the geohash with something like:
location.lat = ...
location.lng = ...
const hash = geokit.Geokit.hash(location);
And then set it into your Firestore document with something like:
documentRef.update(hash);

Avoid looping over visited URLs generated by mongodb and express

I have a mongodb collection, in which each document is accessible through a specific url.
The goal is to display a random document url to the user, which has not been visited before, until the user has seen all documents in the collection, then the whole collection shall be 'cleared' so it is accessible again.
I have thought about using cookies to achieve it, but I haven't found a way to do it.
The application is built using express for nodejs, with the mongoose module for mongodb.
Model.class:
var mongoose = require('mongoose'),
URLSlugs = require('mongoose-url-slugs'),
AutoIncrement = require('mongoose-sequence')(mongoose),
Schema = mongoose.Schema;
var dilemmaSchema = new Schema({
dilemma_title: String,
red_dilemma: String,
blue_dilemma: String,
red_dilemma_votes: {
type: Number,
default: 0
},
blue_dilemma_votes: {
type: Number,
default: 0
}
});
dilemmaSchema.plugin(AutoIncrement, {
inc_field: 'id'
});
dilemmaSchema.plugin(URLSlugs('dilemma_title'));
module.exports = mongoose.model('Dilemma', dilemmaSchema);
Code snippet from Router:
dilemmaRouter.route('/next')
.get(function (req, res) {
Dilemma.count().exec(function (err, count) {
var random = Math.floor(Math.random() * count);
Dilemma.findOne().skip(random).exec(function (err, dilemma) { //This function is supposed to redirect to an unvisited URL, and mark it as visited
dilemmaID = dilemma._id;
res.redirect('/' + dilemma.id + '/' + dilemma.slug);
})
})
})
How the database entries are looked up
dilemmaRouter.route('/:id/:slug')
.get(function (req, res) {
const _id = req.params.id;
const _slug = req.params.slug;
let query = {
id: _id,
slug: _slug
}
Dilemma.findOne(query, function (err, dilemma) {
if (err) {
console.log(err);
} else {
if (dilemma === null) {
res.redirect('/');
} else {
res.render('index', {
dilemma: dilemma
})
}
}
})
})

If you cycle through documents with an order and then not random, you just can add +X to your current id.
If it's random, you need to store for all users all documents they already have seen. So you need another 'table' in your database, or you need to add a field inside your user model where your store all documents seen.
The 'best' solution is think right now would be to add this field in your user model (or be able to know that IP X already saw document A and B). When your user try to access your page, you get the list of all id for the document you have, remove the id saw by the user, and do a random inside this list.
dilemmaRouter.route('/next')
.get(function (req, res) {
Dilemma.count().exec(function (err, count) {
// find all documents
User.find({'idUserOrIP' : 'userIPorID'}).exec(function(user) {
var userListSaw = user.listSaw;
})
// create a list with all your document id
var allDocs = [1...100];
// remove id already seen (user saw id 1 to 3)
allDocs = [4...100];
// random now store the index of the id of the document you want to display
var random = Math.floor(Math.random() * allDocs.length);
// just find your document with the id you just get
Dilemma.find({'id' : allDocs[random]}).exec(function (err, dilemma) { //This function is supposed to redirect to an unvisited URL, and mark it as visited
dilemmaID = dilemma._id;
res.redirect('/' + dilemma.id + '/' + dilemma.slug);
})
})
})

Node express app calling mssql is saying that Connection is closed

I have another app which uses express and routes but this new app i was slimming it down. I know the connection string stuff is correct
script.getQuestions(connection);
script.getQuestions = function(connection,req, res){
console.log(connection);
}
I have read that some people said online to change to use a promise for async fixes this... problem is that with my function having req and res i don't know how to pass those in when i even try to refactor with a promise
"ConnectionError: Connection is closed"
"(module.js:487:32) code: 'ECONNCLOSED', name: 'ConnectionError' }"
What I call up (script) is
var sql = require('mssql');
exports.getQuestions = function(connection, req,res){
console.log(connection);
var request = new sql.Request(connection);
var query = 'select * from Question'
request.query(query).then(function(resultset){
res.json(resultset.recordset);
}).catch(function(err){
console.log(err);
//res.json(err)
})
}

it's a bit hard to understand what you're doing there. But here is an promise example to use mssql
const sql = require('mssql')
sql.connect(config).then(pool => {
// Query
return pool.request()
.input('input_parameter', sql.Int, value)
.query('select * from mytable where id = #input_parameter')
}).then(result => {
console.dir(result)
// Stored procedure
return pool.request()
.input('input_parameter', sql.Int, value)
.output('output_parameter', sql.VarChar(50))
.execute('procedure_name')
}).then(result => {
console.dir(result)
}).catch(err => {
// ... error checks
})
sql.on('error', err => {
// ... error handler
})
source: https://www.npmjs.com/package/mssql#promises