Develop Reference

Web Scraping Journal "El Peruano" - Python/Scrapy

im trying to scrap some info from "El Peruano" journal, but i cannot at first sight it look have to:
El Peruano Website
Put a Date in a Formbox.
Do a click in SearchBox.
Get all links for get all: "Title","Resolution#", "Body"
This is my code:
import scrapy
class SpiderPeruano(scrapy.Spider):
name = "peruano"
start_urls = [
"https://diariooficial.elperuano.pe/Normas"
]
custom_settings= {
"FEED_URI": "peruano.json",
"FEED_FORMAT": "json",
"FEED_EXPORT_ENCODING": "utf-8"
}
def parse_click(self, response):
#i put here a condition but i think is not necessary
#button = response.xpath("//div[#id='busqueda']/form[#action]/button[#id='btnBuscar']").get()
#if buttom:
yield scrapy.FormRequest.from_response(
response,
formxpath= "//form[#id='space_PortalNormasLegalesN']",
formdata={"cddesde": "08/03/2022", "cdhasta:": "08/03/2022"},
dont_click=True,
dont_filter=True,
callback=self.parse
)
def parse(self, response):
links = response.xpath("//div[#class='ediciones_texto']/h5/a/#href").getall()
for link in links:
yield response.follow(link, callback=self.parse_link)
def parse_link(self, response):
title = response.xpath("//div[#class='story']/h1[#class='sumilla']/text()").get()
num = response.xpath("//div[#class='story']/h2[#class='resoluci-n']/text()").getall()
body = response.xpath("//div[#class='story']/p/text()").getall()
yield {
"title": title,
"num": num,
"body": body
}
#call
#scrapy crawl peruano
#url = "https://diariooficial.elperuano.pe/normas"
#Form_BOX: "//form[#action]"
#Box_desde = "//form[#action]/input[#id='cddesde']"
#Box_hasta = "//form[#action]/input[#id='cdhasta']"
#Button= "//div[#id='busqueda']/form[#action]/button[#id='btnBuscar']"
#links = "//div[#class='ediciones_texto']/h5/a/#href"
#titles= "//div[#class='story']/h1[#class='sumilla']/text()"
#resolutionNum= "//div[#class='story']/h2[#class='resoluci-n']/text()"
#body= "//div[#class='story']/p/text()"
So, i need some help for know what i'm doing wrong on my code cuz this run well but dont get the data.
Thx a lot for your time and help!

I found two mistakes:
First:
Scrapy gets url from start_urls and sends response to parse (as default callback) but you expect it in parse_click (to send Form). If I rename functions then it sends form.
Second:
Small typo. In formdata= you use string "cdhasta:" with : at the end and this made problems.
import scrapy
class SpiderPeruano(scrapy.Spider):
name = "peruano"
start_urls = [
"https://diariooficial.elperuano.pe/Normas"
]
custom_settings= {
"FEED_URI": "peruano.json",
"FEED_FORMAT": "json",
"FEED_EXPORT_ENCODING": "utf-8"
}
def parse(self, response):
print('[parse] url:', response.url)
yield scrapy.FormRequest.from_response(
response,
formxpath= "//form[#id='space_PortalNormasLegalesN']",
formdata={"cddesde": "01/03/2022", "cdhasta": "03/03/2022", "btnBuscar":""},
dont_click=True,
dont_filter=True,
#headers={'Referer':"https://diariooficial.elperuano.pe/Normas", 'X-Requested-With': 'XMLHttpRequest'},
callback=self.parse_result
)
def parse_result(self, response):
print('[parse_result] url:', response.url)
links = response.xpath("//div[#class='ediciones_texto']/h5/a/#href").getall()
for link in links:
yield response.follow(link, callback=self.parse_link)
def parse_link(self, response):
print('[parse_link] url:', response.url)
title = response.xpath("//div[#class='story']/h1[#class='sumilla']/text()").get()
num = response.xpath("//div[#class='story']/h2[#class='resoluci-n']/text()").getall()
body = response.xpath("//div[#class='story']/p/text()").getall()
yield {
"title": title,
"num": num,
"body": body
}
# --- run without project ---
from scrapy.crawler import CrawlerProcess
c = CrawlerProcess({
'USER_AGENT': 'Mozilla/5.0 (X11; Linux x86_64; rv:98.0) Gecko/20100101 Firefox/98.0',
# save in file CSV, JSON or XML
'FEEDS': {'output.csv': {'format': 'csv'}}, # new in 2.1
})
c.crawl(SpiderPeruano)
c.start()
EDIT:
Meanwhile I tested it also with requests but I didn't try to get links from response to search details.
import requests
# --- GET ---
headers = {
# 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:98.0) Gecko/20100101 Firefox/98.0',
}
url = 'https://diariooficial.elperuano.pe/Normas'
response = requests.get(url, headers=headers)
print(response)
# --- POST ---
url = 'https://diariooficial.elperuano.pe/Normas/Filtro?dateparam=03/08/2022 00:00:00'
params = {
'cddesde': '01/03/2022',
'cdhasta': '03/03/2022',
# 'X-Requested-With': 'XMLHttpRequest',
}
headers = {
# 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:98.0) Gecko/20100101 Firefox/98.0',
# 'Referer': "https://diariooficial.elperuano.pe/Normas",
# 'X-Requested-With': 'XMLHttpRequest'
}
response = requests.post(url, data=params, headers=headers)
print(response)
print(response.text[:1000])

Filepond revert method for delete does not send unique file id

I am trying to intergrate File Pond with my Flask server and I can not find the unique file id it sends with requests. With normal uploads it will show me request data on the server, but without the unique ID I am looking for. I want to use this unique ID to attach to a database in order to delete the file if the revert is sent later.
Going over the docs, it is suppsoed to send a unique ID with every upload, but I can't tell if it's a bad config in file pond or I suck at reading request.files. Just ignore all my prints where I try to read the contents of request.
Here is my FilePond Config:
FilePond.setOptions({
//allowImageEdit: true,
//allowImageCrop: true,
//allowMultiple: true,
//allowFileEncode: true,
allowImageExifOrientation: true,
credits: false,
server: {
process: {
url: './pondupload',
method: 'POST',
headers: {
'x-customheader': 'Hello World',
},
withCredentials: document.getElementById('csrf_token'),
onload: (response) => response.key,
onerror: (response) => response.data,
ondata: (formData) => {
formData.append('csrf_token', document.getElementById('csrf_token').value)
return formData;
},
},
revert: './pondupload',
restore: './restore/',
load: './load/',
fetch: './fetch/',
},
});
Here is my route in flask:
#bp.route('/pondupload', methods=['POST', 'DELETE'])
def pondupload():
print ('data: '+str(request.data))
print('form: '+str(request.form))
print('files: '+str(request.files))
print('args: '+str(request.args))
form=EmptyForm()
#TODO add csrf to revert
if request.method == "DELETE":
print('delete stuff')
if request.method == "POST" and form.validate_on_submit():
upload_dir = current_app.config['UPLOAD_FOLDER']
fn = ""
file_names = []
# get file object from request.files (see: http://flask.pocoo.org/docs/1.0/api/#flask.Request.files)
for key in request.files:
print('key: '+str(key))
file = request.files[key]
fn = secure_filename(file.filename)
if allowed_file(fn) == True:
file_names.append(fn)
try:
file.save(os.path.join(upload_dir, fn))
return jsonify({}), 200
except:
return jsonify(filename=file_names), 402
return jsonify({}), 400

Converting JS file upload with fetch() to python

I have to convert js code to python.
The js code performs a file upload via a POST request, using fetch().
This is the js code:
<input type="file" />
<button onclick="upload()">Upload data</button>
<script>
upload = async() =>
{
const fileField = document.querySelector('input[type="file"]');
await uploadDoc(fileField.files[0] );
};
uploadDoc = async( file ) =>
{
let fd = new FormData();
fd.append( 'file', file );
fd.append( 'descr', 'demo_upload' );
fd.append( 'title', name );
fd.append( 'contentType', 'text' );
fd.append( 'editor', user );
let resp = await fetch( url, { method: 'POST', mode: 'cors', body: fd });
};
</script>
The code works and complies with the fetch() docs, provided here:
https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch#uploading_a_file
Now when i try to recreate this in python, i get a 500 HTTP Status Code
This is the python code:
from urllib import request
from urllib.parse import urlencode
import json
with open('README.md', 'rb') as f:
upload_credentials = {
"file": f,
"descr": "testing",
"title": "READMEE.md",
"contentType": "text",
"editor": username,
}
url_for_upload = "" #here you place the upload URL
req = request.Request(url_for_upload, method="POST")
form_data = urlencode(upload_credentials)
form_data = form_data.encode()
response = request.urlopen(req, data=form_data)
http_status_code = response.getcode()
content = response.read()
print(http_status_code)
print(content)
This doesn't work however and i get this error:
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 500:
Someone with both js and python experience might be able to see what is wrong in the python side, or how to convert the fetch() function to python.

I think the problem is that you're reading the file in binary mode (rb). You need just r:
with open('README.md', 'r') as f: # r instead of rb
However, I still reccomend the requests module, which is more widely used, and is easier in this case.
import requests # pip install requests
url = "https://www.example.com/upload_file"
headers = {
"content-type": "text/plain" # force text file content type
}
files = {
"my_file": ("FILE_NAME.md", open("README.md","r")) # tuple containing file name, and io.BytesIO file buffer
}
data = { # in case you want to send a request payload too
"foo": "bar"
}
r = requests.post(url, headers=headers, files=files, data=data)
if r.status_code == 200:
print(r.text) # print response as a string (r.content for bytes, if response is binary like an image)

How can I enable the user to download a file from a server using Django?

I am using Django 3.1.2 to build a website, and I want the user of my website to be able to download a .zip file created on the backend when they click a button on the webpage.
However, when testing, I found that the download would not start when the button is clicked. No error was thrown either, and I just could not work out where I went wrong.
JavaScript:
document.querySelector("#download").onclick = function() {
// #download is the button I am talking about; it is in reality a div element
var formData = new FormData();
for (let i = 2; i < formElementsNumber; i++) {
let element = formElements[i];
formData.append(element.name, element.value);
}
formData.append("txtFile", formElements[1].files[0]);
formData.append("token", token);
// Up to here, everything works fine
$.ajax({
url: "/download/",
type: 'post',
data: formData,
// Essential; otherwise will receive an "illegal invocation" error
contentType: false,
processData: false,
success: function(data) {
// What should I add here to make the downloading start?
}
})
}
urls.py:
urlpatterns = [
path('', views.index),
path('process/', views.process),
path('download/', views.download)
]
views.py:
def download(request):
// Creates an .zip file
property_list = ['imageWidth', 'imageHeight', 'fontFamily', 'fontSize', 'rotationDirection', 'rotationAngle', 'foregroundColor', 'backgroundColor', 'token']
image_info = {}
for item in property_list:
image_info[item] = request.POST.get(item)
image_info['txtFile'] = request.FILES.get('txtFile').read().decode('gbk').split('\r\n')
// zip_name is the absolute path to the newly created .zip file
zip_name = w2p.make_zip(image_info, image_info['token'])
// Everything above works fine; the .zip file is successfully created
// What modification should I make here to make the downloading start properly
with open(zip_name, 'rb') as f:
response = HttpResponse(f.read())
response['content_type'] = "application/octet-stream"
response['Content-Disposition'] = 'attachment; filename=word2pic.zip'
return response
A thousand thanks in advance for your time.

import os
from django.conf import settings
from django.http import HttpResponse, Http404
def download(request, path):
file_path = os.path.join(settings.MEDIA_ROOT, path)
if os.path.exists(file_path):
with open(file_path, 'rb') as fh:
response = HttpResponse(fh.read(), content_type="application/filename")
response['Content-Disposition'] = 'inline; filename=' + os.path.basename(file_path)
return response
raise Http404

ExtJS, Flask and AJAX: cross-domain request

I'm developing a RESTful application with ExtJS (client) and Flask (server): client and server are linked by a protocol.
The problem comes when I try to do an AJAX request to the server, like this:
Ext.Ajax.request ({
url: 'http://localhost:5000/user/update/' + userId ,
method: 'POST' ,
xmlData: xmlUser ,
disableCaching: false ,
headers: {
'Content-Type': 'application/xml'
} ,
success: function (res) {
// something here
} ,
failure: function (res) {
// something here
}
});
With the above request, the client is trying to update the user information.
Unfortunately, this is a cross-domain request (details).
The server handles that request as follows:
#app.route ("/user/update/<user_id>", methods=['GET', 'POST'])
def user_update (user_id):
return user_id
What I see on the browser console is an OPTIONS request instead of POST.
Then, I tried to start the Flask application on the 80 port but it's not possible, obviously:
app.run (host="127.0.0.1", port=80)
In conclusion, I don't understand how the client can interact with the server if it cannot do any AJAX request.
How can I get around this problem?

Here's an excellent decorator for CORS with Flask.
http://flask.pocoo.org/snippets/56/
Here's the code for posterity if the link goes dead:
from datetime import timedelta
from flask import make_response, request, current_app
from functools import update_wrapper
def crossdomain(origin=None, methods=None, headers=None,
max_age=21600, attach_to_all=True,
automatic_options=True):
if methods is not None:
methods = ', '.join(sorted(x.upper() for x in methods))
if headers is not None and not isinstance(headers, basestring):
headers = ', '.join(x.upper() for x in headers)
if not isinstance(origin, basestring):
origin = ', '.join(origin)
if isinstance(max_age, timedelta):
max_age = max_age.total_seconds()
def get_methods():
if methods is not None:
return methods
options_resp = current_app.make_default_options_response()
return options_resp.headers['allow']
def decorator(f):
def wrapped_function(*args, **kwargs):
if automatic_options and request.method == 'OPTIONS':
resp = current_app.make_default_options_response()
else:
resp = make_response(f(*args, **kwargs))
if not attach_to_all and request.method != 'OPTIONS':
return resp
h = resp.headers
h['Access-Control-Allow-Origin'] = origin
h['Access-Control-Allow-Methods'] = get_methods()
h['Access-Control-Max-Age'] = str(max_age)
if headers is not None:
h['Access-Control-Allow-Headers'] = headers
return resp
f.provide_automatic_options = False
return update_wrapper(wrapped_function, f)
return decorator

You get around the problem by using CORS
http://en.wikipedia.org/wiki/Cross-origin_resource_sharing

The module Flask-CORS makes it extremely simple to perform cross-domain requests:
app = Flask(__name__)
cors = CORS(app, resources={r"/api/*": {"origins": "*"}})
See also: https://pypi.python.org/pypi/Flask-Cors