Develop Reference

scrape a Javascript:_dopostback()

I am trying to automate a website and I am having trouble with a Javascript issue. After my first request '__EVENTTARGET' = 'ctl00$content$ctl01$btn'. New href pop-up on same page, and now i want to request this new javascript link '__EVENTTARGET' = 'ctl00$content$ctl01$' . I don't know how to scrape a Javascript:_dopostback() using scrapy spider. I tried looking into this issue and cannot find anything.
<a id="ctl00_content_" href="javascript:__doPostBack('ctl00$content$ctl01$,'')">SYED ALI </a>
my spider code::
URL = 'xyz'
class ExitRealtySpider(scrapy.Spider):
name = "campSpider"#name = "exit_realty"
allowed_domains = ["xyz"]
start_urls = [URL]
def parse(self, response):
# submit a form (first page)
self.data = {}
soup = BeautifulSoup(urlopen(URL), 'html.parser')
viewstate = soup.find('input', {'id': '__VIEWSTATE' })['value']
generator = soup.find('input', {'id': '__VIEWSTATEGENERATOR'})['value']
validation = soup.find('input', {'id': '__EVENTVALIDATION' })['value']
self.data['__VIEWSTATE']= viewstate,
self.data['__VIEWSTATEGENERATOR'] = generator,#'',
self.data['__VIEWSTATEENCRYPTED'] = '',
self.data['__EVENTVALIDATION'] = validation,
self.data['typAirmenInquiry'] = '7',
self.data['ctl00$content$ctl01$txtbxLastName'] = 'a',
self.data['ctl00$content$ctl01$txtbxCertNo'] = '123',
self.data['ctl00$content$ctl01$btnSearch'] = 'Search',
self.data['__EVENTTARGET'] = 'ctl00$content$ctl01$'
return FormRequest.from_response(response,
method='POST',
callback=self.parse_page,
formdata=self.data,
#encoding = 'utf-8',
#meta={'page': 1},
dont_filter=True
#headers=HEADERS
)
def parse_page (self,response):
print("\n\n\n\n\n",response.body,"\n\n\n\n\n")
self.data = {}
soup = BeautifulSoup(urlopen(URL), 'html.parser')
viewstate = soup.find('input', {'id': '__VIEWSTATE' })['value']
generator = soup.find('input', {'id': '__VIEWSTATEGENERATOR'})['value']
validation = soup.find('input', {'id': '__EVENTVALIDATION' })['value']
self.data['__EVENTARGUMENT']= '',
self.data['__LASTFOCUS']= '',
self.data['__VIEWSTATE']= viewstate,
self.data['__VIEWSTATEGENERATOR'] = generator,#'',
self.data['__VIEWSTATEENCRYPTED'] = '',
self.data['__EVENTVALIDATION'] = validation,
self.data['typAirmenInquiry'] = '7',
self.data['__EVENTTARGET'] = 'ctl00$content$ctl01$'
ans = FormRequest.from_response(response,
method='POST',
callback=self.parse_page2,
formdata=self.data,
#encoding = 'utf-8',
#meta={'page': 1},
dont_filter=True
#headers=HEADERS
)
return ans
def parse_page2 (self,response):
print("\n\n\n\n\n"),response.body,"\n\n\n\n\n")

How to reload Flask-based webpage after receiving data

I have a simple web project which gets data from, processes it and needs to output result on the page. After the spider works, the data is written to sqlite, but not displayed on the page. How to refresh page after writing data in sqlite?
button
<button id="scrape" class="btn btn-success mr-2">Scrape</button>
js
$.post('/wellness', {'specialty': specialty, 'state': state, 'city': city}, (res) => {
});
flask
#app.route('/<page_id>', methods=['GET', 'POST'])
def page(page_id):
file_html = f"{page_id}.html"
file_py = f"{page_id}.py"
file_db = f"{page_id}.db"
specialty = request.form.getlist('specialty[]')
state = request.form.getlist('state[]')
city = request.form.getlist('city[]')
settings = ''
with open('settings.json', 'r') as f:
for line in f.read():
settings += line
settings = json.loads(settings)
settings['specialty'] = specialty
settings['state'] = state
settings['city'] = city
with open('settings.json', 'w') as f:
f.write(json.dumps(settings, indent=4))
process = subprocess.Popen('python e:/Python/sqlite/spiders/' + file_py, shell=True)
process.wait()
try:
db = sqlite3.connect(file_db)
cursor = db.cursor()
cursor.execute('SELECT * FROM wellness ORDER BY id DESC')
cards = cursor.fetchall()
db.close()
return render_template(file_html, cards = cards)
except:
return render_template(file_html)

Write the appropriate append/fill HTML code to the div where you want to show
(res) => {
# Write append or fill here
});
Example
$.ajax({
url: "/update/",
type: "POST",
data: {"data": data},
success: function(resp){
$('div.stats').html(resp.data); # Filling the data in appropriate div
}
});
You would have to do similar thing for your use case.

solution:
$.post('/wellness', {'specialty': specialty, 'state': state, 'city': city}, (res) => {
$(location).attr('href', 'http://127.0.0.1:5000/wellness')
});

Ajax Call Returns HTML page as response instead of rendering page (Rails)

Hey having a issue with the rendering of a .erb file, in my AJAX call I make a call to my create action on rails where I validate and process the form data and sent back the completed order data as render: json which works fine.
I have a conditional that checks to see if parameter exists, it if does then the completed order data is passed back as a response via render: json
It if doesn't exists it will render a receipt page.
The problem is when I render the receipt page, the full HTML receipt page comes back as a response instead of rendering the page. Please Help!
$scope.placeOrder = function() {
var body = composeOrderBody();
var isValid = validateForm(body.order);
if(isValid) {
var orderComplete = '<%= #orderComplete %>';
var baseUrl = '<%= request.base_url %>';
console.log('Passing order object: ', body.order);
$http({
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
url: checkout_url,
data: {
order: body.order,
xhr_request: true
},
}).then((function(_this) {
return function(response) {
if(typeof response.data == 'undefined' || response.data == null || !response.data) {
console.log('Error: missing Order Number from Order Confirmation data.', response.data);
}
console.log('Order Confirmation response data object:' , response.data);
if(orderComplete) {
var redirectUrl = 'http://' + orderComplete
var order_params = `?oid=${response.data.oid}?cart=${response.data.cart}?total=${response.data.total}`
window.location.href = redirectUrl + order_params;
} else {
console.log('Base Url: ', baseUrl);
// window.location.href = `${baseUrl}/receipt`;
}
};
})(this));
} else {
console.log('Form Validation or Stripe Validation Failed');
}
} // end placeOrder
Rails Code
# Redirect to orderComplete URL if it's set
if !#orderComplete.blank?
puts 'orderComplete parameter is not blank'
# Sum up all the line item quantities
qty = #order.line_items.inject(0) {|sum, line_item| sum + line_item.quantity}
# Get all of the coupons (and values) into a string
coupons = #order.applied_coupons.map { |coupon| coupon.coupon }.join(',')
coupon_values = #order.applied_coupons.map { |coupon| '%.2f' % coupon.applied_value.to_f }.join(',')
order_params = {
"oid" => URI::escape(#order.number),
"cart" => URI::escape(#cart),
"total" => URI::escape('%.2f' % #order.total),
}
#redirectUrl = URI.parse(URI.escape(#orderComplete))
#redirectUrl.query = [#redirectUrl.query, order_params.to_query].compact.join('&')
#redirectUrl = #redirectUrl.to_s
if params[:xhr_request]
render json: order_params.to_json
return
end
render 'receipt_redirect', :layout => 'receipt_redirect'
else
puts 'OrderComplete Parameter is blank'
render 'receipt', :layout => 'receipt', :campaign => #campaign
end

Django Fullcalendar ajax no showing events

Hi I can't get events from ajax call to calendar please help
template:
$('#calendar').fullCalendar({
.....
events:{
url:'{% url 'rezerwacje' %}',
type: 'GET',
},
url.py
url(r'^rezerwacje/json2/', views.RezerwacjeLista, name='rezerwacje'),
views.py
def RezerwacjeLista(request):
if request.is_ajax():
try:
start = request.GET.get('start', False)
print start
end = request.GET.get('end',False)
except ValueError:
start = datetime.now()
end = start + timedelta(days=7)
rezerwacje = Rezerwacje.objects.filter(start__gte=start).filter(end__lte=end)
json_list = []
lista = []
for rezerwacja in rezerwacje:
id = rezerwacja.id
start = rezerwacja.start.strftime("%Y-%m-%d %H:%M:%S")
stanowisko = rezerwacja.stanowisko.nazwa
end = rezerwacja.end.strftime("%Y-%m-%d %H:%M:%S")
kolor = rezerwacja.kolor
status = rezerwacja.status
#allDay = False
json_entry = {"id":'1', 'start':start, 'end':end,'resourceId':stanowisko, 'kolor':kolor, 'status': status}
json_list.append(json_entry)
lista = ast.literal_eval(json.dumps(json_list))
return HttpResponse(lista, content_type='application/json')
That view returns json:
{'kolor': '1', 'status': 1, 'end': '2017-05-03 15:00:00', 'resourceId': 'K1', 'start': '2017-05-03 14:00:00', 'id': '1'}
That is good but when i select date in calendar there is no result on events. In response i get url and json
http://127.0.0.1:8000/rezerwacje/json2/?start=2017-05-03&end=2017-05-04&_=1493937740198
How to pass result to ajax?

Chat Room in Django Channels

I have models "Project" and "Room". Every project has own chat room. I am tring to use Django Channels. Unfortunarly, I have error. What I miss and how to fix this error.
ERROR:
Traceback (most recent call last):
File "C:\Users\Nurzhan\AppData\Local\Programs\Python\Python35-32\lib\site-packages\django\core\handlers\exception.py", line 39, in inner
response = get_response(request)
File "C:\Users\Nurzhan\AppData\Local\Programs\Python\Python35-32\lib\site-packages\django\core\handlers\base.py", line 187, in _get_response
response = self.process_exception_by_middleware(e, request)
File "C:\Users\Nurzhan\AppData\Local\Programs\Python\Python35-32\lib\site-packages\channels\handler.py", line 237, in process_exception_by_middleware
return super(AsgiHandler, self).process_exception_by_middleware(exception, request)
File "C:\Users\Nurzhan\AppData\Local\Programs\Python\Python35-32\lib\site-packages\django\core\handlers\base.py", line 185, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "C:\Users\Nurzhan\AppData\Local\Programs\Python\Python35-32\lib\site-packages\django\contrib\auth\decorators.py", line 23, in _wrapped_view
return view_func(request, *args, **kwargs)
File "C:\Users\Nurzhan\PycharmProjects\RMS\project\views.py", line 176, in chat_room
room, created = Room.objects.get_or_create(project=project_code)
File "C:\Users\Nurzhan\AppData\Local\Programs\Python\Python35-32\lib\site-packages\django\db\models\manager.py", line 85, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "C:\Users\Nurzhan\AppData\Local\Programs\Python\Python35-32\lib\site-packages\django\db\models\query.py", line 475, in get_or_create
return self._create_object_from_params(lookup, params)
File "C:\Users\Nurzhan\AppData\Local\Programs\Python\Python35-32\lib\site-packages\django\db\models\query.py", line 505, in _create_object_from_params
obj = self.create(**params)
File "C:\Users\Nurzhan\AppData\Local\Programs\Python\Python35-32\lib\site-packages\django\db\models\query.py", line 397, in create
obj = self.model(**kwargs)
File "C:\Users\Nurzhan\AppData\Local\Programs\Python\Python35-32\lib\site-packages\django\db\models\base.py", line 537, in __init__
setattr(self, field.name, rel_obj)
File "C:\Users\Nurzhan\AppData\Local\Programs\Python\Python35-32\lib\site-packages\django\db\models\fields\related_descriptors.py", line 211, in __set__
self.field.remote_field.model._meta.object_name,
ValueError: Cannot assign "'1744d0bc-e439-4562-9c29-4e7b2451f39c'": "Room.project" must be a "Project" instance.
urls.py:
url(r'^(?P<project_code>[0-9a-f-]+)/chat_room/$', chat_room, name='chat_room'),
settings.py:
CHANNEL_LAYERS = {
"default": {
"BACKEND": "asgi_redis.RedisChannelLayer",
"CONFIG": {
"hosts": [os.environ.get('REDIS_URL', 'redis://localhost:6379')],
},
"ROUTING": "chat.routing.channel_routing",
},
}
models.py:
class Room(models.Model):
project = models.ForeignKey(Project, on_delete=models.CASCADE)
code = models.UUIDField(_('Code'), primary_key=True, default=uuid.uuid4, editable=False)
def __str__(self):
return self.code
def get_absolute_url(self):
return reverse('project:chat_room', args=[self.project_code])
class Message(models.Model):
room = models.ForeignKey(Room, related_name='messages')
handle = models.TextField()
message = models.TextField()
timestamp = models.DateTimeField(default=timezone.now, db_index=True)
def __unicode__(self):
return '[{timestamp}] {handle}: {message}'.format(**self.as_dict())
def __str__(self):
return self.__unicode__()
#property
def formatted_timestamp(self):
return self.timestamp.strftime('%b %-d %-I:%M %p')
def as_dict(self):
return {'handle': self.handle, 'message': self.message, 'timestamp': self.formatted_timestamp}
views.py:
def chat_room(request, project_code):
# If the room with the given project_code doesn't exist, automatically create it upon first visit.
room, created = Room.objects.get_or_create(project=project_code)
# We want to show the last 50 messages, ordered most-recent-last
messages = reversed(room.messages.order_by('-timestamp')[:50])
return render(request, "project/chat/room.html", {
'room': room,
'messages': messages,
})
consumers.py:
#channel_session
def ws_connect(message):
prefix, project_code = message['path'].strip('/').strip('/').strip('/').strip('/').strip('/').strip('/')
room = Room.objects.get(project=project_code)
message.reply_channel.send({"accept": True})
Group('chat' + project_code).add(message.reply_channel)
message.channel_session['room'] = room.project
#channel_session
def ws_receive(message):
project_code = message.channel_session['room']
room = Room.objects.get(project=project_code)
data = json.loads(message['text'])
m = room.messages.create(handle=data['handle'], message=data['message'])
Group('chat' + project_code).send({'text': json.dumps(m.as_dict())})
#channel_session
def ws_disconnect(message):
project_code = message.channel_session['room']
Group('chat' + project_code).discard(message.reply_channel)
routing.py:
channel_routing = [
route("websocket.connect", ws_connect),
route("websocket.receive", ws_receive),
route("websocket.disconnect", ws_disconnect),
]
chat.js:
$(document).ready(function(){
socket = new WebSocket("ws://" + window.location.host + "/chat" + window.location.pathname);
var name = "";
swal({
title: 'Your name:',
input: 'text',
inputValidator: function (value) {
return new Promise(function (resolve, reject) {
if (value) {
resolve()
} else {
reject('You need to write something!')
}
})
}
}).then(function (result) {
name = result;
swal({
type: 'success',
html: 'Hello ' + result
})
});
socket.onopen = function(){
$('#chatform').on('submit', function(event) {
var message = {
handle: name,
message: $('#message').val(),
}
socket.send(JSON.stringify(message));
$('#message').val('');
return false;
});
}
socket.onmessage = function(message) {
var data = JSON.parse(message.data);
$('#chat').append('<tr>'
+ '<td>' + data.timestamp + '</td>'
+ '<td>' + data.handle + '</td>'
+ '<td>' + data.message + ' </td>'
+ '</tr>');
Push.create("Django Channels Example | " + room , {
body: "New Message",
icon: 'http://pa1.narvii.com/6184/91c955faf54e625b3467093bea75a6d25b813871_128.gif',
timeout: 4000,
onClick: function () {
window.focus();
this.close();
}
});
};
if (socket.readyState == WebSocket.OPEN) socket.onopen();
});