I'm pretty new to webdev and am trying to wrap my head around this. Presently I have a fairly simple express.js server running on a VPS that pulls and parses a JSON datafeed from a 3rd party source, then outputs that data in a readable way. That JSON feed refreshes every 15 seconds, but at the moment I'm doing all the work on receipt of a GET request, and then not checking again until a new request comes in.
My question then, I suppose, is can I pull and parse that datafeed, say, every 60 seconds, and then re-render the page if it's different? How would I go about doing that?
Here's the router.get function that's doing all the work at present:
router.get('/', async function(req, res) {
const json = await fetch("https://data.vatsim.net/v3/vatsim-data.json")
.then(jsonRes => jsonRes.json());
let isControlling = json.controllers.filter(function(controller) {
return controller.cid == cid &&
/(DEL|GND|TWR|APP|DEP|CTR|FSS)$/.test(controller.callsign) &&
controller.frequency != "199.998";
});
let atisArray = json.atis.filter(function(atis) {
return atis.cid == cid;
});
let atises = [];
for (var atis of atisArray) {
let atisAirport = atis.callsign.slice(0, 4);
if (!atises.includes(atisAirport))
{
atises.push(atisAirport);
}
}
if(isControlling.length > 0) {
res.render('controlling', {params:{callsign: isControlling[0].callsign, frequency: isControlling[0].frequency, atisList: atises}});
}
else {
res.render('not_controlling');
}
});
Thanks in advance
Related
I get the svg url using Google Autodraw API. The problem is that some of these urls(The svg url I processed using the picture name obtained from the API) jaggedly don't exist.
I only need to filter out the list of valid urls. For this, I used the method of sending a axios request. However, this method is accompanied by time lag problem when fetching list of urls each time.
Is there any other way to solve this problem? I don't have my server..just client-side.
Here is the code I wrote.
const validateSuggestions = async (suggestions) => {
const validSuggestions = [];
for (let i = 0; i < suggestions.length; i++) {
const suggestion = suggestions[i];
try {
await axios(suggestion.url);
validSuggestions.push(suggestion);
} catch (error) {
continue;
}
}
return validSuggestions;
};
Firstly, some background as to what my test script will cover.
Pressing a button on a website will fire off an email to a test mailbox.
This email can take anything between 10 and 30 minutes to arrive in the test mailbox.
So using the following code from imap-simple ;
'get new email info': function(browser) {
imaps.connect(config).then(function (connection) {
return connection.openBox('INBOX').then(function () {
var searchCriteria = ['UNSEEN'];
var fetchOptions = {
bodies: ['HEADER', 'TEXT'],
markSeen: false
};
return connection.search(searchCriteria, fetchOptions).then(function (results) {
var subjects = results.map(function (res) {
return res.parts.filter(function (part) {
return part.which === 'HEADER';
})[0].body.subject[0];
});
console.log(subjects);
Correctly comes back with a blank subjects array, as the email hasn't been received by the test mailbox yet.
Adding a 30 minutes pause to the beginning of the script 'works', as after the 30 minutes the subjects array is populated as the email is (99.99% of the time) sent within a 30 minute window.
However, it is definitely far from ideal as the email might be received within 15 minutes, meaning the test is 'wasting' 15 minutes.
So what I'd ideally like to do is write some form of loop (?) that tests if the subjects array is populated or not.
So if the array is populated, carry on with the rest of the test script (which entails testing that the array contains a certain text).
If the array is not populated, wait for another minute before trying again.
Continue this trying every minute until the array is populated.
I've tried setInterval, For loops, While loops, etc but I can't seem to get them to work and I'm out of ideas to be honest.
Any advice, help, references would be greatly appreciated and any more info can be promptly added if required.
One way to do that could be using recursion.
const createPromise = ms => new Promise((resolve, reject) => {
setTimeout(() => resolve(ms), ms)
});
function findUnseenEmails(connection) {
return connection.openBox('INBOX').then(function () {
var searchCriteria = [
'UNSEEN'
];
var fetchOptions = {
bodies: ['HEADER', 'TEXT'],
markSeen: false
};
return connection.search(searchCriteria, fetchOptions).then(function (results) {
var subjects = results.map(function (res) {
return res.parts.filter(function (part) {
return part.which === 'HEADER';
})[0].body.subject[0];
});
console.log(subjects);
return subjects.length > 0 ? subjects : createPromise(5000).then(function() { return findUnseenEmails(connection)});
});
});
}
imaps.connect(config).then(function (connection) {
return findUnseenEmails(connection);
}).then((subjects) => console.log('finished', subjects));
Of course there is a possibility and danger of stack overflow, but in such scenario feel free to come back to stack overflow to find here with the help of our community non-recursive solution.
Result:
EDIT:
Answering your question regarding closing connection:
I'd do it like this (in findUnseenEmails function)
if (subjects.length > 0) {
connection.end();
return subjects;
} else {
return createPromise(5000).then(function() { return findUnseenEmails(connection)});
}
I'm writing a cloud function that uses request-promise and cheerio to scrape a website and then check that information against a user document.
I am not entirely familiar with Javascript and Cloud Functions.
I've come so far that I managed to extract the information I need and navigate to the user's document and compare the data. Now the last piece of this function is to give the user points for each matching data point, so I need to update a map inside the user document.
This function has to loop through all users and change their document if the data point matches. I'm not sure the way I've written the code is the most optimal in terms of performance and billing if the userbase gets huge... Any pointers to how I could minimize the impact on the task would be of great help, as im new with JS.
So this is the code:
exports.getV75Results = functions.pubsub.schedule('every 2 minutes').onRun(async (context) => {
let linkMap = new Map();
const url = `https://www.example.com`
const options = {
uri: url,
headers: { 'User-Agent': 'test' },
transform: (body) => cheerio.load(body)
}
await rp(options)
.then(($) => {
for(let i = 1; i <= 7; i++)
{
//Find player from game
const lopp1 = $(`#mainContentHolder > div > div.mainContentStyleTrot > div > div.panel-body > table:nth-child(1) > tbody > tr:nth-child(${i}) > td:nth-child(2) > span`).text()
const lopp1StrR1 = lopp1.replace("(", "");
const lopp1StrR2 = lopp1StrR1.replace(")", "");
const lopp1StrR3 = lopp1StrR2.replace(" ", "");
linkMap.set(i, lopp1StrR3.toUpperCase());
}
console.log(linkMap);
return linkMap;
}).then(async () => {
//Start lookup users
let usersRef = db.collection('fantasyfotball').doc('users');
usersRef.listCollections().then(collections => {
collections.forEach( collection => {
var user = collection.doc(collection.id);
let batch = new admin.firestore().batch();
user.get().then(function(doc) {
let json = doc.data();
//Look in users collection if players document exist
Object.keys(json).forEach((name) => {
if(name != null) {
//Document with users active fotball players
if(name == 'players') {
let i = 0;
Object.values(json[name]).forEach((value) => {
i++;
if(value.localeCompare(linkMap.get(i)) == 0) {
//Loop through user keys and find owned players if user has the correct player
Object.keys(json).forEach((map) => {
if(map != null)
{
//Document with a map of player owned fotball players, each respective player has a key = 'fotball player' and value = '[price, points]'
if(map == 'ownedplayers')
{
Object.entries(json[map]).forEach((players) => {
if(players[0].localeCompare(value) == 0) {
console.log(players[1][1]);
//Add points to respective player field
//PROBABLY NOT HOW TO CHANGE A DOCUMENT FILED, THIS DOESNT WORK..
players[1][1]++;
}
});
//EACH TIME THIS RUNS IT SAYS: "Cannot modify a WriteBatch that has been committed"
batch.update(user, {'ownedplayers': json[map]});
}
}
});
}
});
}
} else {
console.log('user does not have a playermode document.');
}
});
});
return batch.commit().then(function () {
console.log("Succesfully commited changes.");
return null;
});
});
});
}).catch((err) => {
return err;
});
});
The issues i get in the console are "Cannot modify a WriteBatch that has been committed." and I fail to modify and add points to the player field inside the users document.
This is the console:
This is the firestore document structure:
I'm completely stuck on this.. Feels like I've tried all different approaches, but I think i dont fully understand cloud functions and javascript, so i would gladly recive feedback and help on how to make this work.
Cheers,
Finally.... i managed to update the document successfully. I put the commit outside another ".then()". Thought I tried that, but yay I guess :P
}).then(() => {
return batch.commit().then(function () {
console.log("Succesfully commited changes.");
return null;
});
The problem now is that it commits every loop. I think the most optimal here would be to batch update ALL users before committing?
And again, is there a more optimal way to do this, in terms of minimizing the operation and impact? I'm afraid I go too deep with for loops instead of directly navigating to the document, but haven't found an easier way to do that.
Any thoughts?
I want to download several data files from this URL: https://pselookup.vrymel.com/
The site contains a date field and a download button. I want to download data for multiple years (which would mean a lot of requests) and I want to make it automatically.
I've created a Javascript snippet, however, it keeps downloading just the same file over and over again.
$dateField = document.getElementsByClassName('csv_download_input__Input-encwx-1 dDiqPH')[2]
$dlButton = document.getElementsByClassName('csv_download_input__Button-encwx-0 KLfyv')[2]
var now = new Date();
var daysOfYear = [];
for (var d = new Date(2016, 0, 1); d <= now; d.setDate(d.getDate() + 1)) {
daysOfYear.push(new Date(d).toISOString().substring(0,10));
}
(function theLoop (i) {
setTimeout(function () {
$dlButton.click()
$dateField.value = daysOfYear[i]
if (--i) { // If i > 0, keep going
theLoop(i); // Call the loop again, and pass it the current value of i
}
}, 3000);
})(daysOfYear.length-1);
How could I download all of the files automatically?
First off, javascript in the client is probably not the best language to do this nor the best approach to make this happen. It might work, but it's better to know what is best when choosing an approach to a problem. Also, it will avoid for you clicking ~800 times in the popup accepting the download.
You can get the files in a programatically way by just learning what you browser is doing to get the file and trying to reproduce it in bunch.
After inspecting the calls you can see that it's calling an endpoint and that endpoint is returning a link which contains the file that you can download.
Well, that is going to be easy, so now you just need to make the script in any language to be able to retrieve them.
I've chosen javascript but not client side, but nodejs which means that this has to run from your computer.
You could do the same with bash, python or any other language.
To run this do the following:
Go to a new empty directory
Run npm install axios
Create a file with the code I pasted let's call it crawler.js
Run node crawler.js
This has been tested using node v8.15.0
// NOTE: Require this to make a request and save the link as file 20190813:Alevale
const axios = require('axios');
const fs = require('fs');
let now = new Date();
let daysOfYear = [];
const baseUrl = 'https://a4dzytphl9.execute-api.ap-southeast-1.amazonaws.com/prod/eod/'
for (var d = new Date(2016, 0, 1); d <= now; d.setDate(d.getDate() + 1)) {
daysOfYear.push(new Date(d).toISOString().substring(0,10));
}
const waitFor = (time) => {
return new Promise((resolve => setTimeout(resolve, time)))
}
const getUrls = async () =>{
let day
for (day of daysOfYear) {
console.log('getting day', baseUrl + day)
// NOTE: Throttle the calls to not overload the server 20190813:Alevale
await waitFor(4000)
await axios.get(baseUrl + day)
.then(response => {
console.log(response.data);
console.log(response);
if (response.data && response.data.download_url) {
return response.data.download_url
}
return Promise.reject('Could not retrieve response.data.download_url')
})
.then((url) =>{
axios({
method: 'get',
url,
responseType: 'stream'
})
.then(function (response) {
// NOTE: Save the file as 2019-08-13 20190813:Alevale
response.data.pipe(fs.createWriteStream(`${day}.csv`))
})
.catch(console.error)
})
.catch(error => {
console.log(error);
});
}
}
getUrls()
You can instead of simulating the user, get the link to download from:
https://a4dzytphl9.execute-api.ap-southeast-1.amazonaws.com/prod/eod/2019-08-07
just change the date at the end to the date of the file you want to download. And use axios to get this URL.
This will save you sometime (in case you don't really need to simulate the click of the user etc)
Then you will get a response like this:
{
download_url":"https://d3u9ukmkxau9he.cloudfront.net/eod/2019-08-07.csv?Expires=1566226156&Signature=QRUk3tstuNX5KYVPKJSWrXsSXatkWS-eFBIGUufaTEMJ~rgpVi0iPCe1AXl5pbQVdBQxOctpixCbyNz6b9ycDgYNxEdZqPr2o2pDe8cRL655d3zXdICnEGt~dU6p35iMAJkMpPSH~jbewhRSCPUwWXQBfOiEzlHwxru9lPnDfsdSnk3iI3GyR8Oc0ZP50EdUMHF7MjWSBRbCIwnu6wW4Jh0bPmZkQDQ63ms5QxehsmtuGLOgcrC6Ky1OffVQj~ihhmBt4LGhZTajjK4WO18hCP3urKt03qpC4bOvYvJ3pxvRkae0PH1f-vbTWMDkaWHHVCrzqZhkAh3FlvMTWj8D4g__&Key-Pair-Id=APKAIAXOVAEOGN2AYWNQ"
}
and then you can use axios to GET this url and download your file.
I'm learning FRP using Bacon.js, and would like to assemble data from a paginated API in a stream.
The module that uses the data has a consumption API like this:
// UI module, displays unicorns as they arrive
beautifulUnicorns.property.onValue(function(allUnicorns){
console.log("Got "+ allUnicorns.length +" Unicorns");
// ... some real display work
});
The module that assembles the data requests sequential pages from an API and pushes onto the stream every time it gets a new data set:
// beautifulUnicorns module
var curPage = 1
var stream = new Bacon.Bus()
var property = stream.toProperty()
var property.onValue(function(){}) # You have to add an empty subscriber, otherwise future onValues will not receive the initial value. https://github.com/baconjs/bacon.js/wiki/FAQ#why-isnt-my-property-updated
var allUnicorns = [] // !!! stateful list of all unicorns ever received. Is this idiomatic for FRP?
var getNextPage = function(){
/* get data for subsequent pages.
Skipping for clarity */
}
var gotNextPage = function (resp) {
Array.prototype.push.apply(allUnicorns, resp) // just adds the responses to the existing array reference
stream.push(allUnicorns)
curPage++
if (curPage <= pageLimit) { getNextPage() }
}
How do I subscribe to the stream in a way that provides me a full list of all unicorns ever received? Is this flatMap or similar? I don't think I need a new stream out of it, but I don't know. I'm sorry, I'm new to the FRP way of thinking. To be clear, assembling the array works, it just feels like I'm not doing the idiomatic thing.
I'm not using jQuery or another ajax library for this, so that's why I'm not using Bacon.fromPromise
You also may wonder why my consuming module wants the whole set instead of just the incremental update. If it were just appending rows that could be ok, but in my case it's an infinite scroll and it should draw data if both: 1. data is available and 2. area is on screen.
This can be done with the .scan() method. And also you will need a stream that emits items of one page, you can create it with .repeat().
Here is a draft code (sorry not tested):
var itemsPerPage = Bacon.repeat(function(index) {
var pageNumber = index + 1;
if (pageNumber < PAGE_LIMIT) {
return Bacon.fromCallback(function(callback) {
// your method that talks to the server
getDataForAPage(pageNumber, callback);
});
} else {
return false;
}
});
var allItems = itemsPerPage.scan([], function(allItems, itemsFromAPage) {
return allItems.concat(itemsFromAPage);
});
// Here you go
allItems.onValue(function(allUnicorns){
console.log("Got "+ allUnicorns.length +" Unicorns");
// ... some real display work
});
As you noticed, you also won't need .onValue(function(){}) hack, and curPage external state.
Here is a solution using flatMap and fold. When dealing with network you have to remember that the data can come back in a different order than you sent the requests - that's why the combination of fold and map.
var pages = Bacon.fromArray([1,2,3,4,5])
var requests = pages.flatMap(function(page) {
return doAjax(page)
.map(function(value) {
return {
page: page,
value: value
}
})
}).log("Data received")
var allData = requests.fold([], function(arr, data) {
return arr.concat([data])
}).map(function(arr) {
// I would normally write this as a oneliner
var sorted = _.sortBy(arr, "page")
var onlyValues = _.pluck(sorted, "value")
var inOneArray = _.flatten(onlyValues)
return inOneArray
})
allData.log("All data")
function doAjax(page) {
// This would actually be Bacon.fromPromise($.ajax...)
// Math random to simulate the fact that requests can return out
// of order
return Bacon.later(Math.random() * 3000, [
"Page"+page+"Item1",
"Page"+page+"Item2"])
}
http://jsbin.com/damevu/4/edit