//express is the framework we're going to use to handle requests
const express = require('express');
//Create a new instance of express
const app = express();
const FormData = require("form-data");
const bodyParser = require("body-parser");
const http = require('http');
const async = require('async');
//This allows parsing of the body of POST requests, that are encoded in JSON
app.use(bodyParser.json());
var router = express.Router();
//AccuWeather API key
const weatherKey = process.env.WEATHER_KEY_TWO;
cityCode = ""; //City code
cityName = "";
//Current Conditions Vars
var ccWeatherText = ""; //Text for weather at location
var ccTemp = 0; //Degrees Farenheit
var ccIcon = 0; //weather icon number https://developer.accuweather.com/weather-icons
var ccURL = "test"; //URL for get
var hourlyData = [];
var fiveDayData = [];
router.post('/', (req, res) => {
let lat = req.body['lat'];
let lon = req.body['lon'];
var latLongCityCodeURL = ("http://dataservice.accuweather.com/locations/v1/cities/geoposition/search?apikey=" + weatherKey + "&q=" + lat + "," + lon);
//Get city code
const httpGet = url => {
return new Promise((resolve, reject) => {
http.get(url, res => {
let body = '';
res.on('data', chunk => body += chunk);
res.on('end', () => {
try {
body = JSON.parse(body);
} catch (err) {
reject(new Error(err));
}
resolve({
code: body.Key,
name: body.EnglishName
});
});
}).on('error', reject);
});
};
//Current Conditions
const ccGet = url => {
return new Promise((resolve, reject) => {
http.get(url, res => {
let body = '';
res.on('data', chunk => body += chunk);
res.on('end', () => {
try {
body = JSON.parse(body);
} catch (err) {
reject(new Error(err));
}
resolve({
text: body[0].WeatherText,
temp: body[0].Temperature.Imperial.Value,
icon: body[0].WeatherIcon
});
});
}).on('error', reject);
});
};
//12 hour
const twelveGet = url => {
return new Promise((resolve, reject) => {
http.get(url, res => {
let body = '';
res.on('data', chunk => body += chunk);
res.on('end', () => {
try {
body = JSON.parse(body);
} catch (err) {
reject(new Error(err));
}
resolve({
body: body
});
});
}).on('error', reject);
});
};
//5 day
const fiveGet = url => {
return new Promise((resolve, reject) => {
http.get(url, res => {
let body = '';
res.on('data', chunk => body += chunk);
res.on('end', () => {
try {
body = JSON.parse(body);
} catch (err) {
reject(new Error(err));
}
resolve({
body: body
});
});
}).on('error', reject);
});
};
//Get city code from lat lon
httpGet(latLongCityCodeURL).then(data => {
cityCode = data.code;
cityName = data.name;
ccURL = ("http://dataservice.accuweather.com/currentconditions/v1/" + cityCode + "?apikey=" + weatherKey);
twelveURL = ("http://dataservice.accuweather.com/forecasts/v1/hourly/12hour/" + cityCode + "?apikey=" + weatherKey);
fiveURL = ("http://dataservice.accuweather.com/forecasts/v1/daily/5day/" + cityCode + "?apikey=" + weatherKey);
//Get Current Conditions
ccGet(ccURL).then(dataCC => {
ccTemp = dataCC.temp;
ccWeatherText = dataCC.text;
ccIcon = dataCC.icon;
//Get 12 hour forecast
twelveGet(twelveURL).then(dataTwelve => {
//Generate hourly data
for (i = 0; i < dataTwelve.length; i++) {
hourlyData[i] = {
time: dataTwelve[i].EpochDateTime,
temp: dataTwelve[i].Temperature.Value,
text: dataTwelve[i].IconPhrase,
icon: dataTwelve[i].WeatherIcon
};
}
console.log("Hourly Data: " + hourlyData);
}).catch(err => console.log(err));
fiveGet(fiveURL).then(dataFive => {
//Generate five day data
for (i = 0; i < dataFive.length; i++) {
fiveDayData[i] = {
time: dataFive[i].EpochDate,
min: dataFive[i].Temperature.Minimum.Value,
max: dataFive[i].Temperature.Maximum.Value,
iconDay: dataFive[i].Day.Icon,
iconNight: dataFive[i].Night.Icon,
dayPhrase: dataFive[i].Day.IconPhrase,
nightPhrase: dataFive[i].Night.IconPhrase
};
console.log("5 Day Data:" + fiveDayData);
}
res.send({
success: true,
cityName: cityName,
cityCode: cityCode,
currentConditions: {
temp: ccTemp,
icon: ccIcon,
text: ccWeatherText
},
hourlyData: hourlyData,
fiveDayData: fiveDayData
});
}).catch(err => console.log(err));
}).catch(err => console.log(err));
}).catch(err => console.log('Got error ', err));
});
module.exports = router;
Ok so right now I'm creating an endpoint in NodeJS that is POST method which gets the arguments for latitude and longitude. When it gets those it makes calls to Accuweather's API. I got all the accuweather stuff working and returning proper results, but then I cut and pasted that code into my POST method router.post... and now it isn't working. I know it is an ASYNC issue, and I am just getting really lost with async, since I have like 3 or 4 nested async calls inside the router.post, which is another async call. So I'm thinking there is some way to maybe wrap the router.post into its own async call, which waits on the weather calls before returning results?
My end goal: For the user to send a POST with lat and lon, my code does all the weather calls, and returns the data for the POST.
What you want to do is called promise chaining:
new Promise(function(resolve, reject) {
setTimeout(() => resolve(1), 1000); // (*)
}).then(function(result) { // (**)
alert(result); // 1
return result * 2;
}).then(function(result) { // (***)
alert(result); // 2
return result * 2;
}).then(function(result) {
alert(result); // 4
return result * 2;
});
Also, check out MDN's page on using promises
Related
I have the following NodeJS code running in an AWS Lambda:
const https = require('https');
exports.handler = async event => {
try {
const currentWeekNumber = getWeekNumber(new Date(Date.now()));
const currentTournamentId = await getCurrentTournamentIdByWeekNumber(currentWeekNumber);
// note: Issue is currentTournamentId is returned as undefined from above, so below method fails
const leaderBoard = await getLeaderboardByTournId(currentTournamentId);
return {
statusCode: 200,
headers: {'Content-Type': 'application/json'},
body: JSON.stringify(leaderBoard),
};
} catch (error) {
console.log('Error: ️', error);
return {
statusCode: 400,
body: error.message,
};
}
};
function getCurrentTournamentIdByWeekNumber(currentWeekNumber) {
const promise = new Promise((resolve, reject) => {
const options = {
host: 'MY_HOST',
path: '/MY_PATH',
headers: {
'key': 'value'
}
};
const req = https.get(options, res => {
let rawData = '';
res.on('data', chunk => {
rawData += chunk;
});
res.on('end', () => {
try {
resolve(JSON.parse(rawData));
} catch (err) {
reject(new Error(err));
}
});
});
req.on('error', err => {
reject(new Error(err));
});
});
promise.then(result => {
for (let i = 0; i <result.schedule.length; i++) {
let weekNumber = result.schedule[i].date.weekNumber;
if(currentWeekNumber == weekNumber) {
return result.schedule[i].tournId;
}
}
});
}
function getLeaderboardByTournId(tournId) {
return new Promise((resolve, reject) => {
const options = {
host: 'MY_HOST',
path: '/MY_PATH',
headers: {
'key': 'value'
}
};
const req = https.get(options, res => {
let rawData = '';
res.on('data', chunk => {
rawData += chunk;
});
res.on('end', () => {
try {
resolve(JSON.parse(rawData));
} catch (err) {
reject(new Error(err));
}
});
});
req.on('error', err => {
reject(new Error(err));
});
});
}
function getWeekNumber(d) {
// Copy date so don't modify original
d = new Date(Date.UTC(d.getFullYear(), d.getMonth(), d.getDate()));
// Set to nearest Thursday: current date + 4 - current day number
// Make Sunday's day number 7
d.setUTCDate(d.getUTCDate() + 4 - (d.getUTCDay()||7));
// Get first day of year
var yearStart = new Date(Date.UTC(d.getUTCFullYear(),0,1));
// Calculate full weeks to nearest Thursday
var weekNo = Math.ceil(( ( (d - yearStart) / 86400000) + 1)/7);
// Return array of year and week number
return weekNo;
}
I am having an issue where currentTournamentId is being passed into getLeaderboardByTournId as undefined, I think this is due to the execution order of my functions.
I know that the logic is correct in all the functions.
How can I ensure that getCurrentTournamentIdByWeekNumber completes and returns a result before getLeaderboardByTournId is called?
This is my first attempt at node lambda and only getting to grips with JS execution order!
You are correctly awaiting your getCurrentTournamentIdByWeekNumber method, however inside of that method, you're not returning anything so there's no promise to wait for.
If you're add a return it will probably work:
return promise.then(result => {
...
I have a Cloud Function written in Node JS that accesses data from BigQuery, converts this to CSV and exports to a Google Storage bucket.
Currently it executes and returns a 200, but does not run any of the code within my try/catch.
When testing it just returns:
Function execution took x ms. Finished with status code: 200
I've attempted to debug by adding console logs at various points, but it doesn't log anything - it just returns a 200.
exports.run_checks = (req, res) => {
"use strict";
let parsedBody = req.body;
let startCount = parsedBody.start;
let endCount = parsedBody.end;
(async function () {
try {
for (let i = startCount; i < endCount; i += 1) {
//Exclude overly large files here
if (i != 100) {
const query =
`SELECT *
FROM \`bq_dataset\`
WHERE id_number = ${i}`;
const options = {
query: query,
location: "europe-west2",
};
const [job] = await bigquery.createQueryJob(options);
console.log(`Job ${job.id} started.`);
const [rows] = await job.getQueryResults();
let id = rows[0].id;
const createFile = storage.bucket(bucketName).file(`${id}.csv`);
const csv = parse(rows, { fields });
const dataStream = new stream.PassThrough();
dataStream.push(csv);
dataStream.push(null);
await new Promise((resolve, reject) => {
console.log(`Writing ${id} to GCS`);
dataStream
.pipe(
createFile.createWriteStream({
resumable: false,
validation: false,
metadata: { "Cache-Control": "public, max-age=31536000" },
})
)
.on("error", (error) => {
console.error("Stream failed", error);
reject(error);
})
.on("finish", () => {
resolve(true);
});
});
}
}
res.status(200).send();
} catch (err) {
res.send(err);
}
})();
};
Your function is not async. The host has no idea that you are still doing something in your function, it returns without any error.
Modify your arrow function to async, and no need for IIFE, remove it, or await it, that is also important!
exports.run_checks = async (req, res) => {
"use strict";
let parsedBody = req.body;
let startCount = parsedBody.start;
let endCount = parsedBody.end;
try {
for (let i = startCount; i < endCount; i += 1) {
//Exclude overly large files here
if (i != 100) {
const query =
`SELECT *
FROM \`bq_dataset\`
WHERE id_number = ${i}`;
const options = {
query: query,
location: "europe-west2",
};
const [job] = await bigquery.createQueryJob(options);
console.log(`Job ${job.id} started.`);
const [rows] = await job.getQueryResults();
let id = rows[0].id;
const createFile = storage.bucket(bucketName).file(`${id}.csv`);
const csv = parse(rows, { fields });
const dataStream = new stream.PassThrough();
dataStream.push(csv);
dataStream.push(null);
await new Promise((resolve, reject) => {
console.log(`Writing ${id} to GCS`);
dataStream
.pipe(
createFile.createWriteStream({
resumable: false,
validation: false,
metadata: { "Cache-Control": "public, max-age=31536000" },
})
)
.on("error", (error) => {
console.error("Stream failed", error);
reject(error);
})
.on("finish", () => {
resolve(true);
});
});
}
}
res.status(200).send();
} catch (err) {
res.send(err);
}
};
I'm trying to use await on var application = await SchedulerService().getIssues(issueId)
And it returns the error: SyntaxError: await is only valid in async function
I'm starting in node.js. What can I do to fix it?
I've tried already
Add async before initial function const SchedulerService = await function(){ at line 1
Add async on first return return async () => { where's return { at line 3
import schedulerConf from '../../config/scheduler';
import authConf from '../../config/auth';
import applicationConf from '../../config/application';
import request from 'request';
import schedule from 'node-schedule';
import dateformat from 'dateformat';
let interations = 0;
var serviceRecords = [];
var issueRecords = [];
const SchedulerService = function(){
return {
initialize: async () => {
console.log(`***** Starting Scheduler on ${dateformat(new Date(), "dd/mm/yyyy HH:MM:ss")}`);
var j = schedule.scheduleJob('*/1 * * * *', function(){
console.time('└─ Scheduler execution time');
if(interations === 0){
console.log(`Setting scheduler runtime to full time.`);
}else{
console.log(`------------------------`);
}
interations++;
console.log(`Job execution number: ${interations}.`);
SchedulerService().execute()
.then(response => {
console.log(`└─ Job ${interations} was successfully executed.`);
console.log(`└─ Execution date ${dateformat(new Date(), "dd/mm/yyyy HH:MM:ss")}`);
console.timeEnd('└─ Scheduler execution time');
}).catch(error => {
console.log(`└─ Job ${interations} returned error while executing.`);
});
});
},
execute: async () => {
return SchedulerService().getRecords2Sync()
.then(() => {
SchedulerService().sync().then(() => {
}).catch(error => {console.log({error})});
}).catch(error => {console.log({error})});
},
getRecords2Sync: async () => {
serviceRecords = [];
var options = {
url: `http://localhost:${authConf.serverPort}/application`,
method: 'GET',
headers: {
authorization: `${authConf.secret}`
}
}
return new Promise(function (resolve, reject) {
request(options, function (error, res, body) {
if (!error && res.statusCode == 200) {
const srs = JSON.parse(body);
const response = srs['response'];
for(let i =0;i < response.length;i++){
const { id, info } = response[i];
var status = "";
var issueId = "";
var statusClass = "";
for(let x = 0; x < info.length; x++){
if(info[x].key === "status"){
status = info[x].value;
statusClass = info[x].valueClass;
}
if(info[x].key === applicationConf.fields.issueId){
issueId = info[x].value;
}
}
if(statusClass === 0){
if(issueId !== null && issueId !== ""){
serviceRecords.push({id, status, issueId});
}
}
}
//console.log(serviceRecords);
resolve(serviceRecords);
} else {
//console.log(error);
reject(error);
}
});
});
},
getIssues : async (issueId) => {
issueRecords = [];
return new Promise(function(resolve, reject) {
var options = {
url: `http://localhost:${authConf.serverPort}/application2/${issueId}`,
method: 'GET',
headers: {
authorization: `${authConf.secret}`
}
}
request(options, function(error, res, body) {
if (!error && res.statusCode == 200) {
const issues = JSON.parse(body);
const { issue } = issues.response;
const { id, status, custom_fields } = issue;
issueRecords.push({id, status, custom_fields});
resolve(issueRecords);
} else {
reject(error);
}
});
});
},
sync : async () => {
return new Promise(function(resolve, reject) {
for (let i = 0; i < serviceRecords.length; i++) {
const application_id = serviceRecords[i].id;
const application_status = serviceRecords[i].status;
const application_issueId = serviceRecords[i].issueId;
//console.log('issueRecords.length: ', issueRecords);
//console.log('issueRecords.length: ', issueRecords.length);
console.log(`********** application`);
console.log(`└─ id ${application_id}`);
console.log(`└─ status ${application_status}`);
console.log(`└─ issueId ${application_issueId}`);
var application2 = await SchedulerService().getIssues(application_issueId)
.then(response => {
resolve(() => {
console.log(`i've found a record by issue_id ${application_issueId}`);
});
}).catch(error => {
reject(error);
});
}
});
}
}
}
export default new SchedulerService();
Thank you so much!
If you had getIssues resolve with issueId and issueRecords you might do something like this in sync:
sync: async () => {
// `map` over the serviceRecords and return a getIssues promise for each issueId
const promises = serviceRecords.map(({ issueId }) => SchedulerService().getIssues(issueId));
// Wait for all the promises to resolve
const data = await Promise.all(promises);
// Loop over the resolved promises and log the issueId
data.forEach((issueId, issueRecords) => console.log(issueId));
}
I need to save data from GET requests to a variable and then save it in a file. However, in some cases GET request does not save data to global variables.
var fs = require("fs");
var http = require("http");
var request = require('request');
var tmp_json = {};
var g_last = 0;
var data = {};
//request 1
http.get('server:api', (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
tmp_json.server1 = {};
tmp_json.server1 = JSON.parse(data);
g_last = tmp_json.height; // 100500
console.log(g_last); // 100500
});
}).on("error", (err) => {
console.log("Error: " + err.message);
});
//request 2
http.get('server2:api', (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
tmp_json.server2 = {};
tmp_json.server2 = JSON.parse(data);
g_last = tmp_json.height; // 256
console.log(g_last); // 256
});
}).on("error", (err) => {
console.log("Error: " + err.message);
});
console.log(g_last); // 0
data = JSON.stringify(tmp_json);
fs.writeFile('data.json', data, 'utf8'); // empty file
Also I was trying to do it with fs.createWriteStream, but again I can save one request to file, but if there more then one I catch only buffer data.
Your problem is that request1 and request2 are happening while you are writing the file. This is because of the async nature of node. The execution order looks something like this:
Declare empty variables
Request1 goes out
Request2 goes out
Write empty variables to file
Request1 comes back and writes to variables
Request2 comes back and writes to variables
One way to fix this would be with Promises. The following allows for the function in then to be executed after the promises in Promise.all([ ... ]) have resolved:
var fs = require("fs");
var http = require("http");
var tmp_json = {};
var g_last = 0;
var data = {};
//request 1
var req1 = new Promise((resolve, reject) => {
http.get('server:api', (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
tmp_json.server1 = {};
tmp_json.server1 = JSON.parse(data);
g_last = tmp_json.height; // 100500
console.log(g_last); // 100500
resolve()
});
}).on("error", (err) => {
console.log("Error: " + err.message);
reject(error)
});
});
//request 2
var req2 = new Promise((resolve, reject) => {
http.get('server2:api', (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
tmp_json.server2 = {};
tmp_json.server2 = JSON.parse(data);
g_last = tmp_json.height; // 256
console.log(g_last); // 256
resolve()
});
}).on("error", (err) => {
console.log("Error: " + err.message);
reject(error)
});
});
Promise.all([ req1, req2 ]).then(() => {
console.log(g_last);
data = JSON.stringify(tmp_json);
fs.writeFile('data.json', data, 'utf8');
})
Edit:
function handleGet (url) {
return new Promise((resolve, reject) => {
http.get(url, (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
tmp_json.server1 = {};
tmp_json.server1 = JSON.parse(data);
g_last = tmp_json.height; // 100500
console.log(g_last); // 100500
resolve()
});
}).on("error", (err) => {
console.log("Error: " + err.message);
reject(error)
});
})
}
// Then use it
Promise.all([
handleGet('http://google.ca'),
handleGet('http://somesite.com')
])
I'm trying to download every image on a website using a node script.
I wrote it and for the most part it seems to be working, however, it's only downloading the first image. It downloads it equal to the number of images on the site.
Here is my code.
const http = require('http'),
cheerio = require('cheerio'),
fs = require('fs');
var document = '';
var imageData = '';
http.get('http://www.wikihow.com/Start-a-Blog', function(res){
res.on('data', function(chunk){
document+=chunk;
})
res.on('end', function(){
let $ = cheerio.load(document);
var array = [];
var array = $("img").toArray();
var data = [];
array.forEach(function (ele) {
if (ele.attribs.src !== undefined)
data.push(ele.attribs.src);
})
var counter = 0;
data.forEach(function (ele) {
ripImage(ele, counter);
counter ++;
})
})
});
function ripImage(ele, counter){
http.get(ele, function(res){
console.log(res);
res.setEncoding('binary')
res.on('data', function(chunk){
imageData += chunk;
})
res.on('end', function(){
//console.log(ele);
fs.writeFile("dump/file" + counter + ".jpg", imageData, 'binary', function(err){
if (err) throw err
//console.log('File saved.')
});
//res.pipe(file);
})
});
}
I think the problem lies somewhere in the ripImage() function. If you guys can see the problem, and help me fix it, that'd be really appreciated.
Thanks guys.
#Mr.Phoenix is right, the async library is meant for this type of thing. It allows you to iterate over a collection with an asynchronous function, and fire a callback when all of the async functions have completed. Working code:
const http = require('http')
const cheerio = require('cheerio')
const fs = require('fs')
const async = require('async')
let document = ''
http.get('http://www.wikihow.com/Start-a-Blog', (res) => {
res.on('data', (chunk) => {
document += chunk
})
res.on('end', () => {
const $ = cheerio.load(document)
const data = $('img')
.toArray()
.filter((ele) => ele.attribs.src)
.map((ele) => ele.attribs.src)
async.eachOf(data, ripImage, (err) => {
if (err) throw err
console.log('all done!')
})
})
})
function ripImage (ele, i, callback) {
http.get(ele, (res) => {
let imageData = ''
res.setEncoding('binary')
res.on('data', (chunk) => {
imageData += chunk
})
res.on('end', () => {
fs.writeFile('./dump/file' + i + '.jpg', imageData, 'binary', callback)
})
})
}