I'm trying to use await on var application = await SchedulerService().getIssues(issueId)
And it returns the error: SyntaxError: await is only valid in async function
I'm starting in node.js. What can I do to fix it?
I've tried already
Add async before initial function const SchedulerService = await function(){ at line 1
Add async on first return return async () => { where's return { at line 3
import schedulerConf from '../../config/scheduler';
import authConf from '../../config/auth';
import applicationConf from '../../config/application';
import request from 'request';
import schedule from 'node-schedule';
import dateformat from 'dateformat';
let interations = 0;
var serviceRecords = [];
var issueRecords = [];
const SchedulerService = function(){
return {
initialize: async () => {
console.log(`***** Starting Scheduler on ${dateformat(new Date(), "dd/mm/yyyy HH:MM:ss")}`);
var j = schedule.scheduleJob('*/1 * * * *', function(){
console.time('└─ Scheduler execution time');
if(interations === 0){
console.log(`Setting scheduler runtime to full time.`);
}else{
console.log(`------------------------`);
}
interations++;
console.log(`Job execution number: ${interations}.`);
SchedulerService().execute()
.then(response => {
console.log(`└─ Job ${interations} was successfully executed.`);
console.log(`└─ Execution date ${dateformat(new Date(), "dd/mm/yyyy HH:MM:ss")}`);
console.timeEnd('└─ Scheduler execution time');
}).catch(error => {
console.log(`└─ Job ${interations} returned error while executing.`);
});
});
},
execute: async () => {
return SchedulerService().getRecords2Sync()
.then(() => {
SchedulerService().sync().then(() => {
}).catch(error => {console.log({error})});
}).catch(error => {console.log({error})});
},
getRecords2Sync: async () => {
serviceRecords = [];
var options = {
url: `http://localhost:${authConf.serverPort}/application`,
method: 'GET',
headers: {
authorization: `${authConf.secret}`
}
}
return new Promise(function (resolve, reject) {
request(options, function (error, res, body) {
if (!error && res.statusCode == 200) {
const srs = JSON.parse(body);
const response = srs['response'];
for(let i =0;i < response.length;i++){
const { id, info } = response[i];
var status = "";
var issueId = "";
var statusClass = "";
for(let x = 0; x < info.length; x++){
if(info[x].key === "status"){
status = info[x].value;
statusClass = info[x].valueClass;
}
if(info[x].key === applicationConf.fields.issueId){
issueId = info[x].value;
}
}
if(statusClass === 0){
if(issueId !== null && issueId !== ""){
serviceRecords.push({id, status, issueId});
}
}
}
//console.log(serviceRecords);
resolve(serviceRecords);
} else {
//console.log(error);
reject(error);
}
});
});
},
getIssues : async (issueId) => {
issueRecords = [];
return new Promise(function(resolve, reject) {
var options = {
url: `http://localhost:${authConf.serverPort}/application2/${issueId}`,
method: 'GET',
headers: {
authorization: `${authConf.secret}`
}
}
request(options, function(error, res, body) {
if (!error && res.statusCode == 200) {
const issues = JSON.parse(body);
const { issue } = issues.response;
const { id, status, custom_fields } = issue;
issueRecords.push({id, status, custom_fields});
resolve(issueRecords);
} else {
reject(error);
}
});
});
},
sync : async () => {
return new Promise(function(resolve, reject) {
for (let i = 0; i < serviceRecords.length; i++) {
const application_id = serviceRecords[i].id;
const application_status = serviceRecords[i].status;
const application_issueId = serviceRecords[i].issueId;
//console.log('issueRecords.length: ', issueRecords);
//console.log('issueRecords.length: ', issueRecords.length);
console.log(`********** application`);
console.log(`└─ id ${application_id}`);
console.log(`└─ status ${application_status}`);
console.log(`└─ issueId ${application_issueId}`);
var application2 = await SchedulerService().getIssues(application_issueId)
.then(response => {
resolve(() => {
console.log(`i've found a record by issue_id ${application_issueId}`);
});
}).catch(error => {
reject(error);
});
}
});
}
}
}
export default new SchedulerService();
Thank you so much!
If you had getIssues resolve with issueId and issueRecords you might do something like this in sync:
sync: async () => {
// `map` over the serviceRecords and return a getIssues promise for each issueId
const promises = serviceRecords.map(({ issueId }) => SchedulerService().getIssues(issueId));
// Wait for all the promises to resolve
const data = await Promise.all(promises);
// Loop over the resolved promises and log the issueId
data.forEach((issueId, issueRecords) => console.log(issueId));
}
Related
I am not sure if I am approaching this the correct way. I have tried a few different versions of the implementation but am putting the one here that works when the backoff path is NOT used.
So, I have an index.js that is just:
import { Lizard } from './lizard.js';
const lizard = new Lizard();
const global_data = await lizard.global();
console.log(global_data);
In my lizard.js I have a class with functions but for the sake of saving space and noise I will only place the ones that matter here:
export class Lizard {
global() {
const path = '/global';
return this._request(path);
};
_buildRequestOptions(path, params) {
if (isObject(params)) params = querystring.stringify(params);
else params = undefined;
if (params == undefined) path = `/api/v${API_VERSION}${path}`;
else path = `/api/v${API_VERSION}${path}?${params}`;
// Return options
return {
path,
method: 'GET',
host: HOST,
port: 443,
timeout: Lizard.TIMEOUT,
};
};
async _request(path, params) {
const options = this._buildRequestOptions(path, params);
const maxRetries = 10;
function waitFor(milliseconds) {
return new Promise((resolve) => setTimeout(resolve, milliseconds));
}
async function request(options, retries) {
if (retries > 0) {
const timeToWait = 15000 * retries;
console.log(`waiting for ${timeToWait}ms...`);
await waitFor(timeToWait);
}
return new Promise((resolve, reject) => {
const req = https.request(options, (res) => {
let body = [];
res.on('data', (chunk) => {
body.push(chunk);
});
res.on('end', () => {
try {
body = Buffer.concat(body);
body = body.toString();
if (body.startsWith('<!DOCTYPE html>')) {
_WARN_('Invalid request', 'There was a problem with your request. The parameter(s) you gave are missing or incorrect.');
} else if (body.startsWith('Throttled')) {
_WARN_('Throttled request', 'There was a problem with request limit.');
}
body = JSON.parse(body);
} catch (error) {
reject(error);
};
const returnObject = ReturnObject(
!(res.statusCode < 200 || res.statusCode >= 300),
res.statusMessage,
res.statusCode,
body
)
if (returnObject.code != 429) {
resolve(returnObject);
} else {
if (retries < maxRetries) {
console.log('retrying...');
return request(options, retries + 1);
} else {
console.log("Max retries reached. Bubbling the error up");
resolve(returnObject);
}
}
});
});
req.on('error', (error) => reject(error));
req.on('timeout', () => {
req.abort();
reject(new Error(`Lizard API request timed out. Current timeout is: ${Lizard.TIMEOUT} milliseconds`));
});
// End request
req.end();
});
}
return await request(options, 0);
};
}
I was trying to do this in a very difficult way. For anyone else that may stumble upon this here was my ultimate solution:
lizard.js:
async function request(path, params, retries = 0, maxRetries = 10) {
let options = await buildRequestOptions(path, params);
return new Promise((resolve, reject) => {
let req = https.request(options, (res) => {
let body = [];
// Set body on data
res.on('data', (chunk) => {
body.push(chunk);
});
// On end, end the Promise
res.on('end', async () => {
try {
body = Buffer.concat(body);
body = body.toString();
// Check if page is returned instead of JSON
if (body.startsWith('<!DOCTYPE html>')) {
_WARN_('Invalid request', 'There was a problem with your request. The parameter(s) you gave are missing or incorrect.');
} else if (body.startsWith('Throttled')) {
_WARN_('Throttled request', 'There was a problem with request limit.');
}
// Attempt to parse
body = JSON.parse(body);
} catch (error) {
reject(error);
};
if (res.statusCode == 429 && retries < maxRetries) {
const timeToWait = 60000 + (1000 * retries);
console.error('Throttled request ' + retries + ' time(s)');
console.log(`Retrying in ${timeToWait}ms...`);
setTimeout(() => {
resolve(request(path, params, retries + 1));
}, timeToWait);
} else {
resolve(
objectify(
!(res.statusCode < 200 || res.statusCode >= 300),
res.statusMessage,
res.statusCode,
body
)
);
}
});
});
// On error, reject the Promise
req.on('error', (error) => reject(error));
// On timeout, reject the Promise
req.on('timeout', () => {
req.abort();
reject(new Error(`Lizard API request timed out. Current timeout is: ${TIMEOUT} milliseconds`));
});
// End request
req.end();
});
};
I still resolve the object on fail as 429 is too many requests, so anything else needs to bubble up. On top of that if max retries is met, then if I see a 429 I know that I exceeded.
How to return a Promise from the multiple axios get requests?
I have below code.
async function main() {
const URL_1 = 'abc.com/get1/data1';
const result_1 = await getData(URL_1);
const URL_2 = 'abc.com/get2/data2';
const result_2 = await getData(URL_2);
}
async function getData(dataURI) {
let getURI = dataURI;
const config = {
headers: {
Authorization: `Bearer ${my-token-text}`,
},
};
var finalData = [];
// until we get the next URL keep sending the requests
while (getURI != null) {
try {
const getResult = await axios.get(getURI, config);
if (getResult.status === 200) {
const receivedData = getResult.data.value;
finalData.push(...receivedData);
// check if we have nextLink in the payload
if (Object.prototype.hasOwnProperty.call(getResult.data, 'nextLink')) {
getURI = getResult.data.nextLink;
} else {
getURI = null;
return finalData;
}
}
} catch (err) {
break;
}
}
return null;
}
What I am trying to achieve is:
async function main() {
const URL_1 = 'abc.com/get1/data1';
const result_1 = getData(URL_1);
promisesArray.push(result_1);
const URL_2 = 'abc.com/get2/data2';
const result_2 = getData(URL_2);
promisesArray.push(result_2);
await Promise.allSettled(promisesArray).then((results) => {
console.log('Promise All Done: ', results);
});
}
This why I can perform all the requests in parallel.
But when I update the function getData(dataURI) to return return new Promise then I get error for await axios.
async function getData(dataURI) {
return new Promise((resolve, reject) => {
// Same code as above
});
}
I get error:
SyntaxError: await is only valid in async function
As Promise is not async I cannot await in the Promise.
Have you tried:
return new Promise(async (resolve, reject) => {
// Same code as above
});
I have a Cloud Function written in Node JS that accesses data from BigQuery, converts this to CSV and exports to a Google Storage bucket.
Currently it executes and returns a 200, but does not run any of the code within my try/catch.
When testing it just returns:
Function execution took x ms. Finished with status code: 200
I've attempted to debug by adding console logs at various points, but it doesn't log anything - it just returns a 200.
exports.run_checks = (req, res) => {
"use strict";
let parsedBody = req.body;
let startCount = parsedBody.start;
let endCount = parsedBody.end;
(async function () {
try {
for (let i = startCount; i < endCount; i += 1) {
//Exclude overly large files here
if (i != 100) {
const query =
`SELECT *
FROM \`bq_dataset\`
WHERE id_number = ${i}`;
const options = {
query: query,
location: "europe-west2",
};
const [job] = await bigquery.createQueryJob(options);
console.log(`Job ${job.id} started.`);
const [rows] = await job.getQueryResults();
let id = rows[0].id;
const createFile = storage.bucket(bucketName).file(`${id}.csv`);
const csv = parse(rows, { fields });
const dataStream = new stream.PassThrough();
dataStream.push(csv);
dataStream.push(null);
await new Promise((resolve, reject) => {
console.log(`Writing ${id} to GCS`);
dataStream
.pipe(
createFile.createWriteStream({
resumable: false,
validation: false,
metadata: { "Cache-Control": "public, max-age=31536000" },
})
)
.on("error", (error) => {
console.error("Stream failed", error);
reject(error);
})
.on("finish", () => {
resolve(true);
});
});
}
}
res.status(200).send();
} catch (err) {
res.send(err);
}
})();
};
Your function is not async. The host has no idea that you are still doing something in your function, it returns without any error.
Modify your arrow function to async, and no need for IIFE, remove it, or await it, that is also important!
exports.run_checks = async (req, res) => {
"use strict";
let parsedBody = req.body;
let startCount = parsedBody.start;
let endCount = parsedBody.end;
try {
for (let i = startCount; i < endCount; i += 1) {
//Exclude overly large files here
if (i != 100) {
const query =
`SELECT *
FROM \`bq_dataset\`
WHERE id_number = ${i}`;
const options = {
query: query,
location: "europe-west2",
};
const [job] = await bigquery.createQueryJob(options);
console.log(`Job ${job.id} started.`);
const [rows] = await job.getQueryResults();
let id = rows[0].id;
const createFile = storage.bucket(bucketName).file(`${id}.csv`);
const csv = parse(rows, { fields });
const dataStream = new stream.PassThrough();
dataStream.push(csv);
dataStream.push(null);
await new Promise((resolve, reject) => {
console.log(`Writing ${id} to GCS`);
dataStream
.pipe(
createFile.createWriteStream({
resumable: false,
validation: false,
metadata: { "Cache-Control": "public, max-age=31536000" },
})
)
.on("error", (error) => {
console.error("Stream failed", error);
reject(error);
})
.on("finish", () => {
resolve(true);
});
});
}
}
res.status(200).send();
} catch (err) {
res.send(err);
}
};
I am new to Scraping.
This is my PDF downloading code.
I want to use Async Await in this code.
I don't know where I have to use async await in my code.
function scrapPdf(config, search_url, message) {
console.log('PDF downloading');
got(search_url).then(response => {
const $ = cheerio.load(response.body);
$('.search-result').find('li > a').each((idx, elem) => {
if($(elem).text().trim() == 'PDF'){
const item = $(elem).attr('href');
pdf_lists.push(item);
}
})
$('ul.pagination').find('li.page-item').each((idx, elem) => {
if($(elem).attr('class').includes('page-item active navigation')){
if($(elem).next().hasClass('page-item navigation')){
scrapPdf(config, $(elem).next('li').find('a').attr('href'), message);
} else {
const search_result_dir = `./${message.date_ini}-${message.date_end}`;
if(!fs.existsSync(search_result_dir)){
fs.mkdirSync(search_result_dir)
}
for(let i = 0;i < pdf_lists.length; i++){
const download = new DownloaderHelper(pdf_lists[i], search_result_dir);
download.on('end', () => console.log('Download Completed'))
download.start();
}
console.log(`${pdf_lists.length} files Downloaded!`);
uploadFile(search_result_dir);
return ;
}
console.log($(elem).next('li').find('a').attr('href'));
}
});
}).catch(err => {
console.log(err);
});
}
Here's one possibility assuming you only want to use async/await on the Promise returned from the .then:
async function scrapPdf(config, search_url, message) {
let response = null
console.log('PDF downloading');
try {
response = await got(search_url);
} catch (err) {
console.log(err);
}
if (response) {
const $ = cheerio.load(response.body);
$('.search-result').find('li > a').each((idx, elem) => {
if($(elem).text().trim() == 'PDF'){
const item = $(elem).attr('href');
pdf_lists.push(item);
}
})
$('ul.pagination').find('li.page-item').each((idx, elem) => {
if($(elem).attr('class').includes('page-item active navigation')){
if($(elem).next().hasClass('page-item navigation')){
scrapPdf(config, $(elem).next('li').find('a').attr('href'), message);
} else {
const search_result_dir = `./${message.date_ini}-${message.date_end}`;
if(!fs.existsSync(search_result_dir)){
fs.mkdirSync(search_result_dir)
}
for(let i = 0;i < pdf_lists.length; i++){
const download = new DownloaderHelper(pdf_lists[i], search_result_dir);
download.on('end', () => console.log('Download Completed'))
download.start();
}
console.log(`${pdf_lists.length} files Downloaded!`);
uploadFile(search_result_dir);
return ;
}
console.log($(elem).next('li').find('a').attr('href'));
}
});
}
}
I tried to make the code asynchronous but I couldn't. What i need to do?
This is my functions:
1.
router.post('/urls', (req, response) => {
count = 2;
webUrl = req.body.url;
depth = req.body.depth;
letstart(webUrl, response);
});
function letstart(urlLink, response) {
request(urlLink, function (error, res, body) {
console.error('error:', error); // Print the error if one occurred
console.log('statusCode:', res && res.statusCode); // Print the response status code if a response was received
//console.log('body:', body); // Print the HTML for the Google homepage.
if (!error) {
getLinks(body);
if (!ifFinishAll) {
GetinsideLinks(linkslinst, response);
}
else {
console.log("Finish crawl");
}
}
else {
console.log("sorry");
return "sorry";
}
});
}
function GetinsideLinks(list, response) {
count++;
if (count <= depth) {
for (let i = 0; i < list.length; i++) {
const link = list[i].toString();
var includeUrl = link.includes(webUrl);
if (!includeUrl) {
request(link, function (error, res, body) {
console.error('error2:', error); // Print the error if one occurred
console.log('statusCode2:', res && res.statusCode); // Print the response status code if a response was received
if (!error) {
getLinks(body);
}
else {
console.log("sorry2");
}
});
}
}
ifFinishAll = true;
}
else {
console.log("finish");
ifFinishAll = true;
response.status(200).send(resArray);
};
return resArray;
}
function getLinks(body) {
const html = body;
const $ = cheerio.load(html);
const linkObjects = $('a');
const links = [];
linkObjects.each((index, element) => {
countLinks = linkObjects.length;
var strHref = $(element).attr('href');
var strText = $(element).text();
var existUrl = linkslinst.includes(strHref);
var existText = textslist.includes(strText);
if (strText !== '' && strText !== "" && strText !== null && strHref !== '' && strHref !== "" && strHref !== null && strHref !== undefined && !existUrl && !existText) {
var tel = strHref.startsWith("tel");
var mail = strHref.startsWith("mailto");
var linkInStart = isUrlValid(strHref);
if (!tel && !mail) {
if (linkInStart) {
links.push({
text: $(element).text(), // get the text
href: $(element).attr('href'), // get the href attribute
});
linkslinst.push($(element).attr('href'));
textslist.push($(element).text());
}
else {
links.push({
text: $(element).text(), // get the text
href: webUrl.toString() + $(element).attr('href'), // get the href attribute
});
linkslinst.push(webUrl.toString() + $(element).attr('href'))
textslist.push($(element).text());
}
}
}
});
const result = [];
const map = new Map();
for (const item of links) {
if (!map.has(item.text)) {
map.set(item.text, true); // set any value to Map
result.push({
text: item.text,
href: item.href
});
}
}
if (result.length > 0) {
resArray.push({ list: result, depth: count - 1 });
}
console.log('res', resArray);
return resArray;
}
I want to return/response finally to the "resArray". I tried to add async and await to function number 1 and number 2 but it didn't succeed. Maybe I need to add async/await to all functions? How can I fix that?
You can achieve your goal by using async-await.
An async function is a function declared with the async keyword, and the await keyword is permitted within them. The async and await keywords enable asynchronous, promise-based behavior to be written in a cleaner style, avoiding the need to explicitly configure promise chains.
Basic example:
function resolveImmediately() {
return new Promise(resolve => {
resolve(true);
});
}
function resolveAfter2Seconds() {
return new Promise(resolve => {
setTimeout(() => {
resolve('resolved');
}, 2000);
});
}
async function asyncCall() {
console.log('calling');
const result = await resolveImmediately();
console.log(result);
if(result) {
const anotherResult = await resolveAfter2Seconds();
console.log(anotherResult);
}
}
asyncCall();
Note: Your code is too long to debug. As a result, to make you understand about the approach (what & how to do), i have added a simple example into my answer.