I want to wait in my code until two values are the same. For this I use
await new Promise((resolve, reject) => {
if(curCount == maxTests) resolve;
});
But I think, this is only called one time. How I can make it that if both values are the same the promise is resolved? How to avoid that it will never send a resolve?
UPDATE:
Some requested the function that makes trouble. Here is the whole function, without it sub function. The function will fill the q-queue to fullfill the tests sync. The problem ist that req.body.selection.forEach immediately returns but I want to wait until the whole queue is ready. So my idea was to add a promise to the end and hit until current and max are the same.
router.post('/imgtest', async (req, res) => {
console.log('Start Image Test');
//Er kommt an.
req.setTimeout(5000000); // Nach Adam Riese 83 Minuten.
process.setMaxListeners(0);
io = req.app.get('socketio');
//Caluclate the max amounnt of tests
const maxTests = req.body.selection.length * req.body.servers.length;
var curCount = 0;
//RETURN IF THIS IS READY. CURRENTLY IT RETURNS IMMEDIATLY
req.body.selection.forEach(async function(entry) {
//Jetzt erstmal die Domain aus der DB holen
var dbUrl = await getUrl(entry);
console.log('tapsi');
var bildFormat = '';
var arrQuestionmark = dbUrl.split('?');
if(arrQuestionmark.length==2){
if(arrQuestionmark[1].includes('&')){
var arrAnd = arrQuestionmark[1].split('&');
arrAnd.forEach(function(entry) {
if(entry.includes('format=')){
var arrFormat = entry.split('=');
bildFormat = arrFormat[1];
}
});
}
}
var masterName = uuidv1();
const orgpath = path.resolve(__basedir, 'tests/downloads', masterName + '-.' + bildFormat);
//Download the MAsterimage
(async () => {
await queue.add(() =>downloadImage(dbUrl, 'c11', req.body.domain, bildFormat, orgpath) );
})();
req.body.servers.forEach(async function(xserver) {
var fileName = masterName + '-' + xserver + '.' + bildFormat;
const dpath = path.resolve(__basedir, 'tests/downloads', fileName);
(async () => {
await queue.add(() => downloadImage(dbUrl, xserver, req.body.domain, bildFormat, dpath));
//console.log('Done ' + entry);
})();
(async () => {
await queue.add(async() => startCompare(orgpath, dpath, 'null:').then(function(result) {
console.log(result);
curCount++;
messageIO(curCount,maxTests);
}));
//console.log('done compare ' + entry);
//fs.unlinkSync(dpath);
})();
});
});
console.log('Need to wait');
res.sendStatus(200);
});
You're correct in assuming that will only be called once. A way around that is to, within the function, create a loop via setInterval - doing a regular check and resolving if true and clearing the loop.
Not too sure what trying to achieve but one thing is for certain, Array.prototype.forEach() will not await even if its callback is async and performing a test inside a new Promise(...) constructor won't help.
Good news though, for loops will await.
Here's the code with for loops instead of .forEach() (twice), unnecessary stuff removed, and otherwise tidied up.
First a suggestion ...
// https://nodejs.org/api/querystring.html
// The querystring module provides utilities for parsing and formatting URL query strings.
const querystring = require('querystring');
... then:
router.post('/imgtest', async (req, res) => {
req.setTimeout(5000000); // Nach Adam Riese 83 Minuten.
process.setMaxListeners(0);
// io = req.app.get('socketio'); // not used
const masterName = uuidv1(); // moved from inner loop
for (i=0; i<req.body.selection.length; i++) {
let entry = req.body.selection[i];
let dbUrl = await getUrl(entry);
let bildFormat = querystring.parse(dbUrl).format;
let orgpath = path.resolve(__basedir, 'tests/downloads', `${masterName}-.${bildFormat}`);
await queue.add(() => downloadImage(dbUrl, 'c11', req.body.domain, bildFormat, orgpath));
for (j=0; j<req.body.servers.length; j++) {
let xserver = req.body.servers[j];
let dpath = path.resolve(__basedir, 'tests/downloads', `${masterName}-${xserver}.${bildFormat}`);
await queue.add(() => downloadImage(dbUrl, xserver, req.body.domain, bildFormat, dpath));
await queue.add(async() => startCompare(orgpath, dpath, 'null:'); // probably safe to remove `async`
// fs.unlinkSync(dpath); // ???
}
}
res.sendStatus(200);
});
This should get you started. I expect that you still have some way to go. At the very least you need to add a try/catch structure and be prepared to send error/status back to the client.
Related
i am back with a same issue for my promise returning undefined please help.
Here, i am using ipfs to save data with savedata() which takes in a json string,a document name and a secret phrase.
i am not quit sure why promise is returning undefined i have checked everything
here is the new code
index.js
exports.savedata = async function savedata(jsondata,Docname,secretuuid){
const docname = Docname
let ipfss = await main();
let datavar = await ipfss.add(jsondata);
//check if input file exists or not?
const fpath = __dirname + "\\"+ "input.json"
if (fs.existsSync(fpath)) {
}
else{
const defobj = {defaultID:"defaultID"}
fs.writeFile("input.json",JSON.stringify(defobj),function(err){
// console.log('saved!')
})
}
//create an object and put an array with defaultid:defaultid to it
//take that object and keep concatenating the new arrays[new documents]
fs.readFile("input.json","utf-8",function(err,data){
if(err){
console.log(err)
}
const rembrk1 = data.replaceAll("{","")
const rembrk2 = rembrk1.replaceAll("}","")
const newstring = JSON.stringify({[docname]: datavar.path})
const URK = uuidv4() + "rkbyavds"
const CAT = CryptoJS.AES.encrypt(String(datavar.path),secretuuid);
var ENCAT = CAT.toString()
const fstring = "{" + rembrk2 + "," + docname + ":" + CAT + "}"
fs.writeFile("input.json", JSON.stringify(fstring),function(err){
if(err){
console.log(err)
}
})
return new Promise((resolve,reject) => {
// console.log('saved')
const retobj = {CAT:ENCAT,URK:URK}
resolve(retobj)
});
})
}
test.js
obj = JSON.stringify({user:"MqwMedz2edemusaaa",age:1291})
const op = savedata(obj,"doc1","verysecretuuid")
op.then(x=>{
console.log(x)
})
Well, the RegisterUser function executes the actions and retrieves the data. What it doesn't do is return any value. This means it'll return a Promise by default, but this Promise won't have any value to resolve.
Depending on which object you want to have returned, you need to return it or create and return a new Promise from which you can call the resolve-function.
You can read more about it here: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/async_function
I am scraping multiple pages with cheerio and axios in node.js
I am having a hard time with Promises, can someone help me return the JSON if I hit the last page? Thanks!
const getWebsiteContent = async (url) => {
await axios.get(url).then(res => {
const $ = cheerio.load(res.data)
pageNum = getTotalpages($); // Get the pagination
console.log(url);
//Some scraping here
})
indexPage++; // Increment to the next page
const nextPageLink = baseUrl + '&page=' + indexPage; // get next page
if (indexPage > pageNum) {
var editedText = text.slice(0, text.length - 1);
editedText += ']}';
editedText = JSON.parse(editedText); // I want to return this and use elsewhere
return editedText;
}
setTimeout(async () => {
getWebsiteContent(nextPageLink); // Call itself
}, 1000);
}
var myJSON= await getWebsiteContent(baseUrl); // something like this
I would write getPages as an async generator -
async function* getPages (href, initPage = 0) {
const res = await axios.get(setPage(href, initPage))
const $ = cheerio.load(res.data)
const pages = getTotalpages($)
yield { page: initPage, dom: $ }
for (let p = initPage; p < pages; p++) {
await sleep(1000)
const r = await axios.get(setPage(href, p))
yield { page: p, dom: cheerio.load(r.data) }
}
}
This depends on helper setPage that manipulates the href page number using the url module, which is much safer than hobbling together strings by hand -
function setPage (href, page) {
const u = new URL(href)
u.searchParams.set("page", page)
return u.toString()
}
And another helper, sleep, which prevents the mixing of setTimeout with async-based code. This allows us to easily pause between pages -
async function sleep (ms) {
return new Promise(r => setTimeout(r, ms))
}
Finally we write scrape which is a simple wrapper around getPages. This allows us to reuse the getPages function to scrape various elements as needed. A benefit of using this approach is that the caller can determine what happens with each page. Below we push to result array, but as another example we could write each page to disk using the fs module. Obviously this for you to decide -
async function scrape (href) {
const result = []
for await (const {page, dom} of getPages(href)) {
console.log("scraped page", page) // some status message
result.push(getSomeData(dom)) // get something from each page
}
return result
}
scrape(myUrl).then(console.log, console.error)
You shouldn't be using then with your async / await code.
pagination should look something like this:
let response = await axios.get(url)
let $ = cheerio.load(response.data)
// do some scraping
while(url = $('[rel=next]').attr('href')){
response = await axios.get(url)
$ = cheerio.load(response.data)
// do more scraping
}
I'm building a guessing game with Node JS. After collecting some data on the back-end, I send it to the front-end and the game starts. The data contains all 10 levels, so the game can run on a single page. Each level runs for 10 seconds. After the time is up, the user selection is sent to the server, and a result comes back. The answer is displayed, and then the content is changed to the "next level" (using the content in the big data object, therefore no refresh is needed).
I'm having some issues with having 10 levels run for 10 seconds each (or ~12 seconds with a delay for displaying the results).
This can't be done in some type of loop, since all awaits for each level will run at once. For instance:
function timeout(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
for (let i = 0; i < 10; i++) {
displayPage(i);
await timeout(10000);
const result = await $.post(...) // gets results
displayResults(result);
await timeout(2000);
}
all the timeouts will run at once, and it won't work.
I thought of using a setInterval, but I'm not sure how to.. since I want to wait 10 seconds until checking the input, and then display the results for 2 seconds, and then move on.
For now the result I came up with is:
displayPage(level1);
await timeout(10000);
const result = await $.post(...)
displayResults(result);
await timeout(2000);
displayPage(level2);
await timeout(10000);
const result = await $.post(...)
displayResults(result);
await timeout(2000);
displayPage(level3);
await timeout(10000);
const result = await $.post(...)
displayResults(result);
await timeout(2000);
displayPage(level4);
await timeout(10000);
const result = await $.post(...)
displayResults(result);
await timeout(2000);
...
This does not seem efficient and I think there's a better way to do this, but I'm not sure how.
Any suggestions would be appreciated. Thanks!
I think this is what you are looking for:
const pages = [1, 2, 3, 4, 5];
run();
async function run() {
for (let i = 0; i < pages.length; i++) {
await displayPage(i);
const result = 'Some result';
await displayResult(result);
}
}
function displayPage(number) {
text.innerText = 'Page ' + number;
return new Promise(res => {
setTimeout(res, 10000);
});
}
function displayResult(result) {
text.innerText = 'Result: ' + result;
return new Promise(res => {
setTimeout(res, 2000);
});
}
<div id="text"><div>
Another solution, without promises and loops:
const pages = [1, 2, 3, 4, 5];
let currentPageIndex = 0;
displayPage();
function displayPage() {
const index = currentPageIndex++;
if (pages[index] === undefined) return;
const pageNumber = pages[index];
text.innerText = 'Page ' + pageNumber;
const result = 'Some result';
setTimeout(() => {
displayResult(result);
}, 10000);
}
function displayResult(result) {
text.innerText = 'Result: ' + result;
setTimeout(() => {
displayPage();
}, 2000);
}
<div id="text"></div>
Your first option seems to work assuming it is wrapped within an async function:
function timeout(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
const testFunc = async () =>{
for (let i = 0; i < 10; i++) {
console.log('page' , i+1 , '- question')
await timeout(3000);
console.log('page' , i+1 , '- answer')
await timeout(1000);
}
}
testFunc()
use setInterval on about 1000ms to create worker and add a state-machine that toggles the playing(10s) and the waiting(2s). You need a procedure that does the post call to the server and an object to keep the data(levels and stuff).
for example:
setInterval(funcWorker,1000,s_gameObj);
function funcWorker(f_context){
var l_dateNow = new Date();
if(f_context.is_playing){
var l_elapsed = l_dateNow.getTime() - f_context.dateLevelStart.getTime();
if(l_elapsed.totalSeconds() >= f_context.LEVEL_TIME){
f_context.end_level();
}
}else if(f_context.is_waiting_user){
//same check as above but on the waiting start
....
f_context.next_level();
}else if(f_context.is_waiting_server){
//do whatever
}
}
the end_level() should set the state flag in the context(game) object, to sending and send to the server. As the server returns in the response function set the state to waiting user and init the corresponding time variable to now(). The next_level() should set the state to playing and init the corresponding time variable to now() so that the timer can count. Consider the code above as a reference and not as a copy-paste resource.
I stuck by looping through an array that receive values from a promise and push values into a new array which is available outside the foreach.
What i have:
app.post('/submit', function (req, res) {
uploadPics(req, res, function (err) {
if (err instanceof multer.MulterError) {
res.send(JSON.stringify({UploadResult: err.message}));
console.log(err.message + ' ' +'Redirect /home');
} else if (err) {
console.log(err);
} else {
res.send(JSON.stringify({UploadResult: 'Success'}));
var filesarray = req.files;
var picinfos = [];
filesarray.forEach(function(file){
GetFileMetaInfo.filemetainfo(file.path).then(function (metadata){
//Stuck here! Can push values into an array (picinfos) but only available in the foreach. not outside..
})
})
//I need picinfos array here....
}
})
})
How i receive my metadata:
var exif = require('exif-parser');
var fs = require('fs');
exports.filemetainfo = function (filepath) {
return new Promise((resolve) => {
var file = filepath;
var buffer = fs.readFileSync(file);
var parser = exif.create(buffer);
var result = parser.parse();
resolve (result);
}).then(function (metadata){
if (metadata.tags.CreateDate !== undefined){
date = new Date (metadata.tags.CreateDate*1000);
datevalues = [
date.getFullYear(),
date.getMonth()+1,
date.getDate(),
date.getHours(),
date.getMinutes(),
date.getSeconds(),
];
CreateDate = date.getFullYear()+'-'+(date.getMonth()+1)+'-'+date.getDate();
CreateTime = date.getHours()+':'+date.getMinutes()+':'+date.getSeconds();
console.log("CrDate:" +CreateDate, "CrTime:" +CreateTime );
} else {
console.log("No Metadata Creation Infos found in " +filepath);
CreateDate = "";
CretaeTime = "";
}
if (metadata.tags.GPSLatitude !== undefined){
GPSLat = metadata.tags.GPSLatitude;
GPSLon = metadata.tags.GPSLongitude;
console.log("GPSLat:" + GPSLat , "GPSLon:" +GPSLon);
}
else {
console.log("No Metadata GPS Infos found in " +filepath)
GPSLat = "";
GPSLon = "";
}
return MetaData = {
GPSLat: GPSLat ,
GPSLon: GPSLon,
CreateDate: CreateDate,
CreateTime: CreateTime,
}
})
}
May i ask someone to give a hand. How can i make my array available outside the foreach. thank you very much!
The reason you're getting empty array at the end of forEach is because, GetFileMetaInfo.filemetainfo() returns a promise and forEach won't wait for async actions.
You could use async/await with for...of loop to get your desired result.
app.post('/submit', function (req, res) {
uploadPics(req, res, async function (err) { // note async here
if (err instanceof multer.MulterError) {
res.send(JSON.stringify({UploadResult: err.message}));
console.log(err.message + ' ' +'Redirect /home');
} else if (err) {
console.log(err);
} else {
res.send(JSON.stringify({UploadResult: 'Success'}));
var filesarray = req.files;
var picinfos = [];
for(let file of filesarray) {
const metadata = await GetFileMetaInfo.filemetainfo(file.path);
// push metadata into your array here
picinfos.push(metadata);
}
// You will have picinfos here
}
})
})
Although the question is already answered by Dinesh Pandiyan there are still some adjustments that can be made. The following code in his answer runs sequential, meaning that every async request is made after the previously returned result is resolved.
for(let file of filesarray) {
const metadata = await GetFileMetaInfo.filemetainfo(file.path);
// ^- pauses the execution of the current running code
// push metadata into your array here
picinfos.push(metadata);
}
async call #1 ╌╌await╌╌> async call #2 ╌╌await╌╌> async call #3 ╌╌await╌╌> result
You could make the code concurrent by first executing all async statements and then wait until all results are resolved. This can be done by simply changing the following:
// execute all the async functions first, reducing the wait time
for(let file of filesarray) {
const metadata = GetFileMetaInfo.filemetainfo(file.path);
// ^- remove the await
// push metadata into your array here
picinfos.push(metadata);
}
// wait for all results to be resolved
picinfos = await Promise.all(picinfos);
// ^- instead await here
async call #1 ╌╌┐
async call #2 ╌╌┼╌╌await all╌╌> result
async call #3 ╌╌┘
The above could be further simplified by simply using an Array.map() in combination with the already shown Promise.all().
var filesarray = req.files;
var picinfos = await Promise.all(filesarray.map(file => {
return GetFileMetaInfo.filemetainfo(file.path);
}));
// picinfos should be present
Or if you want to avoid working with async/await:
var filesarray = req.files;
Promise.all(filesarray.map(file => {
return GetFileMetaInfo.filemetainfo(file.path);
})).then(picinfos => {
// picinfos should be present
});
I'm trying to use async/await in my firebase function but I am getting an error. I have marked the function as async but when I try to use await inside of it I get error: Expression has type void. Put it on its own line as a statement.
I think this is strange because I think it each await function call is already in its own line as a statement. So, I'm not sure what to do. Any help would be appreciated.
For clarity, I am making a request with the Cheerio web scrape library, and then trying to make two async function calls during each loop with the .each method.
export const helloWorld = functions.https.onRequest((req, response) => {
const options = {
uri: 'https://www.cbssports.com/nba/scoreboard/',
transform: function (body) {
return cheerio.load(body);
}
};
request(options)
.then(($) => {
$('.live-update').each((i, element) => {
const homeTeamAbbr = $(element).find('tbody').children('tr').eq(0).find('a').html().split("alt/").pop().split('.svg')[0];
const awayTeamAbbr = $(element).find('tbody').children('tr').eq(1).find('a').html().split("alt/").pop().split('.svg')[0];
const homeTeam = $(element).find('tbody').children('tr').eq(0).find('a.team').text().trim();
const awayTeam = $(element).find('tbody').children('tr').eq(1).find('a.team').text().trim();
let homeTeamStatsURL = $(element).find('tbody').children('tr').eq(0).find('td').html();
let awayTeamStatsURL = $(element).find('tbody').children('tr').eq(1).find('td').html();
const gameTime = $(element).find('.pregame-date').text().trim();
homeTeamStatsURL = homeTeamStatsURL.match(/href="([^"]*)/)[1] + "roster";
awayTeamStatsURL = awayTeamStatsURL.match(/href="([^"]*)/)[1] + "roster";
const matchupString = awayTeamAbbr + "#" + homeTeamAbbr;
const URLString = "NBA_" + urlDate + "_" + matchupString;
// var docRef = database.collection('NBASchedule').doc("UpcommingSchedule");
// var boxScoreURL = "www.cbssports.com/nba/gametracker/boxscore/" + URLString;
// var setAda = docRef.set({[URLString]:{
// homeTeam: homeTeam,
// awayTeam: awayTeam,
// date: gameTime,
// homeTeamAbbr: homeTeamAbbr,
// awayTeamAbbr: awayTeamAbbr,
// homeTeamStatsURL: homeTeamStatsURL,
// awayTeamStatsURL: awayTeamStatsURL,
// boxScoreURL: boxScoreURL
// }}, { merge: true });
getTeamPlayers(homeTeamStatsURL, matchupString);
getTeamPlayers(awayTeamStatsURL, matchupString);
console.log("retrieved schedule for "+ matchupString + " on " + urlDate)
});
response.send("retrieved schedule");
})
.catch(function (err) {
console.log("error " + err);
});
});
The function I am calling just makes another request and then I'm trying to log some data.
function getTeamPlayers(playerStatsURL, matchupString) {
const options = {
uri: playerStatsURL,
transform: function (body) {
return cheerio.load(body);
}
};
console.log(playerStatsURL + " stats url");
request(options)
.then(($) => {
console.log('inside cheerio')
$('tbody').children('tr').each(function(i, element){
const playerName = $(element).children('td').eq(1).children('span').eq(1).find('a').text().trim();
const injury = $(element).children('td').eq(1).children('span').eq(1).children('.icon-moon-injury').text().trim();
const news = $(element).children('td').eq(1).children('span').eq(1).children('.icon-moon-news').text().trim();
const playerUrl = $(element).children('td').eq(1).children('span').eq(1).find('a').attr('href');
const playerLogsUrl = "https://www.cbssports.com" + playerUrl.replace('playerpage', 'player/gamelogs/2018');
console.log(playerName + ": Inj: " + injury + " News: " + news);
// database.collection('NBAPlayers').add({[playerName]:{
// '01 playerName': playerName,
// '03 playerLogsUrl': playerLogsUrl,
// '04 inj': injury,
// '05 news': news
// }})
// .then(docRef => {
// console.log("ID " + docRef.id);
// //getPlayerLogs(playerLogsUrl, playerName, docRef.id);
// })
// .catch(error => console.error("Error adding document: ", error));
});
});
}
async/await is supported in the Node version 8 (which is deployable as Cloud Functions for Firebase). Specifically, use 8.6.1 (at the time of this writing).
As for awaiting x2 inside a loop - I don't think this is best practice.
Instead, push all these requests into an array, then Promise.all so as to fetch all in parallel.
just in case someone comes after. As Ron stated before, you shouldn't use async/await inside a traditional for loop if the second call doesn't need the value of the first, as you will increase the run time of the code.
Furthermore you can't use async/await inside of a forEach=> or a map=> loop, as it won't stop and wait for the promise to resolve.
The most efficient way is to use Promise.all([]). Here I leave a great youtube video of a crack that explains async/await and Promises => https://www.youtube.com/watch?v=vn3tm0quoqE&t=6s
As to one of the questions in the comments by DarkHorse:
But all I want is for getTeamPlayers() to execute and write to the database so I don't know why I need to return anything.
In Firebase Functions all functions need to return something before the final response.
For example in this case, he has created an http function. Before he ends the function with response.send("retrieved schedule"); you need to finish each and every function triggered. As Firebase Functions will clean up after the final response, erasing and stoping anything that it is still running. So any function that hasn't finish will be killed before doing its job.
Returning a promise is the way Firebase Functions knows when all executions have finished and can clean up.
Hop it helps :)