Develop Reference

How to wait for all the code in a promise to finish before resolving it? (but a little more complex)

Sorry for the very confusing question, I have this code that gets information from a website without any node modules or libraries. It is a list of users separated into different pages use ?page= at the end of the URL. I have managed to iterate through the pages and split up the raw HTML just right. However, my promise resolves before all the data is collected. How can I wait for everything to finish before I resolve the promise? I have tried countless solutions, but none seem to work. Please don't ask to use a node package, as my goal is to not use one :) A friend helped with the regex and splitting it up. Here is the code I am using:
function getData() {
return new Promise((resolve, reject) => {
let final = [] //the array of users returned in the end
const https = require("https"), url = "https://buildtheearth.net/buildteams/121/members";
https.get(url + "?page=1", request => { //initial request, gets the number of user pages.
let rawList = '';
request.setEncoding("utf8"),
request.on("data", data => {rawList += data}),
request.on("end", () => {
if(request = (request = (request = rawList.substring(rawList.indexOf('<div class="pagination">'))).substring(0, request.indexOf("</div>"))).match(/<a(.+)>(.+)<\/a>/g)) {
for(let t = parseInt(request[request.length - 1].match(/(\d+)(?!.*\d)/g)), a = 1; a < t + 1; a++) { //iterates through member pages
https.get(url + "?page=" + a, request2 => { //https request for each page of members
let rawList2 = '';
request2.setEncoding('utf8'),
request2.on("data", data => {rawList2 += data}),
request2.on("end", () => {
let i = rawList2.match(/<td>(.+)<\/td>/g); //finds table in HTML
if (i)
for (var t = 1; t < i.length; t += 3) //iterates through rows in table
console.log(i[t].replace(/<td>/g, "").replace(/<\/td>/g, "")), /* logs element to the console (for testing) */
final.push(i[t].replace(/<td>/g, "").replace(/<\/td>/g, "")); //pushes element to the array that is resolved in the end
})
})
}
}
resolve(final) //resolves promise returning final array, but resolves before elements are added with code above
})
})
})
}
If this helps, here is the website I am trying to get info from.
I am still a little new to JS so if you could help, I would really appreciate it :)

I ended up turning each action into an async function with a try and catch block and then chained the functions together with .then() For the base (getting data from a website) I took inspiration from an article on Medium. Here is the site I am pulling data from, and here is the function to get data from a website:
const getData = async (url) => {
const lib = url.startsWith('https://') ? https : http;
return new Promise((resolve, reject) => {
const req = lib.get(url, res => {
if (res.statusCode < 200 || res.statusCode >= 300) {
return reject(new Error(`Status Code: ${res.statusCode}`));
}
const data = [];
res.on('data', chunk => data.push(chunk));
res.on('end', () => resolve(Buffer.concat(data).toString()));
});
req.on('error', reject);
req.end();
});
};
and then I got the number of pages (which can be accessed by appending ?page=<page number> to the end of the url) with this this function:
const pages = async () => {
try {
let html = await getData('https://buildtheearth.net/buildteams/121/members',);
let pages = await (html = (html = html.substring(html.indexOf('<div class="pagination">'))).substring(0, html.indexOf("</div>"))).match(/<a(.+)>(.+)<\/a>/g)
let pageCount = await parseInt(pages[pages.length - 1].match(/(\d+)(?!.*\d)/g))
return pageCount
} catch (error) {
console.error(error);
}
}
and then I used the page count to iterate through the pages and add the HTML of each to an array with this function:
const getPages = async pageCount => {
let returns = []
try {
for (page = 1; page <= pageCount; page++) {
try {
let pageData = await getData('https://buildtheearth.net/buildteams/121/members?page=' + page)
returns.push(pageData)
} catch (error) {
return error
}
}
} catch (error) {
return error
} finally {return returns}
}
and then I iterated through the array of strings of HTML of each page, and extracted the data I needed out of each with this function which would return the list of members I need:
const iteratePages = async pages => {
if (!Array.isArray(pages)) return
try {
let returns = []
await pages.forEach(page => {
let list = page.match(/<td>(.+)<\/td>/g);
if (list)
for (var element = 1; element < list.length; element += 3)
returns.push(list[element].replace(/<td>/g, "").replace(/<\/td>/g, ""));
})
return returns
} catch (error) {
return error
}
}
And then it was a matter of chaining each together to get the array I needed:
pages().then(pageCount => getPages(pageCount)).then(pages => iteratePages(pages)).then(finalList => {console.log(finalList); console.log(finalList.length)})

firebase cloud functions oncall returns null

wondering what's the weird error.
I am using the onCall method from firebase cloud functions, but when I read it from my app it returns null value. I am trying to return some test data but it doesn't seem to be working. Am i returning the data wrongly?
index.js
exports.handleMassFollowAnalytics = functions.https.onCall((data, context) => {
const brandArray = data.brandArray;
const followed = data.followed;
let done = 0;
for (var i = 0; i < brandArray.length; i++) {
let brand = brandArray[i];
admin.database()
.ref(`brands/${brand}/followers`)
.transaction(function(post) {
if (post !== null) {
post--;
}
return post;
},
function(error, committed, snapshot) {
done++;
if (done === brandArray.length) {
// returning result.
return {
data: "testabc",
};
}
}
);
}
});
app.js
const handleMassFollowAnalytics = firebase
.functions()
.httpsCallable("handleMassFollowAnalytics");
handleMassFollowAnalytics({
brandArray: array,
followed: true,
}).then((result) => {
console.log("result: ", result) // returns null everytime
});

Your function needs to return a promise that resolves with the data to send to the client. Right now, your function returns nothing. The return statement inside the transaction callback is not returning from the main function.
Also, the code is ignoring the promises returned by the transactions you're performing. The final promise returned from the function must resolves only after all the other promises resolve.

So, I used Doug's information and arrived at the following answer, for reference to anyone in future.
This seems to return correctly for me.
Return individual promises
Return final promise
index.js
exports.handleMassFollowAnalytics = functions.https.onCall((data, context) => {
const brandArray = data.brandArray;
const followed = data.followed;
var promises = [];
for (var i = 0; i < brandArray.length; i++) {
let brand = brandArray[i];
promises.push(admin.database()
.ref(`brands/${brand}/followers`)
.transaction(function(post) {
if (post !== null) {
post--;
}
return post;
});
);
}
return Promise.all(promisess).then((result)=>{
return {
data: "testabc",
}
})
});

Why is my promise's 'resolve' being sent before the functions ends execution?

I have a function (that contains promises internally so it itself runs synchronously) that seems to be running asynchronously within my main code. No matter how I format my promise it seems like the resolve gets sent before the functions ends execution:
This problem is also logically recursive, in that if I try adding another promise around the nameExists function (within this very promise) and then putting the resolve in a 'then', i just run into the same issue with the nested resolve...
document.getElementById("config-select").addEventListener("input", function(){
//check if the doc name exists: returns doc id
//promise that doc_obj is created before moving on
let doc_obj = {};
let promise = new Promise(function (resolve, reject) {
let doc_name = document.getElementById("config-select").value;
doc_obj = nameExists(doc_name);
resolve('done'); //this executes BEFORE nameExists is done processing...bringing back the original asynch issue i was trying to fix in the first place...
});
promise.then(function (result) {
alert("then: "+doc_obj);
if(doc_obj.bool === true){//it does exist:
alert("replacing id");
document.getElementById("config-select").setAttribute("doc-id", doc_obj.id);
}
else{//it doesn't:
alert("resetting id");
document.getElementById("config-select").setAttribute("doc-id", "");
}
}
);
});
The nameExists function:
//check if the name in config-select is an existing doc (assumes name is a unique document field)
const nameExists = function(name){
//get all docs
localDB.allDocs({include_docs: true}).then(function (result) {
//return object set to default state if no match is found
let doc_obj = {bool: false, id: ""};
alert("Entering the match checker...");
for(let i =0; i<result.total_rows; i++) {
if(result.rows[i].doc.name == name){
alert(result.rows[i].doc.name);
alert(name);
doc_obj.bool = true;
doc_obj.id = result.rows[i].doc._id;
//found a match
break;
}
}
//return the result
alert("returned obj.id: "+doc_obj.bool);
return doc_obj;
}).catch(function (err) {console.log(err);});
};
Ideally, I would like the doc_obj or some return value object to be populated with data from the nameExists function, before evaluating my 'if statements'. How can I format my promise/resolve statement to achieve this?

You should drop that new Promise - it doesn't change anything about whether you will be able to wait for nameExists' result or not. You will need to return the promise that then() creates inside the nameExists function:
function nameExists(name) {
return localDB.allDocs({include_docs: true}).then(function (result) {
//^^^^^^
for (let i =0; i<result.total_rows; i++) {
if (result.rows[i].doc.name == name){
return {bool: true, id: result.rows[i].doc._id};
}
}
return {bool: false, id: ""};
});
// ^ don't catch errors here if you cannot handle them and provide a fallback result
}
Then you can just wait for it in your event listener:
document.getElementById("config-select").addEventListener("input", function() {
const doc_select = document.getElementById("config-select");
const doc_name = doc_select.value;
// check if the doc name exists: returns doc id
nameExists(doc_name).then(function(doc_obj) {
//^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^
console.log("then", doc_obj);
if (doc_obj.bool) { // it does exist:
alert("replacing id");
} else { // it doesn't:
alert("resetting id");
}
doc_select.setAttribute("doc-id", doc_obj.id); // id is "" when it doesn't exist
}).catch(function (err) {
console.log(err);
})
});

the only async call you have is inside the nameExists function which is the database call so there is no need to write two promises, only one is enough to solve your issue.
the first event should be like that:
document.getElementById("config-select").addEventListener("input", function(){
nameExists(doc_name).then(function(doc_obj) {
alert("then: "+doc_obj);
if(doc_obj.bool === true){//it does exist:
alert("replacing id");
document.getElementById("config-select").setAttribute("doc-id", doc_obj.id);
}
else{//it doesn't:
alert("resetting id");
document.getElementById("config-select").setAttribute("doc-id", "");
}
}).catch(function (err) { console.log(err) });
});
and the nameExists function should look like that:
//check if the name in config-select is an existing doc (assumes name is a unique document field)
const nameExists = function(name){
//get all docs
return localDB.allDocs({include_docs: true}).then(function (result) {
//return object set to default state if no match is found
let doc_obj = {bool: false, id: ""};
alert("Entering the match checker...");
for(let i =0; i<result.total_rows; i++) {
if(result.rows[i].doc.name == name){
alert(result.rows[i].doc.name);
alert(name);
doc_obj.bool = true;
doc_obj.id = result.rows[i].doc._id;
//found a match
break;
}
}
//return the result
alert("returned obj.id: "+doc_obj.bool);
return(doc_obj); // here is where the code runs then statement inside the event
});
};

Call same promise within itself

Still trying to wrap my head around promises and how they work. Im querying the Google webmaster API to return Search Analytics data. I've set up a promise which returns the data if i call it once however i need to call it again based on the result of the previous.
For example:
startRow = 0;
data = [];
Query(startRow).then((results) => {
if (results != null) {
data.push(results)
startRow++;
// RUN SAME QUERY AGAIN
};
});
startRow needs to increase by 1 then call the same promise (with the updated startRow) if the promise returned data. Is this possible or am i looking at this totally the wrong way?

You can't call the same Promise more than once, only create new ones.
startRow = 0;
data = [];
function startQuery() {
// Generally a good idea to always return Promises,
// so you can chain them if needed
return Query(startRow).then(processResults);
}
function processResults(results) {
if (results == null) return;
data.push(results);
startRow++;
return startQuery();
};
startQuery();
Or, in a more compact way:
startRow = 0;
data = [];
function startQuery() {
return Query(startRow).then((results) => {
if (results == null) return;
data.push(results);
startRow++;
return startQuery();
});
}
startQuery();

What you can do is create a function that recursively returns all of the results from a certain page onward, then call that with an initial page value of 0:
function queryPaged(pageNum, soFar) {
return Query(pageNum).then(function (results) {
return results
? queryPaged(pageNum + 1, soFar.concat(results))
: soFar;
});
}
queryPaged(0, []).then(function (allResults) {
console.log(allResults);
});

Why can´t I call a synchronous function inside a async promise?

I´m using express/ node, mysql and bluebird.
I´m performing three async database queries using Promise.all(). After these were performed I have to do some calculation.
If I do this I can call the function getDateRange() successfully, but this leads me to the problem, that I have trouble doing some error handling.
return Promise.all([departmentDatabase.getVacation(departmentID), departmentDatabase.countUser(departmentID), departmentDatabase.blockedDaysOfResponsible(departmentID)])
.then(departmentData => [departmentData, departmentData[0].map(entry => this.getDateRange(new Date(entry.dateFrom), new Date(entry.dateTo)))] )
.spread(function(departmentData, dateRange){
var mergedDateRange = [].concat.apply([], dateRange);
var counts = {};
mergedDateRange.forEach(function(x) { counts[x] = (counts[x] || 0)+1; }); //Zähle doppelt vorkommende Daten
var departmentMinStock = departmentData[0][0].departmentMinStock;
var blockedDays = [];
for (var p in counts) {
if( counts.hasOwnProperty(p) ) {
if( (parseInt(departmentData[1][0].amount) - parseInt(counts[p])) <= parseInt(departmentMinStock)) {
blockedDays.push(p);
}
}
}
return [{"vacationRequest": departmentData[0], "blockedDays": blockedDays}, departmentData[2]];
})
.catch(err => {
// ...do something with it...
// If you want to propagate it:
return Promise.reject(err);
// Or you can do:
// throw err;
});
In case that departmentData is undefined, which can happen quit often, I have no real option (that I know) to stop all following .then() calls and just return an empty array. I could return an empty array, but then I would have to do the check in every .then() is have.
So I thought, because I use Promise.all() I don´t really have following async operation, therefore I could just do all my logic inside the callback of .all().
But if I try to do it this way:
return Promise.all([departmentDatabase.getVacation(departmentID), departmentDatabase.countUser(departmentID), departmentDatabase.blockedDaysOfResponsible(departmentID)])
.then(function(data){
var d = [];
for(var i = 0; data[0].length > i; i++){
//console.log(this.getDateRange(new Date(data[0][0].dateFrom), new Date(data[0][0].dateTo)));
//console.log(data[0][i].dateFrom);
//console.log(data[0][i].dateTo);
var x = this.getDateRange(data[0][i].dateFrom, data[0][i].dateTo);
console.log(x);
// d.push(this.getDateRange(new Date(data[0][0].dateFrom), new Date(data[0][0].dateTo)));
}
return 1;
})
It results in an error TypeError: Cannot read property 'getDateRange' of undefined. But console.log(data[0][i].dateFrom); has a value I have checked that.
This is my getDateRange function:
getDateRange(startDate, stopDate) {
var dateArray = [];
var currentDate = moment(startDate);
while (currentDate <= stopDate) {
dateArray.push(moment(currentDate).format('YYYY-MM-DD'))
currentDate = moment(currentDate).add(1, 'days');
}
return dateArray;
}
Can someone exlpain me why this is happening? And best would be an example on how to do it right in this case?