Develop Reference

Wait for all Firebase data query requests before executing code

I am trying to fetch data from different collections in my cloud Firestore database in advance before I process them and apply them to batch, I created two async functions, one to capture the data and another to execute certain code only after all data is collected, I didn't want the code executing and creating errors before the data is fetched when i try to access the matchesObject after the async function to collect data is finished, it keeps saying "it cannot access a property matchStatus of undefined", i thought took care of that with async and await? could anyone shed some light as to why it is undefined one moment
axios.request(options).then(function(response) {
console.log('Total matches count :' + response.data.matches.length);
const data = response.data;
var matchesSnapshot;
var marketsSnapshot;
var tradesSnapshot;
var betsSnapshot;
matchesObject = {};
marketsObject = {};
tradesObject = {};
betsObject = {};
start();
async function checkDatabase() {
matchesSnapshot = await db.collection('matches').get();
matchesSnapshot.forEach(doc => {
matchesObject[doc.id] = doc.data();
console.log('matches object: ' + doc.id.toString())
});
marketsSnapshot = await db.collection('markets').get();
marketsSnapshot.forEach(doc2 => {
marketsObject[doc2.id] = doc2.data();
console.log('markets object: ' + doc2.id.toString())
});
tradesSnapshot = await db.collection('trades').get();
tradesSnapshot.forEach(doc3 => {
tradesObject[doc3.id] = doc3.data();
console.log('trades object: ' + doc3.id.toString())
});
betsSnapshot = await db.collection('bets').get();
betsSnapshot.forEach(doc4 => {
betsObject[doc4.id] = doc4.data();
console.log('bets object: ' + doc4.id.toString())
});
}
async function start() {
await checkDatabase();
// this is the part which is undefined, it keeps saying it cant access property matchStatus of undefined
console.log('here is matches object ' + matchesObject['302283']['matchStatus']);
if (Object.keys(matchesObject).length != 0) {
for (let bets of Object.keys(betsObject)) {
if (matchesObject[betsObject[bets]['tradeMatchId']]['matchStatus'] == 'IN_PLAY' && betsObject[bets]['matched'] == false) {
var sfRef = db.collection('users').doc(betsObject[bets]['user']);
batch11.set(sfRef, {
accountBalance: admin.firestore.FieldValue + parseFloat(betsObject[bets]['stake']),
}, {
merge: true
});
var sfRef = db.collection('bets').doc(bets);
batch12.set(sfRef, {
tradeCancelled: true,
}, {
merge: true
});
}
}
}
});

There are too many smaller issues in the current code to try to debug them one-by-one, so this refactor introduces various tests against your data. It currently won't make any changes to your database and is meant to be a replacement for your start() function.
One of the main differences against your current code is that it doesn't unnecessarily download 4 collections worth of documents (two of them aren't even used in the code you've included).
Steps
First, it will get all the bet documents that have matched == false. From these documents, it will check if they have any syntax errors and report them to the console. For each valid bet document, the ID of it's linked match document will be grabbed so we can then fetch all the match documents we actually need. Then we queue up the changes to the user's balance and the bet's document. Finally we report about any changes to be done and commit them (once you uncomment the line).
Code
Note: fetchDocumentById() is defined in this gist. Its a helper function to allow someCollectionRef.where(FieldPath.documentId(), 'in', arrayOfIds) to take more than 10 IDs at once.
async function applyBalanceChanges() {
const betsCollectionRef = db.collection('bets');
const matchesCollectionRef = db.collection('matches');
const usersCollectionRef = db.collection('users');
const betDataMap = {}; // Record<string, BetData>
await betsCollectionRef
.where('matched', '==', false)
.get()
.then((betsSnapshot) => {
betsSnapshot.forEach(betDoc => {
betDataMap[betDoc.id] = betDoc.data();
});
});
const matchDataMap = {}; // Record<string, MatchData | undefined>
// betIdList contains all IDs that will be processed
const betIdList = Object.keys(betDataMap).filter(betId => {
const betData = betDataMap[betId];
if (!betData) {
console.log(`WARN: Skipped Bet #${betId} because it was falsy (actual value: ${betData})`);
return false;
}
const matchId = betData.tradeMatchId;
if (!matchId) {
console.log(`WARN: Skipped Bet #${betId} because it had a falsy match ID (actual value: ${matchId})`);
return false;
}
if (!betData.user) {
console.log(`WARN: Skipped Bet #${betId} because it had a falsy user ID (actual value: ${userId})`);
return false;
}
const stakeAsNumber = Number(betData.stake); // not using parseFloat as it's too lax
if (isNaN(stakeAsNumber)) {
console.log(`WARN: Skipped Bet #${betId} because it had an invalid stake value (original NaN value: ${betData.stake})`);
return false;
}
matchDataMap[matchId] = undefined; // using undefined because its the result of `doc.data()` when the document doesn't exist
return true;
});
await fetchDocumentsById(
matchesCollectionRef,
Object.keys(matchIdMap),
(matchDoc) => matchDataMap[matchDoc.id] = matchDoc.data()
);
const batch = db.batch();
const queuedUpdates = 0;
betIdList.forEach(betId => {
const betData = betDataMap[betId];
const matchData = matchDataMap[betData.tradeMatchId];
if (matchData === undefined) {
console.log(`WARN: Skipped /bets/${betId}, because it's linked match doesn't exist!`);
continue;
}
if (matchData.matchStatus !== 'IN_PLAY') {
console.log(`INFO: Skipped /bets/${betId}, because it's linked match status is not "IN_PLAY" (actual value: ${matchData.matchStatus})`);
continue;
}
const betRef = betsCollectionRef.doc(betId);
const betUserRef = usersCollectionRef.doc(betData.user);
batch.update(betUserRef, { accountBalance: admin.firestore.FieldValue.increment(Number(betData.stake)) });
batch.update(betRef, { tradeCancelled: true });
queuedUpdates += 2; // for logging
});
console.log(`INFO: Batch currently has ${queuedUpdates} queued`);
// only uncomment when you are ready to make changes
// batch.commit();
}
Usage:
axios.request(options)
.then(function(response) {
const data = response.data;
console.log('INFO: Total matches count from API:' + data.matches.length);
return applyBalanceChanges();
}

Return value from async / await try..catch

I have a test folder with files
file
file (1)
file (2)
If the file exists I add a suffix to a new filename, to prevent overwriting the file. For example
if file exists new name should be file (1)
if file (1) exists new name should be file (2)
if file (2) exists new name should be file (3)
and so on.
The following function works fine, except the value is not returned so I can assign it later.
async function dest_exists_new_name(file) {
const access = fs.promises.access
try {
await access(file, fs.F_OK)
// file exists - generate new name
const info = path.parse(file)
const dir = info.dir
let name = info.name
const ext = info.ext
// generate suffix
let suffix = ' (1)'
const suffix_regex = / \([0-9]+\)$/
if (suffix_regex.test(name)) { // if suffix exists -> increment it
const num = name.split(' ').reverse()[0].replace(/[()]/g,'')
const next_num = parseInt(num) + 1
suffix = ' (' + next_num + ')'
name = name.replace(suffix_regex, '') // remove old suffix
}
// generate suffix end
const new_name = path.join(dir, name + suffix + ext)
// recurse until dest not exists
await dest_exists_new_name(new_name)
} catch {
// file not exist - return its name
// console.log works OK
console.log('new name ->', file)
// return doesn't work - returns undefined if the previous name exists, but works ok if the name doesn't exists
return file
}
}
await dest_exists_new_name('/path/file')
new name -> /path/file (3) // console.log - works OK
undefined // returns undefined, if file previously exists
The question is how can I correctly return the new file name value?
If there are any culprits in such a solution like
accidental file rewriting
infinite recursion
other issues
I will be grateful for the hints on how to improve the function.

Your function will return file, but being an async function, you need to await its return and you cannot do so outside of an async scope. Thus, if you just console.log() its "istantaneous" value, it will indeed return a pending promise, as the return value has not been resolved yet. You may retrieve the correct return value by including your function in an async scope, like this:
let a = async () => {
console.log(await dest_exists_new_name('/path/file'))
}
a();
This will output:
new name -> /path/file
/path/file //here's your file
Now, by adding return await dest_exists_new_name(new_name) you should be able to achive what you want and both console.log() and return the new, non-existent, file name. Here's a complete, reproducible example:
const fs = require('fs');
const path = require('path');
async function dest_exists_new_name(file) {
const access = fs.promises.access
try {
await access(file, fs.F_OK)
const info = path.parse(file)
const dir = info.dir
let name = info.name
const ext = info.ext
let suffix = ' (1)'
const suffix_regex = / \([0-9]+\)$/
if (suffix_regex.test(name)) {
const num = name.split(' ').reverse()[0].replace(/[()]/g, '')
const next_num = parseInt(num) + 1
suffix = ' (' + next_num + ')'
name = name.replace(suffix_regex, '')
}
const new_name = path.join(dir, name + suffix + ext)
return await dest_exists_new_name(new_name)
} catch {
console.log('new name ->', file)
return file
}
}
//Here, make sure that the path to "file" is correct
let a = async() => console.log(await dest_exists_new_name(path.join(__dirname, './file')));
a();
Output, having up to file (2) in the same folder:
new name -> /path/to/file (3)
/path/to/file (3)

Check you try catch and how you are receiving your variable.
async function dest_exists_new_name(file) {
try {
const res = await dest_exists_new_name(file1); // recursion result
} catch (err) {
return Promise.resolve("file not found");
}
}
// usage
let res = await dest_exists_new_name(fileArg);

First of all, you should use await, since it's async function:
// recurse until dest not exists
await dest_exists_new_name(new_name)
About recursion - IMHO, it's always better to use cycle (if it doesn't make code too complicated).
Mixing async-await & promises is not very good. Ideally you should use one style.
I prefer to use destructuring, lambda functions, and other modern features.
So, my variant for async-await, without recursion:
const fs = require('fs');
const path = require('path');
// better to create outside of func, since it doesn't depend on context
const suffix_regex = / \([0-9]+\)$/
const defaultSuffix = ' (1)'
const access = async (...params) => new Promise((resolve) => fs.access(...params, (err) => (err) ? resolve(false) : resolve(true)))
const generate_new_suffix = ({ dir, name, ext }) => {
if (suffix_regex.test(name)) { // if suffix exists -> increment it
const num = name.split(' ').reverse()[0].replace(/[()]/g, '')
const suffix = `(${+num + 1})`;
const cleanName = name.replace(suffix_regex, '') // remove old suffix
return path.join(dir, cleanName + suffix + ext)
}
return path.join(dir, name + defaultSuffix + ext)
}
const dest_exists_new_name = async (file) => {
let newNameGenerated = false
let newFileName = file
while (await access(newFileName, fs.F_OK)) {
console.log(newFileName);
const info = path.parse(newFileName)
newFileName = generate_new_suffix(info)
};
console.log('new name ->', newFileName)
return newFileName
}
(async () => {
console.log(await dest_exists_new_name(path.join(__dirname, 'file')))
})();

Why aren't error messages printing correctly when this test fails?

I have this test, and when it fails and prints the variable named errorMessage, it prints in a way that seems impossible to me. Here's my code:
function getNamesFromTypescriptFiles(): string[] {
const items = fs.readdirSync('./tsc/names');
const filenames = _.filter(items, (fn) => {
return fn.indexOf('.spec.') === -1;
});
const names = _.map(filenames, (fn) =>
fn.replace('.ts', '')
);
return names;
}
it('has an array element for each typescript file', () => {
const names = getNamesFromTypescriptFiles();
expect(names.length).toBeGreaterThan(0);
for (const name of names) {
expect(
(nameArray as ReadonlyArray<string>).includes(
name
)
).toBe(
true,
'The nameArray did not include ' + name
);
const path = './story/names/' + name + '-content.twee';
const exists = fs.existsSync(path);
expect(exists).toBe(true, `Could not find ${path}`);
const content = fs.readFileSync(path, { encoding: 'utf8' });
const passageLines = _.filter(content.split('\n'), (line) => {
return line.startsWith('::');
});
expect(passageLines.length).toBeGreaterThan(0);
for (const passageLine of passageLines) {
if (!/.*\[.*\].*/.test(passageLine)) {
const errorMessage =
name +
' has a passage named "' +
passageLine +
'" that has no tags. You must at least put an exclude tag.';
console.log(errorMessage);
expect(true).toBe(false, errorMessage);
}
}
}
});
When this prints, it looks like this:
Message:
" that has no tags. You must at least put an exclude tag.'. Passage name
Stack:
" that has no tags. You must at least put an exclude tag.'.med ":: Passage name
at <Jasmine>
I cannot figure out how such an error message is possible (it says the same thing in the console.log I have above it): It's as if it's cutting the string and half and rearranging it. How is this possible? It seems as if I have some sort of race condition, but as far as I can tell, this is not running in parallel.
I am using typescript and running the test in Jasmine. For what it's worth, this test always fails appropriately, it's only the message that is wrong.

Unhandled rejection SequelizeDatabaseError when passing dynamic query in where clause

I am trying to decompose a user's get request and put it in a variable named query. then pass the var query into the sequelize's findAll method using it's where clause, it seems like Sequelize thinks i am looking for a table CALLED query when in reality i am trying to pass the object. I'm sorry if i can not explain very well, but here is the code and the error:
var info = [];
//link example: localhost:8081/filter/?descripiton=san+francisco&houseType=house&numOfBedroom=3&numOfBathroom=2&houseSize=500&price=1200
exports.filterListings = function(req) {
//create an object literal which we will return, and has a nested object named filteredList inside.
//filteredList contains an array named listings where we will put listings that match our filter inside
let response = {
filteredList: {listings: []},
};
//now we need to see how the user wants us to filter the listings
const query = req.query;
//do some logic where we decompose query
if(query.descripiton != undefined) {
//info = info + 'descripiton: ' + query.descripiton+', ';
info.push('descripiton: ' + query.descripiton+', ');
console.log(info);
}
if(query.houseType != undefined) {
//info = info + 'houseType: ' + query.houseType+', ';
info.push('houseType: ' + query.houseType+', ');
//console.log(info);
}
if(query.numOfBedroom != undefined) {
//info = info + 'numOfBedroom: ' + query.numOfBedroom+', ';
info.push('numOfBedroom: ' + query.numOfBedroom+', ');
}
if(query.numOfBathroom != undefined) {
//info = info + 'numOfBathroom: ' + query.numOfBathroom+', ';
info.push('numOfBathroom: ' + query.numOfBathroom+', ');
}
if(query.houseSize != undefined) {
//info = info + 'houseSize: ' + query.houseSize+', ';
info.push('houseSize: ' + query.houseSize+', ');
}
if(query.price != undefined) {
//info = info + 'price: ' + query.price;
info.push('price: ' + query.price);
}
and then when i try to pass the info variable
listingModel.findAll({
//error because it wont recognize the variable search nor will it recognize info
where: {info}
}).then(listings => {
// so we loop through listings and insert what we have found into the response (which we are going to return)
for(var i = 0; i < listings.length; i++) {
response.filteredList.listings.push(listings[i]);
}; // loop where we insert data into response done
I want it to find all listings based on the dynamic query but i am getting the error:
Unhandled rejection SequelizeDatabaseError: Unknown column 'Listing.info' in 'where clause'
Thank you very much for the potential help!

Let's try to sort through your problems one by one. Sorry for the pun :p
Instead of using multiple if for creating your filtered list. Use for ... in. Then use that array of objects along with Sequelize.Op to create your query.
Example:
const Op = require('sequelize').Op;
const whereClause = [];
const query = req.query;
for(const key in query) {
if(query[key] !== '' && query[key] !== null) {
//object will be pushed to the array like "houseType:big"
whereClause.push({key:query[key]})
}
}
//you now have the where clause
//use it in your query with Op.and
listingModel.findAll({
where: {
[Op.and]: whereClause,
}
});
More info about querying with Sequelize - Operators

Variables Not Being Retained

Alrighty.
My problem is with the Javascript Objects itemList and priceList - they are not retained properly into the .then() section.
Here's the part that I'm having trouble with:
var stmt = `SELECT * FROM SRV${msg.guild.id}`;
var itemList = {};
var priceList = {};
var num = -1;
sql.each(stmt, function(err, row) {
num++
if(row.items!="ITEMS") { itemList[num] = row.items; }
if(row.prices!="PRICES") { priceList[num] = row.prices; }
console.log("---------\n" + JSON.stringify(itemList, null, 4) + "\n" + JSON.stringify(priceList, null, 4));
})
.then((itemList, priceList) => {
console.log("----E----\n" + JSON.stringify(itemList, null, 4) + "\n" + JSON.stringify(priceList, null, 4) + "\n----E----");
let cmd = require("./itemsPROCESS.js");
cmd.run(transfer, msg, args, itemList, priceList);
});
And here's the important bit that it outputs:
I've tried using strings instead of Javascript Objects, but it doesn't seem to like that either, and it outputs the same.
The 5 and undefined part should resemble the bit above it, but it doesn't.
Might it have something to do with the null parts? If so, will that mean it won't work with true - false - undefined etc?

remove the arguments itemList and priceList and your code should just be fine.
The itemList you are referring inside the .then function is actually what is resolved by the SQL promise.
.then(() => {...});

You are using itemList and priceList of global scope so don't redefine them in local scope of then. Just remove those arguments from then
.then(() => {
console.log("----E----\n" + JSON.stringify(itemList, null, 4) + "\n" + JSON.stringify(priceList, null, 4) + "\n----E----");
let cmd = require("./itemsPROCESS.js");
cmd.run(transfer, msg, args, itemList, priceList);
});