Develop Reference

Async function returns 200 without executing

I have a Cloud Function written in Node JS that accesses data from BigQuery, converts this to CSV and exports to a Google Storage bucket.
Currently it executes and returns a 200, but does not run any of the code within my try/catch.
When testing it just returns:
Function execution took x ms. Finished with status code: 200
I've attempted to debug by adding console logs at various points, but it doesn't log anything - it just returns a 200.
exports.run_checks = (req, res) => {
"use strict";
let parsedBody = req.body;
let startCount = parsedBody.start;
let endCount = parsedBody.end;
(async function () {
try {
for (let i = startCount; i < endCount; i += 1) {
//Exclude overly large files here
if (i != 100) {
const query =
`SELECT *
FROM \`bq_dataset\`
WHERE id_number = ${i}`;
const options = {
query: query,
location: "europe-west2",
};
const [job] = await bigquery.createQueryJob(options);
console.log(`Job ${job.id} started.`);
const [rows] = await job.getQueryResults();
let id = rows[0].id;
const createFile = storage.bucket(bucketName).file(`${id}.csv`);
const csv = parse(rows, { fields });
const dataStream = new stream.PassThrough();
dataStream.push(csv);
dataStream.push(null);
await new Promise((resolve, reject) => {
console.log(`Writing ${id} to GCS`);
dataStream
.pipe(
createFile.createWriteStream({
resumable: false,
validation: false,
metadata: { "Cache-Control": "public, max-age=31536000" },
})
)
.on("error", (error) => {
console.error("Stream failed", error);
reject(error);
})
.on("finish", () => {
resolve(true);
});
});
}
}
res.status(200).send();
} catch (err) {
res.send(err);
}
})();
};

Your function is not async. The host has no idea that you are still doing something in your function, it returns without any error.
Modify your arrow function to async, and no need for IIFE, remove it, or await it, that is also important!
exports.run_checks = async (req, res) => {
"use strict";
let parsedBody = req.body;
let startCount = parsedBody.start;
let endCount = parsedBody.end;
try {
for (let i = startCount; i < endCount; i += 1) {
//Exclude overly large files here
if (i != 100) {
const query =
`SELECT *
FROM \`bq_dataset\`
WHERE id_number = ${i}`;
const options = {
query: query,
location: "europe-west2",
};
const [job] = await bigquery.createQueryJob(options);
console.log(`Job ${job.id} started.`);
const [rows] = await job.getQueryResults();
let id = rows[0].id;
const createFile = storage.bucket(bucketName).file(`${id}.csv`);
const csv = parse(rows, { fields });
const dataStream = new stream.PassThrough();
dataStream.push(csv);
dataStream.push(null);
await new Promise((resolve, reject) => {
console.log(`Writing ${id} to GCS`);
dataStream
.pipe(
createFile.createWriteStream({
resumable: false,
validation: false,
metadata: { "Cache-Control": "public, max-age=31536000" },
})
)
.on("error", (error) => {
console.error("Stream failed", error);
reject(error);
})
.on("finish", () => {
resolve(true);
});
});
}
}
res.status(200).send();
} catch (err) {
res.send(err);
}
};

Why is this asynchronous function being called twice

I am trying to create a user with email and password using firebase, but when I call the function that creates it, it is called twice and I get an error because I am trying to register the email that is already in use.
I noticed that the console.log('CALLED') is called once, I don't understand why RegisterWithEmail is called twice. My auth flow only creates the userDocument in the confirmation phase, for this reason userSnap.length equals zero in the second call and tries to create again.
How can I call this function once?
FILE: emailconfirm.page.tsx
registerEmail = async data => {
const { setRegStatus, createDoc } = this.props;
console.log('CALLED')
await RegisterWithEmail(data).then(res => {
console.log('Final response ', res)
if(res === 'EMAIL_VERIFIED') {
createDoc()
setRegStatus({ status: 'created', data: res })
}
else if(res === 'SOMETHING_WENT_WRONG'){
setRegStatus({ status: 'error', data: res })
}
}).catch(err => {
console.log('Error ', err)
setRegStatus({ status: 'error', data: err })
})
}
FILE: firebase.utils.tsx
export const RegisterWithEmail = async user => {
console.log("Called Once...");
if(!user) return 'SOMETHING_WENT_WRONG';
else {
const snap = await firestore.collection('users').where('email', '==', user.email).get();
const docs = snap.docs.map((doc) => doc.data());
if (docs.length !== 0) return 'EMAIL_HAS_ALREADY_BEEN_TAKEN';
try {
console.log("Trying to register email...");
return await auth.createUserWithEmailAndPassword(user.email, user.password).then(async usr => {
await usr.user.updateProfile({
displayName: user.name
}) // SETTING NAME
const sendVerifyEmail = usr.user.sendEmailVerification().then(() => setTimer(usr.user, 5))
return await sendVerifyEmail.then(msg => {
console.log('Finishing...', msg)
if(msg.txt !== 'waiting') {
if(msg.error) {
throw msg.txt
}
else return msg.txt
}
}).catch(() => {
throw 'EMAIL_NOT_SENT'
})
}).catch(() => {
throw 'USER_NOT_CREATED'
})
} catch (err) {
throw 'USER_ALREADY_REGISTERED'
}
}
}
Developer console:

You shouldn't be mixing and matching .then()s in async functions for your own sanity's sake.
Something like
export const RegisterWithEmail = async (user) => {
if (!user) return false;
const snap = await firestore.collection("users").where("email", "==", user.email).get();
const docs = snap.docs.map((doc) => doc.data());
if (docs.length !== 0) return false;
console.log("Trying to register email...");
try {
const resp = await auth.createUserWithEmailAndPassword(user.email, user.password);
// then ...
return true;
} catch (err) {
// catch ...
}
};
might work better for you.

I need more code to be sure, but I think you should add await
registerEmail = async data => {
console.log('CALLED')
await RegisterWithEmail(data)
}

How to wait for function to finish before running another one or rest of the code?

I can't figure this one out. I have one function that connects to an SFTP server and downloads files. Then, I have a second function that reads the contents, puts the data in an array, and returns the array.
The problem is that the second function always runs first. I tried different methods but I can't get it to work. That connection to SFTP is quite slow, it can take like 10+ seconds to finish. But I need to somehow wait for it to finish before doing anything else.
const SFTPConfig = require('../config/keys').sftpconfig;
const getCSATFiles = async function(targetDate) {
try {
let Client = require('ssh2-sftp-client');
let sftp = new Client();
const date = moment(targetDate);
var dir = `../csv/${targetDate}/`;
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir);
}
sftp
.connect(SFTPConfig, 'once')
.then(() => {
return sftp.list('/In/Archives/');
})
.then(data => {
data.forEach(item => {
const fileName = item.name;
const remotePath = '/In/Archives/' + fileName;
const localePath = path.join(dir + fileName);
if (
moment(item.modifyTime)
.format('YYYY-MM-DD hh:mm')
.toString()
.slice(0, 10) ===
date
.format('YYYY-MM-DD hh:mm')
.toString()
.slice(0, 10)
) {
sftp
.fastGet(remotePath, localePath, {})
.then(() => {
console.log('finished getting the files!');
sftp.end();
})
.catch(err => {
sftp.end();
console.log(err, 'fastGet method error');
});
}
});
});
} catch (error) {
console.log(error);
}
};
const readCSVFiles = async function(targetDate) {
try {
const casesBO = [];
var dir = `../csv/${targetDate}/`;
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir);
}
const allLocalFiles = path.join(__dirname, dir);
const readDir = util.promisify(fs.readdir);
const files = await readDir(allLocalFiles);
for (let file of files) {
fs.createReadStream(allLocalFiles + file)
.pipe(csv.parse({ headers: true, delimiter: ';' }))
.on('error', error => console.error(error))
.on('data', row => {
if (row['[REGION2]'] !== 'FR') {
casesBO.push(row['[CALLERNO_EMAIL_SOCIAL]']);
console.log(
`${row['[AGENT]']} is ${row['[REGION2]']} and case = ${
row['[CALLERNO_EMAIL_SOCIAL]']
}`
);
}
})
.on('end', rowCount => {
console.log(`Parsed ${rowCount} rows`);
});
}
return casesBO;
} catch (error) {
console.log(error);
}
};
const testFunc = async () => {
const csatfiles = await getCSATFiles('2021-02-03');
const boData = await readCSVFiles('2021-02-03');
console.log(boData);
};
testFunc();

#1 as #messerbill suggested, you need to return the promise from your function.
#2 your promise has a loop inside of it that have more promises. In this case, you need to collect those promises and use Promise.all to resolve them before you second function runs. I put comments on the lines you need to change below. Try this:
const SFTPConfig = require('../config/keys').sftpconfig;
const getCSATFiles = function(targetDate) {
try {
let Client = require('ssh2-sftp-client');
let sftp = new Client();
const date = moment(targetDate);
var dir = `../csv/${targetDate}/`;
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir);
}
// I return the promise here
return sftp
.connect(SFTPConfig, 'once')
.then(() => {
return sftp.list('/In/Archives/');
})
.then(data => {
// I set up my promises as a blank array
const promises = [];
data.forEach(item => {
const fileName = item.name;
const remotePath = '/In/Archives/' + fileName;
const localePath = path.join(dir + fileName);
if (
moment(item.modifyTime)
.format('YYYY-MM-DD hh:mm')
.toString()
.slice(0, 10) ===
date
.format('YYYY-MM-DD hh:mm')
.toString()
.slice(0, 10)
) {
// I collect the promises here
promises.push(sftp
.fastGet(remotePath, localePath, {})
.then(() => {
console.log('finished getting the files!');
sftp.end();
})
.catch(err => {
sftp.end();
console.log(err, 'fastGet method error');
}));
}
});
// I resolve them here
return Promise.all(promises);
});
} catch (error) {
console.log(error);
}
};
const readCSVFiles = async function(targetDate) {
try {
const casesBO = [];
var dir = `../csv/${targetDate}/`;
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir);
}
const allLocalFiles = path.join(__dirname, dir);
const readDir = util.promisify(fs.readdir);
const files = await readDir(allLocalFiles);
for (let file of files) {
fs.createReadStream(allLocalFiles + file)
.pipe(csv.parse({ headers: true, delimiter: ';' }))
.on('error', error => console.error(error))
.on('data', row => {
if (row['[REGION2]'] !== 'FR') {
casesBO.push(row['[CALLERNO_EMAIL_SOCIAL]']);
console.log(
`${row['[AGENT]']} is ${row['[REGION2]']} and case = ${
row['[CALLERNO_EMAIL_SOCIAL]']
}`
);
}
})
.on('end', rowCount => {
console.log(`Parsed ${rowCount} rows`);
});
}
return casesBO;
} catch (error) {
console.log(error);
}
};
const testFunc = async () => {
const csatfiles = await getCSATFiles('2021-02-03');
const boData = await readCSVFiles('2021-02-03');
console.log(boData);
};
testFunc();

You need to take care, that you return the promise you want to resolve inside the function body in order to get the promise resolved at the right time.
async function promiseNotReturned() {
new Promise(resolve => setTimeout(resolve.bind(null), 5000))
}
async function promiseReturned() {
return new Promise(resolve => setTimeout(resolve.bind(null), 5000))
}
async function run() {
await promiseNotReturned()
console.log("does not wait for 5 seconds")
await promiseReturned()
console.log("waits for 5 seconds")
}
run()

Try/catch not catching all errors in express async functions?

I have the following code.
However, it's not catching all errors and I am still getting "throw er; // Unhandled 'error' event".
Why is this?
app.post('/api/properties/zip/:zip/bedrooms/:bedrooms', async (req, res, next) => {
try {
const file = await apiCall(req.params.zip, req.params.bedrooms);
const records = await parse(file);
const seq = await sequelize();
const result = await dbImport(seq, records);
return await res.status(200).json(`${result.length} properties successfully imported to the database`);
} catch (err) {
return next(err);
}
});
// Middleware error handling
app.use((err, req, res, next) => {
console.error(err.message);
if (!err.statusCode) err.statusCode = 500;
return res.status(err.statusCode).json(err.message);
});
For example, it didn't catch the error in the parse() function, until I added a specific error handler. Shouldn't my try/catch catch this error even without adding this?
const fs = require('fs');
const parse = filename => new Promise(((resolve, reject) => {
// Converts a line from the file, parses it to JSON, and stores it an array
const func = (data, records) => {
const json = JSON.parse(data);
records.push(json);
};
// Read in each line of the file and pass that line to func
const readLines = (input) => {
const records = [];
let remaining = '';
// ******** HAD TO ADD THIS *********
input.on('error', (err) => {
reject(err);
});
input.on('data', (data) => {
remaining += data;
let index = remaining.indexOf('\n');
let last = 0;
while (index > -1) {
const line = remaining.substring(last, index);
last = index + 1;
func(line, records);
index = remaining.indexOf('\n', last);
}
remaining = remaining.substring(last);
});
input.on('end', () => {
if (remaining.length > 0) {
func(remaining, records);
}
resolve(records);
});
};
const input = fs.createReadStream(filename);
readLines(input, func);
}));
module.exports = parse;
Thanks in advance!

Perhaps this will demonstrate for you how a try/catch will work with when using await. When a promise is rejected it will throw the resulting value. If the underlying promise resolves it will return that value.
(async () => {
try {
const val1 = await Promise.resolve('resolved val');
const val2 = await Promise.reject('reject val');
console.log(val1);
} catch (err) {
console.error(err);
}
})();

Where is my error in using promises in node.js

I am still very new to node.js. In my current test project I want to send a confirmation email or other emails, depending on the loaded template. The template is stored in MySQL.
The result I am getting is:
{
"message": {
"error": {},
"foo": "bar"
}
}
So the error bit is empty and I don't know why...
If I reject manually at a different point in the code it works just fine, so the problem is not with the middleware, router or server.js file.
Also I have rejected "Foo: Bar" back, to check which catch block catched the error.
Here is my mailer.js file:
const nodemailer = require('nodemailer');
let conDB;
module.exports = (injectedMySql) => {
conDB = injectedMySql
return {
sendMail: sendMail
}
}
const sendMail = (mail) => {
return new Promise((resolve,reject) => {
loadTemplate(mail.templateId, mail.languageId)
.then(data => {
const mailserver = {
host: "something.com",
port: 465,
secure: true, // use TLS
auth: {
user: "something#something.com",
pass: "PASSWORD"
},
tls: {
// do not fail on invalid certs
rejectUnauthorized: false
}
};
const body = {
from: 'something#something.com',
to: mail.toAdress,
subject: allReplace(data.subject, mail.subjectReplace),
text: allReplace(data.body, mail.textReplace),
html: allReplace(data.html, mail.htmlReplace)
}
// create a nodemailer transporter using smtp
let transporter = nodemailer.createTransport(mailserver)
transporter.sendMail(body)
.then(data => {console.log(data)
resolve(data)
})
.catch(err => {reject("sendMail problem")})
})
.catch(error => {reject({"error": error, "foo": "bar"})})
})
}
function allReplace (str, obj) {
var retStr = str;
for (var x in obj) {
retStr = retStr.replace(new RegExp(x, 'g'), obj[x]);
}
return retStr;
};
const loadTemplate = (mailTemplate, languageId) => {
return new Promise((resolve,reject) => {
if(mailTemplate === null || languageId === null)
reject("nop, something is missing");
else
{
if (typeof conDB.query === "function")
{
conDB.query('SELECT * FROM email_template WHERE language_id = ? AND template_id = ?', [mailTemplate,languageId])
.then(data => {resolve(data)})
.catch(err => {reject("mysql has a problem")})
}
else
{
reject("function is not available");
}
}
})
}
Here is my mysql.js file:
var mysql = require('mysql2/promise');
const databaseConfigs = {
host: 'localhost',
user: 'USERNAME',
password: 'PASSWORD',
database: 'DBNAME'
};
const createID = table => {
return new Promise((resolve,reject) => {
//execute the query to register the user
let query = '';
let id = Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15)
query = `SELECT * FROM ${table} WHERE id = ?`
this.query(query,[table,id])
.then(data => {
console.log(data[0].length)
if(data[0].length==0)
{
resolve(id)
}
else
{
createID(table)
.then(data => {resolve(data)})
.catch(error => {reject(error)})
}
})
.catch(error => {reject(error)})
})
}
async function query (sql,att) {
let connection = await mysql.createConnection(databaseConfigs);
return new Promise( ( resolve, reject ) => {
console.log(`Query: '${sql}'`);
connection.query(sql,att)
.then(data => {resolve(data)})
.catch(error => {reject(error)})
connection.end();
});
}
async function transaction(queries, queryValues) {
if (queries.length !== queryValues.length) {
return Promise.reject(
'Number of provided queries did not match the number of provided query values arrays'
)
}
const connection = await mysql.createConnection(databaseConfigs)
try {
await connection.beginTransaction()
const queryPromises = []
queries.forEach((query, index) => {
queryPromises.push(connection.query(query, queryValues[index]))
})
const results = await Promise.all(queryPromises)
await connection.commit()
await connection.end()
return results
} catch (err) {
await connection.rollback()
await connection.end()
return Promise.reject(err)
}
}
module.exports.transaction = transaction;
module.exports.query = query;
module.exports.createID = createID;
Thanks to you all!
Chris

I cleand up your code a bit. Specially the error handling as you always mask your errors with your Promise.reject("message").
I think what confused you is that you're already using libraries which work with promise (you don't need to wrap those into promises again). Thats quite good as you just can use async/await then.
I hope it helps. If something is unclear just ask.
const nodemailer = require('nodemailer');
let conDB;
module.exports = (injectedMySql) => {
conDB = injectedMySql
return {
sendMail: sendMail
}
}
// your load template function already uses promises no need to wrap it
const sendMail = async mail => {
const data = await loadTemplate(mail.templateId, mail.languageId)
const mailserver = {
host: "something.com",
port: 465,
secure: true, // use TLS
auth: {
user: "something#something.com",
pass: "PASSWORD"
},
tls: {
// do not fail on invalid certs
rejectUnauthorized: false
}
};
const body = {
from: 'something#something.com',
to: mail.toAdress,
subject: allReplace(data.subject, mail.subjectReplace),
text: allReplace(data.body, mail.textReplace),
html: allReplace(data.html, mail.htmlReplace)
}
// create a nodemailer transporter using smtp
let transporter = nodemailer.createTransport(mailserver)
try {
// Return the value of sendmail
return await transporter.sendMail(body);
} catch (err) {
// handle error or throw it. I'll throw as you rejected the Promise here it.
// this part will actually help you as you now can see the correct error instead of your rejected "foo bar" erro object
throw err;
}
}
function allReplace(str, obj) {
var retStr = str;
for (var x in obj) {
retStr = retStr.replace(new RegExp(x, 'g'), obj[x]);
}
return retStr;
};
const loadTemplate = async (mailTemplate, languageId) => {
if (mailTemplate === null || languageId === null)
throw new Error("nop, something is missing");
else {
if (typeof conDB.query === "function") {
try {
const data = await conDB.query('SELECT * FROM email_template WHERE language_id = ? AND template_id = ?', [mailTemplate, languageId]);
} catch (err) {
// it's better to use the real error you always hide the real reason why something went wrong with your promise reject :).
throw err;
}
}
else {
throw new error("function is not available");
}
}
}
.
var mysql = require('mysql2/promise');
const databaseConfigs = {
host: 'localhost',
user: 'USERNAME',
password: 'PASSWORD',
database: 'DBNAME'
};
const createID = async table => {
// use GUID? https://www.npmjs.com/package/guid
let id = Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15)
let query = `SELECT * FROM ${table} WHERE id = ?`
try {
data = await this.query(query, [table, id]);
} catch (error) {
// as we throw the error in query we got to catch it here
// handle it or throw it (I throw it because I can't handle it ;).)
throw error;
}
console.log(data[0].length)
if (data[0].length == 0) {
return id;
} else {
return await createID(table);
}
}
const query = async (sql, att) => {
let connection = await mysql.createConnection(databaseConfigs);
console.log(`Query: '${sql}'`);
try {
const data = await connection.query(sql, att);
return data;
} catch (error) {
// Handle error or throw it again
// you rejected the promise so i throw it here
throw error;
} finally {
connection.end();
}
}
// I changed it to make it the same as the other functions from this
// async function transaction(queries, queryValues) { to
const transaction = async (queries, queryValues) => {
if (queries.length !== queryValues.length) {
// just throw an error
throw new Error('Number of provided queries did not match the number of provided query values arrays');
}
const connection = await mysql.createConnection(databaseConfigs)
try {
await connection.beginTransaction()
const queryPromises = []
queries.forEach((query, index) => {
queryPromises.push(connection.query(query, queryValues[index]))
})
const results = await Promise.all(queryPromises)
await connection.commit()
await connection.end()
return results
} catch (err) {
await connection.rollback()
await connection.end()
// this is not needed
// return Promise.reject(err)
// if you don't want to handle it here just throw the error
throw err;
}
}
module.exports.transaction = transaction;
module.exports.query = query;
module.exports.createID = createID;