This question already has answers here:
Using async/await with a forEach loop
(33 answers)
Closed 4 years ago.
I am having an issue with using async/await using node v8.1. It appears my issue is that I am not returning a promise from my async functions. This is causing the flow of the program to run out of order. I thought by making a function async that the function automatically returns a promise, but that is not the case I am running into.
I expect the program below to output:
Validating xlsx file...
(text from validateParsedXlsx)
Adding users to cognito...
(text from addUsersToCognito)
Adding users to dynamodb...
(text from addUsersToDynamodb)
Instead, I get:
Validating xlsx file...
Adding users to cognito...
Adding users to dynamodb...
(text from validateParsedXlsx)
(text from addUsersToCognito)
(text from addUsersToDynamodb)
The issue seems to be pretty obvious, that validateParsedXlsx() addUsersToCognito() and addUsersToDynamodb() are not returning promises. Again, I thought that by using the async keyword, the function automatically took care of this.
Thanks for the help.
Here is my script:
const xlsx = require('xlsx');
const AWS = require('aws-sdk');
AWS.config.update({region: 'us-west-2'});
const documentClient = new AWS.DynamoDB.DocumentClient({convertEmptyValues: true});
async function main(){
if (!process.argv[2]) {
console.log('\nAbsolute filepath missing. Pass the absolute filepath in as command line argument.\n')
process.exit(1);
}
const xlsxFilePath = process.argv[2];
let parsedXlsx = [];
try {
parsedXlsx = parseXlsx(xlsxFilePath);
} catch (error) {
if(error.code === 'ENOENT') {
console.log(`\nThe file path: ${process.argv[2]} cannot be resolved\n`)
} else {
console.log(error);
}
}
console.log('\n\nValidating xlsx file...\n');
await validateParsedXlsx(parsedXlsx);
console.log('\n\nAdding users to cognito...\n');
await addUsersToCognito(parsedXlsx);
console.log('\n\nAdding users to dynamodb...\n');
await addUsersToDynamodb(parsedXlsx);
}
function parseXlsx(filePath) {
const workbook = xlsx.readFile(filePath);
const sheetNameList = workbook.SheetNames;
const parsedXlsxSheets = sheetNameList.map(function (y) {
const worksheet = workbook.Sheets[y];
const headers = {};
const data = [];
for (z in worksheet) {
if(z[0] === '!') continue;
//parse out the column, row, and value
const col = z.substring(0,1);
const row = parseInt(z.substring(1));
const value = worksheet[z].v;
//store header names
if(row == 1) {
headers[col] = value;
continue;
}
if(!data[row]) data[row] = {};
data[row][headers[col]] = value;
}
//drop those first two rows which are empty
data.shift();
data.shift();
return data;
});
return parsedXlsxSheets[0]
}
async function validateParsedXlsx(users) {
let error = false;
users.forEach(async (user, index) => {
if (!user.email) {
console.log(`User at row ${index + 2} doesn't have 'email' entry in xlsx file.`);
error = true;
}
if (!user.displayName) {
console.log(`User at row ${index + 2} doesn't have 'displayName' entry in xlsx file.`);
error = true;
}
if (!user.serviceProviderId) {
console.log(`Userat row ${index + 2} doesn't have 'displayName' entry in xlsx file.`);
error = true;
} else {
const params = {
TableName: 'service-providers',
Key: {
serviceProviderId: user.serviceProviderId
}
}
const response = await documentClient.get(params).promise();
if (!response.Item) {
console.log(`User at row ${index +2} does not have a valid serviceProviderId.`);
error = true;
} else {
console.log(`User ${user.email} is valid, assigned to service provider: ${response.Item.displayName}`);
}
}
if (error) {
console.log(`Every user in xlsx file must have these attributes, spelled correctly: email, displayName, and serviceProviderId\n\nIn addition, make sure the serviceProviderId is correct by checking the service-providers dynanomdb table.`);
process.exit(1);
}
});
}
async function addUsersToCognito(users) {
const cognitoIdentityServiceProvider = new AWS.CognitoIdentityServiceProvider();
const results = await cognitoIdentityServiceProvider.listUserPools({MaxResults: 10}).promise();
let serviceProviderUserPoolId = '';
results.UserPools.forEach((userPool) => {
if(userPool.Name === 'service-provider-users') {
serviceProviderUserPoolId = userPool.Id;
}
});
users.forEach(async (user) => {
const params = {
UserPoolId: serviceProviderUserPoolId,
Username: user.email,
DesiredDeliveryMediums: ['EMAIL'],
TemporaryPassword: 'New_User1',
UserAttributes: [
{
Name: 'email',
Value: user.email
},
{
Name: 'custom:service_provider_id',
Value: user.serviceProviderId
}
]
}
try {
await cognitoIdentityServiceProvider.adminCreateUser(params).promise();
console.log(`Added user ${user.email} to cognito user pool`);
} catch (error) {
if (error.code === 'UsernameExistsException') {
console.log(`Username: ${user.email} already exists. No action taken.`);
}
else {
console.log(error);
}
}
});
}
async function addUsersToDynamodb(users) {
users.forEach(async (user) => {
const params = {
TableName: 'service-provider-users',
Item: {
serviceProviderId: user.serviceProviderId,
userId: user.email,
displayName: user.displayName,
isActive: false,
role: 'BASIC'
},
ConditionExpression: 'attribute_not_exists(userId)'
}
try {
await documentClient.put(params).promise();
console.log(`Added user ${user.email} to dynamodb user table`);
} catch (error) {
if (error.code === 'ConditionalCheckFailedException') {
console.log(`User ${user.email} already in the dynamodb table service-provider-users`);
} else {
console.log(error);
}
}
});
}
main();
users.forEach(async (user, index) => {
That starts a few promising actions but never awaits them. May do:
await Promise.all(users.map(async (user, index) => {
... to execute them in parallel or do this:
await users.reduce((chain, user, index) => async (user, index) => {
await chain;
//...
}, Promise.resolve());
To execute them one after another.
PS: Using process.exit should be the very last option to end your program
Related
I need to query some data from Redshift in my aws lambda code and I do it several times with different parameters, pretty soon I get the error:
ActiveStatementsExceededException: Active statements exceeded the allowed quota (200).
How can I send thousands of queries to Redshift without hitting a limit?
Here is my JS code:
...
let results = await redshiftQuery(connectionInfo, "SELECT * from db.games where game_id=:game_id", {game_id: 12345})
...
async function redshiftQuery(conInfo, query, params = {}) {
let rsParams = []
for(let key in params) {
rsParams.push( { name: key, value: params[key].toString() })
}
const executeStatementInput = {
ClusterIdentifier: conInfo.clusterId,
Database: conInfo.database,
SecretArn: conInfo.secret,
WithEvent: false,
Sql: query
}
if(rsParams.length) {
executeStatementInput.Parameters = rsParams
}
let results = []
try {
let exResponse = await redshiftDataApiClient.executeStatement(executeStatementInput).promise()
if(!exResponse.Id) return results
let describeStatementInfo = null
while(true) {
let { Status: queryStatus, ...rest } = await redshiftDataApiClient.describeStatement({ Id: exResponse.Id }).promise()
describeStatementInfo = rest
if(["FAILED", "FINISHED"].includes(queryStatus)) {
break
}
await sleepMillis(200)
}
if (describeStatementInfo && describeStatementInfo.HasResultSet) {
let result = await redshiftDataApiClient.getStatementResult({ Id: exResponse.Id }).promise()
convertResult(result, results)
while(result.NextToken) {
result = await redshiftDataApiClient.getStatementResult({ Id: exResponse.Id, NextToken: result.NextToken }).promise()
convertResult(result, results)
}
}
} catch (e) {
console.error("Redshift Error", e)
}
return results
}
Ihave some issues trying to find a index of an expecific product in a mongo database.
const cart = await this.model.findOne({ user: { $eq: user } });
if (cart) {
const itemFound = cart.products.findIndex(
(item) => item._id === new ObjectId(obj._id)
);
I'm sending the id from the frontend as a string and I transform it with the new ObjectId, the issue is it gives me -1, when I console log the item._id and new ObjectId(obj._id). Are the same but I dont know why it gives me -1.
this is the whole fuction I want to do:
async editCart(obj, user) {
try {
const cart = await this.model.findOne({ user: { $eq: user } });
if (cart) {
const itemFound = cart.products.findIndex(
(item) => item._id === new ObjectId(obj._id)
);
if (itemFound !== -1) {
let product = cart.products[itemFound];
product.count += obj.count;
const saved = await cart.save();
return saved;
} else {
cart.products.push(obj);
const saved = await cart.save();
return saved;
}
} else {
const newCart = new this.model({
products: obj,
user: user,
});
const saved = await newCart.save();
return saved;
}
} catch (error) {
logger.error(`Error to edit cart ${error}`);
throw new Error(error);
}
}
If you find another way to do it I will be really greatfull
You can use .toString() when you want to compare to ObjectId values:
const itemFound = cart.products.findIndex(
(item) => item._id.toString() === obj._id.toString()
);
I want to populate my database with some random data. I have used Faker.js for generating that data. I'm using MongoDB on my localhost and all the data is properly following all the validation rules from the schema. I'm having problem with the closing connection of my connection after insertion of data. I want to close the connection soon after the data is populated. I'm using async function to be aware of all the things but something is not going right.
Here is my code seeds.js which is the script im using to populate database
const path = require("path");
require("dotenv").config({ path: path.resolve(__dirname, "../.env") });
var mongoose = require("mongoose");
mongoose.connect(process.env.MONGODB_URI);
require("../models/User");
require("../models/Item");
require("../models/Comment");
var Item = mongoose.model("Item");
var Comment = mongoose.model("Comment");
var User = mongoose.model("User");
const ItemData = require("../data/item.json");
const CommentData = require("../data/comment.json");
const UserData = require("../data/user.json");
async function InsertData() {
ItemData.forEach(async (item) => {
item.seller = item.seller.$oid;
const oldItem = await Item.find({ title: item.title });
if (!oldItem.length) {
var newItem = new Item(item);
await newItem.save();
} else {
console.log(item.slug);
}
});
UserData.forEach(async (user) => {
const oldUser = await User.find({ username: user.username });
if (!oldUser.length) {
var user = new User(user);
await user.save();
} else {
console.log(user.username);
}
});
CommentData.forEach(async (comment) => {
comment.item = comment.item.$oid;
comment.seller = comment.seller.$oid;
var newComment = new Comment(comment);
const oldComment = await Comment.find({ _id: newComment.id });
if (!oldComment.length) {
await newComment.save();
} else {
console.log(comment.body);
}
});
}
async function cleanup() {
await Item.deleteMany({}, () => console.log("Data Cleared Item"));
await Comment.deleteMany({}, () => console.log("Data Cleared Comment"));
await User.deleteMany({}, () => console.log("Data Cleared User"));
}
async function main() {
InsertData().then(async () => {
console.debug('Data Inserted. Closing connection.');
await mongoose.connection.close();
});
}
main();
Here is the stack trace of the error
/Users/karnikkanojia/Desktop/Anythink-Market-21cto/backend/node_modules/mongodb/lib/core/connection/pool.js:841
cb(new MongoError('pool destroyed'));
^
MongoError: pool destroyed
at Pool.write (/Users/karnikkanojia/Desktop/Anythink-Market-21cto/backend/node_modules/mongodb/lib/core/connection/pool.js:841:8)
at _command (/Users/karnikkanojia/Desktop/Anythink-Market-21cto/backend/node_modules/mongodb/lib/core/wireprotocol/command.js:120:10)
at command (/Users/karnikkanojia/Desktop/Anythink-Market-21cto/backend/node_modules/mongodb/lib/core/wireprotocol/command.js:28:5)
at Object.query (/Users/karnikkanojia/Desktop/Anythink-Market-21cto/backend/node_modules/mongodb/lib/core/wireprotocol/query.js:66:3)
at Server.query (/Users/karnikkanojia/Desktop/Anythink-Market-21cto/backend/node_modules/mongodb/lib/core/topologies/server.js:644:16)
at FindOperation.execute (/Users/karnikkanojia/Desktop/Anythink-Market-21cto/backend/node_modules/mongodb/lib/operations/find.js:38:12)
at /Users/karnikkanojia/Desktop/Anythink-Market-21cto/backend/node_modules/mongodb/lib/operations/execute_operation.js:144:17
at Server.selectServer (/Users/karnikkanojia/Desktop/Anythink-Market-21cto/backend/node_modules/mongodb/lib/core/topologies/server.js:832:3)
at Server.selectServer (/Users/karnikkanojia/Desktop/Anythink-Market-21cto/backend/node_modules/mongodb/lib/topologies/topology_base.js:342:32)
at executeWithServerSelection (/Users/karnikkanojia/Desktop/Anythink-Market-21cto/backend/node_modules/mongodb/lib/operations/execute_operation.js:131:12)
at /Users/karnikkanojia/Desktop/Anythink-Market-21cto/backend/node_modules/mongodb/lib/operations/execute_operation.js:70:9
at maybePromise (/Users/karnikkanojia/Desktop/Anythink-Market-21cto/backend/node_modules/mongodb/lib/utils.js:685:3)
at executeOperation (/Users/karnikkanojia/Desktop/Anythink-Market-21cto/backend/node_modules/mongodb/lib/operations/execute_operation.js:34:10)
at Cursor._initializeCursor (/Users/karnikkanojia/Desktop/Anythink-Market-21cto/backend/node_modules/mongodb/lib/core/cursor.js:534:7)
at Cursor._initializeCursor (/Users/karnikkanojia/Desktop/Anythink-Market-21cto/backend/node_modules/mongodb/lib/cursor.js:186:11)
at nextFunction (/Users/karnikkanojia/Desktop/Anythink-Market-21cto/backend/node_modules/mongodb/lib/core/cursor.js:737:10)
error Command failed with exit code 1.
I have been struggling with this for hours and have tried a lot of different variations I have found around the web and also on stack overflow but I keep getting stuck on the same thing.
This is my registration code:
// REGISTER USER
app.post("/register", async (request, response) => {
const saltRounds = 10;
const emailAddress = request.body.emailAddress;
const password = await bcrypt.hash(request.body.password, saltRounds);
console.log(password)
// CHECK IF A USER EXISTS
const sqlSearch = "SELECT * FROM users WHERE emailAddress = ?"
const search_query = mysql.format(sqlSearch, [emailAddress])
// INSERT NEW USER
const sqlInsert = "INSERT INTO users (emailAddress, password) VALUES (?,?)"
const insert_query = mysql.format(sqlInsert, [emailAddress, password])
await usersDB.query(search_query, async (err, result) => {
if (err) throw (err)
if (result.length != 0) {
console.log("------> User already exists")
response.send("exists")
} else {
await usersDB.query(insert_query, (err, result) => {
if (err) throw (err)
response.send("created")
})
}
})
})
This is my login code:
// LOGIN (AUTHENTICATE USER)
app.post("/login", async (request, response) => {
const emailAddress = request.body.emailAddress
const password = request.body.password
const sqlSearch = "SELECT * FROM users WHERE emailAddress = ?"
const search_query = mysql.format(sqlSearch, [emailAddress])
await usersDB.query(search_query, async (err, result) => {
if (err) throw (err)
if (result.length == 0) {
console.log("--------> User does not exist")
response.sendStatus(404)
} else {
// Get the hashed password from result
const hashedPassword = result[0].Password
await bcrypt.compare(password, hashedPassword, function(err, result) {
if (result) {
console.log("---------> Login Successful")
response.send(`${emailAddress} is logged in!`)
} else {
console.log("---------> Password Incorrect")
console.log(password)
console.log(hashedPassword)
response.send("Password incorrect!")
}
});
}
})
})
I don't really understand what is going wrong in the compare considering the hashes are the same, I also tried pulling the salt rounds out and declaring them as a variable as you can see, this was recommended on another answer. I have changed the compare await in several different ways but they all give the same result.
I did also check the typeof on each var and they are all strings as they need to be.
My output:
The first hash you see is what is going into the database, the password being "test" and the second hash is from the compare statement along with the plaintext being shown.
$2b$10$wXGSrneIiovWHG7wk6a0BOIXwhzelTlCcxeoLsVJ8Au4iiOcoBBhe
---------> Password Incorrect
test
$2b$10$wXGSrneIiovWHG7wk6a0BOIXwhzelTlCcxeoLsVJ8Au4iiOcoBBhe
Any help would be greatly appreciated.
Note: The password column in my DB is a VARCHAR(255)
You can make a 2 seperate function for achieve the bcrypt functions. Here is the helper file which holds the bcrypt functions
const logger = require('./logger');
const bcrypt = require('bcrypt');
const encryptUtil = {};
// It make a hash password
encryptUtil.oneWayEncrypt = async (text) => {
try {
const salt = await bcrypt.genSalt(parseInt(process.env.SALT_ROUND, 10));
const encoded = await bcrypt.hash(text, salt);
return { encoded, salt };
} catch (err) {
logger.error('[ERROR] From oneWayEncrypt in encryptUtils', err);
throw err;
}
};
// It will validate plain text with the hashed one
encryptUtil.validateBcryptHash = async (text, hash) => {
try {
const isExactMatch = await bcrypt.compare(text, hash);
return isExactMatch;
} catch (err) {
logger.error('[ERROR] From validateBcryptHash in encryptUtils', err);
throw err;
}
};
module.exports = encryptUtil;
Here is the usecase of that function in signup and login
const encryptUtil = require('../../../helper/encryptUtil');
const logger = require('../../../helper/logger');
const jwt = require('../../../helper/jwt');
const userUtils = {};
userUtils.signUp = async (obj) => {
try {
const { name, password } = obj;
const email = obj.email.toLowerCase();
const condition = { email };
const querying = {
attributes: ['id', 'name', 'email''],
where: { email },
};
const isEmailExist = await Model.user.findOne(querying);
if (isEmailExist) {
const errorObj = { code: 400, error: l10n.t('ERR_EMAIL_ALREADY_EXIST') };
throw errorObj;
}
const { encoded: encPassword } = await encryptUtil.oneWayEncrypt(password);
const insertObj = {
name,
email,
password: encPassword,
};
const result = await Model.user.create(insertObj);
const userId = result.id;
const token = jwt.getAuthToken({ userId });
return { token, msg: l10n.t('MSG_SIGNUP_SUCCESS'), user: { name, email, userId } };
} catch (error) {
logger.error('[ERROR] From signUp in userUtils', error);
throw error;
}
};
userUtils.login = async (obj) => {
try {
const { password } = obj;
const email = obj.email.toLowerCase();
const querying = {
attributes: ['id', 'name', 'email', 'password'],
where: { email },
};
const user = await Model.user.findOne(querying);
if (!user) {
const errorObj = { code: 400, error: l10n.t('ERR_CREDENTIAL_NOT_MATCHED') };
throw errorObj;
}
// Here it validates the simple text with hashed text which store in a dbatabase
const isExactMatch = await encryptUtil.validateBcryptHash(password, user.password);
if (!isExactMatch) {
const errorObj = { code: 400, error: l10n.t('ERR_CREDENTIAL_NOT_MATCHED') };
throw errorObj;
}
const token = jwt.getAuthToken({ userId: user.id });
const result = {
token,
user: {
userId: user.id,
name: user.name,
email: user.email,
};
return result;
} catch (error) {
logger.error('[ERROR] From login in userUtils', error);
throw error;
}
};
module.exports = userUtils;
guys i create a function to limit the user`s upload images everything works fine but the problem that
i got an Error on the console (Cannot set headers after they are sent to the client)
and when i copy the whole code of the function note(! not the function itself ) . on the controllers file i didnt get the error however ( its the same code and same everything
but the code is too long so it looks bad
please check the image before looking into the code
const Joi = require('joi');
const appError = require('./appError')
module.exports.validateHadith = (req,res,next)=>{
const hadithSchema = Joi.object({
Hadith: Joi.object({
narrator: Joi.string().required(),
description:Joi.string().required().min(15),
hadith:Joi.string().required().min(15),
}).required()
})
const {error} = hadithSchema.validate(req.body);
if(error){
const msg = error.details.map(el=>el.message).join(',')
throw new appError(msg,400)
} else{
next()
}
};
module.exports.reviewValidation = (req,res,next) => {
const reviewJoiSchema = Joi.object({
review: Joi.object({
comment: Joi.string().required(),
rating: Joi.number().required(),
}).required()
})
const {error} = reviewJoiSchema.validate(req.body);
if(error) {
const msg = error.details.map(el=>el.message).join(',')
throw new appError(msg,400)
} else {
next()
}
}
module.exports.imageValidation = (req,res,next) => {
const imageSchema = Joi.array().min(1).max(6).required()
const {error} = imageSchema.validate(req.files)
if(error) {
const msg = error.details.map(el=>el.message).join(',')
throw new appError(msg,400)
} else{
next()
}
}
module.exports.imageEditValidation = async (updatedHadith,images,req,res,id) => {
const length = updatedHadith.images.length
const max = 6
if(images.length+length <= max) {
updatedHadith.images.push(...images)
await updatedHadith.save()
console.log(updatedHadith)
} else{
req.flash('error','Sorry you can`t have more than 6 images')
return res.redirect(`/hadith/${id}`)
}
}
You need to add await
await imageEditValidation(updateHadith, images,req, res, id)
Currently, it's not awaiting, so it's jumping to req.flash("success", "successfully editing the Hadith") without waiting for it to be returned.
You also need to add a return after `console.log(updatedHadith)
Also need to remove req.flash and res.redirect('/hadith/${id}') statement, and replace it with a return
the validator code will be
module.exports.imageEditValidation = async (updatedHadith,images,req,) => {
const length = updatedHadith.images.length
const max = 6
if(images.length+length <= max) {
updatedHadith.images.push(...images)
await updatedHadith.save()
console.log(updatedHadith)
return true
} else{
req.flash('error','Sorry you can`t have more than 6 images')
return false
}
}
and controller code will be
module.exports.postEditForm =async (req, res, next) => {
const {id} = req.params;
const updatedHadith = await HadithModel.findByIdAndUpdate(
id,
{...req.body.Hadith},
{ runValidators: true }
);
const images = req.files.map(f => ({url : f.path ,filename: f.filename}))
if(await imageEditValidation (updatedHadith,images,req)){
req.flash("success", " The Hadith is successfully Edited :)");
}
res.redirect(`/hadith/${id}`);
}