I have a database with cars and a 3rd party api where I can update some info about them. I have written this code in order to update every x days, but the problem is that the 3rd party database doesn't allow more than x connections per hour. So what I want to do is a function that does one request sequentially every 40s
My database looks like
[{id: 1, name: "car1", rating: 4.6, external_db_id: 12342},
{id: 2, name: "car2", rating: 4.7, external_db_id: 99999}]
Updated code:
async () => {
const allCars = await functionThatGetsCars()
for ( var i = 0; i < allCars.length; i++ ) {
let car = allCars[i]
let canUpdate = null
try{
canUpdate = await fetchExternalData(car.external_db_id)
} catch(err){
console.log("Error fetching external data", err)
canUpdate = null
}
if(canUpdate){
await new Promise((resolve, reject) => setTimeout(resolve, 40000))
const updated = await functionThatUpdatesCarsInMyDB({
id: car.id,
},
{
rating: canUpdate ? canUpdate.external_rating : car.rating
}
return updated
}
}
}
Problem is that right now that it does just update the first item from the loop. What would be the way to make it sequentially with a timeout of 40s? Thanks in advance!
Solved thanks to #FelixKling suggestions:
async () => {
const allCars = await functionThatGetsCars()
for ( var i = 0; i < allCars.length; i++ ) {
let car = allCars[i]
let canUpdate = null
try{
canUpdate = await fetchExternalData(car.external_db_id)
} catch(err){
console.log("Error fetching external data", err)
canUpdate = null
}
if(canUpdate){
await new Promise((resolve, reject) => setTimeout(resolve, 40000))
const updated = await functionThatUpdatesCarsInMyDB({
id: car.id,
},
{
rating: canUpdate ? canUpdate.external_rating : car.rating
}
}
}
}
Related
I need to query some data from Redshift in my aws lambda code and I do it several times with different parameters, pretty soon I get the error:
ActiveStatementsExceededException: Active statements exceeded the allowed quota (200).
How can I send thousands of queries to Redshift without hitting a limit?
Here is my JS code:
...
let results = await redshiftQuery(connectionInfo, "SELECT * from db.games where game_id=:game_id", {game_id: 12345})
...
async function redshiftQuery(conInfo, query, params = {}) {
let rsParams = []
for(let key in params) {
rsParams.push( { name: key, value: params[key].toString() })
}
const executeStatementInput = {
ClusterIdentifier: conInfo.clusterId,
Database: conInfo.database,
SecretArn: conInfo.secret,
WithEvent: false,
Sql: query
}
if(rsParams.length) {
executeStatementInput.Parameters = rsParams
}
let results = []
try {
let exResponse = await redshiftDataApiClient.executeStatement(executeStatementInput).promise()
if(!exResponse.Id) return results
let describeStatementInfo = null
while(true) {
let { Status: queryStatus, ...rest } = await redshiftDataApiClient.describeStatement({ Id: exResponse.Id }).promise()
describeStatementInfo = rest
if(["FAILED", "FINISHED"].includes(queryStatus)) {
break
}
await sleepMillis(200)
}
if (describeStatementInfo && describeStatementInfo.HasResultSet) {
let result = await redshiftDataApiClient.getStatementResult({ Id: exResponse.Id }).promise()
convertResult(result, results)
while(result.NextToken) {
result = await redshiftDataApiClient.getStatementResult({ Id: exResponse.Id, NextToken: result.NextToken }).promise()
convertResult(result, results)
}
}
} catch (e) {
console.error("Redshift Error", e)
}
return results
}
I am having a hard time understanding serverTimestamp in firestore.
When I save a document in database in a firebase function using Fieldvalue.serverTimestamp() or in a javascript client code using serverTimestamp() it sometimes doesn't save the same thing in the database.
See screenshots below :
Sometime I get an object with {nanoseconds: xxx, seconds: xxx} and sometimes I get a timestamp formatted date...
The problem is when I try to query my orders using query(collectionRefOrders, orderBy('createdAt', 'desc'), limit(10)).
The orders with the object appears before the others ones even if they are created after...
Any clue why this happens ? What am I doing wrong ?
Thanks a lot.
EDIT :
Here is the code I use to add documents in the my firebase function (it is a request function I call in a website) :
const { getFirestore, FieldValue } = require('firebase-admin/firestore');
const firebaseDB = getFirestore();
exports.createOrderFromTunnel = functions.region('europe-west3')
.runWith({
timeoutSeconds: 10,
memory: "4GB",
})
.https
.onRequest(async (req, res) => {
cors(req, res, async () => {
try {
const { apiKey } = req.body;
const project = await getProjectFromApiKey(apiKey);
if (!project) {
return res.json({
success: false,
error: 'Unauthorized: invalid or missing api key'
});
}
const contactData = {
address: {},
createdAt: FieldValue.serverTimestamp()
};
const orderData = {
accounting: {
totalHT: 0,
totalTTC: 0,
totalTVA: 0,
},
createdAt: FieldValue.serverTimestamp(),
status: 'NEW',
};
const refProject = firebaseDB
.collection('projects')
.doc(project.id);
const colOrder = firebaseDB.collection(`projects/${project.id}/orders`)
const refOrder = colOrder.doc();
const colContact = firebaseDB.collection(`projects/${project.id}/contacts`)
const refContact = colContact.doc();
await firebaseDB.runTransaction(async transaction => {
const snapProject = await transaction.get(refProject);
const dataProject = snapProject.data();
const sequenceContact = dataProject.sequenceContact;
const sequenceOrder = dataProject.sequenceOrder;
contactData.sequence = sequenceContact;
orderData.sequenceNumber = sequenceOrder;
await transaction.set(refContact, contactData);
orderData.customer.id = refContact.id;
orderData.customer.sequence = sequenceContact;
await transaction.set(refOrder, orderData);
await transaction.update(refProject, {
sequenceContact: sequenceContact + 1,
sequenceOrder: sequenceOrder + 1,
totalContacts: dataProject.totalContacts + 1,
totalOrders: dataProject.totalOrders + 1,
});
return refOrder.id;
});
return res.json({
success: true
});
} catch (err) {
functions.logger.error(err);
return res.json({
success: false,
err
});
}
});
});
Here is the code I use to add documents in my client code (it is a web app in javascript) :
const createOrder = async (projectId) => {
try {
const orderData = {
accounting: {
totalHT: 0,
totalTTC: 0,
totalTVA: 0,
},
createdAt: serverTimestamp(),
status: 'NEW',
surface: 0,
};
const refProject = doc(firebaseDB, 'projects', projectId);
const colOrder = collection(firebaseDB, `projects/${projectId}/orders`)
const refOrder = doc(colOrder);
return await runTransaction(firebaseDB, async (transaction) => {
const snapProject = await transaction.get(refProject);
if (!snapProject.exists()) {
throw "Document does not exist!";
}
const dataProject = snapProject.data();
const sequence = dataProject.sequenceOrder;
orderData.sequenceNumber = sequence;
transaction.set(refOrder, orderData);
transaction.update(refProject, { sequenceOrder: sequence + 1, totalOrders: dataProject.totalOrders + 1 });
return refOrder.id;
});
} catch (e) {
console.error(e);
return null;
}
};
I have a sns lambda function that returns void (https://docs.aws.amazon.com/lambda/latest/dg/with-sns.html). This event orderId and one message'status: success' are what I'm publishing. I check if the 'orderId' exists in my data database in the sns subscription lambda event. If it already exists, update the database; if it doesn't, console error it.
I created an integration test in which I transmit a random 'uuid' that isn't a valid 'orderId,' but it appears that my promise doesn't capture the'rejected'. It should show in console error failed to find order... I'm not sure where I'm going wrong. Also My promise syntax looks complicated, is there any neat way, I can do it. Thank you in advance 🙏🏽
This is sns event, which listen the publishing
interface PromiseFulfilledResult<T> {
status: "fulfilled" | "rejected";
value: T;
}
const parseOrdersFromSns = (event: SNSEvent) => {
try {
return event.Records.flatMap((r) => JSON.parse(r.Sns.Message))
} catch (error) {
console.error('New order from SNS failed at parsing orders', { event }, error)
return []
}
}
export const handlerFn = async (event: SNSEvent): Promise<void> => {
const orders = parseOrdersFromSns(event)
if (orders.length === 0) return
const existingOrdersPromiseResult = await Promise.allSettled(
orders.map(
async (o) => await findOrderStateNode(tagOrderStateId(o.orderId))
)
); // This returns of data if the order exsiit other it will return undefined
const existingOrders = existingOrdersPromiseResult // should returns arrays of data
.filter(({ status }) => status === "fulfilled")
.map(
(o) =>
(
o as PromiseFulfilledResult<
TaggedDatabaseDocument<
OrderStateNode,
TaggedOrderStateId,
TaggedOrderStateId
>
>
).value
);
const failedOrders = existingOrdersPromiseResult.filter( // should stop the opeartion if the data is exsit
({ status }) => status === "rejected"
);
failedOrders.forEach((failure) => {
console.error("failed to find order", { failure });
});
const updateOrder = await Promise.all(
existingOrders.map((o) => {
const existingOrderId = o?.pk as TaggedOrderStateId;
console.log({ existingOrderId }); // Return Undefined
})
);
return updateOrder;
};
this is my test suite
describe('Creating and updating order', () => {
integrationTest(
'Creating and updating the order',
async (correlationId: string) => {
CorrelationIds.set('x-correlation-id', correlationId)
const createdOrder = await createNewOrder(correlationId) // This create random order
if (!createdOrder.id) {
fail('order id is not defined')
}
const order = await getOrder(createdOrder.id)
// Add new order to table
await initializeOrderState([order])
const exisitingOrder = await findOrderStateNode(tagOrderStateId(order.id))
if (!exisitingOrder) fail(`Could not existing order with this orderId: ${order.id}`)
const event = {
Records: [
{
Sns: {
Message: JSON.stringify([
{
orderId: uuid(), // random order it
roundName,
startTime,
},
{
orderId: order.id,
roundName,
startTime,
},
{
orderId: uuid(),
roundName,
startTime,
},
]),
},
},
],
} as SNSEvent
await SnsLambda(event)
const updateOrderState = await findOrderStateNode(tagOrderStateId(order.id))
expect(updateOrderState?.status).toEqual('success')
},
)
})
const generatePropertyAmenities = async (property) => {
let result = [];
property.amenities.map(async (res, key) => {
/**
* Get the amenities type id, creates it if it doesn't exist
* const type_existed_before: defines if the type already existed
*/
let type_id = null;
const type_existed_before = await strapi
.query("amenities-categories")
.findOne({ name: res.type.trim() });
if (type_existed_before) type_id = type_existed_before.id;
else
type_id = await strapi
.query("amenities-categories")
.create({ name: res.type }).id;
res.items.map(async (amenity) => {
const amenity_exists = await strapi
.query("amenities")
.findOne({ name: amenity.name.trim() });
if (amenity_exists) {
// console.log(amenity_exists)
if (
!type_existed_before ||
(type_existed_before &&
!amenity_exists.amenities_categories.find((x) => x === type_id))
) {
await strapi.query("amenities").update(
{ id: amenity_exists.id },
{
amenities_categories: [
...amenity_exists.amenities_categories,
type_id,
],
}
);
}
result = [...result, amenity_exists.id];
} else {
result = [
...result,
await strapi
.query("amenities")
.create({
name: amenity.name.trim(),
amenities_categories: [type_id],
}).id,
];
}
});
});
console.log(result);
return result;
};
result is always empty, am I missing something here ? I really don't see what I'm doing wrong. I checked everything, amenity_exists.id is correct, everything seems correct but something is not I really need some help here guys
Ok so I managed to make it work, I don't think I can explain why and how exactly it works I may need some explanations from you guys, but here's my fix:
const generatePropertyAmenities = async (property) => {
const getAmenityCategoryId = (item) =>
new Promise(async (resolve, reject) => {
const type_existed_before = await strapi
.query("amenities-categories")
.findOne({ name: item.type.trim() });
if (type_existed_before)
resolve({
type_id: type_existed_before.id,
type_existed_before: false,
});
else
resolve({
type_id: await strapi
.query("amenities-categories")
.create({ name: item.type }).id,
type_existed_before: false,
});
});
const getAmenity = (amenity, type_id, type_existed_before) =>
new Promise(async (resolve, reject) => {
const amenity_exists = await strapi
.query("amenities")
.findOne({ name: amenity.name.trim() });
if (amenity_exists) {
// console.log(amenity_exists)
if (
!type_existed_before ||
(type_existed_before &&
!amenity_exists.amenities_categories.find((x) => x === type_id))
) {
await strapi.query("amenities").update(
{ id: amenity_exists.id },
{
amenities_categories: [
...amenity_exists.amenities_categories,
type_id,
],
}
);
}
resolve(amenity_exists.id);
} else {
resolve(
await strapi.query("amenities").create({
name: amenity.name.trim(),
amenities_categories: [type_id],
}).id
);
}
});
const createArrayOfAmenities = () =>
new Promise(async (resolve, reject) => {
let ret = [];
for (let index = 0; index < property.amenities.length; index++) {
const element = property.amenities[index];
let { type_id, type_existed_before } = await getAmenityCategoryId(
element
);
for (let k = 0; k < element.items.length; k++) {
const amenity = element.items[k];
ret = [
...ret,
await getAmenity(amenity, type_id, type_existed_before),
];
}
}
resolve(ret)
});
return await createArrayOfAmenities();
};
Try
await res.items.map(async ...
I have a loop. On each round I need to add Question data into MongoDB database. This works fine. However, I want to get _id of the new inserted Question before the loop goes into the next round. This is where I have a problem. It takes certain amount of time before the server returns _id and loop goes to the next round by that time. Therefore, I need a way to wait for the server response and only after that move to the next round of the loop.
Here is my back-end code:
router.post("/createQuestion", (req, res) => {
const newQuestion = new Question({
description: req.body.description,
type: req.body.type,
model: req.body.model
});
newQuestion.save().then(question => res.json(question._id))
.catch(err => console.log(err));
});
Here is my axios function, which is in a separate file and imported into the class:
export const createQuestion = (questionData) => dispatch => {
axios.post("/api/scorecard/createQuestion", questionData)
.then(res => {
return res.data;
}).catch(err =>
console.log("Error adding a question")
);
};
Here is my code inside my class:
JSON.parse(localStorage.getItem(i)).map(question => {
const newQuestion = {
description: question.description,
type: question.questionType,
model: this.props.model
}
const question_id = this.props.createQuestion(newQuestion);
console.log(question_id);
}
Console shows undefined.
i faced the same issue i solved the same by sending the array question to the node and read one by one question and update with the next Question ID.
router.post("/createQuestion", (req, res) => {
let d =[questionarray];
let i = 0;
let length = d.length;
var result = [];
try {
const timeoutPromise = (timeout) => new Promise((resolve) => setTimeout(resolve, timeout));
for (i = 0; i < length; i++) {
await timeoutPromise(1000); // 1000 = 1 second
let CAT_ID = parseInt(d[i].CAT_ID);
let TOPIC_ID = parseInt(d[i].TOPIC_ID);
let Q_DESC = (d[i].Q_DESC);
let OPT_1 = (d[i].OPT_1);
let OPT_2 = (d[i].OPT_2);
let OPT_3 = (d[i].OPT_3);
let OPT_4 = (d[i].OPT_4);
let ANS_ID = (d[i].ANS_ID);
let TAGS = (d[i].TAGS);
let HINT = (d[i].HINT);
let LEVEL = d[i].LEVEL;
let SRNO = d[i].SrNo;
let qid;
const savemyData = async (data) => {
return await data.save()
}
var myResult = await Question.find({ TOPIC_ID: TOPIC_ID }).countDocuments(function (err, count) {
if (err) {
console.log(err);
}
else {
if (count === 0) {
qid = TOPIC_ID + '' + 10001;
const newQuestion = new Question({
Q_ID: qid,
CAT_ID: CAT_ID,
TOPIC_ID: TOPIC_ID,
Q_ID: qid,
Q_DESC: Q_DESC,
OPT_1: OPT_1,
OPT_2: OPT_2,
OPT_3: OPT_3,
OPT_4: OPT_4,
ANS_ID: ANS_ID,
HINT: HINT,
TAGS: TAGS,
LEVEL: LEVEL,
Q_IMAGE: ''
})
await savemyData(newQuestion)
.then(result => { return true })
.catch(err => { return false });
//`${SRNO} is added successfully`
//`${SRNO} is Failed`
}
else if (count > 0) {
// console.log(count)
Question.find({ TOPIC_ID: TOPIC_ID }).sort({ Q_ID: -1 }).limit(1)
.then(question => {
qid = question[0].Q_ID + 1;
const newQuestion = new Question({
Q_ID: qid,
CAT_ID: CAT_ID,
TOPIC_ID: TOPIC_ID,
Q_ID: qid,
Q_DESC: Q_DESC,
OPT_1: OPT_1,
OPT_2: OPT_2,
OPT_3: OPT_3,
OPT_4: OPT_4,
ANS_ID: ANS_ID,
HINT: HINT,
TAGS: TAGS,
LEVEL: LEVEL,
Q_IMAGE: ''
})
await savemyData(newQuestion)
.then(result => { return true })
.catch(err => { return false });
})
.catch(err => console.log(err));
}
}
});
if (myResult)
result.push(`${SRNO} is added successfully`);
else
result.push(`${SRNO} is Failed`);
}
// console.log(result)
return res.json(result);
}
catch (err) {
//res.status(404).json({ success: false })
console.log(err)
}
});
First your function createQuestion doesn't return a value so the assigning to question_id would always be undefined. Anyways, since u have a dispatch in your createQuestion function, I am assuming u r using redux, so I would suggest you to using redux-thnk, split the fetching new action logic to a thunk action, and use the questionID value from the redux state rather than returning a value from createQuestion. In your class u can be listening for a change of the questionID and if that happens, dispatch the saving of the next question.