Develop Reference

Knexjs how to handle multiple updates

I´m quite unsure on how to handle multiple updates / inserts in knex and return whatever it was successfull on the end or not.
I´m passing an array through req.body loop through it and trigger actions based on informations inside the array.
Example:
const data = [...req.body]
for(let i = 0; i < data.length; i++) {
data[i].totals.length
for(let y = 0; y < data[i].totals.length; y++) {
if(data[i].totals[y].info === "Holiday") {
calcHoliday(data[i].totals[y].total, data[i].id)
} else if(data[i].totals[y].info === "ZA") {
calcZA(data[i].totals[y].total, data[i].id)
}
}
calcOvertime(data[i].totalSum, data[i].id)
if(i === data.length -1) {
res.json("Success")
}
}
The Array I´m passing in looks like this:
[
{
"id": 1,
"totals": [
{
"info": "Holiday",
"total": 4
}
]
},
{
"id": 1,
"totals": [
{
"info": "Holiday",
"total": 4
}
]
}
]
Function Example which gets called in for loop:
const calcHoliday = (hours, userid) => {
knex.transaction(trx => {
trx.insert({
created_at: convertedTime,
info: "Booking Holiday - Hours: " + hours,
statuscode: 200
}).into("logs")
.then(() => {
return trx("hours")
.decrement("holiday_hours", hours)
}).then(trx.commit)
.catch(trx.rollback)
}).then(() => console.log("WORKED"))
.catch(err => console.log(err))
}
This is working perfectly fine but I can´t figure out how to gather the results from each table update in order to respond if everything worked or an error appeared. If I call e.g. after one calcHoliday call .then(resp => res.json(resp) I receive only the response from the first operation.
In short I need a way on how to res.json if everything succeeded or an error appeared somewhere.
Thanks in advance!

TLDR;
Turning your insert calls into an array of promises and then using await and a Promise.all() / Promise.allSettled() structure might solve this problem, but there are some UX decisions to make on what to rollback and how to return errors.
Error Handling Choices:
Any error --> all insertions in all loop iterations should be rolled back
Do you want partial success? The way the code is written now, rollback only applies to items in one function call. If one of the hour-decrement calls fails, it will roll back one log insert, but not any that succeeded for previous data in the loop. If you want the whole dataset to rollback, you'd need to pass the txn through each function call or do a bulk insert of all of your rows in one function call, which might be nice for performance reasons anyway depending on the use case.
Partial success --> commits successes, rolls back single loop iterations that fail, sends detailed list of errors and successes
You'd want to use Promise.allSettled(), which aggregates the successes and errors as an array from all promises in the loop.
Partial success --> commits the successes, rolls back single loop iterations that fail, sends just one error
Opinion: This can be a misleading UX unless the error is "some of the insertions were unsuccessful" and the endpoint is idempotent
This looks closest to what you're describing you want. If this is the case, you'd want to use Promise.all(), which throws an error as soon as one promise in the array errors.
Example Implementation:
Since the original code is incomplete, this is a loose, incomplete example of what option 2/3 might look like. This could easily be transformed into option 1.
First, it might help to modify all of your functions with asynchronous calls to be fulfillable as promises. Async/await helps avoid .then() trees that are hard to reason about.
const calcHoliday = async (hours, userid) => {
try {
const result = await knex.transaction(async(trx) => {
await trx.insert({
created_at: convertedTime,
info: "Booking Holiday - Hours: " + hours,
statuscode: 200
}).into("logs")
return trx("hours").decrement("holiday_hours", hours)
}
return result
} catch(err) {
console.log("It didn't work.")
throw new Error(`Error: Failure to insert for user ${userid}:`, err)
}
}
Here are some utilities to get the data transformed, and to get the appropriate unfulfilled promise to supply to the map in Promise.all/allSettled.
/*
Here's an example of how you might transform the original data with maps in order to avoid nested for-loops:
[
{ id: 1, info: 'Holiday', total: 4 },
{ id: 1, info: 'Holiday', total: 4 }
]
*/
const flatData = data.map(item => {
return item.totals.map(total => ({
id: item.id,
...total
}))
}).flat()
// Returns the appropriate promise based on data
const getTotalsPromises = (row) => {
const {info, id, total} = row
if(info === "Holiday") {
return calcHoliday(total, id)
} else if(info === "ZA") {
return calcZA(total, id)
}
}
const getcalcOvertimePromises = (rowInData) = {
// work left to reader
return calcOvertime(rowInData.correctData, rowInData.otherData)
}
If you want option 2:
// Replaces the loop
// Fulfills *all* the promises, creating an array of errors and successes
const responses = await Promise.allSettled([
...flatData.map(getTotalsPromises),
...data.map(getCalcOvertimePromises)
])
// insert loop here to do something with errors if you want
res.send(responses)
OR Option 3
Create an array of all of the promises you want to run, run them, and process up to one error.
// Replaces the loop
// Runs the promises and waits for them all to finish or the first error.
try {
const responses = await Promise.all([
...flatData.map(getTotalsPromises),
...data.map(getCalcOvertimePromises)
])
res.send(responses)
} catch(err){
// Reached if one of the rows errors
res.send(err)
}
Docs:
Promise.allSettled
Promise.all

How do I recursively search for results in Firestore?

I'm new to Node and Firebase.
I'm currently working on a crafting calculator for a game and have the game items stored in Firebase. Some items are composite items, for example:
1 Lumber Plank = 5 Logs
Based on such requirements, I've structured all the items as a single collection titled as items in Firebase.
Log would be persisted as:
{
"type": "basic",
"name": "log"
}
While lumber plank would be:
{
"type": "composite",
"name": "lumber plank",
"materials": ["log"],
"material_values": [5]
}
With such a structure, I'm trying to construct a crafting tree by recursively searching through the database. A final structure would look as such:
{
"name": "board",
"count": 1,
"materials": [
{
"name": "lumber plank",
"count": 1,
"materials": [
{
"name": "log",
"count": 5,
"materials": null
}
]
}
]
}
I'm having trouble with understanding the callbacks while debugging and this piece of code currently returns undefined followed by log (I'm assuming this comes from the console.log within the search function).
async function search(item, result, count) {
let calcItem = {
name: item,
count: count
};
db.collection("items")
.doc(item)
.get()
.then(doc => {
const data = doc.data();
if (data.type === basic) {
calcItem.materials = null;
result.push(calcItem);
return result;
} else {
let materials = data.materials;
let materialsCount = data.material_values;
calcItem.materials = [];
for (let i = 0; i < materials.length; i++) {
console.log(materials[i]);
search(materials[i], calcItem.materials, materialsCount[i]);
}
}
});
}
let item = "lumber plank";
search(item, [], 1).then(result => console.log(result));
Would appreciate any pointers/tips here. Thanks
Following feedback from Doug,
I've kinda refactored my code based on your comments and I'm seeing some progress.
function recursiveSearch(item, count, result) {
let calcItem = {
name: item,
count: count
};
dbSearch(item).then(function (doc) {
const data = doc.data();
console.log(data);
if (data.type === basic) {
calcItem.materials = null;
result.push(calcItem);
return result;
} else {
let materials = data.materials;
let materialsCount = data.material_values;
calcItem.materials = [];
for (let i = 0; i < materials.length; i++) {
recursiveSearch(materials[i], materialsCount[i], calcItem.materials);
}
}
});
}
function dbSearch(item) {
return Promise.resolve(db.collection("items")
.doc(item)
.get()
.then());
}
Log now outputs the search correctly.
{
material_values: [ 5 ],
materials: [ 'log' ],
name: 'lumber plank',
type: 'composite'
}
{
name: 'log',
type: 'basic'
}
However, if I understand it correctly, if I were to add in this line it's still going to return undefined, am I right?
console.log(recursiveSearch("lumber plank", 1, [])
If so, how do I actually log out the entire item structure whilst completing all the recursive searches?
Sorry if the question sounds kinda dumb. I primarily come from a Java background and dealing with promises/async/await is entirely new to me

You're not dealing with promises correctly. search doesn't actually return a "real" promise, but the caller is expecting it to do so. Since it's async, but doesn't return a value directly, it's actually returning is a promise that always resolves to undefined. It's also apparently intended to be a recursive function, which is making it harder to understand (did you mean to return the promise from the inner call to search?).
Minimally, you should start by making search return the promise chain that it establishes:
return db.collection("item")...get().then()
This will let you receive the value returned by the then callback.
I'll also point out that you're started to use async/await syntax, but never committed to using await to make this code more easy to read, which is a bit confusing.

What is the behavior of an observable within a forEach loop?

When querying an observable within a forEach loop, does a new "instance" of the observable get created on every loop or does the existing observable get overwritten if the next loop query is requested before the previous observable returns a value?
For example, if I have messages to send to my users, I need to query for all their devices and send to all devices. An example of a topic in the topics array that is passed to createTopics(topics) would be:
Single Item in Topics Array
{
"submission": {
"id": 52,
"artistID": 111,
"title": "Sego Sucks",
"Artist": {
"name": "Sego"
}
},
"users": [
{
"id": 7,
"userUID": "ZvOBNBqxbgRYoibSYEwkL9YKtWG2"
}
]
}
My question is, when I run the forEach loop, I have an observable in findUserDevices that is iterated over several times really quickly, will my observable be "overwritten" by the next loop if it doesn't return a value before the next loop comes about? So far when I run this code, it executes as planned, but I am not sure if this is the best way to handle the observable as my data scales and there are more users per topic especially since pushToDevices() is an async function.
function createTopics(topics) {
topics.forEach((topic: any) => {
const message = {
notification: {
title: `New music from ${topic.submission.Artist.name}!`,
body: `Listen to "${topic.submission.title}" now`
}
}
topic.users.forEach((user) => {
findUserDevices(user.userUID, message);
})
})
}
function findUserDevices(uid: string, message) {
collectionData(fb.firestore().collection('devices').where('userId', '==', uid)).subscribe((devices: any) => {
var userDeviceTokens: string[] = devices.map((device: any) => device.token);
if (userDeviceTokens.length != 0) {
message['tokens'] = userDeviceTokens;
pushToDevices(message);
}
})
}
async function pushToDevices(message) {
await admin.messaging().sendMulticast(message).then((response) => {
console.log('done!')
})
}
Thanks for any insight!

return a promise that resolves to a boolean true or false if a value is found

I need to make an api call that returns me a response of id's and values associated with it. The promise then resolves and returns a true or false if the id queried was found or not found.
How can i achieve this? New to using promises. As is promises seem confusing
here is the end point on consuming the API, UserService returns an array of id's and salaries. I need to check if the id exists and the salary matches to the query and then the promise needs to resolve to true or false.
here is the object of id's and incomes
[{
"id": 1,
"name": "Primary",
"sal": [{
"id": "1",
"sal": "10"
}, {
"id": "2",
"sal": "20"
}]
},{
"id": 2,
"name": "Secondary",
"sal": [{
"id": "1",
"sal": "100"
}, {
"id": "2",
"sal": "200"
}
]
}];
UserService.hasUserValue(id, income).then(function(qualifiesforlowIncome){
var isLowIncome = qualifiesforlowIncome
}
qualifiesforlowIncome is a boolean that returns a true or false. I am using angular so in this case should i do a $q.defer and return a defer.promise ?
A little unclear on this

Sure, all you need to do is add a then case which determines if any of the objects match the query then return true or false from it. That value will get carried into the next then case.
In this example, I'm using the some method to see if any of the objects in the array match the condition.
Promise.resolve([
{
id: 1
},
{
id: 2
},
{
id: 3
}
])
.then(results =>
results.some(r => r.id === 2)
)
.then(foundResult => console.log(foundResult));
Or rewritten in ES5:
Promise.resolve([
{
id: 1
},
{
id: 2
},
{
id: 3
}
])
.then(function(results) {
return results.some(r => r.id === 2);
})
.then(function(foundResult) {
console.log(foundResult);
});
This works exactly the same even if you return a Promise which later resolves.
Promise.resolve([
{
id: 1
},
{
id: 2
},
{
id: 3
}
])
.then(results =>
// Return a promise, delay sending back the result
new Promise((resolve, reject) => {
setTimeout(() =>
resolve(results.some(r => r.id === 2)),
500
);
})
)
.then(foundResult => console.log(foundResult));

How do you make your api calls? Let's assume you use fetch.
const UserService = {
getUsersWithIncome(income) {
return fetch("your.api/users?income="+income).then(resp => resp.json());
}
hasUserValue(id, income) {
return UserService.getUsersWithIncome(income).then(users => {
return users.some(u => u.id == id);
});
}
};
But by the way, the logic seems to be reversed here. So you have an endpoint that lists low income users and then look for a specific ID? Why dont you haven a rest endpoint that tells directly if a user with a given id has low income?

Edit: Don't do this! See Bergi's comment below for the reason why.
You'll want to use $q to return a deferred promise from hasUserValue(), make your API request within that method, and then resolve the promise after the API request finishes and you have a chance to crunch the numbers:
hasUserValue(id, income) {
var dfd = $q.defer();
SomeService.getSalaries().then(function(resp) {
// Do some logic here with the response
var qualifiesforlowIncome = someFuncThatDeterminesEligibility(resp);
// This is the value that will be returned for `qualifiesforlowIncome` in your then() callback
dfd.resolve(qualifiesforlowIncome);
});
return dfd.promise;
}
This is basically pseudo-code, the important bit is creating a deferred promise with AngularJS's $q and then returning the promise from the function as dfd.promise. The dfd object will have resolve() and reject() methods that can be used to pass back custom values in then() or catch() callbacks that you register when you call the method.

RxJS Observable fire onCompleted after a number of async actions

I'm trying to create an observable that produces values from a number of asynchronous actions (http requests from a Jenkins server), that will let a subscriber know once all the actions are completed. I feel like I must be misunderstanding something because this fails to do what I expect.
'use strict';
let Rx = require('rx');
let _ = require('lodash');
let values = [
{'id': 1, 'status': true},
{'id': 2, 'status': true},
{'id': 3, 'status': true}
];
function valuesObservable() {
return Rx.Observable.create(function(observer) {
_.map(values, function(value) {
var millisecondsToWait = 1000;
setTimeout(function() { // just using setTimeout here to construct the example
console.log("Sending value: ", value);
observer.onNext(value)
}, millisecondsToWait);
});
console.log("valuesObservable Sending onCompleted");
observer.onCompleted()
});
}
let observer = Rx.Observer.create((data) => {
console.log("Received Data: ", data);
// do something with the info
}, (error) => {
console.log("Error: ", error);
}, () => {
console.log("DONE!");
// do something else once done
});
valuesObservable().subscribe(observer);
Running this, I get output:
valuesObservable Sending onCompleted
DONE!
Sending value: { id: 1, status: true }
Sending value: { id: 2, status: true }
Sending value: { id: 3, status: true }
While what I would like to see is something more like:
Sending value: { id: 1, status: true }
Received Data: { id: 1, status: true }
Sending value: { id: 2, status: true }
Received Data: { id: 2, status: true }
Sending value: { id: 3, status: true }
Received Data: { id: 3, status: true }
valuesObservable Sending onCompleted
DONE!
I don't actually care about the order of the items in the list, I would just like the observer to receive them.
I believe what is happening is that Javascript asynchronously fires the timeout function, and proceeds immediately to the observer.onCompleted() line. Once the subscribing observer receives the onCompleted event (is that the right word?), it decides that it's done and disposes of itself. Then when the async actions complete and the observable fires onNext, the observer no longer exists to take any actions with them.
If I'm right about this, I'm still stumped about how to make it behave in the way I would like. Have I stumbled into an antipattern without realising it? Is there a better way of approaching this whole thing?
Edit:
Since I used setTimeout to construct my example, I realised I can use it to partially solve my problem by giving the observable a timeout.
function valuesObservable() {
return Rx.Observable.create(function(observer) {
let observableTimeout = 10000;
setTimeout(function() {
console.log("valuesObservable Sending onCompleted");
observer.onCompleted();
}, observableTimeout);
_.map(values, function(value) {
let millisecondsToWait = 1000;
setTimeout(function() {
console.log("Sending value: ", value);
observer.onNext(value)
}, millisecondsToWait);
});
});
}
This gets me all of the information from the observable in the order I want (data, then completion) but depending on the choice of timeout, I either may miss some data, or have to wait a long time for the completion event. Is this just a inherent problem of asynchronous programming that I have to live with?

Yes there is a better way. The problem right now is that you are relying on time delays for your synchronization when in fact you can use the Observable operators to do so instead.
The first step is to move away from directly using setTimeout. Instead use timer
Rx.Observable.timer(waitTime);
Next you can lift the values array into an Observable such that each value is emitted as an event by doing:
Rx.Observable.from(values);
And finally you would use flatMap to convert those values into Observables and flatten them into the final sequence. The result being an Observable that emits each time one of the source timers emits, and completes when all the source Observables complete.
Rx.Observable.from(values)
.flatMap(
// Map the value into a stream
value => Rx.Observable.timer(waitTime),
// This function maps the value returned from the timer Observable
// back into the original value you wanted to emit
value => value
)
Thus the complete valuesObservable function would look like:
function valuesObservable(values) {
return Rx.Observable.from(values)
.flatMap(
value => Rx.Observable.timer(waitTime),
value => value
)
.do(
x => console.log(`Sending value: ${value}`),
null,
() => console.log('Sending values completed')
);
}
Note the above would work as well if you weren't using demo stream, i.e. if you had really http streams you could even simplify by using merge (or concat to preserve order)
Rx.Observable.from(streams)
.flatMap(stream => stream);
// OR
Rx.Observable.from(streams).merge();
// Or simply
Rx.Observable.mergeAll(streams);

The best way to construct an observable is to use the existing primitive and then a combination of the existing operators. This avoids a few headaches (unsubscription, error management etc.). Then Rx.Observable.create is certainly useful when nothing else fits your use case. I wonder if generateWithAbsoluteTime would fit.
Anyways, here the issue you run into is that you complete your observer before you send him data. So basically you need to come up with a better completion signal. Maybe :
complete x seconds after last value emitted if no new value is emitted
complete when a value is equal to some 'end' value

With thanks to #paulpdaniels, this is the final code that did what I wanted, including the calls to Jenkins:
'use strict';
let Rx = require('rx');
let jenkinsapi = require('jenkins'); // https://github.com/silas/node-jenkins/issues
let jenkinsOpts = {
"baseUrl": "http://localhost:8080",
"options": {"strictSSL": false},
"job": "my-jenkins-job",
"username": "jenkins",
"apiToken": "f4abcdef012345678917a"
};
let jenkins = jenkinsapi(JSON.parse(JSON.stringify(jenkinsOpts)));
function jobInfoObservable(jenkins, jobName) {
// returns an observable with a containing a single list of builds for a given job
let selector = {tree: 'builds[number,url]'};
return Rx.Observable.fromNodeCallback(function(callback) {
jenkins.job.get(jobName, selector, callback);
})();
}
function buildIDObservable(jenkins, jobName) {
// returns an observable containing a stream of individual build IDs for a given job
return jobInfoObservable(jenkins, jobName).flatMap(function(jobInfo) {
return Rx.Observable.from(jobInfo.builds)
});
}
function buildInfoObservable(jenkins, jobName) {
// returns an observable containing a stream of http response for each build in the history for this job
let buildIDStream = buildIDObservable(jenkins, jobName);
let selector = {'tree': 'actions[parameters[name,value]],building,description,displayName,duration,estimatedDuration,executor,id,number,result,timestamp,url'};
return buildIDStream.flatMap(function(buildID) {
return Rx.Observable.fromNodeCallback(function(callback) {
jenkins.build.get(jobName, buildID.number, selector, callback);
})();
});
}
let observer = Rx.Observer.create((data) => {
console.log("Received Data: ", data);
// do something with the info
}, (error) => {
console.log("Error: ", error);
}, () => {
console.log("DONE!");
// do something else once done
});
buildInfoObservable(jenkins, jenkinsOpts.job).subscribe(observer);
By relying on the Rx built-in operators I managed to avoid messing about with timing logic altogether. This is also much cleaner than nesting multiple Rx.Observable.create statements.