I´m quite unsure on how to handle multiple updates / inserts in knex and return whatever it was successfull on the end or not.
I´m passing an array through req.body loop through it and trigger actions based on informations inside the array.
Example:
const data = [...req.body]
for(let i = 0; i < data.length; i++) {
data[i].totals.length
for(let y = 0; y < data[i].totals.length; y++) {
if(data[i].totals[y].info === "Holiday") {
calcHoliday(data[i].totals[y].total, data[i].id)
} else if(data[i].totals[y].info === "ZA") {
calcZA(data[i].totals[y].total, data[i].id)
}
}
calcOvertime(data[i].totalSum, data[i].id)
if(i === data.length -1) {
res.json("Success")
}
}
The Array I´m passing in looks like this:
[
{
"id": 1,
"totals": [
{
"info": "Holiday",
"total": 4
}
]
},
{
"id": 1,
"totals": [
{
"info": "Holiday",
"total": 4
}
]
}
]
Function Example which gets called in for loop:
const calcHoliday = (hours, userid) => {
knex.transaction(trx => {
trx.insert({
created_at: convertedTime,
info: "Booking Holiday - Hours: " + hours,
statuscode: 200
}).into("logs")
.then(() => {
return trx("hours")
.decrement("holiday_hours", hours)
}).then(trx.commit)
.catch(trx.rollback)
}).then(() => console.log("WORKED"))
.catch(err => console.log(err))
}
This is working perfectly fine but I can´t figure out how to gather the results from each table update in order to respond if everything worked or an error appeared. If I call e.g. after one calcHoliday call .then(resp => res.json(resp) I receive only the response from the first operation.
In short I need a way on how to res.json if everything succeeded or an error appeared somewhere.
Thanks in advance!
TLDR;
Turning your insert calls into an array of promises and then using await and a Promise.all() / Promise.allSettled() structure might solve this problem, but there are some UX decisions to make on what to rollback and how to return errors.
Error Handling Choices:
Any error --> all insertions in all loop iterations should be rolled back
Do you want partial success? The way the code is written now, rollback only applies to items in one function call. If one of the hour-decrement calls fails, it will roll back one log insert, but not any that succeeded for previous data in the loop. If you want the whole dataset to rollback, you'd need to pass the txn through each function call or do a bulk insert of all of your rows in one function call, which might be nice for performance reasons anyway depending on the use case.
Partial success --> commits successes, rolls back single loop iterations that fail, sends detailed list of errors and successes
You'd want to use Promise.allSettled(), which aggregates the successes and errors as an array from all promises in the loop.
Partial success --> commits the successes, rolls back single loop iterations that fail, sends just one error
Opinion: This can be a misleading UX unless the error is "some of the insertions were unsuccessful" and the endpoint is idempotent
This looks closest to what you're describing you want. If this is the case, you'd want to use Promise.all(), which throws an error as soon as one promise in the array errors.
Example Implementation:
Since the original code is incomplete, this is a loose, incomplete example of what option 2/3 might look like. This could easily be transformed into option 1.
First, it might help to modify all of your functions with asynchronous calls to be fulfillable as promises. Async/await helps avoid .then() trees that are hard to reason about.
const calcHoliday = async (hours, userid) => {
try {
const result = await knex.transaction(async(trx) => {
await trx.insert({
created_at: convertedTime,
info: "Booking Holiday - Hours: " + hours,
statuscode: 200
}).into("logs")
return trx("hours").decrement("holiday_hours", hours)
}
return result
} catch(err) {
console.log("It didn't work.")
throw new Error(`Error: Failure to insert for user ${userid}:`, err)
}
}
Here are some utilities to get the data transformed, and to get the appropriate unfulfilled promise to supply to the map in Promise.all/allSettled.
/*
Here's an example of how you might transform the original data with maps in order to avoid nested for-loops:
[
{ id: 1, info: 'Holiday', total: 4 },
{ id: 1, info: 'Holiday', total: 4 }
]
*/
const flatData = data.map(item => {
return item.totals.map(total => ({
id: item.id,
...total
}))
}).flat()
// Returns the appropriate promise based on data
const getTotalsPromises = (row) => {
const {info, id, total} = row
if(info === "Holiday") {
return calcHoliday(total, id)
} else if(info === "ZA") {
return calcZA(total, id)
}
}
const getcalcOvertimePromises = (rowInData) = {
// work left to reader
return calcOvertime(rowInData.correctData, rowInData.otherData)
}
If you want option 2:
// Replaces the loop
// Fulfills *all* the promises, creating an array of errors and successes
const responses = await Promise.allSettled([
...flatData.map(getTotalsPromises),
...data.map(getCalcOvertimePromises)
])
// insert loop here to do something with errors if you want
res.send(responses)
OR Option 3
Create an array of all of the promises you want to run, run them, and process up to one error.
// Replaces the loop
// Runs the promises and waits for them all to finish or the first error.
try {
const responses = await Promise.all([
...flatData.map(getTotalsPromises),
...data.map(getCalcOvertimePromises)
])
res.send(responses)
} catch(err){
// Reached if one of the rows errors
res.send(err)
}
Docs:
Promise.allSettled
Promise.all
Related
Via a microservice, I retrieve several packages of JSON data and spit them out onto a Vue.js-driven page. The data looks something like this:
{"data":{"getcompanies":
[
{"id":6,"name":"Arena","address":"12 Baker Street","zip":"15090"},
{"id":7,"name":"McMillan","address":null,"zip":"15090"},
{"id":8,"name":"Ball","address":"342 Farm Road","zip":"15090"}
]
}}
{"data":{"getusers":
[{"id":22,"name":"Fred","address":"Parmesean Street","zip":"15090"},
{"id":24,"name":"George","address":"Loopy Lane","zip":"15090"},
{"id":25,"name":"Lucy","address":"Farm Road","zip":"15090"}]}}
{"data":{"getdevices":
[{"id":2,"name":"device type 1"},
{"id":4,"name":"device type 2"},
{"id":5,"name":"device type 3"}]}}
...and I successfully grab them individually via code like this:
getCompanies() {
this.sendMicroServiceRequest({
method: 'GET',
url: `api/authenticated/function/getcompanies`
})
.then((response) => {
if(response.data) {
this.dataCompanies = response.data.getcompanies
} else {
console.error(response)
}
}).catch(console.error)
}
...with getUsers() and getDevices() looking respectively the same. getCompanies() returns:
[{"id":6,"name":"Arena","address":"12 Baker Street","zip":"15090"},
{"id":7,"name":"McMillan","address":null,"zip":"15090"},
{"id":8,"name":"Ball","address":"342 Farm Road","zip":"15090"}]
...which I relay to the Vue template in a table, and this works just fine and dandy.
But this is obviously going to get unwieldy if I need to add more microservice calls down the road.
What I'm looking for is an elegant way to jump past the response.data.*whatever* and get to those id-records with a re-useable call, but I'm having trouble getting there. response.data[0] doesn't work, and mapping down to the stuff I need either comes back undefined, or in bits of array. And filtering for response.data[0].id to return just the rows with ids keeps coming back undefined.
My last attempt (see below) to access the data does work, but looks like it comes back as individual array elements. I'd rather not - if possible - rebuild an array into a JSON structure. I keep thinking I should be able to just step past the next level regardless of what it's called, and grab whatever is there in one chunk, as if I read response.data.getcompanies directly, but not caring what 'getcompanies' is, or needing to reference it by name:
// the call
this.dataCompanies = this.getFullData('companies')
getFullData(who) {
this.sendMicroServiceRequest({
method: 'GET',
url: 'api/authenticated/function/get' + who,
})
.then((response) => {
if(response) {
// attempt 1 to get chunk below 'getcompanies'
Object.keys(response.data).forEach(function(prop) {
console.log(response.data[prop])
})
// attempt 2
// for (const prop in response.data) {
// console.log(response.data[prop])
// }
let output = response.data[prop] // erroneously thinking this is in one object
return output
} else {
console.error(response)
}
}).catch(console.error)
}
...outputs:
(63) [{…}, {…}, {…}] <-- *there are 63 of these records, I'm just showing the first few...*
0: {"id":6,"name":"Arena","address":"12 Baker Street","zip":"15090"}
1: {"id":7,"name":"McMillan","address":null,"zip":"15090"},
2: {"id":8,"name":"Ball","address":"342 Farm Road","zip":"15090"}...
Oh, and the return above comes back 'undefined' for some reason that eludes me at 3AM. >.<
It's one of those things where I think I am close, but not quite. Any tips, hints, or pokes in the right direction are greatly appreciated.
I feel it's better to be explicit about accessing the object. Seems like the object key is consistent with the name of the microservice function? If so:
getData(functionName) {
return this.sendMicroServiceRequest({
method: 'GET',
url: "api/authenticated/function/" + functionName
})
.then( response => response.data[functionName] )
}
getCompanies(){
this.getData("getcompanies").then(companies => {
this.dataCompanies = companies
})
}
let arrResponse = {data: ['x']};
let objResponse = {data: {getcompanies: 'x'}};
console.log(arrResponse.data[0]);
console.log(Object.values(objResponse.data)[0]);
response.data[0] would work if data was an array. To get the first-and-only element of an object, use Object.values(response.data)[0] instead. Object.values converts an object to an array of its values.
Its counterparts Object.keys and Object.entries likewise return arrays of keys and key-value tuples respectively.
Note, order isn't guaranteed in objects, so this is only predictable in your situation because data has exactly a single key & value. Otherwise, you'd have to iterate the entry tuples and search for the desired entry.
firstValue
Let's begin with a generic function, firstValue. It will get the first value of an object, if present, otherwise it will throw an error -
const x = { something: "foo" }
const y = {}
const firstValue = t =>
{ const v = Object.values(t)
if (v.length)
return v[0]
else
throw Error("empty data")
}
console.log(firstValue(x)) // "foo"
console.log(firstValue(y)) // Error: empty data
getData
Now write a generic getData. We chain our firstValue function on the end, and be careful not to add a console.log or .catch here; that is a choice for the caller to decide -
getData(url) {
return this
.sendMicroServiceRequest({ method: "GET", url })
.then(response => {
if (response.data)
return response.data
else
return Promise.reject(response)
})
.then(firstValue)
}
Now we write getCompanies, getUsers, etc -
getCompanies() {
return getData("api/authenticated/function/getcompanies")
}
getUsers() {
return getData("api/authenticated/function/getusers")
}
//...
async and await
Maybe you could spruce up getData with async and await -
async getData(url) {
const response =
await this.sendMicroServiceRequest({ method: "GET", url })
return response.data
? firstValue(response.data)
: Promise.reject(response)
}
power of generics demonstrated
We might even suggest that these get* functions are no longer needed -
async getAll() {
return {
companies:
await getData("api/authenticated/function/getcompanies"),
users:
await getData("api/authenticated/function/getusers"),
devices:
await getData("api/authenticated/function/getdevices"),
// ...
}
}
Above we used three await getData(...) requests which happen in serial order. Perhaps you want all of these requests to run in parallel. Below we will show how to do that -
async getAll() {
const requests = [
getData("api/authenticated/function/getcompanies"),
getData("api/authenticated/function/getusers"),
getData("api/authenticated/function/getdevices")
]
const [companies, users, devices] = Promise.all(requests)
return { companies, users, devices }
}
error handling
Finally, error handling is reserved for the caller and should not be attempted within our generic functions -
this.getAll()
.then(data => this.render(data)) // some Vue template
.catch(console.error)
question is possibly a duplicate but I haven't found anything that provides an appropriate answer to my issue.
I have an ExpressJS server which is used to provide API requests to retrieve data from a MongoDB database. I am using mongoosejs for the MongoDB connection to query/save data.
I am building a route that will allow me to find all data that matches some user input but I am having trouble when doing the query. I have spent a long while looking online for someone with a similar issue but coming up blank.
I will leave example of the code I have at the minute below.
code for route
// -- return matched data (GET)
router.get('/match', async (req, res) => {
const style_data = req.query.style; // grab url param for style scores ** this comes in as a string **
const character_data = req.query.character; // grab url param for character scores ** this comes in as a string **
// run matcher systems
const style_matches = style_match(style_data);
res.send({
response: 200,
data: style_matches
}); // return data
});
code for the query
// ---(Build the finder)
const fetch_matches_using = async function(body, richness, smoke, sweetness) {
return await WhiskyModel.find({
'attributes.body': body,
'attributes.richness': richness,
'attributes.smoke': smoke,
'attributes.sweetness': sweetness
});
}
// ---(Start match function)---
const style_match = async function (scores_as_string) {
// ---(extract data)---
const body = scores_as_string[0];
const richness = scores_as_string[1];
const smoke = scores_as_string[2];
const sweetness = scores_as_string[3];
const matched = [];
// ---(initialise variables)---
let match_count = matched.length;
let first_run; // -> exact matches
let second_run; // -> +- 1
let third_run; // -> +- 2
let fourth_run; // -> +- 3
// ---(begin db find loop)---
first_run = fetch_matches_using(body, richness, smoke, sweetness).then((result) => {return result});
matched.push(first_run);
// ---(return final data)---
return matched
}
example of db object
{
_id: mongoid,
meta-data: {
pagemd:{some data},
name: whiskyname
age: whiskyage,
price: price
},
attributes: {
body: "3",
richness: "3",
smoke: "0",
sweetness: "3",
some other data ...
}
}
When I hit the route in postman the JSON data looks like:
{
response: 200,
data: {}
}
and when I console.log() out matched from within the style match function after I have pushed the it prints [ Promise(pending) ] which I don't understand.
if I console.log() the result from within the .then() I get an empty array.
I have tried using the populate() method after running the find which does technically work, but instead of only returning data that matches it returns every entry in the collection so I think I am doing something wrong there, but I also don't see why I would need to use the .populate() function to access the nested object.
Am I doing something totally wrong here?
I should also mention that the route and the matching functions are in different files just to try and keep things simple.
Thanks for any answers.
just posting an answer as I seem to have fixed this.
Issue was with my .find() function, needed to pass in the items to search by and then also a call back within the function to return error/data. I'll leave the changed code below.
new function
const fetch_matches_using = async function(body, richness, smoke, sweetness) {
const data = await WhiskyModel.find({
'attributes.body': body,
'attributes.richness': richness,
'attributes.smoke': smoke,
'attributes.sweetness': sweetness
}, (error, data) => { // new ¬
if (error) {
return error;
}
if (data) {
console.log(data)
return data
}
});
return data; //new
}
There is still an issue with sending the found results back to the route but this is a different issue I believe. If its connected I'll edit this answer with the fix for that.
I'm developing a cloud based billing system and I have two tables in my database namely bill_history and sold_items. I want to store the Bill number, date, customer name, phone number and total amount and then to return the bill number from bill_history and store the array of objects containing item no, item name, price, quantity, amount with the returned bill no in sold_items. I'm using the following code:
app.post('/billed', (req, res) => {
const { items, total, date } = req.body;
console.log(items, total, date);
db.transaction(trx => {
db.insert({
total: total,
date: date,
}).into('billhead')
.transacting(trx)
.returning('billno')
.then(num => {
for (var i = 0; i < items.length; i++) {
trx.insert({
billno: num,
prodname: items[i].name,
quantity: items[i].quantity,
netprice: items[i].amount
}).into('billdetails')
}).then(trx.commit())
.catch(trx.rollback())
})
})
Now Entries are found in bill_history but not entered in sold_items. I can't find the mistake! Help me with this error. The console and terminal shows No Error
Important thing to remember when working with knex queries: they are promises and they will only execute if:
You call then on the knex object itself
You return the knex query inside a promise chain, and call then somewhere down the chain
Inside your for loop, you have only stated what the knex object should do and because of syntax errors didn't call then on the knex object itself.
.into('billdetails').then(inserts => { /// })
It does work if you return trx.insert()...
That being said, it wouldn't suit your use case, as when inserting multiple values inside a transaction, you need to make sure all inserts have been succesfull. Using for loops the way you did in async fashion is dangerous and won't guarantee all individual inserts have completed without errors and that it's safe to commit the transaction.
One way of achieving this in a safe manner would be modifying this section of your code:
// ...
.returning('billno')
.then(num => {
// We create an array of individual inserts
// Each element in the array will be a single knex
// object/promise that inserts one row into the database
const billDetailInserts = items.map(item => trx.insert({
billno: num,
prodname: item.name,
quantity: item.quantity,
netprice: item.amount
).into('billdetails')
})
// we utilize the Promise.all method that will resolve when
// all individual inserts have completed succesfully
return Promise.all(billDetailInserts);
})
.then(inserts => {
// ... commits, rollbacks, logging etc
I'm calling an API and receiving an array of results, I'm checking for pagination and if more pages exist I call the next page, repeat until no more pages.
For each array of results, I call another endpoint and do the exact same thing: I receive an array of results, check for another page and call endpoint again. Wash, rinse repeat.
For instance:
I want to grab a list of countries that might be a paginated response, then for each country I want to grab a list of cities, which might also be paginated. And for each city I execute a set of transformations and then store in a database.
I already tried this, but got stuck:
const grabCountries = Observable.create(async (observer) => {
const url = 'http://api.com/countries'
let cursor = url
do {
const results = fetch(cursor)
// results = {
// data: [ 'Canada', 'France', 'Spain' ],
// next: '47asd8f76358df8f4058898fd8fab'
// }
results.data.forEach(country => { observer.next(country) })
cursor = results.next ? `${url}/${results.next}` : undefined
} while(cursor)
})
const getCities = {
next: (country) => {
const url = 'http://api.com/cities'
let cursor = url
do {
const results = fetch(cursor)
// results = {
// data: [
// 'Montreal', 'Toronto',
// 'Paris', 'Marseilles',
// 'Barcelona', 'Madrid'
// ],
// next: '89ghjg98nd8g8sdfg98gs9h868hfoig'
// }
results.data.forEach(city => {
`**** What do I do here?? ****`
})
cursor = results.next ? `${url}/${results.next}` : undefined
} while(cursor)
}
}
I tried a few approaches:
Making a subject (sometimes I'll need to do parallel processed base on the results of 'grabCountries'. For example I may want to store the countries in a DB in parallel with grabbing the Cities.)
const intermediateSubject = new Subject()
intermediateSubject.subscribe(storeCountriesInDatabase)
intermediateSubject.subscribe(getCities)
I also tried piping and mapping, but it seems like it's basically the same thing.
As I was writing this I thought of this solution and it seems to be working fine, I would just like to know if I'm making this too complicated. There might be cases where I need to make more that just a few API calls in a row. (Imagine, Countries => States => Cities => Bakeries => Reviews => Comments => Replies) So this weird mapping over another observer callback pattern might get nasty.
So this is what I have now basically:
// grabCountries stays the same as above, but the rest is as follows:
const grabCities = (country) =>
Observable.create(async (observer) => {
const url = `http://api.com/${country}/cities`
let cursor = url
do {
const results = fetch(cursor)
// results = {
// data: [
// 'Montreal', 'Toronto',
// 'Paris', 'Marseilles',
// 'Barcelona', 'Madrid'
// ],
// next: '89ghjg98nd8g8sdfg98gs9h868hfoig'
// }
results.data.forEach(city => {
observer.next(city)
})
cursor = results.next ? `${url}/${results.next}` : undefined
} while (cursor)
})
const multiCaster = new Subject()
grabCountries.subscribe(multiCaster)
multiCaster.pipe(map((country) => {
grabCities(country).pipe(map(saveCityToDB)).subscribe()
})).subscribe()
multiCaster.pipe(map(saveCountryToDB)).subscribe()
tl;dr - I call an API that receives a paginated set of results in an array and I need to map through each item and call another api that receives another paginated set of results, each set also in an array.
Is nesting one observable inside another and mapping through the results via 'callApiForCountries.pipe(map(forEachCountryCallApiForCities))' the best method or do you have any other recommendations?
Here's the code that should work with sequential crawling of next url.
You start with a {next:url} until res.next is not available.
of({next:http://api.com/cities}).pipe(
expand(res=>results.next ? `${url}/${results.next}` : undefined
takeWhile(res=>res.next!==undefined)
).subscribe()
OK, so I have spent a lot of brain power on this and have come up with two solutions that seem to be working.
const nestedFlow = () => {
fetchAccountIDs.pipe(map(accountIds => {
getAccountPostIDs(accountIds) // Has the do loop for paging inside
.pipe(
map(fetchPostDetails),
map(mapToDBFormat),
map(storeInDB)
).subscribe()
})).subscribe()
}
const expandedflow = () => {
fetchAccountIDs.subscribe((accountId) => {
// accountId { accountId: '345367geg55sy'}
getAccountPostIDs(accountId).pipe(
expand((results) => {
/*
results : {
postIDs: [
131424234,
247345345,
],
cursor: '374fg8v0ggfgt94',
}
*/
const { postIDs, cursor } = results
if (cursor) return getAccountPostIDs({...accountId, cursor})
return { postIDs, cursor }
}),
takeWhile(hasCursor, true), // recurs until cursor is undefined
concatMap(data => data.postIDs),
map(data => ({ post_id: data })),
map(fetchPostDetails),
map(mapToDBFormat),
map(storeInDB)
).subscribe()
})
}
Both seem to be working with similar performance. I read some where that leaving the data flow is a bad practice and you should pipe everything, but I don't know how to eliminate the first exit in the 'expandedFlow' because the 'expand' needs to call back an observable, but maybe it can be done.
Now I just have to solve the race condition issues from the time the 'complete' is called in getAccountPostIDs the the last record is stored in the DB. Currently in my test, the observer.complete is finishing before 3 of the upsert actions.
Any comments are appreciated and I hope this helps someone out in the future.
What you need is the expand operator. It behaves recursively so it fits the idea of having paginated results.
I'm trying to create an observable that produces values from a number of asynchronous actions (http requests from a Jenkins server), that will let a subscriber know once all the actions are completed. I feel like I must be misunderstanding something because this fails to do what I expect.
'use strict';
let Rx = require('rx');
let _ = require('lodash');
let values = [
{'id': 1, 'status': true},
{'id': 2, 'status': true},
{'id': 3, 'status': true}
];
function valuesObservable() {
return Rx.Observable.create(function(observer) {
_.map(values, function(value) {
var millisecondsToWait = 1000;
setTimeout(function() { // just using setTimeout here to construct the example
console.log("Sending value: ", value);
observer.onNext(value)
}, millisecondsToWait);
});
console.log("valuesObservable Sending onCompleted");
observer.onCompleted()
});
}
let observer = Rx.Observer.create((data) => {
console.log("Received Data: ", data);
// do something with the info
}, (error) => {
console.log("Error: ", error);
}, () => {
console.log("DONE!");
// do something else once done
});
valuesObservable().subscribe(observer);
Running this, I get output:
valuesObservable Sending onCompleted
DONE!
Sending value: { id: 1, status: true }
Sending value: { id: 2, status: true }
Sending value: { id: 3, status: true }
While what I would like to see is something more like:
Sending value: { id: 1, status: true }
Received Data: { id: 1, status: true }
Sending value: { id: 2, status: true }
Received Data: { id: 2, status: true }
Sending value: { id: 3, status: true }
Received Data: { id: 3, status: true }
valuesObservable Sending onCompleted
DONE!
I don't actually care about the order of the items in the list, I would just like the observer to receive them.
I believe what is happening is that Javascript asynchronously fires the timeout function, and proceeds immediately to the observer.onCompleted() line. Once the subscribing observer receives the onCompleted event (is that the right word?), it decides that it's done and disposes of itself. Then when the async actions complete and the observable fires onNext, the observer no longer exists to take any actions with them.
If I'm right about this, I'm still stumped about how to make it behave in the way I would like. Have I stumbled into an antipattern without realising it? Is there a better way of approaching this whole thing?
Edit:
Since I used setTimeout to construct my example, I realised I can use it to partially solve my problem by giving the observable a timeout.
function valuesObservable() {
return Rx.Observable.create(function(observer) {
let observableTimeout = 10000;
setTimeout(function() {
console.log("valuesObservable Sending onCompleted");
observer.onCompleted();
}, observableTimeout);
_.map(values, function(value) {
let millisecondsToWait = 1000;
setTimeout(function() {
console.log("Sending value: ", value);
observer.onNext(value)
}, millisecondsToWait);
});
});
}
This gets me all of the information from the observable in the order I want (data, then completion) but depending on the choice of timeout, I either may miss some data, or have to wait a long time for the completion event. Is this just a inherent problem of asynchronous programming that I have to live with?
Yes there is a better way. The problem right now is that you are relying on time delays for your synchronization when in fact you can use the Observable operators to do so instead.
The first step is to move away from directly using setTimeout. Instead use timer
Rx.Observable.timer(waitTime);
Next you can lift the values array into an Observable such that each value is emitted as an event by doing:
Rx.Observable.from(values);
And finally you would use flatMap to convert those values into Observables and flatten them into the final sequence. The result being an Observable that emits each time one of the source timers emits, and completes when all the source Observables complete.
Rx.Observable.from(values)
.flatMap(
// Map the value into a stream
value => Rx.Observable.timer(waitTime),
// This function maps the value returned from the timer Observable
// back into the original value you wanted to emit
value => value
)
Thus the complete valuesObservable function would look like:
function valuesObservable(values) {
return Rx.Observable.from(values)
.flatMap(
value => Rx.Observable.timer(waitTime),
value => value
)
.do(
x => console.log(`Sending value: ${value}`),
null,
() => console.log('Sending values completed')
);
}
Note the above would work as well if you weren't using demo stream, i.e. if you had really http streams you could even simplify by using merge (or concat to preserve order)
Rx.Observable.from(streams)
.flatMap(stream => stream);
// OR
Rx.Observable.from(streams).merge();
// Or simply
Rx.Observable.mergeAll(streams);
The best way to construct an observable is to use the existing primitive and then a combination of the existing operators. This avoids a few headaches (unsubscription, error management etc.). Then Rx.Observable.create is certainly useful when nothing else fits your use case. I wonder if generateWithAbsoluteTime would fit.
Anyways, here the issue you run into is that you complete your observer before you send him data. So basically you need to come up with a better completion signal. Maybe :
complete x seconds after last value emitted if no new value is emitted
complete when a value is equal to some 'end' value
With thanks to #paulpdaniels, this is the final code that did what I wanted, including the calls to Jenkins:
'use strict';
let Rx = require('rx');
let jenkinsapi = require('jenkins'); // https://github.com/silas/node-jenkins/issues
let jenkinsOpts = {
"baseUrl": "http://localhost:8080",
"options": {"strictSSL": false},
"job": "my-jenkins-job",
"username": "jenkins",
"apiToken": "f4abcdef012345678917a"
};
let jenkins = jenkinsapi(JSON.parse(JSON.stringify(jenkinsOpts)));
function jobInfoObservable(jenkins, jobName) {
// returns an observable with a containing a single list of builds for a given job
let selector = {tree: 'builds[number,url]'};
return Rx.Observable.fromNodeCallback(function(callback) {
jenkins.job.get(jobName, selector, callback);
})();
}
function buildIDObservable(jenkins, jobName) {
// returns an observable containing a stream of individual build IDs for a given job
return jobInfoObservable(jenkins, jobName).flatMap(function(jobInfo) {
return Rx.Observable.from(jobInfo.builds)
});
}
function buildInfoObservable(jenkins, jobName) {
// returns an observable containing a stream of http response for each build in the history for this job
let buildIDStream = buildIDObservable(jenkins, jobName);
let selector = {'tree': 'actions[parameters[name,value]],building,description,displayName,duration,estimatedDuration,executor,id,number,result,timestamp,url'};
return buildIDStream.flatMap(function(buildID) {
return Rx.Observable.fromNodeCallback(function(callback) {
jenkins.build.get(jobName, buildID.number, selector, callback);
})();
});
}
let observer = Rx.Observer.create((data) => {
console.log("Received Data: ", data);
// do something with the info
}, (error) => {
console.log("Error: ", error);
}, () => {
console.log("DONE!");
// do something else once done
});
buildInfoObservable(jenkins, jenkinsOpts.job).subscribe(observer);
By relying on the Rx built-in operators I managed to avoid messing about with timing logic altogether. This is also much cleaner than nesting multiple Rx.Observable.create statements.