RxJs: Paginate through API recursively and find value from list - javascript

I am using rxjs v6.4.0. I am trying to paginate through an API searching for a very specific channel where name equals "development". I am using expand to recursively call the API and get new pages. The end result gives me a concatenated list of channels. Then I filter out all channels where name not equal to "development". However I am getting an error: TypeError: You provided 'undefined' where a stream was expected. You can provide an Observable, Promise, Array, or Iterable.
const Rx = require('rxjs')
const Rx2 = require('rxjs/operators')
const getChannel = (cursor) => {
return this.service.getData(`${url}?cursor=${cursor || ''}`)
.pipe(Rx2.map(resp => JSON.parse(resp.body)))
.pipe(Rx2.expand(body => { // recurse the api until no more cursors
return body.response_metadata &&
body.response_metadata.next_cursor ?
getChannel(body.response_metadata.next_cursor) : Rx.EMPTY
}))
.pipe(Rx2.pluck('channels'))
.pipe(Rx2.mergeAll()) // flattens array
.pipe(Rx2.filter(c => {
console.log('finding', c.name === 'development')
return c.name === 'development'
}))
}

The find callback should return a boolean, not an Observable. E.g.
find(c => c.name === 'development')
UPDATE
Heres a modified example of yours. I've removed generators as they are more complicated then our case needs.
const { of, EMPTY, throwError } = rxjs;
const { filter, tap, expand, pluck, mergeAll } = rxjs.operators;
const DATA =
[ {channels: [{id: 123, name: 'test'}, {id:4, name: 'hello'}], cursor: 1}
, {channels:[{id: 1, name: 'world'}, {id: 2, name: 'knows'}], cursor: 2}
, {channels:[{id: 3, name: 'react'}, {id: 5, name: 'devcap'}], cursor: false}
];
function getChannel(){
return getBlock()
.pipe(
expand(x => x.cursor ? getBlock(x.cursor) : EMPTY),
pluck('channels'),
mergeAll(),
filter(c => c.name === 'devcap')
)
}
getChannel().subscribe({
next: console.log,
error: console.error
});
function getBlock(index = 0) {
if (index >= DATA.length){
throwError('Out of bounds');
}
return of(DATA[index]);
}
UPDATE 2
Your solution didn't work due to recursion being done through solely getChannel(). When you do the expand -- you run another cycle through getChannel(). Meaning that you run pluck-mergeAll-filter chain twice on each recursively fetched value! Plucking and flattering it twice gives you undefined -- therefore the error.
In your playground -- try separating out this code
let getValue = ()=>{
const next = gen.next();
if (!next || !next.value) { return EMPTY; }
return next.value;
}
and use it in the expand, like this:
let getChannel = () => {
return getValue()
.pipe(
expand(body => {
return body.cursor ? getValue() : EMPTY
}),
pluck('channels'),
mergeAll(),
filter(c => c.name === 'devcap'),
)
}

Let me know if this is the functionality you are looking for https://codepen.io/jeremytru91/pen/wOQxbZ?editors=1111
const {
of,
EMPTY
} = rxjs;
const {
filter,
tap,
expand,
take,
pluck,
concatAll,
flatMap,
first
} = rxjs.operators;
function* apiResponses() {
yield of({channels: [{id: 123, name: 'test'}, {id:4, name: 'hello'}], cursor: 1});
yield of({channels:[{id: 3, name: 'react'}, {id: 5, name: 'devcap'}], cursor:3});
yield of({channels:[{id: 1, name: 'world'}, {id: 2, name: 'knows'}], cursor: 2});
yield of({channels:[{id: 4, name: 'react'}, {id: 6, name: 'devcap'}], cursor:4});
}
let gen = apiResponses();
function getChannel() {
return gen.next().value // simulate api call
.pipe(
expand(body => {
return body.cursor ? gen.next().value : EMPTY
}),
pluck('channels'),
flatMap(channels => {
const filtered = channels.filter(channel => channel.name === 'devcap')
if(filtered.length) {
return filtered;
}
return EMPTY;
}),
first()
);
}
getChannel().subscribe(data => {
console.log('DATA!! ', data)
}, e => {
console.log('ERROR', e)
throw e
})

Related

How to return false from traverse function as well as from inner traverse function to stop the traverse and also need unit test for this?

//a user will call the traverse function like this for example stop the traversal when item found
let foundItem: IGXA.TAnyItem | undefined;
traverse([...], item => {
if (item.ID === "test") {
foundItem = item;
return false
}
})
/**
* Calls the callback for every direct and deep child item in the IGXA item list.
*
*
* #param {TItems} items
* #param {(item: TAnyItem) => void} cb
*/
export const traverse = (
items: TItems,
cb: (item: TAnyItem) => void | false,
) => {
for (const item of items) {
cb(item);
if (
item.Type === EItemType.Folder ||
item.Type === EItemType.MultiArticle
) {
traverse(item.SubItems, cb);
}
}
};
//unit test for the above function items is the array we passed to traverse
describe("traverse", () => {
it("calls the callback for every item", () => {
const items: TItems = [
{
ID: "folder",
Type: EItemType.Folder,
SubItems: [
{
ID: "folder_1",
Type: EItemType.Folder,
SubItems: [
{
ID: "article",
Type: EItemType.ArticleGfx,
Caption: {},
},
],
Caption: {},
},
],
Caption: {},
},
{
ID: "multiarticle",
Type: EItemType.MultiArticle,
Caption: {},
MultiArticleId: "",
SubItems: [
{
ID: "article2",
Type: EItemType.ArticleGfx,
Caption: {},
},
],
},
{
ID: "article3",
Type: EItemType.ArticleGfx,
Caption: {},
},
];
const cb = jest.fn();
traverse(items, cb);
expect(cb).toHaveBeenCalledTimes(6);
expect(cb).toHaveBeenCalledWith(items[items.length - 1]);
});
});
Generators are a perfect fit for this -
function* traverse (t) {
switch (t?.constructor) {
case Array:
for (const v of t)
yield *traverse(v)
break
case Object:
yield t
yield *traverse(t?.SubItems)
break
}
}
Q: "How to return false from traverse function as well as from inner traverse function to stop the traverse?"
A: Generators are pauseable and cancellable so you can easily stop iteration -
function findById (graph, id) {
for (const node in traverse(graph))
if (node.ID === id)
return node // <- return stops iteration and stops the generator
}
console.log(findById(TItems, "folder_1"))
{
ID: "folder_1",
Type: EItemType.Folder,
SubItems: [
{
ID: "article",
Type: EItemType.ArticleGfx,
Caption: {},
},
],
Caption: {},
}
Q: "...and also need unit test for this?"
A: Make a simple graph as the test input -
const mygraph = [
{
id: 1,
SubItems: [
{ id: 2 },
{ id: 3 }
]
},
{ id: 4 },
{
id: 5,
SubItems: [
{
id: 6,
SubItems: [ { id: 7 } ]
},
{ id: 8 }
]
}
]
Then write some simple tests -
it("traverse should visit all of the nodes", _ => {
const ids = Array.from(traverse(mygraph), node => node.id)
assert.strictEqual(ids, [1,2,3,4,5,6,7,8])
})
it("findById should find a node", _ => {
const match = findById(mygraph, 6)
assert.strictEqual(match.id, 6)
assert.strictEqual(match.SubItems.length, 1)
})
it("findById should return undefined for unmatched id", _ => {
const match = findById(mygraph, 99)
assert.strictEqual(match, undefined)
})
Q: "thanks for u detailed answer but i need a callback version too? how to achieve it"
A: rewrite traverse and implement your own yield which passes a resume control to the caller -
function traverse (t, yield, resume = () => void 0) {
switch (t?.constructor) {
case Array:
if (empty(t))
return resume()
else
return traverse(head(t), yield, _ => traverse(tail(t), yield, resume))
case Object:
return yield(t, _ => traverse(t?.SubItems, yield) || resume())
}
}
This depends on a few helpers, empty, head and tail. These aren't necessary but keep the code in traverse a bit cleaner and easier to read -
const head = t => t[0]
const tail = t => t.slice(1)
const empty = t => t.length < 1
Here's findById. Traversal will only continue if resume is called -
function findById (graph, id) {
let found = undefined
traverse(graph, (node, resume) =>
node.id === id ? (found = node) : resume()
)
return found
}
console.log(findById(mygraph, 6))
{ id: 6, SubItems: [ { id: 7 } ] }
Another difference is callback-based traverse cannot plug directly into Array.from. To build an array of all visited nodes, we can push to an array in each callback and unconditionally resume -
function getIds (t) {
const r = []
traverse(t, (node, resume) =>
r.push(node.id) && resume()
)
return r
}
console.log(getIds(mygraph))
[1,2,3,4,5,6,7,8]
For this version, you may wish to implement another unit test that verifies traversal is lazy -
it("should stop traversal if resume is not called", _ => {
const input = [ { id: 1 }, { id: 2 } ]
const actual = []
traverse(input, (node, resume) => {
actual.push(node.id)
// do not resume()
})
assert.strictEqual(actual, [1], "should only contain the first item")
})
Expand the snippet below to verify the the program in your own browser -
const head = t => t[0]
const tail = t => t.slice(1)
const empty = t => t.length < 1
function traverse (t, yield, resume = () => void 0) {
switch (t?.constructor) {
case Array:
if (empty(t))
return resume()
else
return traverse(head(t), yield, _ => traverse(tail(t), yield, resume))
case Object:
return yield(t, _ => traverse(t?.SubItems, yield) || resume())
}
}
function getIds (graph) {
const r = []
traverse(graph, (node, resume) =>
r.push(node.id) && resume()
)
return r
}
function findById (graph, id) {
let found = undefined
traverse(graph, (node, resume) =>
node.id === id ? (found = node) : resume()
)
return found
}
const mygraph =
[{id:1,SubItems:[{id:2},{id:3}]},{id:4},{id:5,SubItems:[{id:6,SubItems:[{id:7}]},{id:8}]}]
console.log(JSON.stringify(getIds(mygraph)))
console.log(findById(mygraph, 6))

RxJs: getting which of the object keys changed in sequence filtered via distinctUntilChanged() without keeping external state

Let orders$ is RxJs sequence of objects as shown below. We listen for changes with distinctUntilChanged() and take different actions based on which key changed. Can we tell inside the tap() which object key actually changed, without keeping track of the previous change via the external previousItem variable ?
Example:
let orders$ = of([
{ main: 'pizza', drink: 'cola', id: 0},
{ main: 'pizza', drink: 'cola', id: 1},
...
{ main: 'pizza', drink: 'cola', id: 3332},
{ main: 'pizza', drink: 'fanta', id: 3333}, // <-- drink changes
]);
let previousItem = null;
orders$.pipe(
distinctUntilChanged((o0, o1) => o0.main === o1.main || o0.drink === o1.drink),
tap(item => {
/* Which value actually changed -'main' or 'drink' ?! */
if (previousItem && previousItem.main !== item.main) {
doSthForMainChanged();
}
doSthForDrinkChanged();
previousItem = item; // <-- ... so that not to tweak via external state
})).subscribe();
I could suggest to do it with scan:
source$.pipe(
scan((last: any, item: any) => {
let changes: any = {}
last.main !== item.main ? changes.main = true : false
last.drink !== item.drink ? changes.drink = true : false
return {item, changes}
}, {}),
tap(({item, changes}) => {
if (changes.main) {
doSthForMainChanged();
}
if (changes.drink) {
doSthForDrinkChanged();
}
})
)
and you can also add distinctUntilChanged at the end if needed
You can split the observable into two streams for each type of change, and then use map to track those changes.
const main$ = orders$.pipe(
distinctUntilChanged((o0, o1) => o0.main === o1.main),
map(value => ({changed:'main', value}))
);
const drink$ = orders$.pipe(
distinctUntilChanged((o0, o1) => o0.drink === o1.drink),
map(value => ({changed:'drink', value}))
);
merge(main$, drink$).pipe(
tap(item => {
if (item.changed === 'main') {
doSthForMainChanged();
} else if(item.changed === 'drink') {
doSthForDrinkChanged();
}
})).subscribe();
The only side effect of the above is that the tap will be executed twice if both main and drink change for a single emitted value. I don't know if this is a problem for your business logic or not.

mapping one object to another using some functional programming

I would like to map one array of object into another in a more functional style, I am using typescript.
Basically I am using delete to remove a property on a object, I would like to know if there is a better way to write it.
const data = props.data.map(d => ({
order: d.position,
logs: d.batches.map(b => {
let log= {
amount: b.scrap,
batchNumber: '', // NO GOOD
}
if (!b.batch || b.batch.length === 0) {
delete log.batchNumber // NO GOOD
}
return log
}),
}))
example input data:
const data = [
position: 1,
batches: [
{batchNumber: '', ammount: 3}
]
]
result:
const data = [{
order: 1,
logs:[ {ammount:3}]
}
]
You can do another map on the batches to return a new array of objects, and attach that to your returned object instead:
const out = data.map(({ position: order, batches }) => {
const logs = batches.map(({ batchNumber, ammount }) => {
if (batchNumber) return { batchNumber, ammount };
return { ammount };
});
return { order, logs }
});
DEMO
One approach would be to make a shallow copy of the target omitting keys you want to delete, for example:
let drop = key => obj => Object.keys(obj).reduce((r, k) =>
k === key ? r : {...r, [k]: obj[k]}, {});
let test = [
{foo:11, bar:2, baz: 3},
{foo:22, bar:2, baz: 3},
{foo:33, bar:2, baz: 3},
];
console.log(test.map(drop('bar')));
To add another option to the mix: it is possible to use Object.assign to optionally assign the property:
const data = [{
position: 1,
batches: [{batchNumber: '',ammount: 3}, {batchNumber: 'withNr',ammount: 4}]
}];
const res = data.map(d =>
({
order: d.position,
logs : d.batches.map(({ammount, batchNumber}) => Object.assign({ammount}, batchNumber ? {batchNumber} : null ))
})
);
console.log(res);

The proper ways to stack optioned promises

What would be the proper or the best way to collect all data from DB with promises, but with using native Node promises.
The goal is only to present what is selected:
const allPromises = [];
const selected = {
sectionA: true,
sectionB: false,
sectionCIds: [ 1, 2, 4 ],
};
if (selected.sectionA) {
allPromises.push(getSectionADataFromDbPromise());
}
if (selected.sectionB) {
allPromises.push(getSectionBDataFromDbPromise());
}
if (selected.sectionCIds.length > 0) {
allPromises.push(selected.sectionCIds
.map(getSectionCDataFromDbPromise)
);
}
Promise.all(allPromises)
.then((allResults) => {
if (selected.sectionA) {
dataA = allResults[0];
}
if (selected.sectionA) {
dataB = allResults[1];
}
if (selected.sectionC) {
dataC = allResults[2]; // <-- this will not work if B is not selected
}
// ... same logic to build report: return Promise.all()...
});
Possible solutions:
Track index for each data selected (eg. index of C will be 1)
Object Map
Add else { allPromises.push(Promise.resolve(null)) } to every if
Is there maybe an easier or one of this will be the proper way?
Don't use push on the arrays conditionally, but always put the same value at the same index. Even if the value is nothing - Promise.all will handle that just fine.
const selected = {
sectionA: true,
sectionB: false,
sectionCIds: [ 1, 2, 4 ],
};
Promise.all([
selected.sectionA ? getSectionADataFromDbPromise() : null,
selected.sectionB ? getSectionBDataFromDbPromise() : null,
Promise.all(selected.sectionCIds.map(getSectionCDataFromDbPromise))
]).then(([dataA, dataB, dataC]) => {
if (selected.sectionA) {
// use dataA
}
if (selected.sectionA) {
// use dataB
}
if (dataC.length) { // same as selected.selectionCIds.length
// use dataC
}
});
What do you think about this ? It's bigger, it's heavier, it's more difficult, but it's all automatized and fully evolutive. Wanna handle a new parameter ? A parameter have data now ? Change the map only.
I create a map that would contains everything we need to use a loop. The state of the data (activated or not), the function to call to get the data and so on.
const mapSelected = {
sectionA: {
state: true,
func: getSectionADataFromDbPromise,
},
sectionB: {
state: false,
func: getSectionBDataFromDbPromise,
},
sectionC: {
state: true,
func: getSectionCDataFromDbPromise,
data: [
1,
2,
4,
],
},
};
Then we create the promise array using the map we has created. Handling the case with data and without data.
const promises = Object.values(mapSelected).reduce((tmp, {
state,
func,
data,
}) => {
if (!state) return tmp;
if (data && data.length) {
return [
...tmp,
...data.map(x => func.call(this, x)),
];
}
return [
...tmp,
func.call(this),
];
});
Then we create arrays from the promise return for each key on the map. You can change how I present the data, I didn't knew what you really wanted there.
Promise.all(promises)
.then((allResults) => {
let i = 0;
const [
dataA,
dataB,
dataC,
] = Object.values(mapSelected).reduce((tmp, {
state,
data,
}, xi) => {
if (!state) return tmp;
if (data && data.length) {
data.forEach(x => (tmp[xi].push(allPromises[i++])));
return tmp;
}
tmp[xi].push(allPromises[i++]);
return tmp;
}, Object.values(mapSelected).map(() => []));
});
#EDIT
I just did a snippet about the code I've made, run it
function a() {
return 1;
}
const mapSelected = {
sectionA: {
state: true,
func: a,
},
sectionB: {
state: false,
func: a,
},
sectionC: {
state: true,
func: a,
data: [
1,
2,
4,
],
},
};
const allPromises = [
0,
1,
2,
3,
4,
];
let i = 0;
const [
dataA,
dataB,
dataC,
] = Object.values(mapSelected).reduce((tmp, {
state,
data,
}, xi) => {
if (!state) return tmp;
if (data && data.length) {
data.forEach(x => (tmp[xi].push(allPromises[i++])));
return tmp;
}
tmp[xi].push(allPromises[i++]);
return tmp;
}, Object.values(mapSelected).map(() => []));
console.log(dataA);
console.log(dataB);
console.log(dataC);
Unfortunately, unlike libraries such as Q, the standard Promise does not expose a variant of all taking an object of promises.
However, we can use the new ES2015 and ES2017 Object utility methods to assist us in keeping the code readable.
const allPromises = {};
const selected = {
sectionA: true,
sectionB: false,
sectionCIds: [1, 2, 4],
};
if (selected.sectionA) {
allPromises.sectionA = getSectionADataFromDbPromise();
}
if (selected.sectionB) {
allPromises.sectionB = getSectionBDataFromDbPromise();
}
if (selected.sectionBIds.length > 0) {
allPromises.sectionC = Promise.all(selected.sectionBIds
.map(getSectionCDataFromDbPromise)
);
}
Now we can write
Promise.all(Object.entries(allPromises).map(([key, promise]) =>
promise.then(value => ({[key]: value}))
))
.then(allResults => {
const results = Object.assign({}, ...allResults);
const data = {
a: results.sectionA,
b: results.sectionB,
c: results.sectionB && results.sectionC
};
// ... same logic to build report: return Promise.all()...
});

How to use if statement within map in javascript/vuejs

I'm building a small application in Vuejs where I'm getting a response data and I'm mapping it to a variable, I've got few elements which has empty array, so while mapping I want to check the condition and map accordingly. Here is my code:
this.model = a.map(i => Object.assign({
'id': i.id,
'meeting_date': i.schedule,
'meeting_call': i.type,
'event_type': i.event_type,
'venue': i.venue,
'with_client': i.with_client
},{
if(i.meeting.meeting_summaries)
{
'meeting_summaries': i.meeting_summaries.map(ms => ({
client_name: ms.client_name,
nature: ms.nature,
action: ms.action,
mention: ms.user_id,
feedback: ms.feedback
}))
}
},
map is purely functional, it doesn't modify the elements instead return a newly formed array, so you can do like this:
this.model = a.map(i => {
var item = {}
item['id']= i.id,
item['meeting_date']= i.schedule,
item['meeting_call']= i.type,
item['event_type']= i.event_type,
item['venue']= i.venue,
item['with_client']= i.with_client
if(i.meeting && i.meeting.meeting_summaries) {
item['meeting_summaries']= i.meeting.meeting_summaries.map(ms =>({
client_name: ms.client_name,
nature: ms.nature,
action: ms.action,
mention: ms.user_id,
feedback: ms.feedback
}))
}else {
item['meeting_summaries'] = []
}
return item
}
In your case you can just swap to ternary expression:
this.model = a.map(i => Object.assign({
'id': i.id,
'meeting_date': i.schedule,
'meeting_call': i.type,
'event_type': i.event_type,
'venue': i.venue,
'with_client': i.with_client
}, (i.meeting.meeting_summaries) ? { // if condition is met
'meeting_summaries': i.meeting_summaries.map(ms => ({
client_name: ms.client_name,
nature: ms.nature,
action: ms.action,
mention: ms.user_id,
feedback: ms.feedback
}))
} : {} // otherwise don't do anything
The idea is the following:
const a = [1, 2, 3, 4, 5];
const b = a.map(number => Object.assign({a: number},
(number > 2) ? {b: ++number} : {}, // add 'b' for numbers > 2
(number % 2 === 0) ? {c: number + ' is even'} : {} // add 'c' for even numbers
))
console.log(b)

Categories

Resources