Develop Reference

How to assign an object returned by an async function to a variable

I have a function that scrapes a data from a website. THe function is this:
const getSharesInfo = async () => {
try {
const { data } = await axios.get(url);
const $ = cheerio.load(data);
const elmSelector =
"#ctl00_ContentPlaceHolder1_LiveTrading > table > tbody > tr";
const keys = [
"Name",
"LTP",
"% Change",
"Open",
"High",
"Low",
"Qty",
"Close Price",
];
$(elmSelector).each((parentIdx, parentElm) => {
let keyIdx = 0;
const sharesObj = {};
$(parentElm)
.children()
.each((childIdx, childElm) => {
const tdValue = $(childElm).text();
if (tdValue) {
sharesObj[keys[keyIdx]] = tdValue;
keyIdx++;
}
});
console.log(sharesObj);
return sharesObj;
});
} catch (err) {
console.error(err);
}
};
It makes an object from the data scraped. Then when I run the function it console log the shareObj and output is like this:
{
Name: 'USLB',
LTP: '890.00',
'% Change': '0.79',
Open: '890.00',
High: '888.10',
Low: '888.10',
Qty: '24'
}
{
Name: 'VLBS',
LTP: '849.00',
'% Change': '0',
Open: '850.00',
High: '833.00',
Low: '834.00',
Qty: '285'
}
{
Name: 'WNLB',
LTP: '927.30',
'% Change': '-4.4',
Open: '965.00',
High: '927.30',
Low: '965.00',
Qty: '270'
}
I want to catch it and pass it to the home route. I have setup router like this.
app.get("/", async (req, res) => {
const shares = await getSharesInfo();
res.render("home", { shares });
});
But when I console log shares I get nothing. No errors no data nothing. It just prints some empty spaces. How can I solve this ??

Move the const sharesObj = {} declaration outside of the call to .each (= move it up 3 lines) and move the line return sharesObj; out of the .each loop, too (e. g.: move it down two lines).
As of now you create the constant binding inside .each and then return it inside the .each callback, not in getSharesInfo.
Example:
const getSharesInfo = async () => {
try {
const { data } = await axios.get(url);
const $ = cheerio.load(data);
const elmSelector =
"#ctl00_ContentPlaceHolder1_LiveTrading > table > tbody > tr";
const keys = [
"Name",
"LTP",
"% Change",
"Open",
"High",
"Low",
"Qty",
"Close Price",
];
const sharesObj = {}; // <-- new position
$(elmSelector).each((parentIdx, parentElm) => {
let keyIdx = 0;
//const sharesObj = {}; <-- old position
$(parentElm)
.children()
.each((childIdx, childElm) => {
const tdValue = $(childElm).text();
if (tdValue) {
sharesObj[keys[keyIdx]] = tdValue;
keyIdx++;
}
});
console.log(sharesObj);
//return sharesObj; <-- old position
});
return sharesObj; // <-- new position
} catch (err) {
console.error(err);
}
};

Iterating through multiple documents

I'm running into some difficulties iterating through a list of documents. I'm working on an app for distributing season tickets amongst friends and I've created a document store in mongodb. When I post the teamId and number of tickets (groups) it creates a document with the below structure.
{
draftOwner: '',
draftState: 'active',
draftCreate: 1659240148635,
draftGroups: 2,
homeTeam: '',
homeId: 17,
draftInvites: '',
homeSchedule: [
{
date: '2022-10-14T23:30:00Z',
home_id: 17,
home_name: 'Detroit Red Wings',
away_id: 8,
away_name: 'Montréal Canadiens',
ticketName_0: '',
ticketOwner_0: '',
ticketName_1: '',
ticketOwner_1: ''
},
{
date: '2022-10-17T23:30:00Z',
home_id: 17,
home_name: 'Detroit Red Wings',
away_id: 26,
away_name: 'Los Angeles Kings',
ticketName_0: '',
ticketOwner_0: '',
ticketName_1: '',
ticketOwner_1: ''
},
{
date: '2022-10-23T21:00:00Z',
home_id: 17,
home_name: 'Detroit Red Wings',
away_id: 24,
away_name: 'Anaheim Ducks',
ticketName_0: '',
ticketOwner_0: '',
ticketName_1: '',
ticketOwner_1: ''
}, ... ]
Example of my post method that creates documents for reference.
const createDraft = (req, res, next) => {
const teamId = parseInt(req.query.id);
const draftGroups = parseInt(req.query.groups);
const url = nhlScheduleAPI + teamId.toString() + season;
let settings = { method: "Get"};
fetch(url, settings)
.then(res => res.json())
.then((json) => {
let games = json['dates'];
let draftSchedule = [];
for (let i = 0; i < games.length; i++) {
let row = {};
if (games[i]['games'][0]['teams']['home']['team']['id'] === teamId) {
Object.assign(row, {date: games[i]['games'][0]['gameDate']});
Object.assign(row, {home_id: games[i]['games'][0]['teams']['home']['team']['id']});
Object.assign(row, {home_name: games[i]['games'][0]['teams']['home']['team']['name']});
Object.assign(row, {away_id: games[i]['games'][0]['teams']['away']['team']['id']});
Object.assign(row, {away_name: games[i]['games'][0]['teams']['away']['team']['name']});
for (let n = 0; n < draftGroups; n++) {
let ticketName = "ticketName_" + n.toString();
let ticketOwner = "ticketOwner_" + n.toString();
Object.assign(row, {[ticketName]: ""})
Object.assign(row, {[ticketOwner]: ""})
}
draftSchedule.push(row);
}
}
let newDraftObj = new Object({ draftOwner: "", draftState: "active", draftCreate: Date.now(),
draftGroups: draftGroups, homeTeam: "", homeId: teamId, draftInvites: "", homeSchedule: draftSchedule });
const client = new MongoClient(uri);
async function run() {
try {
const database = client.db("ticketdrafterDB");
const drafts = database.collection("drafts");
const result = await drafts.insertOne(newDraftObj);
console.log(result);
console.log(newDraftObj);
} finally {
await client.close();
}
}
run().catch(console.dir);
res.send(newDraftObj)
})
};
So now what I am running into issues with is iterating through the list of the currently {draftState: active} drafts. I'm trying to show a page that just has a table output of each object showing [draftCreate, homeTeam, draftOwner]. Here is what I have created so far and I'm getting console output, so the objects are being retrieved but I just can't display them to the client for some reason.
controller/home.js
const homeView = (req, res, next) => {
const client = new MongoClient(uri);
async function run() {
try {
const database = client.db("ticketdrafterDB");
const collection = database.collection("drafts");
const query = { draftState: "active" };
const options = { projection: { homeSchedule: 1 }};
const drafts = collection.find(query, options);
if ((await drafts.count()) === 0) {
console.log("No documents found!")
}
res.render('index', {drafts: drafts});
} finally {
await client.close();
}
}
run().catch(console.dir);
index.pug
each draft in drafts
li.list-group-item
p #{draft.draftState}
Would really appreciate any guidance here, thanks in advance!

try turning the drafts cursor into an array with method toArray:
const results = await drafts.toArray();
res.render('index', {drafts: results});

Result won't update VAR

I am trying to run a query, inside AXIOS which gets data from a 3rd party URL. Then uses some of that data to search our mongoDB database.
However it seems it won't update var total = 0
While the query below does function correctly, the return result won't allow me to set that it to the query.
Promise.all(arr.forEach( async (id,index) => {
//(CODE REMOVED JUST TO GET THIS FUNCTION TO WORK)
const search = await geoLocation.find({
'location': {
'$geoWithin': {
'$box': [
[-35.2418503, -13.5076852], [112.8656697, 129.0020486]
]
}
}}).toArray();
total = search.length;
}));
See the full code below
var array = [];
var pointarray = []
var total = 0;
areas.forEach((id,index) => {
if(id.type == "Point"){
pointarray[index] = "N"+id.id;
}else{
array[index] = "R"+id.id;
}
});
var arraySearch = "https://nominatim.openstreetmap.org/lookup?osm_ids="+array.toString()+"&polygon_geojson=1&bbox=1&format=json";
var pointSearch = "https://nominatim.openstreetmap.org/lookup?osm_ids="+pointarray.toString()+"&polygon_geojson=1&bbox=0&format=json"
const requestOne = axios.get(arraySearch);
const requestTwo = axios.get(pointSearch);
axios.all([requestOne, requestTwo])
.then(axios.spread((...responses) => {
const responseOne = responses[0]
const responseTwo = responses[1]
/*
process the responses and return in an array accordingly.
*/
return [
responseOne.data,
responseTwo.data,
];
}))
.then(arr => {
Promise.all(arr.forEach( async (id,index) => {
//const middleIndex = id[index].boundingbox.length / 2;
//const firstHalf = id[index].boundingbox.splice(0, middleIndex);
//const secondHalf = id[index].boundingbox.splice(-middleIndex);
//res.send(secondHalf[0]);
const query = [{
$match: {
location: {
$geoWithin: {$box:[[Number(firstHalf[0]),Number(firstHalf[1])],[Number(secondHalf[0]),Number(secondHalf[1])]]
}
}
}
},{
$count: 'id'
}]
const search = await geoLocation.find({
'location': {
'$geoWithin': {
'$box': [
[-35.2418503, -13.5076852], [112.8656697, 129.0020486]
]
}
}}).toArray();
total = search.length;
// total = search.length;
// const search = geoLocation.aggregate(query).toArray.length;
}));
})
.catch(errors => {
console.log("ERRORS", errors);
})
.then(function () {
res.send(total);
});

Run Promise all in an await for loop

I have a nested array of Promise function. Eg:
let callSet = [
[ func1 , func2 , func3],
[ func4 , func5 , func6],
[ func7 , func8 , func9],
]
the response of await func1() will be in below structure:
{
data : [ {id: 1} , {id:2}],
status: 400
}
I want to run it in a for loop so that they run in batch sequentially and incrementally load the data into array as they come in. Tried below code but I am lost on how to do it:
const finalData = [];
private asyncForEach(PromiseGroups): Promise<any> {
return PromiseGroups.reduce(async (acc, cItem) => {
const results = await acc;
const res = await Promise.all(cItem) as any;
finalData = [...finalData , ...[].concat(...res.map(r => r.data))];
return results
}, Promise.resolve([]))
}
I would like to load it in as:
[ {id: 1}, {id:2} , {id: 3} ..... ]
and this should get updated as the Promise all is getting resolved
I want to wait till func1 , func2 , func3 is resolved and then move to func4 , func5 , func6 . and once I get data of func4 , func5 , func6 , I want to push it with the data of func1 , func2 , func3

Try the below code:
private init(){
let callSet = [
[ func1 , func2 , func3],
[ func4 , func5 , func6],
[ func7 , func8 , func9],
];
this.asyncForEach(callSet,this.fetchInBatch.bind(this) )
}
private asyncForEach(funcGrpList, execFunc) {
return funcGrpList.reduce((p,funcList) => {
return p.then(() => execFunc(funcList));
}, Promise.resolve());
}
private fetchInBatch(pageGroupList) {
return Promise.all(pageGroupList).then((res: any) => {
this.finalData = [...this.finalData , ...[].concat(...res.map(r => r.data))];
})
}
This should work as expected

Edit: Assuming the last array result is return directly because there is no need to wait for next loop to be finished.
async function run(callSet) {
const output = [];
let prev = [];
const len = callSet.length;
for (let i = 0; i < len; i++) {
const array = await Promise.all(callSet[i].map(func => func()));
const data = array.map(item => item.data);
if (i === 0) {
// no need to append item to output
// just append item to previous array for next loop to use.
prev.push(...data);
} else if (i < len) {
// append item to output from previous result.
output.push(...prev);
prev = [];
// append data to previous result for next loop.
prev.push(...data);
} else {
//last loop, just append data from previous result and current result
output.push(...prev);
output.push(...data);
}
}
console.log(output);
}

This will call the execution sets in the requested order and timing and add the return data as soon as the group of promises returns
const finalData = [];
async function execute() {
for (const set of callSet) {
const resp = await Promise.all(set.map(f => f()));
finalData.push(...resp.map(r => r.data).flat());
}
}
Once execute() is called finalData will be updated asynchronously once for each 'row' of functions.
For future readers
resp.map(r => r.data).flat() is due to the specified promises payload. If someone needs just to pack the results together the code would be:
for (const set of callSet) {
const resp = await Promise.all(set);
finalData.push(...resp);
}

If you want to load the data in chunks but produce flat array of results, your easiest option is to use async/await syntax:
interface Data {
id: number
}
interface DataResponse {
data: Data[];
status: number;
}
type AsyncCall = () => Promise<DataResponse>;
/* ... */
const result: Data[] = [];
for(const chunk of callSet) {
const chunkResult = await Promise.all(chunk.map(f => f()));
result.push(...chunkResult.flatMap(x => x.data));
}
Playground Link
JavaScript demo:
/* mock section */
const fakeFunc = (id1, id2) => ()=>
Promise.resolve({
data : [{id: id1} , {id: id2}],
status: 400
});
const func1 = fakeFunc(1, 2),
func2 = fakeFunc(3, 4),
func3 = fakeFunc(5, 6),
func4 = fakeFunc(7, 8),
func5 = fakeFunc(9, 10),
func6 = fakeFunc(11, 12),
func7 = fakeFunc(13, 14),
func8 = fakeFunc(15, 16),
func9 = fakeFunc(17, 18)
;
/* /mock section */
async function main() {
let callSet = [
[ func1 , func2 , func3],
[ func4 , func5 , func6],
[ func7 , func8 , func9],
];
const result = [];
for(const chunk of callSet) {
const chunkResult = await Promise.all(chunk.map(f => f()));
result.push(...chunkResult.flatMap(x => x.data));
}
return result;
}
main()
.then(r => console.log(r));
I you prefer to use the Promise API only, instead of async/await, then you can reduce into a promise like this:
interface Data {
id: number
}
interface DataResponse {
data: Data[];
status: number;
}
type AsyncCall = () => Promise<DataResponse>;
/* ... */
const result = callSet.reduce((p: Promise<Data[]>, chunk: AsyncCall[]) =>
p.then(acc =>
Promise.all(chunk.map(f => f()))
.then(chunkResult => acc.concat(chunkResult.flatMap(x => x.data))))
, Promise.resolve([]));
Playground Link
JavaScript demo:
/* mock section */
const fakeFunc = (id1, id2) => () =>
Promise.resolve({
data : [{id: id1} , {id: id2}],
status: 400
});
const func1 = fakeFunc(1, 2),
func2 = fakeFunc(3, 4),
func3 = fakeFunc(5, 6),
func4 = fakeFunc(7, 8),
func5 = fakeFunc(9, 10),
func6 = fakeFunc(11, 12),
func7 = fakeFunc(13, 14),
func8 = fakeFunc(15, 16),
func9 = fakeFunc(17, 18)
;
/* /mock section */
function main() {
let callSet = [
[ func1 , func2 , func3],
[ func4 , func5 , func6],
[ func7 , func8 , func9],
];
const result = callSet.reduce((p, chunk) =>
p.then(acc =>
Promise.all(chunk.map(f => f()))
.then(chunkResult => acc.concat(chunkResult.flatMap(x => x.data))))
, Promise.resolve([]));
return result;
}
main()
.then(r => console.log(r));
However, it is a bit ugly. It can be improved by extracting some functions:
interface Data {
id: number
}
interface DataResponse {
data: Data[];
status: number;
}
type AsyncCall = () => Promise<DataResponse>;
/* ... */
const combineWith = (acc: Data[]) => (chunkResult: DataResponse[]) =>
acc.concat(chunkResult.flatMap(x => x.data));
const process = (chunk: AsyncCall[]) => (acc: Data[]) =>
Promise.all(chunk.map(f => f()))
.then(combineWith(acc));
/* ... */
const result = callSet.reduce((p: Promise<Data[]>, chunk: AsyncCall[]) =>
p.then(process(chunk))
, Promise.resolve([]))
Playground Link
JavaScript demo:
/* mock section */
const fakeFunc = (id1, id2) => () =>
Promise.resolve({
data : [{id: id1} , {id: id2}],
status: 400
});
const func1 = fakeFunc(1, 2),
func2 = fakeFunc(3, 4),
func3 = fakeFunc(5, 6),
func4 = fakeFunc(7, 8),
func5 = fakeFunc(9, 10),
func6 = fakeFunc(11, 12),
func7 = fakeFunc(13, 14),
func8 = fakeFunc(15, 16),
func9 = fakeFunc(17, 18)
;
/* /mock section */
const combineWith = (acc) => (chunkResult) =>
acc.concat(chunkResult.flatMap(x => x.data));
const process = (chunk) => (acc) =>
Promise.all(chunk.map(f => f()))
.then(combineWith(acc));
function main() {
let callSet = [
[ func1 , func2 , func3],
[ func4 , func5 , func6],
[ func7 , func8 , func9],
];
const result = callSet.reduce((p, chunk) =>
p.then(process(chunk))
, Promise.resolve([]));
return result;
}
main()
.then(r => console.log(r));

Mock ReadableStream

Consider following code:
fetch("/").then(response => {
const reader = response.body.getReader();
const decoder = new TextDecoder();
let res = 0;
return reader.read().then(function processResult(result) {
if (result.done) {
return res;
}
const part = decoder.decode(result.value, { stream: true });
res += part.length;
return reader.read().then(processResult);
});
}).then(res => console.log(res));
Now I want to test it. I'm mocking fetch to return fake response that should provide some reader. I want that reader to return 2 portions of data (see pieces array):
import { stub } from "sinon";
const pieces = [
new Uint8Array([65, 98, 99, 32, 208]), // "Abc " and first byte of "й"
new Uint8Array([185, 209, 139, 209, 141]), // Second byte of "й" and "ыэ"
];
const fetchStub = stub(window, "fetch");
fetchStub.returns(Promise.resolve({
body: {
getReader() {
// What's here?
},
},
}));
Is there something I can simply write in getReader or I should fully mock it like I do with fetch?

Mocked it manually:
fetchStub = stub(window, "fetch");
fetchStub.returns(Promise.resolve({
body: {
getReader() {
let i = 0;
return {
read() {
return Promise.resolve(
i < pieces.length
? { value: pieces[i++], done: false }
: { value: undefined, done: true }
);
},
};
},
},
}));