Develop Reference

JS - StreamSaver downlad does not start

I download data from API in chunks decrypt it and than pass to ReadableStream.
But after last chunk, the file is not downloaded.
I work with axios and StreamSaver.js
Code:
Above in the code I declare:
this.filestream = streamSaver.createWriteStream('sample.jpg');
this.writer = await this.filestream.getWriter();
let readableStream;
readableStream = new ReadableStream({
start(ctrl) {
const nextChunk = async () => {
let fileDataResponse = await that.$api.post(
'endpoint', {
file_id: UUID,
chunk_index: index
}, {
headers: {
...
}
}
);
done =
fileDataResponse.data.length <=
fileDataResponse.data.current_index;
if (fileDataResponse.data.data) {
let data = await that.decryptData(fileDataResponse.data.data);
ctrl.enqueue(data);
}
if (!done) {
index += 1;
nextChunk();
} else {
ctrl.close();
}
};
nextChunk();
}
});
const reader = readableStream.getReader();
const close = () => {
that.writer.close();
};
const pump = () =>
reader.read().then((res) => {
if (!res.done) {
that.writer.write(res.value).then(pump);
} else {
close();
}
});
pump();
Where could be my error here?
Thank you a lot!

Issue was the res.value is not an Int8Array

How to make recursive promise calls?

I am working with an API that gives me data with a limit per request (here 25). Therefore, I have to recursively make promises with fetch. However, while most of my logic works, when I try to return from the function it will return an empty array. The image below will make it more clear.
const url = (conf_name) => {
return (
"https://api.elsevier.com/content/search/scopus?view=COMPLETE&cursor=*&query=CONFNAME(" +
conf_name +
")%20AND%20DOCTYPE(cp)%20AND%20SRCTYPE(p)&field=dc:creator,dc:title,dc:description,affilname,affiliation-city,affiliation-country,authkeywords,prism:doi,prism:doi,prism:coverDate"
);
};
const savePapers = (json, conf_name) => {
let temp = new Array();
for (let i = 0; i < json["search-results"]["entry"].length; i++) {
temp[i] = {
title: json["search-results"]["entry"][i]["dc:title"],
author: json["search-results"]["entry"][i]["dc:creator"],
publication_date: json["search-results"]["entry"][i]["prism:coverDate"],
doi: json["search-results"]["entry"][i]["prism:doi"],
abstract: json["search-results"]["entry"][i]["dc:description"],
author_keywords: json["search-results"]["entry"][i]["authkeywords"],
proceeding: conf_name,
};
}
return temp;
};
async function getPapers(final, url, conf_name) {
let total_amount_of_papers;
let next;
let position = 2;
try {
let response = await fetch(url, options);
let json = await response.json();
total_amount_of_papers = json["search-results"]["opensearch:totalResults"];
if (json["search-results"]["link"][position]["#ref"] == "prev")
next = json["search-results"]["link"][position + 1]["#href"];
next = json["search-results"]["link"][position]["#href"];
final = final.concat(savePapers(json, conf_name));
if (final.length === 50) {
console.log("hey",final.length);
return final;
}
await getPapers(final, next, conf_name);
} catch (error) {
console.log(error);
}
}
const createNewConf = async (conferences) => {
let final = new Array();
try {
var temp = new Conference({
acronym: conferences.acronym,
name: conferences.fullname,
area: conferences.area,
subarea: conferences.subarea,
location: conferences.location,
url: conferences.url,
description: conferences.description,
papers: await getPapers(final, url(conferences.acronym),conferences.acronym),
});
console.log(temp.papers.length);
} catch (error) {
console.log(error);
}
return temp;
};
describe("Saving records", function () {
it("Saved records to the database", async function (done) {
var conferences = [];
try {
for (var i = 0; i <= 1; i++) {
conferences[i] = await createNewConf(json_conferences[i]);
conferences[i].save().then(function () {
assert(conferences[i].isNew === True);
done();
});
}
mongoose.connection.close();
} catch (error) {
console.log(error);
}
});
});
Below you can see the length of my final array after passing the if to stop fetching more. and the second number is what I receive in the initial call
Console
Maybe anyone has an idea of the undefined behavior that occurs during return.
Your help is much appreciated.

How to measure the time of several calls of async function

I have a problem with understanding how do async functions in js work. So I need to call async function for 5 times and measure the duration of every execution. In the end I should have an array with 5 time values.
I have smth like this
for (let i = 0; i < 5; i++) {
const run = async () => {
await test(thisTest, filename, unitTestDict);
};
measure(run).then(report => {
// inspect
const {tSyncOnly, tSyncAsync} = report;
// array.push(tSyncAsync)
console.log(`∑ = ${tSyncAsync}ms \t`);
}).catch(e => {
console.error(e)
});
}
Here I found the realization of time measure function:
class Stack {
constructor() {
this._array = []
}
push(x) {
return this._array.push(x)
}
peek() {
return this._array[this._array.length - 1]
}
pop() {
return this._array.pop()
}
get is_not_empty() {
return this._array.length > 0
}
}
class Timer {
constructor() {
this._records = new Map
/* of {start:number, end:number} */
}
starts(scope) {
const detail =
this._records.set(scope, {
start: this.timestamp(),
end: -1,
})
}
ends(scope) {
this._records.get(scope).end = this.timestamp()
}
timestamp() {
return performance.now()
}
timediff(t0, t1) {
return Math.abs(t0 - t1)
}
report(scopes, detail) {
let tSyncOnly = 0;
let tSyncAsync = 0;
for (const [scope, {start, end}] of this._records)
if (scopes.has(scope))
if (~end) {
tSyncOnly += end - start;
tSyncAsync += end - start;
const {type, offset} = detail.get(scope);
if (type === "Timeout")
tSyncAsync += offset;
}
return {tSyncOnly, tSyncAsync}
}
}
async function measure(asyncFn) {
const stack = new Stack;
const scopes = new Set;
const timer = new Timer;
const detail = new Map;
const hook = createHook({
init(scope, type, parent, resource) {
if (type === 'TIMERWRAP') return;
scopes.add(scope);
detail.set(scope, {
type: type,
offset: type === 'Timeout' ? resource._idleTimeout : 0
})
},
before(scope) {
if (stack.is_not_empty) timer.ends(stack.peek());
stack.push(scope);
timer.starts(scope)
},
after() {
timer.ends(stack.pop())
}
});
return await new Promise(r => {
hook.enable();
setTimeout(() => {
asyncFn()
.then(() => hook.disable())
.then(() => r(timer.report(scopes, detail)))
.catch(console.error)
}, 1)
})
}
I can call it inside the loop and get console.log with time for every iteration:
measure(run).then(report => {
// inspect
const {tSyncOnly, tSyncAsync} = report;
// array.push(tSyncAsync)
console.log(`∑ = ${tSyncAsync}ms \t`);
}).catch(e => {
console.error(e)
});
But when I try to push this console values to array, nothing comes out, the array remains empty. Is there a way to fill it in?

How to bypass jest setTimeout error of 5000ms by managing promises (Async and Await)

I wrote an Async/Await function to return promises for drivers report and analysis.
I have three different promise API files I extracted details from to do my analysis. However running test ith jest I get the error
Timeout - Async callback was not invoked within the 5000ms timeout specified by jest.setTimeout.Error:
I have refactored my code more than three times in two days but the error returns.
I will like to know how to manage my promises, perhaps there is something am not doing well and I am keen on this for optimization.
Is there a way to manage the promises in the code below to bypass the jest error?
any other suggestion will be highly appreciated.
NB: sorry I have post all the code for better insight.
code
const { getTrips } = require('api');
const { getDriver } = require('api')
const { getVehicle } = require('api')
/**
* This function should return the data for drivers in the specified format
*
* Question 4
*
* #returns {any} Driver report data
*/
async function driverReport() {
// Your code goes here
let trip = await getTrips()
trip = trip.map(item => {
item.billedAmount = parseFloat(item.billedAmount.toString().replace(',', '')).toFixed(2);
return item;
})
let getId = trip.reduce((user, cur) => {
user[cur.driverID] ? user[cur.driverID] = user[cur.driverID] + 1 : user[cur.driverID] = 1
return user
}, {})
// console.log(getId)
let mapId = Object.keys(getId)
// console.log(mapId)
let eachTripSummary = mapId.reduce((acc, cur) => {
let singleTrip = trip.filter(item => item.driverID == cur)
acc.push(singleTrip)
return acc
}, [])
// eachTripSummary = eachTripSummary[0]
// console.log(eachTripSummary)
// console.log(trip)
let reducedReport = eachTripSummary.reduce(async(acc, cur) =>{
acc = await acc
// console.log(acc)
let user = {}
let cash = cur.filter(item => item.isCash == true)
// console.log(cash.length)
let nonCash = cur.filter(item => item.isCash == false)
let driverSummary = await getDriverSummary(cur[0]['driverID'])
let trips = []
let customer = {}
cur[0].user ? (customer['user'] = cur[0]['user']['name'], customer['created'] = cur[0]['created'], customer['pickup'] = cur[0]['pickup']['address'],
customer['destination'] = cur[0]['destination']['address'], customer['billed'] = cur[0]['billedAmount'], customer['isCash'] = cur[0]['isCash']) : false
trips.push(customer)
let vehicles = []
if(driverSummary == undefined){
// console.log(cur)
user = {
id: cur[0]['driverID'],
vehicles: vehicles,
noOfCashTrips: cash.length,
noOfNonCashTrips: nonCash.length,
noOfTrips: cur.length,
trips: trips
}
acc.push(user)
// console.log(user)
return acc
}
let driverInfo = driverSummary[0]
let vehicleInfo = driverSummary[1]
let { name, phone } = driverInfo
let { plate, manufacturer } = vehicleInfo[0]
// console.log(plate)
let vpm = {
plate,
manufacturer
}
vehicles.push(vpm)
// console.log(cash.length)
user ={
fulName: name,
phone,
id: cur[0]['driverID'],
vehicles: vehicles,
noOfCashTrips: cash.length,
noOfNonCashTrips: nonCash.length,
noOfTrips: cur.length,
trips: trips
}
acc.push(user)
// console.log(acc)
return acc
}, [])
// reducedReport.then(data =>{console.log(data)})
return reducedReport
}
async function getDriverSummary(param) {
let driverDetails = await getDriver(param)
.then(data => {return data}).catch(err => {return err})
// console.log(driverDetails)
let vehicleDetails;
let { vehicleID } = driverDetails
if(driverDetails != "Error" & vehicleID != undefined){
// console.log(vehicleID)
vehicleDetails = vehicleID.map(async item => {
let vehicleSummary = getVehicle(item)
return vehicleSummary
})
// console.log(await vehicleDetails)
return await Promise.all([driverDetails, vehicleDetails])
}
}
driverReport().then(data => {
console.log(data)
})
module.exports = driverReport;

Use jest.setTimeout(30000); to increase the timeout. It will increase the timeout globally.
// jest.config.js
module.exports = {
setupTestFrameworkScriptFile: './jest.setup.js'
}
// jest.setup.js
jest.setTimeout(30000)
Or you can use user test example like this
describe("...", () => {
test(`...`, async () => {
...
}, 30000);
});

Node js - function to return array of objects read from sequelize database

I'm trying to create a function in node js which reads database values in and pushes them into an array, and then at the end returns this array. My functions looks like this at the moment:
function getInvoicesCount() {
let promiseInvoices = [];
let userInvCount = 0;
let deletedUserInvCount = 0;
let userInvAmount = 0;
let deletedUserInvAmount = 0;
let monthWiseInvCount = [];
db.userInvoices
.findAll({
attributes: [
'deleted_at',
[sequelize.fn('COUNT', sequelize.col('id')), 'count'],
[sequelize.fn('SUM', sequelize.col('invoice_amount')), 'amount'],
[sequelize.fn('MONTH', sequelize.col('invoice_date')), 'month']
],
group: ['invoice_date', 'deleted_at'],
paranoid: false
})
.then(result => {
result.forEach(function(element) {
userInvCount += element.dataValues.count;
userInvAmount += element.dataValues.amount;
if (element.dataValues.deleted_at != null) {
deletedUserInvAmount += element.dataValues.amount;
deletedUserInvCount += element.dataValues.count;
}
monthWiseInvCount.push(element.dataValues);
});
if (monthWiseInvCount.map(a => a === 'deleted_at')) {
monthWiseInvCount.map(a => delete a.deleted_at);
}
promiseInvoices.push(
userInvCount,
userInvAmount,
deletedUserInvCount,
deletedUserInvAmount,
monthWiseInvCount
);
});
return promiseInvoices;
}
In the main part of the code I would like to call this funtion and use a .then to get the returned array
Can you help me out how I can return a promise in the function and how will the array be accessible in the .then part?

Here are the changes you need to do to get expected result :
function getInvoicesCount() {
...
return db.userInvoices.findAll({ //<-------- 1. First add return here
...
}).then(result => {
...
return promiseInvoices; //<----------- 2. Put this line inside the then
});
// return promiseInvoices; //<----------- Remove this line from here
}
getInvoicesCount().then(data => {
console.log(data); // <------- Check the output here
})
Explanation for this :
To get .then when you can function , function should return promise ,
but in you case you are just returning a blank array ,
As per the sequlize doc , db.userInvoices.findAll returns the
promise so all you need to do is add return before the this function
, First step is done
You were return promiseInvoices; at wrong place , why ? , coz at
that you will never get the result , as the code above it run in async
manner as it is promise , so you will always get the balnk array , to
get the expected result you should return it from
db.userInvoices.findAll's then function as shown above in code

You should return the results from then to access it in the chain.
function getInvoicesCount() {
const promiseInvoices = []
let userInvCount = 0
let deletedUserInvCount = 0
let userInvAmount = 0
let deletedUserInvAmount = 0
const monthWiseInvCount = []
return db.userInvoices
.findAll({
attributes: [
'deleted_at',
[sequelize.fn('COUNT', sequelize.col('id')), 'count'],
[sequelize.fn('SUM', sequelize.col('invoice_amount')), 'amount'],
[sequelize.fn('MONTH', sequelize.col('invoice_date')), 'month'],
],
group: ['invoice_date', 'deleted_at'],
paranoid: false,
})
.then((result) => {
result.forEach((element) => {
userInvCount += element.dataValues.count
userInvAmount += element.dataValues.amount
if (element.dataValues.deleted_at != null) {
deletedUserInvAmount += element.dataValues.amount
deletedUserInvCount += element.dataValues.count
}
monthWiseInvCount.push(element.dataValues)
})
if (monthWiseInvCount.map(a => a === 'deleted_at')) {
monthWiseInvCount.map(a => delete a.deleted_at)
}
return promiseInvoices.push(
userInvCount,
userInvAmount,
deletedUserInvCount,
deletedUserInvAmount,
monthWiseInvCount,
)
})
}
You can now use
getInvoicesCount().then(() => {
//do something here
})

getData(htmlData,tags)
.then(function(data) {
console.log(data); //use data after async call finished
})
.catch(function(e) {
console.log(e);
});
function getData() {
return new Promise(function(resolve, reject) {
//call async task and pass the response
resolve(resp);
});
}

In general you can return a promise like this:
function returnPromise() {
return new Promise((resolve, reject) => {
resolve({foo: 'bar'});
});
}
So while the other answer is completely correct, you can use the above structure to create a function that returns a promise like this:
function getInvoicesCount() {
return new Promise((resolve, reject) => {
let promiseInvoices = [];
let userInvCount = 0;
let deletedUserInvCount = 0;
let userInvAmount = 0;
let deletedUserInvAmount = 0;
let monthWiseInvCount = [];
db.userInvoices
.findAll({
attributes: [
'deleted_at',
[sequelize.fn('COUNT', sequelize.col('id')), 'count'],
[sequelize.fn('SUM', sequelize.col('invoice_amount')), 'amount'],
[sequelize.fn('MONTH', sequelize.col('invoice_date')), 'month']
],
group: ['invoice_date', 'deleted_at'],
paranoid: false
})
.then(result => {
result.forEach(function(element) {
userInvCount += element.dataValues.count;
userInvAmount += element.dataValues.amount;
if (element.dataValues.deleted_at != null) {
deletedUserInvAmount += element.dataValues.amount;
deletedUserInvCount += element.dataValues.count;
}
monthWiseInvCount.push(element.dataValues);
});
if (monthWiseInvCount.map(a => a === 'deleted_at')) {
monthWiseInvCount.map(a => delete a.deleted_at);
}
promiseInvoices.push(
userInvCount,
userInvAmount,
deletedUserInvCount,
deletedUserInvAmount,
monthWiseInvCount
);
resolve(promiseInvoices); // When you are done, you resolve
})
.catch(err => reject(err)); // If you hit an error - reject
});
}
However that is a rather big function, would recommend splitting it up in smaller parts.