I'm working with a really big array in JS and I can see most of the time is used for loading and parsing the json data.
// is a Chrome Extension (but maybe I can move it to a nodejs app)
This is basically how I loading my data:
async function loadData(jsonFiles){
const fullData = [];
for(const jsonFile of jsonFiles){
const localUrl = 'http://localhost/'+jsonFile;
const response = await fetch(jsonFile);
if ( response.ok ){
try{
const data = await response.json();
const L = data.length;
for (let k = 0; k < L; k++) {
fullData.push(data[k]);
}
}
catch(e){
}
}
}
return fullData;
}
Is there any faster way to do that? even if it implies to save the data in another way/format
You can do the fetch calls in parallel, but other than that there's not a lot more you can do:
function loadData(jsonFiles){
return Promise.all(
jsonFiles.map(async file => {
const localUrl = 'http://localhost/'+jsonFile;
const response = await fetch(jsonFile);
if (response.ok) {
try{
return await response.json();
} catch (e) {
return null;
}
} else {
return null;
}
})
).then(results => {
return results.filter(result => result); // Filter out the `null`s
}).then(results => {
return results.flat(); // Flatten the results into one array
});
}
Make call and transform to json parallel
async function loadData(jsonFiles){
const calls = [];
for(const jsonFile of jsonFiles){
calls.push(fetch(jsonFile).then(response => response.json()));
}
return await Promise.allSettled(calls)
.then(parts => parts.filter(({status}) => status === "fulfilled"))
.then(parts => parts.map(({value}) => value))
.then(parts => parts.flat());
}
Related
I have a button when user click on it I will send a request and receive answer. If user click 100 times on this button I want to send 100 requests to server and each request send after previous. because I need previous response in next request.
example:
<button #click="sendRequest">send</button>
methods:{
sendRequest:function(){
axios.post('https:/url/store-project-item', {
'id': this.project.id,
"items": this.lists,
'labels': this.labels,
'last_update_key': this.lastUpdateKey,
'debug': 'hYjis6kwW',
}).then((r) => {
if (r.data.status) {
this.change = false
this.lastUpdateKey = r.data.lastUpdateKey;
this.showAlert('success')
} else {
if (r.data.state == "refresh") {
this.showAlert('error')
this.getProject()
} else {
this.showAlert('error')
}
}
}).catch(() => {
this.showAlert('error')
})
}}
I keep a higher-order function (i.e. a function that returns a function) withMaxDOP (DOP = degrees-of-parallelism) handy for this kind of thing:
const withMaxDOP = (f, maxDop) => {
const [push, pop] = createAsyncStack();
for (let x = 0; x < maxDop; ++x) {
push({});
}
return async(...args) => {
const token = await pop();
try {
return await f(...args);
} finally {
push(token);
}
};
};
The function makes use of an async stack data structure (implementation is in the attached demo), where the pop function is async and will only resolve when an item is available to be consumed. maxDop tokens are placed in the stack. Before invoking the supplied function, a token is popped from the stack, sometimes waiting if no token is immediately available. When the supplied completes, the token is returned to the stack. This has the effect of limiting concurrent calls to the supplied function to the number of tokens that are placed in the stack.
You can use the function to wrap a promise-returning (i.e. async) function and use it to limit re-entrancy into that function.
In your case, it could be used as follows:
sendRequest: withMaxDOP(async function(){ /*await axios.post...*/ }, 1)
to ensure that no call to this function ever overlaps another.
Demo:
const createAsyncStack = () => {
const stack = [];
const waitingConsumers = [];
const push = (v) => {
if (waitingConsumers.length > 0) {
const resolver = waitingConsumers.shift();
if (resolver) {
resolver(v);
}
} else {
stack.push(v);
}
};
const pop = () => {
if (stack.length > 0) {
const queueItem = stack.pop();
return typeof queueItem !== 'undefined' ?
Promise.resolve(queueItem) :
Promise.reject(Error('unexpected'));
} else {
return new Promise((resolve) => waitingConsumers.push(resolve));
}
};
return [push, pop];
};
const withMaxDOP = (f, maxDop) => {
const [push, pop] = createAsyncStack();
for (let x = 0; x < maxDop; ++x) {
push({});
}
return async(...args) => {
const token = await pop();
try {
return await f(...args);
} finally {
push(token);
}
};
};
// example usage
const delay = (duration) => {
return new Promise((resolve) => setTimeout(() => resolve(), duration));
};
async function doSomething(name) {
console.log("starting");
// simulate async IO
await delay(1000);
const ret = `hello ${name}`;
console.log(`returning: ${ret}`);
return ret;
}
const limitedDoSomething = withMaxDOP(doSomething, 1);
//call limitedDoSomething 5 times
const promises = [...new Array(5)].map((_, i) => limitedDoSomething(`person${i}`));
//collect the resolved values and log
Promise.all(promises).then(v => console.log(v));
I write some code to getting info
const stock = await Stock.find({
exchange: exchange
});
// Here stock array length is 5300
stock.forEach(async (stockEl) => {
const EOD_API = process.env.EOD_HISTORICAL_API
const {data} = await axios.get(`https://eodhistoricaldata.com/api/fundamentals/${stockEl.code}?api_token=${EOD_API}&filter=General::Industry`);
console.log(data);
});
Here I place get request for every stock array element by forEach function. Then it give me error like image-
Click to see images
But When I place it outside of forEach function like this-
const EOD_API = process.env.EOD_HISTORICAL_API
const {data} = await axios.get(`https://eodhistoricaldata.com/api/fundamentals/${stockEl.code}?api_token=${EOD_API}&filter=General::Industry`);
console.log(data);
Then it gives no error. For Remembering Stock has 5300 element, that means axios run 5300 times.
Any solution or idea?
You need to make a few changes:
Replace forEach with for because forEach is not promise aware
Use try, catch => catch any errors
Use Promise.allSettled => it allows you to run all promisses together without waiting each other which in return will enhance your app performance. It returns an array with status ("fulfilled", "rejected")
const fetchSingleStockElement = async (stockEl) => {
try {
const EOD_API = process.env.EOD_HISTORICAL_API,
{ data } = await axios(
`https://eodhistoricaldata.com/api/fundamentals/${stockEl.code}?api_token=${EOD_API}&filter=General::Industry`
);
return data;
} catch (err) {
throw new Error(err);
}
};
const fetchAllStockData = async () => {
let promisesArray = [];
try {
//fetch stock array
const { data } = await Stock.find({
exchange: exchange
});
//fetch single stock
for (let i = 0; i < data.length; i++) {
promisesArray.push(fetchSingleStockElement(data[i].id));
}
const results = await Promise.allSettled(promisesArray);
console.log('results', results);
} catch (err) {
console.log('results error', err);
}
};
Here is a working example with fake API of 4466 entries:
const fetchSingleAirline = async (airlineId) => {
try {
const { data } = await axios(`https://api.instantwebtools.net/v1/airlines/${airlineId}`);
return data;
} catch (err) {
throw new Error(err);
}
};
const fetchAllAirlineData = async () => {
let promisesArray = [];
try {
const { data } = await axios('https://api.instantwebtools.net/v1/airlines');
for (let i = 0; i < data.length; i++) {
promisesArray.push(fetchSingleAirline(data[i].id));
}
const results = await Promise.allSettled(promisesArray);
console.log('results', results);
} catch (err) {
console.log('results error', err);
}
};
Doing await in forEach doesn't hold the process since forEach is not promise-aware. Try this instead:
(async () => {
for (let index = 0; index < stock.length; index++) {
const EOD_API = process.env.EOD_HISTORICAL_API
const {data} = await axios.get(`https://eodhistoricaldata.com/api/fundamentals/${stock[i].code}?api_token=${EOD_API}&filter=General::Industry`);
console.log(data);
}
})();
More information.
I have been working with a stock api for the past day or two, and I'm trying to make a search engine for it with a filter method. It works, but it can be slow because the initial api fetch response is an array that is about 28,000 indeces. Is there a way to limit the response I get? I looked in the documentation and there doesn't seem to be a solution through the link itself. Any help would be appreciated!
input.addEventListener('keyup', (e) => {
const searchString = e.target.value.toLowerCase();
const filteredCompanies = stockCompanies.filter((company) => {
return (
company.description.toLowerCase().includes(searchString) ||
company.displaySymbol.toLowerCase().includes(searchString)
);
});
displayCompanies(filteredCompanies);
});
const loadCompanies = async () => {
try {
const res = await fetch(`https://finnhub.io/api/v1/stock/symbol?exchange=US&token=${apiKey}`);
stockCompanies = await res.json();
displayCompanies(stockCompanies);
}catch (err) {
console.error(err);
}
};
const displayCompanies = (data) => {
var dynamic = [];
for (var i = 0; i < data.length; i++){
dynamic.push(`
<div class="listings_card">
<h1>${data[i].description}</h1>
<p>${data[i].displaySymbol}</p>
</div>`);
}
document.querySelector('.listings').innerHTML = dynamic.join('');
};
loadCompanies();
If api doesn't support the feature you can not. But you can eliminate the list after you receive the response. Try this way:
const loadCompanies = async () => {
try {
const res = await fetch(`https://finnhub.io/api/v1/stock/symbol?exchange=US&token=${apiKey}`);
const res_data = await res.json();
stockCompanies = res_data.slice(0, 50)
displayCompanies(stockCompanies);
} catch (err) {
console.error(err);
}
};
So I want to pull data from two different endponts, and send them both to a function at the same time, as the parameters for said function.
function otherFunc(jsonData1, jsondata2){
//do stuff
}
function getJsonData(url1,url2){
fetch(url1);
fetch(url2);
.then((response) => response.json())
.then((data) => otherFunc(data));
}
getJsonData(someUrl1, someOtherUrl2);
So I know how to send one set of data to a function, and I know how to make multiple get requests but I don't know how I can send both sets of jsonData to the same function as params.
Thanks in advance
A little clearer using async/await
function otherFunc(jsonData1, jsonData2) {
//do stuff
}
async function getJsonData(url1, url2) {
const res1 = await fetch(url1);
const res2 = await fetch(url2);
const json1 = await res1.json();
const json2 = await res2.json();
otherFunc(json1, json2);
}
getJsonData(someUrl1, someOtherUrl2);
Alternatively using Promise.all()
function otherFunc(jsonData1, jsonData2) {
//do stuff
}
async function getJsonData(url1, url2) {
const resArr = await Promise.all(
[url1, url2].map(url => fetch(url))
);
const [json1, json2] = await Promise.all(
resArr.map(res => res.json())
);
otherFunc(json1, json2)
}
getJsonData(someUrl1, someOtherUrl2);
or...
function otherFunc(jsonData1, jsonData2) {
//do stuff
}
function getJsonData(url1, url2) {
Promise.all([
fetch(url1),
fetch(url2)
]).then(responses => //map the returned promises to resolve json
Promise.all(responses.map(res => res.json()))
).then(([json1, json2]) => // destructure resolved json
otherFunc(json1, json2)
).catch(error =>
// if there's an error, log it
console.log(error));
}
getJsonData(someUrl1, someOtherUrl2);
function otherFunc(jsonData1, jsondata2){
//do stuff
}
function getJsonData(url1,url2){
fetch(url1).then(res1 => {
fetch(url2).then(res2 => {
// here you have both responses ready
otherFunc(res1.json(), res2.json());
})
})
}
getJsonData(someUrl1, someOtherUrl2);
Solution 2 using async-await:
async function getData(url = '') {
const response = await fetch(url);
return response.json();
}
function otherFunc(jsonData1, jsondata2){
//do stuff
}
async function getJsonData(url1,url2){
const res1 = await getData(url1);
const res2 = await getData(url2);
otherFunc(res1, res2);
}
getJsonData(someUrl1, someOtherUrl2);
I would add the two URLs into an array in getJsonData and loop through them storing the results into an array
function getJSONData(url1,url2){
const urlArr = [url1, url2]
const resultsArr = []
for(let x = 0; x < arr.length){
fetch(arr[x])
}
return resultsArr
}
Inspired by this Wait for multiple promises to finish maybe this can work and avoir having any cascade wait.
function otherFunc(jsonData1, jsondata2){
//do stuff
}
function getJsonData(url1,url2){
var json1, json2;
const promise1 = fetch(url1).then(res1 => { json1 = res1.json() });
const promise2 = = fetch(url2).then(res1 => { json2 = res1.json() });
Promise.all([promise1, promise2]).then((values) => {
otherFunc(values[0], values[1]);
//Or if order of parmeter matter
otherFunc(json1, json2);
});
}
getJsonData(someUrl1, someOtherUrl2);
I know that old school for loop works in the traditional way - that it waits for the await to finish getting results.
But in my use case, I need to read a file from local/s3 and process it line by line, and for each line I need to call an External API.
Generally I use await inside the loop because all are running inside a lambda and I don't want to use all memory for running it parallelly.
Here I am reading the file using a stream.on() method, and in order to use await inside that, I need to add async in read method, like so:
stream.on('data',async () =>{
while(data=stream.read()!==null){
console.log('line');
const requests = getRequests(); // sync code,no pblms
for(let i=0;i<requests.length;i++){
const result = await apiCall(request[i);
console.log('result from api')
const finalResult = await anotherapiCall(result.data);
}
}
});
This is working but order in which the lines are processed is not guaranteed. I need all in a sync manner. Any help?
Complete Code
async function processSOIFileLocal (options, params) {
console.log('Process SOI file');
const readStream = byline.createStream(fs.createReadStream(key));
readStream.setEncoding('utf8');
const pattern = /^UHL\s|^UTL\s/;
const regExp = new RegExp(pattern);
readStream.on('readable', () => {
let line;
while (null !== (line = readStream.read())) {
if (!regExp.test(line.toString())) {
totalRecordsCount++;
dataObject = soiParser(line);
const { id } = dataObject;
const XMLRequests = createLoSTRequestXML(
options,
{ mapping: event.mapping, row: dataObject }
);
console.log('Read line');
console.log(id);
try {
for (let i = 0;i < XMLRequests.length;i++) {
totalRequestsCount++;
console.log('Sending request');
const response = await sendLoSTRequest(
options,
{ data: XMLRequests[i],
url: LOST_URL }
);
console.log("got response");
const responseObj = await xml2js.
parseStringPromise(response.data);
if (Object.keys(responseObj).indexOf('errors') !== -1) {
fs.writeFileSync(`${ERR_DIR}/${generateKey()}-${id}.xml`, response.data);
failedRequestsCount++;
} else {
successRequestsCount++;
console.log('Response from the Lost Server');
console.log(response[i].data);
}
}
} catch (err) {
console.log(err);
}
}
}
})
.on('end', () => {
console.log('file processed');
console.log(`
************************************************
Total Records Processed:${totalRecordsCount}
Total Requests Sent: ${totalRequestsCount}
Success Requests: ${successRequestsCount}
Failed Requests: ${failedRequestsCount}
************************************************
`);
});
}
async function sendLoSTRequest (options, params) {
const { axios } = options;
const { url, data } = params;
if (url) {
return axios.post(url, data);
// eslint-disable-next-line no-else-return
} else {
console.log('URL is not found');
return null;
}
}
Code needs to flow like so:
read a line in a sync way
process the line and transform the line into an array of two members
for every member call API and do stuff
once line is complete, look for another line, all done in order
UPDATE: Got a workaround..but it fires stream.end() without waiting stream to finish read
async function processSOIFileLocal (options, params) {
console.log('Process SOI file');
const { ERR_DIR, fs, xml2js, LOST_URL, byline, event } = options;
const { key } = params;
const responseObject = {};
let totalRecordsCount = 0;
let totalRequestsCount = 0;
let failedRequestsCount = 0;
let successRequestsCount = 0;
let dataObject = {};
const queue = (() => {
let q = Promise.resolve();
return fn => (q = q.then(fn));
})();
const readStream = byline.createStream(fs.createReadStream(key));
readStream.setEncoding('utf8');
const pattern = /^UHL\s|^UTL\s/;
const regExp = new RegExp(pattern);
readStream.on('readable', () => {
let line;
while (null !== (line = readStream.read())) {
if (!regExp.test(line.toString())) {
totalRecordsCount++;
dataObject = soiParser(line);
const { id } = dataObject;
const XMLRequests = createLoSTRequestXML(
options,
{ mapping: event.mapping, row: dataObject }
);
// eslint-disable-next-line no-loop-func
queue(async () => {
try {
for (let i = 0;i < XMLRequests.length;i++) {
console.log('Sending request');
console.log(id);
totalRequestsCount++;
const response = await sendLoSTRequest(
options,
{ data: XMLRequests[i],
url: LOST_URL }
);
console.log('got response');
const responseObj = await xml2js.
parseStringPromise(response.data);
if (Object.keys(responseObj).indexOf('errors') !== -1) {
// console.log('Response have the error:');
// await handleError(options, { err: responseObj, id });
failedRequestsCount++;
fs.writeFileSync(`${ERR_DIR}/${generateKey()}-${id}.xml`, response.data);
} else {
console.log('Response from the Lost Server');
console.log(response[i].data);
successRequestsCount++;
}
}
} catch (err) {
console.log(err);
}
});
}
}
})
.on('end', () => {
console.log('file processed');
console.log(`
************************************************
Total Records Processed:${totalRecordsCount}
Total Requests Sent: ${totalRequestsCount}
Success Requests: ${successRequestsCount}
Failed Requests: ${failedRequestsCount}
************************************************
`);
Object.assign(responseObject, {
failedRequestsCount,
successRequestsCount,
totalRecordsCount,
totalRequestsCount
});
});
}
Thank You
The sample code at the top of your question could be rewritten like
const queue = (() => {
let q = Promise.resolve();
return (fn) => (q = q.then(fn));
})();
stream.on('data', async() => {
while (data = stream.read() !== null) {
console.log('line');
const requests = getRequests(); // sync code,no pblms
queue(async () => {
for (let i = 0; i < requests.length; i++) {
const result = await apiCall(request[i]);
console.log('result from api');
const finalResult = await anotherapiCall(result.data);
}
});
}
});
Hopefully that will be useful for the complete code
If anyone want a solution for synchronisely process the file, ie, linebyline read and execute some Async call, it's recommended to use inbuilt stream transform. There we can create a transform function and return a callback when finishes.
That's will help of any one face this issues.
Through2 is a small npm library that also can be used for the same.