I write some code to getting info
const stock = await Stock.find({
exchange: exchange
});
// Here stock array length is 5300
stock.forEach(async (stockEl) => {
const EOD_API = process.env.EOD_HISTORICAL_API
const {data} = await axios.get(`https://eodhistoricaldata.com/api/fundamentals/${stockEl.code}?api_token=${EOD_API}&filter=General::Industry`);
console.log(data);
});
Here I place get request for every stock array element by forEach function. Then it give me error like image-
Click to see images
But When I place it outside of forEach function like this-
const EOD_API = process.env.EOD_HISTORICAL_API
const {data} = await axios.get(`https://eodhistoricaldata.com/api/fundamentals/${stockEl.code}?api_token=${EOD_API}&filter=General::Industry`);
console.log(data);
Then it gives no error. For Remembering Stock has 5300 element, that means axios run 5300 times.
Any solution or idea?
You need to make a few changes:
Replace forEach with for because forEach is not promise aware
Use try, catch => catch any errors
Use Promise.allSettled => it allows you to run all promisses together without waiting each other which in return will enhance your app performance. It returns an array with status ("fulfilled", "rejected")
const fetchSingleStockElement = async (stockEl) => {
try {
const EOD_API = process.env.EOD_HISTORICAL_API,
{ data } = await axios(
`https://eodhistoricaldata.com/api/fundamentals/${stockEl.code}?api_token=${EOD_API}&filter=General::Industry`
);
return data;
} catch (err) {
throw new Error(err);
}
};
const fetchAllStockData = async () => {
let promisesArray = [];
try {
//fetch stock array
const { data } = await Stock.find({
exchange: exchange
});
//fetch single stock
for (let i = 0; i < data.length; i++) {
promisesArray.push(fetchSingleStockElement(data[i].id));
}
const results = await Promise.allSettled(promisesArray);
console.log('results', results);
} catch (err) {
console.log('results error', err);
}
};
Here is a working example with fake API of 4466 entries:
const fetchSingleAirline = async (airlineId) => {
try {
const { data } = await axios(`https://api.instantwebtools.net/v1/airlines/${airlineId}`);
return data;
} catch (err) {
throw new Error(err);
}
};
const fetchAllAirlineData = async () => {
let promisesArray = [];
try {
const { data } = await axios('https://api.instantwebtools.net/v1/airlines');
for (let i = 0; i < data.length; i++) {
promisesArray.push(fetchSingleAirline(data[i].id));
}
const results = await Promise.allSettled(promisesArray);
console.log('results', results);
} catch (err) {
console.log('results error', err);
}
};
Doing await in forEach doesn't hold the process since forEach is not promise-aware. Try this instead:
(async () => {
for (let index = 0; index < stock.length; index++) {
const EOD_API = process.env.EOD_HISTORICAL_API
const {data} = await axios.get(`https://eodhistoricaldata.com/api/fundamentals/${stock[i].code}?api_token=${EOD_API}&filter=General::Industry`);
console.log(data);
}
})();
More information.
Related
I have been working with a stock api for the past day or two, and I'm trying to make a search engine for it with a filter method. It works, but it can be slow because the initial api fetch response is an array that is about 28,000 indeces. Is there a way to limit the response I get? I looked in the documentation and there doesn't seem to be a solution through the link itself. Any help would be appreciated!
input.addEventListener('keyup', (e) => {
const searchString = e.target.value.toLowerCase();
const filteredCompanies = stockCompanies.filter((company) => {
return (
company.description.toLowerCase().includes(searchString) ||
company.displaySymbol.toLowerCase().includes(searchString)
);
});
displayCompanies(filteredCompanies);
});
const loadCompanies = async () => {
try {
const res = await fetch(`https://finnhub.io/api/v1/stock/symbol?exchange=US&token=${apiKey}`);
stockCompanies = await res.json();
displayCompanies(stockCompanies);
}catch (err) {
console.error(err);
}
};
const displayCompanies = (data) => {
var dynamic = [];
for (var i = 0; i < data.length; i++){
dynamic.push(`
<div class="listings_card">
<h1>${data[i].description}</h1>
<p>${data[i].displaySymbol}</p>
</div>`);
}
document.querySelector('.listings').innerHTML = dynamic.join('');
};
loadCompanies();
If api doesn't support the feature you can not. But you can eliminate the list after you receive the response. Try this way:
const loadCompanies = async () => {
try {
const res = await fetch(`https://finnhub.io/api/v1/stock/symbol?exchange=US&token=${apiKey}`);
const res_data = await res.json();
stockCompanies = res_data.slice(0, 50)
displayCompanies(stockCompanies);
} catch (err) {
console.error(err);
}
};
When running the following code tidied in functions or not, it still writes to my file incorrectly. One thing that did work was wrapping those functions inside of a setTimeout method, with the seconds somewhere around 10. I just didn't like the idea of hardcoding those values and taking anymore time to complete than it should. What's a better way of going about this? I need help understanding async/await a little more as you can tell but what better way than to fail and ask for help!
genPriceChangeScripts: async () => {
const priceScript = `...`;
const changeData = await get24hrChange();
const globalCmds = [];
const globalPol = [];
const filtered = changeData.filter(function (item) {
return (
!item.symbol.includes("BTCUSDT_") && !item.symbol.includes("ETHUSDT_")
);
});
async function scripts() {
filtered.forEach((e) => {
const data = e.symbol;
const change = priceScript.replace("CHANGE", data);
fs.writeFile(
`../scripts/price_change/${data.toLowerCase()}_price_change.sh`,
change,
function (err) {
if (err) return console.log(err);
}
);
});
console.log("scripts finished");
}
scripts();
async function commands() {
for (let i = 0; i < filtered.length; i++) {
var pushCmds = `"#($CURRENT_DIR/scripts/price_change/${filtered[
i
].symbol.toLowerCase()}_price_change.sh)"`;
globalCmds.push(pushCmds);
}
const commands = globalCmds.join("\n");
const cmdsWithComment = commands.concat("\n#CHANGE3");
fs.readFile("../binance.tmux", "utf-8", (err, data) => {
if (err) {
throw err;
}
const addCmds = data.replace("#CHANGE1", cmdsWithComment);
fs.writeFile("../binance.tmux", addCmds, (err) => {
if (err) {
throw err;
}
});
});
console.log("cmds finished");
}
commands();
async function pols() {
for (let i = 0; i < filtered.length; i++) {
const pushPol = `"\\#{${filtered[
i
].symbol.toLowerCase()}_price_change}"`;
globalPol.push(pushPol);
}
const pol = globalPol.join("\n");
const polWithComment = pol.concat("\n#CHANGE4");
fs.readFile("../binance.tmux", "utf-8", (err, data) => {
if (err) {
throw err;
}
const addPol = data.replace("#CHANGE2", polWithComment);
fs.writeFile("../binance.tmux", addPol, (err) => {
if (err) {
throw err;
}
});
});
console.log("pols finished");
}
pols();
return prompt.end();
},
The issue is that making a function async doesn't make it automatically wait for anything asynchronous going on inside it
async/await is syntax "sugar" for working with Promises, and Promises only
So, if you use the promise version of writeFile/readFile like so
import * as fs from 'fs/promise';
you can write your code as follows
genPriceChangeScripts: async() => {
const priceScript = `...`;
const changeData = await get24hrChange();
const globalCmds = [];
const globalPol = [];
const filtered = changeData.filter(function (item) {
return (!item.symbol.includes("BTCUSDT_") && !item.symbol.includes("ETHUSDT_"));
});
async function scripts() {
const promises = filtered.map((e) => {
const data = e.symbol;
const change = priceScript.replace("CHANGE", data);
return fs.writeFile(`../scripts/price_change/${data.toLowerCase()}_price_change.sh`, change);
});
await Promise.all(promises);
console.log("scripts finished");
}
await scripts();
async function commands() {
for (let i = 0; i < filtered.length; i++) {
var pushCmds = `"#($CURRENT_DIR/scripts/price_change/${filtered[i].symbol.toLowerCase()}_price_change.sh)"`;
globalCmds.push(pushCmds);
}
const commands = globalCmds.join("\n");
const cmdsWithComment = commands.concat("\n#CHANGE3");
const data = await fs.readFile("../binance.tmux", "utf-8");
const addCmds = data.replace("#CHANGE1", cmdsWithComment);
await fs.writeFile("../binance.tmux", addCmds);
console.log("cmds finished");
}
await commands();
async function pols() {
for (let i = 0; i < filtered.length; i++) {
const pushPol = `"\\#{${filtered[i].symbol.toLowerCase()}_price_change}"`;
globalPol.push(pushPol);
}
const pol = globalPol.join("\n");
const polWithComment = pol.concat("\n#CHANGE4");
const data = await fs.readFile("../binance.tmux", "utf-8");
const addPol = data.replace("#CHANGE2", polWithComment);
await fs.writeFile("../binance.tmux", addPol);
console.log("pols finished");
}
await pols();
return prompt.end();
},
You need to await the File System operations in order to wait for the asynchronous functions to return a response before proceeding.
await fs.readFile and await fs.writeFile
See this question for further explanation and examples.
This is in addition to adding await to your other async functions to propagate the promise chain correctly.
Not that I'm saying all your code works, but here are the kind changes I would make to get you in the correct direction:
// code above
function scripts(){
const a = [];
filtered.forEach(e=>{
const data = e.symbol, change = priceScript.replace('CHANGE', data);
a.push(new Promise((resolve, reject)=>{
fs.writeFile(`../scripts/price_change/${data.toLowerCase()}_price_change.sh`,
change,
err=>{
if(err){
reject(err);
}
else{
resolve();
}
}
);
}));
});
return Promise.all(a);
}
await scripts();
// code below
Well... actually I would define the function somewhere else or not use it at all, but I think this is what you need to see.
I know that old school for loop works in the traditional way - that it waits for the await to finish getting results.
But in my use case, I need to read a file from local/s3 and process it line by line, and for each line I need to call an External API.
Generally I use await inside the loop because all are running inside a lambda and I don't want to use all memory for running it parallelly.
Here I am reading the file using a stream.on() method, and in order to use await inside that, I need to add async in read method, like so:
stream.on('data',async () =>{
while(data=stream.read()!==null){
console.log('line');
const requests = getRequests(); // sync code,no pblms
for(let i=0;i<requests.length;i++){
const result = await apiCall(request[i);
console.log('result from api')
const finalResult = await anotherapiCall(result.data);
}
}
});
This is working but order in which the lines are processed is not guaranteed. I need all in a sync manner. Any help?
Complete Code
async function processSOIFileLocal (options, params) {
console.log('Process SOI file');
const readStream = byline.createStream(fs.createReadStream(key));
readStream.setEncoding('utf8');
const pattern = /^UHL\s|^UTL\s/;
const regExp = new RegExp(pattern);
readStream.on('readable', () => {
let line;
while (null !== (line = readStream.read())) {
if (!regExp.test(line.toString())) {
totalRecordsCount++;
dataObject = soiParser(line);
const { id } = dataObject;
const XMLRequests = createLoSTRequestXML(
options,
{ mapping: event.mapping, row: dataObject }
);
console.log('Read line');
console.log(id);
try {
for (let i = 0;i < XMLRequests.length;i++) {
totalRequestsCount++;
console.log('Sending request');
const response = await sendLoSTRequest(
options,
{ data: XMLRequests[i],
url: LOST_URL }
);
console.log("got response");
const responseObj = await xml2js.
parseStringPromise(response.data);
if (Object.keys(responseObj).indexOf('errors') !== -1) {
fs.writeFileSync(`${ERR_DIR}/${generateKey()}-${id}.xml`, response.data);
failedRequestsCount++;
} else {
successRequestsCount++;
console.log('Response from the Lost Server');
console.log(response[i].data);
}
}
} catch (err) {
console.log(err);
}
}
}
})
.on('end', () => {
console.log('file processed');
console.log(`
************************************************
Total Records Processed:${totalRecordsCount}
Total Requests Sent: ${totalRequestsCount}
Success Requests: ${successRequestsCount}
Failed Requests: ${failedRequestsCount}
************************************************
`);
});
}
async function sendLoSTRequest (options, params) {
const { axios } = options;
const { url, data } = params;
if (url) {
return axios.post(url, data);
// eslint-disable-next-line no-else-return
} else {
console.log('URL is not found');
return null;
}
}
Code needs to flow like so:
read a line in a sync way
process the line and transform the line into an array of two members
for every member call API and do stuff
once line is complete, look for another line, all done in order
UPDATE: Got a workaround..but it fires stream.end() without waiting stream to finish read
async function processSOIFileLocal (options, params) {
console.log('Process SOI file');
const { ERR_DIR, fs, xml2js, LOST_URL, byline, event } = options;
const { key } = params;
const responseObject = {};
let totalRecordsCount = 0;
let totalRequestsCount = 0;
let failedRequestsCount = 0;
let successRequestsCount = 0;
let dataObject = {};
const queue = (() => {
let q = Promise.resolve();
return fn => (q = q.then(fn));
})();
const readStream = byline.createStream(fs.createReadStream(key));
readStream.setEncoding('utf8');
const pattern = /^UHL\s|^UTL\s/;
const regExp = new RegExp(pattern);
readStream.on('readable', () => {
let line;
while (null !== (line = readStream.read())) {
if (!regExp.test(line.toString())) {
totalRecordsCount++;
dataObject = soiParser(line);
const { id } = dataObject;
const XMLRequests = createLoSTRequestXML(
options,
{ mapping: event.mapping, row: dataObject }
);
// eslint-disable-next-line no-loop-func
queue(async () => {
try {
for (let i = 0;i < XMLRequests.length;i++) {
console.log('Sending request');
console.log(id);
totalRequestsCount++;
const response = await sendLoSTRequest(
options,
{ data: XMLRequests[i],
url: LOST_URL }
);
console.log('got response');
const responseObj = await xml2js.
parseStringPromise(response.data);
if (Object.keys(responseObj).indexOf('errors') !== -1) {
// console.log('Response have the error:');
// await handleError(options, { err: responseObj, id });
failedRequestsCount++;
fs.writeFileSync(`${ERR_DIR}/${generateKey()}-${id}.xml`, response.data);
} else {
console.log('Response from the Lost Server');
console.log(response[i].data);
successRequestsCount++;
}
}
} catch (err) {
console.log(err);
}
});
}
}
})
.on('end', () => {
console.log('file processed');
console.log(`
************************************************
Total Records Processed:${totalRecordsCount}
Total Requests Sent: ${totalRequestsCount}
Success Requests: ${successRequestsCount}
Failed Requests: ${failedRequestsCount}
************************************************
`);
Object.assign(responseObject, {
failedRequestsCount,
successRequestsCount,
totalRecordsCount,
totalRequestsCount
});
});
}
Thank You
The sample code at the top of your question could be rewritten like
const queue = (() => {
let q = Promise.resolve();
return (fn) => (q = q.then(fn));
})();
stream.on('data', async() => {
while (data = stream.read() !== null) {
console.log('line');
const requests = getRequests(); // sync code,no pblms
queue(async () => {
for (let i = 0; i < requests.length; i++) {
const result = await apiCall(request[i]);
console.log('result from api');
const finalResult = await anotherapiCall(result.data);
}
});
}
});
Hopefully that will be useful for the complete code
If anyone want a solution for synchronisely process the file, ie, linebyline read and execute some Async call, it's recommended to use inbuilt stream transform. There we can create a transform function and return a callback when finishes.
That's will help of any one face this issues.
Through2 is a small npm library that also can be used for the same.
here's my problem in this function I'm trying to return an array of objects. When i do a console.log in the forEach loop as you can see the array is filled but when I try to print it juste before my return statement it's empty.
and here's my full js file :
const Discord = require("discord.js");
const Maths = require("../Utils/Maths");
const Stop = require("../Commands/stop");
const EventMitter = require("events");
const emitter = new EventMitter();
const fs = require("fs");
module.exports.play = async(client, channel, players) => {
players.push(client.guilds.cache.first().members.cache.array().filter(mem => mem.user.username === "user1").map(mem => mem.user)[0]);
players.push(client.guilds.cache.first().members.cache.array().filter(mem => mem.user.username === "users2").map(mem => mem.user)[0]);
try {
await beginMessage(players, channel);
let playersCards = await beginGame(client, players);
console.log(playersCards);
emitter.on("stop", async () => {
await channel.delete();
console.log("jeu fini");
});
}
catch(e) {
console.error(e);
}
};
async function beginMessage(players, channel) {
let message = "que le jeu commence ";
players.forEach(player => {
message += `${player.toString()} `;
});
await channel.send(message);
}
async function beginGame(client, players) {
let playersCards = [];
fs.readFile("Utils/cartes.json", 'utf8', async (err, data) => {
if(err) console.log(err);
let cartes = JSON.parse(data);
cartes = cartes.cartes;
players.forEach(async player => {
let playerCards = await distributeCards(player, cartes);
playersCards.push(playerCards);
//console.log(playersCards);
});
console.log(playersCards);
return playersCards;
});
}
async function distributeCards(player, cartes) {
let playerCards = [];
for(let i = 0; i < 4; i++) {
let carte = cartes[Maths.getRandomNumber(0,12)];
carte.count--;
playerCards.push(carte.name);
}
let dmChannel = await player.createDM();
await dmChannel.send(playerCards);
return playerCards;
}
module.exports.finishGame = function (client) {
emitter.emit("stop");
};
Must be a initialisation mistake or something like that I've been searching in my code but can't find it.
EDIT 3:
Since fs.readFile is not async we need to convert it to a promised based function something like below.
function readFile(path) {
return new Promise((resolve, reject) => {
fs.readFile(path, 'utf8', function (err, data) {
if (err) {
reject(err);
}
resolve(data);
});
});
}
I have updated the codesandbox snippet. Please check and see it that solves your issue.
https://codesandbox.io/s/nodejs-async-foreach-promise-v16w6
EDIT 2:
You need to implement custom forEach that will execute and return values as async. Something like this:
async function asyncForEach(array, callback) {
for (let index = 0; index < array.length; index++) {
await callback(array[index], index, array);
}
}
I have written a codesandbox snippet that might help you.
https://codesandbox.io/s/nodejs-async-foreach-promise-v16w6
EDIT:
Since you are using promise inside the forEach loop, the values are pushed
only after the promise is resolved. Since forEach is completes even before the values are pushed to playersCards you will not be able to see any values.
You can do something like this. I have updated the code. Haven't tested the code, but can help if there is issue.
async fs.readFile("Utils/cartes.json", 'utf8', async (err, data) => {
if(err) console.log(err);
let cartes = JSON.parse(data);
cartes = cartes.cartes;
await players.forEach(async player => {
let playerCardsPromiseValue = await distributeCards(player, cartes);
playersCards.push(playerCardsPromiseValue);
//console.log(playersCards);
});
console.log(playersCards);
return playersCards;
});
I have this request handler on my node server. It has three MongoDB queries, and I want all the results to be returned, before the response is sent.
api.get('/getStats/:productID', (req,res)=>{
let data = {};
let dailySales = [];
let avgProduct = "";
let customers = [];
Sales.find({productID: productID}).then(
sales => {
dailySales = sales;
}
);
Products.find({}).then(
products => {
// Calculate Avg product here
avgProduct = result;
}
);
Customers.find({}).then(
customers => {
customers = customers;
}
);
data = {
dailySales,
avgProduct,
customers
};
res.json(data);
});
But running this returns
data: {
dailySales: [],
avgProduct: "",
customers: []
}
i.e. The Mongo response is returning before the data is run. Please how to I fix. Thank You
wait for all the promises to resolve before sending the actual response
const sales = Sales.find({productID: productID});
const allProducts = Products.find({});
const allCustomers = Customers.find({});
Promise.all([sales, allProducts, allCustomers])
.then(data => res.json(data));
you can try using the Promise.all where you can pass the MongoDB queries as parameter to it ,the promise will be resolved when all the queries return the result in the array
Try using the in-built util.promisify function along with
async-await to get data correctly!
const promisify = require('utils').promisify;
const salesFindOnePromise = promisify(Sales.find);
const productsFindAllPromise = promisify(Products.find);
const customersFindAllPromise = promisify(Customers.find);
findDailySalesByIdAsync = async (productID) => {
try {
return await salesFindOnePromise({ productID: productID });
} catch(err) {
throw new Error('Could not fetch the appropriate sales with productID');
}
}
findProductsAsync = async () => {
try {
return await productsFindAllPromise({});
} catch (err) {
throw new Error('Could not fetch sales!');
}
}
findCustomersAsync = async () => {
try {
return await customersFindAllPromise({});
} catch (err) {
throw new Error('Could not fetch customers!');
}
}
api.get('/getStats/:productID', async (req,res)=>{
try {
const dailySales = await findDailySalesByIdAsync(productID);
const avgProduct = await findProductsAsync();
const customers = await findCustomersAsync();
const data = {
dailySales,
avgProduct,
customers
};
return res.status(200).send(data);
} catch(err) {
console.err(`Failed because: {err}`);
throw new Error('Could not fetch data because of some error!');
}
});