When calling the following method:
getLyrics: async function(song) {
const body = await this.getSongBody(song);
const lyrics = await cheerio.text(body('.lyrics'));
return lyrics;
}
as such:
genius.getLyrics('What a wonderful')
.then((res) => console.log(res))
.catch((err) => console.log(err.message));
Everything works fine and the lyrics of "What a wonderful world" by Louise Armstrong pops up in the console.
However, when I run the same code but without "await" in front of "cheerio.text..." sometimes the lyrics are produced and other times "undefined" shows up in the console. What has been making me scratch my head for a while now is that "cheerio.text..." does not return a promise (albeit "getSongBody" does), so to my understanding, there is no need to "wait" for it to finish.
I'm clearly missing something about async/await but have no idea what. Any help would be greatly appreciated!
Thanks
EDIT: Added a reproducible example as requested below:
const fetch = require('node-fetch');
const cheerio = require('cheerio');
// API
function geniusApi(token) {
this._token = token;
this._auth = {'Authorization': 'Bearer ' + this._token};
};
geniusApi.prototype = {
getSongURL : async function(search_keyword){
const res = await fetch('https://api.genius.com/search?q=' +
search_keyword,{headers: this._auth});
const body = await res.text();
const body_parsed = JSON.parse(body);
if (body_parsed.response.hits.length == 0){
console.log('No such song found');
throw Error('No such song found');
}
const url = body_parsed.response.hits[0].result.url;
return url;
},
getSongBody: async function (song){
const url = await this.getSongURL(song);
const response = await fetch(url);
const body = await response.text();
const body_parsed = cheerio.load(body);
return body_parsed;
},
getLyrics: async function(song) {
const body = await this.getSongBody(song);
const lyrics = cheerio.text(body('.lyrics'));
return lyrics;
}
}
// TEST EXAMPLE
const token =
'OTh1EYlsNdO1kELVwcevqLPtsgq3FrxfShIXg_w0EaEd8CHZrJWbWvN8Be773Cyr';
const genius = new geniusApi(token);
genius.getLyrics('What a wonderful')
.then((res) => console.log(res))
.catch((err) => console.log(err.message));
For anyone who ever stumbles upon the same issue, the problem in this case had nothing to do with async, promise or any other JS feature. It was merely a coincidence that the code had functioned correctly while using async, it later turned out that it didn't always work with async either.
The reason was simply that the Genius API that I was using to fetch the data, would return different source codes for identical API queries.
Two different source codes were returned, one contained a div called "lyrics" while the other did not. Therefore, sometimes the lyrics were found using cheerio, other times, they were not.
Related
I have a series of API calls I need to make in order to render a grid of image tiles for selection by the user. Right now it takes 3-5 seconds for the page to load and I think it's because I've accidentally added some extra loops, but I'm struggling to discern where the wasted flops are. This is technically a question about NFT data, but the problem is algorithmic not crypto related.
The call sequence is:
Call "Wallet" API to get all assets associated with an address - API doc
On success, call "Asset Metadata" API to get further info about each asset API Doc
Loop step 2 until all assets have a metadata response
This is my code that works (unless there is no assets associated with a wallet), but is just very slow. I'm sure there is a better way to handle this, but I'm struggling to see how. Thanks for your time!
// API Request
var myHeaders = new Headers();
myHeaders.append("X-API-Key", CENTER_API_KEY); //API Key in constants file
var requestOptions = {
method: 'GET',
headers: myHeaders,
redirect: 'follow'
};
const [nftData, updatenftData] = useState();
const [apiState, updateapiState] = useState("init");
const [renderNFT, updaterenderNFT] = useState([]);
useEffect(() => {
const getData = async () => {
let resp = await fetch(walletAPICall, requestOptions);
let json = await resp.json()
updatenftData(json.items);
updateapiState("walletSuccess");
}
const getRender = async () => {
let nftTemp = [];
for (let i=0;i<nftData.length;i++) {
let tempAddress = nftData[i].address;
let tempTokenId = nftData[i].tokenId;
let resp = await fetch(`https://api.center.dev/v1/ethereum-mainnet/${tempAddress}/${tempTokenId}`, requestOptions)
let json = await resp.json()
// console.log(json);
nftTemp.push(json);
}
updaterenderNFT(nftTemp);
updateapiState("NftDataSuccess");
}
if (apiState=="init") {
getData();
}
else if (apiState=="walletSuccess") {
getRender();
}
}, [requestOptions]);
getRender fetches data items sequentially.
You should do it in parallel using Promise.all or Promise.allSettled
Something like this...
function fetchItem(item) {
const res = await fetch(item.url);
return res.json();
}
await Promise.all[...data.map(fetchItem)]
I am having trouble figuring out the problem to an issue where when I write data (create, update, delete) then write a query to get the data after, the data that I receive back is the data prior to the write.
For example:
Let's say I have two functions createApple() and getAppleById. I have a utility function called getConnection() that gets a connection from a pool to be used for transactional queries. I have an endpoint that creates an apple and I get back to the insertId from mysql then I use that to get the apple but when I return it as a response, I get an empty object.
const createApple = async ({ type }) => {
const connection = await getConnection();
await connection.beginTransaction();
return await connection.query(`INSERT INTO apple (type) VALUES (?)`, [type]);
}
const getAppleById = async (appleId) => {
const connection = await getConnection();
return await connection.query(`SELECT * FROM apple WHERE id = ?`, [appleId]);
}
router.post(`/api/apples`, async (req, res) => {
const { insertId: createdAppleId } = await createApple({ ...req.body });
const apple = await getAppleById(createdAppleId);
res.status(201).send(apple); // <-- this returns {}
});
I noticed that if I add a console.log() before sending the data back, it does get back the data, for example:
router.post(`/api/apples`, async (req, res) => {
const { insertId: createdAppleId } = await createApple({ ...req.body });
const apple = await getAppleById(createdAppleId);
console.log(apple);
res.status(201).send(apple); // <-- this now returns the newly created apple
});
Any ideas on why this may be happening? Also, is this considered a good way of getting a newly created/updated entity or would it be better to make two separate calls:
First call to create/edit the entity (a POST or PATCH call)
Second call to get the entity (a GET call)
Any help is appreciated!
Thanks!
const createApple = async ({ type }) => {
const connection = await getConnection();
await connection.beginTransaction();
await connection.query(`INSERT INTO apple (type) VALUES (?)`, [type]);
await connection.commit();
}
I think error this function when you use transaction, you should commit or rollback transaction after finish query
This is best practice for me, I hope it useful for you
const createApple = async ({ type }) => {
const connection = await getConnection();
await connection.beginTransaction();
try{
await connection.query(`INSERT INTO apple (type) VALUES (?)`, [type]);
await connection.commit();
}catch{
await connection.rollback()
}
}
I'm trying to send a direct message using slack web api to a user but I think my getSlackUser method which gets all the available users does not complete in time for when I call slackId;
the console.log(slackId) gives undefined meaning it doesn't complete my api call with bolt
how do I ensure getSlackUser method finishes (make it blocking) before it moves on to the rest?
const { WebClient } = require('#slack/web-api');
const { App } = require('#slack/bolt')
const rtm = new RTMClient(process.env.SLACK_OAUTH_TOKEN);
const web = new WebClient(process.env.SLACK_OAUTH_TOKEN);
const app = new App({
token: process.env.SLACK_OAUTH_TOKEN,
signingSecret: process.env.SLACK_SIGNING_SECRET
});
exports.handler = async (event) => {
const slackId = await getSlackUser('example_real_name').id;
console.log(slackId);
await sendSlackMessage(slackId, 'Bot message');
}
sendSlackMessage = async (channel, message) => {
await web.chat.postMessage({
channel: channel,
text: message,
as_user: true
});
}
getSlackUser = async(real_name) => {
const result = await app.client.users.list({
token: process.env.SLACK_OAUTH_TOKEN
});
console.log(result);
return result.members.find((user) => user.real_name == real_name);
}
The problem is precedence on this line:
const slackId = await getSlackUser('example_real_name').id;
Since member access has a higher precedence (evaluated before) than await, it is effectively the same as:
const slackId = await (getSlackUser('example_real_name').id);
getSlackUser returns a Promise object, then its id member is undefined. Await waits for the undefined, which is undefined.
To fix this, make sure that the await is evaluated before the .id:
const slackId = (await getSlackUser('example_real_name')).id;
I once wrote a very simple code to fetch&display data from certain api. Now that that API doesn't work anymore, I wanted to keep the code intact and make it display an error message that the API doesn't work.
async function getInfected() {
const api_url = //API Address that doesn't work anymore//;
const response = await fetch(api_url);
// console.log(response);
if (response.ok){
const data = await response.json();
const todayInfected = data.data[0][3];
const todayDeath = data.data[0][2];
document.getElementById('todayInfected').textContent = todayInfected;
document.getElementById('todayDeath').textContent = todayDeath;
} else {
const errorMessage = "Error! Can't fetch API!";
document.getElementsByClassName('nowp').textContent = errorMessage;
}
}
getInfected();
This code does nothing and console.log(response) also doesn't work. And unfortunately, I have very little understanding of fetch(). Any Ideas?
I came across a problem in a book which I can't seem to figure out. Unfortunately, I don't have a live link for it, so if anyone could help me with my approach to this theoretically, I'd really appreciate it.
The process:
I get from a fetch call an array of string codes (["abcde", "fghij", "klmno", "pqrst"]).
I want to make a call to a link with each string code.
example:
fetch('http://my-url/abcde').then(res => res.json()).then(res => res).catch(error => new Error(`Error: ${error}`)); // result: 12345
fetch('http://my-url/fghij').then(res => res.json()).then(res => res).catch(error => new Error(`Error: ${error}`)); // result: 67891
...etc
Each of the calls is going to give me a number code, as shown.
I need to get the highest number of the 5 and get its afferent string code and make another call with that.
"abcde" => 1234
"fghij" => 5314
"klmno" => 3465
"pqrst" => 7234 <--- winner
fetch('http://my-url/pqrst').then(res => res.json()).then(res => res).catch(error => new Error(`Error: ${error}`));
What I tried:
let codesArr = []; // array of string codes
let promiseArr = []; // array of fetch using each string code in `codesArr`, meant to be used in Promise.all()
let codesObj = {}; // object with string code and its afferent number code gotten from the Promise.all()
fetch('http://my-url/some-code')
.then(res => res.json())
.then(res => codesArr = res) // now `codesArr` is ["abcde", "fghij", "klmno", "pqrst"]
.catch(error => new Error(`Error: ${error}`);
for(let i = 0; i < codesArr.length; i++) {
promiseArr.push(
fetch(`http://my-url/${codesArr[i]}`)
.then(res => res.text())
.then(res => {
codesObj[codesArr[i]] = res;
// This is to get an object from which I can later get the highest number and its string code. Like this:
// codesObj = {
// "abcde": 12345,
// "fghij": 67891
// }
})
.catch(error => new Error(`Error: ${error}`));
// I am trying to make an array with fetch, so that I can use it later in Promise.all()
}
Promise.all(promiseArray) // I wanted this to go through all the fetches inside the `promiseArr` and return all of the results at once.
.then(res => {
for(let i = 0; i < res.length; i++) {
console.log(res[i]);
// this should output the code number for each call (`12345`, `67891`...etc)
// this is where I get lost
}
})
One of the problems with my approach so far seems to be that it makes too many requests and I get 429 error. I sometimes get the number codes all right, but not too often.
Like you already found out the 429 means that you send too many requests:
429 Too Many Requests
The user has sent too many requests in a given amount of time ("rate
limiting").
The response representations SHOULD include details explaining the
condition, and MAY include a Retry-After header indicating how long to
wait before making a new request.
For example:
HTTP/1.1 429 Too Many Requests
Content-Type: text/html
Retry-After: 3600
<html>
<head>
<title>Too Many Requests</title>
</head>
<body>
<h1>Too Many Requests</h1>
<p>I only allow 50 requests per hour to this Web site per
logged in user. Try again soon.</p>
</body>
</html>
Note that this specification does not define how the origin server
identifies the user, nor how it counts requests. For example, an
origin server that is limiting request rates can do so based upon
counts of requests on a per-resource basis, across the entire server,
or even among a set of servers. Likewise, it might identify the user
by its authentication credentials, or a stateful cookie.
Responses with the 429 status code MUST NOT be stored by a cache.
To handle this issue you should reduce the amount of requests made in a set amount of time. You should iterate your codes with a delay, spacing out the request by a few seconds. If not specified in the API documentation or the 429 response, you have to use trial and error approach to find a delay that works. In the example below I've spaced them out 2 seconds (2000 milliseconds).
The can be done by using the setTimeout() to execute some piece of code later, combine this with a Promise to create a sleep function. When iterating the initially returned array, make sure to await sleep(2000) to create a 2 second delay between each iteration.
An example could be:
const fetch = createFetchMock({
"/some-code": ["abcde", "fghij", "klmno", "pqrst"],
"/abcde": 12345,
"/fghij": 67891,
"/klmno": 23456,
"/pqrst": 78912,
});
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
(async function () {
try {
const url = "https://my-url/some-code";
console.log("fetching url", url);
const response = await fetch(url);
const codes = await response.json();
console.log("got", codes);
const codesObj = {};
for (const code of codes) {
await sleep(2000);
const url = `https://my-url/${code}`;
console.log("fetching url", url);
const response = await fetch(url);
const value = await response.json();
console.log("got", value);
codesObj[code] = value;
}
console.log("codesObj =", codesObj);
} catch (error) {
console.error(error);
}
})();
// fetch mocker factory
function createFetchMock(dataByPath = {}) {
const empty = new Blob([], {type: "text/plain"});
const status = {
ok: { status: 200, statusText: "OK" },
notFound: { status: 404, statusText: "Not Found" },
};
const blobByPath = Object.create(null);
for (const path in dataByPath) {
const json = JSON.stringify(dataByPath[path]);
blobByPath[path] = new Blob([json], { type: "application/json" });
}
return function (url) {
const path = new URL(url).pathname;
const response = (path in blobByPath)
? new Response(blobByPath[path], status.ok)
: new Response(empty, status.notFound);
return Promise.resolve(response);
};
}
In this case... You should run and wait each fetch run finish before run new fetch by using async/await
runFetch = async (codesArr) => {
for(let i = 0; i < codesArr.length; i++){
const rawResponse = await fetch(`http://my-url/${codesArr[i]}`);
const codeResponse = rawResponse.json();
console.log(rawResponse);
codesObj[codesArr[i]] = codeResponse;
}
}
hope that help you.