I m trying to build a search API which takes the location as the argument and returns the latitude and longitude of that location.
I m using http://geocode.xyz to get the details of the location.
For example :
https://geocode.xyz/?locate=Warsaw,Poland&json=1
This will return the required latitude and longitude of that location.
This is the code snippet I have:
const url = 'https://geocode.xyz/?locate=Warsaw,Poland&json=1';
const getData = async url => {
try {
const response = await fetch(url);
const json = await response.json();
console.log(json);
} catch (error) {
console.log(error);
}
};
getData(url);
With this, this is the error I am seeing:
node geocode.js
{success: false, error: { code: '006', message: 'Request Throttled.'}}
I guess the error is because I need to throttle the number of requests that is hitting the API https://geocode.xyz. I m not sure how to use rate limiter with async/await. Any help would be appreciated.
EDIT:
Based on the answers below, I still see 'Request Throttled'
var request = require('request');
const fetch = require("node-fetch");
const requester = {
lastRequest: new Date(),
makeRequest: async function(url) {
// first check when last request was made
var timeSinceLast = (new Date()).getTime() - this.lastRequest.getTime();
if (timeSinceLast < 2000) {
this.lastRequest = new Date(this.lastRequest.getTime() + (2000 - timeSinceLast));
await new Promise(resolve => setTimeout(resolve, 2000-timeSinceLast));
// await new Promise(resolve => setTimeout(resolve, timeSinceLast));
}
const response = await fetch(url);
const json = await response.json();
return json;
}
};
requester.makeRequest('https://geocode.xyz/?locate=Warsaw,Poland&json=1')
.then(console.log)
I m still getting the same error:
{ success: false, error: { code: '006', message: 'Request Throttled.' } }
Is this because of geocode pricing? Is there any way to limit the rate of requests from the user side?
You could make an object that is used for sending all requests out, and that object can keep track of when the last time a request was made. For example
const requester = {
lastRequest: new Date(2000,0,1),
makeRequest: async function (url) {
// first check when last request was made
var timeSinceLast = (new Date()).getTime() - this.lastRequest.getTime();
this.lastRequest = new Date();
if (timeSinceLast < 1000) {
this.lastRequest = new Date(this.lastRequest.getTime() + (1000 - timeSinceLast));
await new Promise(resolve => setTimeout(resolve, 1000-timeSinceLast));
}
// make request here and return result
const response = await fetch(url);
const json = await response.json();
return json;
}
};
https://jsfiddle.net/3k7b0grd/
if you use requester.request, it only lets you make a request once a second, and waits long enough (even for successive requests) to obey that rate limit
Related
I have a series of API calls I need to make in order to render a grid of image tiles for selection by the user. Right now it takes 3-5 seconds for the page to load and I think it's because I've accidentally added some extra loops, but I'm struggling to discern where the wasted flops are. This is technically a question about NFT data, but the problem is algorithmic not crypto related.
The call sequence is:
Call "Wallet" API to get all assets associated with an address - API doc
On success, call "Asset Metadata" API to get further info about each asset API Doc
Loop step 2 until all assets have a metadata response
This is my code that works (unless there is no assets associated with a wallet), but is just very slow. I'm sure there is a better way to handle this, but I'm struggling to see how. Thanks for your time!
// API Request
var myHeaders = new Headers();
myHeaders.append("X-API-Key", CENTER_API_KEY); //API Key in constants file
var requestOptions = {
method: 'GET',
headers: myHeaders,
redirect: 'follow'
};
const [nftData, updatenftData] = useState();
const [apiState, updateapiState] = useState("init");
const [renderNFT, updaterenderNFT] = useState([]);
useEffect(() => {
const getData = async () => {
let resp = await fetch(walletAPICall, requestOptions);
let json = await resp.json()
updatenftData(json.items);
updateapiState("walletSuccess");
}
const getRender = async () => {
let nftTemp = [];
for (let i=0;i<nftData.length;i++) {
let tempAddress = nftData[i].address;
let tempTokenId = nftData[i].tokenId;
let resp = await fetch(`https://api.center.dev/v1/ethereum-mainnet/${tempAddress}/${tempTokenId}`, requestOptions)
let json = await resp.json()
// console.log(json);
nftTemp.push(json);
}
updaterenderNFT(nftTemp);
updateapiState("NftDataSuccess");
}
if (apiState=="init") {
getData();
}
else if (apiState=="walletSuccess") {
getRender();
}
}, [requestOptions]);
getRender fetches data items sequentially.
You should do it in parallel using Promise.all or Promise.allSettled
Something like this...
function fetchItem(item) {
const res = await fetch(item.url);
return res.json();
}
await Promise.all[...data.map(fetchItem)]
I'm trying to send a direct message using slack web api to a user but I think my getSlackUser method which gets all the available users does not complete in time for when I call slackId;
the console.log(slackId) gives undefined meaning it doesn't complete my api call with bolt
how do I ensure getSlackUser method finishes (make it blocking) before it moves on to the rest?
const { WebClient } = require('#slack/web-api');
const { App } = require('#slack/bolt')
const rtm = new RTMClient(process.env.SLACK_OAUTH_TOKEN);
const web = new WebClient(process.env.SLACK_OAUTH_TOKEN);
const app = new App({
token: process.env.SLACK_OAUTH_TOKEN,
signingSecret: process.env.SLACK_SIGNING_SECRET
});
exports.handler = async (event) => {
const slackId = await getSlackUser('example_real_name').id;
console.log(slackId);
await sendSlackMessage(slackId, 'Bot message');
}
sendSlackMessage = async (channel, message) => {
await web.chat.postMessage({
channel: channel,
text: message,
as_user: true
});
}
getSlackUser = async(real_name) => {
const result = await app.client.users.list({
token: process.env.SLACK_OAUTH_TOKEN
});
console.log(result);
return result.members.find((user) => user.real_name == real_name);
}
The problem is precedence on this line:
const slackId = await getSlackUser('example_real_name').id;
Since member access has a higher precedence (evaluated before) than await, it is effectively the same as:
const slackId = await (getSlackUser('example_real_name').id);
getSlackUser returns a Promise object, then its id member is undefined. Await waits for the undefined, which is undefined.
To fix this, make sure that the await is evaluated before the .id:
const slackId = (await getSlackUser('example_real_name')).id;
I once wrote a very simple code to fetch&display data from certain api. Now that that API doesn't work anymore, I wanted to keep the code intact and make it display an error message that the API doesn't work.
async function getInfected() {
const api_url = //API Address that doesn't work anymore//;
const response = await fetch(api_url);
// console.log(response);
if (response.ok){
const data = await response.json();
const todayInfected = data.data[0][3];
const todayDeath = data.data[0][2];
document.getElementById('todayInfected').textContent = todayInfected;
document.getElementById('todayDeath').textContent = todayDeath;
} else {
const errorMessage = "Error! Can't fetch API!";
document.getElementsByClassName('nowp').textContent = errorMessage;
}
}
getInfected();
This code does nothing and console.log(response) also doesn't work. And unfortunately, I have very little understanding of fetch(). Any Ideas?
I came across a problem in a book which I can't seem to figure out. Unfortunately, I don't have a live link for it, so if anyone could help me with my approach to this theoretically, I'd really appreciate it.
The process:
I get from a fetch call an array of string codes (["abcde", "fghij", "klmno", "pqrst"]).
I want to make a call to a link with each string code.
example:
fetch('http://my-url/abcde').then(res => res.json()).then(res => res).catch(error => new Error(`Error: ${error}`)); // result: 12345
fetch('http://my-url/fghij').then(res => res.json()).then(res => res).catch(error => new Error(`Error: ${error}`)); // result: 67891
...etc
Each of the calls is going to give me a number code, as shown.
I need to get the highest number of the 5 and get its afferent string code and make another call with that.
"abcde" => 1234
"fghij" => 5314
"klmno" => 3465
"pqrst" => 7234 <--- winner
fetch('http://my-url/pqrst').then(res => res.json()).then(res => res).catch(error => new Error(`Error: ${error}`));
What I tried:
let codesArr = []; // array of string codes
let promiseArr = []; // array of fetch using each string code in `codesArr`, meant to be used in Promise.all()
let codesObj = {}; // object with string code and its afferent number code gotten from the Promise.all()
fetch('http://my-url/some-code')
.then(res => res.json())
.then(res => codesArr = res) // now `codesArr` is ["abcde", "fghij", "klmno", "pqrst"]
.catch(error => new Error(`Error: ${error}`);
for(let i = 0; i < codesArr.length; i++) {
promiseArr.push(
fetch(`http://my-url/${codesArr[i]}`)
.then(res => res.text())
.then(res => {
codesObj[codesArr[i]] = res;
// This is to get an object from which I can later get the highest number and its string code. Like this:
// codesObj = {
// "abcde": 12345,
// "fghij": 67891
// }
})
.catch(error => new Error(`Error: ${error}`));
// I am trying to make an array with fetch, so that I can use it later in Promise.all()
}
Promise.all(promiseArray) // I wanted this to go through all the fetches inside the `promiseArr` and return all of the results at once.
.then(res => {
for(let i = 0; i < res.length; i++) {
console.log(res[i]);
// this should output the code number for each call (`12345`, `67891`...etc)
// this is where I get lost
}
})
One of the problems with my approach so far seems to be that it makes too many requests and I get 429 error. I sometimes get the number codes all right, but not too often.
Like you already found out the 429 means that you send too many requests:
429 Too Many Requests
The user has sent too many requests in a given amount of time ("rate
limiting").
The response representations SHOULD include details explaining the
condition, and MAY include a Retry-After header indicating how long to
wait before making a new request.
For example:
HTTP/1.1 429 Too Many Requests
Content-Type: text/html
Retry-After: 3600
<html>
<head>
<title>Too Many Requests</title>
</head>
<body>
<h1>Too Many Requests</h1>
<p>I only allow 50 requests per hour to this Web site per
logged in user. Try again soon.</p>
</body>
</html>
Note that this specification does not define how the origin server
identifies the user, nor how it counts requests. For example, an
origin server that is limiting request rates can do so based upon
counts of requests on a per-resource basis, across the entire server,
or even among a set of servers. Likewise, it might identify the user
by its authentication credentials, or a stateful cookie.
Responses with the 429 status code MUST NOT be stored by a cache.
To handle this issue you should reduce the amount of requests made in a set amount of time. You should iterate your codes with a delay, spacing out the request by a few seconds. If not specified in the API documentation or the 429 response, you have to use trial and error approach to find a delay that works. In the example below I've spaced them out 2 seconds (2000 milliseconds).
The can be done by using the setTimeout() to execute some piece of code later, combine this with a Promise to create a sleep function. When iterating the initially returned array, make sure to await sleep(2000) to create a 2 second delay between each iteration.
An example could be:
const fetch = createFetchMock({
"/some-code": ["abcde", "fghij", "klmno", "pqrst"],
"/abcde": 12345,
"/fghij": 67891,
"/klmno": 23456,
"/pqrst": 78912,
});
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
(async function () {
try {
const url = "https://my-url/some-code";
console.log("fetching url", url);
const response = await fetch(url);
const codes = await response.json();
console.log("got", codes);
const codesObj = {};
for (const code of codes) {
await sleep(2000);
const url = `https://my-url/${code}`;
console.log("fetching url", url);
const response = await fetch(url);
const value = await response.json();
console.log("got", value);
codesObj[code] = value;
}
console.log("codesObj =", codesObj);
} catch (error) {
console.error(error);
}
})();
// fetch mocker factory
function createFetchMock(dataByPath = {}) {
const empty = new Blob([], {type: "text/plain"});
const status = {
ok: { status: 200, statusText: "OK" },
notFound: { status: 404, statusText: "Not Found" },
};
const blobByPath = Object.create(null);
for (const path in dataByPath) {
const json = JSON.stringify(dataByPath[path]);
blobByPath[path] = new Blob([json], { type: "application/json" });
}
return function (url) {
const path = new URL(url).pathname;
const response = (path in blobByPath)
? new Response(blobByPath[path], status.ok)
: new Response(empty, status.notFound);
return Promise.resolve(response);
};
}
In this case... You should run and wait each fetch run finish before run new fetch by using async/await
runFetch = async (codesArr) => {
for(let i = 0; i < codesArr.length; i++){
const rawResponse = await fetch(`http://my-url/${codesArr[i]}`);
const codeResponse = rawResponse.json();
console.log(rawResponse);
codesObj[codesArr[i]] = codeResponse;
}
}
hope that help you.
I'm trying to send ethereum transaction that sends ERC20 tokens to someone with Ledger Nano S through Node.JS but I'm not able to successfully sign and send this transaction.
First of all, I signed the transaction through the method, signTransaction, of ledgerhq API and then after signing it, I sended it to the main net by using sendSignedTransaction. When I execute below code, Ledger receives request and shows details of a transaction. However, after pressing Ledger's confirm button, the console returns error 'Returned error: Invalid signature: Crypto error (Invalid EC signature)'.
import AppEth from "#ledgerhq/hw-app-eth";
import TransportU2F from "#ledgerhq/hw-transport-u2f";
import TransportNodeHid from "#ledgerhq/hw-transport-node-hid";
import EthereumTx from "ethereumjs-tx"
const Web3 = require('web3');
import { addHexPrefix, bufferToHex, toBuffer } from 'ethereumjs-util';
const web3 = new Web3(new Web3.providers.HttpProvider('http://localhost:8545'));
var destAddresses = ['0xa6acFa18468786473269Dc1521fd4ff40F6481D9'];
var amount = 1000000000000;
var i=0;
var contract = new web3.eth.Contract([token contract ABI... ], '0x74a...');
const data1 = contract.methods.transfer(destAddresses[0], amount).encodeABI();
const exParams = {
gasLimit: 6e6,
gasPrice: 3e9,
from: '0x1A...',
data : data1,
to: '0x74a...',
value: '0x00',
nonce: "0x0",
chainId: 1,
v: "0x01",
r: "0x00",
s: "0x00"
}
async function makeSign(txParams) {
const tx = new EthereumTx(txParams);
const txHex = tx.serialize().toString("hex");
const signedTransaction = '0x' + txHex;
let transport;
try {
transport = await TransportNodeHid.create();
let eth2 = new AppEth(transport);
const result = await eth2.signTransaction("m/44'/60'/0'/0", txHex).then(result => {
web3.eth.sendSignedTransaction('0x' + txHex)
.then(res => {
console.log(res);
}).catch(err => {
console.log('sendSignedTransaction');
console.log(err);
});
}).catch(err => {
console.log('signTransaction');
console.log(err);
});
txParams.r = `0x${result.r, 'hex'}`;
txParams.s = `0x${result.s, 'hex'}`;
txParams.v = `0x${result.v, 'hex'}`;
return result;
} catch (e) {
console.log(e);
}
}
makeSign(exParams).then(function () {
console.log("Promise Resolved2");
}.catch(function () {
console.log("Promise Rejected2");
});
When I only use signTransaction function, I can confirm the transaction in the ledger device and return txhash on the console. However, ultimately I want to broadcast a transaction to the main net. Could you please give me any idea? I want any feedback. Also, if there are any examples of creating and broadcasting a raw transaction by using the ledger, notice me please.
Your code already sends the transaction to the network. However, just awaiting the "send" promise only gives you the transaction hash, not the receipt. You need to treat it as an event emitter and wait for the 'confirmation' event.
const serializedTx = tx.serialize();
web3.eth.sendSignedTransaction(serializedTx.toString('hex'))
.once('transactionHash', hash => console.log('Tx hash', hash))
.on('confirmation', (confNumber, receipt) => {
console.log(`Confirmation #${confNumber}`, receipt);
})
.on('error', console.error);
To send it to mainnet as you mention, you can either run a local geth node on port 8545 and use your code unchanged, or point web3 at infura or similar.