So i have just started to learn javascript today and i'm trying to fetch text from a url and then split and stringify it using josn.
Then i'm trying to pick a random proxy from the stringify json and log it in my self invoking function but it logs as undefined and i can't workout what i'm doing wrong and i was hoping maybe someone could tell me what i'm doing wrong.
My code:
const doFetch = async () => {
try {
let proxies = [];
let socks = await fetch("https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt");
let response = await socks.text();
let list = response.toString().split('\n');
for (let i = 0; i < list.length; i++) {
let splitText = list[i].split(':');
proxies.push(JSON.stringify({'type': 'socks4', 'ip': splitText[0], 'port': splitText[1]}));
}
return proxies
} catch (error) {
console.log('fetch error:', error.message);
}
}
(async () => {
let proxies = await doFetch();
let proxie = proxies[Math.floor(Math.random() * proxies.length)];
console.log(proxie); // logs fine
console.log(proxie.type, proxie.ip, proxie.port); // logs as undefined
})();
proxie.type, proxie.ip, proxie.port are undefined because proxie is a string and not an object. Rewrite your code in this way:
(async () => {
let proxies = await doFetch();
let proxie = proxies[Math.floor(Math.random() * proxies.length)];
console.log(proxie); // logs fine
let proxieObj = JSON.parse(proxie);
console.log(proxieObj.type, proxieObj.ip, proxieObj.port);
})();
Related
I am fairly new to Javascript and I understand that it executes asynchronously. I tried using the callback method to fetch the secret values and then execute next block of code. But it is not waiting.
This is the function that fetches the keyvault secret values
function getsecret_values(client,secret_name,callback) {
let val = []
for (let i =0;i<secret_name.length;i++){
client.getSecret(secret_name[i]).then((latestSecret) => {
val[i] = latestSecret.value;
})
}
callback(val)
}
I am calling getsecret_values function from main block
let vaultName = result.database;
const url = `https://${vaultName}.vault.azure.net`;
const credential = new ClientSecretCredential(result.host, result.user, result.password);
const client = new SecretClient(url, credential);
let secret_values = []
getsecret_values(client, secrets, function(result) {
secret_values = result
console.log(secret_values)
});
console.log(secret_values)
\\next code block
Both the console.log returns empty array.
I want my code to wait till the secret values are fetched and put into secret_values array and then proceed to next block of code. How do I achieve this?
the easiest way is to use Async Await pattern, which uses promises in the background. Trying not to change your code much:
async function getsecret_values(client,secret_name) {
let val = []
for (let i =0;i<secret_name.length;i++){
const latestSecret = await client.getSecret(secret_name[i])
val[i] = latestSecret.value;
}
return val
}
in your main block:
getsecret_values(client, secrets).then(function(result) {
secret_values = result
console.log(secret_values)
});
console.log(secret_values) // will still be an empty array as the then function has not been executed yet....
my approach would be:
async function getsecret_values(client,secret_name) {
let val = []
for (let i =0;i<secret_name.length;i++){
const latestSecret = await client.getSecret(secret_name[i])
val[i] = latestSecret.value;
}
return val
}
// main:
async function main() {
let vaultName = result.database;
const url = `https://${vaultName}.vault.azure.net`;
const credential = new ClientSecretCredential(result.host, result.user, result.password);
const client = new SecretClient(url, credential);
const secret_values = await getsecret_values(client, secrets)
console.log(secret_values)
}
main()
I have a CAR file object in javascript and want to read it using js-car github. But I keep getting unexpected end of the file error. Here is my code I am trying
let arrayBuffer = await files[0].arrayBuffer();
let bytes=new Uint8Array(carFile);
const reader = await CarReader.fromBytes(bytes) //throws error here
const indexer = await CarIndexer.fromBytes(bytes) //throws error here
I also tired this
let str = await files[0].stream()
const reader = await CarReader.fromIterable(files[0].stream()) //throws error here
and none of them work. However with the same file this code works
const inStream = fs.createReadStream('test.car')
const reader = await CarReader.fromIterable(inStream)
I checked and I know that CarReader.fromBytes needs a Unit8Arrey and I am sure files[0] is not null. Does anyone knows what I am missing here?
for the people might face similar issue in future this is my solution:
I used res.body directly and converted it to an async stream and read it using fromIterable
async function* streamAsyncIterator(stream) {
// Get a lock on the stream
const reader = stream.getReader();
try {
while (true) {
// Read from the stream
const { done, value } = await reader.read();
// Exit if we're done
if (done) return;
// Else yield the chunk
yield value;
}
}
finally {
reader.releaseLock();
}
}
const info = await w3StorageClient.status(response)
if (info) {
// Fetch and verify files from web3.storage
const res = await w3StorageClient.get(response);
const reader = await CarReader.fromIterable(streamAsyncIterator(res.body))
// read the list of roots from the header
const roots = await reader.getRoots()
// retrieve a block, as a { cid:CID, bytes:UInt8Array } pair from the archive
const got = await reader.get(roots[0])
// also possible: for await (const { cid, bytes } of CarIterator.fromIterable(inStream)) { ... }
let decoded = cbor.decode(got.bytes)
console.log('Retrieved [%s] from example.car with CID [%s]',
decoded,
roots[0].toString())
}
I'm trying to fetch data one by one in order using async await and push the response.json to a array. However, the code I use below does not display the result in console.log(b);. Any one know what's the issue?
Thank you in advance!
async function fetch_data(){
let b = [];
for (let i = 1; i < 10; i++) {
let response = await fetch('https://SOME_URL/' + i, {
method: "GET",
headers: {"Key": "123456"}})
if (!response.ok) {
var error_detail = `An error has occured: ${response.status}`;
throw new Error(error_detail);
}
var data = await response.json();
// await b.push(data);
b.push(data);
}
// await Promise.all(b).then(function (b) {
// console.log(b)})
console.log(b);
return b;
}
When I run the script, it does not return anything in the console
fetch_data().catch(error => {
error.error_detail;
});
UPDATE: seems solved using my answer below. not sure why though.
The issue in which console.log(b); not displaying the output is because for example 'https://SOME_URL/' + i as i increases, IF when i = 5 and it returns error, then the console.log(b); will not return anything. If I set the i < 5, then console.log(b); will return the output b. So which means if any fetch in the loop returns error within the limit of i, then the push will not work (b will be empty) and hence console.log(b); will not return anything.
Anyone have idea how to solve this?
You need to rename your function. The error you get is Maximum call stack size exceeded since when you call fetch inside the loop you actually call the outer function, and not the function you want.
I updated several places
One of the issues would be the function name: "fetch" as same as the default fetch function name.
const fetchData = async(symbol) => {
try {
let promiseGroup = [];
const subFetch = async(i) => {
const BASE_URL = `https://reqres.in/api/products`
return fetch(`${BASE_URL}/${i}`, {
method: "GET",
headers: {
"Key": "123456"
}
}).then(res => res.json())
}
for (let i = 1; i < 10; i++) {
promiseGroup.push(i);
}
const results = await Promise.all(promiseGroup.map(subFetch))
console.log(results)
return result;
} catch (error) {
const error_detail = `An error has occured: ${error}`;
throw new Error(error_detail);
}
}
console.log(fetchData())
Here is a working version (at least its working for me):
Basically I just removed the header and indented it a bit, so your example should work as well. Maybe you have some issue on the backend ?
async function fetch_data() {
let b = [];
for (let i = 1; i < 10; i++) {
let response = await fetch('https://jsonplaceholder.typicode.com/todos/' + i, {
method: "GET",
});
if (!response.ok) {
var error_detail = `An error has occured: ${response.status}`;
throw new Error(error_detail);
}
var data = await response.json();
// await b.push(data);
b.push(data);
}
console.log(b);
return b;
}
fetch_data()
.then(data => console.log(data.length))
.catch(err => console.error(err));
Changing throw new Error(error_detail); to break seems solved the issue and able to display the result.
So I have two simple functions, the first function makes an api call and retrieves 100 category ids and stores them in an array. I use lodash to randomly pick 6 of these category ids. The second function is suppose to make use of these 6 unique category ids and use them in the query string for the next 6 api calls made in the second function.
async function getCategoryIds() {
const res = await axios.get('http://jservice.io//api/categories?count=100');
for (let cat of res.data) {
categories.push(cat.id)
}
var sampleCategories = _.sampleSize(categories, 6);
console.log(sampleCategories);
return sampleCategories;
}
getCategoryIds()
.then(getCategory)
async function getCategory(sampleCategories) {
const res1 = await axios.get(`http://jservice.io/api/category?id=${sampleCategories[0]}`);
const res2 = await axios.get(`http://jservice.io/api/category?id=${sampleCategories[1]}`);
const res3 = await axios.get(`http://jservice.io/api/category?id=${sampleCategories[2]}`);
const res4 = await axios.get(`http://jservice.io/api/category?id=${sampleCategories[3]}`);
const res5 = await axios.get(`http://jservice.io/api/category?id=${sampleCategories[4]}`);
const res6 = await axios.get(`http://jservice.io/api/category?id=${sampleCategories[5]}`);
}
getCategory();
However, no matter how I rework it I still cannot get this error to go away:
Uncaught (in promise) TypeError: Cannot read property '0' of undefined
Could somebody steer my in the right direction?
Thank you.
if your backend is exactly sending response an exact array then you should
dont forget to give args when u r calling getCategory function
then edit your getCategory function
async function getCategory(sampleCategories) {
let arr = []
const res1 = await axios.get('any url you want')
//repeat 6 times
arr = [res1, res2, ...otherElems]
return arr
}
with 'then' syntax
getCategoryIds()
.then(response => getCategory(response))
with async await syntax
const firstResponseArr = await getCategoryIds();
const secondResponseArr = await getCategory(firstResponseArr);
Your Mistake
Calling getCategory(); with passing any argument. Clearly async function getCategory(sampleCategories) needs an argument - sampleCategories which you failed to pass.
Error intepretation
Uncaught (in promise) TypeError: Cannot read property '0' of undefined # sampleCategories
Call getCategory() with an argument
This has nothing to do with axios but with your careless mistake. (Don't worry, it happens to even the best of us)
Alright so what I was struggling with was trying to figure out how to properly utilize async/await -- this seemed to work for me:
let categories = []
var sampleCategories = []
async function getCategoryIds() {
const res = await axios.get('http://jservice.io//api/categories?count=100');
for (let cat of res.data) {
categories.push(cat.id)
}
var sampleCategories = _.sampleSize(categories, 6);
return sampleCategories;
}
getCategoryIds()
async function getCategory() {
var y = await getCategoryIds();
const res1 = await axios.get(`http://jservice.io/api/category?id=${y[0]}`);
const res2 = await axios.get(`http://jservice.io/api/category?id=${y[1]}`);
const res3 = await axios.get(`http://jservice.io/api/category?id=${y[2]}`);
const res4 = await axios.get(`http://jservice.io/api/category?id=${y[3]}`);
const res5 = await axios.get(`http://jservice.io/api/category?id=${y[4]}`);
const res6 = await axios.get(`http://jservice.io/api/category?id=${y[5]}`);
let arr = [res1, res2, res3, res4, res5, res6]
return arr
}
getCategory();
I am trying to scrap wikipedia page to fetch list of airlines by first scrapping first page and then going to each individual page of airline to get the website url. I have divided the code in two functions. One to scrap main page and get a new url, and second function to scrap another page from the created url to get the website name from that page. I have used request-promise module for getting the html and then cheerio to parse the data.
export async function getAirlinesWebsites(req,res) {
let response = await request(options_mainpage);
console.log(`Data`);
let $ = cheerio.load(response);
console.log('Response got');
$('tr').each((i,e)=>{
let children = '';
console.log('inside function ', i);
if($(e).children('td').children('a').attr('class') !== 'new') {
children = $(e).children('td').children('a').attr('href');
let wiki_url = 'https://en.wikipedia.org' + children;
console.log(`wiki_url = ${wiki_url}`);
let airline_url = getAirlineUrl(wiki_url);
console.log(`airline_url = ${airline_url}`);
}
})
And then the getAirlineUrl() function will parse another page based on the provided url.
async function getAirlineUrl(url){
const wiki_child_options = {
url : url,
headers : headers
}
let child_response = await request(wiki_child_options);
let $ = cheerio.load(child_response);
let answer = $('.infobox.vcard').children('tbody').children('tr').children('td').children('span.url').text();
return answer;
})
However when I console log the answer variable in the parent function, I get a [object Promise] value instead of a String. How do I resolve this issue?
Async function return promise.In case of that,you need to use then to get resolved response or use await.
This should work if other part of your code is ok.
export async function getAirlinesWebsites(req, res) {
let response = await request(options_mainpage);
console.log(`Data`);
let $ = cheerio.load(response);
console.log("Response got");
$("tr").each(async (i, e) => {
let children = "";
console.log("inside function ", i);
if ($(e).children("td").children("a").attr("class") !== "new") {
children = $(e).children("td").children("a").attr("href");
let wiki_url = "https://en.wikipedia.org" + children;
console.log(`wiki_url = ${wiki_url}`);
let airline_url = await getAirlineUrl(wiki_url);
console.log(`airline_url = ${airline_url}`);
}
});
}
Since your getAirlineUrl function returns a promise, you need to await that promise. You can't have await nested inside of the .each callback because the callback is not an async function, and if it was it wouldn't work still. The best fix is the avoid using .each and just use a loop.
export async function getAirlinesWebsites(req,res) {
let response = await request(options_mainpage);
console.log(`Data`);
let $ = cheerio.load(response);
console.log('Response got');
for (const [i, e] of Array.from($('tr')).entries()) {
let children = '';
console.log('inside function ', i);
if($(e).children('td').children('a').attr('class') !== 'new') {
children = $(e).children('td').children('a').attr('href');
let wiki_url = 'https://en.wikipedia.org' + children;
console.log(`wiki_url = ${wiki_url}`);
let airline_url = await getAirlineUrl(wiki_url);
console.log(`airline_url = ${airline_url}`);
}
}
}