I am using zomato API which sends only 20 names in one api call and I want atleast 60 responses so I thought of calling the same api three times and combining all the responses together.
app.get('/locations/:query', async (req, res) => {
const query = req.params.query;
const data = await zomato.cities({ q: query,count: 1 })
const cityId= await (data[0].id);
const restaurants1 = await zomato.search({ entity_id: cityId, entity_type: 'city', start:0, count:20, sort:'rating', order:'desc' })
const restaurants2 = await zomato.search({ entity_id: cityId, entity_type: 'city', start:20, count:40, sort:'rating', order:'desc' })
const restaurants = Object.assign(restaurants1,...restaurants2);
res.send(restaurants);
})
As of now I have only tried till 40 but even this does not work. If another constant restaurant3 is there which has start 40 and end 60, how do I merge these three and send them back?
I haven't worked with zomato-api before but according to their API-documentation, you should be able to do this:
app.get('/locations/:query', async (req, res) => {
const query = req.params.query;
const data = await zomato.cities({ q: query,count: 1 })
const cityId= data[0].id;
const result = [];
const nrOfRequests = 2; // change this accordingly to increase the nr of requests
let currCount = 0;
const nrOfEntries = 20;
for(let i=0; i < nrOfRequests ; i++) {
const response = await zomato.search({ entity_id: cityId, entity_type: 'city', start:currCount, count:nrOfEntries, sort:'rating', order:'desc' });
result.push(...response.restaurants);
currCount += nrOfEntries;
}
res.send(result);
});
Basically you should be able to loop for the desired amount of requests by updating the start parameter for each iteration and then storing the resulting restaurants in your result.
You shouldn't be spreading restaurants2 in your Object.assign(). Use one of the following:
const restaurants = Object.assign(restaurants1, restaurants2);
// or
const restaurants = { ...restaurants1, ...restaurants2 };
Related
I have this code:
const fetchPokemonData = async () => {
const data = await fetch('https://pokeapi.co/api/v2/pokemon?limit=151', options)
let pokemons = []
await data.json().then(pokemon => pokemons.push(pokemon.results))
return pokemons}
const getPokemonsUrl = async () => {
const pokemonsUrls = []
const pokemonData = await fetchPokemonData()
pokemonData[0].map(pokemons => pokemonsUrls.push(pokemons.url))
return pokemonsUrls}
const createPokemonObject = async () => {
const urls = await getPokemonsUrl()
const arrayOfPokemons = []
urls.map(async url =>{
const data = await fetch(url)
const pokemon = await data.json()
const { name, id, sprites: {other: {dream_world: {front_default}}}, types, weight, stats } = pokemon
arrayOfPokemons.push(
{
name: name,
id: id,
image: front_default, types: types,
weight: weight,
stats: stats
}
)
})
console.log(arrayOfPokemons) //works
arrayOfPokemons.map(pokemon => console.log(pokemon)) // works only after setTimeout() delay }
The problem is when I try to log each pokemon outside urls.map() array function, there is no each individual Pokemon, because the data isn`t fetched yet (I tested that put by putting arrayOfPokemons inside setTimeout() function and after some delay time, each Pokemon was shown). Can someone please rewrite or explain the way to rewrite this code do that I get all individual pokemons outside of urls.map() function, because that is the best way for me to learn.
There are several issues:
push returns the length of the array, not data. Instead of using push, really map the data with a .map and capture the returned array.
.map(async will execute the async callbacks immediately and continue. There is no waiting for those callbacks to terminate their asynchronous code. Either use a normal for loop or use await Promise.all
Here is a correction of your code:
const fetchPokemonData = async () => {
const data = await fetch('https://pokeapi.co/api/v2/pokemon?limit=151');
const pokemon = await data.json();
return pokemon.results;
}
const getPokemonsUrl = async () => {
const pokemons = await fetchPokemonData();
return pokemons.map(pokemon => pokemon.url);
}
const createPokemonObject = async () => {
console.log("wait for it...");
const urls = await getPokemonsUrl();
const arrayOfPokemons = await Promise.all(urls.map(async url => {
const data = await fetch(url);
const pokemon = await data.json();
const { name, id, sprites: {other: {dream_world: {front_default}}}, types, weight, stats } = pokemon;
return {
name: name,
id: id,
image: front_default, types: types,
weight: weight,
stats: stats
};
}));
console.log(arrayOfPokemons);
}
createPokemonObject();
newbie here. I was on web scraping project. And I wanted some guide on web scraping pagination technique. I'm scraping this site https://www.imoney.my/unit-trust-investments. As you can see ,I wanted to retrieve different "Total return" percentage based on Xyears. Right now I'm using cheerio and axios.
const http = require("http");
const axios = require("axios");
const cheerio = require("cheerio");
http
.createServer(async function (_, res) {
try {
const response = await axios.get(
"https://www.imoney.my/unit-trust-investments"
);
const $ = cheerio.load(response.data);
const funds = [];
$("[class='list-item']").each((_i, row) => {
const $row = $(row);
const fund = $row.find("[class*='product-title']").find("a").text();
const price = $row.find("[class*='is-narrow product-profit']").find("b").text();
const risk = $row.find("[class*='product-title']").find("[class*='font-xsm extra-info']").text().replace('/10','');;
const totalreturn = $row.find("[class*='product-return']").find("[class='font-lg']").find("b").text().replace('%','');
funds.push({ fund, price, risk, totalreturn});
});
res.statusCode = 200;
res.write(JSON.stringify(funds, null, 4));
} catch (err) {
res.statusCode = 400;
res.write("Unable to process request.");
}
res.end();
})
.listen(8080);
do note, the URL does not change when different year is selected, only the value for total return is changed
This happens because the page uses javascript to generate the content. In this case, you need something like Puppeteer. That's what you need:
const puppeteer = require("puppeteer");
const availableFunds = "10000";
const years = 2; // 3 for 0.5 years; 2 for 1 year; 1 for 2 years, 0 for 3 years.
async function start() {
const browser = await puppeteer.launch({
headless: false,
});
const page = await browser.newPage();
await page.goto("https://www.imoney.my/unit-trust-investments");
await page.waitForSelector(".product-item");
await page.focus("#amount");
for (let i = 0; i < 5; i++) {
await page.keyboard.press("Backspace");
}
await page.type("#amount", availableFunds);
await page.click("#tenure");
for (let i = 0; i < years; i++) {
await page.keyboard.press("ArrowUp");
}
await page.keyboard.press("Enter");
const funds = await page.evaluate(() => {
const funds = [];
Array.from(document.querySelectorAll(".product-item")).forEach((el) => {
const fund = el.querySelector(".title")?.textContent.trim();
const price = el.querySelector(".investmentReturnValue")?.textContent.trim();
const risk = el.querySelector(".col-title .info-desc dd")?.textContent.trim();
const totalreturn = el.querySelector(".col-rate.text-left .info-desc .ir-value")?.textContent.trim();
if (fund && price && risk && totalreturn) funds.push({ fund, price, risk, totalreturn });
});
return funds;
});
console.log(funds);
browser.close();
}
start();
Output:
[
{
fund: 'Aberdeen Standard Islamic World Equity Fund - Class A',
price: 'RM 12,651.20',
risk: 'Medium\n 7/10',
totalreturn: '26.51'
},
{
fund: 'Affin Hwang Select Balanced Fund',
price: 'RM 10,355.52',
risk: 'Medium\n 5/10',
totalreturn: '3.56'
},
... and others
I try to get specific documents from MongoDB with Node.js and insert them into array.
const getStockComments = async (req) => {
const stockname = req.params.stockName;
var comments = [];
var data = [];
const stock = await stockModel.findOne({ name: stockname });
comments = stock.comments;
comments.forEach(async (commentId) => {
const comm = await commentModel.findOne({ _id: commentId });
data.push(comm);
console.log(data); // This returns the data in loops, because its inside a loop.
});
console.log(data); // This not returns the data and i don't know why.
return data;
};
The first console.log(data) returns the same data a lot of times because its inside a loop.
But the second console.log(data) dosen't returns the data at all.
What I'm doing wrong?
Instead of using loop , you can use $in operator to simplify things .
const getStockComments = async (req) => {
const stockname = req.params.stockName;
var comments = [];
var data = [];
const stock = await stockModel.findOne({ name: stockname });
comments = stock.comments;
commentModel.find({ _id: { $in: comments } }, (err, comments) => {
data = comments;
});
console.log(data);
return data;
};
I hope you are well. I am getting data from one API and sending to Shopify store API. As its working fine but its entering some products as when it iterates in the loop API is busy with index suppose 0,1,2 and then indexes 3,4,...10 bypassed. So , according to me I should delay the foreach loop with 10,15 seconds. Please help me to do this . I tried it many times with SETTimer etc but foreach loop structure is difficult for me as a new person. Please check the below code. Thanks
const request = require('request');
const { json } = require('express');
const { Parser } = require('json2csv');
const fastcsv = require('fast-csv');
//const csv = require('csv-parser');
const fs = require('fs');
const { privateDecrypt } = require('crypto');
const { time } = require('console');
const fields = ['Vendor', 'Price', 'SKU','error'];
const opts = { fields };
const createCsvWriter = require('csv-writer').createObjectCsvWriter;
const csvWriter = createCsvWriter({
path: 'C:/Users/IT City/Desktop/arslan.csv',
header: [
{id: 'Vendor', title: 'Vendor'},
{id: 'Price', title: 'Price'},
{id: 'SKU', title: 'SKU'},
{id: 'error', title: 'error'},
]
});
let new_products = {
product_final: {
Vendor: String,
Price: String,
SKU: String,
Error: String,
}
};
//////////First API from which I am getting the data
const options = {
method: 'GET',
url: 'https://api.dexi.io/runs/f58f7795-c11a-478c-b670-c7ae5df8677b/latest/result',
headers: {
accept: 'application/json',
json: true,
'x-dexiio-access': '9d56e967dfXXXXXXX725e234b311655c96',
'x-dexiio-account': '5e597feb-99axxxxxxxx-37f1315723ab'
}
};
products = ['Vendor','Price','SKU','Error']
let product = {};
request(options, function (error, response, body) {
if (error) throw new Error(error);
pro = JSON.parse(body);
/////Looping through the each Item from the first API and sending data to Shopify. But its entering only 10-12 products
//// As handle is busy to entering the product.
/// I have to delay the foreach loop 10, 15 seconds
pro.rows.forEach(
row => {
for (let z = 0; z < row.length; z++)
{
product[pro.headers[z]] = row[z];
product_final2[pro.headers[z]] = row[z];
}
productssdata.push(product_final2)
products.push(product)
var Price = product.Price;
var SKU = product.SKU;
var Vendor = product.Vendor;
var body_html = "THISFSDFSDFSDFSDFSFSDF";
let new_products = {
product: {
title: Vendor,
body_html: Price,
vendor: Vendor,
product_type: SKU,
tags: Price
}
};
const options = {
method: 'POST',
url:
'https://0abcfsdfsdf4bb6532f3b#amjad.myshopify.com/admin/api/2020-07/products.json',
headers: {
accept: 'application/json',
'apiKey': '07649cABSDCCSD8ffbae7af02',
'password': 'sSDCSDFDF',
body: new_products,
json: true,
};
request(options, function (error, response, body) {
if (error) throw new Error(error);
console.log(body)
});
}
);
}
);
you don't need to use setTimeOut() to delay the loop. thats what async and await are for let me share you an example how to make the forEach loop delay with await!!.
step1 : return a function with promise and use await until it is complete.
const wait =async props => {
return new Promise((reslove,reject) => {
return reslove(Math.random());
})
}
const x = [1,2,3,4]
x.forEach(async number =>{
const num = await wait();
console.log('start')
console.log(num);
console.log('end');
})
Request is deprecated
It can't be used with await anyway, which makes it inconvenient. There used to be another module, request-promise, that as wrapping request and return a Promise, so one could await it, but it's still deprecated.
For these reasons, use Axios or Fetch instead
You can't use await or delay a .forEach() loop, but you can in a for loop.
You think you need to delay the calls, because they are asynchronous, but in reality you should simply await each call. Delaying calls with an arbitrary timeout is a dirty workaround.
In the end, you can do something like :
( async () => {
let options = {
url : "http://......"
};
const response = await axios.get(options); // Add try/catch block around this to manage errors
const pro = response.data;
for( let row of pro.rows) {
options = {
url : "http://some.other.url"
}
const response = await axios.get(options); // Each call will be done one by one and awaited in order
}
})()
setTimeout(pro.rows.forEach((row) => {...}), 10000)
This executes the forEach after 10 seconds.
I'm building an app in Express and using Postgres for my database, Sequelize for ORM.
In my database each Post can have one of the 5 types, 1, 2, 3, 4, 5.
I want to show the amount of all the posts by type.
router.route('/post).get(async (req, res) => {
const postOne = await Post.findAll({
where: {
state: 1
}
});
const postTwo = await Post.findAll({
where: {
state: 2
}
});
res.send({ postOne: postOne.length, postTwo: postTwo.length });
I can write like this for all of the 5 types, but I was wondering if there was any shorter way to do it so that I don't have to write the same code 5 times.
Thanks!
const getPostWithType = (type) => Post.findAll({
where: {
state: type
}
});
Now you can await getPostWithType(1) instead. Or even better:
const postTypes = [1,2,3,4,5];
const allPosts = await Promise.all(postTypes.map(getPostWithType));
// allPosts is now an array with the results. You can map it again to get the lengths
const allPostLengths = allPosts.map(p => p.length);
what about using an array? Like below...
let promiseArray = [];
for (let i = 1; i <= 5; i++) {
let promise = Post.findAll({
where: {
state: i
}
});
promiseArray.push(promise);
}
Promise.all(promiseArray).then(function(response) {
//Do whatever...
});