I've been trying to configure the OMDB API to return multiple values (multiple movies) on my webpage
However, I'm not sure how to go about this and how to display more than one movie
Can anyone tell me what I did wrong through this code? I'm still pretty new to coding and jumping through hoops trying to figure it out
fetch("https://www.omdbapi.com/?s=batman&apikey=API-KEY")
.then((response) => {
if (response.ok) {
return response.json();
} else {
throw new Error("NETWORK RESPONSE ERROR");
}
})
.then(data => {
console.log(data);
displaymovie(data);
})
.catch((error) => console.error("FETCH ERROR:", error));
function displaymovie(data) {
const movie = data.Search[1];
const movieDiv = document.getElementById("movie");
const movieName = movie.Title;
const heading = document.createElement("h1");
heading.innerHTML = movieName;
movieDiv.appendChild(heading);
const movieImg = document.createElement("img");
movieImg.src = movie.Poster;
movieDiv.appendChild(movieImg);
const movieYear = document.createElement("year");
movieYear.innerHTML = movie.Year;
movieDiv.appendChild(movieYear);
}
I've tried using multiple variables and changing the inputs, but nothing seems to be working. When I try to use multiple array values, no results show up at all.
Any help would be appreciated
So I'm building a program that scrapes Poshmark webpages and extracts the usernames of each seller on the page!
I want it to go through every page using the 'next' button, but theres 6 buttons all with the same class name...
Heres the link: https://poshmark.com/category/Men-Jackets_&_Coats?sort_by=like_count&all_size=true&my_size=false
(In my google chrome this page has an infinite scroll (hence the scrollToBottom async function i started writing) but i realized inside puppeteer's chrome it has 'next page' buttons.)
The window displays page 1-5 and then the 'next page' button.
The problem is that all of the buttons share the same html class name, so I'm confused on how to differentiate.
const e = require('express');
const puppeteer = require('puppeteer');
const url = "https://poshmark.com/category/Men-Jackets_&_Coats?sort_by=like_count&all_size=true&my_size=false";
let usernames = [];
const initItemArea = async (page) => {
const itemArea = await page.evaluate(() => {
return Array.from(document.querySelectorAll('.tc--g.m--l--1.ellipses')).map(x => x.textContent);
});
}
const pushToArray = async (itemArea, page) => {
itemArea.forEach(function (element) {
//console.log('username: ', $(element).text());
usernames.push(element);
});
};
const scrollToBottom = async (itemArea, page) => {
while (true) {
previousHeight = await page.evaluate('document.body.scrollHeight');
await page.evaluate('window.scrollTo(0, document.body.scrollHeight)');
await page.waitForFunction(`document.body.scrollHeight > ${previousHeight}`);
await new Promise((resolve) => setTimeout(resolve, 1000));
await page.screenshot({path : "ss.png"})
}
};
const gotoNextPage = async (page) => {
await page.waitForSelector(".button.btn.btn--pagination");
const nextButton = await page.evaluate((page) => {
document.querySelector(".button.btn.btn--pagination")
});
await page.click(nextButton);
console.log('Next Page Loading')
};
async function main() {
const client = await puppeteer.launch({
headless: false,
executablePath: "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
});
const page = await client.newPage();
await page.goto(url);
await page.waitForSelector(".tc--g.m--l--1.ellipses");
const itemArea = await page.evaluate(() => {
return Array.from(document.querySelectorAll('.tc--g.m--l--1.ellipses')).map(x => x.textContent);
});
gotoNextPage(page)
};
main();
Currently, my gotoNextPage function doesnt even find the button, so i thought i'd entered the selector wrong...
Then when I went to find the selector, I realized all buttons have the same one anyway...
My html knowledge is basically nonexistent, but I want to finish this project out. All help is very appreciated.
Bonus: my initPageArea function doesn't work when I call as a function like that, so I hardcoded it into main()...
I'll be diving deep into this problem later on, as I've seen it before, but any quick answers / direction would be awesome.
Thanks a lot.
you can try selecting the buttons using their position in the page.
For example, you can select the first button using the following CSS selector:
.button.btn.btn--pagination:nth-child(1)
to select the second button:
.button.btn.btn--pagination:nth-child(2)
Got the idea? :)
you can refactor your gotoNextPage function to use this approach, consider this example:
const gotoNextPage = async (page, buttonIndex) => {
await page.waitForSelector(".button.btn.btn--pagination");
// Select the button using its position in the page
const nextButton = await page.evaluate((buttonIndex) => {
return document.querySelector(`.button.btn.btn--pagination:nth-child(${buttonIndex})`);
}, buttonIndex);
// Click on the button
await page.click(nextButton);
console.log("Next Page Loading");
};
Whenever you're messing with buttons and scroll, it's a good idea to think about where the data is coming from. It's usually being delivered to the front-end via a JSON API, so you might as well try to hit that API directly rather than mess with the DOM.
const url = maxId => `https://poshmark.com/vm-rest/channel_groups/category/channels/category/collections/post?request={%22filters%22:{%22department%22:%22Men%22,%22category_v2%22:%22Jackets_%26_Coats%22,%22inventory_status%22:[%22available%22]},%22sort_by%22:%22like_count%22,%22facets%22:[%22color%22,%22brand%22,%22size%22],%22experience%22:%22all%22,%22sizeSystem%22:%22us%22,%22max_id%22:%22${maxId}%22,%22count%22:%2248%22}&summarize=true&pm_version=226.1.0`;
(async () => {
const usernames = [];
for (let maxId = 1; maxId < 5 /* for testing */; maxId++) {
const response = await fetch(url(maxId)); // Node 18 or install node-fetch
if (!response.ok) {
throw Error(response.statusText);
}
const payload = await response.json();
if (payload.error) {
break;
}
usernames.push(...payload.data.map(e => e.creator_username));
}
console.log(usernames.slice(0, 10));
console.log("usernames.length", usernames.length);
})()
.catch(err => console.error(err));
The response blob has a ton of additional data.
I would add a significant delay between requests if I were to use code like this to avoid rate limiting/blocking.
If you're set on Puppeteer, something like this should work as well, although it's slower and I didn't have time to run to the end of the 5k (or more?) users:
const puppeteer = require("puppeteer"); // ^19.1.0
const url = "Your URL";
let browser;
(async () => {
browser = await puppeteer.launch();
const [page] = await browser.pages();
await page.goto(url, {waitUntil: "domcontentloaded"});
const usernames = [];
const sel = ".tc--g.m--l--1.ellipses";
for (;;) {
try {
await page.waitForSelector(sel);
const users = await page.$$eval(sel, els => {
const text = els.map(e => e.textContent);
els.forEach(el => el.remove());
return text;
});
console.log(users); // optional for debugging
usernames.push(...users);
await page.$$eval(
".btn--pagination",
els => els.find(el => el.textContent.includes("Next")).click()
);
}
catch (err) {
break;
}
}
console.log(usernames);
console.log(usernames.length);
})()
.catch(err => console.error(err))
.finally(() => browser?.close());
I don't think navigations are triggered by the "Next" button, so my strategy for detecting when a page transition has occurred involves destroying the current set of elements after scraping the usernames, then waiting until the next batch shows up. This may seem inelegant, but it's easy to implement and seems reliable, not making assumptions about the usernames themselves.
It's also possible to use Puppeteer and make or intercept API requests, armed with a fresh cookie. This is sort of halfway between the two extremes shown above. For example:
const puppeteer = require("puppeteer");
const url = "Your URL";
let browser;
(async () => {
browser = await puppeteer.launch();
const [page] = await browser.pages();
await page.goto(url, {waitUntil: "domcontentloaded"});
const usernames = await page.evaluate(async () => {
const url = maxId => `https://poshmark.com/vm-rest/channel_groups/category/channels/category/collections/post?request={%22filters%22:{%22department%22:%22Men%22,%22category_v2%22:%22Jackets_%26_Coats%22,%22inventory_status%22:[%22available%22]},%22sort_by%22:%22like_count%22,%22facets%22:[%22color%22,%22brand%22,%22size%22],%22experience%22:%22all%22,%22sizeSystem%22:%22us%22,%22max_id%22:%22${maxId}%22,%22count%22:%2248%22}&summarize=true&pm_version=226.1.0`;
const usernames = [];
try {
for (let maxId = 1; maxId < 5 /* for testing */; maxId++) {
const response = await fetch(url(maxId)); // node 18 or install node-fetch
if (!response.ok) {
throw Error(response.statusText);
break;
}
const json = await response.json();
if (json.error) {
break;
}
usernames.push(...json.data.map(e => e.creator_username));
}
}
catch (err) {
console.error(err);
}
return usernames;
});
console.log(usernames);
console.log("usernames.length", usernames.length);
})()
.catch(err => console.error(err))
.finally(() => browser?.close());
The above code limits to 4 requests to keep it simple and easy to validate.
Blocking images and other unnecessary resources can help speed the Puppeteer versions up, left as an exercise (or just use the direct fetch version shown at top).
I have the following get request where I call a bunch of data and pass it through to my EJS view.
router.get('/currentrentals', ensureAuthenticated, async (req, res) => {
const companies = await Company.getCompanies();
const barges = await Barge.getBarges();
const parishes = await Parish.getParishes();
const states = await State.getStates();
const pickupdropoff = await PickupDropoff.getPickupDropoff();
var notifications = await Notification.getNotifications();
JSON.stringify(barges);
JSON.stringify(companies);
JSON.stringify(parishes);
JSON.stringify(states);
JSON.stringify(pickupdropoff);
JSON.stringify(notifications);
var notifications = await notifications.sort((a, b) => b.id - a.id).slice(0,3);
res.render('currentrentals', {
name: req.user.name, companies: companies, barges: barges, parishes: parishes, states: states, pickupdropoff : pickupdropoff, notifications : notifications
});
}
);
Two questions:
I have multiple get requests that requires the same information. Is there a way to make this data available across the entirety of my site, so I don't have to rewrite this for each get path?
Is there a more succinct way to write the existing code I have? Perhaps looping through them or something of the sort? Simply for learning purposes.
The code currently works as-is.
Thanks!
If the data is constant, you can try this:
let data = null;
async function getData() {
if (!data) {
data = {
companies: await Company.getCompanies(),
barges: await Barge.getBarges();
parishes: await Parish.getParishes(),
states: await State.getStates(),
pickupdropoff: await PickupDropoff.getPickupDropoff(),
notifications: (await Notification.getNotifications()).sort((a, b) => b.id - a.id).slice(0,3)
};
}
return data;
}
router.get('/currentrentals', ensureAuthenticated, async (req, res) => {
res.render('currentrentals', { name: req.user.name, ...(await getData()) });
}
);
// other routes
I am succesfully updating my user's profile picture on their profile and on all of their reviews posted with this function:
export const storeUserProfileImage = (url) => {
const { currentUser } = firebase.auth();
firebase.database().ref(`/users/${currentUser.uid}/profilePic`)
.update({ url });
firebase.database().ref('reviews')
.orderByChild('username')
.equalTo('User3')
.once('value', (snapshot) => {
snapshot.forEach((child) => {
child.ref.update({ profilePic: url });
});
});
};
I am aware that I should be using an atomic update to do this so the data updates at the same time (in case a user leaves the app or something else goes wrong). I am confused on how I can accomplish this when querying over child values.
Any help or guidance would be greatly appreciated!
Declare a variable to store all the updates. Add the updates as you read them on your listener's loop. When the loop is finished, run the atomic update.
export const storeUserProfileImage = (url) => {
const { currentUser } = firebase.auth();
firebase.database().ref('reviews')
.orderByChild('username')
.equalTo('User3')
.once('value', (snapshot) => {
var updates = {};
updates[`/users/${currentUser.uid}/profilePic`] = url;
snapshot.forEach((child) => {
updates[`/reviews/${child.key}/profilePic`] = url;
});
firebase.database().ref().update(updates);
});
};
I'm trying to learn NodeJS. I'm using mongoose & mLab. I'm new to every one of these technologies.
My model at the moment looks like this. I will add a few things to the schema later.
const mongoose = require("mongoose");
const fetchData = require("../seed");
const schema = mongoose.Schema;
const dataSchema = new Schema({});
module.exports = recallData = mongoose.model("recalls", dataSchema);
I also made a seed file for fetching data..
const Recall = require("./models/Recall");
module.exports = function getData(req, res) {
const urls = [url1, url2, url3];
urls.map(url => {
fetch(url)
.then(res => res.json())
.then(data =>
data.results.map(recalls => {
let recs = new Recall(recalls);
recs.save;
})
);
});
}
my question is how do I make the fetch run and populate the database? Is there a command or a mongoose function that will do that?
I know that I'm basically trying to emulate Rails with a seed file. Maybe it's not the way to do it in Node. Any help is super appreciated.
Turns out it's pretty simple. All I needed was a nights sleep. I needed to connect to mongoose and after save(), disconnect.
Now the code looks like this. I still need to add and edit some stuffs in it. Any smart refactoring advice is appreciated.
const mongoose = require("mongoose");
const Recall = require("./models/Recall");
const db = require("./config/keys").mongoURI;
const fetch = require("node-fetch");
const URLS = require("./config/seedURLs");
let resultData;
let saveCounter = 0;
mongoose
.connect(db)
.then(() => console.log("mongodb connection success"))
.catch(error => console.log(error));
URLS.map(async url => {
try {
const response = await fetch(url);
const json = await response.json();
resultData = [...json.results];
for (let i = 0; i < resultData.length; i++) {
let temp = new Recall({
key1: resultData[i].key1,
key2: resultData[i].key2,
.
.
.
});
temp.save(() => {
saveCounter++;
if (saveCounter === resultData.length) {
mongoose
.disconnect()
.then(() => console.log("mongodb disconnected"))
.catch(error => console.log(error));
}
});
}
} catch (error) {
console.log(error);
}
});
Run node seed.js command.
This is the general idea.