I want to call an API here
https://api.jikan.moe/v4/top/anime to get the data in raw format and then create an array of only useful data out of it. What is the reason the data is not being consoled
const initialAnime = {
anime: [],
genresLoaded: false,
genres: [],
};
function createAnimeFromRawData(rawData, animeArray) {
const data = rawData.data;
data.forEach((animeData) => {
const anime = {
mal_id: animeData.mal_id,
title: animeData.title,
title_english: animeData.title_english,
type: animeData.type,
episodes: animeData.episodes,
status: animeData.status,
duration: animeData.duration,
rating: animeData.rating,
rank: animeData.rank,
synopsis: animeData.synopsis,
};
console.log(animeArray);
animeArray.push(anime);
});
}
const RawdataAnime = async (api, genre, paging) => {
const Animearray = [];
for (let i = 1; Animearray.length < 60 && i < 10; i++) {
const {
data: { results },
} = await axios.get(`${api}`);
createAnimeFromRawData(results, Animearray);
}
return Animearray;
};
export const fetchAnime = createAsyncThunk(
"myanimelist/topAnime",
async (thunkAPI) => {
const {
myanimelist: { genres },
} = thunkAPI.getState();
return RawdataAnime(`https://api.jikan.moe/v4/top/anime`, genres, false);
}
);
const animeSlice = createSlice({
name: "Myanimelist",
initialState: initialAnime,
extraReducers: (builder) => {
builder.addCase(getGenresAnime.fulfilled, (state, action) => {
state.genres = action.payload;
state.genresLoaded = true;
});
builder.addCase(fetchAnime.fulfilled, (state, action) => {
state.anime = action.payload;
});
},
});
export const store = configureStore({
reducer: {
netflix: netflixSlice.reducer,
anime: animeSlice.reducer,
},
});
I tried the code above to get an array of only useful parts of data in the code but there was nothing in the console. There was no error and no output.
Whereas the response.data will be something similar to the json below::
{
"pagination":{
...
},
"data":[
...
],
"links":{
...
},
"meta":{
...
}
}
I believe the error is in the snippet
const { data: { results }} = await axios.get(`${api}`); // There are no results in the returned content
createAnimeFromRawData(results, Animearray);
Try something like
const { data } = await axios.get(`${api}`); // Equivalent to response.data
const results = data?.data || []
createAnimeFromRawData(results, Animearray);
Related
I am trying to create a scraping application using redux toolkit for learning purposes.Whenever I dispatch the action the data gets scraped and console logged but the action state is never fullfilled and is always pending
MY ASYNC THUNK
export const loadData = createAsyncThunk(
"alldata/getdata",
async ({ pageNo, language }, thunkAPI) => {
const data = await fetch(
`http://localhost:5000/scrape?pageNo=${encodeURIComponent(
pageNo
)}&language=${encodeURIComponent(language)}`
);
const res=await data.json()
return {
payload: res,
};
}
);
MY SLICE
const projectSlice = createSlice({
name: "allprojects",
initialState: {
projectState: [],
workingState: [],
isLoading: false,
hasError: false,
},
reducers: {
addProject: (state, action) => {
return state.workingState.push(action.payload);
},
removeProject: (state, action) => {
return state.workingState.filter(
(project) => project.link !== action.payload.link
);
},
},
extraReducers: {
[loadData.pending]: (state, action) => {
state.isLoading = true;
state.hasError = false;
},
[loadData.fulfilled]: (state, { payload }) => {
state.projectState = payload;
state.isLoading = false;
state.hasError = false;
},
[loadData.rejected]: (state, action) => {
state.isLoading = false;
state.hasError = true;
},
},
});
export const { addProject, removeProject } = projectSlice.actions;
const Projectreducer = projectSlice.reducer;
export default Projectreducer;
export const projectSelector = (state) => state.allprojects;
REACT COMPONENT
const { workingState, projectState, isLoading, hasError } =
useSelector(projectSelector);
const dispatch = useDispatch();
const [selectData, setSelectData] = React.useState({ languages: "" });
const [pageData, setPageData] = React.useState({ pageNo: 1 });
const handleClick = (event) => {
event.preventDefault();
dispatch(
loadData({ pageNo: pageData.pageNo, language: selectData.languages })
);
};
So how do I get the action to be fullfilled and push the data in the ProjectState array after the async request
EDIT:
API
app.get("/scrape", async (req, res) => {
const { pageNo, language } = req.query;
const browser = await puppeteer.launch({ headless: true });
const page = await browser.newPage();
await page.goto(
`https://github.com/search?p=${pageNo}&q=language%3A${language}`,
{
waitUntil: "domcontentloaded",
}
); // URL is given by the "user" (your client-side application)
const data = await page.evaluate(() => {
const list = [];
const items = document.querySelectorAll(".repo-list-item");
for (const item of items) {
list.push({
projectName: item.querySelector(".f4 > a").innerText,
about: item.querySelector("p").innerText,
link: item.querySelector("a").getAttribute("href"),
});
}
return list;
});
console.log(data);
await browser.close();
});
Store
import { configureStore } from "#reduxjs/toolkit";
import Projectreducer from "./Slices/slice";
export const store = configureStore({
reducer: {
allprojects: Projectreducer,
},
});
Its possible that the api that you are fetching is throwing an error so in this case it always recommended to have a catch block and throw an error to that its falls into loadData.rejected state.
So, do the check network tab in the dev tools of the browser that you are using, so that you can confirm if the api is responding or not.
Also can you share the projectSelector selector ? could be the something wrong in the selector.
action:
export const loadData = createAsyncThunk(
'alldata/getdata',
async ({ pageNo, language }, { rejectWithValue }) => {
try {
const data = await fetch(
`http://localhost:5000/scrape?pageNo=${encodeURIComponent(
pageNo
)}&language=${encodeURIComponent(language)}`
);
const res = await data.json();
return {
payload: res,
};
} catch (error) {
return rejectWithValue({ payload: error?.message || error });
}
}
);
reducer:
extraReducers: {
...,
[loadData.rejected]: (state, { payload }) => {
state.isLoading = false;
state.hasError = true;
state.message = payload;
},
},
One more thing to mention here is that redux toolkit recommends to use builder callback, for more details check here:
https://redux-toolkit.js.org/api/createslice#extrareducers
in your example it would be like
extraReducers: (builder) => {
builder
.addCase(loadData.pending, (state) => {
...
})
.addCase(loadData.fulfilled, (state, action) => {
...
})
.addCase(loadData.rejected, (state, action) => {
...
});
I use next-redux-wrapper, MSW, #mswjs/data and redux-toolkit for storing my data in a store as well as mocking API calls and fetching from a mock Database.
I have the following scenario happening to me.
I am on page /content/editor and in the console and terminal, I can see the data was fetched from the mock database and hydrated from getStaticProps of Editor.js. So now IDs 1 to 6 are inside the store accessible.
Now I click on the PLUS icon to create a new project. I fill out the dialog and press "SAVE". a POST request starts, it's pending and then it gets fulfilled. The new project is now in the mock DB as well as in the store, I can see IDs 1 to 7 now.
Since I clicked "SAVE" and the POST request was successful, I am being routed to /content/editor/7 to view the newly created project.
Now I am on Page [id].js, which also fetched data from the mock DB and then it gets stored and hydrated into the redux store. The idea is, it takes the previous store's state and spreads it into the store, with the new data (if there are any).
Now the ID 7 no longer exists. And IDs 1 to 6 also don't exist anymore, instead, I can see in the console and terminal that IDs 8 to 13 were created, and the previous ones are no more.
Obviously, this is not great. When I create a new project and then switch the route, I should be able to access the newly created project as well as the previously created ones. But instead, they all get overwritten.
It either has something to do with the next-redux-wrapper or MSW, but I am not sure how to make it work. I need help with it. I will post some code now:
Code
getStaticProps
// path example: /content/editor
// Editor.js
export const getStaticProps = wrapper.getStaticProps(
(store) =>
async ({ locale }) => {
const [translation] = await Promise.all([
serverSideTranslations(locale, ['editor', 'common', 'thesis']),
store.dispatch(fetchProjects()),
store.dispatch(fetchBuildingBlocks()),
]);
return {
props: {
...translation,
},
};
}
);
// path example: /content/editor/2
// [id].js
export const getStaticProps = wrapper.getStaticProps(
(store) =>
async ({ locale, params }) => {
const { id } = params;
const [translation] = await Promise.all([
serverSideTranslations(locale, ['editor', 'common', 'thesis']),
store.dispatch(fetchProjects()),
// store.dispatch(fetchProjectById(id)), // issue: fetching by ID returns null
store.dispatch(fetchBuildingBlocks()),
]);
return {
props: {
...translation,
id,
},
};
}
);
Mock Database
Factory
I am going to shorten the code to the relevant bits. I will remove properties for a project, as well es helper functions to generate data.
const asscendingId = (() => {
let id = 1;
return () => id++;
})();
const isDevelopment =
process.env.NODE_ENV === 'development' || process.env.STORYBOOK || false;
export const projectFactory = () => {
return {
id: primaryKey(isDevelopment ? asscendingId : nanoid),
name: String,
// ... other properties
}
};
export const createProject = (data) => {
return {
name: data.name,
createdAt: getUnixTime(new Date()),
...data,
};
};
/**
* Create initial set of tasks
*/
export function generateMockProjects(amount) {
const projects = [];
for (let i = amount; i >= 0; i--) {
const project = createProject({
name: faker.lorem.sentence(faker.datatype.number({ min: 1, max: 5 })),
dueDate: date(),
fontFamily: getRandomFontFamily(),
pageMargins: getRandomPageMargins(),
textAlign: getRandomTextAlign(),
pageNumberPosition: getRandomPageNumberPosition(),
...createWordsCounter(),
});
projects.push(project);
}
return projects;
}
API Handler
I will shorten this one to GET and POST requests only.
import { db } from '../../db';
export const projectsHandlers = (delay = 0) => {
return [
rest.get('https://my.backend/mock/projects', getAllProjects(delay)),
rest.get('https://my.backend/mock/projects/:id', getProjectById(delay)),
rest.get('https://my.backend/mock/projectsNames', getProjectsNames(delay)),
rest.get(
'https://my.backend/mock/projects/name/:id',
getProjectsNamesById(delay)
),
rest.post('https://my.backend/mock/projects', postProject(delay)),
rest.patch(
'https://my.backend/mock/projects/:id',
updateProjectById(delay)
),
];
};
function getAllProjects(delay) {
return (request, response, context) => {
const projects = db.project.getAll();
return response(context.delay(delay), context.json(projects));
};
}
function postProject(delay) {
return (request, response, context) => {
const { body } = request;
if (body.content === 'error') {
return response(
context.delay(delay),
context.status(500),
context.json('Server error saving this project')
);
}
const now = getUnixTime(new Date());
const project = db.project.create({
...body,
createdAt: now,
maxWords: 10_000,
minWords: 7000,
targetWords: 8500,
potentialWords: 1500,
currentWords: 0,
});
return response(context.delay(delay), context.json(project));
};
}
// all handlers
import { buildingBlocksHandlers } from './api/buildingblocks';
import { checklistHandlers } from './api/checklist';
import { paragraphsHandlers } from './api/paragraphs';
import { projectsHandlers } from './api/projects';
import { tasksHandlers } from './api/tasks';
const ARTIFICIAL_DELAY_MS = 2000;
export const handlers = [
...tasksHandlers(ARTIFICIAL_DELAY_MS),
...checklistHandlers(ARTIFICIAL_DELAY_MS),
...projectsHandlers(ARTIFICIAL_DELAY_MS),
...buildingBlocksHandlers(ARTIFICIAL_DELAY_MS),
...paragraphsHandlers(ARTIFICIAL_DELAY_MS),
];
// database
import { factory } from '#mswjs/data';
import {
buildingBlockFactory,
generateMockBuildingBlocks,
} from './factory/buildingblocks.factory';
import {
checklistFactory,
generateMockChecklist,
} from './factory/checklist.factory';
import { paragraphFactory } from './factory/paragraph.factory';
import {
projectFactory,
generateMockProjects,
} from './factory/project.factory';
import { taskFactory, generateMockTasks } from './factory/task.factory';
export const db = factory({
task: taskFactory(),
checklist: checklistFactory(),
project: projectFactory(),
buildingBlock: buildingBlockFactory(),
paragraph: paragraphFactory(),
});
generateMockProjects(5).map((project) => db.project.create(project));
const projectIds = db.project.getAll().map((project) => project.id);
generateMockTasks(20, projectIds).map((task) => db.task.create(task));
generateMockBuildingBlocks(10, projectIds).map((block) =>
db.buildingBlock.create(block)
);
const taskIds = db.task.getAll().map((task) => task.id);
generateMockChecklist(20, taskIds).map((item) => db.checklist.create(item));
Project Slice
I will shorten this one as well to the relevant snippets.
// projects.slice.js
import {
createAsyncThunk,
createEntityAdapter,
createSelector,
createSlice,
current,
} from '#reduxjs/toolkit';
import { client } from 'mocks/client';
import { HYDRATE } from 'next-redux-wrapper';
const projectsAdapter = createEntityAdapter();
const initialState = projectsAdapter.getInitialState({
status: 'idle',
filter: { type: null, value: null },
statuses: {},
});
export const fetchProjects = createAsyncThunk(
'projects/fetchProjects',
async () => {
const response = await client.get('https://my.backend/mock/projects');
return response.data;
}
);
export const saveNewProject = createAsyncThunk(
'projects/saveNewProject',
async (data) => {
const response = await client.post('https://my.backend/mock/projects', {
...data,
});
return response.data;
}
);
export const projectSlice = createSlice({
name: 'projects',
initialState,
reducers: {
// irrelevant reducers....
},
extraReducers: (builder) => {
builder
.addCase(HYDRATE, (state, action) => {
// eslint-disable-next-line no-console
console.log('HYDRATE', action.payload);
const statuses = Object.fromEntries(
action.payload.projects.ids.map((id) => [id, 'idle'])
);
return {
...state,
...action.payload.projects,
statuses,
};
})
.addCase(fetchProjects.pending, (state, action) => {
state.status = 'loading';
})
.addCase(fetchProjects.fulfilled, (state, action) => {
projectsAdapter.addMany(state, action.payload);
state.status = 'idle';
action.payload.forEach((item) => {
state.statuses[item.id] = 'idle';
});
})
.addCase(saveNewProject.pending, (state, action) => {
console.log('SAVE NEW PROJECT PENDING', action);
})
.addCase(saveNewProject.fulfilled, (state, action) => {
projectsAdapter.addOne(state, action.payload);
console.group('SAVE NEW PROJECT FULFILLED');
console.log(current(state));
console.log(action);
console.groupEnd();
state.statuses[action.payload.id] = 'idle';
})
// other irrelevant reducers...
},
});
This should be all the relevant code. If you have questions, please ask them and I will try to answer them.
I have changed how the state gets hydrated, so I turned this code:
.addCase(HYDRATE, (state, action) => {
// eslint-disable-next-line no-console
console.log('HYDRATE', action.payload);
const statuses = Object.fromEntries(
action.payload.projects.ids.map((id) => [id, 'idle'])
);
return {
...state,
...action.payload.projects,
statuses,
};
})
Into this code:
.addCase(HYDRATE, (state, action) => {
// eslint-disable-next-line no-console
console.group('HYDRATE', action.payload);
const statuses = Object.fromEntries(
action.payload.projects.ids.map((id) => [id, 'idle'])
);
state.statuses = { ...state.statuses, ...statuses };
projectsAdapter.upsertMany(state, action.payload.projects.entities);
})
I used the adapter to upsert all entries.
I have this app which uses the first createAsyncThunk to get the first page from the API, then I want the second createAsyncThunk, which gets the next page, to fire when the user reaches the bottom of the page and get the data in the infinite scrolling method.
// Gets the First 10 Posts from the API
export const getPosts = createAsyncThunk(
"post/getPosts",
async (apiAddress) => {
const response = await fetch(apiAddress);
if (!response.ok) throw new Error("Request Failed!");
const data = await response.json();
return data;
}
);
// Loads the Next 10 Posts
export const getMorePosts = createAsyncThunk(
"post/getMorePosts",
async (apiAddress) => {
const response = await fetch(apiAddress);
if (!response.ok) throw new Error("Request Failed!");
const data = await response.json();
return data;
}
);
const redditPostSlice = createSlice({
name: "post",
initialState: {
redditPost: {},
isLoading: false,
hasError: false,
moreIsLoading: false,
moreHasError: false,
},
extraReducers: (builder) => {
builder
.addCase(getPosts.pending, (state) => {
state.isLoading = true;
state.hasError = false;
})
.addCase(getPosts.fulfilled, (state, action) => {
state.redditPost = action.payload.data;
state.isLoading = false;
state.hasError = false;
})
.addCase(getPosts.rejected, (state) => {
state.isLoading = false;
state.hasError = true;
})
.addCase(getMorePosts.pending, (state) => {
state.moreIsLoading = true;
state.moreHasError = false;
})
.addCase(getMorePosts.fulfilled, (state, action) => {
state.redditPost = action.payload.data;
state.moreIsLoading = false;
state.moreHasError = false;
})
.addCase(getMorePosts.rejected, (state) => {
state.moreIsLoading = false;
state.moreHasError = true;
});
},
});
My problem is that the state of the app changes to the second page and the first page contents are gone.
I know my problem is here state.redditPost = action.payload.data but I don't know how I can append this new state to the previous one.
I've been at this for hours and don't really know what to do anymore.
Is there any way to append the new state to the previous state?
I would assume that the payload data has an array of children. Like this example of response found online:
{
kind: "Listing",
data: {
...
children: [
{kind: "t3", data: {...}}
{kind: "t3", data: {...}}
{kind: "t3", data: {...}}
...
]
...
}
}
So you would need to make redditPost be an array. Also semantically is should be redditPosts to denote array.
initialState: {
redditPost: {},
...
and then when you're updating it one of the easiest ways is using ES6 spread
state.redditPost = {
...state.redditPost,
after: action.payload.data.after,
children: [
...state.redditPost.children,
...action.payload.data.children
]
}
I'm trying to wrap my head around how to use Promise.all() in this code. I've read on articles that you can run async operations in parallel with Promise.all() to optimize for speed. Here's the current code in nested for-loops (bad):
type ListGroup = {
listId: string
groupIds: Array<string>
}
const listsAndGroups: Array<ListGroup> = []; // <-- put everything here
const { lists } = await mailchimp.get('/lists');
for (const list of lists) {
const listObj = { listId: list.id };
const { categories } = await mailchimp.get(
`/lists/${list.id}/interest-categories`,
);
for (const category of categories) {
const { interests } = await mailchimp.get(
`/lists/${list.id}/interest-categories/${category.id}/interests`,
);
Object.defineProperty(listObj, 'groupIds', {
value: interests.map((interest) => interest.id),
enumerable: true,
});
}
listsAndGroups.push(listObj);
}
Here's how I'm doing so far, I think I'm just running blindly here without really knowing what I'm doing:
const listsAndGroups: Array<ListGroup> = await getListsGroups(); // <-- put everything here
const getListsGroups = async () => {
const { lists } = await mailchimp.get('/lists');
const listGroups = lists.map((list) =>
getCategories(list.id).then((groups) =>
groups.map((group: Record<'groupIds', string>) => {
return {
listId: list.id,
...group,
};
}),
),
);
return Promise.all(listGroups);
};
const getCategories = async (listId: string) => {
const { categories } = await mailchimp.get(
`/lists/${listId}/interest-categories`,
);
const groups = categories.map((category) =>
getInterests(listId, category.id),
);
return Promise.all(groups);
};
const getInterests = async (listId: string, categoryId: string) => {
const { interests } = await mailchimp.get(
`/lists/${listId}/interest-categories/${categoryId}/interests`,
);
return { groupIds: interests.map((interest) => interest.id) };
};
You could simplify your operation many way, Here is one:
type ListGroup = {
listId: string
groupIds: Array<string>
}
const listsAndGroups: Array<ListGroup> = []; // <-- put everything here
const { lists } = await mailchimp.get('/lists');
const pandingLists = lists.map(list =>
mailchimp.get(`/lists/${list.id}/interest-categories`)
.then(data => [data, { listId: list.id }])
);
for (const [{ categories }, listObj] of await Promise.all(pandingLists)) {
const batch = categories.map(({ id }) =>
mailchimp.get(`/lists/${listObj.listId}/interest-categories/${id}/interests`).then(interests => {
Object.defineProperty(listObj, 'groupIds', {
value: interests.map(({ id }) => id),
enumerable: true,
});
}));
await Promise.all(batch).then(() => listsAndGroups.push(listObj));
}
This is the code I tried in gatsby-node.js, using gatsby develop to interface with graphql... I'm trying to source data from a blockchain indexer to display on my website.
const fetch = require('node-fetch');
const NODE_TYPE = 'objkt';
exports.sourceNodes = async ({ actions, createContentDigest, createNodeId }) => {
const { createNode } = actions;
const response = await fetch('https://staging.api.tzkt.io/v1/bigmaps/523/keys?value.issuer=tz1V9ZviaGUWZjGx4U7cGYFEyUGyqpFnVGXx&active=true');
const json = await response.json();
const { results = [] } = json;
const objkt = await Promise.all(results.map(async result => {
const { url } = result;
const objResponse = await fetch(url);
return await objResponse.json();
}));
objkt.forEach((node, index) => {
createNode({
...node,
id: createNodeId(`${NODE_TYPE}-${node.id}`),
parent: null,
children: null,
internal: {
type: NODE_TYPE,
content: JSON.stringify(node),
contentDigest: createContentDigest(node)
}
});
});
};
creates error:
{
"errors": [
{
"message": "Syntax Error: Expected Name, found \"}\".",
"locations": [
{
"line": 4,
"column": 3
}
],
"stack": [
"GraphQLError: Syntax Error: Expected Name, found \"}\".",
data I'm trying to source
I'm very lost as to why this error happens...
SOLUTION:
const fetch = require("node-fetch")
const NODE_TYPE = `objkt`
exports.sourceNodes = async ({
actions,
createContentDigest,
createNodeId,
}) => {
const { createNode } = actions
const response = await fetch(
"https://staging.api.tzkt.io/v1/bigmaps/523/keys?value.issuer=tz1V9ZviaGUWZjGx4U7cGYFEyUGyqpFnVGXx&active=true"
)
const objkt = await response.json()
objkt.forEach((node, index) => {
createNode({
...node,
id: createNodeId(`${NODE_TYPE}-${node.id}`),
parent: null,
children: [],
internal: {
type: NODE_TYPE,
content: JSON.stringify(node),
contentDigest: createContentDigest(node),
},
})
})
}
exports.onPreInit = () => console.log("Loaded gatsby-node")
GraphQL code:
query MyQuery {
objkt {
value {
issuer
objkt_id
objkt_amount
xtz_per_objkt
}
internal {
content
contentDigest
}
}
}