I think this more of a general async/await loop question, but I'm trying to do it within the bounds of an Airtable API request and within getStaticProps of Next.js so I thought that is important to share.
What I want to do is create an array of base IDs like ["appBaseId01", "appBaseId02", "appBaseId03"] and output the contents of a page. I have it working with 1 base, but am failing at getting it for multiple.
Below is the code for one static base, if anyone can help me grok how I'd want to loop over these. My gut says that I need to await each uniquely and then pop them into an array, but I'm not sure.
const records = await airtable
.base("appBaseId01")("Case Overview Information")
.select()
.firstPage();
const details = records.map((detail) => {
return {
city: detail.get("City") || null,
name: detail.get("Name") || null,
state: detail.get("State") || null,
};
});
return {
props: {
details,
},
};
EDIT
I've gotten closer to emulating it, but haven't figured out how to loop the initial requests yet.
This yields me an array of arrays that I can at least work with, but it's janky and unsustainable.
export async function getStaticProps() {
const caseOneRecords = await setOverviewBase("appBaseId01")
.select({})
.firstPage();
const caseTwoRecords = await setOverviewBase("appBaseId02")
.select({})
.firstPage();
const cases = [];
cases.push(minifyOverviewRecords(caseOneRecords));
cases.push(minifyOverviewRecords(caseTwoRecords));
return {
props: {
cases,
},
};
}
setOverviewBase is a helper that establishes the Airtable connection and sets the table name.
const setOverviewBase = (baseId) =>
base.base(baseId)("Case Overview Information");
You can map the array of base IDs and await with Promise.all. Assuming you have getFirstPage and minifyOverviewRecords defined as below, you could do the following:
const getFirstPage = (baseId) =>
airtable
.base(baseId)("Case Overview Information")
.select({})
.firstPage();
const minifyOverviewRecords = (records) =>
records.map((detail) => {
return {
city: detail.get("City") || null,
name: detail.get("Name") || null,
state: detail.get("State") || null,
};
});
export async function getStaticProps() {
const cases = await Promise.all(
["appBaseId01", "appBaseId02", "appBaseId03"].map(async (baseId) => {
const firstPage = await getFirstPage(baseId);
return minifyOverviewRecords(firstPage);
})
);
return {
props: {
cases
}
};
}
Related
I have a Next.js application here which needs to read a CSV file from a URL in the same repo in multiple places, but I cannot seem to be able to retrieve this data. You can find the relevant file in my repo here.
Note, the URL I'm trying to pull data from is this: https://raw.githubusercontent.com/ivan-rivera/balderdash-next/main/public/test_rare_words.csv
Here is what I've tried so far:
Approach 1: importing the data
let vocab = {};
...
async function buildVocab() {
const words = await import(VOCAB_URL); // this works when I point to a folder in my directory, but it does not work when I deploy this app. If I point at the URL address, I get an error saying that it cannot find the module
for (let i = 0; i < words.length; i++) {
vocab[words[i].word] = words[i].definition;
}
}
Approach 2: papaparse
const papa = require("papaparse");
let vocab = {};
...
export async function buildVocab() {
await papa.parse(
VOCAB_URL,
{
header: true,
download: true,
delimiter: ",",
step: function (row) {
console.log("Row:", row.data); // this prints data correctly
},
complete: function (results) {
console.log(results); // this returns an object with several attributes among which is "data" and "errors" and both are empty
},
}
);
// this does not work because `complete` does not return anything
vocab = Object.assign({}, ...raw.map((e) => ({ [e.word]: e.definition })));
console.log(vocab);
}
Approach 3: needle
const csvParser = require("csv-parser");
const needle = require("needle");
let vocab = {};
...
let result = [];
needle
.get(VOCAB_URL)
.pipe(csvParser())
.on("data", (data) => {
result.push(data);
});
vocab = Object.assign({}, ...result.map((e) => ({ [e.word]: e.definition })));
// This approach also returns nothing, however, I noticed that if I force it to sleep, then I do get the results I want:
setTimeout(() => {
console.log(result);
}, 1000); // now this prints the data I'm looking for
What I cannot figure out is how to force this function to wait for needle to retrieve the data. I've declared it as an async function and I'm calling it with await buildVocab() but it doesn't help.
Any ideas how I can fix this? Sorry, I'm a JS beginner, so it's probably something fundamental that I'm missing :(
After spending hours on this, I think I finally found a solution:
let vocab = {};
export async function buildVocab() {
await fetch(VOCAB_URL)
.then((resp) => resp.text())
.then((text) => {
papa.parse(text, { header: true }).data.forEach((row) => {
vocab[row.word] = row.definition;
});
});
}
The only oddity that I still can't work out is this: I'm calling my buildVocab function inside another async function and I noticed that if I do not include a console.log statement in that function, then the vocab still does not get populated in time. Here is the function:
export async function sampleWord() {
await buildVocab();
const keys = Object.keys(vocab);
const index = Math.floor(Math.random() * keys.length);
console.log(`selected word: ${keys[index]}`); // this is important!
return keys[index];
}
I decided to try using recoil instead of redux and ran into the problem of caching mutable collections. To be more precise, the problem is not in the response caching itself, but in the further data management, without re-requesting. For example, I requested an array of free time slots that will be displayed in the calendar, the user can change them (unblock or block). After the request (unblock or block) I can use useRecoilRefresher_UNSTABLE to reset the selector cache and request updated slots again, this works as expected, but why do this if you can remember the result after the first request and then just update it. My code is:
export const _cacheFreeSlotsA = atom<SlotsArray | null>({
key: '_cacheFreeSlotsA',
default: null,
});
export const freeSlotsS = selector<SlotsArray>({
key: 'freeSlotsS',
get: async ({ get }) => {
const cache = get(_cacheFreeSlotsA);
if (cache) {
return cache;
}
const res = await slotsAPI.getFreeSlots();
return res.slots;
},
});
export const useUpdateFreeSlotsS = () => {
const setFreeSlots = useSetRecoilState(_cacheFreeSlotsA);
const currentSlots = useRecoilValue(freeSlotsS);
return async (req: ChangeSlotsRequest) => {
await slotsAPI.changeSlots(req);
switch (req.operation) {
case 'open':
setFreeSlots([...currentSlots, ...req.slots]);
break;
case 'close':
setFreeSlots(difference(currentSlots, req.slots));
break;
}
};
};
// export const useUpdateFreeSlotsS = () => {
// const refresh = useRecoilRefresher_UNSTABLE(freeSlotsS);
// return async (req: ChangeSlotsRequest) => {
// await slotsAPI.changeSlots(req);
// refresh();
// };
// };
It works but look like workaround. Is there a more elegant and clear way to implement this behavior? (it would be just perfect if inside the freeSlotsS get method I could get access to the set method, but unfortunately it is not in the arguments)
Im trying to figure this out.
I want to get all my users from my database, cache them
and then when making a new request I want to get those that Ive cached + new ones that have been created.
So far:
const batchUsers = async ({ user }) => {
const users = await user.findAll({});
return users;
};
const apolloServer = new ApolloServer({
schema,
playground: true,
context: {
userLoader: new DataLoader(() => batchUsers(db)),// not sending keys since Im after all users
},
});
my resolver:
users: async (obj, args, context, info) => {
return context.userLoader.load();
}
load method requiers a parameter but in this case I dont want to have a specific user I want all of them.
I dont understand how to implement this can someone please explain.
If you're trying to just load all records, then there's not much of a point in utilizing DataLoader to begin in. The purpose behind DataLoader is to batch multiple calls like load(7) and load(22) into a single call that's then executed against your data source. If you need to get all users, then you should just call user.findAll directly.
Also, if you do end up using DataLoader, make sure you pass in a function, not an object as your context. The function will be ran on each request, which will ensure you're using a fresh instance of DataLoader instead of one with a stale cache.
context: () => ({
userLoader: new DataLoader(async (ids) => {
const users = await User.findAll({
where: { id: ids }
})
// Note that we need to map over the original ids instead of
// just returning the results of User.findAll because the
// length of the returned array needs to match the length of the ids
return ids.map(id => users.find(user => user.id === id) || null)
}),
}),
Note that you could also return an instance of an error instead of null inside the array if you want load to reject.
Took me a while but I got this working:
const batchUsers = async (keys, { user }) => {
const users = await user.findAll({
raw: true,
where: {
Id: {
// #ts-ignore
// eslint-disable-next-line no-undef
[op.in]: keys,
},
},
});
const gs = _.groupBy(users, 'Id');
return keys.map(k => gs[k] || []);
};
const apolloServer = new ApolloServer({
schema,
playground: true,
context: () => ({
userLoader: new DataLoader(keys => batchUsers(keys, db)),
}),
});
resolver:
user: {
myUsers: ({ Id }, args, { userLoader }) => {
return userLoader.load(Id);
},
},
playground:
{users
{Id
myUsers
{Id}}
}
playground explained:
users basically fetches all users and then myusers does the same thing by inhereting the id from the first call.
I think I choose a horrible example here since I did not see any gains in performence by this. I did see however that the query turned into:
SELECT ... FROM User WhERE ID IN(...)
I have graphql User type that needs information from multiple REST api's and different servers.
Basic example: get the user firstname from rest domain 1 and get lastname from rest domain 2. Both rest domain have a common "userID" attribute.
A simplefied example of my resolver code atm:
user: async (_source, args, { dataSources }) => {
try {
const datasource1 = await dataSources.RESTAPI1.getUser(args.id);
const datasource2 = await dataSources.RESTAPI2.getUser(args.id);
return { ...datasource1, ...datasource2 };
} catch (error) {
console.log("An error occurred.", error);
}
return [];
}
This works fine for this simplefied version, but I have 2 problems with this solution:
first, IRL there is a lot of logic going into merging the 2 json results. Since some field are shared but have different data (or are empty). So it's like cherry picking both results to create a combined result.
My second problem is that this is still a waterfall method. First get the data from restapi1, when thats done call restapi2. Basicly apollo-server is reintroducing rest-waterfall-fetch graphql tries to solve.
Keeping these 2 problems in mind.. Can I optimise this piece of code or rewrite is for better performance or readability? Or are there any packages that might help with this behavior?
Many thanks!
With regard to performance, if the two calls are independent of one another, you can utilize Promise.all to execute them in parallel:
const [dataSource1,dataSource2] = await Promise.all([
dataSources.RESTAPI1.getUser(args.id),
dataSources.RESTAPI2.getUser(args.id),
])
We normally let GraphQL's default resolver logic do the heavy lifting, but if you're finding that you need to "cherry pick" the data from both calls, you can return something like this in your root resolver:
return { dataSource1, dataSource2 }
and then write resolvers for each field:
const resolvers = {
User: {
someField: ({ dataSource1, dataSource2 }) => {
return dataSource1.a || dataSource2.b
},
someOtherField: ({ dataSource1, dataSource2 }) => {
return someCondition ? dataSource1.foo : dataSource2.bar
},
}
}
Assuming your user resolver returns type User forsake...
type User {
id: ID!
datasource1: RandomType
datasource1: RandomType
}
You can create individual resolvers for each field in type User, this can reduce the complexity of the user Query, to only the requested fields.
query {
user {
id
datasource1 {
...
}
}
}
const resolvers = {
Query: {
user: () => {
return { id: "..." };
}
},
User: {
datasource1: () => { ... },
datasource2: () => { ... } // i wont execute
}
};
datasource1 & datasource2 resolvers will only execute in parallel, after Query.user executes.
For parallel call.
const users = async (_source, args, { dataSources }) => {
try {
const promises = [
dataSources.RESTAPI1,
dataSources.RESTAPI2
].map(({ getUser }) => getUser(args.id));
const data = await Promise.all(promises);
return Object.assign({}, ...data);
} catch (error) {
console.log("An error occurred.", error);
}
return [];
};
I am creating a program that...
1. Detects all of the drives on any given system.
2. Scans those drives for files of specific file types. For example, it may search all of the drives for any jpeg, png, and svg files.
3. The results are then stored in a JSON file in the following desired format.
{
"C:": {
"jpeg": [
...
{
"path": "C:\\Users\\John\\Pictures\\example.jpeg",
"name": "example",
"type": "jpeg",
"size": 86016
},
...
],
"png": [],
"svg": []
},
...
}
The code...
async function scan(path, exts) {
try {
const stats = await fsp.stat(path)
if (stats.isDirectory()) {
const
childPaths = await fsp.readdir(path),
promises = childPaths.map(
childPath => scan(join(path, childPath), exts)
),
results = await Promise.all(promises)
// Likely needs to change.
return [].concat(...results)
} else if (stats.isFile()) {
const fileExt = extname(path).replace('.', '')
if (exts.includes(fileExt)){
// Likely needs to change.
return {
"path": path,
"name": basename(path, fileExt).slice(0, -1),
"type": fileExt,
"size": stats.size
}
}
}
return []
}
catch (error) {
return []
}
}
const results = await Promise.all(
config.drives.map(drive => scan(drive, exts))
)
console.log(results) // [ Array(140), Array(0), ... ]
// And I would like to do something like the following...
for (const drive of results) {
const
root = parse(path).root,
fileExt = extname(path).replace('.', '')
data[root][fileExt] = []
}
await fsp.writeFile('./data.json', JSON.stringify(config, null, 2))
The global results is of course divided into individual arrays that correspond to each drive. But currently it combines all of the objects into one giant array despite their corresponding file types. There is also currently no way for me to know which array belongs to each drive, especially if the drive's array does not contain any items that I can parse to retrieve the root directory.
I can obviously map or loop thru the global results again, and then sort everything out, as illustrated below, but it would be a lot cleaner to have scan() handle everything from the get go.
// Initiate scan sequence.
async function initiateScan(exts) {
let
[config, data] = await Promise.all([
readJson('./config.json'),
readJson('./data.json')
]),
results = await Promise.all(
// config.drives.map(drive => scan(drive, exts))
['K:', 'D:'].map(drive => scan(drive, exts))
)
for (const drive of results) {
let root = false
for (const [i, file] of drive.entries()) {
if (!root) root = parse(file.path).root.slice(0,-1)
if (!data[root][file.type] || !i) data[root][file.type] = []
data[root][file.type].push(file)
}
}
await fsp.writeFile('./data.json', JSON.stringify(config, null, 2))
}
Due to my lack of experience with asynchronicity and objects in general, I am not quite sure how to best handle the data in map( ... )/scan. I am really not even sure how to best structure the output of scan() so that the structure of the global results is easily manipulable.
Any help would be greatly appreciated.
Mutating an outer object as asynchronously-derived results arrive is not particularly clean, however it can be done fairly simply and safely as follows:
(async function(exts, results) { // async IIFE wrapper
async function scan(path) { // lightly modified version of scan() from the question.
try {
const stats = await fsp.stat(path);
if (stats.isDirectory()) {
const childPaths = await fsp.readdir(path);
const promises = childPaths.map(childPath => scan(join(path, childPath)));
return Promise.all(promises);
} else if (stats.isFile()) {
const fileExt = extname(path).replace('.', '');
if (results[path] && results[path][fileExt]) {
results[path][fileExt].push({
'path': path,
'name': basename(path, fileExt).slice(0, -1),
'type': fileExt,
'size': stats.size
});
}
}
}
catch (error) {
console.log(error);
// swallow error by not rethrowing
}
}
await Promise.all(config.drives.map(path => {
// Synchronously seed the results object with the required data structure
results[path] = {};
for (fileExt of exts) {
results[path][fileExt] = []; // array will populated with data, or remain empty if no qualifying data is found.
}
// Asynchronously populate the results[path] object, and return Promise to the .map() callback
return scan(path);
}));
console.log(results);
// Here: whatever else you want to do with the results.
})(exts, {}); // pass `exts` and an empty results object to the IIFE function.
The results object is synchronously seeded with empty data structures, which are then populated asynchronously.
Everything is wrapped in an async Immediately Invoked Function Expression (IIFE), thus:
avoiding the global namespace (if not already avoided)
ensuring availabillty of await (if not already available)
making a safe closure for the results object.
This still needs some work, and it is iterating through the generated files collection a second time.
// This should get you an object with one property per drive
const results = Object.fromEntries(
(await Promise.all(
config.drives.map(async drive => [drive, await scan(drive, exts)])
)
)
.map(
([drive, files]) => [
drive,
// we reduce each drive's file array to an object with
// one property per file extension
files.reduce(
(acc, file) => {
acc[file.type].push(file)
return acc
},
Object.fromEntries(exts.map(ext => [ext, []]))
)
]
)
)
nodejs supports Object.fromEntries from version 12.0.0, so if you can guarantee your application will always be run in that version or a later one, Object.fromEntries should be fine here.
You can use the glob npm library to get all of the filenames and then just transform that array to your object like this:
import {basename, extname} from 'path';
import {stat} from 'fs/promises'; // Or whichever library you use to promisify fs
import * as glob from "glob";
function searchForFiles() {
return new Promise((resolve, reject) => glob(
"/**/*.{jpeg,jpg,png,svg}", // The files to search for and where
{ silent: true, strict: false}, // No error when eg. something cannot be accessed
(err, files) => err ? reject() : resolve(files)
));
}
async function getFileObject() {
const fileNames = await searchForFiles(); // An array containing all file names (eg. ['D:\\my\path\to\file.jpeg', 'C:\\otherfile.svg'])
// An array containing all objects describing your file
const fileObjects = await Promise.all(fileNames.map(async filename => ({
path: filename,
name: basename(path, fileExt).slice(0, -1),
type: extname(path).replace('.', ''),
size: stat(path).size,
drive: `${filename.split(':\\')[0]}:`
})));
// Create your actual object
return fileObjects.reduce((result, {path, name, type, size, drive}) => {
if (!result[drive]) { // create eg. { C: {} } if it does not already exist
result.drive = {};
}
if (!result[drive][type]) { // create eg. {C: { jpeg: [] }} if it does not already exist
result[drive][type] = [];
}
// Push the object to the correct array
result[drive][type].push({path, name, type, size});
return result;
}, {});
}
The function must traverse the file system recursively, looking for files that match your criteria. The recursion can be simplified by the fact that the result doesn't need to retain any hierarchy, so we can just carry a flat array (files) as a parameter.
let exts = [...]
async function scan(path, files) {
const stats = await fsp.stat(path)
if (stats.isDirectory()) {
childPaths = await fsp.readdir(path)
let promises = childPaths.map(childPath => {
return scan(join(path, childPath), files)
})
return Promise.all(promises)
} else if (stats.isFile()) {
const fileExt = extname(path).replace('.', '')
if (exts.includes(fileExt)) {
files.push({
path: path,
name: basename(path, fileExt).slice(0, -1),
type: fileExt,
size: stats.size
})
}
}
}
let files = []
await scan('/', files)
console.log(files)