We are trying to import a csv with around 30.000 lines of data into a MongoDB database using Mongoose. We have created a model with some validators so only correct rows will be added to the database.
First we read the csv with papaparse and create an array out of it. Next we insert the data into the database with the mongoose Model.insertMany method. To continu inserting data when a row fails we use the option ordered: false.
This all works but we are looking for a way to collect the failed rows so we can check why a row didn't get passed the validation.
Is there a way to get the failed rows with the insertMany method?
...
const readCSV = async (filePath) => {
const csvFile = fs.readFileSync(filePath);
const csvData = csvFile.toString();
return new Promise(resolve => {
Papa.parse(csvData, {
header: true,
transformHeader: header => header.trim(),
complete: results => {
console.log('Complete', results.data.length, 'records.');
resolve(results.data);
}
});
});
};
const start = async () => {
try {
await connectDB(process.env.DATABASE_URL);
const parsedData = await readCSV(csvFilePath);
const response = await Company.insertMany(parsedData, { ordered: false });
console.log(response);
} catch(error) {
console.log(error);
}
}
start();
Related
So I am using supabase and I want to allow a user to create partner and that lives in one table called orgs but the ids have a relationship in a partners table. General flow;
mermaidERD
I am trying the follow but get an empty array on the partner update.
const handleSubmit = async () => {
try {
const { data: newOrgData, error: newOrgError } = await supabase
.from('organizations')
.insert([{ name: partnerOrgName }])
.select();
console.log(newOrgData);
setPartnerOrgName(newOrgData.name);
setPartnerOrgId(newOrgData.id);
if (newOrgError) {
throw newOrgError;
}
const { data: newPartnerData, error: newPartnerError } = await supabase
.from('partners')
.insert([{
organization_id: user.organization_id,
partner_organization_id: newOrgData.id,
}])
.select();
console.log(newPartnerData);
if (newPartnerError) {
throw newPartnerError;
}
setIsOpen(false);
await fetchOrganizations();
} catch (error) {
console.error(error);
}
};
I would expect it to take the user.organizatio_id and insert it and the newOrgData.id and insert it into the appropriate rows on the partners table
I am pulling data from a Google Sheet and after some help, I have reached the point where the discord bot is responding to the inquiry, with the correct data, but it responds with each row of the map array in separate responses rather than gathering the data and sending it in one message. So I believe I need to use an async function and the await feature. I just can't seem to figure out where or how to put it in. I've been stuck on this for a few hours now and can't seem to get it to work?
Here is my code:
const { google } = require('googleapis');
const { sheets } = require('googleapis/build/src/apis/sheets');
const keys = require('../Data/client_secret.json');
const { Team, DiscordAPIError } = require('discord.js');
const Command = require("../Structures/Command");
const gclient = new google.auth.JWT(
keys.client_email,
null,
keys.private_key,
['https://www.googleapis.com/auth/spreadsheets']
);
module.exports = new Command({
name: "freeagents",
description: "Potential free agents for next season.",
async run(message, args, client) {
gclient.authorize(function(err,tokens){
if(err){
console.log(err);
return;
} else {
console.log("Connected!");
gsrun(gclient);
}
});
async function gsrun(cl){
const gsapi = google.sheets({version:'v4', auth: cl });
gsapi.spreadsheets.values.get({
spreadsheetId: "11e5nFk50pDztDLngwTSmossJaNXNAGOaLqaGDEwrbQM",
range: 'Keepers!C1:F',
}, (err, res) => {
if (err) return console.log('The API returned an error: ' + err);
const rows = res.data.values;
if (rows.length) {
const cells = rows.filter(cell => cell[3])
cells.map((cell) => {
console.log(`${cell[0]}, ${cell[1]}`);
console.log(`${cells}`);
return message.reply(`${cell[0]}, ${cell[1]}`);
});
} else {
console.log('No data found.');
}
})
};
}
});
Any help would be greatly appreciated!
You're receiving multiple messages because you're sending the message inside cells.map((cell) => {}). That method will call a function on each member of the array. You should instead iterate over the cells array to produce a single string that you can then send as a message, for example:
const strings = [];
for (let cell of cells) {
strings.push(`${cell[0]}, ${cell[1]}`);
}
return message.reply(strings.join("\n"));
I thought this would be a straightforward task but what I am trying to do is go through all the documents (users) in my collection using a cursor and saving some data into a JS array or set about a specific field of the user or all of the data on that user. I am able to see each document printed on the console, but after looping through all the documents, my array is still empty when printed. Where am I going wrong and if there is an alternative approach please let me know.
const mongoose = require('mongoose');
let Users = require('./models/user.model');
const uri = process.env.ATLAS_URI;
mongoose.connect(uri, { useNewUrlParser: true, useCreateIndex:true, useUnifiedTopology: true});
const connection = mongoose.connection;
connection.once('open', () => {
console.log("MongoDB connection success");
})
let arr = [];
async function getRecords() {
let cursor = Users.find({}).cursor();
for (let doc = await cursor.next(); doc != null; doc = await cursor.next()) {
arr.push(doc); //does not work :(
console.log(doc); //this works
}
}
getRecords();
console.log("ARRAY:",arr); //prints []
Suggestion
Your code should read as such
let arr = [];
// This is an async function (returns a promise)
async function getRecords() {
let docs = await Users.find({}).lean();
arr = docs.filter((doc) => doc !== null); // As an example, however, enter appropriate condition for filter
return arr;
}
// Call the async funtion
getRecords().then(docs => {
console.log("ARRAY:", arr);
});
I am using Firebase authentication to store users. I have two types of users: Manager and Employee. I am storing the manager's UID in Firestore employee along with the employee's UID. The structure is shown below.
Firestore structure
Company
|
> Document's ID
|
> mng_uid: Manager's UID
> emp_uid: Employee's UID
Now I want to perform a query like "Retrieve employees' info which is under the specific manager." To do that I tried to run the below code.
module.exports = {
get_users: async (mng_uid, emp_uid) => {
return await db.collection("Company").where("manager_uid", "==", mng_uid).get().then(snaps => {
if (!snaps.empty) {
let resp = {};
let i = 0;
snaps.forEach(async (snap) => {
resp[i] = await admin.auth().getUser(emp_uid).then(userRecord => {
return userRecord;
}).catch(err => {
return err;
});
i++;
});
return resp;
}
else return "Oops! Not found.";
}).catch(() => {
return "Error in retrieving employees.";
});
}
}
Above code returns {}. I tried to debug by returning data from specific lines. I got to know that the issue is in retrieving the user's info using firebase auth function which I used in forEach loop. But it is not returning any error.
Thank you.
There are several points to be corrected in your code:
You use async/await with then() which is not recommended. Only use one of these approaches.
If I understand correctly your goal ("Retrieve employees' info which is under the specific manager"), you do not need to pass a emp_uid parameter to your function, but for each snap you need to read the value of the emp_uid field with snap.data().emp_uid
Finally, you need to use Promise.all() to execute all the asynchronous getUser() method calls in parallel.
So the following should do the trick:
module.exports = {
get_users: async (mng_uid) => {
try {
const snaps = await db
.collection('Company')
.where('manager_uid', '==', mng_uid)
.get();
if (!snaps.empty) {
const promises = [];
snaps.forEach(snap => {
promises.push(admin.auth().getUser(snap.data().emp_uid));
});
return Promise.all(promises); //This will return an Array of UserRecords
} else return 'Oops! Not found.';
} catch (error) {
//...
}
},
};
Init code:
let dbPormise = null;
const OBJECT_STORE_NAME = 'pages';
const DB_NAME = 'tracking-log';
To initiate an ObjectStore:
dbPromise = idb.open(DB_NAME, 3, upgradeDB => {
upgradeDB.createObjectStore(OBJECT_STORE_NAME, {
autoIncrement: true,
keypath: 'id'
});
});
This is how I generate a blank record in the IndexedDB:
const tx = db.transaction(OBJECT_STORE_NAME, 'readwrite');
tx.objectStore(OBJECT_STORE_NAME).put(
{ id: newBucketID, data: [] });
Now, at a later point, I have some elements that I want to append to the data array for a particular id.
This is how I tried doing it:
const tx = db.transaction(OBJECT_STORE_NAME, 'readwrite');
tx.objectStore(OBJECT_STORE_NAME).put(
{ id: localStorage.getItem("currentBucket"), data: item }
);
Schema
{
data: Array
}
Every item has a unique key generated and provided by me.
However, this doesn't work and returns an error: "Key already exists in the object store."
So, how can I append a value to a field inside a IDB objectt?
Not sure about the error, but regardless of that, the basic way of adding an item would be something like this:
function addItem(db, bucketId, item) {
return new Promise(addItemExecutor.bind(null, db, bucketId, item));
}
function addItemExecutor(db, bucketId, item, resolve, reject) {
// Start a single writable transaction that we will use for two requests. One to
// find the corresponding bucket, and one to update it.
const tx = db.transaction(OBJECT_STORE_NAME, 'readwrite');
// If all requests completed without error, we are done
tx.oncomplete = resolve;
// If any request fails, the operation fails
tx.onerror = event => reject(event.target.error);
const store = tx.objectStore(OBJECT_STORE_NAME);
// Go find the corresponding bucket object to update
const findRequest = store.get(bucketId);
findRequest.onsuccess = findRequestOnsuccess.bind(findRequest, bucketId, item, reject);
}
// React to the resolution of the get request
function findRequestOnsuccess(bucketId, item, reject, event) {
const bucket = event.target.result;
// If no bucket exists for that id then fail
if(!bucket) {
const error = new Error('No bucket found for id ' + bucketId);
reject(error);
return;
}
// Lazily init the data array property
if(!bucket.data) {
bucket.data = [];
}
// Add our item to the data array
bucket.data.push(item);
// Save the bucket object back into the bucket object store, completely replacing
// the bucket that was there before.
const bucketStore = event.target.source;
bucketStore.put(bucket);
}
async function someCallingCodeExampleAvoidingTopLevelAwait() {
const bucketId = localStorage.currentBucket;
const item = {foo:bar};
const db = evilUnreliableGlobalDbVariableFromSomewhereMagicalForeverOpenAssumeInitialized;
try {
await addItem(db, bucketId, item);
} catch(error) {
console.debug(error);
}
// Leave the database connection open for page lifetime
}
Without a reduced example it's difficult to figure out what's going on. The best way to get help is to create a reduced example of the problem, as in, the smallest amount of code needed to recreate the issue you're seeing, then put it on something like jsbin.com or glitch.com so folks only have to click a link to see the error you're seeing.
I wasn't able to recreate the error you're seeing. You have keypath when it should be keyPath, but I don't think that creates the error you're seeing.
Anyway, here's how to modify a record in IDB:
async function main() {
// Set up the database.
const OBJECT_STORE_NAME = 'pages';
const DB_NAME = 'tracking-log';
const db = await idb.open(DB_NAME, 1, upgradeDB => {
upgradeDB.createObjectStore(OBJECT_STORE_NAME, {
autoIncrement: true,
keyPath: 'id'
});
});
// The OP didn't make it clear what this value was, so I'll guess.
const newBucketID = 1;
{
// Create the record.
const tx = db.transaction(OBJECT_STORE_NAME, 'readwrite');
tx.objectStore(OBJECT_STORE_NAME).put({ id: newBucketID, data: ['first value'] });
}
{
const tx = db.transaction(OBJECT_STORE_NAME, 'readwrite');
// Get the record.
const record = await tx.objectStore(OBJECT_STORE_NAME).get(newBucketID);
// Modify it.
record.data.push('second value');
// Put the modified record back.
tx.objectStore(OBJECT_STORE_NAME).put(record);
}
{
// Read the value to confirm everything worked.
const tx = db.transaction(OBJECT_STORE_NAME);
const value = await tx.objectStore(OBJECT_STORE_NAME).get(newBucketID);
console.log(value);
}
}
main();
And here's that example running: https://jsbin.com/dineguq/edit?js,console