I have some mock data for below 2 URLS:
1. Get the list of users from 'https://myapp.com/authors'.
2. Get the list of Books from 'https://myapp.com/books'.
Now my task is to sort the Books by name and write the sorted list to the file mysortedbooks.json as JSON
Then I have to create an array of authors with books property that has all the books of that author.
If the author has no books then this array should be empty. Sorting is not needed for this case, and data should be stored in file authorBooks.json as JSON.
Now I have to return a promise that resolves when the above steps are complete. For example, I should return the final saveToFile call in below code.
const fs = require('fs');
function getFromURL(url) {
switch (url) {
case 'https://myapp.com/authors':
return Promise.resolve([
{ name: "Chinua Achebe", id: "1" },
{ name: "Hans Christian Andersen", id: "2" },
{ name: "Dante Alighieri", id: "3" },
]);
case 'https://myapp.com/books':
return Promise.resolve([
{ name: "Things Fall Apart", authorId: "1" },
{ name: "The Epic Of Gilgamesh", authorId: "1" },
{ name: "Fairy tales", authorId: "2" },
{ name: "The Divine Comedy", authorId: "2" },
{ name: "One Thousand and One Nights", authorId: "1" },
{ name: "Pride and Prejudice", authorId: "2" },
]);
}
}
const outFile = fs.createWriteStream('...out-put-path...');
function saveToFile(fileName, data) {
outFile.write(`${fileName}: ${data}\n`);
return Promise.resolve();
}
function processData() {
const authors = getFromURL('https://myapp.com/authors').then(author => {
return authors;
});
const books = getFromURL('https://myapp.com/authors').then(books => {
return books.sort();
});
return saveToFile('mysortedbooks.json', JSON.stringify(books)).then(() => {
const authorAndBooks = authors.map(author => {
var jsonData = {};
jsonData['name'] = author.name;
jsonData['books'] = [];
for(var i=0; i<books.length; i++) {
if(authod.id == books[i].authorId) {
jsonData['books'].push(books[i].name);
}
}
});
saveToFile('authorBooks.json', authorAndBooks);
});
}
processData().then(() => outFile.end());
The main logic I have to implement is in processData method.
I tried adding code to solve the requirement but I got stuck how to return promise after all the operations. Also how to build my authorAndBooks JSON content.
Please help me with this.
const authors = getFromURL('https://myapp.com/authors').then(author => {
return authors;
});
const books = getFromURL('https://myapp.com/authors').then(books => {
return books.sort();
});
//authors and books are both promises here, so await them
return Promise.all([authors, books]).then(function(results){
authors = results[0];
books = results[1];
return saveToFile(...);
});
alternatively declare your function async and do
const authors = await getFromURL('https://myapp.com/authors').then(author => {
return authors;
});
const books = await getFromURL('https://myapp.com/authors').then(books => {
return books.sort();
});
return await saveToFile(...);
Refactored code with Promise Chaining and to create multiple file streams
const fs = require('fs');
function getFromURL(url) {
switch (url) {
case 'https://myapp.com/authors':
return Promise.resolve([
{ name: "Chinua Achebe", id: "1" },
{ name: "Hans Christian Andersen", id: "2" },
{ name: "Dante Alighieri", id: "3" },
]);
case 'https://myapp.com/books':
return Promise.resolve([
{ name: "Things Fall Apart", authorId: "1" },
{ name: "The Epic Of Gilgamesh", authorId: "1" },
{ name: "Fairy tales", authorId: "2" },
{ name: "The Divine Comedy", authorId: "2" },
{ name: "One Thousand and One Nights", authorId: "1" },
{ name: "Pride and Prejudice", authorId: "2" },
]);
}
}
function saveToFile(fileName, data) {
const outFile = fs.createWriteStream(`/var/${fileName}`);
outFile.write(data);
return Promise.resolve(outFile);
}
function authorBookMapping(data) {
let [authors, books] = data;
var jsonData = {};
authors.map(author => {
jsonData['name'] = author.name;
jsonData['books'] = [];
for(var i=0; i<books.length; i++) {
if(author.id == books[i].authorId) {
jsonData['books'].push(books[i].name);
}
}
});
return {
books: books,
authorAndBooks: jsonData
};
}
function writeFile(data) {
if(data) {
const {books, authorAndBooks} = data;
const book = saveToFile('mysortedbooks.json', JSON.stringify(books));
const author = saveToFile('authorBooks.json', JSON.stringify(authorAndBooks));
return Promise.all([book, author]);
}
}
function processData() {
const authors = getFromURL('https://myapp.com/authors');
const books = getFromURL('https://myapp.com/authors');
return Promise.all([authors, books])
.then(authorBookMapping)
.then(writeFile)
}
processData().then((stream) => {
for(let s in stream) {
stream[s].close();
}
})
.catch((err) => {
console.log("Err :", err);
}) ;
lot of mistakes in your code. I will try to explain one by one, read comments between code. I would recommend you to read some basics of file operations and promises. Problem is in your saveToFile method and how you are chaining promises in processData method.
Change your saveToFIle function as following. You can also use promise supporting fs libraries like fs-extra but I'm not sure if you want to use an external library.
const path = require('path');
const basePath = '.';//whatever base path of your directories
function saveToFile(fileName, data) {
// fs.writeFile method uses callback, you can use many ways to convert a callback method to support promises
// this is one of the simple and doesn't require any libraries to import
return new Promise((resolve,reject)=>{
let filePath = path.join(basePath,fileName);
return fs.writeFile(filePath,data,(err, data)=>{
if(err) reject(err);
else resolve();
});
})
}
Now change your processData function to use promise.all and to sort boooks in right way
function processData() {
let books, authors;
//Promise.all can be used when operations are not interdependent, fteches result fasetr as works like parallel requests
return Promise.all([
getFromURL('https://myapp.com/books'),
getFromURL('https://myapp.com/authors')
]).then(data => {
books = data[0];
authors = data[1];
let authorAndBooks = authors.map(author => {
let jsonData = {};
jsonData['name'] = author.name;
jsonData['books'] = [];
for(var i=0; i<books.length; i++) {
if(author.id == books[i].authorId) {
jsonData['books'].push(books[i].name);
}
}
return jsonData;
console.log(jsonData);
});
// you will have to use a comparator to sort objects, given below it will sort books based on names.
books.sort((first,second)=>{ return first.name>second.name ?1:-1})
return Promise.all([
saveToFile("mysortedbooks.json",JSON.stringify(books)),
saveToFile("authorBooks.json",JSON.stringify(authorAndBooks))])
}).then(data=>{
console.log('All operations complete');
})
}
processData();
Have you considered looking at this in a different way? If this is going to be the case for other APIs I'd think about aggregating those APIs in an aggregator service or the API itself if you can.
It is always better to receive all the data you need at once rather than multiple calls, you will incur latency and complexity.
Related
I have a use case where I'm trying to loop through an array of objects, where I need to make some GraphQL requests that may have some pagination for a given object in the array. I'm trying to speed up performance by pushing the recursive function to an array of promises, and then use Promse.all to resolve all of those.
I'm running into an issue though where I'm getting an undefined response from Promise.all - The end goal is to have the following response for each unique object in the array:
[{
account: test1,
id: 1,
high: 2039,
critical: 4059
},
{
account: test2,
id: 2,
high: 395,
critical: 203
}]
...where I'm only returning anAccount object after recursion is done paginating/making all requests for a given account object.
Here is the sample code:
const fetch = require('isomorphic-fetch');
const API_KEY = '<key>';
async function main() {
let promises = [];
let accounts = [{'name': 'test1', 'id': 1}, {'name': 'test2' , 'id': 2}];
for (const a of accounts) {
let cursor = null;
let anAccountsResults = [];
promises.push(getCounts(a, anAccountsResults, cursor));
}
let allResults = await Promise.all(promises);
console.log(allResults);
}
async function getCounts(acct, results, c) {
var q = ``;
if (c == null) {
q = `{
actor {
account(id: ${acct.id}) {
aiIssues {
issues(filter: {states: ACTIVATED}) {
issues {
issueId
priority
}
nextCursor
}
}
}
}
}`
} else {
q = `{
actor {
account(id: ${acct.id}) {
aiIssues {
issues(filter: {states: ACTIVATED}, cursor: "${c}") {
issues {
issueId
priority
}
nextCursor
}
}
}
}
}`
}
const resp = await fetch('https://my.api.com/graphql', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'API-Key': API_KEY
},
body: JSON.stringify({
query: q,
variables: ''}),
});
let json_resp = await resp.json();
let aSingleResult = json_resp.data.actor.account.aiIssues.issues.issues;
let nextCursor = json_resp.data.actor.account.aiIssues.issues.nextCursor;
console.log(nextCursor);
if (nextCursor == null) {
results = results.concat(aSingleResult);
} else {
results = results.concat(aSingleResult);
await getCounts(acct, results, nextCursor);
}
let criticalCount = results.filter(i => i.priority == 'CRITICAL').length;
let highCount = results.filter(i => i.priority == 'HIGH').length;
let anAccount = {
account: acct.name,
id: acct.id,
high: highCount,
critical: criticalCount
};
return anAccount;
}
main();
logging anAccount in function getCounts has the correct detail, but when returning it, logging the output of Promise.all(promises) yields undefined. Is there a better way to handle this in a way where I can still asynchronously run multiple recursive functions in parallel within the loop with Promise.all?
Your main problem appears to be that results = results.concat(aSingleResult); does not mutate the array you passed, but only reassigns the local variable results inside the function, so the anAccount only will use the aSingleResult from the current call.
Instead of collecting things into a results array that you pass an a parameter, better have every call return a new array. Then in the recursive await getCounts(acct, results, nextCursor) call, do not ignore the return value.
async function main() {
let promises = [];
const accounts = [{'name': 'test1', 'id': 1}, {'name': 'test2' , 'id': 2}];
const promises = accounts.map(async acct => {
const results = await getIssues(acct);
const criticalCount = results.filter(i => i.priority == 'CRITICAL').length;
const highCount = results.filter(i => i.priority == 'HIGH').length;
return {
account: acct.name,
id: acct.id,
high: highCount,
critical: criticalCount
};
});
const allResults = await Promise.all(promises);
console.log(allResults);
}
const query = `query ($accountId: ID!, $cursor: IssuesCursor) {
actor {
account(id: $accountId) {
aiIssues {
issues(filter: {states: ACTIVATED}, cursor: $cursor) {
issues {
issueId
priority
}
nextCursor
}
}
}
}
}`;
async function getIssues(acct, cursor) {
const resp = await fetch('https://my.api.com/graphql', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'API-Key': API_KEY
},
body: JSON.stringify({
query: q,
variables: {
accountId: acct.id,
cursor,
}
}),
});
if (!resp.ok) throw new Error(resp.statusText);
const { data, error } = await resp.json();
if (error) throw new Error('GraphQL error', {cause: error});
const { nextCursor, issues } = data.actor.account.aiIssues.issues;
if (nextCursor == null) {
return issues;
} else {
return issues.concat(await getIssues(acct, nextCursor));
}
}
I have a csv that I have to read in and have to upload to a MySql database using Prisma. However... Writing all the students to the database works but the entire 'if a group exists, use that group and if it doesn't exist, create it' part doesn't work because he keeps adding duplicate groups into the database, I have tried so many options by now and none seem to work and I'm getting desperate...
router.post('/', async (req, res) => {
fs.createReadStream("./Upload/StudentenEnGroepen001.csv")
.pipe(parse({ delimiter: ",", from_line: 2 }))
.on("data", async (row) => {
let group = null;
let inschrijving = row[6];
if (inschrijving === "Student") {
let parts = row[8].split(",");
let groep = parts[1];
let groepNaam = groep.split(" - ")[0].trim();
const student = await prisma.student.create({
data: {
Code: row[0],
Gebruikersnaam: row[1],
Familienaam: row[2],
Voornaam: row[3],
Sorteernaam: row[4],
Email: row[5],
},
})
const groupCount = await prisma.groep.count({
where: {
Naam: { equals: groepNaam },
}
});
if (groupCount > 0) {
const existingGroup = await prisma.groep.findFirst({
where: {
Naam: { equals: groepNaam },
}
});
group = existingGroup;
} else {
group = await prisma.groep.create({
data: {
Naam: groepNaam,
}
});
}
await prisma.groepstudent.create({
data: {
GroepID: group.ID,
StudentID: student.ID,
},
});
}
})
res.json({message: "Studenten en groepen zijn toegevoegd."});
});
My latest attempt was this but this also doesn't work.
const groupCount = await prisma.groep.count({
where: {
Naam: { equals: groepNaam },
}
});
if (groupCount > 0) {
const existingGroup = await prisma.groep.findFirst({
where: {
Naam: { equals: groepNaam },
}
});
group = existingGroup;
} else {
group = await prisma.groep.create({
data: {
Naam: groepNaam,
}
});
}
I tried to just use 'findFirst' but that also didn't work...
group = await prisma.groep.findFirst({
where: {
Naam: { equals: groepNaam },
}
});
if (group) {
// then use this group
else{
// create group..
...
Would you please include the errors you're getting?
Adding to what Pointy said, try going back to the point where your app was actually working. Then, start adding in small pieces at a time. Run your app each time to see when it bombs out on you. Check your debugger for clues as to when exactly this is happening. These errors should point you in the right direction. Good luck!
I am new in nodejs and mongodb. Its really very confusing to use promise in loop in nodejs for new developer.I require the final array or object. which then() give me final result. Please correct this.
I have a controller function described below.
let League = require('../../model/league.model');
let Leaguetype = require('../../model/leagueType.model');
let Leaguecategories = require('../../model/leagueCategories.model');
let fetchLeague = async function (req, res, next){
let body = req.body;
await mongo.findFromCollection(Leaguetype)
.then(function(types) {
return Promise.all(types.map(function(type){
return mongo.findFromCollection(Leaguecategories, {"league_type_id": type._id})
.then(function(categories) {
return Promise.all(categories.map(function(category){
return mongo.findFromCollection(League, {"league_category_id": category._id})
.then(function(leagues) {
return Promise.all(leagues.map(function(league){
return league;
}))
.then(function(league){
console.log(league);
})
})
}))
});
}))
})
.then(function(final){
console.log(final);
})
.catch (error => {
console.log('no',error);
})
}
mongo.findFromCollection function is looking like this.
findFromCollection = (model_name, query_obj = {}) => {
return new Promise((resolve, reject) => {
if (model_name !== undefined && model_name !== '') {
model_name.find(query_obj, function (e, result) {
if (!e) {
resolve(result)
} else {
reject(e);
}
})
} else {
reject({ status: 104, message: `Invalid search.` });
}
})
}
and here is my model file
var mongoose = require('mongoose');
const league_categories = new mongoose.Schema({
name: {
type: String,
required: true
},
active: {
type: String,
required: true
},
create_date: {
type: Date,
required: true,
default: Date.now
},
league_type_id: {
type: String,
required: 'league_type',
required:true
}
})
module.exports = mongoose.model('Leaguecategories', league_categories)
First i recommend you stop using callbacks wherever you can, its a bit dated and the code is much harder to read and maintain.
I re-wrote your code a little bit to look closer to what i'm used to, this does not mean this style is better, i just personally think its easier to understand what's going on.
async function fetchLeague(req, res, next) {
try {
//get types
let types = await Leaguetype.find({});
//iterate over all types.
let results = await Promise.all(types.map(async (type) => {
let categories = await Leaguecategories.find({"league_type_id": type._id});
return Promise.all(categories.map(async (category) => {
return League.find({"league_category_id": category._id})
}))
}));
// results is in the form of [ [ [ list of leagues] * per category ] * per type ]
// if a certain category or type did not have matches it will be an empty array.
return results;
} catch (error) {
console.log('no', error);
return []
}
}
I am using node version v10.15.3 and "sequelize": "^4.22.8". When using bulkCreate I am getting double values in my db:
My model looks like the following:
module.exports = (sequelize, DataTypes) => {
const Company = sequelize.define('Company', {
name: DataTypes.STRING,
url: DataTypes.STRING,
symbol: DataTypes.STRING,
description: DataTypes.STRING,
}, {})
Company.associate = function(models) {
Company.hasMany(models.Rating)
};
return Company
};
I have created a custom createOrUpdateCompany() function. Find below my minimum executable example:
const models = require('./src/models');
const Company = require('./src/models').Company;
async function getAllCompanies() {
try {
let res = await Company.findAll()
return res;
} catch (error) {
console.log(error)
}
}
async function createOrUpdateCompany(dataArray) {
if (dataArray === undefined || dataArray.length === 0) return ''
let iss = []
const allCompanies = await getAllCompanies()
// flatten array
dataArray = [].concat.apply([], dataArray)
if (allCompanies !== undefined) {
// 1. remove exact dedupes from dataArray
dataArray = [...new Map(dataArray.map(obj => [JSON.stringify(obj), obj])).values()]
// 2. compare dataArray to allCompanies and remove difference
// dataArray = dataArray.filter(cv => !allCompanies.find(e => e.symbol === cv.symbol))
dataArray = dataArray.filter(cv => !allCompanies.find(e => e.symbol === cv.symbol))
// 3. Remove null values for link and "" values for name
dataArray = dataArray.filter(cv => !(cv.name === '' || cv.url === null))
}
try {
iss = await Company.bulkCreate(dataArray, {
fields: [
'name',
'url',
'symbol',
'description',
]
})
} catch (error) {
console.log(error)
}
return iss
}
let data = [{
"date": "9/14/2019",
"issuer": "Issuer6",
"name": "Name1",
"symbol": "Symbol2",
"url": "www.url.com"
}, {
"date": "9/11/2029",
"issuer": "Issuer3",
"name": "Name1",
"symbol": "Symbol1",
"url": "www.url.com"
}, {
"date": "8/13/2019",
"issuer": "Issuer1",
"name": "Name1",
"symbol": "Symbol1",
"url": "www.url.com"
}]
async function main() {
// setup db
await models.sequelize.sync({
force: true
})
await createOrUpdateCompany(data)
await createOrUpdateCompany(data)
await createOrUpdateCompany(data)
await createOrUpdateCompany(data)
console.log("##################### DONE #####################");
}
main()
When executing the above example I get the following in my db:
However, I would like to get only the following as result:
As you can see I only get two entries based on the unique result.
Any suggestions why my createOrUpdateCompany() is wrong?
I appreciate your replies!
There are 2 ways to achieve uniqueness in your result
Filter your array data before execution (Javascript stuff)
Unique elements from array
Make specific fields unique in the DB (composite unique) (Database stuff)Add Composite unique key in MySQL
in the mapping I have two objects which will go to default in switch and 1 record which will go to ORDER_OPEN case and the object won't enter to if statements and just it will push to orderArray but when the API is executed I only receive two objects from default and when I log the orderArray it is pushing into the objectArray after the execution of API.
router.get('/orderByPhone/:id', async (req, res) => {
const { ORDER_OPEN, ORDER_FILL, BITY_FILL, BITY_CANCEL, getOrderStatusValue } = require('../../lib/constants/orderStatus');
const statusUtils = require('../../lib/constants/orderStatus');
const apiUtils = require('../../lib/apiUtils');
const neo4jUtils = require('../../lib/neo4jUtils');
const orderArray = [];
try {
const id = req.params.id;
const response = await neo4jUtils.getOrders(1, id);
response.records.map(async (record) => {
switch (record._fields[0].properties.orderStatus) {
case ORDER_OPEN:
const ret = await apiUtils.fetchOrderStatus(record._fields[0].properties.bityId, record._fields[0].properties.token);
if (ret.legacy_status == BITY_FILL) {
await neo4jUtils.updateOrderStatus(record._fields[0].properties.bityId, getOrderStatusValue(ret.legacy_status))
} else if (ret.legacy_status == BITY_CANCEL) {
await neo4jUtils.updateOrderStatus(record._fields[0].properties.bityId, getOrderStatusValue(ret.legacy_status))
}
orderArray.push({
input: {
amount: ret.input.amount,
currency: ret.input.currency
},
ouput: {
amount: ret.output.amount,
currency: ret.output.currency
},
status: {
status: statusUtils.getOrderStatusValue(ret.legacy_status)
}
});
break;
case ORDER_FILL:
orderArray.push({
input: {
amount: record._fields[0].properties.fromAmount,
currency: record._fields[0].properties.fromCurrency
},
ouput: {
amount: record._fields[0].properties.toAmount,
currency: record._fields[0].properties.toCurrency
},
status: {
status: record._fields[0].properties.orderStatus
}
});
break;
default:
orderArray.push({
input: {
amount: record._fields[0].properties.fromAmount,
currency: record._fields[0].properties.fromCurrency
},
ouput: {
amount: record._fields[0].properties.toAmount,
currency: record._fields[0].properties.toCurrency
},
status: {
status: record._fields[0].properties.orderStatus
}
});
break;
}
});
} catch (error) {
res.status(500).send(errorHandleing.FiveZeroZero)
}
res.status(200).json(orderArray);
});
response.records.map(async (record) => {...} is a sync function, it will return a promises array, your code will not wait until all action in {...} finish. This is main reason your request only takes small time to response.
Correct way, just wait until all jobs are finish:
let promises = response.records.map(async (record) => {...}
await Promise.all(promises); // waiting....