Sorry if my title is not very explicit I dont know how to explain this properly.
I am trying to use the distinct function for my app that uses loopback 3 and mongodb. It seems to work right but my endpoint wont hit the return inside my function.
This is my code
const distinctUsers = await sellerCollection.distinct('userId',{
hostId : host.id,
eventId:{
"$ne" : eventId
}
}, async function (err, userIds) {;
if(!userIds || userIds.length ==0)
return [];
const filter = {
where:{
id: {
inq: userIds
}
}
};
console.log("should be last")
return await BPUser.find(filter);
});
console.log(distinctUsers);
console.log("wtf??");
//return [];
If I uncomment the return [] it will send the return and later it will show the should be last, so even when I dont have the return it seems to finish. It is now waiting for the response. I dont like the way my code looks so any pointer of how to make this look better I will take it.
It looks like the sellerCollection.distinct takes a callback as one of it's parameters, therefore, you cannot use async/await with a callback-style function, since it's not a promise.
I would suggest turning this call into a promise if you'd like to use async/await:
function findDistinct(hostId, eventId) {
return new Promise((resolve, reject) => {
sellerCollection.distinct(
'userId',
{ hostId, eventId: { "$ne": eventId } },
function (error, userIds) {
if (error) {
reject(error);
return;
}
if (!userIds || userIds.length === 0) {
resolve([]);
return;
}
resolve(userIds);
}
)
})
}
Then, you can use this new function with async/await like such:
async function getDistinctUsers() {
try {
const hostId = ...
const eventId = ...
const distinctUsers = await findDistinct(hostId, eventId)
if (distinctUsers.length === 0) {
return
}
const filter = {
where: {
id: { inq: userIds }
}
}
const bpUsers = await BPUser.find(filter) // assuming it's a promise
console.log(bpUsers)
} catch (error) {
// handle error
}
}
Related
mongoCheckFile = async (fileData) => {
try {
let offerIdsValues = [];
//Extract all fileIds from CSV
fileData.data.forEach((element, idx, fileData) => {
fileIdsInCsv.push(`${element.fileId}`)
});
fileData.fileOffers.forEach(element => {
offerIdsValues.push(element.id);
});
let matchingClickDocs = [];
//The following functions are not waited for before returning
await Click.find({
$and: [ {fileId: { $in: fileIdsInCsv} }, { fileId: {$in: offerIdsValues} } ]
}).populate('furtherInfo').exec(async (error, clicks) => {
if (error) {
console.log(error);
} else {
await Promise.all(clicks.map(async (click) => {
const furtherDatas = await furtherData.findOne({ fileId: click.fileId });
if (furtherDatas) {
click.furtherInfo = furtherDatas;
await click.save();
matchingClickDocs.push(click);
//this pushes correct values and creates correct array.
}
}));
console.log(matchingClickDocs.length, 'first')
// This returns after global function with correct val
}
console.log(matchingClickDocs.length, 'second')
// This returns after above with correct val
});
console.log( matchingClickDocs.length, 'third')
// Returns first with incorrect Val.
}
catch (err) {
return (err);
}
};
This function returns before the promise returns its array.
I have console.logged the output as I run through and its all good until it comes to returning it.
I've hit a brick wall trying to return these values before the function returns.
I guess its a problem with the global function not awaiting the promises at the end.
I've tried a few different implementations but keep getting stuck on it.
I'm seeking guidance on how best to handle the following scenario. I'm fairly new to JS and async development, so I'm trying to understand the best way to handle this. I call one api (callAPI) which returns an array of items. I need to take those items and send them to another api (callAPI2) but that api doesn't have a bulk method, so I have to call the api for each item in the array. Below is how I have it structured: getArray promise returns the first array, I send the array to promise 2 (getIndividualData) where I loop and call the second api. I'm wondering if there are better ways to structure this? If I pass in a large array, I may need to pace the calls to the api so I don't get throttled... so maybe I need some version of Promise.all?
let getArray = function() {
return new Promise(function(resolve,reject) {
callApi.get().on('success', function(result, response) {
resolve(result);
});
});
}
let getIndividualData = function(arrayOfItems) {
return new Promise(function(resolve,reject) {
var responseArray = [];
for(var i = 0; i < arrayOfItems.length; i++) {
callApi2.get(arrayOfItems[i]).on('success', function(result, response) {
responseArray.push(result);
});
}
resolve(responseArray);
});
}
let failureCallback = function() {
return "Error!";
}
getArray().then(function(response) {
return getIndividualData(response);
}).then(function(finalArray) {
console.log(`The final array is ${JSON.stringify(finalArray)}`);
}).catch(failureCallback);
You can make a request for each item in a large array without getting throttled by implementing a concurrency throttler with a Set of Promises and async/await syntax. I've duplicated your code below, modifying the implementation of getIndividualData and passing in concurrency as an option.
let getArray = function() {
return new Promise(function(resolve,reject) {
callApi.get().on('success', function(result, response) {
resolve(result);
});
});
}
let getIndividualData = async function(arrayOfItems, { concurrency }) {
var promiseSet = new Set(),
responseArray = [],
i = 0;
while (i < arrayOfItems.length) {
if (promiseSet.size >= concurrency) {
await Promise.race(promiseSet)
}
const promise = new Promise(function(resolve,reject) {
callApi2.get(arrayOfItems[i]).on('success', function(result, response) {
resolve(result)
})
})
responseArray.push(promise.then(result => {
promiseSet.delete(promise)
return result
}))
i += 1
}
return Promise.all(responseArray)
}
let failureCallback = function() {
return "Error!";
}
getArray().then(function(response) {
return getIndividualData(response, { concurrency: 10 });
}).then(function(finalArray) {
console.log(`The final array is ${JSON.stringify(finalArray)}`);
}).catch(failureCallback);
Reformulating this to a helper that promisifies an object that has an .on('success') event handler and an async function for the top-level .then()ish code gives us something like this...
To pace the API calls, add in p-limit or similar to getIndividualData.
function promisifyOnSuccessObj(onSuccessObj) {
return new Promise((resolve) => {
onSuccessObj.on("success", (result, response) => resolve(result));
// TODO: what about `response`?
// TODO: onSuccessObj.on('error')..?
});
}
function getIndividualData(arrayOfItems) {
// Returns an array of promises
return arrayOfItems.map((item) =>
promisifyOnSuccessObj(callApi2.get(item)),
);
}
async function doThings() {
const result = await promisifyOnSuccessObj(callApi.get());
const individualDatas = await Promise.all(getIndividualData(result));
console.log(`The final array is ${JSON.stringify(individualDatas)}`);
}
You could combine Promise.all, map and async...await syntax and in the end get one array of resolved individual promises based on the previous resolved array promise.
const mockApi = {
request: (response) => {
return new Promise((resolve, reject) => {
setTimeout(() => resolve(response), 1000)
})
},
getArray() {
return this.request(['foo', 'bar', 'baz'])
},
getItem(item) {
return this.request(`Resolved: ${item}`)
}
}
async function getData() {
const array = await mockApi.getArray();
const final = await Promise.all(array.map(e => mockApi.getItem(e)));
console.log(final)
}
getData()
I am working on a site using Express.js, node.js, Axios, and ejs. I am making REST calls to a Oracle SQL REST services using Axios. I am having trouble working with Promises or Async/Await. I could use some guidance, if possible.
I have a repository layer to interface with the Oracle DB. For example:
dataaccess.js
const axios = require('axios');
exports.IsManufacturerCategory = function (categoryId) {
axios.get(`DB ADDRESS ${categoryId}`)
.then(response => {
console.error('GET IsManufacturerCategory categoryId = ' + categoryId);
console.error('Response = ' + JSON.stringify(response.data));
return (response.data);
})
.catch(rej => {
console.error('ERROR IsManufacturerCategory categoryId = ' + categoryId);
console.error('ERR = \n' + rej.data);
return (rej.data);
});
}
Which is called in my middleware. When I call var isManufacturerCat = exports.IsManufacturerCategory(categoryId); it is undefined. I am attempting to use the data retrieved from the Axios call to return a ejs view to my router, which I can provide if needed.
category.js
var isManufacturerCat = exports.IsManufacturerCategory(categoryId);
if (isManufacturerCat) {
var models = dataaccess.GetCategorySubCategories(categoryId);
return ("manufacturers", {
data: {
Canonical: cononical,
Category: category,
IsAManufacturerCategory: iAManufacturerCat,
Models: models
}
});
}
I am open to any advice in my project structure, usage of Promises, Async/Await, etc.
Thank you in advance.
EDIT
After working with some of the answers given, I have made some progress but I am having issues with layers of async calls. I end up getting into a spot where I need to await a call, but I am in a function that I am not able/do not want to do so (i.e. my router).
indexMiddleware.js
exports.getRedirectURL = async function (fullOrigionalpath) {
if (fullOrigionalpath.split('.').length == 1 || fullOrigionalpath.indexOf(".aspx") != -1) {
if (fullOrigionalpath.indexOf(".aspx") != -1) {
//some string stuff to get url
}
else if (fullOrigionalpath.indexOf("/solutions/") != -1) {
if (!fullOrigionalpath.match("/solutions/$")) {
if (fullOrigionalpath.indexOf("/t-") != -1) {
//some stuff
}
else {
var solPart = fullOrigionalpath.split("/solutions/");
solPart = solPart.filter(function (e) { return e });
if (solPart.length > 0) {
var solParts = solPart[solPart.length - 1].split("/");
solParts = solParts.filter(function (e) { return e });
if (solParts.length == 1) {
waitForRespose = true;
const isASolutionCategory = await dataaccess.isASolutionCategory(solParts[0]); // returns void
if (isASolutionCategory != undefined && isASolutionCategory.length > 0 && isASolutionCategory[0].Count == 1) {
// set redirecturl
}
}
else {
redirecturl = "/solutions/solutiontemplate";
}
}
}
}
}
else if (URL STUFF) {
// finally if none of the above fit into url condition then verify current url with Category URL or product url into database and if that matches then redirect to proper internal URL
if (fullOrigionalpath.lastIndexOf('/') == (fullOrigionalpath.length - 1)) {
fullOrigionalpath = fullOrigionalpath.substring(0, fullOrigionalpath.lastIndexOf('/'));
}
waitForRespose = true;
const originalURL = await exports.getOriginalUrl(fullOrigionalpath); //returns string
redirecturl = originalURL;
return redirecturl;
}
}
if (!waitForRespose) {
return redirecturl;
}
}
exports.getOriginalUrl = async function (friendlyUrl) {
var originalUrl = '';
var urlParts = friendlyUrl.split('/');
urlParts = urlParts.filter(function (e) { return e });
if (urlParts.length > 0) {
var skuID = urlParts[urlParts.length - 1];
const parts = await dataaccess.getFriendlyUrlParts(skuID); //returns void
console.log("Inside GetOriginalUrl (index.js middleware) FriendlyUrlParts: " + parts);//undefined
if (parts != undefined && parts != null && parts.length > 0) {
//some stuff
}
else {
// verify whether it's category URL then return the category local URL
console.log('Getting CategoryLocalUrl');
const categoryLocalUrl = await dataaccess.getCategoryLocalUrl(friendlyUrl); // returns void
console.log('CategoryLocalUrl Gotten ' + JSON.stringify(categoryLocalUrl)); //undefined
if (categoryLocalUrl != undefined && categoryLocalUrl.length > 0) {
//set originalUrl
return originalUrl;
}
}
}
else { return ''; }
}
index.js router
router.use(function (req, res, next) {
//bunch of stuff
index.getRedirectURL(url)
.then(res => {
req.url = res;
})
.catch(error => {
console.error(error);
})
.finally(final => {
next();
});
}
I am getting undefined in my console.logs after the awaits. I'm not really sure what I'm doing I guess.
Let's start with dataaccess.js. Basically, you're exporting a function that's doing async work, but the function isn't async. Most people want to be able to utilize async/await, so rather than have IsManufacturerCategory accept a callback function, it would better to have the function return a promise. The easiest way to do that is to make the function an async function. Async functions return promises which are resolved/rejected more easily than by returning an explicit promise. Here's how that could be rewritten:
const axios = require('axios');
exports.IsManufacturerCategory = async function (categoryId) {
try {
const response = await axios.get(`DB ADDRESS ${categoryId}`);
console.log('GET IsManufacturerCategory categoryId = ' + categoryId);
console.log('Response = ' + JSON.stringify(response.data));
} catch (err) {
console.error('ERROR IsManufacturerCategory categoryId = ' + categoryId);
console.error('ERR = \n' + rej.data);
throw err;
}
}
Note that I'm using console.error only for errors. Because you wanted to log some error related details, I used a try/catch statement. If you didn't care about doing that, the function could be simplified to this:
const axios = require('axios');
exports.IsManufacturerCategory = async function (categoryId) {
const response = await axios.get(`DB ADDRESS ${categoryId}`);
console.log('GET IsManufacturerCategory categoryId = ' + categoryId);
console.log('Response = ' + JSON.stringify(response.data));
}
This is because async functions automatically swallow errors and treat them as rejections - so the promise returned by IsManufacturerCategory would be rejected. Similarly, when an async function returns a value, the promise is resolved with the value returned.
Moving on to category.js... This looks strange because you're accessing IsManufacturerCategory from the exports of that module when I think you mean to access it from the import of the dataaccess module, right?
Inside this function, you should put any async work in an async function so that you can use await with function that return promises. Here's how it could be rewritten:
const dataaccess = require('dataccess.js');
async function validateManufacturerCat(categoryId) {
const isManufacturerCat = await dataaccess.IsManufacturerCategory(categoryId);
if (isManufacturerCat) {
const models = await dataaccess.GetCategorySubCategories(categoryId);
return ({
manufacturers: {
data: {
Canonical: cononical,
Category: category,
IsAManufacturerCategory: iAManufacturerCat,
Models: models
}
}
});
}
}
validateManufacturerCat(categoryId)
.then(res => {
console.log(res);
})
.catch(err => {
console.error(err);
});
A couple notes:
I changed the return value in the if statement to be a single value. You should try to always return a single value when working with promises and async/await (since you can only resolve/return one value).
I see lots of functions starting with capital letters. There's a convention in JavaScript where functions with capital letters are constructor functions (meant to be invoked with the new keyword).
Here is a working example. Note that you need a callback function to get the data when it is available from the promise
//client.js
const axios = require("axios")
exports.IsManufacturerCategory = function (url, callback) {
axios.get(url)
.then(response=>
{
callback(response.data);
})
.catch(error=>{
callback( error);
})
};
//app.js
const client = require('./client');
function process(data) {
console.log(data)
}
client.IsManufacturerCategory("http://localhost:3000/persons",process);
documentation
I have been wondering why adding a callback to the mongoose findOneAndUpdate function results in saving the data twice to the DB?
public async addPersonAsFavorite(userId: string, friendId: string) {
if (!await this.isPersonAlreadyFriend(userId, friendId)) {
const friendList = FriendsList.findOneAndUpdate(
{ _id: userId },
{ $push: { friendsList: friendId } },
{ upsert: true, new: true },
(err, data) => {
if (err) console.error(err);
return data;
}
);
return friendList;
}}
public async isPersonAlreadyFriend(userId: string, friendId: string) {
let isFriendFound = false;
await FriendsList.findById(userId, (err, data) => {
if (data) {
console.log(data.friendsList);
}
if (err) console.error(err);
if (data && data.friendsList.indexOf(friendId) > -1) {
isFriendFound = true;
console.log('already friend');
} else {
console.log('not friend');
isFriendFound = false;
}
})
return isFriendFound;
}
If i remove the callback, the data only gets saved once.
EDIT: added second piece of code and new question.
If someone spams the button to add friend. The friend will be added multiple times because before the first friend is added and the check can be done to prevent this it has already added the person multiple times.
How can i make sure that it completes the write to the DB before allowing the function to be called again?
Maybe the problem is in isPersonAlreadyFriend method, because you are trying to call it using async await but then you are passing a callback, what makes the method not return a promise. The rigth way to use promises with mongodb should be something like this:
public async isPersonAlreadyFriend(userId: string, friendId: string) {
let isFriendFound = false;
const data = await FriendsList.findById(userId);
if (data) {
console.log(data.friendsList);
}
if (data && data.friendsList.indexOf(friendId) > -1) {
isFriendFound = true;
console.log('already friend');
} else {
console.log('not friend');
isFriendFound = false;
}
return isFriendFound;
}
Try with this and let me know if it helps
I am using an external api that I use as my datasource. That api gives its data back based on a pagination like type.
So I can get data for page 1, 2, 3 etc. But the problem that I have is that I need all data at once, the api doesn't support this so I write it myself.
I have made the actual call code:
function getTransactionPart(start){
return new Promise(function(resolve, reject) {
const options = {
url: 'myurl?limit=40&start='+start,
json: true
};
let result = {total_transactions : 0, transactions : []};
request(options, function (error, response, body) {
if (error) {
return reject(result);
}
body.data.forEach(function (transaction) {
result.transactions.push({
timestamp: transaction.timestamp,
amount: transaction.amount,
confirmed: transaction.confirmed
});
});
result.total_transactions = body.total
return resolve(result);
})
})
}
The above code returns me the expected results with the limit that I gave. Also I get a number back (result.total_transactions) when this is more than 40 I need to make another call with 40 as start etc etc.
The code where I need to combine it:
function getTransactions(start) {
return new Promise(function(resolve, reject) {
getTransactionPart(start).then(result => {
if(result.total_transactions > 40) {
//next call
} else {
// return all?
}
console.log(result)
}).catch(error => {
console.error(error);
return r.json({
})
});
})
}
So I do the first call with getTransactionPart(0) after that the method itself needs to combine the result form all the sub calls and return the whole result as expected. How can I do this with recursion and promises?
This is easier if you use an async function and await the request:
async function getTransactions(start) {
const result = [];
for(let pos = start; ; pos += 40) {
const { total_transactions, transactions } = await getTransactionPart(pos);
result.push(...transactions);
if(total_transactions < 40) break;
}
return result;
}
For sure you could also do this recursively, but do you really need that?
async function getTransactions(start) {
const { total_transactions, transactions } = await getTransactionPart(pos);
if(total_transactions < 40)
return transactions;
return transactions.concat(await getTransactions(start + 40));
}