Develop Reference

How to get data with axios from all api pages?

What am I doing wrong? I want to get data from all pages in api. After adding the while it stopped working, but I don't know how to start the page loop differently!
getCustomers: function() {
let url = '/crm/customer/';
return axios.get(url).then((response) => {
this.customers = response.data.results;
if (this.customers.length === 100) {
let i = 2;
axios.get('/crm/customer/?page=' + i).then((response) => {
this.c = response.data.results;
i += 1;
for (let item of this.c.values()) {
this.customers.push(item);
}
while (this.c.length === 100) {
axios.get('/crm/customer/?page=' + i).then((response) => {
this.c = response.data.results;
i += 1;
for (let item of this.c.values()) {
this.customers.push(item);
}
}).catch( error => {}).finally(() => (global_waiting_stop()));
}
}).catch( error => {}).finally(() => (global_waiting_stop()));
}
}).catch( error => {}).finally(() => (global_waiting_stop()));
},

What I would do is, first, use an async function called getPageOfResults:
async function getPageOfResults(page) {
const response = await axios.get('/crm/customer/?page=' + page);
return response.data.results; // .results.values()? I don't know
}
Then, inside another async function, you can have a loop that does what you want:
async function getAllResults() {
const customers = [];
let lastResultsLength = 100;
let page = 1;
while (lastResultsLength === 100) {
const newResults = await getPageOfResults(page);
page++;
lastResultsLength = newResults.length;
customers = customers.concat(newResults);
}
return customers;
}
So you've got a variable keeping track of what page your on, and you keep on getting new pages until you get a page with less than 100 results.
You're adding all the results together with the concat function, and returning the entire list at the end.

NodeJS REST API wait for response

I am trying to get a bunch of ID's from an API and then form a sequence of requests which would make further calls to an API to fetch some parameters. These would be totaled and i expect the output results to be pushed as JSON array.
The problem is REST call is async and i've put a promise but not sure when to resolve the promise back to the calling function, the rest call some times take a second or 2 to respond back.
I would like know at what point can i resolve the promise or how to know when the totals have been computed ?
The Route
app.get("/sonar/:x_id",function(req,resp) {
getRestSonar(req.params.x_id).then(function (fromResolve) {
resp.send(fromResolve);
});
});
The function with promise which makes the rest call loops
var getRestSonar = function(requestX) {
return new Promise(function(resolve,reject) {
var unirest = require("unirest");
var reqx = unirest("GET", "http://sonarqubexxServer/api/projects");
var outputJson = {
table: []
};
reqx.end(function (res) {
if (res.error) throw new Error(res.error);
// console.log(res.body);
var result = res.body;
//var needle = req.params.csi_id;
var needle = requestX;
var TotalDuplicateLines = 0;
var TotalBugs = 0;
var TotalNcloc = 0;
var TotalCodeSmells = 0;
var TotalVulnerabilities = 0;
for (var i=0;i<result.length;i++) {
if (result[i].nm.indexOf(needle) !== -1) {
console.log(result[i].k);
var queryUrl = "http://sonarqubexxServer/api/resources?resource="+result[i].k+"&metrics=code_smells,bugs,vulnerabilities,ncloc,coverage,duplicated_lines&format=json"
console.log(queryUrl);
var subrequest = unirest("GET",queryUrl);
subrequest.end(function (resXX) {
if (resXX.error);
var resXXResult = resXX.body;
for (var i=0;i<resXXResult.length;i++) {
// var duplicateData = resXXResult[0].msr.filter(item => item.key == 'duplicated_lines');
resXXResult[i].msr.forEach(m => {
if (m.key === 'duplicated_lines') {
console.log('Duplicated Lines ' + m.val);
TotalDuplicateLines += m.val;
}
else if(m.key === 'bugs' ) {
console.log('Bugs ' + m.val);
TotalBugs += m.val;
}
else if(m.key === 'ncloc' ) {
console.log('Lines of Code ' + m.val);
TotalNcloc += m.val;
}
else if(m.key === 'code_smells' ) {
console.log('Code Smells ' + m.val);
TotalCodeSmells += m.val;
}
else if(m.key === 'vulnerabilities' ) {
console.log('Vulnerabilities ' + m.val);
TotalVulnerabilities += m.val;
outputJson.table.push({totduplines:TotalDuplicateLines},{totVul:TotalVulnerabilities});
}
});
console.log("Iam here with I :: " + i);
if (i === (resXXResult.length - 1)) {
//Should i resolve here makes no sense
console.log("Resolved the promise now..");
}
//The for ends here
}
// I see this is a bad place to resolve..
resolve(outputJson);
});
}
}
});
});
}
EDIT : As suggested in the comments, split the calls into smaller
sections
Now, i fetch the api calls seperatly create an array out of it, then use promises to call back to the API ? how do i resolve each call by looping over it ?
When i try to loop it always resolves request[0] and then comes out of the promise, how can i create a promise array and wait for them to complete ?
app.get("/sonar/:csi_id",function(req,resp) {
var collectiveResult = [];
getRestSonar(req.params.csi_id).then(function (fromResolve) {
return splitReqUrl(fromResolve);
}).then(function(fromSplitUrl) {
console.log("I am from split url ::::" + fromSplitUrl);
return getSubSonarProperties(fromSplitUrl);
}).then(function(fromsubSonar) {
collectiveResult.push(fromsubSonar);
console.log("+++++++++++++++++++++++++++");
console.log(fromsubSonar);
resp.send(collectiveResult);
});
});
var getSubSonarProperties = function(getUrl) {
return new Promise(function(resolve,reject) {
var getSubRest = require("unirest");
console.log("Attempting to GET " + getUrl);
var req = getSubRest("GET",getUrl);
var outputJson = {
table: []
}
var TotalDuplicateLines = 0;
var TotalBugs = 0;
var TotalNcloc = 0;
var TotalCodeSmells = 0;
var TotalVulnerabilities = 0;
req.end(function (res) {
if (res.error);
var resXXResult = res.body;
resolve(resXXResult);
});
});
}
var splitReqUrl = function(request) {
return new Promise(function(resolve,reject) {
resolve(request[1]);
//for(var i=0; i< request.length; i++) {
// resolve(request[i]);
//}
});
}
var getRestSonar = function(requestX) {
return new Promise(function(resolve,reject) {
var unirest = require("unirest");
var reqx = unirest("GET", "http://sonarqubexxx/api/projects");
var outputJson = {
table: []
};
reqx.end(function (res) {
if (res.error) throw new Error(res.error);
// console.log(res.body);
var result = res.body;
//var needle = req.params.csi_id;
var needle = requestX;
var queryArray = [];
for (var i=0;i<result.length;i++) {
if (result[i].nm.indexOf(needle) !== -1) {
console.log(result[i].k);
var queryUrl = "http://sonarxxx/api/resources?resource="+result[i].k+"&metrics=code_smells,bugs,vulnerabilities,ncloc,coverage,duplicated_lines&format=json"
//console.log(queryUrl);
queryArray.push(queryUrl);
}
if (i === (result.length - 1)) {
resolve(queryArray);
}
}
});
});
}

Problem
First of all the problem with your solution is that you're trying to make everything inside a single big new Promise(...) creator.
Even if you manage to make that work it's still a common anti-pattern as Promises are made to be chained using the .then(...) method.
As pointed out by Roamer-1888 there oughta be a fork of unirest that handles Promises directly instead of requiring callbacks as in your example, but let's stick with your version of unirest here.
Solution
So what you need to be doing is create a Promise chain to handle the different steps of your code and pass the results down the chain.
Your steps seem to be:
Make the first call to retrieve initial results.
Filter the results based on the requestX input.
For each item left, make several calls to obtain more data.
Put everything back into an outputJson object.
Basically the only async steps are 1 and 3, but it might be ok to add a third step to build your outputJson and pass it downstream.
So let's start with the first step.
1. Make the first call
In the first link of the Promise chain we need to retrieve the initial results with your first unirest call:
new Promise((resolve, reject) => {
unirest("GET", "http://sonarqubexxServer/api/projects")
.end((res) => {
if (res.error) {
reject(res.error);
} else {
resolve(res.body);
}
});
})
See in this example I already checked if the response contains an error and fired a rejection in that case, otherwise I resolve the promise with the body (the data we need).
The Promise we created above will throw an error if the request fails, and will downstream the body of the response if everything goes fine.
2. Filtering and Sub-calls
Now then we can go ahead and use the full potential of Promises with the .then(...) method:
new Promise((resolve, reject) => {
unirest("GET", "http://sonarqubexxServer/api/projects")
.end((res) => {
if (res.error) {
reject(res.error);
} else {
resolve(res.body);
}
});
}).then((results) => {
results = results.filter((result) => {
return result.nm.indexOf(request) != -1;
});
return Promise.all(results.map((result) => {
return new Promise((resolve, reject) => {
var queryUrl = "http://sonarqubexxServer/api/resources?resource=" + result.k + "&metrics=code_smells,bugs,vulnerabilities,ncloc,coverage,duplicated_lines&format=json"
unirest("GET", queryUrl)
.end((res) => {
if (res.error) {
reject(res.error);
} else {
resolve(res.body);
}
});
})
}))
})
In this step I used some Array methods to make the code cleaner and Promise.all to handle several promises together.
Array.filter is a method which iterates an array and checks for each item if it should be kept in the filtered output or not. So, in your case, we want to keep only those items where result.nm.indexOf(request) != -1.
Array.map is a method which iterates an array and converts each item to something else. Basically the function you provide takes each item as input, converts it to something else and then replaces this new value to the old one in the output array.
Finally Promise.all accepts an array of Promises and returns a Promise itself. This returned Promise will resolve when all the given Promises resolve and will pass downstream an array which items are the results of each single Promise.
So by writing Promise.all(results.map((results) => { return new Promise(...) })) we convert each result in the results array into a Promise that executes the result-specific call and put it into the output array of Promises which is fed to Promise.all so they get executed at once.
3. Build the outputJSON
Now the Promise chain outputs the result of Promise.all which is an array of all the results of each Promise, which are the results of each sub-call.
We can then simply take the downstream data and use your nested iterations to build the outputJSON to be passed downstream:
new Promise((resolve, reject) => {
unirest("GET", "http://sonarqubexxServer/api/projects")
.end((res) => {
if (res.error) {
reject(res.error);
} else {
resolve(res.body);
}
});
}).then((results) => {
results = results.filter((result) => {
return result.nm.indexOf(request) != -1;
});
return Promise.all(results.map((result) => {
return new Promise((resolve, reject) => {
var queryUrl = "http://sonarqubexxServer/api/resources?resource=" + result.k + "&metrics=code_smells,bugs,vulnerabilities,ncloc,coverage,duplicated_lines&format=json"
unirest("GET", queryUrl)
.end((res) => {
if (res.error) {
reject(res.error);
} else {
resolve(res.body);
}
});
})
}))
}).then((allResults) => {
var TotalDuplicateLines = 0;
var TotalBugs = 0;
var TotalNcloc = 0;
var TotalCodeSmells = 0;
var TotalVulnerabilities = 0;
var outputJson = {
table: []
};
for (var i = 0; i < allResults; i++) {
for (var j = 0; j < allResults[i].length; j++) {
allResults[i][j].msr.forEach(m => {
if (m.key === 'duplicated_lines') {
TotalDuplicateLines += m.val;
}
else if (m.key === 'bugs') {
TotalBugs += m.val;
}
else if (m.key === 'ncloc') {
TotalNcloc += m.val;
}
else if (m.key === 'code_smells') {
TotalCodeSmells += m.val;
}
else if (m.key === 'vulnerabilities') {
TotalVulnerabilities += m.val;
outputJson.table.push({ totduplines: TotalDuplicateLines }, { totVul: TotalVulnerabilities });
}
});
}
}
return outputJson;
})
If your return this long Promise chain in your getRestSonar(request) function, then you could write getRestSonar(request).then((outputJson) => { ... do something with your outputJson ... })

Issue after nesting async.eachOfSeries

I was using nested for loops to get a thing done. I figured out that the HTTP call and the database query are asynchronous and eventually gives undesired output. So I used async.eachOfSeries. The execution gets stopped just after it enters the inner most async.eachOfSeries in the first iteration itself. I suppose I have made some error is structuring the code and in returning the callbacks. I need to know basically two things from you:
What is stopping the execution of the code in the first iteration itself?
Will this approach solve the asynchronicity issue? Or is there any better approach to solve it?
Your help would be appreciated.
async.eachOfSeries(res, function (value, camp, callback) {
let _id = res[camp]._id;
let arr = res[camp].campaignID;
async.eachOfSeries(arr, function (value1, i, callback) {
let users = arr[i].users;
let id = arr[i].id;
let loop = Math.ceil(users / 1000);
let limit = 0,
offset = 0;
for (let j = 0; j < loop; j++) {
if (users > 1000) {
limit = 1000;
users -= limit;
} else {
limit = users;
}
console.log(limit + " limit " + offset + " offset");
var start = Date.now();
while (Date.now() < start + 100) {}
const request = mailjet
.get("messagesentstatistics")
.request({
"CampaignID": id,
"AllMessages": true,
"Limit": limit,
"Offset": offset
})
request
.then((result) => {
let data = result.body.Data;
var loop = 0;
async.eachOfSeries(data, function (value2, val, callback) {
console.log("ayyuuuuu");
let jsonObj = data[val];
let email = jsonObj.ToEmail;
jsonObj['retailer'] = res[camp].retailer;
jsonObj['summary'] = 'f';
let tempObj = {};
tempObj[id] = jsonObj;
let options = {
new: true
};
let campId = id;
User.addCampaignResponse(email, campId, tempObj, options, function (err, results) {
if (err) {
throw err;
} else {
Campaign.updateResponse(_id, function (err, results2) {
if (err)
throw err;
}) // console.log(results);
}
})
}, function (err) {
callback();
})
})
.catch((err) => {
console.log(err);
})
offset += limit;
}
}, function (err) {
callback();
})
}, function (err) {
callback(undefined, "doneeeeee");
})

NodeJS - sum async results in recursive function

Hi there I have the following python recursive function that sum values from all child nodes and I want to port in NodeJS but I have some problem with async calls.
def getTree(parent_id, level=1):
c.execute('select * from users where parent_id=?', (parent_id,))
rows = c.fetchall()
total = 0
for child in children:
total += getAsyncValue(child.id)
total += getTree(child.id, level+1)
return total
I tried to do this but I need probably to chain it with promises because the total count is not available while I loop as I get it from an async function
getTree = function(parent_id, level=1) {
c.all("select * from users where parent_id="+parent_id, function(err, children) {
var total = 0;
children.forEach(function(child) {
total += getAsyncValue(child.id)
total += getTree(child.id, level+1)
});
return total;
});
}

without seeing getAsyncValue I can't provide a complete answer - however
var getAsyncValue = function(id) {
return new Promise((resolve, reject) => {
// resolve some value some how
});
};
// helper to make the getTree function "nicer"
c.allAsync = function(str) {
return new Promise((resolve, reject) =>
this.all(str, (err, children) => {
if (err) {
return reject(err);
}
resolve(children);
})
);
};
var getTree = function(parent_id, level=1) {
return c.allAsync("select * from users where parent_id="+parent_id).then(children =>
Promise.all(children.map(child =>
Promise.all([getAsyncValue(child.id), getTree(child.id, level+1)])
.then(([a, b]) => a + b)
)).then(results => results.reduce((a, b) => a + b))
);
};
I think using async/await, the code can be written as:
var getAsyncValue = async function(id) {
return new Promise((resolve, reject) => {
// resolve some value some how
});
};
// helper to make the getTree function "nicer"
c.allAsync = async function(str) {
return new Promise((resolve, reject) =>
this.all(str, (err, children) => {
if (err) {
return reject(err);
}
resolve(children);
})
);
};
var getTree = async function(parent_id, level=1) {
let children = await c.allAsync("select * from users where parent_id="+parent_id);
let total = 0;
for (let i = 0; i < children.length; i++) {
let child = children[i];
total += await getAsyncValue(child.id);
total += await getTree(child.id, level + 1);
}
return total;
};

Convert to CSV in a downloadable format

I want to convert an array of objects into CSV.
Please have a look at the code. What I am doing wrong? I am getting an array when I console.log("csv",array) which is passed from printAccountsGa. Moreover, I have declared JSONGA = [] globally but still I get array with every object on consoling. But when I pass that to csvConverter function it displays undefined.
function printAccountsGa(results) {
if (results && !results.error) {
var accounts = results.items;
for (var i = 0, account; account = accounts[i]; i++) {
//console.log('Account Id: ' + account.id);
listProperties(account.id);
//console.log('Account Name: ' + account.name);
}
console.log(JSONGA);
csvConverter(JSONGA);
}
}
function printAccountsGtm(results) {
console.log("result is",results);
if (results && !results.error) {
var accounts = results.accounts;
console.log(accounts);
for (var i = 0, account; account = accounts[i]; i++) {
//console.log('Account Id: ' + account.accountId);
listContainers(account.accountId);
//console.log('Account Name: ' + account.name);
}
}
}
function listProperties(accountID){
var request = gapi.client.analytics.management.webproperties.list({
'accountId': accountID
});
request.execute(printProperties);
}
function printProperties(results){
if (results && !results.error) {
var properties = results.items;
//console.log(properties[0].accountId);
$.each(properties,function(element){
//console.log(properties[element].accountId);
var tempObj={};
tempObj={
'AccountID':properties[element].accountId,
'PropertyID':properties[element].id,
'name':properties[element].name,
}
JSONGA.push(tempObj);
})
}
}
function csvConverter(array){
console.log("csv",array);
var result = array.map(function(d){
return JSON.stringify(values(d));
})
.join('\n')
.replace(/(^\[)|(\]$)/mg, '');
window.open('data:text/csv;charset=utf-8,' + escape(result));
}
window.onload = function(){
var el = document.getElementById('auth-button-ga');
el.addEventListener('click', authorizeGa);
var elt = document.getElementById("auth-button-gtm");
elt.addEventListener('click', authorizeGtm);
}