If you have to loop and make a bunch of calls to a repository or gateway in my case, how do I do that asynchronously meaning not wrapping my async calls inside a synchronous for loop?
For example, what would be a better approach (restructuring this code) to loop through a set of ids, and make the call to find() below like I'm trying to do?
The goal: I want to take an array of ids, iterate them, and during each iteration, use the id to call find() on my gateway to go get the object for that id, then stuff it into a final array in which I'll return when all said and done.
What I'm using:
q (for promises)
co-pg (to hit the database)
someModule.js
var _gateway = require('./database/someGateway');
var cars = [];
var car;
for (var i = 0; i < results.docs.length; i++){
var carId = results.docs[i].carId;
_gateway.find(carId)
.then(function(data){
console.log('data[0]: ' + data[0].id);
cars.push(data[0]);
})
.done();
}
console.log("cars: " + cars.length); // length here is 0 because my asyn calls weren't done yet
result(cars);
someGateway.js
'use strict';
var Q = require('q');
var _carModel = require('../../models/car');
module.exports = {
models: {
car: _carModel
},
find: _find
};
function _find(carId)
{
return _carModel.find(carId);
};
carModel.js
'use strict';
var Q = require('q');
var pg = require('co-pg')(require('pg'));
var config = require('../../models/database-config');
var car = module.exports = {};
car.find = Q.async(function *(id)
{
var query = 'SELECT id, title, description FROM car WHERE id = ' + id;
var connectionResults = yield pg.connectPromise(config.connection);
var client = connectionResults[0];
var done = connectionResults[1];
var result = yield client.queryPromise(query);
done();
console.log("result.rows[0].id: " + result.rows[0].id);
return result.rows;
});
so I need help understanding how to refactor my code in someModule.js to get that working properly, so that I make a call to find() for each id, stuff each found car into the array, then return the array. The carModel code is async. It goes out to a physical database to perform the actual query lookup.
UPDATE #1
Ok after a couple more hours of trying all sorts of sh** (q.all(), and a ton of other combinations of callback code, etc.) here's what I have at this point:
someModule.js
var _data;
var Q = require('q');
var _solrClient = require('../models/solr/query');
var _solrEndpoint = "q=_text&indent=true&rows=10";
var _postgreSQLGateway = require('./database/postgreSQLGateway');
module.exports = {
data: function(data){
_data = data;
},
find: function (text, result){
if(!searchText){
result(null);
};
_solrClient.query(endpoint, function(results){
var carIds = [];
var cars = [];
var car;
for (var i = 0; i < results.docs.length; i++){
carIds.push(results.docs[i].carId);
}
for (var i = 0; i < carIds.length; i++) {
var car = _postgreSQLGateway.find(carIds[i], function(o){
console.log("i: " + i);
});
};
});
}
};
someGateway.js
'use strict';
var Q = require('q');
var _carModel = require('../../models/postgreSQL/car');
module.exports = {
models: {
car: _carModel
},
find: _find
};
function _find(carId, foundCar)
{
console.log("CALL MADE");
_carModel.find(carId)
.then(function(car){
console.log("car: " + car[0].id);
foundCar(car);
});
};
carModel.js
[same code, has not changed]
Of course I noticed that the for loop fires off all my function calls asyncronously and so when I console.write the i, it's 10 because the for loop is done but then as we know, the rest of the console.logs happen later after the callbacks are done.
So I still can't get this working right...
Also when I was playing around I started down this path but it ended at a brick wall:
var find = Q.async(function(carIds, cars)
{
var tasks = [];
var foundCars = [];
for (var i = 0; i < carIds.length; i++) {
tasks.push(_postgreSQLGateway.find(carIds[' + i + ']));
};
Q.all([tasks.join()]).done(function (values) {
for (var i = 0; i < values.length; i++) {
console.log("VALUES: " + values[0]);
foundCars.push(values[0]);
}
cars(foundCars);
});
});
I ended up with [object promise] every time for values[i] instead of a car for value[i]
I don't know the Q promises library, but here's a solution using generic Promises built into node.js. This runs all the requests in parallel and then when all results have been collected, it runs the final .then() handler with all the results:
var _gateway = require('./database/someGateway');
var promises = [];
for (var i = 0; i < results.docs.length; i++) {
promises.push(_gateway.find(results.docs[i].carId).then(function (data) {
console.log('data[0]: ' + data[0].id);
return data[0];
}));
}
Promise.all(promises).then(function(cars) {
// cars will be an array of results in order
console.log("cars: " + cars.length);
result(cars);
});
Individual promise libraries (like the one I know Bluebird) have features built in that lets you do this kind of activity in even less code, but I've intentionally kept this answer to just using standard promise features.
This is potentially really easy with the vanilla Promise API from es6 (and replicated by Bluebird and other libs). First map the IDs to an array of promises:
var promises = results.docs.map(function(doc) {
return _gateway.find(doc.carId);
});
Then create a promise for the aggregate result:
var allDone = Promise.all(promises);
Then inside the done() callback of the aggregate promise, you'll have a final array of results, in the same length and order as the carId array:
allDone.then(function(results) {
// do something with "results"
});
Related
app.get("/indsalesx/:store/:mm", (req, res) => {
connect();
let ddd = [];
let staffarray = [{}];
let store = req.params.store;
let mm = req.params.mm;
const SP = mongoose.model(`sales${store}`, Sales);
let num = stafflist[store].length - 1;
for (i = 0; i <= num; i++) {
let staffname = stafflist[store][i];
let calc = 0;
SP.find(
{ v_salesperson: stafflist[store][i], v_month: mm },
"v_amount",
(err, doc) => {
let t = doc.length - 1;
doc.map((res) => {
calc = calc + res.v_amount;
});
ddd.name = staffname;
ddd.amount = calc;
staffarray.push(ddd);
}
);
}
console.log(staffarray);
});
The issue I have is: Why is staffarray returning an empty array? staffarray was declared as an empty array of objects, and in a loop function, objects were pushed to to array. But when I console.log(staffarray), it returns the empty array of objects declared initially.
Any help on what to do?
When using find(), you can use 2 approaches.
Pass a callback function
await the function to execute and return the results.
It appears that you used the first approach which means that you are passing a callback into the find() method which handles the result once received.
The console.log() code line will execute before the result will return since it's the next line to execute after the for loop.
So, let's go through what it happening here:
Javascript is executing the find() code line.
That line of code is being placed in the web API which are the pieces of the browser in which concurrency kicks in and makes the call to the server for us.
The console.log() line is being executed with an empty array (since the results haven't been received yet.
After some time, results came back and the callback is being set in the callback queue.
The JS event loop takes the callback from the callback queue and executes it.
This is part of the javascript event loop. you could read more about this here
Mongoose documentation: Model.find()
you can use for of with async/await instead of for
app.get("/indsalesx/:store/:mm", async(req, res) => {
connect();
let ddd = [];
let staffarray = [{}];
let store = req.params.store;
let mm = req.params.mm;
const SP = mongoose.model(`sales${store}`, Sales);
let num = stafflist[store].length - 1;
var list = Array.from(Array(num).keys());
for (let i of list) {
let staffname = stafflist[store][i];
let calc = 0;
let doc = await SP.find(
{ v_salesperson: stafflist[store][i], v_month: mm },
"v_amount"
);
let t = doc.length - 1;
doc.map((res) => {
calc = calc + res.v_amount;
});
ddd.name = staffname;
ddd.amount = calc;
staffarray.push(ddd);
}
console.log(staffarray);
});
I have been able to solve it, all I needed was proper structuring with the async and await statements.
app.get("/indsalesx/:store/:mm", async (req, res) => {
connect();
let ddd = {};
let staffarray = [];
let store = req.params.store;
let mm = req.params.mm;
const SP = mongoose.model(`sales${store}`, Sales);
let num = stafflist[store].length - 1;
for (i = 0; i <= num; i++) {
let staffname = stafflist[store][i];
let calc = 0;
await SP.find(
{ v_salesperson: stafflist[store][i], v_month: mm },
"v_amount",
(err, doc) => {
let t = doc.length - 1;
doc.map((res) => {
calc = calc + res.v_amount;
});
staffarray.push({ name: staffname, amount: calc });
}
);
}
console.log(staffarray);
res.send({ data: staffarray });
});
I want to get the value of power_level value on the last line of the below mentioned code but it is not showing anything. I need that value for further use what to do.
var config = {
//
};
firebase.initializeApp(config);
var database = firebase.database();
var ref = database.ref("power");
ref.on("value", gotData, errData);
var power_level;
function gotData(data) {
var power = data.val();
// Grab the keys to iterate over the object
var keys = Object.keys(power);
console.log(keys);
for (var i = keys.length - 1; i < keys.length; i++) {
var key = keys[i];
// Look at each fruit object!
var power_level = power[key];
console.log(power_level);
}
}
console.log(power_level);
The reason you don't get the value on the last line is because it is asynchronous. Basically you do ref.on and it runs the gotData() function if (and more importantly WHEN) it is successful, but your last console.log() runs regardless of whether gotData() is finished.
An easy way to handle this is by defining a separate function which takes power_level as a parameter and handles the power_level, and run that at the end of gotData().
Something like this:
var config = {
//
};
firebase.initializeApp(config);
var database = firebase.database();
var ref = database.ref("power");
ref.on("value", gotData, errData);
var power_level;
function gotData(data) {
var power = data.val();
// Grab the keys to iterate over the object
var keys = Object.keys(power);
console.log(keys);
for (var i = keys.length-1; i < keys.length; i++) {
var key = keys[i];
// Look at each fruit object!
var power_level = power[key];
console.log(power_level);
////////// call handlePowerlevel here to handle logic further
handlePowerlevel(power_level)
}
}
function handlePowerlevel(pl) {
console.log(pl);
// actual logic
}
I have a ObservableArray.Using ajax async calls I am fetching data for binding.There will be 1000's of data. In each calls will fetch 100's of data. Problem is due to ajax async call order will not be from 1 - 1000. Cannot make it as sync call because browser will not respond. How can i sort the async data in knock out?
var DataVM = ko.observableArray([]);
ko.applyBindings(DataVM, document.getElementById("ControlBlock"));
for (var i = 0; i < totalAjaxCall; i++) {
GetData(guid, start, end, self.DataCallback);
start = start + 100;
end = end +100;
}
DataCallback= function (result) {
var temp = JSON.parse(result.d);
var data = [];
var data = temp.Data;
for (var j = 0; j < data.length; j++) {
var tempItem_ = new Item();
tempItem_.Number = data[j].Number;
// Other codes
DataVM.push(tempItem_ );
}
};
You can remember the blocks as they come in, then reassemble things when you've received them all. See comments:
var DataVM = ko.observableArray([]);
ko.applyBindings(DataVM, document.getElementById("ControlBlock"));
// Remember the results in a temporary array of arrays
var received = 0;
var receivedBlocks = [];
for (var i = 0; i < totalAjaxCall; i++) {
// Tell `DataCallback` which block it's going to get
GetData(guid, start, end, self.DataCallback.bind(null, i));
// No need for `self` ----^^^^^
start = start + 100;
end = end +100;
}
DataCallback = function (i, result) {
// Create and remember the items for this block
receivedBlocks[i] = JSON.parse(result.d).map(function(e) {
var tempItem_ = new Item();
tempItem_.Number = num;
return tempItem_;
});
++received;
// Do we have them all?
if (received == totalAjaxCall) {
// Flatten our array of arrays, now we have all the pieces
var receivedItems = []
receivedBlocks.forEach(function(block) {
receivedItems.push.apply(result, block);
});
// Push all of those onto DataVM as a single operation
// Note: You were using `DataVM.push`, so I used that here,
// but if you wanted to *replace* the contents of `DataVM`,
// (or if you know it's empty), you'd just do:
// DataVM(receivedItems);
// instead.
DataVM.push.apply(DataVM, receivedItems);
}
};
I'm not sure how far you want to deviate from your current code, but I'd like to advertise some of knockout's additional features :)
If you create a small "in-between" model for your requests, you can make use of computed values to automatically keep track of a correctly sorted list of data.
For example, if you define a new Request() like so:
var Request = function(start, end) {
this.completed = ko.observable(false);
this.data = [];
getData(start, end, this.onLoad.bind(this));
};
Request.prototype.onLoad = function(data) {
this.data = data;
this.completed(true);
};
You can change your for loop to create those "in-between" models. This creates a Request for 0 to 100, 101 to 201, etc. Each of these models is stored in an array, in the order of creation.
function getDataRequests(start, end, chunkSize) {
var requests = [];
for (var i = start; i < end; i += chunkSize) {
requests.push(new Request(i, Math.min(i + chunkSize, end)));
}
return requests;
};
Now that you can create an ordered array, you can compute another ordered array of data by merging all completed requests together:
var DataVM = function(start, end, chunkSize) {
// We keep track of a list of requests
var requests = ko.observableArray(
getDataRequests(start, end, chunkSize)
);
// Because requests have an observable completed prop,
// we can automatically keep track of a list of completed
// requests
var completedRequests = ko.pureComputed(() =>
requests().filter(r => r.completed()));
// Now, whenever a requests completes, we flatten the
// `data` parts for `completed` requests
this.data = ko.pureComputed(() => completedRequests()
.reduce((items, r) => items.concat(r.data), []));
};
Because you have the requests array, you can easily compute UI properties. For example: firstLoaded is a computed that returns the completed value of your first request.
Here's a complete example (ES2015):
var DataVM = function(start, end, chunkSize) {
// We keep track of a list of requests
var requests = ko.observableArray(
getDataRequests(start, end, chunkSize)
);
// Because requests have an observable completed prop,
// we can automatically keep track of a list of completed
// requests
var completedRequests = ko.pureComputed(() =>
requests().filter(r => r.completed()));
// Now, whenever a requests completes, we flatten the
// `data` parts for `completed` requests
this.data = ko.pureComputed(() => completedRequests()
.reduce((items, r) => items.concat(r.data), []));
// Shows progress
this.loadingMsg = ko.pureComputed(() => {
var completedCount = completedRequests().length,
allCount = requests().length;
return completedCount === allCount
? `Done loading ${end - start} items in ${allCount} steps`
: `Loading... (${completedCount}/${allCount})`;
});
// Check if the first (if any) request has completed loading
this.firstCompleted = ko.pureComputed(() =>
requests().length && requests()[0].completed());
};
var Request = function(start, end) {
this.completed = ko.observable(false);
this.data = [];
getData(start, end, this.onLoad.bind(this));
};
Request.prototype.onLoad = function(data) {
this.data = data;
this.completed(true);
};
var vm = new DataVM(0, 50, 5);
ko.applyBindings(vm);
// Mock async ajax stuff and data getters
function getDataRequests(start, end, chunkSize) {
var requests = [];
for (var i = start; i < end; i += chunkSize) {
requests.push(new Request(i, Math.min(i + chunkSize, end)));
}
return requests;
};
function getData(start, end, cb) {
setTimeout(function() {
cb(mockData(start, end));
}, Math.random() * 3000 + 500);
}
function mockData(from, to) {
return Array(to - from).fill(from).map(function(_, i) {
return from + i;
});
}
<script src="https://cdnjs.cloudflare.com/ajax/libs/knockout/3.2.0/knockout-min.js"></script>
<div data-bind="text: loadingMsg"></div>
<ul data-bind="foreach: data, visible: firstCompleted" style="border: 1px solid black;">
<li data-bind="text: $data"></li>
</ul>
Here is my code
Parse.Cloud.define('filters', function(request, response){
var _ = require('underscore');
var customerGeoPoint = request.params.geolocation;
var rating = request.params.rating
var finalList = [];
// var arr = [];
var promises = [];
var query = new Parse.Query('ServiceProvider');
query.withinMiles("geoLocation", customerGeoPoint, 10);
query.find().then(function(spobjs){
return spobjs
}).then(function(newval){
var query2 = new Parse.Query('Customer');
for(i in newval){
query2.withinMiles('geoLocation', newval[i].get('geoLocation'), newval[i].get('serviceRadius'));
var sp = query2.first()
if(sp != null)
{
finalList.push(newval[i]);
}
}
return finalList ;
}).then(function(resval){
var arr = [];
var arr = _.sortBy(resval,'averageRating'); ** This Line doesn't work **
console.log(arr[0]);
return arr ;
}).then(function(checkval){
response.success(checkval);
},function(error){
// console.log(error);
response.error(error);
});
});
In the above code the line which reads "This Line doesn't work" does nothing. I have required underscore.js but still it doesn't sort the array. finalList value gets returned to the then promise after but it doesn't sort it and returns the same value as finalList. Can someone tell me what's the issue with this code?
When sorting parse objects with underscorejs, the iteratee must return the value of the attribute using get()...
var arr = _.sortBy(resval, function(o) { return o.get('averageRating'); });
I am trying to go through nodeschool's learnyounode.
This problem is the same as the previous problem (HTTP COLLECT) in
that you need to use http.get(). However, this time you will be
provided with three URLs as the first three command-line arguments.
You must collect the complete content provided to you by each of the
URLs and print it to the console (stdout). You don't need to print out
the length, just the data as a String; one line per URL. The catch is
that you must print them out in the same order as the URLs are
provided to you as command-line arguments.
I'm confused as to why my solution doesn't work exactly since it looks the same to me but more functional and am unsure of their inner test workings:
1. ACTUAL: ""
1. EXPECTED: "As busy as a dead horse also lets get some dero. Built like a sleepout no dramas lets get some chook. She'll be right thingo my she'll be right ute. "
2. ACTUAL: "She'll be right bizzo no worries she'll be right fair dinkum. We're going aerial pingpong no worries as busy as a gyno. "
2. EXPECTED: "She'll be right bizzo no worries she'll be right fair dinkum. We're going aerial pingpong no worries as busy as a gyno. "
3. ACTUAL: "He's got a massive pretty spiffy heaps she'll be right brizzie. He hasn't got a fly wire where shazza got us some strewth. She'll be right spit the dummy with it'll be fair go. We're going gobsmacked with as stands out like arvo. He's got a massive bush bash mate she'll be right slacker. "
3. EXPECTED: "He's got a massive pretty spiffy heaps she'll be right brizzie. He hasn't got a fly wire where shazza got us some strewth. She'll be right spit the dummy with it'll be fair go. We're going gobsmacked with as stands out like arvo. He's got a massive bush bash mate she'll be right slacker. "
4. ACTUAL: ""
4. EXPECTED: ""
my code:
var http = require('http');
var bl = require('bl');
var result = [];
var urls = process.argv.slice(2);
urls.forEach(function(url, i) {
http.get(url, function(response) {
response.pipe(bl(function(err, data) {
if (err) return console.error(err);
result[i] = data.toString();
if (i === urls.length - 1) {
console.log(result.join('\n'));
}
}));
});
});
official solution:
var http = require('http')
var bl = require('bl')
var results = []
var count = 0
function printResults () {
for (var i = 0; i < 3; i++)
console.log(results[i])
}
function httpGet (index) {
http.get(process.argv[2 + index], function (response) {
response.pipe(bl(function (err, data) {
if (err)
return console.error(err)
results[index] = data.toString()
count++
if (count == 3)
printResults()
}))
})
}
for (var i = 0; i < 3; i++)
httpGet(i)
Basically the first test never passes (although if there is only 1 url in the iterated array (instead of 3), the first test passes but not the others). Any insight would be great. I'm not sure where to ask about this and perhaps I'm just missing some JS thing, so sorry if this is not appropriate.
You haven't made sure that all of the urls have been downloaded.
The requests don't necessarily come back in order. Consider if 3 comes back first. You'll skip the other two urls and only print out 3.
The demo code counts the number of responses so it's guaranteed to get everything before it prints out the answer.
I think that you just need wait until all requested results ends or any one error. There are my passed answer:
var http = require('http');
var bl = require('bl');
var urls = process.argv.slice(2)
var count = urls.length;
var results = [];
urls.forEach((url, index) => {
http.get(url, (res) => {
res.pipe(bl((err, data) => {
if (err) throw err;
results[index] = data.toString();
count--;
if (count == 0) {
results.forEach((result) => {
console.log(result)
});
}
}))
})
})
var http = require('http');
var links = [2, 3, 4];
var buffer = [];
(function render(index) {
http.get(process.argv[links[index]], function (response){
response.setEncoding('utf8');
response.on('data', function(chunk){
if(buffer[index] === undefined) {
buffer[index] = '';
}
buffer[index] += chunk;
});
response.on('end', function () {
var newIndex = index+1;
if(links[newIndex] !== undefined) {
render(newIndex);
} else {
return renderOutput();
}
});
response.on('error', console.error);
}).on('error', console.error);
})(0); //self-calling function
function renderOutput() {
buffer.forEach(function (elem) {
console.log(elem);
});
}
I got it working without using bufferList(bl) module and may be more generic approach.
var http = require('http');
var urlList = [];
urlList.push(process.argv[2]);
urlList.push(process.argv[3]);
urlList.push(process.argv[4]);
var results = []
var count = 0
function getURLdata (index) {
http.get(urlList[index], function(response){
var data = {};
data[index] = '';
response.setEncoding('utf-8');
response.on('error', function(err){
console.log(err);
});
response.on('data', function(chunk){
data[index] += chunk;
});
response.on('end', function(){
results[index] = data;
count++;
if (count == urlList.length){
for (var i = 0; i < urlList.length; i++){
console.log(results[i][i]);
}
}
});
});
}
for (var i = 0; i < urlList.length; i++)
getURLdata(i);
I am a beginner so maybe this solution has problems, this is using async/await, by making an array of promises and waiting for them to resolve, this will control the order of responses
const axios = require("axios")
const getURL = async url =>
{
let res = await axios.get(url)
return res.data
}
const getUrlArray = () =>
{
let args = process.argv.slice(2)
.map(e => getURL(e))
return Promise.all(args)
}
getUrlArray()
.then(data => data.forEach(e => console.log(e)))
Pretty simple solution, but gets the job done:
const http = require('http');
const bl = require('bl');
var x;
for (x = 2; x < 5; x++) {
http.get(process.argv[x], function (res) {
res.pipe(bl(function (err, data) {
if (err) { return console.error(err) }
console.log(data.toString());
}));
});
};