I'm having trouble trying to return promise results as a yield back to the original caller.
store.js
module.exports = {
find: function *(storeRequest){
if(!_gateway){
_gateway = realGateway;
}
storeResponse.http.statusCode = 200;
var stores = _gateway.find(storeRequest.id).next().value; // I want to be able to get the stores array back here ultimately by calling next() like I am trying to do here
console.log(stores[0]);
//yield storeResponse;
}
};
storeGateway.js
module.exports = {
find: function *(id){
var stores = [];
var entity;
database.find.then(function(foundStores){
entity = testUtil.createStore(foundStores[0].id, foundStores[0].name);
console.log("ENTITY:");
console.log(entity);
stores.push(entity);
console.log("STORES[0]:");
console.log(stores[0]);
// I am getting the results here successfully so far when I console.log(stores[0])! But now I want to return stores now from here and yield the array so it propogates up to the caller of storeGateway's find()
// yield entity; --- this doesn't work because I think I'm in the promise then scope
}
);
//yield entity; -- and this definitely won't work because it's not in the promise callback (then)
}
};
database.js
var co = require('co');
var pg = require('co-pg')(require('pg'));
var config = require('./postgreSQL-config');
var database = module.exports = {};
var _id;
var _foundStores;
database.find = co(function* poolExample(id) {
var query = "Select id, name from stores";
try {
var connectionResults = yield pg.connectPromise(config.postgres);
var client = connectionResults[0];
var done = connectionResults[1];
var result = yield client.queryPromise(query);
done();
console.log("PRINTING ROWS:");
console.log(result.rows[0]);
_foundStores = yield result.rows;
} catch(ex) {
console.error(ex.toString());
}
console.log("RESULTS!:");
console.log(_foundStores);
return _foundStores;
});
I'm getting data printed on every console.log you see above. I just don't know how to return the stores from the storeGateway's find() method since it's receiving the stores array in the promise result (in the .then()) and I need to be able to yield that back upstream.
(see my comment in code, I'm trying to return the found stores in the promise's then back upstream from my store.js's find generator function).
The point of using generators and co is that you can yield promises to the coroutine runner and get their results back, so that you don't have to use then.
Start by making find a method in your database.js:
database.find = co.wrap(function* poolExample(id) {
// ^^^^^
…
});
Then in storeGateway.js you should be doing
module.exports = {
find: function*(id) {
var foundStores = yield database.find(id);
var entity = testUtil.createStore(foundStores[0].id, foundStores[0].name);
console.log("ENTITY:", entity);
var stores = [entity];
console.log("STORES[0]:", stores[0]);
return stores;
}
};
(maybe wrap the generator function in co.wrap(…)).
Then in store.js you can do
module.exports = {
find: co.wrap(function*(storeRequest) {
if (!_gateway) _gateway = realGateway;
storeResponse.http.statusCode = 200;
var stores = yield* _gateway.find(storeRequest.id);
// or yield _gateway.find(storeRequest.id); if you did wrap it and it
// returns a promise, not a generator
console.log(stores[0]);
return stores;
})
};
There are two ways to do this. You either receive a call back parameter into your function and call it when the promise is resolved (inside your then function) or better, return the result of then. then() itself returns a promise and whatever you return from with in the function are available to the subsequent functions chained to the promise, so if you do this
return database.find.then(function(foundStores){
entity = testUtil.createStore(foundStores[0].id, foundStores[0].name);
console.log("ENTITY:");
console.log(entity);
stores.push(entity);
console.log("STORES[0]:");
console.log(stores[0]);
return stores[0];
}
then you could do gateway.find().then(function(stores){}) and stores would be what you returned i.e. stores[0].
Related
I have Meteor method with a following fragment (below) and it has asynchronous loop inside. As far as i know, this should be wrapped with meteor wrapAsync, but their documentation isn't clear enough to me, so i'm not sure how to implement it properly. What would be the easiest way to use wrapAsync on this example?
if(Meteor.isServer){
Meteor.methods({
listCollections: function(){
// list collections in database
db = MongoInternals.defaultRemoteCollectionDriver().mongo.db;
collections = db.listCollections();
// convert cursor with collections to javascript array:
// method should wait for this loop to finish before returning result
var collectionsArray = [];
collections.each(function(n, collection){
if(collection){
collectionsArray.push(collection.name);
}
});
// return ignores the each loop and empty array is passed as a result
return collectionsArray;
}
});
Meteor.call("listCollections", function(err, res){
console.log(res);
});
}
It is quite hard to understand from your context, but normally, the wrapAsync could be used like this
Meteor.methods({
'yourCurrentMethod': function(sentCollections) {
var processArray = function (collections) {
var collectionsArray = [];
collections.each(function(n, collection){
if(collection){
collectionsArray.push(collection.name);
}
});
// return ignores the each loop and empty array is passed as a result
return collectionsArray;
};
var processArrayCalling = Meteor.wrapAsync(processArray);
var result = processArrayCalling(sentCollections);
// process with the result
}
});
I have resolved this problem using Future, which was really easy to use.
if(Meteor.isServer){
var Future = Npm.require( 'fibers/future' ); // future code
Meteor.methods({
listCollections: function(){
// list collections in database
db = MongoInternals.defaultRemoteCollectionDriver().mongo.db;
collections = db.listCollections();
// create our future instance
var future = new Future(); // future code
// count number of collections inside collection list
var collectionsCount = 0;
collections.each(function(n, collection){
collectionsCount++;
});
// method should wait for this loop to finish before returning result
var collectionsArray = [];
var i = 0;
collections.each(function(n, collection){
if(collection){
collectionsArray.push(collection.name);
}
i++;
if( i == collectionsCount ){
future.return( collectionsArray ); // future code
}
});
return future.wait(); // future code
}
});
Meteor.call("listCollections", function(err, res){
console.log(res);
});
}
Using koa.js, I want to make an API which runs a generator function that runs a long time in the background, but sends a token back to the user immediately.
The user can then use that token to retrieve status of their job later.
'use strict';
var generateToken = function(){
//...
};
var processData = function *(data, token) {
//...
var a = yield analysis(data);
console.log(a) // a is undefined
};
app.post('/process_data', validate, function *(next) {
var token = generateToken();
var that = this;
setTimeout(function() {
for (var i of processData(that.request.body, token)){
continue;
}
});
this.body = "this should return immediately " + token;
return next;
});
Running it within the setTimeout, variable 'a' is not saved. How do I construct this so that processData runs exactly like a normal yield?
You would probably want to have the long running process get handled by a job queue such as Kue
You would queue the job with a http post
then check on the job with a http get
Here is a rough outline of what I think you want to be doing:
var kue = require('kue'),
koa = require('koa'),
route = require('koa-router'),
thunkify = require('thunkify'),
parse = require('co-body'),
co = require('co'),
app = koa(),
jobs = kue.createQueue();
app.use(route(app));
// turn callbacks into thunks for generators
var createJob = thunkify(jobs.create);
var findJob = thunkify(kue.Job.get);
// Process the job here
jobs.process('longProcess', function(job, done){
// do work in here
// call done(err) when completed
// EDIT: if you want to handle job using generators/yield
// you could use a library like co
co(function *(){
var qs = yield doWork(job.data);
done();
}).error(done);
});
// Queue/Start the Job here
app.post('/jobs', function *(){
var body = yield parse(this);
var job = yield createJob('longProcess', body);
this.body = job.id;
});
// Check Status of job here
app.get('/jobs/:token', function *(){
var job = yield findJob(this.params.token);
this.body = job;
// job.status === 'complete' || ...
});
app.listen(3000);
Thanks to Bergi for the solution.
app.post('/process_data', validate, function *(next) {
var token = generateToken();
co(processData(this.request.body, token));
this.body = "this should return immediately " + token;
return next;
});
This is my first stab at attempting to put together a node module and I am still trying to wrap my head around how to structure the asynchronous callbacks. This is a case in point. Right now I am trying to use featureService.getCount() and getting nothing in response. Using breakpoints, I know featureService.getUniqueIds() is working.
Since a callback is in there, I am assuming the reason why I am not getting a length back is the callback in getCount has not responded yet. After looking at this for most of the afternoon and not really coming up with a very good solution other than a recursive loop checking for the value to be populated with a timeout, I am asking for advice how to better structure my code to accomplish the task at hand.
I have read a bit about promises. Is this an applicable instance or even a viable solution? I really have no clue how to implement promises, but it makes logical sense in such an instance.
Obviously I am lost here. Thank you for any help you can offer.
var Client = require('node-rest-client').Client;
var client = new Client();
var featureService = function(endpoint){
var uniqueIds;
var count;
// get list of unique id's
this.getUniqueIds = function(){
if (!uniqueIds) {
var options = {
parameters: {
f: 'json',
where: "OBJECTID LIKE '%'",
returnIdsOnly: 'true'
}
};
client.get(endpoint + '/query', options, function(data, res){
var dataObject = JSON.parse(data);
var uniqueIds = dataObject.objectIds;
return uniqueIds;
});
} else {
return uniqueIds;
}
};
// get feature count
this.getCount = function(){
// get uniqueIds
uniqueIds = this.getUniqueIds();
// get length of uniqueIds
count = uniqueIds.length;
};
// get list of unique attribute values in a single field for typeahead
this.getTypeaheadJson = function(field){};
// find features in a field with a specific value
this.find = function(field, value){};
};
var endpoint = 'http://services.arcgis.com/SgB3dZDkkUxpEHxu/arcgis/rest/services/aw_accesses_20140712b/FeatureServer/1';
var afs = new featureService(endpoint);
console.log(afs.getCount());
exports.featureService = featureService;
Now, after working it over some more and using request as in the bluebird documentation (I could not get the above module to work), I have this working, but cannot figure out how to get the calculated value to work with, the number of iterations.
var Promise = require("bluebird"),
request = Promise.promisifyAll(require("request"));
var FeatureService = function(){
// get count from rest endpoint
var getCount = function(){
var optionsCount = {
url: endpoint + '/query',
qs: {
f: 'json',
where: "OBJECTID LIKE '%'",
returnCountOnly: 'true'
}
};
return request.getAsync(optionsCount)
.get(1)
.then(JSON.parse)
.get('count');
};
// get max record count for each call to rest endpoint
var getMaxRecordCount = function(){
var optionsCount = {
url: endpoint,
qs: {
f: 'json'
}
};
return request.getAsync(optionsCount)
.get(1)
.then(JSON.parse)
.get('maxRecordCount');
};
// divide the total record count by the number of records returned per query to get the number of query iterations
this.getQueryIterations = function(){
getCount().then(function(count){
getMaxRecordCount().then(function(maxCount){
return Math.ceil(count/maxCount);
});
});
};
};
// url to test against
var endpoint = 'http://services.arcgis.com/SgB3dZDkkUxpEHxu/arcgis/rest/services/aw_accesses_20140712b/FeatureServer/1';
// create new feature service object instance
afs = new FeatureService();
// This seems like it should work, but only returns undefined
console.log(afs.getQueryIterations());
// this throws an error telling me "TypeError: Cannot call method 'then' of undefined"
//afs.getQueryIterations().then(function(iterCount){
// console.log(iterCount);
//});
Yes, use promises! They're a powerful tool, made for exactly this purpose, and with a decent library they're easy to use. In your case:
var Promise = require('bluebird'); // for example, the Bluebird libary
var Client = Promise.promisifyAll(require('node-rest-client').Client);
var client = new Client();
function FeatureService(endpoint) {
var uniqueIds;
var count;
// get list of unique id's
this.getUniqueIds = function(){
if (!uniqueIds) { // by caching the promise itself, you won't do multiple requests
// even if the method is called again before the first returns
uniqueIds = client.getAsync(endpoint + '/query', {
parameters: {
f: 'json',
where: "OBJECTID LIKE '%'",
returnIdsOnly: 'true'
}
})
.then(JSON.parse)
.get("objectIds");
}
return uniqueIds;
};
// get feature count
this.getCount = function(){
if (!count)
count = this.getUniqueIds() // returns a promise now!
.get("length");
return count; // return a promise for the length
};
// get list of unique attribute values in a single field for typeahead
this.getTypeaheadJson = function(field){};
// find features in a field with a specific value
this.find = function(field, value){};
};
var endpoint = 'http://services.arcgis.com/SgB3dZDkkUxpEHxu/arcgis/rest/services/aw_accesses_20140712b/FeatureServer/1';
var afs = new FeatureService(endpoint);
afs.getCount().then(function(count) {
console.log(count);
}); // you will need to use a callback to do something with async results (always!)
exports.FeatureService = FeatureService;
Here, using Bluebird's Promise.promisifyAll, you can just use .getAsync() instead of .get(), and will get a promise for the result.
// divide the total record count by the number of records returned per query to get the number of query iterations
this.getQueryIterations = function(){
getCount().then(function(count){
getMaxRecordCount().then(function(maxCount){
return Math.ceil(count/maxCount);
});
});
};
That's the right idea! Only you always want to return something from .then handlers, so that the promise returned by the .then() call will resolve with that value.
// divide the total record count by the number of records returned per query to get the number of query iterations
this.getQueryIterations = function(){
return getCount().then(function(count){
// ^^^^^^ return the promise from the `getQueryIterations` method
return getMaxRecordCount().then(function(maxCount){
// ^^^^^^ return the promise for the iteration number
return Math.ceil(count/maxCount);
});
});
};
Now, you get back a promise for the innermost result, and this will work now:
afs.getQueryIterations().then(function(iterCount){
console.log(iterCount);
});
I know this is wrong, but essentially I want to
connect to a db/orm via a promise
wait on that promise to fulfill and get the models (the return from the promise)
use the results for form a middleware generator function to place the models on the request
I suspect that this isn't the best approach, so essentially I have two questions:
Should I be rewriting my db/orm connect to a generator function (I have a feeling that is more inline with koa style)
Back to the original question (as I am sure I will not get a chance to rewrite all my business logic) - how do I wait on a promise to fulfill and to then return a generator function?
This is my poor attempt - which is not working, and honestly I didn't expect it to, but I wanted to start by writing code, to have something to work with to figure this out:
var connectImpl = function() {
var qPromise = q.nbind(object.method, object);
return qPromise ;
}
var domainMiddlewareImpl = function() {
let connectPromise = connectImpl()
return connectPromise.then(function(models){
return function *(next){
this.request.models = models ;
}
})
}
var app = koa()
app.use(domainMiddlewareImpl())
According to this, you can do the following:
var domainMiddlewareImpl = function() {
return function *(){
this.request.models = yield connectImpl();
};
};
A context sensitive answer based on the info provided by Hugo (thx):
var connectImpl = function() {
var qPromise = q.nbind(object.method, object);
return qPromise ;
}
var domainMiddlewareImpl = function () {
let models = null ;
return function *(next) {
if(models == null){
//////////////////////////////////////////////////////
// we only want one "connection", if that sets up a
// pool so be it
//////////////////////////////////////////////////////
models = yield connectImpl() ;
}
this.request.models = models.collections;
this.request.connections = models.connections;
yield next
};
};
My example the connectImpl is setting up domain models in an ORM (waterline for now), connecting to a database (pooled), and returning a promise for the ORM models and DB connections. I only want that to happen once, and then for every request through my Koa middleware, add the objects to the request.
Say we have a Customer Object, that has a "Foo" collection. I'd like my "getCustomer" function to add all Foos it does not have already, and then return itself, as a promise...
So I'd like a promise to: Get a Customer, then add all missing Foos to this Customer, so that when the promise is resolved, the customer has all missing foos.
Example:
// dataservice.js
// returns Q.Promise<breeze.QueryResult>
function getCustomer(custId) {
var query = breeze.EntityQuery().from("Customers").where("CustomerId", "==", custId);
return this.em.executeQuery(query);
}
// returns Q.Promise<breeze.QueryResult>
function getFoosNotOnCustomer(customer) {
var query = breeze.EntityQuery().from("Foo").where("CustomerId", "!=", customer.Id());
return this.em.executeQuery(query);
}
I am struggling with how to "chain" these together properly, what do do if no customer is found, etc. How can I modify "getCustomer" to do this? I am basically trying to user breeze synchronously. Here is my attempt, but it turns into ugly nested code in a hurry.
// want to return Q.Promise<Customer> that has Foos loaded
// I think this is actually returning something like Q.Promise<Q.Promise<Customer>>
function getCustomer(custId) {
var query = breeze.EntityQuery().from("Customers")
.where("CustomerId", "==", custId);
return this.em.executeQuery(query) // return here?
.then(function(data) {
// what about conditionals?
if(data.results.length == 1) {
getFoosNotOnCustomer(data.results[0]).
then(function (foosArray) {
$.each(foosArray, function(i,el) {
// push foos onto customer instance
}
return custWithFoos; // return here?
}
// return something?
}
}
}
Here's what I ended up doing:
function getCustomer(custId) {
var query = breeze.EntityQuery().from("Customers").where("CustomerId", "==", custId);
return manager.executeQuery(query) // return here?
.then(addFoos)
.then(doSomethingElse);
}
function addFoos(data) {
var myDefer = Q.Defer();
if (data && data.result.length == 1) {
var customer = data.results[0];
var query = // get FOOS Customer doesn't have;
manager.executeQuery(query).then(function (fooData) {
$.each(fooData.results function (i, el) {
customer.Foos.push(el);
});
myDefer.reslove(customer);
});
} else {
myDefer.resolve(undefined);
}
return myDefer.promise;
}
function doSomethingElse(customer) {
var myDefer = Q.Defer();
customer.SomePropert("test");
return myDefer.resovlve(customer);
}
// ----- MVVM
var custPromise = getCustomer(1).then(function (customer) {
// do something
});
I will take your example on face value despite my inability to understand the semantics ... in particular my inability to understand why getting all Foos not belonging to the customer is going to be helpful.
I'll just focus on "chaining" and I'll assume you want the caller to take possession of the selected customer when you're done.
Sequential chaining
In this example, we wait for the customer before getting the Foos
function getCustomer(custId) {
var cust;
var em = this.em;
var query = breeze.EntityQuery().from("Customers")
.where("CustomerId", "==", custId);
// On success calls `gotCustomer` which itself returns a promise
return em.executeQuery(query)
.then(gotCustomer)
.fail(handleFail); // you should handleFail
// Called after retrieving the customer.
// returns a new promise that the original caller will wait for.
// Defined as a nested success function
// so it can have access to the captured `cust` variable
function gotCustomer(data) {
cust = data.results[0];
if (!cust) {
return null; // no customer with that id; bail out now
}
// got a customer so look for non-customer foos
// returning another promise so caller will wait
return breeze.EntityQuery().from("Foos")
.where("CustomerId", "!=", custId)
.using(em).execute()
.then(gotFoos);
}
// Now you have both the customer and the other Foos;
// bring them together and return the customer.
function gotFoos(data) {
var foos = data.results;
// assume `notMyFoos` is an unmapped property that
// should hold every Foo that doesn't belong to this Customer
foos.forEach(function(f) { cust.notMyFoos.push(f); }
return cust; // return the customer to the caller after Foos arrive.
}
}
Parallel async queries
In your scenario you really don't have to wait for the customer query before getting the foos. You know the selection criterion for both the customer and the foos from the start. Assuming you think that there is a high probability that the customer query will return a customer, you might fire off both queries in parallel and then mash up the data when both queries complete. Consider the Q.all for this.
function getCustomer(custId) {
var em = this.em;
var custPromise = breeze.EntityQuery().from("Customers")
.where("CustomerId", "==", custId)
.using(em).execute();
var fooPromise = breeze.EntityQuery().from("Foos")
.where("CustomerId", "!=", custId)
.using(em).execute();
Q.all([custPromise, fooPromise])
.then(success)
.fail(handleFail); // you should handleFail
// Now you have both the customer and the "other" Foos;
// bring them together and return the customer.
// `data` is an array of the results from each promise in the order requested.
function success(data) {
var cust = data[0].results[0];
if (!cust) return null;
var foos = data[1].results;
// assume `notMyFoos` is an unmapped property that
// should hold every Foo that doesn't belong to this Customer
foos.forEach(function(f) { cust.notMyFoos.push(f); }
return cust; // return the customer to the caller after Foos arrive.
}
}
Notice that I don't have to do so much null checking in the success paths. I'm guaranteed to have data.results when the success callback is called. I do have to account for the possibility that there is no Customer with custId.