I am trying to scan the top 100 movie torrents on the pirate bay using node and add a movie poster for each result.
I am using these libraries
thepiratebay
imdb-api
I am able to find top 100 and return the results with no problems
app.get('/movies', function(req, res){
tpb.topTorrents(207).then(function(topMovies){
async.map(topMovies, tpb.getTorrent, function(err, results){
res.send(results);
})
})
});
I am also able to look up movies via an IMDB ID and return the results
app.get('/imdb', function(req, res){
imdb.getReq({ id: 'tt2660888' }, function(err, things) {
res.send(things);
});
});
What I am trying to do is loop over the top 100 results pull the imdb id out of the description field out and query imdb replacing the picture field with result.
app.get('/movies', function(req, res){
tpb.topTorrents(207).then(function(topMovies){
async.map(topMovies, tpb.getTorrent, function(err, results){
for (var value of results) {
if (S(value.description).contains('www.imdb.com/title/')) {
var imdbId = S(value.description).between('www.imdb.com/title/', '/').s
imdb.getReq({ id: imdbId }, function(err, movie) {
value["picture"] = movie.poster
});
}
}
res.send(results);
})
})
});
This isn't working for some reason but it makes sense to me intuitively. If I remove the imdb-api call and replace it with value["picture"] = "foo". It does work. I'm not sure if this is related to how node handles loops. I'm new to the JS world and have a ruby background
Thanks in advance
You are on the right track with the async module but the imdb requests are also asynchronous so res.send just gets called with the initial result of async.map
You can use another async.map for the imdb calls and them chain them with async.waterfall which will pass the results of the first function as an argument to the second (async.apply just invokes the tpb function with your topMovies).
function tpb (topMovies, done) {
async.map(topMovies, tpb.getTorrent, done);
}
function imdb (movies, done) {
function lookup (value, callback) {
if (S(value.description).contains('www.imdb.com/title/')) {
var imdbId = S(value.description).between('www.imdb.com/title/', '/').s
imdb.getReq({ id: imdbId }, function(err, movie) {
value["picture"] = movie.poster
return cb(err, value);
});
} else {
return callback(null);
}
}
async.map(movies, lookup, done);
}
app.get('/movies', function(req, res){
tpb.topTorrents(207).then(function(topMovies){
async.waterfall([async.apply(tpb, topMovies), imdb], function (err, results) {
if (err) {
// do error handling
}
return res.send(results);
});
});
});
Related
I want to access the variable "result" from the function which contains the query.
When I want to access it from another file, in which I am trying to work with the output after a POST Request, the variable is declared as "undefined".
This is the file in which i execute the query:
const db = require('../db/connect');
module.exports = {
getID(name){
db.query(`SELECT CWID FROM user WHERE surname = '${name}'`, function(error, result, fields){
if(error) console.log(error);
console.log(result);
});
}
}
And this is the file where I want to work with the data:
router.post('/test', function(req, res){
const data = queries.getID(req.body.name);
console.log(data);
res.render('new test', {title: "test"});
})
Can anybody help me with this?
Here's an example of querying using mysql and async/await. This should do what you would like:
Query file
const db = require('./db/connect');
module.exports = {
getID(name) {
return new Promise((resolve, reject) => {
db.query(`SELECT CWID FROM user WHERE surname = '${name}'`, function(error, result, fields) {
if (error) {
reject(error);
} else {
resolve(result);
}
});
});
}
}
Main File
router.post('/test', async function(req, res){
const data = await queries.getID(req.body.name);
console.log("Query result: ", data);
res.render('new test', {title: "test"});
});
The reason your result is undefined in your initial example is that you're using asynchronous i/o (normal in Node.js). By returning a Promise from getID, we can make async. calls easily and with some nice code syntax.
I feel like a complete moron but me and a friend are working on a project together and having trouble getting the first route to return the items requested from the Etsy API and have getAllListings add the items to the database. If you can see something glaringly obvious that we are doing wrong please let me know.
I should also mention that while the statement console.dir(body) does print out the items to the terminal it does not look like the contents are being passed to GET '/api/etsy/getListings'
Thanks!
routes.js
//this i want to return a list of active listings from the users shop.
app.get('/api/etsy/getListings',function(req, res){
bEtsy.getAllListings(req, res, function(err, body) {
});
res.json(req.body);
});
bEtsy.js
var standardCallback = function (err, status, body, headers, callback) {
if (err) {
console.log(err);
return callback(err, null);
}
if (body) {
console.dir(body);
return callback(null, body); // this gives me an error
}
}
var getAllListings = function(itemId, callback){
var Item = mongoose.model('Item');
var listingsParams = {
include_private: true
}
etsy.auth().get(
'/shops/'+etsy.shop+'/listings/active',
listingsParams,
function(err, status, body, headers){
var newi = new Item({name: body.title, stock: body.count, owner: "00000000000000000000",
etsy:{listingId: body.listing_id, stock: body.count}});
newi.save(function(err){
if (err) return handError(err);
});
standardCallback(err, status, body, headers, callback);
}
);
};
You are calling this function with three parameters, when it only takes two
bEtsy.getAllListings(req, res, function(err, body) {
});
On top of that the first argument itemId is being passed the request object and then it is also never used inside the function itself but some global variable called listingsParams is?!?
I'm learning node.js so bear with me.
I'm trying to create a node.js web application using express+jade that is basically just a line queue. (I.E. take a number, wait in line, now serving number 4...except the 4 will be a mysql table field). The page will auto-update every 5 seconds. There are three line queues handled by the page (I.E) :3000/1 :3000/2 :3000/3.
To be clear, I have the application working, but I want to make sure I am doing it correctly as opposed to just hacking it together with poor methodology.
In my index.js I have the standard setup:
exports.bio = function(req, res){
res.render('index', {
location: 'Biometrics',
number: bio()
});
};
exports.interview = function(req, res){
res.render('index', {
location: 'Interview',
number: interview()
});
};
exports.docs = function(req, res){
res.render('index', {
location: 'Documentation',
number: doc()
});
};
I am currently also calling the values for the "number:" JSON value from within the index.js as well.
var doc = (function() {
//do javascript and data calls
return a;
});
var bio = (function() {
//do javascript and data calls
return a;
});
var interview = (function() {
//do javascript and data calls
return a;
});
My question is: What would be the recommended way to do this or am I on the right track?
This will work as long as the functions doc(), bio(), and interview() are synchronous, but most likely that won't be the case, particularly if they need to perform some database access.
If these functions were async then your could should look like this:
exports.docs = function(req, res){
// call the doc() function and render the response in the callback
doc(function(err, number) {
res.render('index', {
location: 'Documentation',
number: number
});
});
};
The doc() function will look like this:
var doc = (function(callback) {
// This code very likely be async
// therefore it won't return any value via "return"
// but rather calling your function (callback)
db.doSomething(someparams, callback);
});
Inside db.doSomething() there will be a call to your function callback(err, theValue)
The asynchonous way would be something like:
exports.docs = function(req, res) {
fetchSomeValueFromMysql(req.param('foo'), function (err, data) {
if (err) {
res.send(500, 'boom!');
return;
}
res.render('index', {
location: 'Documentation',
number: data
});
});
};
Say if you had an async operation in bio();
exports.bio = function (req, res) {
bio(function (err, data) {
if (!err && data) {
res.render('index', {
location: 'Biometrics',
number: data
});
} else {
// handle error or no data
res.render('error');
}
});
}
var bio = function(cb) {
//do javascript and data calls
cb(err, data);
});
Again, there are many ways to get this working. But the above should do.
I am building an Express Node.js appication using this Facebook SDK, and here is my current route for root:
app.get('/', Facebook.loginRequired(), function (req, res) {
req.facebook.api('/me', function(err, user) {
res.render('index', {title: 'title', user: user});
});
});
I am making a call to the FB open graph URL http://graph.facebook.com/me, but I'd like to make multiple open graph calls to pass to my template. For example:
var APIcalls = ['/me', '/me/music', '/me/music.listens'];
req.facebook.api(APIcalls, function(err, res1, res2, res3){
res.render('index', {res1: res1, res2: res2, res3: res3});
});
What is the easiest way to make multiple API calls to Facebook and then pass the responses to my template accordingly? You can see my full app.js file here. Thanks!
This scenario seems perfect for async.map
see here: https://github.com/caolan/async#map
the async.map method takes an array of items and performs the same async call for each item in your array in parallel. If no errors are encountered, it will call a callback with a new array of results.
so your example would look something like this:
var async = require('async'),
APIcalls = ['/me', '/me/music', '/me/music.listens'];
async.map(APIcalls,
function(item, callback){
req.facebook.api(item, function(err, user) {
if(err){
return callback(err);
};
callback(null, user);
});},
function(err, results){
if(err){
throw new Error(err);
}
res.render('index', results);
}
);
For this you should use batching so that you can get away with only a single request to the Graph API.
FB.api('/', 'POST', {
batch: [{
relative_url: '/me'
},{
relative_url: '/me/music'
},{
relative_url: '/me/music.listens'
}]
}, function(o) {
console.log(o);
});
Alternatively you can use multi-fql
FB.api('/fql', {
q1: 'select ....',
q2: 'select ...'
}, function(o) {
console.log(o);
});
Once that is done, you simply combine the data into a single model that you pass to the renderer.
Suppose I have a CMS application written in Node.js which persists data on a Redis database. When this application creates a new content, it should increment the id counter, add the new id to a list of ides and then set a new hash with content. What I would do for now is to create a function to perform this execution. This function (let us call it createArticle()) would have a callback and would execute the increment. Once the increment was executed, a callback function would push it into the list of ids. After that, another callback would create the hash. The hash-creating callback would call the function passed as parameter to createArticle():
function createArticle(title, content, callback) {
var client = redis.createClient();
client.incr("idCounter", function(err, id) {
if (err) return callback(err, data);
client.lpush("articleIds", id, function (err, data) {
if (err) return callback(err, data);
var key = "article:"+id;
client.hmset(key, "title", title, "content", content, callback);
});
});
}
I would use this function more or less this way (using Express in this example):
app.post('/createarticle', function(req, res) {
var title = req.body.article.title,
content = req.body.article.content;
createArticle(title, content, function(err, data) {
if (err) return res.render('error', { status: 500, message: 'Internal Server Error' });
res.render('index', { status: 200, message: 'Article created!' });
});
});
However, this code looks a bit cumbersome to me. Is this the way to go. Or is there a better way to do a series of I/O steps? I used Express and Redis in my example, but the answer do not need to use them.
You can make those error-catchers single-lined:
function createArticle(title, content, callback) {
var client = redis.createClient()
client.incr("idCounter", function(err, id) {
if (err) return callback(err, data)
client.lpush("articleIds", id, function (err, data) {
if (err) return callback(err, data)
var key = "article:"+id
client.hmset(key, "title", title, "content", content", callback)
})
})
}
And you could use a helper for handling errors:
function noError(errorCb, cb) {
var slice = Array.prototype.slice
return function (err) {
var currentCb = err ? errorCb : cb
currentCb.apply(this, slice.apply(arguments, err?0:1)
}
}
function createArticle(title, content, cb) {
var client = redis.createClient()
client.incr("idCounter", noError(cb, function(id) {
client.lpush("articleIds", id, noError(function (data) {
var key = "article:"+id
client.hmset(key, "title", title, "content", content", callback)
}))
})
}
Or something like that.