EDIT: Updated with full code.
I'm writing javascript code in node.js that is meant to loop through an array of data that I have scraped from the web. The purpose of the code is to:
1) Check the player table to see if that player's name exists in a record, and
2) If that player doesn't exist, add him to the database.
Here is my code:
var cheerio = require('cheerio');
var request = require('request');
var data = [];
var mysql = require('mysql');
var connection = mysql.createConnection({
host: 'localhost',
user: 'root',
password: 'blahblah',
database: 'test',
port: 3306 });
connection.connect();
request('http://www.basketball-reference.com/friv/dailyleaders.cgi?month=12&day=28&year=2014', function(err, response, body){
if(!err && response.statusCode ==200){
var $ = cheerio.load(body);
$('td', 'tbody').each(function(){
var url = $(this).attr('href');
var text = $(this).text();
data.push(text);
});
for(i=1;i<data.length;i+=26){
var query = connection.query(
"SELECT * FROM player WHERE name = '"+data[i]+"'",function(err, result, fields) {
if (err) throw err;
if(result.length==0){
//var insertQuery = "INSERT INTO player (provider_id, team_id, position_id, name) VALUES (1, (SELECT id FROM team WHERE slug = '"+data[i+1]+"'),1,'"+data[i]+"');";
console.log(i);
}
});
}
}
});
Note that I commented out my INSERT query for testing purposes, but the player name is located in data[i] while the team name is in data[i+1].
To test, I am just printing i to the console, and every loop it prints out the max value of i, which is 3225. However, that console.log command is INSIDE the for loop, so shouldn't it print each value of i as it is incremented?
This is similar to this question: JavaScript closure inside loops – simple practical example
That answer explains the why things are acting the way they are.
This happens in closures, if you are referencing a variable in a callback and that variable was already at that max value before the callback was executed.
Here's an example of what your code is doing (bad with callbacks):
function a(){
var i = 0;
var callbacks = [];
for (i=0;i<100;i+=25){
callbacks.push(function(){
alert(i);
});
}
return callbacks;
}
var callbackArray = a();
for( f in callbackArray){
callbackArray[f]();
}
Here's what it should be doing:
function createFunction(i){
return function(){
alert(i);
}
}
function a(){
var i = 0;
var callbacks = [];
for (i=0;i<100;i+=25){
callbacks.push(createFunction(i));
}
return callbacks;
}
var callbackArray = a();
for( f in callbackArray){
callbackArray[f]();
}
For your fix, you should do something like this:
// outside of the code you posted, as a separate function.
function createPlayerSelectionCallback(data,i){
return function(err, result, fields) {
if (err) throw err;
if(result.length==0){
//var insertQuery = "INSERT INTO player (provider_id, team_id, position_id, name) VALUES (1, (SELECT id FROM team WHERE slug = '"+data[i+1]+"'),1,'"+data[i]+"');";
console.log(i);
}
}
}
// this for loop stays in the same place, just modified to use the new function.
for(i=1;i<data.length;i+=26){
var query = connection.query(
"SELECT * FROM player WHERE name = '"+data[i]+"'",createPlayerSelectionCallback(data, i))
}
Related
How can I save the result of an async function into a variable so I can use it below?
The code looks like this:
app.post('/reg', function(request, response){
var user = request.body.username;
var pass = request.body.password;
var quest = request.body.question;
var ans = request.body.answer;
MongoClient.connect(uri, function(err, db) {
var dbc = db.db("chat");
dbc.collection("accounts").find({username: `${user}`}).toArray(async (err, result) => {
console.log(result.length)
const a = await result.length;
return a;
});
console.log(a); <--------- here i got the error: a is not defined
if(a==1){
response.send("already_exist");
}
else{
response.send("succes");
obj={username: user, password: pass, question: quest, answer: ans};
var dbc = db.db("chat");
dbc.collection("accounts").insertOne(obj);
}
db.close();
// response.send("yep");
});
});
I have tried a lot of options but i cant find anything useful.
The thing that I want to do here is finding if an username already exists in database, if you have another ideas please tell me.
the variable a is not in scope of the main function, try declaring it outside at main/global level.
Also you're returning the value from the function, just use it.
const results = dbc.collection("accounts").find({..
New to Node.js and I'm trying to scrape some data by looping through an array of 3 urls. The scraped data will be used to store in a mongodb collection.
Right now, I am looping through the array of urls and using node's request module inside the for loop for each url and storing data dynamically in an array called products.
My issue is that when i try to print products.length on to the console outside of the request method, the value is 0, indicating an empty array. Here's a part of my code:
//these arrays will store the scraped information from webpage
var prodList = [];
var priceList = [];
//this is the array that will be used to organize and display the scraped info
var products = [];
//store scraped data as an object
function Prod(prodName, price) {
this.prodName = prodName;
this.price = price;
};
var populateArray = function() {
//urls to scrape
var nyxLinks = [
"http://www.nyxcosmetics.ca/en_CA/face?sz=999&viewall=1",
"http://www.nyxcosmetics.ca/en_CA/lips?sz=999&viewall=1",
"http://www.nyxcosmetics.ca/en_CA/eyes?sz=999&viewall=1"
];
//empty all arrays
prodList = [];
priceList = [];
products = [];
for(var i = 0; i < nyxLinks.length; i++) {
//define url to download
var url = nyxLinks[i];
console.log(url);
request(url, function(error, response, body) {
if(!error) {
//load page into cheerio
var $ = cheerio.load(body);
//for each product on the page store in respective arrays
$(".product_tile_wrapper").each(function(i, elem) {
prodList.push($(this).find($(".product_name")).attr("title"));
priceList.push($(this).find($(".product_price")).attr("data-pricevalue"));
});
for(var i = 0; i < prodList.length; i++) {
//store product info as an object
products.push(new Prod(prodList[i], priceList[i]));
}
} else {
console.log("We've encountered an error!")
}
}).on("end", function(err, data) {
if(!err) {
console.log("products length " + products.length);
} else {
console.log(err);
}
});
}
console.log("products length " + products.length);
}
mongoose.connect('mongodb://127.0.0.1:27017/makeupdb');
var db = mongoose.connection;
db.on('error', console.error.bind(console, 'Connection Error:'));
db.once('open', function() {
// we're connected
populateArray();
console.log("number of products in products array " + products.length);
//clear the current collection - db.remove({})
//insert data in mongodb - db.insert(products)
});
The console output from this code is:
Server running at http://127.0.0.01:1337/
http://www.nyxcosmetics.ca/en_CA/face?sz=999&viewall=1
http://www.nyxcosmetics.ca/en_CA/lips?sz=999&viewall=1
http://www.nyxcosmetics.ca/en_CA/eyes?sz=999&viewall=1
products length 0
number of products in products array 0
products length 0
products length 31
products length 119
I believe I need to use a callback to be able to access the products array but I am not sure where I would need to use this call back. Any help will be much appreciated.
Thanks,
Radha
This is because of JavaScript's asynchronous model. The engine won't wait for the .on('end') callback to be executed before moving on to the next iteration of your for loop.
You could have a counter variable that is incremented each time a callback is called, and when the counter reaches the number of requests made, call your final function. Do something like this:
var numRequestsFinished = 0;
var products = [];
var finalCallback = function() {
console.log('Final Products:', products);
};
for (var i = 0; i < nyxLinks.length; i++) {
request(..., function(err, data) {
numRequestsFinished++;
// error checking
products.push(data);
if (numRequestsFinished === nyxLinks.length) {
finalCallback();
}
});
}
Alternately, you could take a look at a Promise library such as Bluebird and the Promise.all API. This would allow you to define a function to be called when all promises in an array have completed.
currently,i learn node-mysql in node.js,but I meet a question,try a lot,i don't how to solve it ?
like this:
var express = require('express');
var app = express();
var mysqlcon=require('./testmysql');
var mysqlconn=mysqlcon.conn;
var http=require("http");
var url=require("url");
app.use('/sql',testExist).listen(3000);
function testExist(req,res,next){
var query = url.parse(req.url,true).query;
var name=query.name;
mysqlconn.connect();
var result=function(err, results){
return results.length;
}
mysqlconn.query('select * from user where name = ?',[name],result);
mysqlconn.end();
if(result>1)
console.log('result:'+result)
res.end("name is already exist!")
}
I want to get the inner function's result in outer function ,for example the length of such query will be 1(results.length==1),but i want to use the result in testExist(),we know that node use callback function,so that i can't get the inner result in outer function. So can any guys help me to solve it.
you need move the using code inside the callback.
function callback(err, results) {
if(results.length >1) console.log('result:' + results.length);
res.end("name is already exists!");
}
mysqlconn.query("select * from user where name= ?", [name], callback);
I have been trying to get the value of a variable after execution of a for loop in node js. But it is showing as undefined. But the same variable is getting the desired value when I call the variable in setTimeout function after 5000ms. I don't want to use setTimeout because it varies from user to user who uses my application.
The code with which I'm trying is
function getQuestions()
{
for(x=0;x<sevenQIDs.length;x++)
{
var query = connection.query('SELECT QUESTION, OP1, OP2, OP3, OP4, ANS FROM QUESTIONS WHERE QUESTION_ID="'+sevenQIDs[x]+'"', function(err,result,fields){
if(err){
throw err;
}
else{
var a = result[0];
var resTime = [], resAns = [];
resTime.length = 5;
resAns.length = 5;
a.resTime = resTime;
a.resAns = resAns;
sevenQues.push(a);
}
});
}
}
socket.on('socket1', function(text)
{
getQuestions();
console.log(sevenQues);
}
Here sevenQues is a global variable and I'm getting undefined in the console. But when I put this
setTimeout(function()
{
console.log(sevenQues);
},5000);
I'm getting the desired value. Please help me to resolve this issue and I heard of some async.js file which can do the foreach loop to send the desired output. But I'm unable to get this done. Anyhelp would be appreciated.
try this:
function getQuestions(callback)
{
for(x=0;x<sevenQIDs.length;x++)
{
var query = connection.query('SELECT QUESTION, OP1, OP2, OP3, OP4, ANS FROM QUESTIONS WHERE QUESTION_ID="'+sevenQIDs[x]+'"', function(err,result,fields){
if(err){
throw err;
}
else{
var a = result[0];
var resTime = [], resAns = [];
resTime.length = 5;
resAns.length = 5;
a.resTime = resTime;
a.resAns = resAns;
sevenQues.push(a);
if(x == sevenQIDs.length-1) callback();
}
});
}
}
socket.on('socket1', function(text)
{
getQuestions(function(){console.log(sevenQues)});
}
How do you access mongodb count results in nodejs so the result can be accessible to the asynchronous request? I can get the result and update the database but the asynchronous request fails to access the vars or the vars are empty and the vars appear to be updated when the next asynchronous request is made. The request must not be waiting for the query to finish and the next request is filled with the previous request's variables.
testOne.increment = function(request) {
var MongoClient = require('mongodb').MongoClient,
format = require('util').format;
MongoClient.connect('mongodb://127.0.0.1:27017/bbb_tracking', function(err, db) {
if (err) throw err;
collection = db.collection('bbb_tio');
collection.count({vio_domain:dom}, function(err, docs) {
if (err) throw err;
if (docs > 0) {
var vio_val = 3;
} else {
var vio_val = 0;
}
if (vio_val === 3) {
event = "New_Event";
var inf = 3;
}
db.close();
console.log("docs " + docs);
});
});
};
In the above, even when the vars are set in scope they are not defined asynchronously. Can I get some guidance on structuring this properly so the vars are populated in the callback. Thank you!
Since the count function is asynchronous, you'll need to pass a callback to the increment function so that when the count is returned from the database, the code can call the callback.
testOne.increment = function(request, callback) {
var MongoClient = require('mongodb').MongoClient,
format = require('util').format;
MongoClient.connect('mongodb://127.0.0.1:27017/bbb_tracking', function(err, db) {
if (err) throw err;
var collection = db.collection('bbb_tio');
// not sure where the dom value comes from ?
collection.count({vio_domain:dom}, function(err, count) {
var vio_val = 0;
if (err) throw err;
if (count > 0) {
vio_val = 3;
event = "New_Event";
var inf = 3;
}
db.close();
console.log("docs count: " + count);
// call the callback here (err as the first parameter, and the value as the second)
callback(null, count);
});
});
};
testOne.increment({}, function(err, count) {
// the count would be here...
});
(I don't understand what the variables you've used mean or why they're not used later, so I just did a bit of a clean-up. Variables are scoped to function blocks and hoisted to the function, so you don't need to redeclare them in each if block like you had done with vio_val).
You could use the 'async' module. It makes the code a lot cleaner and easier to debug. Take a look at the code in GitHub for adduser.js & deleteuser.js in the following post
http://gigadom.wordpress.com/2014/11/05/bend-it-like-bluemix-mongodb-using-auto-scaling-part-2/
Regards
Ganesh
length give you count of result array
const userdata = await User.find({ role: role, 'name': new RegExp(searchkey, 'i') },{date: 0,__v:0,password:0}).
sort(orderObj)
.limit(limit)
.skip(skip);
console.log(userdata.length);