I'm trying to assign a MongoDB document object to a Javascript variable running on a Node.js server. I'm having trouble understanding how to do this with the async behavior of the MongoDB driver on Node. In my main app.js file I have the following
// MongoDB
//-------------------------------------------------
var DOC = require('./test_mongodb.js');
console.log(typeof(DOC)); //<---------------- this fails
console.log('Works:', JSON.stringify(DOC)); //<------------this fails
The required file test_mongodb.js is as follows
const config = require('../config/mongodb.config.js'); // <--- This just pulls in the ip:port and database name...nothing special
const mongodb = require('mongodb');
var DOC = {}; // <------------------I want to store a document here
// Client
const mongoClient = mongodb.MongoClient;
// Connection URL
const mongoUrl = 'mongodb://' + config.db.host;
console.log(config.db.host);
// Use connect method to connect to the server
mongoClient.connect(mongoUrl, function(err, client) {
if (err) {
console.log('Unable to connect to the mongoDB server. Error:', err);
}
else {
console.log("Connected to mongoDB server");
// Select database
const db = client.db('DATA');
// Get the documents collection
var coll = db.collection('TEST');
//We have a cursor now with our find criteria
var cursor = coll.find({"name": "testing"});
//Lets iterate on the result
var count = 0;
cursor.each(function (err, doc) {
if (err) {
console.log(err);
} else {
console.log('Fetched:', doc);
if (count == 3) {
//console.log(typeof(doc));
//DOC = JSON.parse(JSON.stringify(doc)) ;
//console.log(typeof(DOC))
//console.log('Inside:',DOC);
DOC = doc; <----------------------just capture one doc
}
count = count + 1;
}
});
}
// Close connection when done
client.close();
});
console.log(typeof(DOC));
console.log('Inside:',DOC);
module.exports = DOC; // <-------------I want to export the variable here
The output of console.log(typeof(DOC)) in app.js is undefined. Overall, I realize this is a problem with the async behavior, but I don't understand how to correct it. I've read that the solution is to use callback functions, but I don't fully grasp how this is done here with Mongo. In your solution, please give a detailed explanation of how the callbacks are physically working as this is my main confusion.
One other side question...is there a difference between cursor.each and cursor.forEach?
The problem here is that the module.exports assignment is happening before the query is complete.
Your environment probably supports Promises, so you can return one here. Here's how your code should like:
test_mongodb.js
const config = require('../config/mongodb.config.js'); // <--- This just pulls in the ip:port and database name...nothing special
const mongodb = require('mongodb');
var DOC = new Promise(function(resolve, reject) {
// Client
const mongoClient = mongodb.MongoClient;
// Connection URL
const mongoUrl = 'mongodb://' + config.db.host;
console.log(config.db.host);
// Use connect method to connect to the server
mongoClient.connect(mongoUrl, function(err, client) {
if (err) {
console.log('Unable to connect to the mongoDB server. Error:', err);
} else {
console.log("Connected to mongoDB server");
// Select database
const db = client.db('DATA');
// Get the documents collection
var coll = db.collection('TEST');
//We have a cursor now with our find criteria
var cursor = coll.find({
"name": "testing"
});
//Lets iterate on the result
var count = 0;
cursor.each(function(err, doc) {
if (err) {
console.log(err);
} else {
console.log('Fetched:', doc);
if (count == 3) {
resolve(doc);
}
count = count + 1;
}
});
}
// Close connection when done
client.close();
});
})
module.exports = DOC;
Then wait for the promise to resolve in the parent.
app.js
require('./test_mongodb.js').then(function(DOC) {
console.log(DOC);
}, function(err) {
console.log(err);
});
Related
I am receiving data from different GPS devices in my TCP socket server. Each device has a unique id that is coming on with its response. All I want is whenever the new device connects with my server, I want to query my database first and then save that data into an array along with that socket data. So if that device sends data again, I don't need to query my database again. This is what I have tried so far
const Net = require('net');
var socketarray = [];
const server = new Net.Server();
const HOST = "IP";
const PORT = 8888;
var car_id;
var car_no;
var car_username;
server.listen(PORT, HOST);
server.on('connection', function(socket) {
console.log('A new connection has been established.');
socket.on('data', function(data) {
var server_data = data.toString();
var car_details = server_data.split(',').filter(str => str != "");
var car_id = car_details[0];
var find_string = socketarray.findIndex(x => x.car_id == car_id);
if (find_string === -1) {
getCarDetails(car_id, function(car_details) {
var length = socketarray.push({
"car_id": "" + car_id,
"car_no": car_details.car_no,
"car_username": car_details.username,
});
car_id = car_id;
car_no = car_details.car_no;
car_username = car_details.username;
});
index = length - 1;
} else {
index = find_string;
car_id = socketarray[find_string].car_id;
car_no = socketarray[find_string].car_no;
car_username = socketarray[find_string].car_username;
}
});
socket.on('end', function() {
console.log('Closing connection with the client');
});
socket.on('error', function(err) {
console.log(`Error: ${err}`);
});
});
function getCarDetails(car_id,callback) {
console.log("db called");
db.getConnection((err, connection) => {
if(err){
console.log("error is" + err);
//throw err;
return;
}
console.log('connected as id ' + connection.threadId);
let selectQuery = 'SELECT * FROM ?? WHERE ?? = ?';
let query = mysql.format(selectQuery, ["car", "id", car_id]);
// query = SELECT * FROM `todo` where `user` = 'shahid'
connection.query(query, (err, data) => {
connection.release();
if(err) {
console.error(err);
//throw err;
return;
}
// rows fetch
if(data.length > 0){
return callback(data[0]);
} else{
return callback(false);
}
});
});
}
I am using these variables later in my program and also modifying the array by using index variables like this e.g socketarray[index].car_username = "bello". But somehow I am having issues like data got mixed up. Since I am pushing into DB at the end but the issue is sometimes some other car data get inserted into some other car data in MySQL.
Am I doing something wrong in the program? I am having difficulties tracking the exact issue of where on what position arrays got mixed up.
On my studoes with NodeJS I've found an issue/warning the is getting me a bit worried because I would like to avoid memory leaks, but since this language is a bit new to me, I can't find which of my code is doing it.
const MongoClient = require('mongodb').MongoClient;
const assert = require('assert');
const fs = require("fs");
const conversor = require("./converterSiteEmObjeto");
// Connection URL
const url = 'mongodb://localhost:27017';
// Database config
const client = new MongoClient(url, { useNewUrlParser: true, useUnifiedTopology: true });
async function salvarBanco(listaLinks) {
try {
await client.connect();
console.log("Connected successfully to server");
const db = client.db('consultadados');
// Iterates through the list of links
for (link of listaLinks) {
// scrape data from link and return a JS object
let resultado = await conversor(link);
// insert object into DB and asserts
let r = await db.collection('cnpjs').insertOne(resultado);
assert.equal(1, r.insertedCount);
}
} catch (err) {
console.log(err.stack);
}
client.close();
console.log("Done...");
};
function converterSitemapEmListaLinks() {
// Reads file containing links
fs.readFile('links/sitemap.txt', 'utf8', function (err, contents) {
//creates a array of links (100+ links)
let listaLinks = contents.split("\n");
//saves to database
salvarBanco(listaLinks);
});
}
converterSitemapEmListaLinks();
It runs just fine, but after a few dozen iteractions on listaLinks (list of links) I get the following message:
" MaxListenersExceededWarning: Possible EventEmitter memory leak detected. 11 SIGHUP listeners added to [process]. Use emitter.setMaxListeners() to increase limit "
I know what it means, what I would like is to know how to fix it instead of simply setting the listeners to a higher max value.
I am unable to execute the sql, when using the global database connection in node.js.
I have followed the steps as in Azure documentation: https://learn.microsoft.com/en-us/azure/mysql/connect-nodejs and able to display the output on the console. But, I want to put all my Azure SQL database connection in a separate file, but the select query is not printing the output on the console.
DatabaseManager.js
var Connection = require('tedious').Connection;
var Request = require('tedious').Request;
var sqlConnection = function sqlConnection() {
// Create connection to database
var config =
{
userName: 'uname',
password: 'password',
server: 'dbserver.database.windows.net',
options:
{
database: 'mydatabase',
encrypt: true
}
}
var connection = new Connection(config);
// Attempt to connect and execute queries if connection goes through
connection.on('connect', function(err) {
if (err)
{
console.log(err)
}
else
{
console.log('CONNECTED TO DATABASE');
}
}
);
}
module.exports = sqlConnection;
app.js
var restify = require('restify');
var builder = require('botbuilder');
var botbuilder_azure = require("botbuilder-azure");
var azure = require('azure-storage');
var dbconnection = require('./DatabaseManager');
bot.dialog('profileDialog',
(session) => {
session.send('You reached the profile intent. You said \'%s\'.', session.message.text);
console.log('Reading rows from the Table...');
dbconnection("select FNAME from StudentProfile where ID=1"),
function (err, result, fields) {
if (err) throw err;
console.log(result);
}
session.endDialog();
}
Console Output:
Reading rows from the Table...
CONNECTED TO DATABASE
I was expecting the output of FNAME, but nothing is printing on the console. Is there anything, I am missing?
Thank you.
There's a couple of problems here. First off, you should only ever import a module once per file. This is just a performance consideration and won't actually break your code.
Next, pay attention to what you're exporting from your DatabaseManager module. Right now, you're exporting a function that creates the connection and then doesn't do anything with it. We can fix this by using a pattern called a "callback" which lets us provide a function that will then be called with the connection as an argument.
I added a ton of comments to the code explaining things. This code won't run as-is - there's a couple places where I have "do this or this". You'll have to choose one.
var Tedious = require('tedious'); // Only require a library once per file
var Connection = Tedious.Connection;
var Request = Tedious.Request;
// Or using the object spread operator
var { Connection, Request } = require('tedious');
// You called this `sqlConnection`. I'm going to use a verb since it's a
// function and not a variable containing the connection. I'm also going
// to change the declaration syntax to be clearer.
function connect(cb) { // cb is short for callback. It should be a function.
var config = {
userName: 'uname',
password: 'password',
server: 'dbserver.database.windows.net',
options: {
database: 'mydatabase',
encrypt: true
}
}; // Put a semi-colon on your variable assignments
var connection = new Connection(config);
// Attempt to connect and execute queries if connection goes through
connection.on('connect', function(err) {
if (err) {
console.log(err);
return; // Stop executing the function if it failed
}
// We don't need an "else" because of the return statement above
console.log('CONNECTED TO DATABASE');
// We have a connection, now let's do something with it. Call the
// callback and pass it the connection.
cb(connection);
});
}
module.exports = connect; // This exports a function that creates the connection
Then back in your main file, you can use it like so.
var restify = require('restify');
var builder = require('botbuilder');
var botbuilder_azure = require('botbuilder-azure');
var azure = require('azure-storage');
var connect = require('./DatabaseManager'); // renamed to be a verb since it's a function.
bot.dialog('profileDialog', (session) => { // Hey, this is a callback too!
session.send('You reached the profile intent. You said \'%s\'.', session.message.text);
console.log('Creating a connection');
connect((connection) => {
// or with the traditional function notation
connect(function(connection) {
console.log('Reading rows from the Table...');
// Execute your queries here using your connection. This code is
// taken from
// https://github.com/tediousjs/tedious/blob/master/examples/minimal.js
request = new Request("select FNAME from StudentProfile where ID=1", function(err, rowCount) { // Look another callback!
if (err) {
console.log(err);
} else {
console.log(rowCount + ' rows');
}
connection.close();
});
request.on('row', function(columns) { // Iterate through the rows using a callback
columns.forEach(function(column) {
if (column.value === null) {
console.log('NULL');
} else {
console.log(column.value);
}
});
});
connection.execSql(request);
});
so basically, I'm trying to set up a variable that I can directly query the mongodb from. My code looks like this:
db.js
var mongo = require('mongodb').MongoClient;
var DB = null;
var dbURL = 'mongodb://localhost:27017';
exports.connect = function(cb) {
console.log('meep'); //tracing code
mongo.connect(dbURL, function(err, db){
console.log('lmao'); //tracing
if (err) throw Error('Something has went wrong');
else { DB=db; console.log(DB); cb();}
});
console.log(DB); //returns null
};
//if some other file requires this file as mydbjs for example,
//I want to be able to do mydbjs.db().collection('epik').find();
exports.db = function() {
if (DB === null) throw Error('DB Object has not yet been initialized');
return DB;
};
Is there a way to do this or do I have to do everything inside the connect function? I'm assuming it could have to do with callback functions or something. Any help is deeply appreciated.
Your assumption is correct - you can use db variable inside this callback only
function(err, db){
console.log('lmao'); //tracing
if (err) throw Error('Something has went wrong');
else { DB=db; console.log(DB); cb();}
});
Here you can detailed explanation with the examples: nodeJs callbacks simple example
I am newbie in node.js. I am trying to define a constructor for node.js class which basically fetches data from mongo db. The code for same is as follows
var MongoClient = require('mongodb').MongoClient,
Server = require('mongodb').Server;
var ObjectID = require('mongodb').ObjectID;
var mongoHost = 'localHost';
var mongoPort = 27017;
CollectionDriver = function(db) {
this.db = db;
console.log("Collection drive has been set");
};
CollectionDriver = function() {
var mongoClient = new MongoClient(new Server(mongoHost, mongoPort));
mongoClient.open(function(err, mongoClient) {
if (!mongoClient || err) {
console.error("Error! Exiting... Must start MongoDB first");
process.exit(1);
}
var db1 = mongoClient.db("Quiz");
this.db = db1;
console.log("Set Collection Driver by default");
});
};
CollectionDriver.prototype.getCollection = function(collectionName, callback) {
this.db.collection(collectionName, function(error, the_collection) {
if (error) callback(error);
else callback(null, the_collection);
});
};
//find all objects for a collection
CollectionDriver.prototype.findAll = function(collectionName, obj, callback) {
this.getCollection(collectionName, function(error, the_collection) {
if (error) callback(error);
else {
the_collection.find(obj).toArray(function(error, results) {
if (error) callback(error);
else {
console.dir(results);
callback(null, results);
}
});
}
});
};
I am trying to use this in other class as follows:
CollectionDriver = require('./collectionDriver').CollectionDriver;
var collectionDriver = new CollectionDriver(db);
OtherClassName.prototype.find = function(credentials, callback) {
collectionDriver.findAll("user_groups", credentials, function(error, results) {
if (!error) {
// Do something
}
});
// ...
Whenever I try to access find method of Otherclass it says db variable of above class(i.e ConnectionDriver) is undefined. However I could see ConnectionDriver constructor is getting executed correctly and connection to db is opened correctly and this.db=db1 is also gettining executed.Am I doing something wrong? Any help would be appreciated
After banging my head and investing(inwasting) a whole day. I realised the issue. Actually mongoClient.open() call is an asyncronous call and i return the CollectionDriver object even before opening db/getting db instance.
I solved this problem by moving opening of db call outside constructor and setting instance db . The modified code is as follows :
var db = null;
var mongoClient = new MongoClient(new Server(mongoHost, mongoPort));
mongoClient.open(function(err, mongoClient) {
if (!mongoClient || err) {
console.error("Error! Exiting... Must start MongoDB first");
process.exit(1);
}
db = mongoClient.db("Quiz");
});
CollectionDriver = function() {
console.log("Set Collection Driver by default");
};
The above code seem to work , however when I assign db instance to this.db rather than explicitly declaring a variable as done above , I get same undefined/null error(when i initialise db variable to null). Any thoughts why this behavior ??
This is already answered but I believe the correct answer for this is, this gets redefined in function calls. Inside the callback function to open, this now points to the function. The way around this is to use ES6 arrow functions, if you're on node 4+.
collectionDriver = function() {
var mongoClient = new MongoClient(new Server(mongoHost, mongoPort));
mongoClient.open((err, mongoClient) => {
if (!mongoClient || err) {
console.error("Error! Exiting... Must start MongoDB first");
process.exit(1);
}
var db1 = mongoClient.db("Quiz");
this.db = db1;
console.log("Set Collection Driver by default");
});
Note: the function (err, mongoClient) { is changed to (err, mongoClient) => {
this then behaves as expected.