When I execute a findOne query to a collection that is different than that of the surrounding stream, it doesn't really execute the callback (does 1 or 2). Here is my code:
schema.js:
'use strict';
var mongoose = require('mongoose'),
Schema = mongoose.Schema;
var AutocompleteSchema = new Schema({
nGram: String,
locations: [{ type: Schema.Types.ObjectId, ref: 'Location' }]
});
module.exports = mongoose.model('Autocomplete', AutocompleteSchema);
ingest.js:
var Autocomplete = require('./schema');
var nGramAPI = require('ngram');
var cache = [];
function storeNGram(input, location) {
if(cache.indexOf(input) === -1) {
cache.push(input);
Autocomplete
.findOne({ nGram: input })
.populate('locations')
.exec(function (err, nGram) {
console.log(nGram);
if(!nGram) {
var newAutocomplete = {
nGram: input,
locations: [location._id]
};
Autocomplete.create(newAutocomplete, function(err, created) {
cache.splice(cache.indexOf(input), 1);
});
}
else {
nGram.locations.push(location._id);
sortLocations(nGram);
location.save(function(err, saved){
cache.splice(cache.indexOf(input), 1);
});
}
});
}
else {
setTimeout(function() {
storeNGram(input, location);
}, 100);
}
}
exports.ingest = function() {
console.log("Building nGrams");
var stream = Location.find().stream();
stream.on('data', function (location) {
var length = location.dispName.length > 20 ? 20 : location.dispName.length;
for(var i = 1; i <= length; i++) {
_.each(nGramAPI(i)(location.dispName), function(nGram) {
storeNGram(nGram, location);
});
}
});
}
When I execute the Autocomplete.findOne on its own outside of the stream it queries properly, but when inside it fails. Any suggestions?
Related
I am a beginner in nodejs, I was tasked to read a excel file and convert it into json format so that it can be stored into mongodb database.
excel2.js file:
const XlsxStreamReader = require("xlsx-stream-reader");
var fs = require('fs');
const config = require('./excelpush.json');
const db = require('./dbmanager.js');
var finalList = [];
var singlePerson = {};
class ExcelReader {
readFile() {
var workBookReader = new XlsxStreamReader();
workBookReader.on('error', function (error) {
throw (error);
})
workBookReader.on('worksheet', function (workSheetReader) {
if (workSheetReader.id > 1) {
workSheetReader.skip();
return;
}
var isFirstLine = true;
var headerIndex = [];
workSheetReader.on('row', function (row) {
if (isFirstLine) {
headerIndex = row.values.slice(1);
}
else if (!isFirstLine) {
let rowValues = row.values.slice(1);
let valueIndex = 0;
headerIndex.forEach(currentval => {
singlePerson[currentval] = rowValues[valueIndex];
valueIndex++;
});
finalList.push(singlePerson);
}
isFirstLine = false;
});
workSheetReader.on('end', function () {
});
workSheetReader.process()
});
workBookReader.on('end', function () {
//console.log(finalList);
console.log('finished!');
});
fs.createReadStream(config.filePath).pipe(workBookReader);
}
}
excelReader = new ExcelReader();
excelReader.readFile();
db.connectDb();
db.insertDb(finalList);
dbmanager.js file:
var mongoose = require('mongoose');
const config = require('./db.json');
function connectDb() {
mongoose.connect(`mongodb://${config.dbConfig.host}:${config.dbConfig.port}/${config.dbConfig.dbName}`);
mongoose.connection.once('open', function () {
console.log('Connection has been made');
}).on('error', function (error) {
console.log('error is:' + error);
})
}
function insertDb(list) {
var myschema = new mongoose.Schema({}, { strict: false });
var obj = mongoose.model('myschema', myschema, `${config.dbConfig.collectionName}`);
var obj1 = new obj(list);
obj1.save();
}
module.exports = {
connectDb, insertDb
}
db.json file:
{
"dbConfig": {
"host": "localhost",
"port": "27017",
"dbName": "PRACTICE",
"collectionName": "Excel"
}
}
excelpush.json file:
{
"filePath": "D:\\grv_tracx_updt (1).xlsx"
}
here excel2.js is taking excel file from excel2.json and reading it and storing as a array of objects in finalList variable.
In dbmanager.js file db connection and insertion code are written.
Here i am not able to store data in database, The code gets executed perfectly but data in not stored in mongodb database.
Note: excel file is large.
So I have files inside the following folder:
app/controller/token.js
app/controller/news.js
token.js:
"use strict";
var connection = require("../con");
exports.isTokenExists = function(token) {
var checkToken = "SELECT COUNT(`id`) AS 'total' FROM `user` WHERE `token` = '" + token + "'";
var isExists = false;
var count;
var checkResult;
connection.query(checkToken, function(error, rows) {
if (!error) {
checkResult = JSON.parse(JSON.stringify(rows));
for (var i = 0; i < checkResult.length; i++) {
var row = rows[i];
count = row.total;
}
if (count > 0) {
isExists = true;
}
}
});
return isExists;
};
news.js:
"use strict";
var response = require("../response/responses");
var connection = require("../con");
var getToken = require("./token");
exports.news = function(req, res) {
response.send(false, "News API", null, res);
};
exports.allNews = function(req, res) {
var checkTokenExists = getToken.isTokenExists("75d12cc4dc07608d5b87a6cba33cac056df1239c");
if (checkTokenExists) {
var allNewsQuery = "SELECT a.`id`, b.`title` AS `category`, a.`title`, a.`description`, a.`content`, a.`image`, a.`created_date` FROM `news` AS a LEFT JOIN `news_category` AS b ON a.`id_news_category` = b.`id` ORDER BY `created_date` DESC LIMIT 20";
connection.query(allNewsQuery, function(error, rows) {
if (error) {
response.send(true, "" + error, null, res);
} else {
var data = [];
var newsData = JSON.parse(JSON.stringify(rows));
for (var i = 0; i < newsData.length; i++) {
var row = rows[i];
data[i] = {
id: row.id,
idCategory: row.idCategory,
category: row.category,
title: row.title,
description: row.description,
image: row.image,
createdDate: row.created_date
};
}
response.send(false, "News is not empty", data, res);
}
});
} else {
response.send(true, "Error: Token not found", checkTokenExists, res);
}
};
I always getting false value from isTokenExists meanwhile the token is exists in the table.
How do I get true response if the token is exist and how do I get false response if the token is not exists in table?
Any help will be much appreciated.
Regards.
The issue here is that connection.query accepts a callback, but the rest of your code will move passed that without awaiting the result, which is why your isExists always returns false. You can fix this by encapsulating the query with a Promise like this:
"use strict";
const connection = require("../con");
exports.isTokenExists = async function(token) {
const checkToken = "SELECT COUNT(`id`) AS 'total' FROM `user` WHERE `token` = ?";
return new Promise((resolve, reject) => {
connection.query(checkToken, token, function (error, results) {
if (error) return reject(error);
return resolve(results.length > 0);
});
});
};
I also simplified the logic in the callback a bit.
Then, in news.js await the result like this:
exports.allNews = async function(req, res) {
getToken.isTokenExists("75d12cc4dc07608d5b87a6cba33cac056df1239c")
.then(result => {
if (result === true) {
//place your code for handling if the token exists here
}
else {
//place your code for handling if the token does not exist
}
})
.catch(err => {
//handle error
});
}
You are missing async / await concept. You need to wait until your query executes.
1) Write a promise function
export.getCount = function(query) {
return new Promise((res, rej) => {
let count = 0;
connection.query(checkToken, function(error, rows) {
if (!error) {
checkResult = JSON.parse(JSON.stringify(rows));
for (var i = 0; i < checkResult.length; i++) {
var row = rows[i];
count = row.total;
}
}
return res(count);
})
}
2) Write async function which supports await operations
exports.isTokenExists = async function(token) {
var query = "SELECT COUNT(`id`) AS 'total' FROM `user` WHERE `token` = '" + token + "'";
let count = await getCount(query)
return count > 0; // Returns true if count is > 0
};
forgive me the question.
I'm not used to node and sync / await
I have the following function which queries a mongodb returning a json, I'm saving that return on redis.
So far so good.
findLightnings: async (request, h) => {
const q = request.query.q.split(':');
const index = q[0];
const value = q[1].split(',');
const dateInit = new Date(request.query.dateInit);
const dateEnd = new Date(request.query.dateEnd);
const page = request.query.page;
const distance = request.query.distance;
const redis = require('redis');
const client = redis.createClient();
let limit = 300;
let filter = {
$and: [{
data: {
$gte: dateInit.toISOString(),
$lte: dateEnd.toISOString()
}
}]
}
if (index === 'latlng') {
filter['$and'][0]['geo.coordinates'] = {
$near: {
$geometry: {
type: 'Point',
coordinates: value.map(Number),
$minDistance: 0,
$maxDistance: distance
}
}
}
limit = 100;
} else {
filter['$and'][0][`cidade.${index}`] = {
$in: value
}
}
return client.get('elektro', async (err, reply) => {
let resp = null;
if (reply) {
console.log(reply); //<<<<<<<< Return Json OK
resp = reply // <<<<<<<<<< Return TRUE in json's place
} else {
console.log('db')
const query = await Lightning.find(filter).sort('data').skip(page*limit).limit(limit).exec();
client.set('elektro', JSON.stringify(query));
client.expire('elektro', 3600);
resp = query
}
return JSON.stringify(resp);
})
}
The problem is time to recover this data from the redis.
In the console log json appears normal, how much tento returns that value for the main function it comes 'TRUE' and not the json saved in redis.
Someone can give me a helping hand on this.
I really need this function.
const redis = require('redis');
const client = redis.createClient(6379);
const bluebird = require("bluebird");
bluebird.promisifyAll(redis.RedisClient.prototype);
bluebird.promisifyAll(redis.Multi.prototype);
const redisdata = await client.getAsync("user:photos");
if (redisdata) {
console.log(`cache EXISTS`)
return res.json({ source: 'cache', data: JSON.parse(redisdata) })
}
I was able to solve the problem with the redis client.getAsync().
which already has a native async function:
source: Node-redis
The final code is as follows:
findLightnings: async (request, h) => {
const q = request.query.q.split(':');
const index = q[0];
const value = q[1].split(',');
const dateInit = new Date(request.query.dateInit);
const dateEnd = new Date(request.query.dateEnd);
const page = request.query.page;
const distance = request.query.distance;
let limit = 300;
let filter = {
$and: [{
data: {
$gte: dateInit.toISOString(),
$lte: dateEnd.toISOString()
}
}]
}
if (index === 'latlng') {
filter['$and'][0]['geo.coordinates'] = {
$near: {
$geometry: {
type: 'Point',
coordinates: value.map(Number),
$minDistance: 0,
$maxDistance: distance
}
}
}
limit = 100;
} else {
filter['$and'][0][`cidade.${index}`] = {
$in: value
}
}
return getAsync('findLightnings'+ '/' + request.query.q + '/' + request.query.dateInit + '/' + request.query.dateEnd).then(async (res) => {
if(res){
console.log('Fonte Dados => redis')
return res
}else{
console.log('Fonte Dados => db')
try {
const query = await Lightning.find(filter).sort('data').exec();//.skip(page*limit).limit(limit).exec();
client.set('findLightnings'+ '/' + request.query.q + '/' + request.query.dateInit + '/' + request.query.dateEnd, JSON.stringify(query));
return query;
} catch (err) {
return Boom.badData(err);
}
}
client.close();
});
},
var mongoose = require('mongoose');
var genreSchema = mongoose.Schema({
name:{
type: String,
required: true
},
create_date:{
type: Date,
default: Date.now
}
});
var Genre = module.exports = mongoose.model('Genre',genreSchema);
//Get Grenres
module.exports.getGenres = function (callback, limit) {
Genre.find(callback).limit(limit);
}
//Add Genre
module.exports.addGenre = function (genre, callback) {
Genre.create(genre, callback);
}
Above code is working but I felt the code is ugly because I have to write so many times of module.exports. How to simplify above code?
You can have a single object assigned to module.exports like below -
var Genre = mongoose.model('Genre',genreSchema);
//Get Grenres
var getGenres = function (callback, limit) {
Genre.find(callback).limit(limit);
}
//Add Genre
var addGenre = function (genre, callback) {
Genre.create(genre, callback);
}
module.exports = {
getGenres : getGenres,
addGenres : addGenre
};
At the moment I have a list of Team ID's which I have stored in the array teamIds. I also have a function findByKey in my Team mongoose schema which finds a Team based on its key. As you can see from the code I have to put each new Team.findByKey inside the previous one, if I put it outside of the previous findByKey it doesn't function correctly.
I'm having problems putting the Team.findByKey methods into some sort of for loop which can loop through the array teamIds and for each Team in the array. I am developing the function to create a new tournament, the idea is that you pass an array of teamIds to here, and it loops through each ID, searching for that Team and adding it to a teamsList array in order to create the tournament.
I need to put the last bit of code to create the array of Team objects into the last Team.findByKey method which is my problem. How would I fix this so that I can pass it n amount of teams and find each team and store it as an array of Teams?
This is my code:
app.get('/tournament', function(req, res){
function genMatches(nTeams) {
var matchArray = [];
while (nTeams > 1) {
nTeams = nTeams >> 1;
var matches = [];
for (var i = 0; i < nTeams; ++i) {
matches.push([]);
}
matchArray.push(matches);
}
return matchArray;
}
var teamIds = [
1364472344972,
1363173222886,
1363007586845,
1363007557484
]
var tournamentName = 'My Tournament';
Team.findByKey(teamIds[0], function(err, team1) {
if(err) {
util.log("Error occured");
}
if(!team1) {
util.log("The team does not exist");
}
Team.findByKey(teamIds[1], function(err, team2) {
if(err) {
return util.log("Error occured");
}
if(!team2) {
return util.log("The team does not exist");
}
Team.findByKey(teamIds[2], function(err, team3) {
if(err) {
return util.log("Error occured");
}
if(!team3) {
return util.log("The team does not exist");
}
Team.findByKey(teamIds[3], function(err, team4) {
if(err) {
return util.log("Error occured");
}
if(!team4) {
return util.log("The team does not exist");
}
var teamList = [
team1._id,
team2._id,
team3._id,
team4._id
];
var numRounds = Math.log(teamList.length)/Math.log(2);
var matches = genMatches(teamList.length);
for(var i = 0; i < matches[0].length; i++){
matches[0][i] = [ teamList[i+i], teamList[i+(i+1)] ]
}
var tournament = new Tournament({
name: tournamentName,
teams: teamList,
rounds: numRounds,
matches: matches
});
res.send(tournament);
});
});
});
});
});
Team schema:
'use strict';
var util = require('util');
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
var validatePresenceOf = function(value){
return value && value.length;
};
var toLower = function(string){
return string.toLowerCase();
};
var getId = function(){
return new Date().getTime();
};
/**
* The Team schema. we will use timestamp as the unique key for each team
*/
var Team = new Schema({
'key' : {
unique : true,
type : Number,
default: getId
},
'name' : { type : String,
validate : [validatePresenceOf, 'Team name is required'],
index : { unique : true }
}
});
/**
* Get complete team details for all the teams
*/
Team.statics.getAll = function(cb){
var query = this.find({});
query.sort({key : -1});
return query.exec(cb);
};
Team.statics.findByKey = function(key, cb){
return this.find({'key' : key}, cb);
};
Team.statics.findByName = function(name, cb) {
return this.find({'name' : name}, cb);
};
module.exports = mongoose.model('Team', Team);
You can use the $in operator to look up your four teams in one query:
Team.find({key: {$in: teamIds}}, function (err, teams) {
...
});
If for some reason you need to call findByKey multiple times instead of collapsing it all into a single $in, look at using the async library's each method to do that more cleanly.