Develop Reference

Nodejs Mogoose bulkWrite with condition check before upsert, and missing default fields

Problem: How to bulk insert million records by making condition check on a filed
Situation: I have to run a standalone node script multiple times and will get JSON data that needs to be inserted to Mongo DB. This JSON data can have same data which is inserted earlier or can have a change in one field (count) that needs to be updated.
Field 'count' can be greater, equal or lesser than the existing entry.
Requirement : I need to update that document (row) only if the count is greater than existing one.
I should be able to do this by fetching the existing records and then looping every record and do condition check. This is not I'm looking for as there will be millions of records to insert.
Question 1. Is there a way in mongoose bulkWrite to have such condition check before upsert?
Question 2. If bulkWrite is not what I should be using what
is the alternate solution which has better performance and less
overhead on application memory?
MongoDB server Version: 3.4.10
Below sample code inserts 2 records and after some delay, it tries to insert another record with only change in count that is lesser than existing entry in DB.
At this situation, my requirement is not to update the second record count value.
Question 3: Why is the default value fields are not set in DB?
Question 4: When dealing with huge set of data, is there an event to know if all the records are written to disk? As the DB Save is 'async' call, when to trigger database.close and process.exit(), current code is simply waiting for 5 seconds which is wrong way to implement.
If I don't close DB, application wont exit, and if I exit application with some timeout, it might result in exiting code before full data is written to disk.
/**
* Get DB Connection
*/
const mongoose = require("mongoose");
mongoose.Promise = global.Promise;
const url = "mongodb://127.0.0.1:27017/sample";
let DATABASE = {};
DATABASE.connect = () => {
let db = mongoose.createConnection(url, { autoIndex: false, connectTimeoutMS: 30000, reconnectTries: 30, reconnectInterval: 2000 });
console.log("Database connecting to URL ", url);
db.on('error', console.error.bind(console, 'connection error:'));
return db;
}
DATABASE.close = (db) => {
if (db) {
db.close(function () {
console.log('Mongoose default connection with DB disconnected ');
});
}
}
/**
* Now define schema (will be part of module file)
*/
const Schema = mongoose.Schema;
var apiDetailSchema = new Schema({
ip: String,
instance: Number,
component: String,
logDate: Number,
count: Number,
apiMethod: {type:String, default:'DEFAULT METHOD', required: true},
api: String,
status: String,
httpMethod: String,
environment: String,
datacenter: {type:String, default:'INDIA', required:true},
});
apiDetailSchema.index({ ip: 1, instance: 1, component: 1, logDate: 1, api: 1, status: 1, httpMethod:1, environment:1, datacenter:1}, { unique: true });
const API_DETAIL = {};
API_DETAIL.connect = () => {
if (API_DETAIL.db) {
console.log("Returning existing DB for API Schema");
return API_DETAIL.db;
}
console.log("Requesting New DB connection for API Schema");
API_DETAIL.db = DATABASE.connect();
return API_DETAIL.db;
}
API_DETAIL.close = () => {
if (API_DETAIL.db) DATABASE.close(API_DETAIL.db);
}
API_DETAIL.connect();
API_DETAIL.SCHEMA = API_DETAIL.db.model('apiDetail', apiDetailSchema);
/**
* Use of API_DETAIL to insert data
*/
var bulkUpdateApiData = (data) => {
let total = data.length;
return new Promise((resolve, reject) => {
if (total === 0) {
resolve("NO DATA to update API details");
}
console.log("Bulkupdating "+total+" API records");
let db = API_DETAIL.connect(); // Connect to DB
if (!db) {
console.log("Failed to obtain DB connection during API Bulk update");
reject("ERROR: DB Connection failed");
}
let bulkOps = [];
console.log("Going to Bulk update "+total+" API details");
data.forEach(d => {
let { ip, instance, component, logDate, count, api, status, httpMethod, environment, datacenter } = d;
let upsertDoc = {
'updateOne': {
// Filter applied to all field except count, so that it will update count
// TODO: Check if the count is more, then only update
'filter': { ip, instance, component, logDate, api, status, httpMethod, environment, datacenter },
'update': d,
'upsert': true
}
}
bulkOps.push(upsertDoc);
});
API_DETAIL.SCHEMA.bulkWrite(bulkOps).then(BulkWriteOpResultObject => {
console.log(total + " API Details updated to DB");
// console.log(JSON.stringify(BulkWriteOpResultObject, null, 2));
resolve("Updated "+total+ " API Details");
}).catch(e => {
console.log("ERROR upserting addIpDetail", e);
reject(e);
});
});
} // Function : bulkUpdateApiData
let initialData = [
{
"ip": "192.168.1.2",
"instance": 2,
"component": "NODE",
"logDate": "20180114",
"api": "/services/srest/abc/authenticator/login",
"status": "200",
"httpMethod": "POST",
"environment": "production",
"count": 8
},
{
"ip": "192.168.1.2",
"instance": 2,
"component": "NODE",
"logDate": "20180114",
"api": "/services/srest/abc/authenticator/logout",
"status": "204",
"httpMethod": "POST",
"environment": "production",
"count": 8888 // Initially it was more
}];
bulkUpdateApiData(initialData).then(output => {
console.log(output);
}).catch(e => {
console.log("Something went wrong during API Detail bulk update", e);
});
let newData = [
{
"ip": "192.168.1.2",
"instance": 2,
"component": "NODE",
"logDate": "20180114",
"api": "/services/srest/abc/authenticator/logout",
"status": "204",
"httpMethod": "POST",
"environment": "production",
"count": 10 // Now it is lesser than initial one
}];
// Wait for 2 seconds to complete previous write operation,
// if not below bulkWrite will complete first !!!
setTimeout(() => {
console.log("=================================================");
console.log("Bulk updating EXISTING data with lesser count");
bulkUpdateApiData(newData).then(output => {
console.log(output);
}).catch(e => {
console.log("Something went wrong during API Detail bulk update", e);
});
}, 2000);
console.log("-----------------------------------------------");
// As DB write / save is async operation, When should I call this CLOSE connection?
// Is there a way to know when exactly DB write is completed?
setTimeout(API_DETAIL.close, 5000);
Output:
> node bulkWrite.js
Requesting New DB connection for API Schema
Database connecting to URL mongodb://127.0.0.1:27017/sample
Bulkupdating 2 API records
Returning existing DB for API Schema
Going to Bulk update 2 API details
-----------------------------------------------
2 API Details updated to DB
Updated 2 API Details
=================================================
Bulk updating EXISTING data with lesser count
Bulkupdating 1 API records
Returning existing DB for API Schema
Going to Bulk update 1 API details
1 API Details updated to DB
Updated 1 API Details
Mongoose default connection with DB disconnected
DB Output, where the second record / document has update value in filed 'count'
> db.apidetails.find().pretty()
{
"_id" : ObjectId("5a5df2d1952021f65578fc8f"),
"api" : "/services/srest/abc/authenticator/login",
"component" : "NODE",
"count" : 8,
"datacenter" : null,
"environment" : "production",
"httpMethod" : "POST",
"instance" : 2,
"ip" : "192.168.1.2",
"logDate" : 20180114,
"status" : "200"
}
{
"_id" : ObjectId("5a5df2d1952021f65578fc90"),
"api" : "/services/srest/abc/authenticator/logout",
"component" : "NODE",
"count" : 10,
"datacenter" : null,
"environment" : "production",
"httpMethod" : "POST",
"instance" : 2,
"ip" : "192.168.1.2",
"logDate" : 20180114,
"status" : "204"
}

Get request with fetch and react

This is probably a very dumb question, but I cant find any solution on the web, please dont bash me. I set up a create-react-app and want to display some data in a table.
This is my JSON:
[ {
"unitid" : 29,
"createddate" : 1510324778000,
"latitude" : 49.402008056640625,
"longitude" : 11.901968002319336,
"senderid" : 6,
"signalstrength" : 37
}, {
"unitid" : 34,
"createddate" : 1510563384000,
"latitude" : 49.22679901123047,
"longitude" : 12.845210075378418,
"senderid" : 8,
"signalstrength" : 0
},......
And my table needs a JSON Array with all the attributes in order to display every column correctly.
This is the code where I try to fetch the data from an endpoint of mine:
class App extends Component {
constructor(props){
super(props);
this.state = {
vehicleList: [],
};
}
componentDidMount(){
fetch('endpoint-link', {mode: 'no-cors', method: 'get'})
.then(response => response.json())
.then(result => {
this.setState({vehicleList: result.vehicleList})
console.log(result)
})
.catch(error => {
this.setState({isLoaded: true, error})
console.log(error)
});
}
render() {
const {vehicleList} = this.state;
console.log(vehicleList);.......
When I open the Devtools in Chrome, go for "Network" I can see that the endpoint is known and the data is found, but somehow the fetch method is not loading the JSON into the Array.

Your JSON doesn't have a vehicleList property. It just has the array of vehicles. So you need to put that in your state:
this.setState({vehicleList: result})

The problem is this -> ...{mode="no-cors"...}
See this post in order to fix it: Handle response - SyntaxError: Unexpected end of input

ElasticSearch Javascript, Highlight not working

we switched recently to ElasticSearch Angular version and everything is working as expected except the Highlight, which is not returned at all.
This is how I setup a demo query:
$esClient.search({
index: 'myIndex',
body: {
size: 10,
from: 0,
query: query,
highlight: {
fields: {
"_all": { "pre_tags": ["<em>"], "post_tags": ["</em>"] }
}
}
}
}).then(function (result) {
// map the resultset for Row Template
var currentRows = result.hits.hits.map(function (record) {
return {
"type": record._type,
"entity": record._source, // the result
"highlight": record.highlight, // the highlights
"id": record._id // Search record ID
};
});
});
If I use the same code with a classic XmlHttpRequest and pass the query model inlcuding the highlight, I get back a JSON which contains an highlight array per each result, while using the ElasticSearch Angular client the query succeed but I don't get back the highlight.
Am I doing something wrong?

I think you might want to change to this format:
{
"query" : {...},
"highlight" : {
"pre_tags" : ["<tag1>"],
"post_tags" : ["</tag1>"],
"fields" : {
"_all" : {}
}
}}
https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-highlighting.html

Unable to retireve objects from Mongoose-Node

I have a Mongodb database with resume objects like below. I am using a node-express server. And I am querying the mongo database to get objects based on a specific skill. For example: If I query for a skill: jquery, only objects with that skill is returned. The problem is with the get function that is returning objects from the database.
In the highlighted code: If I directly insert the object like:
Resume.find({skills.jQuery : 2},function(err, results){...}
then I get the expected results.
However if I insert it dynamically (skillSet), then it doesnot work. Iit checked the value for skillSet and it give me what I expect('skills.JQuery')
var skillSet = ("'"+'skills.' + req.params.skill +"'");
console.log('skillSet',skillSet) //'skills.jQuery'
Resume.find({skillSet : 2},function(err, results){
Below is the code snippet:
{
"_id" : ObjectId("56031b4353b32084651173fb"),
"uuid" : "acd06792-87c3-4b0e-827a-8bd19d7f9c03",
"creationDate" : ISODate("2015-09-23T21:36:03.728Z"),
"status" : "3",
"name" : "resume_dev",
"__v" : 0,
"skills" : {
"node" : 2,
"react" : 2,
"React" : 3,
"JQUERY" : 2,
"JavaScript" : 15,
"JQuery" : 5,
"Backbone" : 3,
"Node" : 2,
"Angular" : 4,
"Javascript" : 2,
"jQuery" : 17,
"javascript" : 3
}
}
router.get('/skills/:skill', function(req, res){
console.log("req.params.skill",req.params.skill);
var skillSet = ("'"+'skills.' + req.params.skill +"'");
console.log('skillSet',skillSet) //skills.react
Resume.find({skillSet : 2},function(err, results){
console.log('hi');
console.log(skillSet)
console.log(results);
if (err) {
res.status(500).json(err);
}
else {
// console.log("====>",results);
res.status(200).json(results);
// res.render('status',{Resume: JSON.stringify(results)});
}
});
});

When I understand your question correctly, you can do something like this:
var query = {};
query['skills.' + req.params.skill] = 2;
Resume.find(query,function(err, results){
// Do something with the callback
};
EDIT:
If you want to get all numbers greater or equal to 1, you will need $gte. Here are the docs. This is the updated query:
var query = {};
query['skills.' + req.params.skill] = {$gte: 1};
Resume.find(query,function(err, results){
// Do something with the callback
};

Firebase: How do I retrieve records from my data for which a specific key exists?

I have data in firebase that looks like this:
"application": {
"companies": {
"firebase": {
"creation": {
"name": "Firebase Inc",
"location": "USA"
},
"google": {
"creattion": {
"name": "Google Inc",
"location": "USA"
}
}
"facebook": {
},
"apple": {
}
}
}
}
There are tens of thousands of records under companies key. How do i efficiently execute following queries?
How do I query only the records for which key creation is present under their name?
How do I query only the records that DO NOT have key creation present under their name?
I also want to call .on('child_added') on the returned result set so that I can process only those specific records later on. Is it possible?

EDIT: Simpler way without using an extra parameter
Queries
Here are the queries to do this without having to use an extra parameter:
Find the companies without creation:
var ref = new Firebase(fbUrl+'/companies').orderByChild("creation").equalTo(null);
Find the companies with creation:
var ref = new Firebase(fbUrl+'/companies').orderByChild("creation").startAt(!null);
You would add ".indexOn": "creation" to the rules.
Edit 2: I was curious, so I pushed 11,000 records to /companies2 (half with creation children, half without). I was able to retrieve 5500 matching records in ~4 seconds using the above queries (or one of the variants I've shown below).
Edit 3: If you're running these queries frequently, it might be worth it to separate children of /companies into two bins based the presence of creation. That way, you can read the two segments separately without having to rely on queries.
Factory
Here is what the revised factory would look like (I've revised the PLNKR to match):
app.factory("CompaniesFactory",function($q, fbUrl){
return function(hasCreation){
var deferred = $q.defer();
var ref = new Firebase(fbUrl+'/companies').orderByChild("creation");
var query;
if (hasCreation) {
query = ref.startAt(!null);
// or:
// query = ref.startAt(true);
} else {
query = ref.equalTo(null);
// or:
// query = ref.endAt(!null);
// query = ref.endAt(true);
}
query.once("value", function(dataSnapshot){
deferred.resolve(dataSnapshot.val());
}, function (error) {
deferred.reject(error);
});
return deferred.promise;
}
});
And yes, it is possible to call .on('child_added') on the returned dataSnapshot. See DataSnapshot.ref().
Original answer using an extra parameter:
(Keeping this for reference)
Another way to do it would be by adding another parameter called hasCreation to children of companies that have creation, and query by that.
Data
The query would then be var ref = new Firebase(fbUrl+'/companies').orderByChild("hasCreation").equalTo(hasCreation);
If hasCreation in the query is null, the query will return the companies without a hasCreation child.
If hasCreation in the query is true, the query will return the companies with hasCreation===true.
{
"company1" : {
"creation" : {
"name" : "company1"
},
"hasCreation" : true
},
"company2" : {
"name" : "company2"
},
"company3" : {
"name" : "company3"
},
"company4" : {
"creation" : {
"name" : "company4"
},
"hasCreation" : true
}
}
Rules
You would add the ".indexOn" : "hasCreation" to your rules like so:
"so:29179389":{
".read" : true,
".write" : true,
"companies" : {
".indexOn" : "hasCreation"
}
}
Companies Factory
app.factory("CompaniesFactory",function($q, fbUrl){
return function(hasCreation){
var deferred = $q.defer();
if (!hasCreation) {
hasCreation = null;
}
var ref = new Firebase(fbUrl+'/companies').orderByChild("hasCreation").equalTo(hasCreation);
ref.once("value", function(dataSnapshot){
deferred.resolve(dataSnapshot.val());
});
return deferred.promise;
}
});
Controller
app.controller('HomeController',function($scope,fbUrl,CompaniesFactory) {
$scope.getCompanies = function(hasCreation) {
var companies = new CompaniesFactory(hasCreation).then(function(data){
console.log(data);
$scope.companies = data;
});
}
});
HTML
<body ng-app="sampleApp">
<div ng-controller="HomeController">
<button ng-click="getCompanies(true)">Find with creation</button>
<button ng-click="getCompanies(false)">Find without creation</button>
<h2>Companies:</h2>
{{companies}}
</div>
</body>

What I would do, is I would set a condition to verify if your xxx.firebaseio.com/Application/companies/______/creation exists. In a empty blank, you can set for loop to irritate over the array of companies.Then, you can create two arrays with angular.forEach: one including those companies which do have 'creation', and the other array in which the elements do not include the 'creation'.
Hope that helps :)
Edit:
There is another approach to this question, in this thread:
Angularfire: how to query the values of a specific key in an array?