Develop Reference

javascript elasticsearch - works on local instance but not on server?

I have a javascript visualization script that uses tweets which are indexed in elasticsearch (digested by tweepy library), the tweets are getting through to elasticseach and all is working fine, however the visualization only works with my local instance of elasticsearch (even if i connect to it through server and both instances of elasticsearch are running), if i disconnect the local elasticsearch and try to run the javascript it doesnt work and im getting a TypeError: response is undefined and Unable to revive connection: http://localhost:9200/.
What do i need to change so it works from the elasticsearch instance rather than the local instance?
here is the javascript code:
$(function() {
// Data
var colors = ["#8dd3c7","#ffffb3","#bebada","#fb8072","#80b1d3","#fdb462","#b3de69","#fccde5","#d9d9d9","#bc80bd","#ccebc5","#ffed6f"];
var client = new elasticsearch.Client({
host: 'localhost:9200',
log: 'trace'
});
var index = "tw";
var loadData = function() {
var data_count = 10000;
client.search({
index: index,
q: "*.*",
size: data_count,
sort : 'timestamp:desc'
// sort: '_id'
}, function (error, response) {
hits = response.hits.hits;
data = hits.map(function(element, index, array) {
return element._source
});
// console.log(data);
formatTWEEPY(updateVis);
//
});
};
loadData();
// Instantiate vis
var timesets = sm.vis.timesets()
.colors(colors)
// .setMode("background")
// .applyLayout(false)
.verticalMode("ellipse")
.elementMode("gradient")
.showSettings(true)
.margin({ top: 0, right: 180, bottom: 0, left: 25 });
// Update the vis
var updateVis = function() {
// Update dimension
var width = $("svg").width();
var height = $("svg").height();
timesets.width(width).height(height - 30);
d3.select(".sm-timesets-demo").attr("transform", "translate(0, " + height + ")");
redraw();
};
function redraw() {
d3.select(".sm-timesets-demo").datum(data).call(timesets);
}
function formatTWEEPY(callback) {
data.forEach(d => {
d.themes = d.hashtags.map(function(t) {return t.text});
d.time = new Date(d.timestamp);
d.title = d.message;
});
// console.log(data)
var hashtags = flatten(data.map(function(t){return t.themes}))
// console.log(hashtags)
var dict = hashtags.reduce(function (p, c) {
p[c] = (p[c] || 0) + 1;
return p;
}, {});
var items = Object.keys(dict).map(function(key) {
return [key, dict[key]];
});
themes = items.sort((a, b) => d3.descending(a[1], b[1])).slice(0, 10).map(e => e[0])
data = { events: data.slice(), themes: themes };
callback();
}
});
function flatten(arr) {
return arr.reduce(function (flat, toFlatten) {
return flat.concat(Array.isArray(toFlatten) ? flatten(toFlatten) : toFlatten);
}, []);
}
I have tried to add an IP to the server instead of the host: 'localhost:9200' however it made no difference, it still did not work. Ive added these lines to my elasticsearch.yml on both server instance as well as local one so it allows the connection:
http.cors.enabled: true
http.cors.allow-origin: "*"
http.cors.allow-methods : OPTIONS, HEAD, GET, POST
I have tried this example from elastic website and im getting the elasticsearch cluster is down response but not sure how to fix it so i get the correct respose:
client.ping({
requestTimeout: 30000,
}, function (error) {
if (error) {
console.error('elasticsearch cluster is down!');
} else {
console.log('All is well');
}
});
The elasticsearch in running is 6.2
Thanks for the help!

please provide systemctl​ ​status​ ​elasticsearch output
works for me after editing /etc/elasticsearch/elasticsearch.yml
http.cors.enabled:​ ​ true
http.cors.allow-origin:​ ​ "*"
$​ ​apt-get​ ​install​​ elasticsearch-6.2.deb
$​ systemctl​ enable​​ elasticsearch # important
$​ systemctl​ ​start​ ​elasticsearch

remember search query to use in function

I'm using Angularjs with the dirPagination plugin to connect with an Web API. This seems to work fine. I added a search function, to do a server side search:
$scope.searchChanged = function () {
if ($scope.searchFor.length == 0) {
$scope.calculatedValue = 'e';
} else {
vm.getData(vm.pageno, vm.getSearch($scope.searchFor));
}
}
vm.getSearch = function (query) {
if (query == undefined) {
query = 'tech';
} else {
query = query;
}
return query;
}
See Plnkr for the full code
If I start searching (e.g. sales) the API returns results and the paging is correct, the get request is:
/api/students/?category=sales&begin=1&pageSize=10
But if you want to go to another page number, the get request to the server is:
/api/students/?category=tech&begin=2&pageSize=10
How can the view remember the query 'sales', so that the paging and results are correct?

You are making a common mistake here: You don't need to pass in variable from the view if you are already using a scope variable.
Changing to this would be much less error prone
// change this to var getSearch or function getSearch if you don't need it on the view anymore
vm.getSearch = function () {
var query = vm.searchFor;
// you should only use vm, change ng-model to data.searchFor
if (query == undefined) {
query = 'tech';
}
return query;
}
vm.getData = function () {
vm.users = [];
$http.get("/api/students/?category=" + vm.getSearch() + "&begin=" + vm.pageno + "&pageSize=" + vm.itemsPerPage).success(function (response) {
vm.users = response.data;
vm.total_count = response.total_count;
});
};

Your request id good, you need to optimize the sql query so you can get the right results. it should look something like this:
#begin INT = 0,
#pageSize INT = 10
SELECT *
FROM [TableName]
ORDER BY id
OFFSET (#pageSize * #begin )
ROWS FETCH NEXT #pageSize ROWS ONLY;

Angular Factory and Service work but not as expected

My app is looking up google place details and displaying some of the information. I have a list of place id's in a json file broken down by type of establishment. A factory accesses and makes available the ids to the controller. I also have a service that loops through all the id's, looking up the details and adding them to an object that is made available to the controller.
I can get it to work in the sense that I can access the json data, look up the details, and return the object. However, no matter how I do it, if I try and return multiple objects, one for each type of business, I get all the businesses together or an error (more on that in a minute).
I have structured this a number of ways but I will show the code for 2 ways that I have tried. I'm new to Angular so I may have this completely wrong and not even using services and factories correctly so please go easy on me.
locations.json
{
"restaurants": {
"Michaels": "ChIJwaTJAL4n5IgRgyJupbpQhjM",
"Collage": "ChIJw5HgNzAm5IgRqbkEqKXIpC4",
"Scarlet": "ChIJT9ImkZUn5IgREb1hYwKA1Nc",
"Maya": "ChIJofgqBJYn5IgRVa-HQvp6KDk",
"Ice": "ChIJnXpQpewn5IgR7k9yxWXUu1M",
"Sangrias": "ChIJITcc_ZUn5IgR90iEna6FRGM",
"Columbia": "ChIJ8xR18JUn5IgRfwJJByM-quU",
"Harrys": "ChIJ8aLBaJYn5IgR60p2CS_RHIw"
},
"bars":
{
"Scarlet": "ChIJT9ImkZUn5IgREb1hYwKA1Nc",
"Lion": "ChIJqVCL_b0n5IgRpVR5CFZWi4o",
"Tradewinds": "ChIJpwF4ZJYn5IgRTDzwBWvlSIE",
"Ice": "ChIJnXpQpewn5IgR7k9yxWXUu1M",
"Stogies": "ChIJlwkiApYn5IgR6XVFMyqLAS4",
"Rondeazvous": "ChIJkz3V7pUn5IgRQhui26imF1k",
"Meehan": "ChIJK8NZGZYn5IgRA91RrGETwrQ",
"Sangrias": "ChIJITcc_ZUn5IgR90iEna6FRGM",
"NoName": "ChIJA-VeCb4n5IgRmbuF8wdOGaA",
"StGeorge": "ChIJ4yo36JUn5IgRXgiRD7KMDe0"
}
}
Method 1
locations.js
angular.module('app.locations', [])
.factory('restsFact', function($http){
var restaurants = [];
return {
getRests: function(){
return $http.get('locations.json').then(function(response){
restaurants = response.data.restaurants;
return restaurants;
});
}
};
})
.factory('barsFact', function($http){
var bars = [];
return {
getBars: function() {
return $http.get('locations.json').then(function(response){
bars = response.data.bars;
return bars;
});
}
};
})
.service('locationsService', function (ngGPlacesAPI) {
var x, id, details, push, placeDetails = [];
// Takes list of specific type of locations as argument and looks up Place details for each location
this.details = function(type) {
for (x in type) {
if (type.hasOwnProperty(x)) {
id = type[x];
ngGPlacesAPI.placeDetails({placeId: id}).then(push);
}
}
return placeDetails;
};
push = function (data) {
details = data;
placeDetails.push(details);
};
});
Controllers
.controller('RestCtrl', function($scope, locationsService, restsFact) {
// Location Details Object
restsFact.getRests().then(function(locs){
$scope.restaurants= locationsService.details(locs);
});
})
//
// Bar Controller
//
.controller('BarsCtrl', function($scope, locationsService, barsFact){
// Locations Details Object
barsFact.getBars().then(function(locs){
$scope.bars = locationsService.details(locs);
});
})
Method 2
With this method I can load one page but if I move to the next I get an error: [$rootScope:inprog] $digest already in progress. I read up on the error and get the idea of why I get it but just not sure how to go about fixing it.
locations.js
angular.module('app.locations', [])
.factory('locationsFact', function($http){
var locations = [];
return {
getlocations: function(){
return $http.get('locations.json').then(function(response){
locations = response;
return locations;
});
}
}
})
.service('locationsService', function (ngGPlacesAPI) {
var x, id, details, push, placeDetails = [];
// Takes list of specific type of locations as argument and looks up Place details for each location
this.details = function(type) {
for (x in type) {
if (type.hasOwnProperty(x)) {
id = type[x];
ngGPlacesAPI.placeDetails({placeId: id}).then(push);
}
}
return placeDetails;
};
push = function (data) {
details = data;
placeDetails.push(details);
};
});
Controller
.controller('locationsCtrl', function($scope, locationsService, locationsFact){
// Locations Details Object
locationsFact.getlocations().then(function(locs){
$scope.restaurants = locationsService.details(locs.data.restaurants);
$scope.bars = locationsService.details(locs.data.bars);
});
})

So I read a lot over the last week and learned a lot as well. I completely rewrote that mess up above into something resembling decent code, there were a lot of problems with it originally. I got everything working anyway. Here is how it looks now.
Factory
angular.module('app.factories', [])
.factory('data', function($http){
// Get JSON With Place ID's and create array of
// place id objects for each category
var places = {};
places.ids = function(){
return $http.get('locations.json')
.success(function(data){
places.rests = data.restaurants;
places.bars = data.bars;
places.lodg = data.lodging;
places.att = data.attractions;
});
};
return places;
})
.factory('details', function(ngGPlacesAPI, $q){
var details = {};
// Split ID Array into array of arrays <= 10.
// Google won't return more than 10 details request at one time.
details.process = function(type) {
var idSets = {},
size = 10,
i, j, k;
for (i=0, j=type.length, k=0; i<j; i+=size){
idSets[k] = type.slice(i, i+size);
k++;
}
return idSets;
};
// Lookup Details by Place ID
// Loop through ID array and return array of details objects
details.getDetails = function(idSet, pageNum) {
var page = idSet[pageNum],
promises = [];
for(var i=0; i<page.length; i++) {
promises.push(ngGPlacesAPI.placeDetails({placeId: page[i][i]}));
}
return $q.all(promises);
};
// Return Details Object
return details;
});
Controller
//
// Restaurants Controller
//
.controller('restaurantsCtrl', function(details, data, $scope) {
var vm = this;
// Get JSON file with placeIds and set some variables
data.ids().then(function() {
var page = details.process(data.rests),
pageNum = 0,
numPages = page.length;
vm.moreData = true;
// Loads more place details on scroll down
vm.loadMore = function() {
if (pageNum <= numPages - 1) {
pageNum++;
details.getDetails(page, pageNum).then(function(response) {
vm.rests.push(response);
vm.$broadcast('scroll.infiniteScrollComplete');
});
}else{vm.moreData=false}
};
// Load first page of place details
details.getDetails(page, pageNum).then(function(response){
vm.rests = response;
console.log(vm.rests);
});
// Watches for when to load more details
$scope.$on('$stateChangeSuccess', function(){
vm.loadMore();
});
});
})

How can i iterate over elastic search results using angular?

In my dashboard application am using elastic search to retrieve data, which retrieves data fine. Now i need to iterate over the data and get the result in the required way.
Here is my code,
routerApp.controller('SearchCtrl', function($scope, ejsResource) {
var ejs = ejsResource('http://192.168.1.200:9200');
var oQuery = ejs.QueryStringQuery().defaultField('Agent');
var client = ejs.Request()
.indices('nondomain_callrelatedinfo')
.types('callrelatedinfos');
$scope.search = function() {
$scope.results = client
.query(oQuery.query($scope.queryTerm || '*'))
.doSearch();
console.log($scope.results);
};
});
I have added a console.log in the results and it reuturns something like this,
Data:
when i iterate over hits, it says cannot read property of "hits" undefined
$scope.dataRetrieved= $scope.results.value.hits;
for (var i = 0; i < $scope.dataRetrieved.length; i++) {
console.log($scope.dataRetrieved[i]);
};
};

You should use the Promise object that the doSearch() returns :
client
.query(oQuery.query($scope.queryTerm || '*'))
.doSearch().then(function (resp) {
var hits = resp.hits.hits;
// do whatever
}, function (err) {
console.trace(err.message);
});
Knowing angular, if you want to bind the result to $scope.result maybe a $timeout(function() {}) could also help.

https://docs.angularjs.org/api/ng/function/angular.forEach
just nest your forEach:
angular.forEach(results, function(result, key) {
angular.forEach(result, function(value, key) {
this.push(key + ': ' + value);
}, log);
}, log);

$scope.results.hits.hits;
$scope.dataRetrieved= $scope.results.hits.hits;
for (var i = 0; i < $scope.dataRetrieved.length; i++) {
console.log($scope.dataRetrieved[i]);
};
};
the first hits is a object the second is your array. thats why the first hits has no property length

Building OData $filter URLs with an Angular provider

I have an angular provider for querying an OData service.
Now I'm trying to build a $filter function onto that provider, so I can use it throughout my app.
The problem I'm running into, and so far haven't been able to solve, is that the query part of the URL needs to start with '$filter=' which I can handle fine when there is one filter, but multiple filters are added as ' and {query goes here}'.
A sample query would be:
http://www.example.com/odata/Restaurants/?$filter=id eq 2 and city eq 'Los Angeles' and substringof("Lagasse", chef)&$top=20
I'm sending all the filters in an array to the provider. Ideally, I would use "$filter=#{firstFilter}" for the first filter in the array and for the remaining filters use " and #{remainingFilter}" but I'm unsure how to structure this code.
My current code uses multiple if statements to check if a filter is there, but with the nature of building the url, it makes one of the filters mandatory at all times. I'd like to avoid this.
For example:
var filter = "$filter=id eq 2";
if (city) {
filter += " and city eq #{cityName}";
}
if (chef) {
filter += " and substringof(#{chefName}, chef)";
}
Now everytime a user inputs a query, they have to specify an id.
We are NOT using BreezeJS, JayData, or any other library. Strictly AngularJS and specifically $http, NOT $resource.

You can use odata-filter-builder to build $filter part for OData URL query options.
Then just use $http with config params.
Short example:
var filter = ODataFilterBuilder()
.eq('id', 2)
.eq('city', 'Los Angeles')
.and('substringof("Lagasse", chef)')
.toString();
$http
.get(resourceUrl, {params: {$filter: filter, $top: 20}})
.then(function(response) {
// Handle response
})
Full example:
angular
.module('OData', [])
.constant('ODataFilterBuilder', ODataFilterBuilder)
.factory('ODataService', function($http) {
return {
load: function(resourceUrl, queryParams) {
return $http.get(resourceUrl, {params: queryParams})
.then(function(response) {
return response.data.value;
});
}
}
});
angular
.module('app', ['OData'])
.controller('appController', function($http, ODataService, ODataFilterBuilder, $httpParamSerializer) {
// 1. inject ODataFilterBuilder
// use short name for filter builder
var f = ODataFilterBuilder;
// 2. build filter
var filter = f()
.eq('id', 2)
.eq('city', 'Los Angeles')
.and('substringof("Lagasse", chef)')
// 3. creater odata query params
var queryParams = {
$filter: filter.toString(),
$top: 20
// TODO: add other params
};
// 4. prepare odata resourse URL
var odataServiceUrl = 'http://path/to/odata/service/';
var odataResourseUrl = odataServiceUrl + 'entity';
// 5. Do http request with odataResourseUrl and queryParams
// use ODataService or angular $http service
// ODataService
// .load(odataResourseUrl, queryParams)
// .then(function(value) {
// // handle value
// });
// OR
// $http.get(odataResourseUrl, {params: queryParams})
// .then(function(respons) {
// // handle respons.data.value
// });
// Result examles:
// NOTE: $httpParamSerializer - default $http params serializer that converts objects to strings
var queryParamsSerialised = $httpParamSerializer(queryParams);
// put values to 'this' to use it in html
this.queryParams = queryParams;
this.queryParamsSerialised = queryParamsSerialised;
this.queryUrl = odataResourseUrl + '?' + queryParamsSerialised;
});
<div ng-app="app" ng-controller="appController as vm">
<pre>queryParams: {{vm.queryParams|json}}</pre>
<pre>queryParamsSerialised: {{vm.queryParamsSerialised}}</pre>
<pre>queryUrl: {{vm.queryUrl}}</pre>
</div>
<script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.5.5/angular.min.js"></script>
<script src="https://npmcdn.com/odata-filter-builder#^0.1/dist/odata-filter-builder.js"></script>

I know you said you are not using any other library, but in case anyone else stumbles up this...I suggest using joData. It's very expressive and requires no other dependencies: https://github.com/mccow002/joData
Here is your example in joData:
var query = new jo('http://www.example.com/odata/Restaurants');
query.filter(new jo.FilterClause('id').eq(2))
.andFilter(new jo.FilterClause('city').eq('Los Angeles'))
.andFilter(new jo.FilterClause('chef').substringof('Lagasse').eq(true))
.top(2);
Then,
query.toString()
produces...
"http://www.example.com/odata/Restaurants?$top=2&$filter=id eq 2 and city eq 'Los Angeles' and substringof('Lagasse',chef) eq true"
Anyway, to set this up for Angular here is what I did...
Reference joData js in your html before your angular.js reference.
Wrap it in an Angular service
angular.module('app.shared.oData', [])
.service('oDataBuilderService', function () {
this.jo = jo;
};
Now other services can use this odata service to construct odata queries:
angular.module('app.shared.video', [])
.service('videoService', function ($http, oDataBuilderService) {
this.getByVideoCatalogId = function (videoCatalogId) {
var query = new oDataBuilderService.jo('http://www.example.com/VideoCatalog/OData');
var videoCatalogIdEquals = new oDataBuilderService.jo.FilterClause("VideoCatalogId").eq(videoCatalogId);
query.andFilter(videoCatalogIdEquals);
var queryAsString = query.toString();
var promise = $http.get(queryAsString);
return promise;
}
});

You can add $filter= at the end, but then you'd have the same problem with the "and "
Easier would be to just create a simple function
addFilter(currentFilter, filter) {
if (currentFilter.length == 0) {
currentFilter = "$filter=" + filter;
} else {
currentFilter += "and " + filter;
}
return currentFilter;
}
so then just call currentFilter = addFilter(currentFilter, "what to filter")
var currentFilter = "";
if (city) {
currentFilter = addFilter(currentFilter, "city eq #{cityName}");
}
if (chef) {
currentFilter = addFilter(currentFilter, "substringof(#{chefName}, chef)");
}

I've done similar and have what I think is an elegant solution with no dependencies on other libraries. I use Array.join and concatenate the filters with an "and" between them:
var filter = [];
if (city) filter.push("City eq '" + city + "'");
if (chef) filter.push("Chef eq '" + chef + "'");
var filterQuery = filter.join(" and ");
This assumes you always want "and" between them, (not or), but that is the case in my apps and in your example. I also do this to build up the url:
var parts = [baseurl];
parts.push("$top=10");
parts.push("$skip=" + skip);
parts.push(filterQuery);
var url = parts.join("&");