Develop Reference

Error retrieving JSON data in node.js

I'm trying to make an android app which displays the top command data on my AWS ec2 instance. The following is the node.js file I've written. It sends the top command result using a topparser npm module. And using the checkCpuUsage function, it sends a notification if cpu usage is more than 1%.
var express = require('express');
var app = express();
var fs = require('fs');
var spawn = require('child_process').spawn;
var topparser=require("topparser");
var FCM = require('fcm-node');
var server = app.listen(8080);
var regTokens = [];
var bodyParser = require('body-parser');
app.use(bodyParser.json());
let data_builder = "";
const DEFAUT_TOP_DURATION = 2000;
function getTopData(callback, duration) {
duration = duration || DEFAUT_TOP_DURATION;
var top = spawn('top');
top.stdout.on('data', function(data) {
data_builder+=data.toString();
});
setTimeout(function() {
top.kill();
callback(topparser.parse(data_builder,10))
},duration);
}
function sendServerData(req, res) {
getTopData((data)=>{
res.status("200").send(data);
})
}
function checkCpuUsage() {
getTopData((data)=>{
var idl = data.cpu.idle;
if(idl<99) {
sendNotification(idl);
}
}, 2000)
}
function saveRegToken(req, res) {
console.log("saveRegToken req body", req.body)
regTokens.push(req.body.token);
res.send(200);
}
let server_key =JSON.parse(fs.readFileSync(('./config.json'))); ;
function sendNotification(idle){
var idl = idle;
var fcm = new FCM(server_key);
for(i=0; i<regTokens.length; i++) {
var message = {
to: 'regTokens[i]',
notification: {
title: 'CPU Usage!',
body: 'CPU Usage: ' + idl,
},
};
fcm.send(message, function(err, response){
if (err) {
console.log("Something has gone wrong!");
} else {
console.log("Successfully sent with response: ", response);
}
});
}
}
setInterval(()=>checkCpuUsage(),10000);
app.get('/sys', sendServerData);
app.post('/save_token', saveRegToken)
console.log("server listening on 8080");
I'm getting the following error in the checkCpuUsage() function:
var idl = data.cpu.idle;
^
TypeError: Cannot read property 'cpu' of undefined
at getTopData (/home/ubuntu/server_monitor/server1.1.1.js:38:17)
at Timeout._onTimeout (/home/ubuntu/server_monitor/server1.1.1.js:26:3)
at ontimeout (timers.js:469:11)
at tryOnTimeout (timers.js:304:5)
at Timer.listOnTimeout (timers.js:264:5)
Following is an example JSON result from the topparser module:
{
process:[
{
pid:'1990',
user:'alex',
pr:'20',
ni:'0',
virt:'1560516',
res:'90656',
shr:'21864',
s:'S',
cpu:'6.1',
mem:'12.5',
time:'13:46.58',
command:'cinnamon'
},
{
pid:'5381',
user:'alex',
pr:'20',
ni:'0',
virt:'929508',
res:'119792',
shr:'8132',
s:'S',
cpu:'6.1',
mem:'16.5',
time:'11:14.11',
command:'firefox'
},
{
pid:'0245',
user:'alex',
pr:'20',
ni:'0',
virt:'24948',
res:'1508',
shr:'1056',
s:'R',
cpu:'6.1',
mem:'0.2',
time:'0:00.02',
command:'top'
},
{
pid:'1',
user:'root',
pr:'20',
ni:'0',
virt:'37352',
res:'5688',
shr:'488',
s:'S',
cpu:'0.0',
mem:'0.8',
time:'0:04.93',
command:'init'
},
{
pid:'2',
user:'root',
pr:'20',
ni:'0',
virt:'0',
res:'0',
shr:'0',
s:'S',
cpu:'0.0',
mem:'0.0',
time:'0:00.07',
command:'kthreadd'
},
{
pid:'3',
user:'root',
pr:'20',
ni:'0',
virt:'0',
res:'0',
shr:'0',
s:'S',
cpu:'0.0',
mem:'0.0',
time:'0:54.23',
command:'ksoftirqd/0'
},
{
pid:'4',
user:'root',
pr:'20',
ni:'0',
virt:'0',
res:'0',
shr:'0',
s:'S',
cpu:'0.0',
mem:'0.0',
time:'0:00.00',
command:'kworker/0:0'
}
],
task:{
total:194,
running:1,
sleeping:193,
stopped:0,
zombie:0
},
cpu:{
user:0.9,
system:3.1,
ni:0.1,
'idle':95,
wa:0.3,
hi:0.6,
si:0,
st:0
},
ram:{
total:727308,
used:664028,
free:63280,
buffers:7600
},
swap:{
total:753660,
used:309516,
free:444144,
cachedMem:187424
}
}

The problem is here :
var top = spawn('top');
top.stdout.on('data', function(data) {
data_builder+=data.toString();
});
setTimeout(function() {
top.kill();
callback(topparser.parse(data_builder,10))
},duration);
You are trying to call the callback using a timeout which is bad practice. Instead, You call the callback from inside the stdout.on('data') hence, making sure that function is called and there is data available before calling the callback.
Like so :
top.stdout.on('data', function(data) {
data_builder+=data.toString();
top.kill();
callback(topparser.parse(data_builder,10))
});
Ideally, Using promises would help to make the code more readable and structured.

Error clearly pointing TypeError: Cannot read property 'cpu' of undefined
It means here your data is undefined. Here you are not getting data from your getTopData().
You can check this two lines data_builder+=data.toString(); or topparser.parse(data_builder,10)
Add some console statements, You will get the idea.
SUGGESTIONS :
Be consistent in your code. Somewhere you are writing plain JavaScript syntax and somewhere in ES6
Make a habit to follow any one.

CRON Job not working on meteor

In main.js in server folder,
I have this code,
var DDP = require('ddp');
var DDPlogin = require('ddp-login');
var Job = require('meteor-job');
var ddp = new DDP({
host: "127.0.0.1",
port: 3000,
use_ejson: true
});
Meteor.startup(() => {
var myJobs = JobCollection('myJobQueue');
Job.setDDP(ddp);
ddp.connect(function (err) {
if(err) throw err;
DDPlogin(ddp, function (err, token) {
if (err) throw err;
});
});
myJobs.allow({
admin: function (userId, method, params) {
return true;
}
});
Meteor.publish('allJobs', function () {
return myJobs.find({});
});
myJobs.startJobServer();
var workers = Job.processJobs('myJobQueue', 'sendEmail',
function (job, cb) {
console.log(job.data.text);
job.done();
cb(null);
}
);
And in my main.js in client folder,
I have this code,
var myJobs = JobCollection('myJobQueue');
var jobSub = null;
class App extends Component {
componentDidMount(){
if(jobSub !== null)
jobSub.stop();
jobSub = Meteor.subscribe('allJobs');
var job = new Job(myJobs, 'sendEmail',
{
text: 'bozo#clowns.com'
}
);
job.priority('normal')
.retry({ retries: 5,
wait: 60*1000 }) // 1 minute between attempts
.delay(0) // start immediately
.save();
}
...
render(){
console.log(myJobs.find().fetch());
...
}
}
I am using the vsivsi:meteor-job-collection package.
The problem is that console.log() is not executed.
What is wrong in my step by step installation and usage?
I need to console.log() every minute.

benchmarking server functions

I try to use benchmark.js to get some stats on a node server performance but somehow can't get the server function call and no stats, the only stuff I get - the server starts through the benchmarking script.
How to get the stats and not start the server if benchmark.js is suitable for this kind of tasks?
the server:
t-server.js
'use strict';
var
http = require('http');
var self = module.exports = {
init: function(){
self.server = http.createServer(self.handleRequest);
self.server.listen(8080, function(){
console.log("Server up ");
});
}
}
self.handleRequest = function(request, response) {
var requestParams = {a: 'a', b: 1};
self.process2test(response, requestParams);
}
self.process2test = function(response, requestParams) {
console.log("in process2test, got a:"+requestParams.a+",b:"+requestParams.b);
}
self.init();
benchmark script
benchmark-v.js
var Benchmark = require('benchmark');
var suite = new Benchmark.Suite();
var server = require('./t-server');
var response={};
var requestParams = {
"a":"a",
"b": 2
};
suite.add('process2test_a', function(response, requestParams) {
requestParams.b++;
server.process2test(response,requestParams);
})
.add('process2test_b', function(response, requestParams) {
requestParams.a='c';
server.process2test(response,requestParams);
})
.on('cycle', function(event) {
console.log(String(event.target));
})
.on('complete', function() {
console.log('Fastest is ' + this.filter('fastest').map('name'));
})
.run({ 'async': true });

How to create consumer for kafka topic?

I have topic created on kafka server now i am creating consumer to read topic messages from server, but i dont see any data when i try to consume message using consumer.on('message') , Any idea what is implemented wrong in below code , Do i need to set offset ?
consumer.js
var kafka = require('kafka-node');
var config = require('./config.js');
var zk = require('node-zookeeper-client');
var kafkaConn = config.kafkaCon.dit;
var HighLevelConsumer = kafka.HighLevelConsumer;
var Client = kafka.Client;
function start() {
topics = [{
topic: 'test-1'
}];
var groupId = 'push';
var clientId = "consumer-" + Math.floor(Math.random() * 10000);
var options = {
autoCommit: true,
fetchMaxWaitMs: 100,
fetchMaxBytes: 10 * 1024 * 1024,
groupId: groupId
};
console.log("Started consumer: ", clientId);
var consumer_client = new kafka.Client(kafkaConn, clientId);
var client = new Client(consumer_client.connectionString, clientId);
var consumer = new HighLevelConsumer(client, topics, options);
consumer.on('message', function(message) {
var topic = message.topic;
console.log('Message', topic);
});
};
start();

const Kafka = require("node-rdkafka");
const kafkaConf = {
"group.id": "cloudkarafka-example",
"metadata.broker.list": ["localhost:9092"],
"socket.keepalive.enable": true,
//"security.protocol": "SASL_SSL",
//"sasl.mechanisms": "SCRAM-SHA-256",
//"sasl.username": process.env.CLOUDKARAFKA_USERNAME,
//"sasl.password": process.env.CLOUDKARAFKA_PASSWORD,
"debug": "generic,broker,security",
'enable.auto.commit': false,
};
//const prefix = process.env.CLOUDKARAFKA_USERNAME;
const topics = ['topicName'];
const consumer = new Kafka.KafkaConsumer(kafkaConf, {
"auto.offset.reset": "beginning"
});
consumer.on("error", function(err) {
console.error(err);
});
consumer.on("ready", function(arg) {
console.log(`Consumer ${arg.name} ready`);
consumer.subscribe(topics);
consumer.consume();
});
consumer.on("data", function(m) {
console.log(m.value.toString());
});
consumer.on("disconnected", function(arg) {
process.exit();
});
consumer.on('event.error', function(err) {
console.error(err);
process.exit(1);
});
consumer.on('event.log', function(log) {
console.log(log);
});
consumer.connect();

A Kafka Consumer can be written with the kafka-node npm module. For my use case, my consumer was a separate Express server which listened to events and stored them in a database.
import kafka from "kafka-node"
const client = new kafka.Client("http://localhost:2181");
const topics = [
{
topic: "webevents.dev"
}
];
const options = {
autoCommit: true,
fetchMaxWaitMs: 1000,
fetchMaxBytes: 1024 * 1024,
encoding: "buffer"
};
const consumer = new kafka.HighLevelConsumer(client, topics, options);
consumer.on("message", function(message) {
// Read string into a buffer.
var buf = new Buffer(message.value, "binary");
var decodedMessage = JSON.parse(buf.toString());
//Events is a Sequelize Model Object.
return Events.create({
id: decodedMessage.id,
type: decodedMessage.type,
userId: decodedMessage.userId,
sessionId: decodedMessage.sessionId,
data: JSON.stringify(decodedMessage.data),
createdAt: new Date()
});
});
consumer.on("error", function(err) {
console.log("error", err);
});
process.on("SIGINT", function() {
consumer.close(true, function() {
process.exit();
});
});
more info in https://nodewebapps.com/2017/11/04/getting-started-with-nodejs-and-kafka/

Mock 'end' signal from emitter, but it doesn't get called

I am doing a unit testing with mocha + unit.js + proxyquire. Right now I am stuck at mocked emitter. First mocked emitter which is a fake on 'row' signal, it done. Second one I followed the same like first one. Since it is not a big difference, but I at here I have a question.
Why emitter is not emit 'end'?
userHandler_post.js :
var pg = require('pg');
var connectionString = require('./postgres.ini');
var pgdown = require('./pgdown.js');
exports.post = function(req,res, callback){
var cb_is_func = (typeof(callback)==='function');
// Grab data from http request
var adata = [req.body.username,
req.body.password,
req.body.privilege,
req.body.firstname,
req.body.middlename,
req.body.lastname,
req.body.streetaddress, //rename
req.body.subdistrict,
req.body.district,
req.body.province,
req.body.country,
req.body.zipcode,
req.body.email,
req.body.phonecountrycode,
req.body.phoneareacode,
req.body.phonenumber, //rename
req.body.faxcountrycode,
req.body.faxareacode,
req.body.faxnumber]; //rename
// Get a Postgres client from the connection pool
pg.connect(connectionString, function(err, client, done) {
// Handle Errors
if(err) {
done();
pgdown.pgdown(res);
if(cb_is_func){callback();}
}else{
// SQL Query > Insert Data
var func_ = 'SELECT Dugong.Users_Add($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19)';
var addUser_ = client.query(func_, adata);
addUser_.on('error', function(error){
var data = {success : false,
username : req.body.username,
reason : {errmsg : 'Unable to add user',
errid : 'addUser_' }};
done();
res.json(data);
if(cb_is_func){callback(data);}
});
addUser_.on('end',function(result){
var data = {success : true, username : req.body.username};
done();
res.json(data);
if(cb_is_func){callback(data);}
});
}
});
pg.end();
};
userHandler_post.test.js :
var httpMocks = require('node-mocks-http');
var test = require('unit.js');
var real_userHandler = require('../routes/userHandler_post.js');
var proxyquire = require('proxyquire'),
pgStub = { };
var events = require('events');
describe('userHandler_post : ', function () {
var request;
beforeEach(function (){
request = httpMocks.createRequest({
method: 'POST',
url: '/users',
body : { Username : 'Yoda',
... //Omit to save question lines.
FaxNum: '',
}
});
});
it('should show connetion error. If cannot connect to database.', function (done) {
pgStub.connect = function (aconnectionString, cb){
cb('Connection refused',null,null);
};
var response = httpMocks.createResponse();
var userHandler_post = proxyquire('../routes/userHandler_post.js', { 'pg' : pgStub} );
userHandler_post.post(request,response); //Let it update response
var answer = response._getData();
test.value(answer).match('Connection refused');
done(); //Need
});
it('should show query error. If database rejects query.', function (done) {
pgStub.connect = function (aconnectionString, cb){
var client = {};
client.query = function(querystr){
var emitter = new events.EventEmitter();
setTimeout(function() {
emitter.emit('error', {detail: 'Query error'})
}, 0);
return emitter;
};
var done = function(){};
cb(null,client,done);
};
var response = httpMocks.createResponse();
var userHandler_post = proxyquire('../routes/userHandler_post.js', { 'pg' : pgStub} );
userHandler_post.post(request, response, function(){
var answer = response._getData();
test.value(answer).match('Query error');
});
done(); //Need
});
it('should show several records. If database processes query.', function (done) {
pgStub.connect = function (aconnectionString, cb){
var client = {};
client.query = function(querystr){
var emitter = new events.EventEmitter();
console.log('below emitter');
setTimeout(function() {
console.log('-*************--');
emitter.emit('end', {a : 1});
}, 0); //
return emitter;
};
var done = function(){};
cb(null, client, done);
};
var response = httpMocks.createResponse();
var userHandler_post = proxyquire('../routes/userHandler_post.js', { 'pg' : pgStub} );
userHandler_post.post(request, response, function(){
var answer = response._getData();
console.log(answer);
// test.value(answer).match('Connection refused');
});
// console.log(response._getData());
done(); //Need
});
});
Terminal :
mocha testing/userHandler_post.test.js
userHandler_post :
✓ should show connetion error. If cannot connect to database.
✓ should show query error. If database rejects query.
below emitter
✓ should show several records. If database processes query.
3 passing (10ms)
If it really emitted the 'end' signal. Then it should say 'Insert record completed'.

I found it.
Upgrade node from v0.10.29 to v0.12.6
npm cache clean -f
npm install -g n
aptitude install curl
n stable