Develop Reference

Winston logger setting the timestamp of the log to 12/31/1969 instead of current time

I'm setting up logging in an app with winston and occassionally when I run tests a separate file is created with the date 12/31/1969. Is there something explicit I need to put in the creation of the transport so that it knows what the current date is?
What's very interesting is this seems to be a system wide anomaly as the _log method, which doesn't use the new Date() syntax, but the moment.js library also results in a 12-31-1969 inside the log file:
My logger:
class Logger{
constructor(configs){
if (!configs) configs = {};
this.logDirectory = configs.directory ? path.join(__dirname, configs.directory) : path.join(__dirname, '../logs') ;
this.initialize();
this.date = moment().format("YYYY-MM-DD HH:mm:ss");
}
initialize() {
this._createTransportObj();
this._createLoggerObj();
}
_createTransportObj() {
const DailyRotateFile = winston.transports.DailyRotateFile;
this._transport = new DailyRotateFile({
filename: path.join(this.logDirectory, '/log-%DATE%.log'),
datePattern: 'YYYY-MM-DD',
level: 'info'
});
}
_createLoggerObj() {
this._logger = winston.createLogger({
level: 'info',
format: winston.format.json(),
transports: [this._transport],
exitOnError: true
});
if (nodeEnv !== 'production') {
this._logger.add(new winston.transports.Console({
format: winston.format.simple()
}));
}
}
_log(type, msg, options) {
const logMsg = {};
const timestamp = moment().format("YYYY-MM-DD HH:mm:ss");
logMsg.level = type || 'info';
logMsg.time = timestamp;
logMsg.msg = msg || '';
logMsg.desc = options.description || '';
// get the user that made the request if available
if (options.user) logMsg.user = options.user;
// get the url endpoint that was hit if available
if (options.url) logMsg.url = options.url;
// if an error is sent through, get the stack
// and remove it from msg for readability
if (msg.stack) {
logMsg.stack = msg.stack;
msg = msg.message ? msg.message : msg;
}
// get the ip address of the caller if available
if (options.ip) logMsg.ip = options.ip;
// get the body of the request if available
if (options.body) logMsg.body = options.body;
// get the query string of the request if available
if (options.query) logMsg.query = options.query;
// get the params string of the request if available
if (options.params) logMsg.params = options.params;
const jsonString = JSON.stringify(logMsg);
this._logger.log(type, logMsg);
}
info(msg, options) {
return this._log('info', msg, options);
}
error(msg, options) {
return this._log('error', msg, options);
}
warn(msg, options) {
return this._log('warn', msg, options);
}
verbose(msg, options) {
return this._log('verbose', msg, options);
}
debug(msg, options) {
return this._log('debug', msg, options);
}
silly(msg, options) {
return this._log('silly', msg, options);
}
}
module.exports = { Logger };
I'm currently only testing it in a promise handler that my routes flow through:
const asyncHandler = fn => (req, res, next) => {
const logger = new Logger();
Promise.resolve(fn(req, res, next))
.then(result => {
if (req.body.password) delete req.body.password;
logger.info(result,
{ user: req.user.username,
url: req.originalUrl,
body: req.body,
description: '200:OK Response sent back successfully'
});
return res.status(200).json({ result })
})
.catch(e => {
console.log(e);
return res.status(400).json({ error: e.message })
});
};
module.exports = asyncHandler;
UPDATE*
ok, so it seems to not be the logger itself. I ran a batch of tests and noticed it's always the same route that triggers the date change. What's weird is I can't seem to figure out what's happening.
The route is:
and the app.use() statement is as follows:
finally the admin_access middleware is simple enought:
I've figured out if I break the endpoint in the app.js file before it hits admin_access the date is correct. However if I break in admin_access the date is 12-31-1969. So what could be happening between the two? Is there something I could be setting unintentionally on this route?

Figured it out. Turns out the package sinon-test was changing the system time.
I had my test setup like this:
const sinon = require('sinon');
const sinonTest = require('sinon-test');
const test = sinonTest(sinon);
describe('Test suite for route: /video/admin/create', ()=>{
let agent;
beforeEach(async function() {
// create temporary admin
admin.permissionId = 1;
await new NewUser().createUser(admin);
//login as admin
agent = chai.request.agent(server);
await agent
.post('/auth')
.send({ username: admin.username, password: admin.password });
});
afterEach(async function() {
// destroy temp admin
await new UserManager(admin).hardRemove();
});
it('should fail to create a video for not including video info', test(async function(){
const newVideo = { title: 'Test Video' };
const response = await agent.post('/video/admin/create')
.send(newVideo);
expect(response.status).to.equal(400);
}));
it('should create a video', test(async function(){
const newVideo = {
title: 'Test Video',
description: 'This is a description',
embed: 'https://youtu.be/SKbHjjZXdmc',
source: 'youtube',
type: 'lecture',
category: 'physics'
};
const response = await agent.post('/video/admin/create')
.send(newVideo);
// validations
expect(response.status).to.equal(200);
const { result } = response.body;
expect(result.title).to.equal(newVideo.title);
expect(result.description).to.equal(newVideo.description);
expect(result.embed).to.equal(newVideo.embed);
}));
});
When I unwrapped the test from the test() function everything worked correctly.

CRON Job not working on meteor

In main.js in server folder,
I have this code,
var DDP = require('ddp');
var DDPlogin = require('ddp-login');
var Job = require('meteor-job');
var ddp = new DDP({
host: "127.0.0.1",
port: 3000,
use_ejson: true
});
Meteor.startup(() => {
var myJobs = JobCollection('myJobQueue');
Job.setDDP(ddp);
ddp.connect(function (err) {
if(err) throw err;
DDPlogin(ddp, function (err, token) {
if (err) throw err;
});
});
myJobs.allow({
admin: function (userId, method, params) {
return true;
}
});
Meteor.publish('allJobs', function () {
return myJobs.find({});
});
myJobs.startJobServer();
var workers = Job.processJobs('myJobQueue', 'sendEmail',
function (job, cb) {
console.log(job.data.text);
job.done();
cb(null);
}
);
And in my main.js in client folder,
I have this code,
var myJobs = JobCollection('myJobQueue');
var jobSub = null;
class App extends Component {
componentDidMount(){
if(jobSub !== null)
jobSub.stop();
jobSub = Meteor.subscribe('allJobs');
var job = new Job(myJobs, 'sendEmail',
{
text: 'bozo#clowns.com'
}
);
job.priority('normal')
.retry({ retries: 5,
wait: 60*1000 }) // 1 minute between attempts
.delay(0) // start immediately
.save();
}
...
render(){
console.log(myJobs.find().fetch());
...
}
}
I am using the vsivsi:meteor-job-collection package.
The problem is that console.log() is not executed.
What is wrong in my step by step installation and usage?
I need to console.log() every minute.

How to create consumer for kafka topic?

I have topic created on kafka server now i am creating consumer to read topic messages from server, but i dont see any data when i try to consume message using consumer.on('message') , Any idea what is implemented wrong in below code , Do i need to set offset ?
consumer.js
var kafka = require('kafka-node');
var config = require('./config.js');
var zk = require('node-zookeeper-client');
var kafkaConn = config.kafkaCon.dit;
var HighLevelConsumer = kafka.HighLevelConsumer;
var Client = kafka.Client;
function start() {
topics = [{
topic: 'test-1'
}];
var groupId = 'push';
var clientId = "consumer-" + Math.floor(Math.random() * 10000);
var options = {
autoCommit: true,
fetchMaxWaitMs: 100,
fetchMaxBytes: 10 * 1024 * 1024,
groupId: groupId
};
console.log("Started consumer: ", clientId);
var consumer_client = new kafka.Client(kafkaConn, clientId);
var client = new Client(consumer_client.connectionString, clientId);
var consumer = new HighLevelConsumer(client, topics, options);
consumer.on('message', function(message) {
var topic = message.topic;
console.log('Message', topic);
});
};
start();

const Kafka = require("node-rdkafka");
const kafkaConf = {
"group.id": "cloudkarafka-example",
"metadata.broker.list": ["localhost:9092"],
"socket.keepalive.enable": true,
//"security.protocol": "SASL_SSL",
//"sasl.mechanisms": "SCRAM-SHA-256",
//"sasl.username": process.env.CLOUDKARAFKA_USERNAME,
//"sasl.password": process.env.CLOUDKARAFKA_PASSWORD,
"debug": "generic,broker,security",
'enable.auto.commit': false,
};
//const prefix = process.env.CLOUDKARAFKA_USERNAME;
const topics = ['topicName'];
const consumer = new Kafka.KafkaConsumer(kafkaConf, {
"auto.offset.reset": "beginning"
});
consumer.on("error", function(err) {
console.error(err);
});
consumer.on("ready", function(arg) {
console.log(`Consumer ${arg.name} ready`);
consumer.subscribe(topics);
consumer.consume();
});
consumer.on("data", function(m) {
console.log(m.value.toString());
});
consumer.on("disconnected", function(arg) {
process.exit();
});
consumer.on('event.error', function(err) {
console.error(err);
process.exit(1);
});
consumer.on('event.log', function(log) {
console.log(log);
});
consumer.connect();

A Kafka Consumer can be written with the kafka-node npm module. For my use case, my consumer was a separate Express server which listened to events and stored them in a database.
import kafka from "kafka-node"
const client = new kafka.Client("http://localhost:2181");
const topics = [
{
topic: "webevents.dev"
}
];
const options = {
autoCommit: true,
fetchMaxWaitMs: 1000,
fetchMaxBytes: 1024 * 1024,
encoding: "buffer"
};
const consumer = new kafka.HighLevelConsumer(client, topics, options);
consumer.on("message", function(message) {
// Read string into a buffer.
var buf = new Buffer(message.value, "binary");
var decodedMessage = JSON.parse(buf.toString());
//Events is a Sequelize Model Object.
return Events.create({
id: decodedMessage.id,
type: decodedMessage.type,
userId: decodedMessage.userId,
sessionId: decodedMessage.sessionId,
data: JSON.stringify(decodedMessage.data),
createdAt: new Date()
});
});
consumer.on("error", function(err) {
console.log("error", err);
});
process.on("SIGINT", function() {
consumer.close(true, function() {
process.exit();
});
});
more info in https://nodewebapps.com/2017/11/04/getting-started-with-nodejs-and-kafka/

Create Parse.com Roles Programmatically

I'm looking to create Parse.com roles programmatically because from what I've gathered you can not create parent/child relationships or roles referencing roles in the data browser. I currently do not have any roles in Parse. When I run the following code I get 400 (Bad Request) in my JS console.
$scope.activateRoles = function(){
console.log("activating vendor");
Parse.Cloud.run("activateVendor ", {
success: function (IDK) {
alert("The user roles were activated");
},
error: function (error) {
alert("The user roles were not activated.", error);
}
});
}()
Parse.Cloud.define("createRoles", function(request, response){
Parse.Cloud.useMasterKey();
var admin = new Parse.ACL();
admin.setPublicReadAccess(false);
admin.setPublicWriteAccess(false);
admin.getReadAccess("Uh792HOaqi");
admin.getWriteAccess("Uh792HOaqi");
admin.setReadAccess("Uh792HOaqi", true);
admin.setWriteAccess("Uh792HOaqi", true);
var adminRole = new Parse.Role("admin", admin);
adminRole.save();
var agentRole = new Parse.Role("agent", admin);
agentRole.getRoles().add(adminRole);
agentRole.save();
var vendorRole = new Parse.Role("vendor", admin);
vendorRole.getRoles().add(adminRole);
vendorRole.save();
})

Got to make sure to call success/error. You also want to watch out for making sure to add users to a role that's already been saved. You also can only add roles to a role if the role your adding has already been saved.
Here's what will work:
Parse.Cloud.define("createRoles", function(request, response){
Parse.Cloud.useMasterKey();
var admin = new Parse.ACL();
var adminRole = new Parse.Role("admin", admin);
adminRole.save().then(function(adminRoleSaved){
var promises = [];
var agentRole = new Parse.Role("agent", admin);
agentRole.getRoles().add(adminRoleSaved);
promises.push(agentRole.save());
var me = new Parse.Query(Parse.User);
me.get(request.params.id, {
success: function(me) {
adminRoleSaved.getUsers().add(me);
adminRoleSaved.save();
},
error: function(error){
}
});
var vendorRole = new Parse.Role("business", admin);
vendorRole.getRoles().add(adminRoleSaved);
promises.push(vendorRole.save());
Parse.Promise.when(promises).then(function() {
response.success();
}, function(error) {
response.error(error);
});
});
});

cloudcode : add child role to parent ... sample working code but maybe not your best implementation... its parse/backbone/marionette impl.
Parse.Cloud.define("addrole", function(request, response) {
var roleParent = request.params.parentName;
var roleChild = request.params.childName.trim();
var _role , _chrole;
var _error = {
};
var _errorch = {
};
var _errorparm = {
};
var cst = {
parent:"parent",
pid:"pid",
child:"child"
};
if(roleParent == roleChild)response.error(_errorparm);
var cst1 = "parnt";
var qp = new Parse.Query(Parse.Role);
qp.equalTo("name", roleParent);
var qc = new Parse.Query(Parse.Role);
qc.equalTo("name", roleChild);
Parse.Cloud.useMasterKey();
qp.first().then(function(role) {
_role = role;
if (typeof _role === "undefined") {
return Parse.Promise.error(_error);
} else {
cst.parent = _role.get("name");
cst.pid = _role.get("objectId");
return qc.first();
};
}
).then(function(rolechld) {
_chrole = rolechld;
if (typeof _chrole === "undefined") {
return Parse.Promise.error(_errorch);
} else {
cst.child = _chrole.get("name");
console.log(cst);
_role.getACL().setRoleReadAccess(_chrole, true);
_role.getRoles().add(_chrole);
return _role.save();
}
}).then(function(hello) {
response.success(_chrole.toJSON());
}, function(error) {
response.error(error);
});
});

NodeJS memory usage

I am playing with NodeJS and for this purpose created an email extractor. Somehow when i create multiple http requests the node.exe memory useage in windows task manager keeps increasing. I understand that the node needs more memory to process the requests but what i noticed that this memory usage does not come down even after all requests have been successfully processed.
When i start nodejs it consumes about 35000K memory but after about 80-100 request this goes upto 50000K and stays.
Here is my simple email extractor module:
var request = require('request'),
cheerio = require('cheerio'),
async = require('async'),
urlHelper = require('url');
function Extractor(config) {
this.baseUrl = config.url;
this.parsedUrl = urlHelper.parse(config.url);
this.urls = [];
this.emails = [];
}
Extractor.prototype.getEmails = function getEmails(html) {
var foundEmails = html.match(/([a-zA-Z0-9._-]+#[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+)/gi) || [];
if(foundEmails.length) this.emails = this.emails.concat(foundEmails);
}
Extractor.prototype.extract = function extract(html) {
var $ = cheerio.load(html),
that = this;
if($('body')){
this.getEmails($('body').html());
}
if(!this.emails.length){
$("a[href^='http://" + this.parsedUrl.host + "'], a[href^='https://" + this.parsedUrl.host + "'], a[href^='/'], a[href^='./'], a[href^='../']").each(function(k, v) {
that.urls.push(urlHelper.resolve(that.baseUrl, $(v).attr('href')));
});
}
};
/**
* Process the base URL
*/
Extractor.prototype.processBase = function processBase(next) {
request(this.baseUrl, function(err, response, body) {
return next(err, body);
});
}
/**
* Process the internal pages
*/
Extractor.prototype.processInternal = function processInternal(cb) {
var that = this;
async.whilst(
// while this condition returns true
function () { return that.emails.length === 0 && that.urls.length > 0; },
// do this
function (callback) {
request(that.urls.shift(), function (err, response, body) {
var $ = cheerio.load(body);
if($(body)){
that.getEmails($('body').html());
}
callback(); // async internal, needs to be called after we are done with our thing
});
},
// call this if any errors occur. An error also stops the series
// this is also called on successful completion of the series
function (err) {
cb(that);
}
);
}
Extractor.prototype.process = function process(next) {
var that = this;
this.processBase(function(err, html) {
if(err) {
console.log(err);
} else {
that.extract(html);
if(!that.emails.length) {
that.processInternal(function(res) {
return next(null, that);
});
}
}
});
}
module.exports = Extractor;
and here is how i call it:
var express = require('express');
var router = express.Router();
var Extractor = require('../services/Extractor');
router.get('/', function(req, res) {
res.json({msg: 'okay'});
var extractor = new Extractor({url: 'http://lior-197784.use1-2.nitrousbox.com:4000/crawl'});
extractor.process(function(err, res) {});
});
module.exports = router;