I am trying to implement a logger in node js which will create a new log file every day on a custom format for logs
for this i have used three packages
winston
morgan
winston-daily-rotate-file
so the final output should every day a new log file should create in logs folder and it should log all the http(morgan logs) and typed logs (winston logs) into a single file in the below format
Date || Filename || statusCode || logMessage || uuid (for tracing)
eg: Fri Jan 18 2019 13:48:18 GMT+0530 (IST) || [index.js] || 200 || calling the new route || 287dccb0-1afa-11e9-88a0-dfb1c665be9d
so for this i have written three files index.js(root file of nodejs) logger.js(logger implementation and configuration) and logger.test.js(test cases for logger using jest)
additional packages
cors
uuid
http-context
app-root-path
express-http-context
jest
the problems that i have
if i put a logger.error({message: {statusCode:200, logMsg: "the server will be starting in port 3000"}}) in the index.js on app.listen before to console.log() the uuid is null
the test cases that i have written is wrong, i am new to jest i just want to know how can i check all that cases.
why when i test the suits uuid is null, how can i pass the uuid for test cases also
how can i check whether new folder will be created and if already logs folder are there new file is created kind of test cases.
How can i add other levels , info, debuge , warn based on the env. How can i improve this code to implement the logger functionality
// index.js
const app = require('express')();
const cors = require('cors')
const morgan = require('morgan') // HTTP request logger middleware
const logger = require('./config/logger')(module) //Logger
const uuid = require('uuid')
const httpContext = require('express-http-context')
// Use any third party middleware that does not need access to the context here
// app.use(some3rdParty.middleware);
app.use(httpContext.middleware);
// all code from here on has access to the same context for each request
// Run the context for each request.
// Assigning a unique identifier to each request
app.use((req, res, next) => {
httpContext.set('reqId', uuid.v1());
next()
})
// using morgan with winston(logger)
app.use(morgan('combined', {
stream: {
write: (message) => logger.error(message)
}
}))
app.use(cors());
app.use("/new", (req, res) => {
logger.error({
message: {
statusCode: 400,
logMsg: "hitting new route"
}
})
nextLayer(res)
})
const nextLayer = (res) => {
logger.error({
message: {
statusCode: 400,
logMsg: "hitting in nextLayer function"
}
})
res.send("OK")
}
app.listen(4000, () => {
console.log('Server running on port 4000');
})
// Logger.js
const appRoot = require('app-root-path')
const {
createLogger,
format,
transports
} = require('winston')
const {
combine,
timestamp,
label,
printf
} = format
const path = require('path')
require('winston-daily-rotate-file');
const httpContext = require('express-http-context')
/**
* #method checkMessageProp
* #param {message} can be object if developer defined, else it will be string
* if its a network request
* #returns a fixed format how the status code and message should show
*/
const checkMessageProp = (message) => {
switch (typeof message) {
case "object":
const {
statusCode,
logMsg
} = message
return `${statusCode ? statusCode : "Not Defined"} || ${logMsg ? logMsg : "Not Defined"}`;
case "string":
let messageSplit = message.split(`"`)
var message = messageSplit ? `${messageSplit[2].trim().split(" ")[0]} || ${messageSplit[1]}` : null
return message
default:
return message
}
}
/**
* #method customFormat
* #param {log} the log passed by the developer or based on network requests
* #returns a customFormat how it should be logged to the log files
*/
const customFormat = printf(log => {
const now = new Date();
const reqId = httpContext.get('reqId');
return `${log.timestamp ? new Date(log.timestamp) : now} || [${log.label}] || ${checkMessageProp(log.message)} || ${reqId ? reqId : null}`
});
/**
* #method getFileName
* #param {moduleObj} the module realted object passed from the require of logger file
* #returns the file name where the logger was invoked
*/
const getFileName = moduleObj => {
if (Object.keys(moduleObj).length > 0) {
let parts = moduleObj.filename.split(path.sep)
return parts.pop()
} else {
return "Module not passed while requiring the logger"
}
}
// Custom settings for each transport
const options = moduleObj => {
return {
dailyRotateFile: {
filename: `${appRoot}/logs/TPS-UI-%DATE%.log`,
datePattern: 'YYYY-MM-DD',
prepend: true,
level: "error",
timestamp: new Date(),
localTime: true
}
}
}
// Instantiate a Winston Logger with the settings
let logger = moduleObj => {
return createLogger({
format: combine(
label({
label: getFileName(moduleObj)
}),
customFormat
),
transports: [
new transports.DailyRotateFile(options(moduleObj).dailyRotateFile)
],
exitOnError: false // do not exit on handled exceptions
})
}
module.exports = logger
// logger.test.js
const logger = require('./logger')
beforeEach(() => {
mockLoggerMessageObject = {
message: {
statusCode: 400,
logMsg: "Calling in test suite"
}
}
mockLoggerMessageString = `::ffff:127.0.0.1 - - [18/Jan/2019:04:50:57 +0000]
"GET /new HTTP/1.1" 200 2 "http://localhost/" "Mozilla/5.0
(linux) AppleWebKit/537.36 (KHTML, like Gecko) jsdom/11.12.0"`
mockLoggerMessageNumberFormat = 123
mockLoggerMessageArrayFormat = ["data", "test", 123]
})
describe(`Logger test cases`, () => {
test('should invoke the logger function with the mock Logger message object', () => {
expect(logger(module).error(mockLoggerMessageObject)).toBeDefined()
})
test(`should invoke the logger function with empty object`, () => {
expect(logger(module).error({})).toBeDefined()
})
test(`should invoke the logger function without any module object`, () => {
expect(logger({}).error(mockLoggerMessageObject)).toBeDefined()
})
test(`should invoke the logger function without any module and message object`, () => {
expect(logger({}).error({})).toBeDefined()
})
test(`should invoke the logger function with the http request`, () => {
expect(logger(module).error(mockLoggerMessageString)).toBeDefined()
})
test(`should invoke the logger function with the number format`, () => {
expect(logger(module).error(mockLoggerMessageNumberFormat)).toBeDefined()
})
test(`should invoke the logger function with the array format`, () => {
expect(logger(module).error(mockLoggerMessageArrayFormat)).toBeDefined()
})
})
for winston i'm using timestamp(), like this it will automatically add timestamp() property to the object
const {transports, createLogger, format} = require('winston');
const logger = createLogger({
format: format.combine(
format.timestamp(),
format.json()
),
Also to check if it creates file you can mock date, to say 2019-01-01 and check is it create file 2019-01-01.log
than move date to 2019-01-02 and log something else.
Winston will create new folder and gzip archive and you can check is file exists and can be unzipped and contains information
Try to read winston's documentation.
Basically i would say that you may need to use
format.timestamp()
format.json()
colorize()
dailyRotate with zippedArchive:true
If morgan doesn't suits your needs you can try to log directly in
app.use((req, res, next) => {
logger.silly({ message:'start', req,res});
return next().then(r=>logger.silly({ message:'end', req,res}; return r;);
}
Related
I'm trying to retrieve value from firebase database which is RTDB
it doesnt send any error or output any error as i know of so im really confused as to why.
it worked fine a minute ago.
below is my code to retrieve value from table and send through telegram
import * as functions from "firebase-functions";
import * as express from "express";
import * as cors from "cors";
import * as admin from "firebase-admin";
admin.initializeApp();
const bot = express();
bot.use(cors({origin: true}));
bot.post("/", async function(req, res) {
if (
req.body &&
req.body.message &&
req.body.message.chat &&
req.body.message.chat.id &&
req.body.message.from &&
req.body.message.from.first_name
) {
const chatId = req.body.message.chat.id;
const firstName = req.body.message.from.first_name;
const receivedMessage = req.body.message.text;
// Check if the received message is the "/start" command
if (receivedMessage === "/start") {
// Define your RTDB Reference to point to the "Sensor MQ7" parent node
const rtdbReference = admin.database().ref("Sensor MQ7");
// Read the latest unknown child node of "Sensor MQ7"
const wildcardNodeSnapshot = await rtdbReference
.limitToLast(1)
.once("child_added");
// Read the known child nodes of the unknown child node
const coConcentrationSnapshot = wildcardNodeSnapshot.child("MQ7");
const latSnapshot = wildcardNodeSnapshot.child("latitude");
const longSnapshot = wildcardNodeSnapshot.child("longitude");
const timeSnapshot = wildcardNodeSnapshot.child("time");
// Get the values of the known child nodes
const coConcentration = coConcentrationSnapshot.val();
const time = timeSnapshot.val();
const latitude = latSnapshot.val();
const longtitude = longSnapshot.val();
// Check if the known child node value is above 100
if (coConcentration > 100) {
// Return the response with a button interface
return res.status(200).send({
method: "sendMessage",
chatId,
text: `${firstName} is in danger! Carbon Monoxide Concentration is too high: ${coConcentration}. Last known location is at (${latitude},${longtitude}) approximately ${time} o clock.`,
reply_markup: {
inline_keyboard: [
[
{
text: `Call ${firstName}`,
callback_data: "call_user",
},
],
],
},
});
}
}
} else {
// Return an error message if the request body is missing the required information
return res.status(400).send({
status: "An error occurred. The request body is missing the required information.",
});
}
// Return an empty response
return res.status(200).send({});
});
export const router = functions.https.onRequest(bot);
expected output should be like this: output
its strange because when i do /start it doesnt even send a blank reply
i was supposed to output the database value
Getting metric has already been registered when trying to publish metrics from service. To avoid that I used register.removeSingleMetric("newMetric"); but problem is that it clear register and past records every time a new call comes in.
Wondering what to address this problem:
export function workController(req: Request, res: Response) {
resolveFeaturePromises(featurePromises).then(featureAggregate => {
return rulesService.runWorkingRulesForHook(featureAggregate, hook.name, headers)
.then(([shouldLog, result]) => {
...
// publish metric
publishHookMetrics(result);
// publish metric
sendResponse(res, new CoreResponse(result.isWorking, result.responseData), req, featureAggregate, hook);
});
}).catch(err => {
sendResponse(res, new CoreResponse(Action.ALLOW, null), req, null, hook);
console.error(err);
});
}
function publishCustomMetrics(customInfoObject: CustomInfoObject) {
const counter = new promClient.Counter({
name: "newMetric",
help: "metric for custom detail",
labelNames: ["name", "isWorking"]
});
counter.inc({
name: customInfoObject.hook,
isWorking: customInfoObject.isWorking
});
}
stack trace
[Node] [2020-07-30T17:40:09+0500] [ERROR] Error: A metric with the name newMetric has already been registered.
application.ts
export async function startWebServer(): Promise<Server> {
if (!isGlobalsInitilized) {
throw new Error("Globals are noit initilized. Run initGlobals() first.");
}
// Setup prom express middleware
const metricsMiddleware = promBundle({
includeMethod: true,
includePath: true,
metricsPath: "/prometheus",
promClient: {
collectDefaultMetrics: {
}
}
});
// start http server
const app = require("express")();
app.use(bodyParser.json());
app.use(metricsMiddleware);
const routeConfig = require("./config/route-config");
routeConfig.configure(app);
const port = process.env.PORT || 3000;
return app.listen(port, function () {
console.log("service listening on port", port);
});
}
versions:
express-prom-bundle: 6.0.0
prom-client: 12.0.0
The counter should be initialize only once, then used for each call. Simplest modification from code you gave would be something like this.
const counter = new promClient.Counter({
name: "newMetric",
help: "metric for custom detail",
labelNames: ["name", "isWorking"]
});
export function publishCustomMetrics(customInfoObject: CustomInfoObject) {
counter.inc({
name: customInfoObject.hook,
isWorking: customInfoObject.isWorking
});
}
BTW, I found this thread while searching a way to avoid the "already been registered" error in unit tests. Many tests was instantiating the same class (from a library) that was initializing a counter in the constructor.
As it is in tests, and I didn't need consistency over the metrics, I found an easy solution is to clear metrics registers at the beginning of each test.
import { register } from "prom-client";
// ...
register.clear();
in my nodejs program,i use egg.js framework to create an API server.I am using egg-mysql to crud my database,here are error code i don't understand.And there is no mysql in the declaration of ctx.app
nodejs version: 12.16.1,
npm version: 6.14.3
{"code":"ERR_INVALID_ARG_TYPE","message":"The "string" argument must be of type string or an instance of Buffer or ArrayBuffer. Received undefined","stack":"TypeError [ERR_INVALID_ARG_TYPE]: The \"string\" argument must be of type string or an instance of Buffer or ArrayBuffer. Received undefined\n at Function.byteLength (buffer.js:713:11)\n at respond (/root/workspace/node_modules/koa/lib/application.js:261:25)\n at handleResponse (/root/workspace/node_modules/koa/lib/application.js:164:34)\n at processTicksAndRejections (internal/process/task_queues.js:97:5)","name":"TypeError","status":500}
this is my config files
//config/plugin.js
'use strict';
/** #type Egg.EggPlugin */
exports.mysql = {
enable: true,
package: 'egg-mysql',
};
//config/config.default.js
module.exports = appInfo => {
const config = exports = {}
config.mysql = {
client: {
host: 'localhost',
port: '3306',
user: 'aichechaoshi',
password: 'Aiche123',
database: 'aichechaoshi_db',
},
app: true,
agent: false,
};
// use for cookie sign key, should change to your own and keep security
config.keys = appInfo.name + '_1585128324727_7762';
// add your middleware config here
config.middleware = [];
// add your user config here
const userConfig = {
// myAppName: 'egg',
};
return {
...config,
...userConfig,
};
};
in service file
//app/service/mysql.js
async read() {
const { ctx } = this;
const result = await ctx.app.mysql.select('2019_09_zhouqi_brand');
return result;
}
need help!!!
I'm trying to follow the PACT workshop example with some alternate data.
This may be more of a Javascript/Node question but I'm a but stumped, as a novice.
Given a consumer.spec.js file of:
const chai = require('chai');
const nock = require('nock');
const chaiAsPromised = require('chai-as-promised');
const expect = chai.expect;
const API_PORT = process.env.API_PORT || 9123;
chai.use(chaiAsPromised);
const API_HOST = `http://localhost:${API_PORT}`;
describe('Consumer', () => {
describe('when a call to the Provider is made', () => {
const clothingStatus = 'hello';
const {emailClothingOfferStatus} = require('../client');
it('can process the HTML payload from the provider', () => {
nock(API_HOST)
.get('/provider')
.query({validPermStatus:'hello'})
.reply(200, {
test:'NO',
validPermStatus: clothingStatus,
count: 1000,
});
const response = emailClothingOfferStatus(clothingStatus);
return expect(response.body.clothingStatus).to.eventually.equal('hello')
})
})
});
and a client .js file of:
const request = require('superagent');
const API_HOST = process.env.API_HOST || 'http://localhost';
const API_PORT = process.env.API_PORT || 9123;
const API_ENDPOINT = `${API_HOST}:${API_PORT}`;
// Fetch provider data
const emailClothingOfferStatus = emailPermChoice => {
let withEmailClothing = {};
const emailClothingGrantedRegex = 'hello';
if(emailPermChoice){
console.log(emailPermChoice);
withEmailClothing = {validPermStatus: emailPermChoice}
}
return request
.get(`${API_ENDPOINT}/provider`)
.query(withEmailClothing)
.then(
res => {
if (res.body.validPermStatus.match(emailClothingGrantedRegex)) {
return {
clothingStatus: (res.body.validPermStatus),
}
} else {
throw new Error('Could not verify email clothing offer status')
}
},
err => {
throw new Error(`Error from response: ${err.body}`)
}
)
};
module.exports = {
emailClothingOfferStatus,
};
and I have the following in my package.json scripts:
"test:consumer": "./node_modules/.bin/mocha --timeout 150000 pact/consumer/test/consumer.spec.js",
When I run npm run test:consumer, I get:
1) Consumer
when a call to the Provider is made
can process the HTML payload from the provider:
TypeError: Cannot read property 'clothingStatus' of undefined
at Context.it (pact/consumer/test/consumer.spec.js:29:35)
I'm sure it's something obvious but can anyone help?
Two things stand out to me as a problem:
The test above is a normal unit test designed to show how unit tests won't catch contract issues, and leads you into why Pact is useful (In case this wasn't clear). In short, it's not a Pact test at all - I can tell because it's using Nock, meaning the expected requests will never reach Pact. I can also tell because the Pact package doesn't appear to be imported. You want to model from this file https://github.com/DiUS/pact-workshop-js/blob/master/consumer/test/consumerPact.spec.js
The response value is a Promise, which means you can't do return expect(response.body.clothingStatus).to.eventually.equal('hello') because response is a promise, so body will be undefined and clothingStatus is not a property of that. The chai eventually API is useful for this sort of test, but as I understand, it has to work directly with a Promise - you could do expect(response).to... and then chai can go to work.
Your function emailClothingOfferStatus returns response.then() which is a promise and not an actual response.
Therefore response.body is undefined.
You should be able to test the result like this:
const response = emailClothingOfferStatus(clothingStatus);
response.then((res) => {
expect(res.body.clothingStatus).to.eventually.equal('hello')
})
I am experimenting with push-notifications sent from a Node.js app.
Following some tutorial and examples, I now have a working mini-application to start with.
What it does is very basic, when it is loaded into the browser, a notification is fired and the user sees a message popping up.
It is basically composed of four files:
index.js, index.html, worker.js and client.js.
As a first experiment, I would like to implement some slightly more sophisticated behavior.
The app should fire a notification of type A when it starts (as it is already doing) and then fire a notification of type B every 121 minutes.
Is this kind of thing possible or just impossible?
If it is possible, how can I do it?
For reference I put here the two relevant files:
index.js:
const express = require('express'),
webPush = require('web-push'),
bodyParser = require('body-parser'),
path = require('path');
const app = express();
app.use(express.static(path.join(__dirname, 'client')));
app.use(bodyParser.json());
const privateVapIdKey = process.env.privVapIdKey,
publicVapIdKey = process.env.pubVapIdKey;
webPush.setVapidDetails(
'mailto:myemail#example.com',
publicVapIdKey,privateVapIdKey);
// Subscribe Route.
app.post('/subscribe',(req,res) => {
const subscription = req.body; // Get Push Subscription Object.
res.status(201).json({}); // Send 201. Resource created.
// Do a lot of useful things ......
.......
// Create the PayLoad.
const payload = JSON.stringify({
title:'A big title!',
........
});
// Pass Object to sendNotification.
webPush.sendNotification(subscription,payload).catch(err => console.error(err));
});
const port = 5003;
const PORT = process.env.PORT || port;
app.listen(PORT, () => console.log(`Listening on ${ PORT }`));
client.js:
const publicVapIdKey = 'my-secret-3453754...pubVapIdKey';
// Chec for ServiceWorker.
if ('serviceWorker' in navigator) {
send().catch(err => console.error(err));
}
// Register ServiceWorker, Register Push, Send Push.
async function send() {
console.log("Registering ServiceWorker.");
const register = await navigator.serviceWorker.register('/worker.js', {
scope: "/"
});
console.log('ServiceWorker registered.');
console.log("Registering Push.");
//register.pushManager.uns
const subscription = await register.pushManager.subscribe({
userVisibleOnly: true,
applicationServerKey: urlBase64ToUint8Array(publicVapIdKey)
});
console.log('Push registered.');
console.log("Sending Push.");
await fetch('/subscribe', {
method: 'POST',
body: JSON.stringify(subscription),
headers: {
'content-type': 'application/json'
}
});
console.log('Push sent.');
}
function urlBase64ToUint8Array(base64String) {
const padding = '='.repeat((4 - base64String.length % 4) % 4);
const base64 = (base64String + padding)
.replace(/\-/g, '+')
.replace(/_/g, '/');
const rawData = window.atob(base64);
const outputArray = new Uint8Array(rawData.length);
for (let i = 0; i < rawData.length; ++i) {
outputArray[i] = rawData.charCodeAt(i);
}
return outputArray;
}
Simple event timing can be achieved with setTimeout and setInterval: https://nodejs.org/api/timers.html
They work the same as in browser side JavaScript.
For something more advanced, try node-cron
To be able to send notifications to an endpoint, you need its subscription, so we have to store the clients subscriptions:
const subscriptions = [];
app.post('/subscribe',(req,res) => {
const subscription = req.body;
// FIXME: Validate subscription!!!
subscriptions.push(subscription);
res.status(201).json({});
webPush.sendNotification(subscription, { title: "A" });
});
Now every 121 minutes, we go over all subscriptions and deliver our message B:
const MINS = 60 * 1000;
setInterval(() => {
for(const subscription of subscriptions) {
webPush.sendNotification(subscription, {title: "B" });
}
}, 121 * MINS);
PS: You should probably also add an "unsubscribe" endpoint, as you would otherwise deliver notifications to dead endpoints somewhen
Its possible! but i would suggest you use Admin FCM for server side its more latest library than web-push and its way easier to push notification.
//node.js serverside code
const FCM = require("firebase-admin");
//fcm-push-notification.json is where all the configurations are
const serviceAccount = require("fcm-push-notification.json");
FCM.initializeApp({
credential: SERVICE_ACCOUNT,
databaseURL: DBURL
});
// In your Subscribe Route.
app.post('/subscribe',(req,res) => {
FCM.messaging()
.sendToDevice(
DEVICE_TOKEN,
{
data: DATA,
notification: {
title: "A big title!",
body: "HELLO PUSH!"
}
}
)
.then(res => {
// do something here
})
});
here is the service worker
// put firebase-messaging-sw.js service worker
// this is to get notified in the background when the tab is closed on not active
(global => {
importScripts("https://www.gstatic.com/firebasejs/4.8.1/firebase-app.js");
importScripts(
"https://www.gstatic.com/firebasejs/4.8.1/firebase-messaging.js"
);
firebase.initializeApp({
messagingSenderId: SENDER_ID
});
const messaging = firebase.messaging();
console.log("Service Worker started!");
messaging.setBackgroundMessageHandler(payload => {
console.log("Message received In background");
// Customize notification here
const notificationOptions = {
body: "Background Message body.",
icon: "/firebase-logo.png"
};
return global.registration.showNotification("TITLE", notificationOptions);
});
})(self);
in your javascript
//to get notified in forground just do this
import firebase from "firebase";
firebase.initializeApp(FCM_CONF);
let messaging = firebase.messaging();
messaging.usePublicVapidKey(VAPID_KEY);
messaging.onMessage(payload => {
console.log("Message received from foreground ");
});
finally create a manifest.json
//create manifest.json with content like this
{
"gcm_sender_id": SENDER_ID
}
and to fire a notification of type B every 121 minutes. use something like later.js
var later = require('later');
var schedule = later.parse.text('every 121 min');
var timer = later.setTimeout(() => {
// fired every 121 minutes
FCM.messaging()
.sendToDevice(
DEVICE_TOKEN,
{
data: DATA,
notification: {
title: "A big title!",
body: "HELLO PUSH!"
}
}
)
}, schedule);