I want to combine kafka-node and node tail to watch some log files and send any new entry to a kafka cluster. I have the following code which works.
var info={
"zookeepers":"127.0.0.1:9500",
"topics":{
"server":[
"/tmp/server-1.log",
"/tmp/server-2.log"
],
"audit":[
"/tmp/audit-1.log",
"/tmp/audit-2.log"
]
}
}
function dldmProducer (logFiletoTail, fileTopic, zookeepers){
var kafka = require('kafka-node');
var Producer = kafka.Producer;
var client = new kafka.Client(zookeepers);
var producer = new Producer(client);
var partitionNo = 0
producer.on('ready', function () {
var Tail = require('tail').Tail;
var tail = new Tail(logFiletoTail);
tail.on("line", function(data) {
dataArray=data.split("\n")
var payloads = [{ topic: fileTopic, messages: dataArray, partition: partitionNo },];
partitionNo = (partitionNo +1)%3;
producer.send(payloads, function(err, data){
//console.log(payloads);
//console.log(data);
});
});
tail.on("error", function(error) {
//console.log('TAIL ERROR: ', error);
});
});
producer.on('error', function(error){
//console.log('PRODUCER ERROR: ', error)
});
}
for (var topicItem in info['topics']){
for (var i = 0; i<info['topics'][topicItem].length; i++){
dldmProducer(info['topics'][topicItem][i],topicItem,info['zookeepers'])
}
}
But then the problem is, error handling is non-existent, so i'm worried that if things go wrong during producer.send(payloads,function(err,data){}), the payloads would just be lost forever. I rewrote the function dldmProducer to the following trying to catch any unsuccessfully sent payloads and resend them. (if the payloads are successfully sent, the response would be in data. data is undefined otherwise, or at least that's my understanding)
function dldmProducer (logFiletoTail, fileTopic, zookeepers){
var kafka = require('kafka-node');
var Producer = kafka.Producer;
var client = new kafka.Client(zookeepers);
var producer = new Producer(client);
var partitionNo = 0
var buffer = []
var Tail = require('tail').Tail;
var tail = new Tail(logFiletoTail);
tail.on("line", function(data) {
var dataArray=data.split("\n")
console.log(dataArray)
buffer = buffer.concat(dataArray)
});
tail.on("error", function(error) {
console.log('TAIL ERROR: ', error);
});
producer.on('ready', function () {
while (true){
var logItem = buffer.shift()
if (logItem){
console.log(logItem)
var payloads = [{ topic: fileTopic, messages: [logItem], partition: partitionNo },];
partitionNo = (partitionNo +1)%3;
producer.send(payloads, function(error, data){
if (!data){
console.log(data)
buffer.push(logItem)
}
if (error){
console.log('KAFKA RESPONSE ERROR: ', error)
}
});
}
}
});
producer.on('error', function(error){
console.log('PRODUCER ERROR: ', error)
});
}
The function doesn't seem to work. It doesn't even tail on the files correctly and hardly any information from the file is logged on screen even though i'm constantly writing dummy contents into the files. I'm still not used to javascript kind of thinking so any pointer would be great.
Related
I am trying to send a soap request with an attachment. Everything works fine except that the attachment i send is always of zero bytes. The soap server accepts a Base64 encoded file and i had achieved to do it in Java using the code
OutputStream outputStream = new ByteArrayOutputStream()
outputStream.writeTo(fileOutputStream);
Base64.encode(outputStream.toByteArray())//argument passed to the function which sends this to the SOAP API
I want to replicate the same with node but i am unable to do so. Below is the function i am using to achieve this. I am reading some files from the client and trying to send it to the SOAP API. I have marked the place in the code responsible to read and append the data the rest is just for reference.
function createSoapEntryWithAtt(req,response){
var form = new formidable.IncomingForm();
form.parse(req, function (err, fields, files) {
let filesArr = []
for(objkeys in files){
filesArr.push(files[objkeys])
}
return Promise.all(filesArr.map(item => {
return new Promise((res,rej) => {
var oldpath = item.path;
var newpath = 'C:/user/' + item.name;
**var data = fs.readFileSync(oldpath).toString('base64');
let result = []
for (var i = 0; i < data.length; i += 2)// trying to create a 64bit byte array
result.push('0x' + data[i] + '' + data[i + 1])**
console.log(result)
if(data)
res({ [`${item.name}`]: result })
rej("Error occured")
})
})).then(data => {
let url = config.url
var credentials = {
AuthenticationInfo: {
userName: "user",
password: "passwd"
}
}
let args = {
Notes: "Testing From Node App",
}
let count = 0
for (index in data) {
if (count <= 3) {
**for(keys in data[index]){
//console.log(data[index][keys])
args[`Attachment${++count}_Name`] = keys
args[`Attachment${++count}_Data`] = data[index][keys]//Attaching the file read
}
}**
}
soap.createClient(url, function (err, client) {
client.addSoapHeader(credentials)
client.CreateWorkInfo(args, function (err, res) {
if (err) {
console.log("Error is ----->" + err)
} else {
console.log("Response is -----> " + res)
response.end();
}
})
})
})
});
}
Please ignore this question .... and thanks and sorry if anyone wasted time on this question. The error was a careless mistake from my side in the line args["Attachment${++count}_Name"] = keys
args["Attachment${++count}_Data"] = data[index][keys]. Here as i am incrementing the count in both lines there is a mismatch in the sense that Attachment name will be 1 and then in the second line Attachment data will be 02 and hence the name does not contain any data.
I am trying to run this program in Raspberry Pi 3.
I have installed nodejs and ws on my raspberry pi.
Then I installed serial port module.
I am trying to create this project:enter link description here
I have tried to find solutions everywhere but I could not find one.
If any one knows how to solve this problem please help me.
var webSocketUrl = "wss://api.artik.cloud/v1.1/websocket?ack=true";
var device_id = "5bb3ba9304674086bee67fa507a215cf"; //DEVICE ID
var device_token = "36b278345b6d4d11abf764ae213c5c70"; //DEVICE TOKEN
var WebSocket = require('ws');
var isWebSocketReady = false;
var data="";
var ws = null;
var serialport = require("serialport");
var SerialPort = serialport.SerialPort;
var sp = new SerialPort("/dev/ttyACM0", { //for serial communication with arduino
baudrate: 9600,
// The baud rate of uno is 9600
parser: serialport.parsers.readline("\n")
});
/**
* Gets the current time in millis
*/
function getTimeMillis(){
return parseInt(Date.now().toString());
}
/**
* Create a /websocket connection and setup GPIO pin
*/
function start() {
//Create the WebSocket connection
isWebSocketReady = false;
ws = new WebSocket(webSocketUrl);
ws.on('open', function() {
console.log("WebSocket connection is open ....");
register();
});
ws.on('message', function(data) {
//this loop is called whenever the client sends some message
handleRcvMsg(data); //data is send to the function handleRcvMsg()
});
ws.on('close', function() {
console.log("WebSocket connection is closed ....");
});
}
/**
* Sends a register message to /websocket endpoint
*/
//Client will only work when device gets registered from here
function register(){
console.log("Registering device on the WebSocket connection");
try{
var registerMessage = '{"type":"register", "sdid":"'+device_id+'", "Authorization":"bearer '+device_token+'", "cid":"'+getTimeMillis()+'"}';
console.log('Sending register message ' + registerMessage + '\n');
ws.send(registerMessage, {mask: true});
isWebSocketReady = true;
}
catch (e) {
console.error('Failed to register messages. Error in registering message: ' + e.toString());
}
}
//data after receiving is sent here for processing
function handleRcvMsg(msg){
var msgObj = JSON.parse(msg);
if (msgObj.type != "action") return; //Early return;
var actions = msgObj.data.actions;
var actionName = actions[0].name; //assume that there is only one action in actions
console.log("The received action is " + actionName);
}
/**
* Send one message to ARTIK Cloud
*/
//This function is responsible for sending commands to cloud
//function sendStateToArtikCloud(parking,temperature,water){
function sendDataToArtikCloud(pantry){
var result=pantry.split(" ");//data gets split by " " to get the values
try{
ts = ', "ts": '+getTimeMillis();
var data = {
"Garlic": result[1],
"Potato":result[2],
"Temperature":result[3],
"Chilli":result[4],
"Humidity": result[5],
"Ginger":result[6],
"Onion": result[7]
};
var payload = '{"sdid":"'+device_id+'"'+ts+', "data": '+JSON.stringify(data)+', "cid":"'+getTimeMillis()+'"}';
console.log('Sending payload ' + payload + '\n');
ws.send(payload, {mask: true});
} catch (e) {
console.error('Error in sending a message: ' + e.toString() +'\n');
}
}
function exitClosePins() {
console.log('Exit and destroy all pins!');
process.exit();
}
start();
//exectes every second when data is received from arduino (5sec programmed delay from arduino)
sp.on("open", function () {
sp.on('data', function(data) {
console.log("Serial port received data:" + data);
//var result=data.split(" ");//data gets split by " " to get the values
//sendStateToArtikCloud(result[0],result[2],result[1]);//parking,temperature,waterlevel
sendDataToArtikCloud(data);
});
});
process.on('SIGINT', exitClosePins);
I am getting an error on my raspberry pi
enter image description here
Suggest me a solution.
The documentation will tell you that Readline is spelled with a capital R
https://www.npmjs.com/package/serialport#module_serialport--SerialPort.parsers
parser: serialport.parsers.Readline("\n")
~
[TypeError: serialport.parsers.readline is not a function.]
If it has not been resolved yet, try this method.
var serialport = require("serialport")
var SerialPort = serialport.SerialPort;
var sp = new serialport("/dev/ttyACM0"),{
BaudRate: 9600,
parser: new serialport.parsers.Readline("\r\n")
});
I hope your problem is solved.
var client = require('http');
var endpoint = apiEndpoint;
var request = client.get(endpoint, function(responseFromApi) {
var responseString = '';
responseFromApi.setEncoding('utf-8');
responseFromApi.on('data', function(data) {
responseString += data;
});
// To reformat the string response into Json...
responseFromApi.on('end', function() {
var jsonResponse = JSON.parse(responseString);
callback(jsonResponse);
});
});
I am making API calls using the method above, however on random instances my call fails due to the Assertion fail like below. Anyone has any idea how to fix this?
Assertion failed: (handle->type == UV_TCP || handle->type == UV_TTY || handle->type == UV_NAMED_PIPE), function uv___stream_fd, file ../deps/uv/src/unix/stream.c, line 1568.
Environment: Mac, Nodejs
Note: I have tested the same code on an AWS lambda server and never faced this issue. I am guessing this is a Mac only instance. Lord Google informed me that it is a Mac desync issue.
Same is true if trying to get data from a dynamoDB sitting on Amazon server using the code below...
// To get userID.
var userId = getUserIdFromContext(this);
if (!userId) {
callback('userId is not set.');
}
// To get table name.
var table = constants.dynamoDBTableName;
if(!table) {
callback('DynamoDB Table name is not set.');
}
// To set the DocumentClient.
if(!doc) {
doc = new aws.DynamoDB.DocumentClient({apiVersion: '2012-08-10'});
}
// To set the params.
var params = {
Key: {
CustomerId: userId
},
TableName: table,
ConsistentRead: true
};
// To get data from table.
var skillContext = this;
doc.get(params, function(err, data){
if(err) {
console.log('get error: ' + JSON.stringify(err, null, 4));
callback(err);
} else {
if(isEmptyObject(data)) {
callback('The object is empty.');
} else {
var userData = JSON.parse(data.Item['Data']);
extractUserData(skillContext, userData, callback);
}
}
});
}
fist of all im not shure if the following is a non-blocking problem?
im getting started with https://github.com/sahat/hackathon-starter
currently i try to read all files out of a folder and later process all files...
i used EventEmitter to kind of manage the workflow.
i want to clear all arrays if the URL is refeshed or loaded new, but somehow if i reaload the URL there seems to be something inside the arrays which cases multiple outputs of the same data?
at the moment i just would be happy to have a correct console.log output.
/**
* GET /
* Home page.
*/
var fs = require('fs');
//XML
var jsxml = require("node-jsxml");
var Namespace = jsxml.Namespace,
QName = jsxml.QName,
XML = jsxml.XML,
XMLList = jsxml.XMLList;
//EventEmitter
var EventEmitter=require('events').EventEmitter;
var dateinamenEE=new EventEmitter();
var dateiinhaltEE=new EventEmitter();
var dateinamen = [];
var dateiinhalt = [];
exports.index = function(req, res) {
fs.readdir('./data', function (err, files) {
if (!err) {
files.forEach(function(value) {
dateinamen.push(value);
});
dateinamenEE.emit('dateinamen_ready');
} else {
throw err;
}
});
dateinamenEE.on('dateinamen_ready',function(){
dateinamen.forEach(function(value) {
var buf = fs.readFileSync('./data/'+value, "utf8");
var xml = new XML(buf);
var list = xml.descendants("suggestion");
var ergebnis = "";
var basiswort = "";
var buchstabe = "";
var obj = null;
list.each(function(item, index){
ergebnis = item.attribute('data').toString()
//basiswort = value.replace("%2B", " ");
//basiswort = basiswort.replace(".xml", "");
//var pieces = buchstabe.split(" ");
obj = {k: basiswort, b: buchstabe, e: ergebnis};
dateiinhalt.push(obj);
});
});
dateiinhaltEE.emit('dateiinhalt_ready');
});
dateiinhaltEE.on('dateiinhalt_ready',function(){
//console.log(dateiinhalt);
console.log("dateinamen:" + dateinamen.length);
console.log("dateiinhalt:" + dateiinhalt.length);
});
res.render('home', {
title: 'Home'
});
};
If if log the length of the 2 arrays the output on the second reload shows. First time loading the url:
Express server listening on port 3000 in development mode
dateinamen:2
dateiinhalt:20
Second time / refreshing the url:
GET / 200 898.198 ms - -
GET /fonts/fontawesome-webfont.woff2?v=4.3.0 304 12.991 ms - -
GET /favicon.ico 200 4.516 ms - -
dateinamen:4
dateiinhalt:60
dateinamen:4
dateiinhalt:60
dateinamen:4
dateiinhalt:100
dateinamen:4
dateiinhalt:100
GET / 200 139.259 ms - -
What causes the code to extend the arrays while reloading the page?
The non-blocking problem is due do your for(...) loops.
Changing them by : array.forEach(function(elem, index){});
EDIT
The arrays should be initialized inside the index function :
exports.index = function(req, res) {
var dateinamen = [];
var dateiinhalt = [];
...
Also, I'm not sure you need the use of EventEmitter.
Something like
`
fs.readdir('./data', function (err, files) {
if (!err) {
files.forEach(function(file){
var buf = fs.readFileSync('./data/'+file, "utf8");
var xml = new XML(buf);
var list = xml.descendants("suggestion");
var ergebnis = null;
var obj = null;
list.each(function(item, index){
ergebnis = item.attribute('data').toString();
obj = {k: file, v: ergebnis};
dateiinhalt.push(obj);
});
});
console.log(dateiinhalt);
} else {
throw err;
}
});
`
could do the job no?
(I wanted to say this as a comment, but I'm still missing reputation)
Using Node.js, when one requests a HTTP response, in optimal circumstances, the request comes back with a HTTP response.
However, sometimes the request breaks because the site, for example, has a 404 code, or the site does not exist at all. When requesting a batch of URLs, if there is a 404 code on, say, the 200th URL out of 1000 URLs requested, the entire script breaks. Here is my code:
var hostNames = ['www.gizmodo.com','www.sitethatdoesnotexist123.com','www.google.com'];
for (i; i < hostNames.length; i++){
var options = {
host: hostNames[i],
path: '/'
};
(function (i){
http.get(options, function(res) {
var obj = {};
obj.url = hostNames[i];
obj.statusCode = res.statusCode;
obj.headers = res.headers;
db.scrape.save(obj);
}).on('error',function(e){
console.log("Error Site: " + hostNames[i]);
});
})(i);
};
Is there a way, that for example, if the site does not exist, I simply skip to the next URL, instead of having the script break?
EDIT: Fixed. Thanks user DavidKemp
Use a try/catch block to catch any errors that might occur, and then continue on from there.
For example:
var hostNames = ['www.gizmodo.com','www.sitethatdoesnotexist123.com','www.google.com'];
//moved the function out so we do not have to keep redefining it:
var get_url = function (i){
http.get(options, function(res) {
var obj = {};
obj.url = hostNames[i];
obj.statusCode = res.statusCode;
obj.headers = res.headers;
console.log(JSON.stringify(obj, null, 4));
})
};
for (i; i < hostNames.length; i++){
var options = {
host: hostNames[i],
path: '/'
};
try {
get_url(i);
}
catch(err){
//do something with err
}
};
You need to bind an error handler to your request. I also cleaned up the code a bit.
hostNames.forEach(function(hostname), {
var req = http.get({host: hostName}, function(res) {
var obj = {
url: hostName,
statusCode: res.statusCode,
headers: res.headers
};
console.log(JSON.stringify(obj, null, 4));
});
req.on('error', function(err){
console.log('Failed to fetch', hostName);
});
});
You can use uncaughtException event. this let script run even after exception. link
process.on('uncaughtException', function(err) {
console.log('Caught exception: ' + err);
});
var hostNames = ['www.gizmodo.com','www.sitethatdoesnotexist123.com','www.google.com'];
for (i; i < hostNames.length; i++){
var options = {
host: hostNames[i],
path: '/'
};
(function (i){
http.get(options, function(res) {
var obj = {};
obj.url = hostNames[i];
obj.statusCode = res.statusCode;
obj.headers = res.headers;
db.scrape.save(obj);
}).on('error',function(e){
console.log("Error Site: " + hostNames[i]);
});
})(i);
};
Added a callback for when there's an error. Logs the site that returns an error on console. This error is usually triggered by a 404 or request time is too long.
The full docs are at http://nodejs.org/api/http.html#http_http_get_options_callback at the time of writing. loganfsmyth's answer provides a useful example.