Develop Reference

Javascript: parsing emails into json/building json file from text

I have a self-made email parser that pulls email body's that match a specific subject and writes each email to individual text files in a folder, using fs:
module.exports = {
run: function(){
var Imap = require('imap'),
inspect = require('util').inspect;
var fs = require('fs'), fileStream;
var logger = require('./logger.js');
var buffer = '';
var imap = new Imap({
user: "tracker#email.com",
password: "xxxxxxxx",
host: "host.secureserver.net",
port: 993,
tls: true,
connTimeout: 10000,
authTimeout: 5000,
debug: console.log,
tlsOptions: { rejectUnauthorized: true },
mailbox: "INBOX",
searchFilter: ["UNSEEN", "FLAGGED"], // the search filter being used after an IDLE notification has been retrieved
markSeen: false, // all fetched email willbe marked as seen and not fetched next time
fetchUnreadOnStart: true, // use it only if you want to get all unread email on lib start. Default is `false`,
mailParserOptions: { streamAttachments: false }, // options to be passed to mailParser lib.
attachments: false, // download attachments as they are encountered to the project directory
attachmentOptions: { directory: "attachments/" }
});
function openInbox(cb){
imap.openBox('Inbox', false, cb);
}
imap.once('ready', function(){
logger.printWriteLine('Parsing inbox for new error alerts...', 1);
openInbox(function (err, box){
if(err) throw err;
imap.search(
['UNSEEN', ['SUBJECT', 'Error Alerts']],
function(err, results){
if(err) throw err;
else if(!results || !results.length){
logger.printWriteLine('No new emails', 2);
}
else{
var f = imap.fetch(results, {bodies: '1', markSeen: true});
f.on('message', function(msg, seqno){
logger.printWriteLine('message #:'+seqno, 1);
logger.printWriteLine('message type: '+msg.txt, 1);
var prefix = '(#' + seqno + ') ';
msg.on('body', function (stream, info){
stream.on('data', function(chunk){
buffer += chunk.toString('utf8');
//console.log('Buffer: '+buffer);
})
stream.once('end', function(){
if(info.which === '1'){
//console.log('Buffer 2: ' + buffer);
}
});
stream.pipe(fs.createWriteStream('./mailParser/'+ seqno + '-body.txt'));
});
msg.once('end', function () {
logger.printWriteLine(prefix + ' - End of message.', 1);
});
});
f.once('error', function (err) {
console.log('Fetch error: ' + err);
});
f.once('end', function () {
logger.printWriteLine('Done fetching messages.', 1);
imap.end();
});
}
});
});
});
imap.once('error', function (err) {
console.log(err);
});
imap.once('end', function () {
console.log('Connection ended');
});
imap.connect();
}
}
The resulting text files are each formatted like this:
Vehicle ID720
DRIDT0MA12200330
Event TypeMediaError - SD1,Protected
Event Local Time2022-09-20 16:54:18
Event Time GMT2022-09-20 20:54:18
URLView Event
More DataSD1,Protected
Vehicle Registration
Vehicle VIN
Vehicle Manufacturer
Vehicle Model
Vehicle FuelColorTransmissionPolicy HolderPolicy NumberCustom 1Custom 2
My goal is to parse/pull relevant data from these emails, and build the details into a json file. Right now I'm stuck trying to loop through each text file, parse/pull the relevant details with regex/"simple-text-parser", and build into a json file to be consumed by other parts of my program.
goal output would be something like this:
{
"vehicle1": {
"number": "720",
"DRID": "T0MA12200330",
"Type": "Media Error - SD1,Protected"
"Time": "2022-09-20 16:54:18"
},
"vehicle2: {
...
}
}
Obviously this is an inefficient, round-about way to accomplish this (with the text files). I'm wondering if anybody can suggest a better way to parse the emails from my mail parser, so perhaps I can go straight to JSON, or any alternate methods one might use to accomplish what I'm trying to do.
Please understand I'm not a professional dev so my code may be pretty rough.

Search in ldapjs

I am trying to use the search method of Ldap.js in my node.js code. Here is my code for the client side. It adds successfully a user, but searching for the newly added user does not yield any results. (The ldap server is running in a docker container from https://github.com/osixia/docker-openldap)
var ldap = require("ldapjs");
var assert = require("assert");
var client = ldap.createClient({
url: "ldap://localhost:389",
});
client.bind("cn=admin,dc=example,dc=org", "admin", function (err) {
assert.ifError(err);
let newUser = {
cn: "userId7",
userPassword: "password",
objectClass: "person",
sn: "efub",
};
// Here i successfully add this user "userId7"
client.add(
"cn=userId7,dc=example,dc=org",
newUser,
(err, response) => {
if (err) return console.log(err);
return response;
}
);
var options = {
filter: "(objectClass=*)",
scope: "sub",
};
// Now the search, it runs without error, but does never receive a searchEntry
client.search(
"cn=userId7,dc=example,dc=org",
options,
function (error, search) {
console.log("Searching.....");
client.on("searchEntry", function (entry) {
console.log("I found a result in searchEntry");
});
client.on("error", function (error) {
console.error("error: " + error.message);
});
client.unbind(function (error) {
if (error) {
console.log(error.message);
} else {
console.log("client disconnected");
}
});
}
);
});
client.on('error', function (err) {
if (err.syscall == "connect") {
console.log(err);
}
});
Also, if it helps, this is how the newly added user looks like when i display all users from ldap by running docker exec my-openldap-container ldapsearch -x -H ldap://localhost:389 -b dc=example,dc=org -D "cn=admin,dc=example,dc=org" -w admin
# userId7, example.org
dn: cn=userId7,dc=example,dc=org
cn: userId7
userPassword:: cGFzc3dvcmQ=
objectClass: person
sn: efub
Update: I can successfully search for the user "userId7" with the shell command: docker exec ldap-service ldapsearch -LLL -x -D "cn=admin,dc=example,dc=org" -w "admin" -b "cn=userId7,dc=example,dc=org" "(objectclass=*)". How can i make ldapJS also run this search successfully?
Update 2: I can also successfully search by using the frontend "phpLDAPadmin" as seen in the screenshots below:

So i solved it. The correct client.search code is:
client.search(
"cn=userId7,dc=example,dc=org",
options,
function (error, res) {
console.log("Searching.....");
res.on("searchEntry", function (entry) {
console.log("I found a result in searchEntry", JSON.stringify(entry.object));
});
res.on("error", function (error) {
console.error("error: " + error.message);
});
client.unbind(function (error) {
if (error) {
console.log(error.message);
} else {
console.log("client disconnected");
}
});
}
);
Inside function (error, res) { I listened for the events via client.on("searchEntry", instead of res.on("searchEntry", therefore missing the events from the search results. The root cause was a classic copy and paste error and changing the variable while misunderstanding the origin of the event.

Saving and deleting lines to & from file in node.js

I have written code for a chatroom, I am trying to implement a ban list which bans by username.
In the file I want it to look something like this (without the blank lines between each other)..
UserToBan1 Banned-By Reason-For-Ban
UserToBan2 Banned-By Reason-For-Ban
UserToBan3 Banned-By Reason-For-Ban
I want to be able to check if the person is listed in that file by username.
Want to be able to remove the line from the list (unban) and to be able to add someone to the file.
I am new to node.js and javascript but I don't know what would be the best way to do this. I have created a banlist.json file which I know how to open and close but adding, removing lines and checking the first variable is where I am stuck.
EDIT:
This the code I am now working with but seems to produce a null value when I console.log(data) or console.log(content).
s.on('connection', function(ws) {
ws.on('message', function(message){
// START only on connection
message = JSON.parse(message);
if(message.type == "name"){
// start check for double login
var ConnectingUser = message.data;
var found = 0;
s.clients.forEach(function e(client) {
var ConnectedUser = client.personName;
if(ConnectedUser == ConnectingUser) {
client.send(JSON.stringify(
{
name: "Server",
data: "***We do not allow double logins!"
}
));
client.send(JSON.stringify(
{
name: "Server",
data: "🔴 Disconnected..."
}
));
client.close();
}
});
// end check for double login
console.log("Client <"+message.data+"> Connected");
memberJoinedChatMsg(ws, message);
ws.personName = message.data;
return;
}
// -- test stuff start ------------------------------------------------------------
var file = './banlist/banned.json';
fs = require('fs');
fs.readFile(file, function(content) {
var data = JSON.parse(content);
console.log(Object.keys(data));
// Delete line
delete data["UserToBan1"]
console.log(Object.keys(data));
// Convert JSON object to string
var transformed_content = JSON.dumps(data);
// write file here
fs.writeFile(file, transformed_content, function(err) {
if (err) {
console.log("Error writing file: " + (err.stack || err))
} else {
console.log("Saved file")
}
})
});
// -- test stuff end ------------------------------------------------------------

If you know how to read/write a file, you can directly use store the data as JSON in that file, e.g.:
{
"UserToBan1": {
"bannedby": "user who banned UserToBan1",
"reason": "reason for the ban"
},
"UserToBan2": {
"bannedby": "user who banned UserToBan2",
"reason": "reason for the ban"
},
"UserToBan3": {
"bannedby": "user who banned UserToBan3",
"reason": "reason for the ban"
}
}
When reading the file, parse the file content as JSON:
fs = require('fs');
var file = "/path/to/json/file";
fs.readFile(file, function(err, content) {
if (err) {
console.log("Error reading file: " + (err.stack || err))
} else {
var data = JSON.parse(content);
console.log(Object.keys(data));
// Delete line (see: https://stackoverflow.com/questions/3455405/how-do-i-remove-a-key-from-a-javascript-object/28797751)
// example to delete user from list
delete data["UserToBan1"]
// example to add user to list
data["UserToBan4"] = {
"bannedby": "user who banned UserToBan4",
"reason": "reason for banning UserToBan4"
}
console.log(Object.keys(data));
// Convert JSON object to string
var transformed_content = JSON.stringify(data, null, 4);
// write file here
fs.writeFile(file, transformed_content, function(err) {
if (err) {
console.log("Error writing file: " + (err.stack || err))
} else {
console.log("Saved file")
}
})
}
});

MongoClient not returning data in cucumberjs test

I've taken this apart several different ways. The find happens after the remove, and the find never finds anything. If I comment out the this.accounts.remove... the find works. If I leave the remove line in there it doesn't. My understanding of cucumberjs, mongo client and node indicates that the find should work.
I've even tried moving the remove/find sequence into its own file, and it works there. It seems to be only when I'm running it in cucumber that the sequence fails. I suspect because of the way of cucumber loads the files, but I'm not sure.
Can someone help me figure out how to get this working?
World.js:
var db = new Db('FlashCards', new Server('localhost', 27017));
db.open(function(err, opened) {
if (err) {
console.log("error opening: ", err);
done(err);
}
db = opened;
});
var {
defineSupportCode
} = require('cucumber');
function CustomWorld() {
this.db = db;
this.accounts = db.collection('accounts');
hooks.js:
Before(function(result, done) {
//comment this out, and leave a done(), it works!!!!
this.accounts.remove(function(error, result){
if( error) {
console.log("Error cleaning the database: ", error);
done(error);
}
done();
})
});
user_steps.js:
Then('I will be registered', function(done) {
let world = this;
this.accounts.find({
username: world.user.username
}).toArray(
function(err, accounts) {
if (err) {
console.log("Error retrieveing data: ", err);
done(err);
}
console.log("Accounts found: ", accounts);
expect(accounts).to.be.ok;
expect(accounts.length).to.be.equal(1);
done();
});
});
Inovcation:
cucumber-js --compiler es6:babel-core/register

You are missing the item to be removed in the remove method. I am assuming the item to be removed is
this.accounts.remove(function(error, result){

You are missing one parameter to remove method. The parameter is query to remove. I am assuming, the remove query is {username: world.user.username}
var qry={username: world.user.username};
Please try with the following:
Before(function(result, done) { //comment this out, and leave a done(), it works!!!!
var qry={username: world.user.username};
this.accounts.remove(qry, function(error, result){
if( error) {
console.log("Error cleaning the database: ", error);
done(error);
}
done();
}) });

GmailAPI - Node.js - Cannot update user signatures

I have authorised a small node app with the gmail API. I am successfully getting responses. I can, for eg, see my email signature using users.settings.sendAs.list()
However When I try and make a change to the settings, again in this case the signature, the response I get back is empty. So the update is working in so far as i can wipe a signature I added manually in gmail, but I cannot add anything new.
Here's the script
var google = require('googleapis');
var key = require('./Sig-Updater.json');
var gmail = google.gmail('v1');
var jwtClient = new google.auth.JWT(
key.client_email,
null,
key.private_key,
['https://www.googleapis.com/auth/gmail.settings.basic', 'https://www.googleapis.com/auth/gmail.settings.sharing', 'https://www.googleapis.com/auth/gmail.modify', 'https://mail.google.com', 'https://www.googleapis.com/auth/gmail.compose',
'https://www.googleapis.com/auth/gmail.readonly' ],
'some_email#somewhere.com'
);
jwtClient.authorize(function (err, tokens) {
if (err) {
console.log('Auth failed because: ' + err);
return;
}
console.log(tokens)
gmail.users.settings.sendAs.update({
userId: 'me',
auth: jwtClient,
sendAsEmail: 'some_email#somewhere.com',
signature: '<div dir="ltr">Hello there</div>'
}, function (err, resp) {
if(err){
console.log(err);
} else {
console.log(resp);
}
});
});
Sorry for any noob business - new to this API and indeed node.js

Looks like my Noob disclaimer was warranted in the end,
I had omitted the 'resource' object from the request.
Should look more like
gmail.users.settings.sendAs.update({
userId: 'me',
auth: jwtClient,
sendAsEmail: 'some_email#somewhere.com',
fields: 'signature',
resource: {
signature: '<div dir="ltr">Hello there</div>'
}
}, function (err, resp) {
if(err){
console.log(err);
} else {
console.log(resp);
}
});
Hope that helps someone in the future!