I have a self-made email parser that pulls email body's that match a specific subject and writes each email to individual text files in a folder, using fs:
module.exports = {
run: function(){
var Imap = require('imap'),
inspect = require('util').inspect;
var fs = require('fs'), fileStream;
var logger = require('./logger.js');
var buffer = '';
var imap = new Imap({
user: "tracker#email.com",
password: "xxxxxxxx",
host: "host.secureserver.net",
port: 993,
tls: true,
connTimeout: 10000,
authTimeout: 5000,
debug: console.log,
tlsOptions: { rejectUnauthorized: true },
mailbox: "INBOX",
searchFilter: ["UNSEEN", "FLAGGED"], // the search filter being used after an IDLE notification has been retrieved
markSeen: false, // all fetched email willbe marked as seen and not fetched next time
fetchUnreadOnStart: true, // use it only if you want to get all unread email on lib start. Default is `false`,
mailParserOptions: { streamAttachments: false }, // options to be passed to mailParser lib.
attachments: false, // download attachments as they are encountered to the project directory
attachmentOptions: { directory: "attachments/" }
});
function openInbox(cb){
imap.openBox('Inbox', false, cb);
}
imap.once('ready', function(){
logger.printWriteLine('Parsing inbox for new error alerts...', 1);
openInbox(function (err, box){
if(err) throw err;
imap.search(
['UNSEEN', ['SUBJECT', 'Error Alerts']],
function(err, results){
if(err) throw err;
else if(!results || !results.length){
logger.printWriteLine('No new emails', 2);
}
else{
var f = imap.fetch(results, {bodies: '1', markSeen: true});
f.on('message', function(msg, seqno){
logger.printWriteLine('message #:'+seqno, 1);
logger.printWriteLine('message type: '+msg.txt, 1);
var prefix = '(#' + seqno + ') ';
msg.on('body', function (stream, info){
stream.on('data', function(chunk){
buffer += chunk.toString('utf8');
//console.log('Buffer: '+buffer);
})
stream.once('end', function(){
if(info.which === '1'){
//console.log('Buffer 2: ' + buffer);
}
});
stream.pipe(fs.createWriteStream('./mailParser/'+ seqno + '-body.txt'));
});
msg.once('end', function () {
logger.printWriteLine(prefix + ' - End of message.', 1);
});
});
f.once('error', function (err) {
console.log('Fetch error: ' + err);
});
f.once('end', function () {
logger.printWriteLine('Done fetching messages.', 1);
imap.end();
});
}
});
});
});
imap.once('error', function (err) {
console.log(err);
});
imap.once('end', function () {
console.log('Connection ended');
});
imap.connect();
}
}
The resulting text files are each formatted like this:
Vehicle ID720
DRIDT0MA12200330
Event TypeMediaError - SD1,Protected
Event Local Time2022-09-20 16:54:18
Event Time GMT2022-09-20 20:54:18
URLView Event
More DataSD1,Protected
Vehicle Registration
Vehicle VIN
Vehicle Manufacturer
Vehicle Model
Vehicle FuelColorTransmissionPolicy HolderPolicy NumberCustom 1Custom 2
My goal is to parse/pull relevant data from these emails, and build the details into a json file. Right now I'm stuck trying to loop through each text file, parse/pull the relevant details with regex/"simple-text-parser", and build into a json file to be consumed by other parts of my program.
goal output would be something like this:
{
"vehicle1": {
"number": "720",
"DRID": "T0MA12200330",
"Type": "Media Error - SD1,Protected"
"Time": "2022-09-20 16:54:18"
},
"vehicle2: {
...
}
}
Obviously this is an inefficient, round-about way to accomplish this (with the text files). I'm wondering if anybody can suggest a better way to parse the emails from my mail parser, so perhaps I can go straight to JSON, or any alternate methods one might use to accomplish what I'm trying to do.
Please understand I'm not a professional dev so my code may be pretty rough.
Related
I am trying to use the search method of Ldap.js in my node.js code. Here is my code for the client side. It adds successfully a user, but searching for the newly added user does not yield any results. (The ldap server is running in a docker container from https://github.com/osixia/docker-openldap)
var ldap = require("ldapjs");
var assert = require("assert");
var client = ldap.createClient({
url: "ldap://localhost:389",
});
client.bind("cn=admin,dc=example,dc=org", "admin", function (err) {
assert.ifError(err);
let newUser = {
cn: "userId7",
userPassword: "password",
objectClass: "person",
sn: "efub",
};
// Here i successfully add this user "userId7"
client.add(
"cn=userId7,dc=example,dc=org",
newUser,
(err, response) => {
if (err) return console.log(err);
return response;
}
);
var options = {
filter: "(objectClass=*)",
scope: "sub",
};
// Now the search, it runs without error, but does never receive a searchEntry
client.search(
"cn=userId7,dc=example,dc=org",
options,
function (error, search) {
console.log("Searching.....");
client.on("searchEntry", function (entry) {
console.log("I found a result in searchEntry");
});
client.on("error", function (error) {
console.error("error: " + error.message);
});
client.unbind(function (error) {
if (error) {
console.log(error.message);
} else {
console.log("client disconnected");
}
});
}
);
});
client.on('error', function (err) {
if (err.syscall == "connect") {
console.log(err);
}
});
Also, if it helps, this is how the newly added user looks like when i display all users from ldap by running docker exec my-openldap-container ldapsearch -x -H ldap://localhost:389 -b dc=example,dc=org -D "cn=admin,dc=example,dc=org" -w admin
# userId7, example.org
dn: cn=userId7,dc=example,dc=org
cn: userId7
userPassword:: cGFzc3dvcmQ=
objectClass: person
sn: efub
Update: I can successfully search for the user "userId7" with the shell command: docker exec ldap-service ldapsearch -LLL -x -D "cn=admin,dc=example,dc=org" -w "admin" -b "cn=userId7,dc=example,dc=org" "(objectclass=*)". How can i make ldapJS also run this search successfully?
Update 2: I can also successfully search by using the frontend "phpLDAPadmin" as seen in the screenshots below:
So i solved it. The correct client.search code is:
client.search(
"cn=userId7,dc=example,dc=org",
options,
function (error, res) {
console.log("Searching.....");
res.on("searchEntry", function (entry) {
console.log("I found a result in searchEntry", JSON.stringify(entry.object));
});
res.on("error", function (error) {
console.error("error: " + error.message);
});
client.unbind(function (error) {
if (error) {
console.log(error.message);
} else {
console.log("client disconnected");
}
});
}
);
Inside function (error, res) { I listened for the events via client.on("searchEntry", instead of res.on("searchEntry", therefore missing the events from the search results. The root cause was a classic copy and paste error and changing the variable while misunderstanding the origin of the event.
I have written code for a chatroom, I am trying to implement a ban list which bans by username.
In the file I want it to look something like this (without the blank lines between each other)..
UserToBan1 Banned-By Reason-For-Ban
UserToBan2 Banned-By Reason-For-Ban
UserToBan3 Banned-By Reason-For-Ban
I want to be able to check if the person is listed in that file by username.
Want to be able to remove the line from the list (unban) and to be able to add someone to the file.
I am new to node.js and javascript but I don't know what would be the best way to do this. I have created a banlist.json file which I know how to open and close but adding, removing lines and checking the first variable is where I am stuck.
EDIT:
This the code I am now working with but seems to produce a null value when I console.log(data) or console.log(content).
s.on('connection', function(ws) {
ws.on('message', function(message){
// START only on connection
message = JSON.parse(message);
if(message.type == "name"){
// start check for double login
var ConnectingUser = message.data;
var found = 0;
s.clients.forEach(function e(client) {
var ConnectedUser = client.personName;
if(ConnectedUser == ConnectingUser) {
client.send(JSON.stringify(
{
name: "Server",
data: "***We do not allow double logins!"
}
));
client.send(JSON.stringify(
{
name: "Server",
data: "🔴 Disconnected..."
}
));
client.close();
}
});
// end check for double login
console.log("Client <"+message.data+"> Connected");
memberJoinedChatMsg(ws, message);
ws.personName = message.data;
return;
}
// -- test stuff start ------------------------------------------------------------
var file = './banlist/banned.json';
fs = require('fs');
fs.readFile(file, function(content) {
var data = JSON.parse(content);
console.log(Object.keys(data));
// Delete line
delete data["UserToBan1"]
console.log(Object.keys(data));
// Convert JSON object to string
var transformed_content = JSON.dumps(data);
// write file here
fs.writeFile(file, transformed_content, function(err) {
if (err) {
console.log("Error writing file: " + (err.stack || err))
} else {
console.log("Saved file")
}
})
});
// -- test stuff end ------------------------------------------------------------
If you know how to read/write a file, you can directly use store the data as JSON in that file, e.g.:
{
"UserToBan1": {
"bannedby": "user who banned UserToBan1",
"reason": "reason for the ban"
},
"UserToBan2": {
"bannedby": "user who banned UserToBan2",
"reason": "reason for the ban"
},
"UserToBan3": {
"bannedby": "user who banned UserToBan3",
"reason": "reason for the ban"
}
}
When reading the file, parse the file content as JSON:
fs = require('fs');
var file = "/path/to/json/file";
fs.readFile(file, function(err, content) {
if (err) {
console.log("Error reading file: " + (err.stack || err))
} else {
var data = JSON.parse(content);
console.log(Object.keys(data));
// Delete line (see: https://stackoverflow.com/questions/3455405/how-do-i-remove-a-key-from-a-javascript-object/28797751)
// example to delete user from list
delete data["UserToBan1"]
// example to add user to list
data["UserToBan4"] = {
"bannedby": "user who banned UserToBan4",
"reason": "reason for banning UserToBan4"
}
console.log(Object.keys(data));
// Convert JSON object to string
var transformed_content = JSON.stringify(data, null, 4);
// write file here
fs.writeFile(file, transformed_content, function(err) {
if (err) {
console.log("Error writing file: " + (err.stack || err))
} else {
console.log("Saved file")
}
})
}
});
I am trying to use imap and node to get all of the unread emails from an email and log the email sender and the subject. I am trying to run the following code:
var imap = new Imap({
user: Email,
password: Password,
host: 'imap.gmail.com',
port: 993,
tls: true
});
function openInbox(cb) {
imap.openBox('INBOX', true, cb);
}
openInbox(function(err, box) {
if (err) throw err;
var f = imap.seq.fetch(box.messages.total + ':*', { bodies: ['HEADER.FIELDS (FROM)','TEXT'] });
f.on('message', function(msg, seqno) {
console.log('Message #%d', seqno);
var prefix = '(#' + seqno + ') ';
msg.on('body', function(stream, info) {
if (info.which === 'TEXT')
console.log(prefix + 'Body [%s] found, %d total bytes',
inspect(info.which), info.size);
var buffer = '', count = 0;
stream.on('data', function(chunk) {
count += chunk.length;
buffer += chunk.toString('utf8');
if (info.which === 'TEXT')
console.log(prefix + 'Body [%s] (%d/%d)',
inspect(info.which), count, info.size);
});
stream.once('end', function() {
if (info.which !== 'TEXT')
console.log(prefix + 'Parsed header: %s',
inspect(Imap.parseHeader(buffer)));
else
console.log(prefix + 'Body [%s] Finished',
inspect(info.which));
});
});
msg.once('attributes', function(attrs) {
console.log(prefix + 'Attributes: %s', inspect(attrs, false, 8));
});
msg.once('end', function() {
console.log(prefix + 'Finished');
});
});
f.once('error', function(err) {
console.log('Fetch error: ' + err);
});
f.once('end', function() {
console.log('Done fetching all messages!');
imap.end();
});
});
When I run this code I am getting the following error:
Uncaught Error: Not authenticated
at Connection.openBox
What does this error mean? I made sure to enable Less secure app access on every gmail account I am trying to access. I would appreciate anyones help
Assuming you are using the node-imap module, you aren't calling it correctly.
This error happens if you try to call openBox before your connection is set up. (That is, that you are connected and authenticated, and ready to open a mailbox).
You need to do two things:
Add a top level call to imap.connect()
Wrap most of your current openInbox call inside of a call to imap.once() like this:
imap.once('ready', function() {
openInbox(function(err, box) {
...
});
});
I'd recommend you start with the exact example in their documentation until you are confident you have it set up right and working, so you can separate out the code issues from the issues with gmail authentication.
Beyond that, as you note, Gmail is pretty fickle about how you are using low security apps. It probably makes sense to use a normal client to prove that you've gotten the low security app setting enabled, you've proceessed any unlock captchas, etc, that you need to do. See the full list. In particular, you may need to approve one of the logins manually in the security console before it starts to work.
FWIW, the error you get if you're connected, but not authenticating because Gmail doesn't like the login is different, it looks more like this:
{ Error: Please log in via your web browser: https://support.google.com/mail/accounts/answer/78754 (Failure)
at Connection._resTagged (/.../lib/Connection.js:1502:11)
at Parser.<anonymous> (/.../lib/Connection.js:194:10)
I have authorised a small node app with the gmail API. I am successfully getting responses. I can, for eg, see my email signature using users.settings.sendAs.list()
However When I try and make a change to the settings, again in this case the signature, the response I get back is empty. So the update is working in so far as i can wipe a signature I added manually in gmail, but I cannot add anything new.
Here's the script
var google = require('googleapis');
var key = require('./Sig-Updater.json');
var gmail = google.gmail('v1');
var jwtClient = new google.auth.JWT(
key.client_email,
null,
key.private_key,
['https://www.googleapis.com/auth/gmail.settings.basic', 'https://www.googleapis.com/auth/gmail.settings.sharing', 'https://www.googleapis.com/auth/gmail.modify', 'https://mail.google.com', 'https://www.googleapis.com/auth/gmail.compose',
'https://www.googleapis.com/auth/gmail.readonly' ],
'some_email#somewhere.com'
);
jwtClient.authorize(function (err, tokens) {
if (err) {
console.log('Auth failed because: ' + err);
return;
}
console.log(tokens)
gmail.users.settings.sendAs.update({
userId: 'me',
auth: jwtClient,
sendAsEmail: 'some_email#somewhere.com',
signature: '<div dir="ltr">Hello there</div>'
}, function (err, resp) {
if(err){
console.log(err);
} else {
console.log(resp);
}
});
});
Sorry for any noob business - new to this API and indeed node.js
Looks like my Noob disclaimer was warranted in the end,
I had omitted the 'resource' object from the request.
Should look more like
gmail.users.settings.sendAs.update({
userId: 'me',
auth: jwtClient,
sendAsEmail: 'some_email#somewhere.com',
fields: 'signature',
resource: {
signature: '<div dir="ltr">Hello there</div>'
}
}, function (err, resp) {
if(err){
console.log(err);
} else {
console.log(resp);
}
});
Hope that helps someone in the future!
I'm looking for a way to refactor part of my code to be shorter and simpler, but I don't know Mongoose very well and I'm not sure how to proceed.
I am trying to check a collection for the existence of a document and, if it doesn't exist, create it. If it does exist, I need to update it. In either case I need to access the document's contents afterward.
What I've managed to do so far is query the collection for a specific document and, if it's not found, create a new document. If it is found, I update it (currently using dates as dummy data for this). From there I can access either the found document from my initial find operation or the newly saved document and this works, but there must be a better way to accomplish what I'm after.
Here's my working code, sans distracting extras.
var query = Model.find({
/* query */
}).lean().limit(1);
// Find the document
query.exec(function(error, result) {
if (error) { throw error; }
// If the document doesn't exist
if (!result.length) {
// Create a new one
var model = new Model(); //use the defaults in the schema
model.save(function(error) {
if (error) { throw error; }
// do something with the document here
});
}
// If the document does exist
else {
// Update it
var query = { /* query */ },
update = {},
options = {};
Model.update(query, update, options, function(error) {
if (error) { throw error; }
// do the same something with the document here
// in this case, using result[0] from the topmost query
});
}
});
I've looked into findOneAndUpdate and other related methods but I'm not sure if they fit my use case or if I understand how to use them correctly. Can anyone point me in the right direction?
(Probably) Related questions:
How to check if that data already exist in the database during update (Mongoose And Express)
Mongoose.js: how to implement create or update?
NodeJS + Mongo: Insert if not exists, otherwise - update
Return updated collection with Mongoose
Edit
I didn't come across the question pointed out to me in my searching, but after reviewing the answers there I've come up with this. It's certainly prettier, in my opinion, and it works, so unless I'm doing something horribly wrong I think my question can probably be closed.
I would appreciate any additional input on my solution.
// Setup stuff
var query = { /* query */ },
update = { expire: new Date() },
options = { upsert: true };
// Find the document
Model.findOneAndUpdate(query, update, options, function(error, result) {
if (!error) {
// If the document doesn't exist
if (!result) {
// Create it
result = new Model();
}
// Save the document
result.save(function(error) {
if (!error) {
// Do something with the document
} else {
throw error;
}
});
}
});
You are looking for the new option parameter. The new option returns the newly created document(if a new document is created). Use it like this:
var query = {},
update = { expire: new Date() },
options = { upsert: true, new: true, setDefaultsOnInsert: true };
// Find the document
Model.findOneAndUpdate(query, update, options, function(error, result) {
if (error) return;
// do something with the document
});
Since upsert creates a document if not finds a document, you don't need to create another one manually.
Since you wish to refactor parts of your code to be shorter and simpler,
Use async / await
Use .findOneAndUpdate() as suggested in this answer
let query = { /* query */ };
let update = {expire: new Date()};
let options = {upsert: true, new: true, setDefaultsOnInsert: true};
let model = await Model.findOneAndUpdate(query, update, options);
///This is simple example explaining findByIDAndUpdate from my code added with try catch block to catch errors
try{
const options = {
upsert: true,
new: true,
setDefaultsOnInsert: true
};
const query = {
$set: {
description: req.body.description,
title: req.body.title
}
};
const survey = await Survey.findByIdAndUpdate(
req.params.id,
query,
options
).populate("questions");
}catch(e){
console.log(e)
}
Here is an example I am using. I have to return custom responses for UI updates etc. This can be even shorter. User is
const UserScheme = mongoose.Schema({
_id: String,
name: String,
city: String,
address: String,
},{timestamps: true});
const User = mongoose.model('Users', UserScheme);
async function userUpdateAdd(data){
var resp = '{"status": "error"}';
if(data){
var resp = await User.updateOne({ _id: data._id }, data).then(function(err, res){
console.log("database.userUpdateAdd -> Update data saved in database!");
if(err){
var errMessage = err.matchedCount == 0 ? "User Record does not exist, will create new..." : "Record not updated";
// If no match, create new
if(err.matchedCount == 0){
const create_user = new User(data);
resp = create_user.save().then(function(){
console.log("database.userUpdateAdd -> Data saved to database!");
return '{"status":"success", "message": "New User added successfully"}';
});
return resp;
}
// Exists, return success update message
if(err.matchedCount == 1){
return '{"status": "success", "message" : "Update saved successfully"}';
} else {
return '{"status": "error", "code": "' + err.modifiedCount + '", "message": "' + errMessage + '"}';
}
}
})
.catch((error) => {
//When there are errors We handle them here
console.log("database.userUpdateAdd -> Error, data not saved! Server error");
return '{"status": "error", "code": "400", "message": "Server error!"}';
});
}
return resp;
}
Here's an example:
const mongoose = require('mongoose');
mongoose.connect('mongodb://localhost/rsvp', {useNewUrlParser: true, useUnifiedTopology: true});
const db = mongoose.connection;
db.on('error', () => {
console.log('mongoose connection error');
});
db.once('open', () => {
console.log('mongoose connected successfully');
});
const rsvpSchema = mongoose.Schema({
firstName: String,
lastName: String,
email: String,
guests: Number
});
const Rsvp = mongoose.model('Rsvp', rsvpSchema);
// This is the part you will need... In this example, if first and last name match, update email and guest number. Otherwise, create a new document. The key is to learn to put "upsert" as the "options" for the argument.
const findRsvpAndUpdate = (result, callback) => {
Rsvp.findOneAndUpdate({firstName: result.firstName, lastName: result.lastName}, result, { upsert: true }, (err, results) => {
if (err) {
callback(err);
} else {
callback(null, results);
}
})
};
// From your server index.js file, call this...
app.post('/rsvps', (req, res) => {
findRsvpAndUpdate(req.body, (error, result) => {
if (error) {
res.status(500).send(error);
} else {
res.status(200).send(result);
}
})
});