Develop Reference

Using FS to write new files if a certain url is found and remove the file if it's not found anymore

I'm trying to write a script, when a new url is found it will turn the url to a hash. Check if the file already has been written it just ignores it, and if it's not known earlier it should be added.
needle.get(mainUrl, function(err, res) {
if (err) throw err;
if (res.statusCode == 200 && !err ) {
var $ = cheerio.load(res.body)
var href = $('div div a').each(function(index, element) {
urlList.push($(element).attr("href"))
var url =($(element).attr("href"))
var hash = crypto.createHash('md5').update(url).digest('hex');
fs.writeFile('./directory/otherdirectory' + `${hash}`, url, (err) => {
if (err) throw err;
console.log('Hash created: ' + url + ' saved as ' + hash
});
}
)
}
})
This is what I've done so far, but this only writes new files. it doesn't check if files already has been added and doesn't remove files that's not found anymore.
So what I try to do:
I've written a script that fetches a website for urls.
Hash all the urls.
Make FS check if file already has been written, if it has just ignore it.
If it not is known earlier, add it as a new file.
If url isn't found when fetching anymore, delete it from the list.

I think this might be an X/Y problem and for that I'm still awaiting the answer to my comment.
With that said, you can simply ignore the existing files using fs.existsSync, if that returns true just skip saving the current file, otherwise save it. And to remove files that are not available anymore, just get all the files in the directory using fs.readdir and remove files that you whose urls are not in the response using fs.unlink:
needle.get(mainUrl, (err, res) => {
if (err) throw err;
if (res.statusCode == 200) {
let $ = cheerio.load(res.body);
let hashes = []; // list of hashes for this website (to be used later to keep only the items that are still available)
$('div div a').each((index, element) => {
let url = $(element).attr("href");
let hash = crypto.createHash('md5').update(url).digest('hex');
hashes.push(hash); // store the hash of the current url
if (!fs.existsSync('./directory/otherdirectory/' + hash)) { // if this file doesn't exist (notice the "not operator !" before fs.existsSync)
fs.writeFile('./directory/otherdirectory/' + hash, url, err => { // save it
if (err) throw err;
console.log('Hash created: ' + url + ' saved as ' + hash);
});
}
});
fs.readdir('./directory/otherdirectory', (err, files) => { // get a list of all the files in the directory
if (err) throw err;
files.forEach(file => { // and for each file
if(!hashes.includes(file)) { // if it was not encountered above (meaning that it doesn't exist in the hashes array)
fs.unlink('./directory/otherdirectory/' + file, err => { // remove it
if (err) throw err;
});
}
});
});
});
Another approach:
Since you only seem to want to store the urls, the best way to so would be to use one single file to store them all instead of storing each url in its own file. Something like this is more efficient:
needle.get(mainUrl, (err, res) => {
if (err) throw err;
if (res.statusCode == 200) {
let $ = cheerio.load(res.body);
let urls = $('div div a') // get the 'a' elements
.map((index, element) => $(element).attr("href")) // map each one into its href attribute
.get(); // and get them as an array
fs.writeFile('./directory/list-of-urls', urls.join('\n'), err => { // then save all the urls encountered in the file 'list-of-urls' (each on its own line, hence the join('\n'))
if (err) throw err;
console.log('saved all the urls to the file "list-of-urls"');
});
}
});
That way old urls will be removed automatically as the file gets overwritten each time, and new urls will be added automatically. No need to check whether an url is already encountered or not because it will get re-saved anyway.
And if you want to get the list of urls somewhere else, just read the file and split it by '\n' like so:
fs.readFile('./directory/list-of-urls', 'utf8', (err, data) => {
if (err) throw err;
let urls = data.split('\n');
// use urls here
});

How to get text from a constructor?

I'm stuck at using text that is created in a constructor outside the constructor.
Actually, I'm trying to read a pdf file using pdfreader node module. I'm able to print pdf data word by word as text in below constructor.
fs.readFile(pdfFilePath, (err, pdfBuffer) => {
// pdfBuffer contains the file content
new pdfreader.PdfReader().parseBuffer(pdfBuffer, function(err, item){
if (err)
callback(err);
else if (!item)
callback();
else if(item.text !== undefined)
console.log(item.text);
});
});
But I want to get that text into some string or array and I need to check whether a piece of text is there in the whole string or array.
I am not able to do this.

var a = '';
var pdfreader = require('pdfreader');
let pdfFilePath = 'C:/Users/Downloads/';
fs.readFile(pdfFilePath, (err, pdfBuffer) => {
new PdfReader().parseBuffer(pdfBuffer, function(err, item) {
if (err) callback(err);
else if (!item) callback();
else if(item.text !== undefined)
a = a.concat(item.text);
});
});
console.log('after block' + a)
Try this. I'm unsure if the item.text considers a space or not.

error in writing text to a file using node js fs?

I have a folder which is in my public folder called errorlogs.I want to append text to a file in that folder without overwriting the file. However, for some reason, I am unable to do that using the fs.appendfile. Please can someone suggest my error. Here is what I tried.It actually creates a file called errorlogsapikey.txt outside my public folder which is not what I want
var filepath = __dirname+'/errorlogs';
ensureExists(__dirname + '/errorlogs', 0744, function(err) {
if (err){
} // handle folder creation error
else {
console.log("we are all good");
var data = "url:" +" "+url+" "+"urlstatus:"+" "+urlstatus+" "+"time:"+ " "+formatted;
fs.appendFile(filepath + apikey+'.txt', data,function(err){
if(err) throw err;
});
}
});
function ensureExists(path, mask, cb) {
if (typeof mask == 'function') { // allow the `mask` parameter to be optional
cb = mask;
mask = 0777;
}
fs.mkdir(path, mask, function(err) {
if (err) {
if (err.code == 'EEXIST') cb(null); // ignore the error if the folder already exists
else cb(err); // something else went wrong
} else cb(null); // successfully created folder
});
}

How do i display the javascript error object?

I am using Node.js and Express web application framework and
mysql package from here
https://www.npmjs.com/package/mysql
here's my Parent.js file .. it's my model file
var db = require('../dbconnection');
var Parent = {
findIfParentMobileNumberExists: function (parentmobilenumber, callback) {
db.query('SELECT parentmobilenumber from parents where parentmobilenumber=?', parentmobilenumber, function (err, rows) {
if (err) {
callback(err, null);
}
if (rows.length < 1) {
console.log("rows length less than 1");
callback(err, null);
}
else if (rows.length !== 0) {
console.log("rows length greater than 1");
callback(null, rows[0].parentmobilenumber);
}
else {
}
});
}
module.exports = Parent;
and here's my registerParent.js file .. this is my route
var express = require('express');
var router = express.Router();
var Parent = require('../models/Parent');
router.post('/', function (req, res, next) {
var countrycode = req.body.countrycode;
var parentmobilenumber = (countrycode) + (req.body.inputMobileNumber);
//remove hyphens from mobile number
var parentmobilenumberwithouthyphens =
parentmobilenumber.replace(/-/g, "");
//remove spaces from mobile number
var parentmobilenumberwithouthyphensandspaces =
parentmobilenumberwithouthyphens.replace(/ /g, '');
Parent.findIfParentMobileNumberExists(parentmobilenumberwithouthyphensandspaces, function (err, parentmobilenumberfromdb) {
if (err) {
res.json({registerErrorMessage: err.message});
}
else {
if (parentmobilenumberwithouthyphensandspaces === parentmobilenumberfromdb) {
console.log(parentmobilenumberfromdb);
}
else {
res.json({registerErrorMessage: 'The Mobile Number does not exist in our database'});
}
}
});
});
if i enter a correct mobile number i get the mobile number displayed in a console.log message .. however when i enter an incorrect mobile number which is not in the database the resulting rows.length is < 1
i get the message
rows length less than 1
but i don't understand when this block runs
if (rows.length < 1) {
console.log("rows length less than 1");
callback(err, null);
}
the err object is never displayed .. i.e this block is never run in the router.post function
if (err){
res.json({registerErrorMessage: err.message});
}
so i want to display the err object correctly first ..
and then i want to display an error message that is readable by end-users
how can i do this ?
i tried reading the documentation here
https://github.com/mysqljs/mysql#error-handling
but i really don't understand how to properly execute callbacks.

You don't start http server with express.
Try to insert code below to start listening:
const app = express();
app.listen( insert port here )

At first, you might want to terminate your logic if an error occurs:
if(err){
return callback(err);
}
Now the error is passed to the client side, where you can catch it and display a message. E.g. (using jquery):
$.ajax({
url:"your/backend",
dataType: "json",
success(res){
if(res.code){
return alert("an error ("+res.code+") occured. Sorry :/");
}
//continue with valid response
}
});
However, you may change your API to be more general, e.g. always return an error property, thats either false or the error object:
on error:
res.json({error:err});
on success:
res.json({error:false,data:"whatever"});
So that you could do this on client side:
$.ajax({
url:"your/backend",
dataType: "json",
success(res){
if(res.error){
return alert("an error ("+res.error.code+") occured. Sorry :/");
}
//continue with valid response
alert(res.data);
}
});

Returning true or false inside functions, I am a bit confused

I have this function, for example:
app.get('/', function(req, res) {
var token = req.query.token;
if(!token) {
res.render('auth'); // Authentication
} else {
authorizedUsers.forEach(function(user) {
if(user.id == token) {
console.log('found, nickname: ' + user.nickname);
return true;
}
});
console.log('not found');
return false;
}
});
Basically it's looping through the authorizedUsers array, and looking for an entry where entry.id equals the token variable.
What I wanted to do is, if found, to return true and stop the execution of the rest of the app.get('/')... block.
If not found, obviously the forEach loop has already gone through all entries, and eventually the execution arrives to the printing of "not found" and return false;.
For my current code, even though an entry is found, execution continues and I still get the 'not found' log.
Am I missing anything?
To simplify things, what I wanna do is:
Looping through all authorizedUsers entries, comparing entry.id to the token variable.
If found, print "found" to the console and stop the execution.
If not found, print "not found" to the console and stop the
execution.
Thank you.
Edit
Following Michael's solution, this is my working code:
app.get('/', function(req, res) {
var token = req.query.token;
if(!token) {
res.render('auth'); // Authentication
} else {
if(!authorizedUsers.some(function(user) {
if(user.id == token)
return true;
})) {
console.log('No entries found.');
} else {
console.log('Entries found!');
}
}
});

You would use Array.prototype.some for this:
authorizedUsers.some(function(user) {
return user.id == token;
}
The some() method tests whether some element in the array passes the test implemented by the provided function.

The forEach function cannot be interrupted unless the code inside throws an exception.
So you could either do something like this
Or just let it run through all the records doing something like this:
var found = false;
authorizedUsers.forEach(function(user) {
if(user.id == token) found = true;
});
console.log('found? ', found);