I have a test folder with files
file
file (1)
file (2)
If the file exists I add a suffix to a new filename, to prevent overwriting the file. For example
if file exists new name should be file (1)
if file (1) exists new name should be file (2)
if file (2) exists new name should be file (3)
and so on.
The following function works fine, except the value is not returned so I can assign it later.
async function dest_exists_new_name(file) {
const access = fs.promises.access
try {
await access(file, fs.F_OK)
// file exists - generate new name
const info = path.parse(file)
const dir = info.dir
let name = info.name
const ext = info.ext
// generate suffix
let suffix = ' (1)'
const suffix_regex = / \([0-9]+\)$/
if (suffix_regex.test(name)) { // if suffix exists -> increment it
const num = name.split(' ').reverse()[0].replace(/[()]/g,'')
const next_num = parseInt(num) + 1
suffix = ' (' + next_num + ')'
name = name.replace(suffix_regex, '') // remove old suffix
}
// generate suffix end
const new_name = path.join(dir, name + suffix + ext)
// recurse until dest not exists
await dest_exists_new_name(new_name)
} catch {
// file not exist - return its name
// console.log works OK
console.log('new name ->', file)
// return doesn't work - returns undefined if the previous name exists, but works ok if the name doesn't exists
return file
}
}
await dest_exists_new_name('/path/file')
new name -> /path/file (3) // console.log - works OK
undefined // returns undefined, if file previously exists
The question is how can I correctly return the new file name value?
If there are any culprits in such a solution like
accidental file rewriting
infinite recursion
other issues
I will be grateful for the hints on how to improve the function.
Your function will return file, but being an async function, you need to await its return and you cannot do so outside of an async scope. Thus, if you just console.log() its "istantaneous" value, it will indeed return a pending promise, as the return value has not been resolved yet. You may retrieve the correct return value by including your function in an async scope, like this:
let a = async () => {
console.log(await dest_exists_new_name('/path/file'))
}
a();
This will output:
new name -> /path/file
/path/file //here's your file
Now, by adding return await dest_exists_new_name(new_name) you should be able to achive what you want and both console.log() and return the new, non-existent, file name. Here's a complete, reproducible example:
const fs = require('fs');
const path = require('path');
async function dest_exists_new_name(file) {
const access = fs.promises.access
try {
await access(file, fs.F_OK)
const info = path.parse(file)
const dir = info.dir
let name = info.name
const ext = info.ext
let suffix = ' (1)'
const suffix_regex = / \([0-9]+\)$/
if (suffix_regex.test(name)) {
const num = name.split(' ').reverse()[0].replace(/[()]/g, '')
const next_num = parseInt(num) + 1
suffix = ' (' + next_num + ')'
name = name.replace(suffix_regex, '')
}
const new_name = path.join(dir, name + suffix + ext)
return await dest_exists_new_name(new_name)
} catch {
console.log('new name ->', file)
return file
}
}
//Here, make sure that the path to "file" is correct
let a = async() => console.log(await dest_exists_new_name(path.join(__dirname, './file')));
a();
Output, having up to file (2) in the same folder:
new name -> /path/to/file (3)
/path/to/file (3)
Check you try catch and how you are receiving your variable.
async function dest_exists_new_name(file) {
try {
const res = await dest_exists_new_name(file1); // recursion result
} catch (err) {
return Promise.resolve("file not found");
}
}
// usage
let res = await dest_exists_new_name(fileArg);
First of all, you should use await, since it's async function:
// recurse until dest not exists
await dest_exists_new_name(new_name)
About recursion - IMHO, it's always better to use cycle (if it doesn't make code too complicated).
Mixing async-await & promises is not very good. Ideally you should use one style.
I prefer to use destructuring, lambda functions, and other modern features.
So, my variant for async-await, without recursion:
const fs = require('fs');
const path = require('path');
// better to create outside of func, since it doesn't depend on context
const suffix_regex = / \([0-9]+\)$/
const defaultSuffix = ' (1)'
const access = async (...params) => new Promise((resolve) => fs.access(...params, (err) => (err) ? resolve(false) : resolve(true)))
const generate_new_suffix = ({ dir, name, ext }) => {
if (suffix_regex.test(name)) { // if suffix exists -> increment it
const num = name.split(' ').reverse()[0].replace(/[()]/g, '')
const suffix = `(${+num + 1})`;
const cleanName = name.replace(suffix_regex, '') // remove old suffix
return path.join(dir, cleanName + suffix + ext)
}
return path.join(dir, name + defaultSuffix + ext)
}
const dest_exists_new_name = async (file) => {
let newNameGenerated = false
let newFileName = file
while (await access(newFileName, fs.F_OK)) {
console.log(newFileName);
const info = path.parse(newFileName)
newFileName = generate_new_suffix(info)
};
console.log('new name ->', newFileName)
return newFileName
}
(async () => {
console.log(await dest_exists_new_name(path.join(__dirname, 'file')))
})();
Related
I have a command file for a discord bot that contains the command and a piece of parsing logic contained within a function that I want to reuse within my index.js
// file: ./commands/scrumPrompt.js
// The function
const extractDeets = function (f, scrum) {
let items = [];
let re = new RegExp("(\n[ -]*" + f + ".*)", "g");
let replace = new RegExp("[ -]*" + f + "[ ]+");
for (const item of scrum.matchAll(re)) {
items.push(item[1].trim().replace(replace, ""));
}
return items;
};
// The actual command itself within the same file
module.exports = {
name: "scrum",
usage: `!scrum < followed by your message > as per Standup format - refer !show for showing the format`,
description: "Reply to standup prompt",
async execute(message, args) {
if (message.channel.type === "text") {
if (!args.length)
return message.reply(
"Please Provide your scrum as per the format in help menu !scrum < your message >"
);
else {
if (message.author.id !== -1) {
const client = new MongoClient(MONGO_URI);
try {
const database = client.db(DB_NAME);
const members = database.collection("members");
const query = { user_id: message.author.id };
const membersdetail = await members.findOne(query);
if (membersdetail !== null) {
// since this method returns the matched document, not a cursor, print it directly
//console.log("Adding Scrum for ", membersdetail.email);
let userscrum = args.splice(0).join(" ");
// Check if multiple !scrum commands are present in developer scrum message
if (userscrum.includes("!scrum") == false) {
// Expects notations of "-" to exist
let [f, e, b, o, bl] = ["f", "e", "b", "o", "x"];
let features = extractDeets(f, userscrum);
let enhancements = extractDeets(e, userscrum);
let bugs = extractDeets(b, userscrum);
let others = extractDeets(o, userscrum);
let blockers = extractDeets(bl, userscrum);
.
.
.
};
I want to keep the name of the function as extractDeets() itself so that it doesn't mess with the usage within the command as well. I'm not completely sure how to export it into the index.js because it's already kind of being imported here:
// Imports the command file + adds the command to the bot commands collection
for (const file of commandFiles) {
const command = require(`./commands/${file}`);
bot.commands.set(command.name, command);
}
I'm unsure of how to add the function as another import, maybe I should export it into another file and then import it from there? I'm not sure if that's possible or doable here. I've tried directly importing from here but then the command doesn't work, which is troublesome.
You can do it like this:
module.exports = { extractDeets };
Later, you can import it like this:
const { extractDeets } = require('../your_file');
I am trying to fetch data from different collections in my cloud Firestore database in advance before I process them and apply them to batch, I created two async functions, one to capture the data and another to execute certain code only after all data is collected, I didn't want the code executing and creating errors before the data is fetched when i try to access the matchesObject after the async function to collect data is finished, it keeps saying "it cannot access a property matchStatus of undefined", i thought took care of that with async and await? could anyone shed some light as to why it is undefined one moment
axios.request(options).then(function(response) {
console.log('Total matches count :' + response.data.matches.length);
const data = response.data;
var matchesSnapshot;
var marketsSnapshot;
var tradesSnapshot;
var betsSnapshot;
matchesObject = {};
marketsObject = {};
tradesObject = {};
betsObject = {};
start();
async function checkDatabase() {
matchesSnapshot = await db.collection('matches').get();
matchesSnapshot.forEach(doc => {
matchesObject[doc.id] = doc.data();
console.log('matches object: ' + doc.id.toString())
});
marketsSnapshot = await db.collection('markets').get();
marketsSnapshot.forEach(doc2 => {
marketsObject[doc2.id] = doc2.data();
console.log('markets object: ' + doc2.id.toString())
});
tradesSnapshot = await db.collection('trades').get();
tradesSnapshot.forEach(doc3 => {
tradesObject[doc3.id] = doc3.data();
console.log('trades object: ' + doc3.id.toString())
});
betsSnapshot = await db.collection('bets').get();
betsSnapshot.forEach(doc4 => {
betsObject[doc4.id] = doc4.data();
console.log('bets object: ' + doc4.id.toString())
});
}
async function start() {
await checkDatabase();
// this is the part which is undefined, it keeps saying it cant access property matchStatus of undefined
console.log('here is matches object ' + matchesObject['302283']['matchStatus']);
if (Object.keys(matchesObject).length != 0) {
for (let bets of Object.keys(betsObject)) {
if (matchesObject[betsObject[bets]['tradeMatchId']]['matchStatus'] == 'IN_PLAY' && betsObject[bets]['matched'] == false) {
var sfRef = db.collection('users').doc(betsObject[bets]['user']);
batch11.set(sfRef, {
accountBalance: admin.firestore.FieldValue + parseFloat(betsObject[bets]['stake']),
}, {
merge: true
});
var sfRef = db.collection('bets').doc(bets);
batch12.set(sfRef, {
tradeCancelled: true,
}, {
merge: true
});
}
}
}
});
There are too many smaller issues in the current code to try to debug them one-by-one, so this refactor introduces various tests against your data. It currently won't make any changes to your database and is meant to be a replacement for your start() function.
One of the main differences against your current code is that it doesn't unnecessarily download 4 collections worth of documents (two of them aren't even used in the code you've included).
Steps
First, it will get all the bet documents that have matched == false. From these documents, it will check if they have any syntax errors and report them to the console. For each valid bet document, the ID of it's linked match document will be grabbed so we can then fetch all the match documents we actually need. Then we queue up the changes to the user's balance and the bet's document. Finally we report about any changes to be done and commit them (once you uncomment the line).
Code
Note: fetchDocumentById() is defined in this gist. Its a helper function to allow someCollectionRef.where(FieldPath.documentId(), 'in', arrayOfIds) to take more than 10 IDs at once.
async function applyBalanceChanges() {
const betsCollectionRef = db.collection('bets');
const matchesCollectionRef = db.collection('matches');
const usersCollectionRef = db.collection('users');
const betDataMap = {}; // Record<string, BetData>
await betsCollectionRef
.where('matched', '==', false)
.get()
.then((betsSnapshot) => {
betsSnapshot.forEach(betDoc => {
betDataMap[betDoc.id] = betDoc.data();
});
});
const matchDataMap = {}; // Record<string, MatchData | undefined>
// betIdList contains all IDs that will be processed
const betIdList = Object.keys(betDataMap).filter(betId => {
const betData = betDataMap[betId];
if (!betData) {
console.log(`WARN: Skipped Bet #${betId} because it was falsy (actual value: ${betData})`);
return false;
}
const matchId = betData.tradeMatchId;
if (!matchId) {
console.log(`WARN: Skipped Bet #${betId} because it had a falsy match ID (actual value: ${matchId})`);
return false;
}
if (!betData.user) {
console.log(`WARN: Skipped Bet #${betId} because it had a falsy user ID (actual value: ${userId})`);
return false;
}
const stakeAsNumber = Number(betData.stake); // not using parseFloat as it's too lax
if (isNaN(stakeAsNumber)) {
console.log(`WARN: Skipped Bet #${betId} because it had an invalid stake value (original NaN value: ${betData.stake})`);
return false;
}
matchDataMap[matchId] = undefined; // using undefined because its the result of `doc.data()` when the document doesn't exist
return true;
});
await fetchDocumentsById(
matchesCollectionRef,
Object.keys(matchIdMap),
(matchDoc) => matchDataMap[matchDoc.id] = matchDoc.data()
);
const batch = db.batch();
const queuedUpdates = 0;
betIdList.forEach(betId => {
const betData = betDataMap[betId];
const matchData = matchDataMap[betData.tradeMatchId];
if (matchData === undefined) {
console.log(`WARN: Skipped /bets/${betId}, because it's linked match doesn't exist!`);
continue;
}
if (matchData.matchStatus !== 'IN_PLAY') {
console.log(`INFO: Skipped /bets/${betId}, because it's linked match status is not "IN_PLAY" (actual value: ${matchData.matchStatus})`);
continue;
}
const betRef = betsCollectionRef.doc(betId);
const betUserRef = usersCollectionRef.doc(betData.user);
batch.update(betUserRef, { accountBalance: admin.firestore.FieldValue.increment(Number(betData.stake)) });
batch.update(betRef, { tradeCancelled: true });
queuedUpdates += 2; // for logging
});
console.log(`INFO: Batch currently has ${queuedUpdates} queued`);
// only uncomment when you are ready to make changes
// batch.commit();
}
Usage:
axios.request(options)
.then(function(response) {
const data = response.data;
console.log('INFO: Total matches count from API:' + data.matches.length);
return applyBalanceChanges();
}
I try to download, unzip and process a GTFS file in zip format. Downloading and unzipping are working, but I get error message when try to use txt files with gtfs-utils module in gtfsFunc(). Output is undefined. Delays are hardcoded just for testing purpose.
const dl = new DownloaderHelper('http://www.bkk.hu/gtfs/budapest_gtfs.zip', __dirname);
dl.on('end', () => console.log('Download Completed'))
dl.start();
myVar = setTimeout(zipFunc, 30000);
function zipFunc() {
console.log('Unzipping started...');
var zip = new AdmZip("./budapest_gtfs.zip");
var zipEntries = zip.getEntries();
zip.extractAllTo("./gtfsdata/", true);
}
myVar = setTimeout(gtfsFunc, 40000);
function gtfsFunc() {
console.log('Processing started...');
const readFile = name => readCsv('./gtfsdata/' + name + '.txt')
const filter = t => t.route_id === 'M4'
readStops(readFile, filter)
.then((stops) => {
const someStopId = Object.keys(stops)[0]
const someStop = stops[someStopId]
console.log(someStop)
})
}
As #ChickenSoups said, you are trying to filter stops files with route_id field and this txt doesnt have this field.
The fields that stops has are:
stop_id, stop_name, stop_lat, stop_lon, stop_code, location_type, parent_station, wheelchair_boarding, stop_direction
Perhaps what you need is read the Trips.txt file instead Stops.txt, as this file has route_id field.
And you can accomplish this using readTrips function:
const readTrips = require("gtfs-utils/read-trips");
And your gtfsFunc would be:
function gtfsFunc() {
console.log("Processing started...");
const readFile = (name) => {
return readCsv("./gtfsdata/" + name + ".txt").on("error", console.error);
};
//I used 5200 because your Trips.txt contains routes id with this value
const filterTrips = (t) => t.route_id === "5200";
readTrips(readFile, filterTrips).then((stops) => {
console.log("filtered stops", stops);
const someStopId = Object.keys(stops)[0];
const someStop = stops[someStopId];
console.log("someStop", someStop);
});
}
Or if what you really want is to read Stops.txt, you just need to change your filter
const filter = t => t.route_id === 'M4'
to use some valid field, for example:
const filter = t => t.stop_name=== 'M4'
Stop data don't have route_id field.
You should try other data, such as Trip or Route
You can look at the first row in your data file to see which field do they have.
GTFS data structure here
I have a code that accepts a list of nested objects, each of which should be converted to a log row.
The code goes through a loop on each object, then an inner loop on each property, and extracts its property (there are hundreds of properties), then puts all the information of a row - as a map of the object's name and its value, into a variable called returnVar.
We use the 'fast-csv' library, with WriteStream that is named csvStream. also with a fs.createWriteStream pipe.
Finally, we loop over each object and write it with csvStream.write(),
that will insert the properties name in the first line of the file, and the logs (in the same order) in the other lines.
I need to change the code so that instead of doing pipe to file stream, it will print to a string type variable.
This is the code:
let Promise = require('bluebird');
let csv = require('fast-csv');
let fs = Promise.promisifyAll(require('fs'));
...
return new Promise(function (resolve, reject) {
var csvStream = csv.createWriteStream({ headers: propNames })
.transform(function (item) { // every item is a nested object that contains data for a log line
var returnVar = {}; // every returnVar will represents a map of property and value, that will be transform to a log line
for (var prop in item) {
if (item.hasOwnProperty(prop)) {
if (propNames.indexOf(prop) >= 0) {
if (typeof item[prop] === 'object') {
returnVar[prop] = JSON.stringify(item[prop]);
}
else {
returnVar[prop] = item[prop];
}
}
//the object might be a complex item that contains some properties that we want to export...
else if (typeof item[prop] === 'object') {
var nestedItem = item[prop];
for (var nestedProp in nestedItem) {
if (propNames.indexOf(prop + "_" + nestedProp) >= 0) {
returnVar[prop + "_" + nestedProp] = nestedItem[nestedProp];
}
}
}
}
}
return returnVar; // return log line
});
// create file path
var fileId = "Report_" + cryptoService.generateRandomPassword(16) + ".csv";
var filePath = tempPath + fileId;
getOrCreateTempDirectory().then(function () {
var writableStream = fs.createWriteStream(filePath);
writableStream.on("finish", function () {
resolve({
fileId: fileId
});
});
csvStream.pipe(writableStream);
_.each(results.records, function (result) {
// write line to file
csvStream.write(result._source);
});
csvStream.end();
});
});
https://c2fo.io/fast-csv/docs/formatting/methods#writetobuffer
https://c2fo.io/fast-csv/docs/formatting/methods#writetostring
Change
csvStream.write(result._source);
to
csvStream.writeToString(result._source).then(data => console.log(data));
Promise.all(_.map(results.records, result => csvStream.writeToString(result._source)))
.then(rows=>console.log(rows))
// rows should be an array of strings representing all the results
You can also use async/await
The code listed below searches for files that
contains a specified string under it's directory/subdirectories.
to activate it, you type node [jsfilename] [folder] [filename] [ext]
i would also like it to announce: Nothing found in a console.log every time that
a word wasn't found.
ive tried
if (!regex.test(fileContent)) {
console.log(`Noting found`);
it works only if you have one file without your word, but if not ,it loops.
for example if you have 4 files and one of them has the string it wil show
Your word was found in directory: [file]
Noting found
Noting found
Noting found.
So, how can i stop the loop after one !found console.log and how can i prevent it from showing in case of something has found?
const path = require('path');
const fs = require('fs');
function searchFilesInDirectory(dir, filter, ext) {
if (!fs.existsSync(dir)) {
console.log(`Specified directory: ${dir} does not exist`);
return;
}
const files = fs.readdirSync(dir);
const found = getFilesInDirectory(dir, ext);
found.forEach(file => {
const fileContent = fs.readFileSync(file);
const regex = new RegExp('\\b' + filter + '\\b');
if (regex.test(fileContent)) {
console.log(`Your word was found in directory: ${file}`);
}
});
}
function getFilesInDirectory(dir, ext) {
if (!fs.existsSync(dir)) {
console.log(`Specified directory: ${dir} does not exist`);
return;
}
let files = [];
fs.readdirSync(dir).forEach(file => {
const filePath = path.join(dir, file);
const stat = fs.lstatSync(filePath);
if (stat.isDirectory()) {
const nestedFiles = getFilesInDirectory(filePath, ext);
files = files.concat(nestedFiles);
} else {
if (path.extname(file) === ext) {
files.push(filePath);
}
}
});
return files;
}
searchFilesInDirectory(process.argv[2], process.argv[3], process.argv[4]);
Change:
if (!regex.test(fileContent)) {
console.log(`Noting found`);
// ...
to:
if (!printed && !regex.test(fileContent)) {
console.log(`Noting found`);
printed = true;
// ...
and make sure that you have a variable called printed defined in outer scope, originally falsy.