Develop Reference

Async and Sync Functions Node JS

I have a couple of functions I am running to download a zip file from Nexus, then unzip/extract the contents of the zip file, finally a search for a specific file type. All function work, however the synchronous search for some reason is not producing any results. If I simply run the download and extract functions in 1 script, then execute the search in another script I get my expected results. I am almost positive it is due to the search being synchronous whereas the download and extract are both async. Is there a quick way to add the find function at the end after the download & extract functions have run? Below is the code:
//npm modules
const fs = require('fs-extra');
const download = require('download');
const unzipper = require('unzipper');
const path = require('path');
//Custom Variables
const artifact = 'SOME_FILE.zip';
const repo = "SOME NEXUS REPOSITORY";
const url = "http://SOME URL/repository/";
const directory = 'SOME DIRECTORY';
//Get Artifact and Extract it to local directory function
const getArtifact = async () => {
const getArtifact = await download(url+repo, "./unzip")
const file = await fs.writeFileSync(directory+artifact, await download(url+repo))
const readStream = await fs.createReadStream(directory + artifact).pipe(unzipper.Extract({path:
directory}))
}
//Find function which should run after download and extract have been fulfilled
const findFile = function (dir, pattern) {
var results = [];
fs.readdirSync(dir).forEach(function (dirInner) {
dirInner = path.resolve(dir, dirInner);
var stat = fs.statSync(dirInner);
console.log(stat)
if (stat.isDirectory()) {
results = results.concat(findFile(dirInner, pattern));
}
if (stat.isFile() && dirInner.endsWith(pattern)) {
results.push(dirInner);
}
});
console.log(results)
return results;
};
//clear contents of directory before new download and extract
fs.emptyDirSync(directory)
//call download and extract function
getArtifact()
When I run "findFile" after the download & extract by itself in a separate script I get expected array output. However, when I try to incorporate (see below) this into the same script I get the an empty array:
getArtifact().then(function findFile (dir, pattern) {
var results = [];
fs.readdirSync(directory).forEach(function (dirInner) {
dirInner = path.resolve(directory, dirInner);
var stat = fs.statSync(dirInner);
console.log(stat)
if (stat.isDirectory()) {
results = results.concat(findFile(dirInner, pattern))
if (stat.isFile() && dirInner.endsWith(pattern)) {
results.push(dirInner);
}
}
console.log(results)
return results;
})
})
//Output
[]
//If I try the following:
getArtifact().then(findFile(directory, file))
// I get same empty array
[]
//If I run "findFile" in its own script after the download extract I get the following:
[
'SOME_FILE_PATH\\document1',
'SOME_FILE_PATH\\document2',
'SOME_FILE_PATH\\document3',
'SOME_FILE_PATH\\document4',
'SOME_FILE_PATH\\document5',
'SOME_FILE_PATH\\document6',
'SOME_FILE_PATH\\document7
]
Any help with how I can incorporate my findFile function into my existing download&extract function is appreciated...

How to delete a file using cloud function JavaScript firebase?

I wanted to delete the file from storage when a data node is deleted from the realtime database. the name of the file to be deleted is taken before deleted. The name of the file is saved in the node in child named "imageTitle". The code works fine before implementing the file delete code. I mean the nodes get deleted perfectly.
When I implement the file delete code the rest doesn't work, but there is no any errors. The code after file delete doesn't work. I dunno why.
This is for an academic final project.
There's a folder named images in the bucket, and the file I need to delete is in there. The file name is taken from the child in the node which is to be deleted in the realtime database named imageTitle:
'use strict';
const functions = require('firebase-functions');
const admin = require('firebase-admin');
admin.initializeApp();
exports.changetime = functions.database.ref('/posts/{pushId}')
.onCreate((snapshot, context) => {
const editDate = Date.now()
datas = snapshot.val();
return snapshot.ref.update({editDate})
})
const CUT_OFF_TIME = 1 * 60 * 1000;
/**
* This database triggered function will check for child nodes that are older than the
* cut-off time. Each child needs to have a `timestamp` attribute.
*/
exports.deleteOldItems = functions.database.ref('/posts/{pushId}').onWrite(async (change,
context) => {
const ref = change.after.ref.parent; // reference to the parent
const now = Date.now();
const id = context.params.pushId;
const cutoff = now - CUT_OFF_TIME;
const oldItemsQuery = ref.orderByChild('createdDate').endAt(cutoff);
const snapshot = await oldItemsQuery.once('value');
const getImageTitle = admin.database().ref(`/posts/${id}/imageTitle`).once('value');
return getImageTitle.then(imageTitle => {
console.log('post author id', imageTitle.val());
const imageName = imageTitle.val();
const filePath = 'images/' + imageName;
const path = `images/${imageName}`;
const bucket = app.storage().bucket();
return bucket.file(path).delete().then(() =>{
console.log(`File deleted successfully in path: ${imageName}`)
/* The problem is here. the code doesn't work after file.delete function. no errors. but doesn't work
if I remove that piece of code the other codes work fine. I mean the realtime database nodes get deleted and updated perfectly */
const updates = {};
snapshot.forEach(child => {
updates[child.key] = null;
if(updates[child.key] == null){
admin.database().ref(/post-likes/+id).remove();
}
})
return ref.update(updates);
});
});
[my storage looks like this][1]});
There's a folder named images in the bucket, and the file I need to delete is in there. The file name is taken from the child in the node which id to be deleted in the realtime database imageTitle:
enter image description here

I think this is the problem:
const filePath = 'images/' + imageName;
const path = `images/${imageName}`;
You should be using filePath not path in your bucket reference. const path = `images/${imageName}`; is wild-card syntax used when querying, not when assigning variables....think you stayed in 'Dart' mode here:-). Not sure what path contains, therefore, but console.log it to see.
Here is some code that I use to delete images from my storage bucket that works perfectly. First, a couple of things to check (best done using console.log, never assume you know what is in the variable), and do:
Ensure imageName, filePath, path and bucket contain what they should contain.
Include the .catch block in your code to check if there actually is an error
I am not clear if console.log(`File deleted successfully in path: ${imageName}`) is executing but if it isn't then your file.delete must be throwing an error which the catch block should trap.
The code snippet:
const oldProfileImagePath = "profileImages/" + authenticatedUserId + "/" + oldProfileImageName;
return bucket.file(oldProfileImagePath).delete()
.then(function () {
console.info("Old Profile Image: " + oldProfileImagePath + " deleted!");
})
.catch(function (error) {
console.error("Remove Old Profile Image: " + oldProfileImagePath +
" failed with " + error.message)
});

Dir walking to get abs path array of various files

Goal: Get a List of Absolute Paths for all files in a directory recursively leveraging NodeJs.
Info: As a python dev, I normally use the python packages which handle this in a platform independent fashion. My boss wanted some javascript code which would handle this goal... and as a JS dev previously, I was like "oh this is easy. Let's look up the node as I never got a chance to get my hands dirty with it." but I seem to be mistaken.
I don't see anything in node relating to Dir Walking, or a way I could hack together to create such a thing.
I was looking in "Child Process", "Console", "File System", "OS", "Path", and "Process". I didn't see anything which would do anything akin to:
pushd .
cd $dir
for folder in $(ls);
do
pushd .
cd $folder
//call again
ls $(pwd)$flag >> $dir/files_matching.txt
popd
done;
// or any platform independent means of recursively getting
// all files and their abs path which meet flag criterion,
// such as "*.txt" || "_*found*"
I could use child process to carry out Command Line items, but then I need to create a bunch of conditionals based on the OS consuming the app, and figured this would be something which already exists.
I don't want to reinvent the wheel, but figured this has already been done; I just don't see it in the base modules.
Is there a node module I would need which accomplishes this, which is outside of the base modules?
I am trying not to have to hand roll a conditional os based system to get an exhaustive list of abs paths for all files under a directory (or subset due to extensions, etc.)

I'd do it like this:
synchronous:
const fs = require("fs");
const { resolve } = require("path");
const getFiles = dir => {
const stack = [resolve(dir)];
const files = [];
while (stack.length) {
dir = stack.pop();
fs.readdirSync(dir).forEach(item => {
const path = resolve(dir, item);
(fs.statSync(path).isDirectory() ? stack : files).push(path);
});
}
return files;
};
console.log(getFiles("."));
asynchronous:
const fs = require("fs");
const { resolve } = require("path");
const pify = require("util").promisify;
const readdir = pify(fs.readdir);
const stat = pify(fs.stat);
const getFiles = async dir => {
const files = await readdir(resolve(dir));
const filesP = files.map(async file => {
const path = resolve(dir, file);
return (await stat(path)).isDirectory() ? getFiles(path) : path;
});
// return (await Promise.all(filesP)).flat(); // flat supported in node ~11
return [].concat(...(await Promise.all(filesP)));
};
getFiles(".").then(console.log);
async demo https://repl.it/#marzelin/getFiles

So, I was looking at the filesystem module and noticed the function readDir
https://nodejs.org/dist/latest-v8.x/docs/api/fs.html#fs_fs_readdir_path_options_callback
which does the trick in part. I guess it wasnt named a method i would have looking for. I was looking for things involving LIST and DIR, but not READ.
Anyways, here is a way to read Dir.
var fs = require('fs');
if (process.argv.length <= 2) {
console.log("Usage: " + __filename + " path/to/directory");
process.exit(-1);
}
var path = process.argv[2];
fs.readdir(path, function(err, items) {
console.log(items);
for (var i=0; i<items.length; i++) {
console.log(items[i]);
}
});
You notice that this one above is Async, but there is a Sync variant, just add "Sync" to the signature. Now you need to determine if something is a directory:
let file = fs.statSync("path/to/directory")
let isDir = file.isDirectory()
So you can couple this all together.
var fs = require('fs')
function recurse_file_system(path, contains) {
let files = fs.readdirSync(path);
let dArr = [];
let fArr = [];
for (let i in files){
let newPath = path + "/" + files[i]
if (fs.statSync(newPath).isDirectory()){
dArr.push(newPath)
}else{
if (filter(files[i], ".txt")){
fArr.push(newPath)
}
}
}
if (arr.length == 0){
return fArr;
}else{
for (let d in dArr){
let rslt = recurse_file_system(dArr[d]);
for (let i in rslt){
fArr.push(rslt[i])
}
}
return fArr;
}
}
console.log("Files:")
console.log(recurse_file_system("/"))
Now if you want to extend this, all you need to do is add a filter to say, limit the size of returns based on particular criterion, such as file name limitation.
function filter(filename, contains){
let reg = new RegEx(contains)
return reg.test(filename)
}
and you can add it to the base case, where you see filter... OR you can just return the WHOLE set and filter it there with the List method, filter.

get one file from each directory

in my app user can have multiple directory in setting , I want to get one file for each folder, note these directories can have their own directories an nested as possible
my config.json
{"dirs":["F:/music/Ellie Goulding","F:/music/Eminem - Revival [Album] [iTunes Version] - [7tunes]"]}
var findSongs = async function (baseDir,isDir=false,musics) {
try{
musics =musics || []
var files = await getDirFiles(baseDir)
for(let i =0;i<files.length;i++){
try {
let file = files[i]
var musicPath = `${baseDir}/${file}`
let stat = fs.lstatSync(musicPath)
if(stat.isDirectory()){
await findSongs(musicPath,true,musics)
}else{
let meta = await getMusicMeta(musicPath)
if(meta){
musics.push(meta)
}
if(isDir){
break
}
}
issue with this code is it goes get all songs of second folder and doesn't return anything from folder one but if only place one folder like this : F:/music it will work as expected

javascript: Creating directories from file path recursively adding timestamp to the filename

I am using the below code to split up a user provided path, create all intermediate dirs in the path and attach a timestamp to the ending file name. I am splitting the path with / first and then using forEach over the resulting array. Is there any better way/direct lib function to do this in javascript / nodejs.
function test(usrPath) {
var locMatches = usrPath.split("/")
locMatches.forEach (function (item) {
location = pathUtils.join(location,item)
if (!fs.existsSync(location)) {
fs.mkdirSync(location)
}
})
return pathUtils.join (location,usrPath + (new Date).toISOString().replace(/(^\d\d\d\d)|-|:|(\..*)/g,"").replace("T","_")+".log")
}

Ok, so there are path utils that allow to make the implementation better across platform.
Also, it gives a better managed access for working with path elements like root, dir, filename and extension. pathUtils.sep allows working on the dir elements more cross platform.
var pathUtils = require('path')
function test(usrPath) {
var pathElements = pathUtils.parse(usrPath)
pathElements.dir.split(pathUtils.sep).forEach (function (item) {
location = pathUtils.join(location,item)
if (!fs.existsSync(location)) {
fs.mkdirSync(location)
}
})
return pathUtils.join (location,pathElements.name + (new Date).toISOString().replace(/(^\d\d\d\d)|-|:|(\..*)/g,"").replace("T","_")+pathElements.ext.replace(/^$/i,".log"))
}