I'm currently working on a backup script with NodeJS. The script downloads a directory and its files und subdirectories recursively using FTP/FTPS. I'm using the basic-ftp package to do the FTP calls.
When I try to download a big directory with a lot of subdirectories, I get the Maximum call stack size exceeded error, but I don't find why and where it happens. I don't see any infinity loop or any missing return calls. After hours of debugging, I have no more ideas.
I don't use the downloadDirTo method from basic-ftp, because I don't want to stop downloading after a error happend. When an error occures it should keep going and it should add the error to the log file.
The repository is here: https://github.com/julianpoemp/webspace-backup.
As soon as the FTPManager is ready, I call the doBackup method (see method in BackupManager). This method calls the downloadFolder method defined in FTPManager.
export class BackupManager {
private ftpManager: FtpManager;
constructor() {
osLocale().then((locale) => {
ConsoleOutput.info(`locale is ${locale}`);
moment.locale(locale);
}).catch((error) => {
ConsoleOutput.error(error);
});
this.ftpManager = new FtpManager(AppSettings.settings.backup.root, {
host: AppSettings.settings.server.host,
port: AppSettings.settings.server.port,
user: AppSettings.settings.server.user,
password: AppSettings.settings.server.password,
pasvTimeout: AppSettings.settings.server.pasvTimeout
});
this.ftpManager.afterManagerIsReady().then(() => {
this.doBackup();
}).catch((error) => {
ConsoleOutput.error(error);
});
}
public doBackup() {
let errors = '';
if (fs.existsSync(path.join(AppSettings.appPath, 'errors.log'))) {
fs.unlinkSync(path.join(AppSettings.appPath, 'errors.log'));
}
if (fs.existsSync(path.join(AppSettings.appPath, 'statistics.txt'))) {
fs.unlinkSync(path.join(AppSettings.appPath, 'statistics.txt'));
}
const subscr = this.ftpManager.error.subscribe((message: string) => {
ConsoleOutput.error(`${moment().format('L LTS')}: ${message}`);
const line = `${moment().format('L LTS')}:\t${message}\n`;
errors += line;
fs.appendFile(path.join(AppSettings.appPath, 'errors.log'), line, {
encoding: 'Utf8'
}, () => {
});
});
let name = AppSettings.settings.backup.root.substring(0, AppSettings.settings.backup.root.lastIndexOf('/'));
name = name.substring(name.lastIndexOf('/') + 1);
const downloadPath = (AppSettings.settings.backup.downloadPath === '') ? AppSettings.appPath : AppSettings.settings.backup.downloadPath;
ConsoleOutput.info(`Remote path: ${AppSettings.settings.backup.root}\nDownload path: ${downloadPath}\n`);
this.ftpManager.statistics.started = Date.now();
this.ftpManager.downloadFolder(AppSettings.settings.backup.root, path.join(downloadPath, name)).then(() => {
this.ftpManager.statistics.ended = Date.now();
this.ftpManager.statistics.duration = (this.ftpManager.statistics.ended - this.ftpManager.statistics.started) / 1000 / 60;
ConsoleOutput.success('Backup finished!');
const statistics = `\n-- Statistics: --
Started: ${moment(this.ftpManager.statistics.started).format('L LTS')}
Ended: ${moment(this.ftpManager.statistics.ended).format('L LTS')}
Duration: ${this.ftpManager.getTimeString(this.ftpManager.statistics.duration * 60 * 1000)} (H:m:s)
Folders: ${this.ftpManager.statistics.folders}
Files: ${this.ftpManager.statistics.files}
Errors: ${errors.split('\n').length - 1}`;
ConsoleOutput.log('\n' + statistics);
fs.writeFileSync(path.join(AppSettings.appPath, 'statistics.txt'), statistics, {
encoding: 'utf-8'
});
if (errors !== '') {
ConsoleOutput.error(`There are errors. Please read the errors.log file for further information.`);
}
subscr.unsubscribe();
this.ftpManager.close();
}).catch((error) => {
ConsoleOutput.error(error);
this.ftpManager.close();
});
}
}
import * as ftp from 'basic-ftp';
import {FileInfo} from 'basic-ftp';
import * as Path from 'path';
import * as fs from 'fs';
import {Subject} from 'rxjs';
import {FtpEntry, FTPFolder} from './ftp-entry';
import {ConsoleOutput} from './ConsoleOutput';
import moment = require('moment');
export class FtpManager {
private isReady = false;
private _client: ftp.Client;
private currentDirectory = '';
public readyChange: Subject<boolean>;
public error: Subject<string>;
private connectionOptions: FTPConnectionOptions;
public statistics = {
folders: 0,
files: 0,
started: 0,
ended: 0,
duration: 0
};
private recursives = 0;
constructor(path: string, options: FTPConnectionOptions) {
this._client = new ftp.Client();
this._client.ftp.verbose = false;
this.readyChange = new Subject<boolean>();
this.error = new Subject<string>();
this.currentDirectory = path;
this.connectionOptions = options;
this.connect().then(() => {
this.isReady = true;
this.gotTo(path).then(() => {
this.onReady();
}).catch((error) => {
ConsoleOutput.error('ERROR: ' + error);
this.onConnectionFailed();
});
});
}
private connect(): Promise<void> {
return new Promise<void>((resolve, reject) => {
this._client.access({
host: this.connectionOptions.host,
user: this.connectionOptions.user,
password: this.connectionOptions.password,
secure: true
}).then(() => {
resolve();
}).catch((error) => {
reject(error);
});
});
}
private onReady = () => {
this.isReady = true;
this.readyChange.next(true);
};
private onConnectionFailed() {
this.isReady = false;
this.readyChange.next(false);
}
public close() {
this._client.close();
}
public async gotTo(path: string) {
return new Promise<void>((resolve, reject) => {
if (this.isReady) {
ConsoleOutput.info(`open ${path}`);
this._client.cd(path).then(() => {
this._client.pwd().then((dir) => {
this.currentDirectory = dir;
resolve();
}).catch((error) => {
reject(error);
});
}).catch((error) => {
reject(error);
});
} else {
reject(`FTPManager is not ready. gotTo ${path}`);
}
});
}
public async listEntries(path: string): Promise<FileInfo[]> {
if (this.isReady) {
return this._client.list(path);
} else {
throw new Error('FtpManager is not ready. list entries');
}
}
public afterManagerIsReady(): Promise<void> {
return new Promise<void>((resolve, reject) => {
if (this.isReady) {
resolve();
} else {
this.readyChange.subscribe(() => {
resolve();
},
(error) => {
reject(error);
},
() => {
});
}
});
}
public async downloadFolder(remotePath: string, downloadPath: string) {
this.recursives++;
if (this.recursives % 100 === 99) {
ConsoleOutput.info('WAIT');
await this.wait(0);
}
if (!fs.existsSync(downloadPath)) {
fs.mkdirSync(downloadPath);
}
try {
const list = await this.listEntries(remotePath);
for (const fileInfo of list) {
if (fileInfo.isDirectory) {
const folderPath = remotePath + fileInfo.name + '/';
try {
await this.downloadFolder(folderPath, Path.join(downloadPath, fileInfo.name));
this.statistics.folders++;
ConsoleOutput.success(`${this.getCurrentTimeString()}===> Directory downloaded: ${remotePath}\n`);
} catch (e) {
this.error.next(e);
}
} else if (fileInfo.isFile) {
try {
const filePath = remotePath + fileInfo.name;
if (this.recursives % 100 === 99) {
ConsoleOutput.info('WAIT');
await this.wait(0);
}
await this.downloadFile(filePath, downloadPath, fileInfo);
} catch (e) {
this.error.next(e);
}
}
}
return true;
} catch (e) {
this.error.next(e);
return true;
}
}
public async downloadFile(path: string, downloadPath: string, fileInfo: FileInfo) {
this.recursives++;
if (fs.existsSync(downloadPath)) {
const handler = (info) => {
let procent = Math.round((info.bytes / fileInfo.size) * 10000) / 100;
if (isNaN(procent)) {
procent = 0;
}
let procentStr = '';
if (procent < 10) {
procentStr = '__';
} else if (procent < 100) {
procentStr = '_';
}
procentStr += procent.toFixed(2);
ConsoleOutput.log(`${this.getCurrentTimeString()}---> ${info.type} (${procentStr}%): ${info.name}`);
};
if (this._client.closed) {
try {
await this.connect();
} catch (e) {
throw new Error(e);
}
}
this._client.trackProgress(handler);
try {
await this._client.downloadTo(Path.join(downloadPath, fileInfo.name), path);
this._client.trackProgress(undefined);
this.statistics.files++;
return true;
} catch (e) {
throw new Error(e);
}
} else {
throw new Error('downloadPath does not exist');
}
}
public chmod(path: string, permission: string): Promise<void> {
return new Promise<void>((resolve, reject) => {
this._client.send(`SITE CHMOD ${permission} ${path}`).then(() => {
console.log(`changed chmod of ${path} to ${permission}`);
resolve();
}).catch((error) => {
reject(error);
});
});
}
public getCurrentTimeString(): string {
const duration = Date.now() - this.statistics.started;
return moment().format('L LTS') + ' | Duration: ' + this.getTimeString(duration) + ' ';
}
public getTimeString(timespan: number) {
if (timespan < 0) {
timespan = 0;
}
let result = '';
const minutes: string = this.formatNumber(this.getMinutes(timespan), 2);
const seconds: string = this.formatNumber(this.getSeconds(timespan), 2);
const hours: string = this.formatNumber(this.getHours(timespan), 2);
result += hours + ':' + minutes + ':' + seconds;
return result;
}
private formatNumber = (num, length): string => {
let result = '' + num.toFixed(0);
while (result.length < length) {
result = '0' + result;
}
return result;
};
private getSeconds(timespan: number): number {
return Math.floor(timespan / 1000) % 60;
}
private getMinutes(timespan: number): number {
return Math.floor(timespan / 1000 / 60) % 60;
}
private getHours(timespan: number): number {
return Math.floor(timespan / 1000 / 60 / 60);
}
public async wait(time: number): Promise<void> {
return new Promise<void>((resolve) => {
setTimeout(() => {
resolve();
}, time);
});
}
}
export interface FTPConnectionOptions {
host: string;
port: number;
user: string;
password: string;
pasvTimeout: number;
}
Problem
Inside the FtpManager.downloadFolder function, I see recursive calls to the same downloadFolder method with an await. Your Maximum call stack exceeded error could come from there, since your initial call will need to keep everything in memory while traversing all subdirectories.
Proposed solution
Instead of awaiting everything recursively, you could setup a queue system, with an algorithm like this:
Add the current folder to a queue
While that queue is not empty:
Get the first folder in the queue (and remove it from it)
List all entries in it
Download all files
Add all subfolders to the queue
This allows you to download a lot of folders in a loop, instead of using recursion. Each loop iteration will run independently, meaning that the result of the root directory download won't depend on the deeeeeep file tree inside it.
Using a queue manager
There are plenty of queue manager modules for NodeJS, which allow you to have concurrency, timeouts, etc. One I've used in the past is simply named queue. It has a lot of useful features, but will require a little more work to implement in your project. Hence, for this answer, I used no external queue module, so that you can see the logic behind it. Feel free to search for queue, job, concurrency...
Example
I wanted to implement that logic directly into your own code, but I don't use Typescript, so I thought I'd make a simple folder copy function, which uses the same logic.
Note: For simplicity, I've not added any error handling, this is just a proof of concept! You can find a demo project which uses this here on my Github.
Here is how I've done it:
const fs = require('fs-extra');
const Path = require('path');
class CopyManager {
constructor() {
// Create a queue accessible by all methods
this.folderQueue = [];
}
/**
* Copies a directory
* #param {String} remotePath
* #param {String} downloadPath
*/
async copyFolder(remotePath, downloadPath) {
// Add the folder to the queue
this.folderQueue.push({ remotePath, downloadPath });
// While the queue contains folders to download
while (this.folderQueue.length > 0) {
// Download them
const { remotePath, downloadPath } = this.folderQueue.shift();
console.log(`Copy directory: ${remotePath} to ${downloadPath}`);
await this._copyFolderAux(remotePath, downloadPath);
}
}
/**
* Private internal method which copies the files from a folder,
* but if it finds subfolders, simply adds them to the folderQueue
* #param {String} remotePath
* #param {String} downloadPath
*/
async _copyFolderAux(remotePath, downloadPath) {
await fs.mkdir(downloadPath);
const list = await this.listEntries(remotePath);
for (const fileInfo of list) {
if (fileInfo.isDirectory) {
const folderPath = Path.join(remotePath, fileInfo.name);
const targetPath = Path.join(downloadPath, fileInfo.name);
// Push the folder to the queue
this.folderQueue.push({ remotePath: folderPath, downloadPath: targetPath });
} else if (fileInfo.isFile) {
const filePath = Path.join(remotePath, fileInfo.name);
await this.copyFile(filePath, downloadPath, fileInfo);
}
}
}
/**
* Copies a file
* #param {String} filePath
* #param {String} downloadPath
* #param {Object} fileInfo
*/
async copyFile(filePath, downloadPath, fileInfo) {
const targetPath = Path.join(downloadPath, fileInfo.name);
console.log(`Copy file: ${filePath} to ${targetPath}`);
return await fs.copy(filePath, targetPath);
}
/**
* Lists entries from a folder
* #param {String} remotePath
*/
async listEntries(remotePath) {
const fileNames = await fs.readdir(remotePath);
return Promise.all(
fileNames.map(async name => {
const stats = await fs.lstat(Path.join(remotePath, name));
return {
name,
isDirectory: stats.isDirectory(),
isFile: stats.isFile()
};
})
);
}
}
module.exports = CopyManager;
I found the source of the problem. It's the pkg package that emits the maximum callstack exceeded error: www.github.com/zeit/pkg/issues/681.
When I test it directly using node on windows, it work's. I will either downgrade to Node 10 or looking for another solution.
Thanks #blex for the help!
Related
am trying to build a web scraper that downloads all the pdfs in a website. i've written all the logic necessary to do this but for some reason it downloads an empty pdf file which is not suppose to be so, the problem seems to be coming from the downloadFile function when i try to pipe the data which for some reason seems not to be working because i get an empty pdf file after the function is ran. i'll would appreciate it if someone can help me out with this problem, thanks.
here's a sample of my code:
app.js
const fs = require("fs");
const path = require("path");
const cheerio = require("cheerio");
const axiosInstance = require("./getAxios");
const axios = axiosInstance();
const Surl = "https://www.health.gov.ng/";
// linkList sample: "https://www.health.gov.ng/index.php?option=com_content&view=article&id=143&Itemid=512";
let = connectionFailCount = 0;
let linkList = [];
let dlinkList = [];
const getWebsiteLinks = async (Surl) => {
try {
console.log(`Crawling all links from: ${Surl}`);
const response = await axios.get(Surl);
const $ = cheerio.load(response.data);
const ranges = $("a").each(function (idx, el) {
if ($(el).attr("href")) {
return $(el).attr("href");
}
});
for (let index = 0; index < ranges.length; index++) {
let raw_links = $("a")[index].attribs.href;
if (raw_links.startsWith("/")) {
linkList.push(Surl + raw_links);
}
}
if (linkList.length > 0) {
console.log(`Finished crawling links: Found ${linkList.length} links`);
console.log(
"--------------------------------------------------------\n\n"
);
}
return;
} catch (error) {
if (connectionFailCount === 0) {
connectionFailCount += 1;
getWebsiteLinks(Surl);
console.log(`Connection error. \n
Reconnecting to server....`);
} else if (connectionFailCount === 5) {
console.error(`Can not connect to server. Try again later.`);
}
}
};
const downloadLinks = async (linkList) => {
try {
console.log("Crawling links to find pdf links. this may take a while...");
for (const link of linkList) {
const response = await axios.get(link);
// Skip where there's delayed server response
if (response.code === "ECONNRESET") continue;
const $ = cheerio.load(response.data);
$("a").each(function (idx, el) {
if ($(el)?.attr("href")?.endsWith(".pdf")) {
let addr = $(el).attr("href");
let dlink = Surl + addr;
dlinkList.push({
pathName: addr,
url: dlink,
});
}
});
}
console.log(dlinkList);
if (dlinkList.length > 0) {
console.log(`Crawling Finish: Found ${dlinkList.length} pdf links`);
console.log(
"--------------------------------------------------------\n\n"
);
}
} catch (error) {
if (connectionFailCount === 0) {
connectionFailCount += 1;
console.log(`Connection error. \n
Reconnecting to server: ${connectionFailCount} count`);
downloadLinks(linkList);
}
if (connectionFailCount === 3) {
console.error(`Can not connect to server. Try again later.`);
return;
}
// console.error("downloadLinksError: ", error);
}
};
const downloadFiles = async (dlinkList) => {
console.log("Creating directory to save PDF files");
const appRoot = path.dirname(path.resolve(__dirname));
// Had to change and restructure code due to error
const folderName = `PDF/${Surl.split("/").pop()}`;
const subFolderName = Surl.split("/").pop();
try {
if (!fs.existsSync(path.join(appRoot, folderName))) {
fs.mkdirSync(path.join(appRoot, "PDF"));
fs.mkdirSync(path.join(`${appRoot}/PDF`, subFolderName));
}
dlinkList.forEach(async (link) => {
let name = link.pathName;
let url = link.url;
let file = fs
.createWriteStream(
`${appRoot}/${folderName}/${name.split("/").pop()}`,
"utf-8"
)
.on("error", (err) => {
console.error("createWriteStreamError: ", err);
});
try {
console.log("Downloading PDF file...");
const { data } = await axios({
url,
method: "GET",
responseType: "stream",
});
if (data) {
console.log("PDF file Downloaded");
data.pipe(file);
}
} catch (error) {
console.error(error);
}
});
return;
} catch (error) {
console.error("downloadFilesError: ", error);
}
};
(async () => {
await getWebsiteLinks(Surl);
await downloadLinks(linkList);
await downloadFiles(dlinkList);
})();
getAxios.js
const axios = require("axios");
const https = require("https");
module.exports = function () {
const domain = "https://www.health.gov.ng/";
let instance;
if (!instance) {
//create axios instance
instance = axios.create({
baseURL: domain,
timeout: 60000, // Increase time out incase of network delay or delayed server response
maxContentLength: 500 * 1000 * 1000, // Increase maximum response ata length
httpsAgent: new https.Agent({ keepAlive: true }),
headers: { "Content-Type": "application/xml" },
});
}
return instance;
};
Background
Javascript library for Microsoft Office add-ins allows you to get raw content of the DOCX file through getFileAsync() api, which returns a slice of up to 4MB in one go. You keep calling the function using a sliding window approach till you have reed entire content. I need to upload these slices to the server and the join them back to recreate the original DOCX file.
My attempt
I'm using axios on the client-side and busboy-based express-chunked-file-upload middleware on my node server. As I call getFileAsync recursively, I get a raw array of bytes that I then convert to a Blob and append to FormData before posting it to the node server. The entire thing works and I get the slice on the server. However, the chunk that gets written to the disk on the server is much larger than the blob I uploaded, normally of the order of 3 times, so it is obviously not getting what I sent.
My suspicion is that this may have to do with stream encoding, but the node middleware does not expose any options to set encoding.
Here is the current state of code:
Client-side
public sendActiveDocument(uploadAs: string, sliceSize: number): Promise<boolean> {
return new Promise<boolean>((resolve) => {
Office.context.document.getFileAsync(Office.FileType.Compressed,
{ sliceSize: sliceSize },
async (result) => {
if (result.status == Office.AsyncResultStatus.Succeeded) {
// Get the File object from the result.
const myFile = result.value;
const state = {
file: myFile,
filename: uploadAs,
counter: 0,
sliceCount: myFile.sliceCount,
chunkSize: sliceSize
} as getFileState;
console.log("Getting file of " + myFile.size + " bytes");
const hash = makeId(12)
this.getSlice(state, hash).then(resolve(true))
} else {
resolve(false)
}
})
})
}
private async getSlice(state: getFileState, fileHash: string): Promise<boolean> {
const result = await this.getSliceAsyncPromise(state.file, state.counter)
if (result.status == Office.AsyncResultStatus.Succeeded) {
const data = result.value.data;
if (data) {
const formData = new FormData();
formData.append("file", new Blob([data]), state.filename);
const boundary = makeId(12);
const start = state.counter * state.chunkSize
const end = (state.counter + 1) * state.chunkSize
const total = state.file.size
return await Axios.post('/upload', formData, {
headers: {
"Content-Type": `multipart/form-data; boundary=${boundary}`,
"file-chunk-id": fileHash,
"file-chunk-size": state.chunkSize,
"Content-Range": 'bytes ' + start + '-' + end + '/' + total,
},
}).then(async res => {
if (res.status === 200) {
state.counter++;
if (state.counter < state.sliceCount) {
return await this.getSlice(state, fileHash);
}
else {
this.closeFile(state);
return true
}
}
else {
return false
}
}).catch(err => {
console.log(err)
this.closeFile(state)
return false
})
} else {
return false
}
}
else {
console.log(result.status);
return false
}
}
private getSliceAsyncPromise(file: Office.File, sliceNumber: number): Promise<Office.AsyncResult<Office.Slice>> {
return new Promise(function (resolve) {
file.getSliceAsync(sliceNumber, result => resolve(result))
})
}
Server-side
This code is totally from the npm package (link above), so I'm not supposed to change anything in here, but still for reference:
makeMiddleware = () => {
return (req, res, next) => {
const busboy = new Busboy({ headers: req.headers });
busboy.on('file', (fieldName, file, filename, _0, _1) => {
if (this.fileField !== fieldName) { // Current field is not handled.
return next();
}
const chunkSize = req.headers[this.chunkSizeHeader] || 500000; // Default: 500Kb.
const chunkId = req.headers[this.chunkIdHeader] || 'unique-file-id'; // If not specified, will reuse same chunk id.
// NOTE: Using the same chunk id for multiple file uploads in parallel will corrupt the result.
const contentRangeHeader = req.headers['content-range'];
let contentRange;
const errorMessage = util.format(
'Invalid Content-Range header: %s', contentRangeHeader
);
try {
contentRange = parse(contentRangeHeader);
} catch (err) {
return next(new Error(errorMessage));
}
if (!contentRange) {
return next(new Error(errorMessage));
}
const part = contentRange.start / chunkSize;
const partFilename = util.format('%i.part', part);
const tmpDir = util.format('/tmp/%s', chunkId);
this._makeSureDirExists(tmpDir);
const partPath = path.join(tmpDir, partFilename);
const writableStream = fs.createWriteStream(partPath);
file.pipe(writableStream);
file.on('end', () => {
req.filePart = part;
if (this._isLastPart(contentRange)) {
req.isLastPart = true;
this._buildOriginalFile(chunkId, chunkSize, contentRange, filename).then(() => {
next();
}).catch(_ => {
const errorMessage = 'Failed merging parts.';
next(new Error(errorMessage));
});
} else {
req.isLastPart = false;
next();
}
});
});
req.pipe(busboy);
};
}
Update
So it looks like I have found the problem at least. busboy appears to be writing my array of bytes as text in the output file. I get 80,75,3,4,20,0,6,0,8,0,0,0,33,0,44,25 (as text) when I upload the array of bytes [80,75,3,4,20,0,6,0,8,0,0,0,33,0,44,25]. Now need to figure out how to force it to write it as a binary stream.
Figured out. Just in case it helps anyone, there was no problem with busboy or office.js or axios. I just had to convert the incoming chunk of data to Uint8Array before creating a blob from it. So instead of:
formData.append("file", new Blob([data]), state.filename);
like this:
const blob = new Blob([ new Uint8Array(data) ])
formData.append("file", blob, state.filename);
And it worked like a charm.
I am developing a web APP using Angular 12. I used a global error handler to process all my http errors. In my book portal component (in book module), When I called ReaderService
function (getFavorList) from another reader module, got an error: TypeError: this.handleError is not a function. If I call this function in same module is fine, and if I changed catchError code in getFavorList from
catchError(this.handleError('getFavorBookList'))
to
catchError((err)=>console.log(err))
This error will also disappeared, looks like "this" has problem, but I don't know how to fix without changing the error handling function. I also tried to bind(this) to this.handleError but not fixing the issue.
Following is code of book portal component:
ngOnInit(): void {
const bookID = this.route.snapshot.paramMap.get('id');
const readerName = this.tokenService.getUsername();
this.commonService.setSubject(readerName);
const readerID = this.readerAuthService.getReaderID();
//Load book info from database
this.bookService.getBook(bookID).subscribe((eBook: Book) => {
if (eBook && eBook.bookTitle) {
this.book = eBook;
this.logger.info(`Success load profile of ${bookID}`)
} else {
this.logger.warn(`Failed to load ${bookID} profile from server`);
}
//Load the existing comments and display in page
console.log('The comment length is ' + eBook.comments.length);
const existComments = document.querySelector('div.existing-comments');
if (eBook.comments.length > 0) {
this.bookService.getBookComments(bookID).subscribe((comments: BookComment[]) => {
if (comments && comments.length > 0) {
this.logger.info(`Success load comments of book ${bookID}`)
for (const item of comments) {
let p1 = document.createElement('p');
p1.className = 'comment-item';
p1.innerHTML = item.comment;
p1.style.fontSize = 'large';
p1.style.fontFamily = 'Times New Roman';
let p2 = document.createElement('p');
p2.className = 'comment-item';
p2.innerHTML = `---by ${item.readerName}`;
p2.style.fontSize = 'large';
p2.style.fontFamily = 'Times New Roman';
existComments.appendChild(p1);
existComments.appendChild(p2);
}
}
});
} else {
let p1 = document.createElement('p');
p1.className = 'comment-item';
p1.innerHTML = 'Be the first person to write comments!';
p1.style.fontSize = 'large';
p1.style.fontFamily = 'Times New Roman';
existComments.appendChild(p1);
}
this.commentForm.setValue({
bookID: bookID,
readerName: readerName,
title: '',
comment: '',
});
});
//If book is in favor book list, disable add fovoriteBook button
let favorInd = false;
this.readerService.getFavorList(readerID).subscribe((data) => {
console.log(data);
if (data && data.length > 0) {
for (const item of data) {
if (item.bookID === bookID) {
favorInd = true;
break;
}
}
if (favorInd) {
const addFavorButton = document.querySelector('button.add-favorites') as HTMLButtonElement;
addFavorButton.disabled = true;
}
}
});
}
Following is code of getFavorList:
private handleError: HandleError;
getFavorList(readerID): Observable<any> {
return this.http.get(`/api/reader/${readerID}/getfavourlist`).pipe(
catchError(this.handleError('getFavorBookList')), shareReplay()
)
}
Following is code of hanleError part:
export type HandleError =
<T> (operation?: string, result?: T) => (error: HttpErrorResponse) => Observable<T>;
/** Handles HttpClient errors */
#Injectable()
export class HttpErrorHandler {
constructor(
private router: Router,
private logger: NGXLogger,
) { }
createHandleError = (serviceName = '') => {
return <T>(operation = 'operation', result = {} as T) => this.handleError(serviceName, operation, result)
}
/**
* Returns a function that handles Http operation failures.
* This error handler lets the app continue to run as if no error occurred.
* #param serviceName = name of the data service that attempted the operation
* #param operation - name of the operation that failed
* #param result - optional value to return as the observable result
*/
handleError<T>(serviceName = '', operation = 'operation', result = {} as T) {
return (error: HttpErrorResponse): Observable<T> => {
//Generate error message
let errorMessage = '';
if (error.error instanceof ErrorEvent) {
errorMessage = `Client side error: ${error.error.message}`;
} else if (error.status === 401) {
errorMessage = `Server return ${error.status} with body "${error.error}"`;
if (error.error.message.includes('Incorrect username or password')) {
window.alert('Incorrect username or password, please check');
} else {
window.alert('Need login to access the contents.');
if (serviceName.toLowerCase().indexOf('reader') != -1) {
this.router.navigateByUrl('/reader/login');
} else {
this.router.navigateByUrl('/librarian/login');
}
}
} else {
errorMessage = `Server return ${error.status} with body "${error.error}"`;
}
//Generate user friendly error log
const errorLog = `HTTP Error in ${serviceName}: ${operation} failed: ${errorMessage}`;
// TODO: send the error to remote logging infrastructure
this.logger.error(errorLog);
return of(result);
};
}
}
Thanks!
I have this function get the token from api and check if I have admin permission. The problem is my export its firing before of the checking of the function.
const async = [
{ path: '*', redirect: '/404', hidden: true }
]
var hasAdmin = getAdmin()
if (hasAdmin === undefined || hasAdmin === null) {
var token = null
var url = null
var uri = window.location.href.split('?')
if (uri.length === 2) {
var vars = uri[1].split('&')
var getVars = {}
var tmp = ''
vars.forEach(function(v) {
tmp = v.split('=')
if (tmp.length === 2) {
getVars[tmp[0]] = tmp[1]
}
token = getVars.AUTH_ID
url = getVars.DOMAIN
})
getUserAdmin(url, token)
.then(response => {
var hasAdmin = response.result
if (hasAdmin === true) {
console.log('hasAdmin: ' + hasAdmin)
async.push(adminRouter)
}
})
.catch(error => {
console.log(error)
})
}
} else if (hasAdmin === true) {
async.push(adminRouter)
}
export const asyncRoutes = async
console.log('hasAdmin2: ' + hasAdmin)
Here is how Im using the const asyncRoutes:
import { asyncRoutes, constantRoutes } from '#/router'
/**
* Use meta.role to determine if the current user has permission
* #param roles
* #param route
*/
function hasPermission(roles, route) {
if (route.meta && route.meta.roles) {
return roles.some(role => route.meta.roles.includes(role))
} else {
return true
}
}
/**
* Filter asynchronous routing tables by recursion
* #param routes asyncRoutes
* #param roles
*/
export function filterAsyncRoutes(routes, roles) {
const res = []
routes.forEach(route => {
const tmp = { ...route }
if (hasPermission(roles, tmp)) {
if (tmp.children) {
tmp.children = filterAsyncRoutes(tmp.children, roles)
}
res.push(tmp)
}
})
return res
}
const state = {
routes: [],
addRoutes: []
}
const mutations = {
SET_ROUTES: (state, routes) => {
state.addRoutes = routes
state.routes = constantRoutes.concat(routes)
}
}
const actions = {
generateRoutes({ commit }, roles) {
return new Promise(resolve => {
let accessedRoutes
if (roles.includes('admin')) {
accessedRoutes = asyncRoutes || []
} else {
accessedRoutes = filterAsyncRoutes(asyncRoutes, roles)
}
commit('SET_ROUTES', accessedRoutes)
resolve(accessedRoutes)
})
}
}
export default {
namespaced: true,
state,
mutations,
actions
}
I always need reload(f5) the page in first access to gets the menu admin.
Await from promise dont works because de export should be always in the top. Some idea how can I fix this?
obs: attention in variable hasAdmin
I am trying to create a class that will fetch / cache users from my Firestore database. For some reason, I can't seem to save or expose the previous promise that was created. Here is my class:
export class UserCache {
private cacheTimeMilliseconds: number = 600000;
private userCache: any = {};
public getCacheUser(userid: string): Promise<User> {
return new Promise((resolve, reject) => {
let d = new Date();
d.setTime(d.getTime() - this.cacheTimeMilliseconds);
if (this.userCache[userid] && this.userCache[userid].complete && this.userCache[userid].lastAccess > d.getTime()) {
console.log("User cached");
resolve(this.userCache[userid].user);
}
console.log("Need to cache user");
this.userCache[userid] = {
complete: false
};
this.getSetUserFetchPromise(userid).then((data) => {
let user: User = <User>{ id: data.id, ...data.data() };
this.userCache[userid].user = user;
this.userCache[userid].complete = true;
this.userCache[userid].lastAccess = Date.now();
resolve(user);
});
});
}
private getSetUserFetchPromise(userid: string): Promise<any> {
console.log(this.userCache[userid]);
if (this.userCache[userid] && this.userCache[userid].promise) {
return this.userCache[userid].promise;
} else {
console.log("Creating new user fetch request.");
this.userCache[userid].promise = firestore().collection('users').doc(userid).get();
console.log(this.userCache[userid]);
return this.userCache[userid].promise;
}
}
}
Logs: (there are only 2 unique users, so should only be creating 2 new requests)
In the logs I can see that the promise is getting set in getSetUserFetchPromise, but the next time the function is called, the property is no longer set. I suspect it is either a scope or concurrency issue, but I can't seem to get around it.
I am calling getCacheUser in a consuming class with let oCache = new UserCache() and oCache.getCacheUser('USERID')
Edit following Tuan's answer below
UserCacheProvider.ts
import firestore from '#react-native-firebase/firestore';
import { User } from '../static/models';
class UserCache {
private cacheTimeMilliseconds: number = 600000;
private userCache: any = {};
public getCacheUser(userid: string): Promise<User> {
return new Promise((resolve, reject) => {
let d = new Date();
d.setTime(d.getTime() - this.cacheTimeMilliseconds);
if (this.userCache[userid] && this.userCache[userid].complete && this.userCache[userid].lastAccess > d.getTime()) {
console.log("User cached");
resolve(this.userCache[userid].user);
}
console.log("Need to cache user");
this.userCache[userid] = {
complete: false
};
this.getSetUserFetchPromise(userid).then((data) => {
let user: User = <User>{ id: data.id, ...data.data() };
this.userCache[userid].user = user;
this.userCache[userid].complete = true;
this.userCache[userid].lastAccess = Date.now();
resolve(user);
});
});
}
private getSetUserFetchPromise(userid: string): Promise<any> {
console.log(this.userCache[userid]);
if (this.userCache[userid] && this.userCache[userid].promise) {
return this.userCache[userid].promise;
} else {
console.log("Creating new user fetch request.");
this.userCache[userid].promise = firestore().collection('users').doc(userid).get();
console.log(this.userCache[userid]);
return this.userCache[userid].promise;
}
}
}
const userCache = new UserCache();
export default userCache;
ChatProvider.ts (usage)
let promises = [];
docs.forEach(doc => {
let message: Message = <Message>{ id: doc.id, ...doc.data() };
promises.push(UserCacheProvider.getCacheUser(message.senderid).then((oUser) => {
let conv: GCMessage = {
_id: message.id,
text: message.messagecontent,
createdAt: new Date(message.messagedate),
user: <GCUser>{ _id: oUser.id, avatar: oUser.thumbnail, name: oUser.displayname }
}
if (message.type && message.type == 'info') {
conv.system = true;
}
if (message.messageattachment && message.messageattachment != '') {
conv.image = message.messageattachment;
}
return conv;
}));
});
Promise.all(promises).then((values) => {
resolve(values);
});
Without seeing the calling code, it could be that getCacheUser is called twice before firestore resolves.
As an aside, I think refactoring the class may make debugging easier. I wonder why it caches the user, promise completion status, and the promise itself. Why not just cache the promise, something like:
interface UserCacheRecord {
promise: Promise<User>
lastAccess: number
}
export class UserCache {
private cacheTimeMilliseconds: number = 600000;
private userCache: { [userid: string]: UserCacheRecord } = {};
public async getCacheUser(userid: string): Promise<User> {
let d = new Date();
const cacheExpireTime = d.getTime() - this.cacheTimeMilliseconds
if (this.userCache[userid] && this.userCache[userid].lastAccess > cacheExpireTime) {
console.log("User cached");
return this.userCache[userid].promise
}
console.log("Need to cache user");
this.userCache[userid] = {
promise: this.getUser(userid),
lastAccess: Date.now()
}
return this.userCache[userid].promise
}
private async getUser(userid: string): Promise<User> {
const data = firestore().collection('users').doc(userid).get();
return <User>{ id: data.id, ...data.data() };
}
}
Currently, you create new UserCache everytime you access cache users. You have to export the instance of UserCache class, so just single instance is used for your app.
UserCache.ts
class UserCache {
}
const userCache = new UserCache();
export default userCache;
SomeFile.ts
import UserCache from './UserCache';
UserCache.getCacheUser('USERID')
Update
Added some tests
class UserCache {
userCache = {};
getUser(id) {
return new Promise((resolve, reject) => {
if (this.userCache[id]) {
resolve({
...this.userCache[id],
isCache: true,
});
}
this.requestUser(id).then(data => {
resolve(data);
this.userCache[id] = data;
});
});
}
requestUser(id) {
return Promise.resolve({
id,
});
}
}
const userCache = new UserCache();
export default userCache;
userCache.test.ts
import UserCache from '../test';
describe('Test user cache', () => {
test('User cached successfully', async () => {
const user1: any = await UserCache.getUser('test1');
expect(user1.isCache).toBeUndefined();
const user2: any = await UserCache.getUser('test1');
expect(user2.isCache).toBe(true);
});
});