From 296fb8b422b48ab1ee838c35ef3c8bc0b823f049 Mon Sep 17 00:00:00 2001 From: Patrik Braun Date: Sun, 23 Jul 2017 22:37:29 +0200 Subject: [PATCH] backend performance improvement --- .../thumbnail/ThumbnailGeneratorMWs.ts | 21 ++++----- backend/model/memory/UserManager.ts | 1 - backend/model/threading/DiskMangerWorker.ts | 6 +-- backend/model/threading/TaskQue.ts | 46 +++++++++++++++++++ backend/model/threading/ThreadPool.ts | 8 ++-- backend/model/threading/Worker.ts | 6 ++- backend/server.ts | 3 +- backend/tsconfig.json | 16 +++++++ 8 files changed, 84 insertions(+), 23 deletions(-) create mode 100644 backend/model/threading/TaskQue.ts create mode 100644 backend/tsconfig.json diff --git a/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts b/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts index 56bfeb7..ce51b74 100644 --- a/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts +++ b/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts @@ -12,13 +12,13 @@ import {PhotoDTO} from "../../../common/entities/PhotoDTO"; import {Config} from "../../../common/config/private/Config"; import {ThumbnailProcessingLib} from "../../../common/config/private/IPrivateConfig"; import {ThumbnailTH} from "../../model/threading/ThreadPool"; -import {RendererFactory, RendererInput} from "../../model/threading/ThumbnailWoker"; +import {RendererInput} from "../../model/threading/ThumbnailWoker"; +import {ITaskQue, TaskQue} from "../../model/threading/TaskQue"; export class ThumbnailGeneratorMWs { private static initDone = false; - private static ThumbnailFunction: (input: RendererInput) => Promise = null; - private static threadPool: ThumbnailTH = null; + private static taskQue: ITaskQue = null; public static init() { if (this.initDone == true) { @@ -33,11 +33,11 @@ export class ThumbnailGeneratorMWs { Config.Client.concurrentThumbnailGenerations = 1; } - this.ThumbnailFunction = RendererFactory.build(Config.Server.thumbnail.processingLibrary); - if (Config.Server.enableThreading == true && Config.Server.thumbnail.processingLibrary == ThumbnailProcessingLib.Jimp) { - this.threadPool = new ThumbnailTH(Config.Client.concurrentThumbnailGenerations); + this.taskQue = new ThumbnailTH(Config.Client.concurrentThumbnailGenerations); + } else { + this.taskQue = new TaskQue(Config.Client.concurrentThumbnailGenerations); } this.initDone = true; @@ -157,13 +157,8 @@ export class ThumbnailGeneratorMWs { qualityPriority: Config.Server.thumbnail.qualityPriority }; try { - if (this.threadPool !== null) { - await this.threadPool.execute(input); - return next(); - } else { - await ThumbnailGeneratorMWs.ThumbnailFunction(input); - return next(); - } + await this.taskQue.execute(input); + return next(); } catch (error) { return next(new ErrorDTO(ErrorCodes.THUMBNAIL_GENERATION_ERROR, "Error during generating thumbnail", error)); } diff --git a/backend/model/memory/UserManager.ts b/backend/model/memory/UserManager.ts index a25ed63..07c5d23 100644 --- a/backend/model/memory/UserManager.ts +++ b/backend/model/memory/UserManager.ts @@ -65,7 +65,6 @@ export class UserManager implements IUserManager { users.splice(i, 1); } } - console.log(users); return users; } diff --git a/backend/model/threading/DiskMangerWorker.ts b/backend/model/threading/DiskMangerWorker.ts index 8d267cb..25d342c 100644 --- a/backend/model/threading/DiskMangerWorker.ts +++ b/backend/model/threading/DiskMangerWorker.ts @@ -35,7 +35,7 @@ export class DiskMangerWorker { const absoluteDirectoryName = path.join(ProjectPath.ImageFolder, relativeDirectoryName); const stat = fs.statSync(path.join(ProjectPath.ImageFolder, relativeDirectoryName)); - let directory = { + const directory = { name: directoryName, path: directoryParent, lastModified: Math.max(stat.ctime.getTime(), stat.mtime.getTime()), @@ -51,8 +51,8 @@ export class DiskMangerWorker { try { for (let i = 0; i < list.length; i++) { - let file = list[i]; - let fullFilePath = path.normalize(path.resolve(absoluteDirectoryName, file)); + const file = list[i]; + const fullFilePath = path.normalize(path.resolve(absoluteDirectoryName, file)); if (photosOnly == false && fs.statSync(fullFilePath).isDirectory()) { const d = await DiskMangerWorker.scanDirectory(path.join(relativeDirectoryName, file), Config.Server.folderPreviewSize, true diff --git a/backend/model/threading/TaskQue.ts b/backend/model/threading/TaskQue.ts new file mode 100644 index 0000000..6bd1007 --- /dev/null +++ b/backend/model/threading/TaskQue.ts @@ -0,0 +1,46 @@ +import {RendererInput, ThumbnailWoker} from "./ThumbnailWoker"; +import {Config} from "../../../common/config/private/Config"; + + +interface QueTask { + data: RendererInput; + promise: { resolve: Function, reject: Function }; +} + + +export interface ITaskQue { + execute(input: any): Promise; +} + +export class TaskQue implements ITaskQue { + + private tasks: QueTask[] = []; + private taskInProgress: number = 0; + private run = async () => { + if (this.tasks.length == 0 || this.taskInProgress >= this.size) { + return; + } + this.taskInProgress++; + const task = this.tasks.shift(); + try { + task.promise.resolve(await ThumbnailWoker.render(task.data, Config.Server.thumbnail.processingLibrary)); + } catch (err) { + task.promise.reject(err); + } + this.taskInProgress--; + process.nextTick(this.run); + }; + + constructor(private size: number) { + } + + execute(input: RendererInput): Promise { + return new Promise((resolve: Function, reject: Function) => { + this.tasks.push({ + data: input, + promise: {resolve: resolve, reject: reject} + }); + this.run(); + }); + } +} diff --git a/backend/model/threading/ThreadPool.ts b/backend/model/threading/ThreadPool.ts index e3dbca5..08a81ea 100644 --- a/backend/model/threading/ThreadPool.ts +++ b/backend/model/threading/ThreadPool.ts @@ -4,6 +4,7 @@ import {DiskManagerTask, ThumbnailTask, WorkerMessage, WorkerTask, WorkerTaskTyp import {DirectoryDTO} from "../../../common/entities/DirectoryDTO"; import {RendererInput} from "./ThumbnailWoker"; import {Config} from "../../../common/config/private/Config"; +import {ITaskQue} from "./TaskQue"; interface PoolTask { @@ -26,7 +27,6 @@ export class ThreadPool { Logger.silly("Creating thread pool with", size, "workers"); for (let i = 0; i < size; i++) { this.startWorker(); - } } @@ -83,14 +83,14 @@ export class ThreadPool { return; } - const poolTask = this.tasks.pop(); + const poolTask = this.tasks.shift(); worker.poolTask = poolTask; worker.worker.send(poolTask.task); }; } -export class DiskManagerTH extends ThreadPool { +export class DiskManagerTH extends ThreadPool implements ITaskQue { execute(relativeDirectoryName: string): Promise { return super.executeTask({ type: WorkerTaskTypes.diskManager, @@ -99,7 +99,7 @@ export class DiskManagerTH extends ThreadPool { } } -export class ThumbnailTH extends ThreadPool { +export class ThumbnailTH extends ThreadPool implements ITaskQue { execute(input: RendererInput): Promise { return super.executeTask({ type: WorkerTaskTypes.thumbnail, diff --git a/backend/model/threading/Worker.ts b/backend/model/threading/Worker.ts index 16305ba..bcad1e6 100644 --- a/backend/model/threading/Worker.ts +++ b/backend/model/threading/Worker.ts @@ -2,6 +2,7 @@ import {DiskMangerWorker} from "./DiskMangerWorker"; import {Logger} from "../../Logger"; import {RendererInput, ThumbnailWoker} from "./ThumbnailWoker"; import {ThumbnailProcessingLib} from "../../../common/config/private/IPrivateConfig"; + export class Worker { @@ -13,6 +14,9 @@ export class Worker { switch (task.type) { case WorkerTaskTypes.diskManager: result = await DiskMangerWorker.scanDirectory((task).relativeDirectoryName); + if (global.gc) { + global.gc(); + } break; case WorkerTaskTypes.thumbnail: result = await ThumbnailWoker.render((task).input, (task).renderer); @@ -34,7 +38,7 @@ export class Worker { } -export enum WorkerTaskTypes{ +export enum WorkerTaskTypes { thumbnail, diskManager } diff --git a/backend/server.ts b/backend/server.ts index 0228277..1c1239d 100644 --- a/backend/server.ts +++ b/backend/server.ts @@ -16,7 +16,8 @@ import {ThumbnailGeneratorMWs} from "./middlewares/thumbnail/ThumbnailGeneratorM import {DiskManager} from "./model/DiskManger"; import {NotificationRouter} from "./routes/NotificationRouter"; import {ConfigDiagnostics} from "./model/ConfigDiagnostics"; -import _session = require('cookie-session'); + +const _session = require('cookie-session'); const LOG_TAG = "[server]"; diff --git a/backend/tsconfig.json b/backend/tsconfig.json new file mode 100644 index 0000000..a7e384e --- /dev/null +++ b/backend/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compileOnSave": true, + "compilerOptions": { + "module": "commonjs", + "skipLibCheck": true, + "sourceMap": false, + "declaration": false, + "moduleResolution": "node", + "emitDecoratorMetadata": true, + "experimentalDecorators": true, + "target": "es6", + "typeRoots": [ + "../node_modules/@types" + ] + } +}