From c716ff4ca7fb1e3d97febf6f9e51e38343174804 Mon Sep 17 00:00:00 2001 From: "Patrik J. Braun" Date: Mon, 30 Dec 2019 17:52:58 +0100 Subject: [PATCH] improving jobs --- package.json | 2 +- src/backend/model/jobs/jobs/FileJob.ts | 28 ++++++------ .../model/jobs/jobs/PhotoConvertingJob.ts | 6 +-- .../model/jobs/jobs/ThumbnailGenerationJob.ts | 11 ++--- .../model/jobs/jobs/VideoConvertingJob.ts | 13 +++--- src/common/entities/MediaDTO.ts | 10 +++++ .../button/job-button.settings.component.ts | 4 +- .../job-progress.settings.component.html | 4 +- .../photo/photo.settings.component.html | 2 +- .../app/ui/settings/scheduled-jobs.service.ts | 43 ++++++++++++++++--- .../thumbnail.settings.component.html | 2 +- .../video/video.settings.component.html | 2 +- src/frontend/translate/messages.en.xlf | 12 ++++-- src/frontend/translate/messages.fr.xlf | 12 ++++-- src/frontend/translate/messages.hu.xlf | 8 ++++ src/frontend/translate/messages.ro.xlf | 12 ++++-- src/frontend/translate/messages.ru.xlf | 12 ++++-- 17 files changed, 126 insertions(+), 57 deletions(-) diff --git a/package.json b/package.json index a56725c..f985baa 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "pretest": "tsc", "test": "ng test && mocha --recursive test/backend/unit && mocha --recursive test/backend/integration && mocha --recursive test/common/unit ", "start": "node ./src/backend/index", - "run-dev": "ng build --aot --watch --output-path=./dist --i18n-locale hu --i18n-file src/frontend/translate/messages.hu.xlf --i18n-missing-translation warning", + "run-dev": "ng build --aot --watch --output-path=./dist --i18n-locale en --i18n-file src/frontend/translate/messages.en.xlf --i18n-missing-translation warning", "build-stats": "ng build --aot --prod --stats-json --output-path=./dist --i18n-locale en --i18n-file src/frontend/translate/messages.en.xlf --i18n-missing-translation warning", "merge-new-translation": "gulp merge-new-translation", "add-translation": "gulp add-translation" diff --git a/src/backend/model/jobs/jobs/FileJob.ts b/src/backend/model/jobs/jobs/FileJob.ts index b6e0d55..7bf2860 100644 --- a/src/backend/model/jobs/jobs/FileJob.ts +++ b/src/backend/model/jobs/jobs/FileJob.ts @@ -12,7 +12,7 @@ import {MediaEntity} from '../../database/sql/enitites/MediaEntity'; import {PhotoEntity} from '../../database/sql/enitites/PhotoEntity'; import {VideoEntity} from '../../database/sql/enitites/VideoEntity'; import {backendTexts} from '../../../../common/BackendTexts'; -import DatabaseType = ServerConfig.DatabaseType; +import {ProjectPath} from '../../../ProjectPath'; declare var global: NodeJS.Global; @@ -23,12 +23,12 @@ const LOG_TAG = '[FileJob]'; export abstract class FileJob extends Job { public readonly ConfigTemplate: ConfigTemplateEntry[] = []; directoryQueue: string[] = []; - fileQueue: FileDTO[] = []; + fileQueue: string[] = []; protected constructor(private scanFilter: DiskMangerWorker.DirectoryScanSettings) { super(); - if (Config.Server.Database.type !== DatabaseType.memory) { + if (Config.Server.Database.type !== ServerConfig.DatabaseType.memory) { this.ConfigTemplate.push({ id: 'indexedOnly', type: 'boolean', @@ -54,9 +54,9 @@ export abstract class FileJob; + protected abstract async shouldProcess(filePath: string): Promise; - protected abstract async processFile(file: FileDTO): Promise; + protected abstract async processFile(filePath: string): Promise; protected async step(): Promise { if (this.directoryQueue.length === 0 && this.fileQueue.length === 0) { @@ -66,7 +66,7 @@ export abstract class FileJob 0) { if (this.config.indexedOnly === true && - Config.Server.Database.type !== DatabaseType.memory) { + Config.Server.Database.type !== ServerConfig.DatabaseType.memory) { await this.loadAllMediaFilesFromDB(); this.directoryQueue = []; } else { @@ -74,13 +74,12 @@ export abstract class FileJob 0) { this.Progress.Left = this.fileQueue.length; - const file = this.fileQueue.shift(); - const filePath = path.join(file.directory.path, file.directory.name, file.name); + const filePath = this.fileQueue.shift(); try { - if ((await this.shouldProcess(file)) === true) { + if ((await this.shouldProcess(filePath)) === true) { this.Progress.Processed++; this.Progress.log('processing: ' + filePath); - await this.processFile(file); + await this.processFile(filePath); } else { this.Progress.log('skipping: ' + filePath); this.Progress.Skipped++; @@ -102,10 +101,12 @@ export abstract class FileJob path.join(ProjectPath.ImageFolder, f.directory.path, f.directory.name, f.name))); } if (this.scanFilter.noMetaFile !== true) { - this.fileQueue.push(...await this.filterMetaFiles(scanned.metaFile)); + this.fileQueue.push(...(await this.filterMetaFiles(scanned.metaFile)) + .map(f => path.join(ProjectPath.ImageFolder, f.directory.path, f.directory.name, f.name))); } } @@ -134,6 +135,7 @@ export abstract class FileJob path.join(ProjectPath.ImageFolder, f.directory.path, f.directory.name, f.name)))); } } diff --git a/src/backend/model/jobs/jobs/PhotoConvertingJob.ts b/src/backend/model/jobs/jobs/PhotoConvertingJob.ts index 46746b4..f1757fa 100644 --- a/src/backend/model/jobs/jobs/PhotoConvertingJob.ts +++ b/src/backend/model/jobs/jobs/PhotoConvertingJob.ts @@ -21,14 +21,12 @@ export class PhotoConvertingJob extends FileJob { } - protected async shouldProcess(file: FileDTO): Promise { - const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name); + protected async shouldProcess(mPath: string): Promise { return !(await PhotoProcessing.convertedPhotoExist(mPath, Config.Server.Media.Photo.Converting.resolution)); } - protected async processFile(file: FileDTO): Promise { - const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name); + protected async processFile(mPath: string): Promise { await PhotoProcessing.convertPhoto(mPath); } diff --git a/src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts b/src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts index e22c4b4..c347cd6 100644 --- a/src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts +++ b/src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts @@ -1,7 +1,5 @@ import {Config} from '../../../../common/config/private/Config'; import {DefaultsJobs} from '../../../../common/entities/job/JobDTO'; -import {ProjectPath} from '../../../ProjectPath'; -import * as path from 'path'; import {FileJob} from './FileJob'; import {PhotoProcessing} from '../../fileprocessing/PhotoProcessing'; import {ThumbnailSourceType} from '../../threading/PhotoWorker'; @@ -49,8 +47,7 @@ export class ThumbnailGenerationJob extends FileJob<{ sizes: number[], indexedOn return undefined; } - protected async shouldProcess(file: FileDTO): Promise { - const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name); + protected async shouldProcess(mPath: string): Promise { for (let i = 0; i < this.config.sizes.length; ++i) { if (!(await PhotoProcessing.convertedPhotoExist(mPath, this.config.sizes[i]))) { return true; @@ -58,13 +55,11 @@ export class ThumbnailGenerationJob extends FileJob<{ sizes: number[], indexedOn } } - protected async processFile(file: FileDTO): Promise { - - const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name); + protected async processFile(mPath: string): Promise { for (let i = 0; i < this.config.sizes.length; ++i) { await PhotoProcessing.generateThumbnail(mPath, this.config.sizes[i], - MediaDTO.isVideo(file) ? ThumbnailSourceType.Video : ThumbnailSourceType.Photo, + MediaDTO.isVideoPath(mPath) ? ThumbnailSourceType.Video : ThumbnailSourceType.Photo, false); } diff --git a/src/backend/model/jobs/jobs/VideoConvertingJob.ts b/src/backend/model/jobs/jobs/VideoConvertingJob.ts index 691532f..95f1bcf 100644 --- a/src/backend/model/jobs/jobs/VideoConvertingJob.ts +++ b/src/backend/model/jobs/jobs/VideoConvertingJob.ts @@ -1,12 +1,10 @@ import {Config} from '../../../../common/config/private/Config'; import {DefaultsJobs} from '../../../../common/entities/job/JobDTO'; -import {ProjectPath} from '../../../ProjectPath'; -import * as path from 'path'; import {FileJob} from './FileJob'; import {VideoProcessing} from '../../fileprocessing/VideoProcessing'; -import {FileDTO} from '../../../../common/entities/FileDTO'; const LOG_TAG = '[VideoConvertingJob]'; +declare const global: any; export class VideoConvertingJob extends FileJob { @@ -20,14 +18,15 @@ export class VideoConvertingJob extends FileJob { return Config.Client.Media.Video.enabled === true; } - protected async shouldProcess(file: FileDTO): Promise { - const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name); + protected async shouldProcess(mPath: string): Promise { return !(await VideoProcessing.convertedVideoExist(mPath)); } - protected async processFile(file: FileDTO): Promise { - const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name); + protected async processFile(mPath: string): Promise { await VideoProcessing.convertVideo(mPath); + if (global.gc) { + global.gc(); + } } diff --git a/src/common/entities/MediaDTO.ts b/src/common/entities/MediaDTO.ts index 7183348..08a15e5 100644 --- a/src/common/entities/MediaDTO.ts +++ b/src/common/entities/MediaDTO.ts @@ -65,6 +65,16 @@ export module MediaDTO { return false; }; + export const isVideoPath = (path: string): boolean => { + const lower = path.toLowerCase(); + for (const ext of SupportedFormats.WithDots.Videos) { + if (lower.endsWith(ext)) { + return true; + } + } + return false; + }; + export const isVideoTranscodingNeeded = (media: FileDTO): boolean => { const lower = media.name.toLowerCase(); for (const ext of SupportedFormats.WithDots.TranscodeNeed.Videos) { diff --git a/src/frontend/app/ui/settings/jobs/button/job-button.settings.component.ts b/src/frontend/app/ui/settings/jobs/button/job-button.settings.component.ts index d9db5ac..23bdf31 100644 --- a/src/frontend/app/ui/settings/jobs/button/job-button.settings.component.ts +++ b/src/frontend/app/ui/settings/jobs/button/job-button.settings.component.ts @@ -41,7 +41,7 @@ export class JobButtonComponent { this.error.emit(''); try { await this.jobsService.start(this.jobName, this.config, this.soloRun); - this.notification.info(this.i18n('Job started') + ': ' + this.jobName); + this.notification.success(this.i18n('Job started') + ': ' + this.backendTextService.getJobName(this.jobName)); return true; } catch (err) { console.log(err); @@ -57,7 +57,7 @@ export class JobButtonComponent { this.error.emit(''); try { await this.jobsService.stop(this.jobName); - this.notification.info(this.i18n('Job stopped') + ': ' + this.jobName); + this.notification.info(this.i18n('Job stopped') + ': ' + this.backendTextService.getJobName(this.jobName)); return true; } catch (err) { console.log(err); diff --git a/src/frontend/app/ui/settings/jobs/progress/job-progress.settings.component.html b/src/frontend/app/ui/settings/jobs/progress/job-progress.settings.component.html index e1e5499..aa45275 100644 --- a/src/frontend/app/ui/settings/jobs/progress/job-progress.settings.component.html +++ b/src/frontend/app/ui/settings/jobs/progress/job-progress.settings.component.html @@ -25,7 +25,7 @@
-

+

...

diff --git a/src/frontend/app/ui/settings/photo/photo.settings.component.html b/src/frontend/app/ui/settings/photo/photo.settings.component.html index f5d2f62..74e6fe3 100644 --- a/src/frontend/app/ui/settings/photo/photo.settings.component.html +++ b/src/frontend/app/ui/settings/photo/photo.settings.component.html @@ -100,7 +100,7 @@

- diff --git a/src/frontend/app/ui/settings/scheduled-jobs.service.ts b/src/frontend/app/ui/settings/scheduled-jobs.service.ts index 8cd0b10..5ada490 100644 --- a/src/frontend/app/ui/settings/scheduled-jobs.service.ts +++ b/src/frontend/app/ui/settings/scheduled-jobs.service.ts @@ -1,9 +1,12 @@ import {EventEmitter, Injectable} from '@angular/core'; import {BehaviorSubject} from 'rxjs'; -import {JobProgressDTO} from '../../../../common/entities/job/JobProgressDTO'; +import {JobProgressDTO, JobProgressStates} from '../../../../common/entities/job/JobProgressDTO'; import {NetworkService} from '../../model/network/network.service'; import {JobScheduleDTO} from '../../../../common/entities/job/JobScheduleDTO'; import {JobDTO} from '../../../../common/entities/job/JobDTO'; +import {BackendtextService} from '../../model/backendtext.service'; +import {NotificationService} from '../../model/notification.service'; +import {I18n} from '@ngx-translate/i18n-polyfill'; @Injectable() export class ScheduledJobsService { @@ -15,7 +18,10 @@ export class ScheduledJobsService { public jobStartingStopping: { [key: string]: boolean } = {}; private subscribers = 0; - constructor(private _networkService: NetworkService) { + constructor(private _networkService: NetworkService, + private notification: NotificationService, + private backendTextService: BackendtextService, + private i18n: I18n) { this.progress = new BehaviorSubject({}); } @@ -39,6 +45,8 @@ export class ScheduledJobsService { this.jobStartingStopping[jobName] = true; await this._networkService.postJson('/admin/jobs/scheduled/' + jobName + '/' + (soloStart === true ? 'soloStart' : 'start'), {config: config}); + // placeholder to force showing running job + this.addDummyProgress(jobName, config); delete this.jobStartingStopping[jobName]; this.forceUpdate(); } @@ -53,14 +61,20 @@ export class ScheduledJobsService { protected async loadProgress(): Promise { const prevPrg = this.progress.value; this.progress.next(await this._networkService.getJson<{ [key: string]: JobProgressDTO }>('/admin/jobs/scheduled/progress')); - for (const prg in prevPrg) { - if (!this.progress.value.hasOwnProperty(prg)) { + for (const prg of Object.keys(prevPrg)) { + if (!this.progress.value.hasOwnProperty(prg) || + // state changed from running to finished + ((prevPrg[prg].state === JobProgressStates.running || + prevPrg[prg].state === JobProgressStates.cancelling) && + !(this.progress.value[prg].state === JobProgressStates.running || + this.progress.value[prg].state === JobProgressStates.cancelling) + )) { this.onJobFinish.emit(prg); + this.notification.info(this.i18n('Job finished') + ': ' + this.backendTextService.getJobName(prevPrg[prg].jobName)); } } } - protected getProgressPeriodically() { if (this.timer != null || this.subscribers === 0) { return; @@ -76,6 +90,25 @@ export class ScheduledJobsService { this.loadProgress().catch(console.error); } + private addDummyProgress(jobName: string, config: any) { + const prgs = this.progress.value; + prgs[JobDTO.getHashName(jobName, config)] = { + jobName: jobName, + state: JobProgressStates.running, + HashName: JobDTO.getHashName(jobName, config), + logs: [], steps: { + skipped: 0, + processed: 0, + all: 0 + }, + time: { + start: Date.now(), + end: Date.now() + } + }; + this.progress.next(prgs); + } + private incSubscribers() { this.subscribers++; this.getProgressPeriodically(); diff --git a/src/frontend/app/ui/settings/thumbnail/thumbnail.settings.component.html b/src/frontend/app/ui/settings/thumbnail/thumbnail.settings.component.html index fde9cd8..6e5ea85 100644 --- a/src/frontend/app/ui/settings/thumbnail/thumbnail.settings.component.html +++ b/src/frontend/app/ui/settings/thumbnail/thumbnail.settings.component.html @@ -74,7 +74,7 @@ - Reset - diff --git a/src/frontend/translate/messages.en.xlf b/src/frontend/translate/messages.en.xlf index af29f86..98313df 100644 --- a/src/frontend/translate/messages.en.xlf +++ b/src/frontend/translate/messages.en.xlf @@ -904,9 +904,7 @@ app/ui/settings/thumbnail/thumbnail.settings.component.html 59 - ';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 - pixels. - + ';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 pixels. Video support uses ffmpeg. ffmpeg and ffprobe binaries need to be available in the PATH or @@ -2592,6 +2590,14 @@ Random Photo + + Job finished + + src/frontend/app/ui/settings/scheduled-jobs.service.ts + 1 + + Job finished + Thumbnail diff --git a/src/frontend/translate/messages.fr.xlf b/src/frontend/translate/messages.fr.xlf index 86bfc1c..f8d1625 100644 --- a/src/frontend/translate/messages.fr.xlf +++ b/src/frontend/translate/messages.fr.xlf @@ -904,9 +904,7 @@ app/ui/settings/thumbnail/thumbnail.settings.component.html 59 - ';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 - pixels. - + ';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 pixels. Video support uses ffmpeg. ffmpeg and ffprobe binaries need to be available in the PATH or @@ -2592,6 +2590,14 @@ Random Photo + + Job finished + + src/frontend/app/ui/settings/scheduled-jobs.service.ts + 1 + + Job finished + Thumbnail diff --git a/src/frontend/translate/messages.hu.xlf b/src/frontend/translate/messages.hu.xlf index 018b5e7..93b6f76 100644 --- a/src/frontend/translate/messages.hu.xlf +++ b/src/frontend/translate/messages.hu.xlf @@ -2590,6 +2590,14 @@ Véletleg Fotó + + Job finished + + src/frontend/app/ui/settings/scheduled-jobs.service.ts + 1 + + Feladat végzett + Thumbnail diff --git a/src/frontend/translate/messages.ro.xlf b/src/frontend/translate/messages.ro.xlf index e5ff8eb..486f7a6 100644 --- a/src/frontend/translate/messages.ro.xlf +++ b/src/frontend/translate/messages.ro.xlf @@ -904,9 +904,7 @@ app/ui/settings/thumbnail/thumbnail.settings.component.html 59 - ';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 - pixels. - + ';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 pixels. Video support uses ffmpeg. ffmpeg and ffprobe binaries need to be available in the PATH or @@ -2592,6 +2590,14 @@ Random Photo + + Job finished + + src/frontend/app/ui/settings/scheduled-jobs.service.ts + 1 + + Job finished + Thumbnail diff --git a/src/frontend/translate/messages.ru.xlf b/src/frontend/translate/messages.ru.xlf index 5f33f40..3e522da 100644 --- a/src/frontend/translate/messages.ru.xlf +++ b/src/frontend/translate/messages.ru.xlf @@ -904,9 +904,7 @@ app/ui/settings/thumbnail/thumbnail.settings.component.html 59 - ';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 - pixels. - + ';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 pixels. Video support uses ffmpeg. ffmpeg and ffprobe binaries need to be available in the PATH or @@ -2592,6 +2590,14 @@ Random Photo + + Job finished + + src/frontend/app/ui/settings/scheduled-jobs.service.ts + 1 + + Job finished + Thumbnail