diff --git a/.travis.yml b/.travis.yml index 69c0851..0c010b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,12 @@ dist: trusty language: node_js node_js: -- '10' -- '11' - + - '10' + - '11' +env: + - MYSQL_HOST='localhost' MYSQL_PASSWORD='' MYSQL_USERNAME='root' MYSQL_DATABASE='pigallery2_test' +services: + - mysql addons: chrome: stable before_install: diff --git a/backend/middlewares/AdminMWs.ts b/backend/middlewares/AdminMWs.ts index 0175fa9..1984ae1 100644 --- a/backend/middlewares/AdminMWs.ts +++ b/backend/middlewares/AdminMWs.ts @@ -42,6 +42,25 @@ export class AdminMWs { } } + + public static async getDuplicates(req: Request, res: Response, next: NextFunction) { + if (Config.Server.database.type === DatabaseType.memory) { + return next(new ErrorDTO(ErrorCodes.GENERAL_ERROR, 'Statistic is only available for indexed content')); + } + + + const galleryManager = ObjectManagerRepository.getInstance().GalleryManager; + try { + req.resultPipe = await galleryManager.getPossibleDuplicates(); + return next(); + } catch (err) { + if (err instanceof Error) { + return next(new ErrorDTO(ErrorCodes.GENERAL_ERROR, 'Error while getting duplicates: ' + err.toString(), err)); + } + return next(new ErrorDTO(ErrorCodes.GENERAL_ERROR, 'Error while getting duplicates', err)); + } + } + public static async updateDatabaseSettings(req: Request, res: Response, next: NextFunction) { if ((typeof req.body === 'undefined') || (typeof req.body.settings === 'undefined')) { @@ -399,7 +418,7 @@ export class AdminMWs { public static startIndexing(req: Request, res: Response, next: NextFunction) { try { const createThumbnails: boolean = (req.body).createThumbnails || false; - ObjectManagerRepository.getInstance().IndexingManager.startIndexing(createThumbnails); + ObjectManagerRepository.getInstance().IndexingTaskManager.startIndexing(createThumbnails); req.resultPipe = 'ok'; return next(); } catch (err) { @@ -413,7 +432,7 @@ export class AdminMWs { public static getIndexingProgress(req: Request, res: Response, next: NextFunction) { try { - req.resultPipe = ObjectManagerRepository.getInstance().IndexingManager.getProgress(); + req.resultPipe = ObjectManagerRepository.getInstance().IndexingTaskManager.getProgress(); return next(); } catch (err) { if (err instanceof Error) { @@ -425,7 +444,7 @@ export class AdminMWs { public static cancelIndexing(req: Request, res: Response, next: NextFunction) { try { - ObjectManagerRepository.getInstance().IndexingManager.cancelIndexing(); + ObjectManagerRepository.getInstance().IndexingTaskManager.cancelIndexing(); req.resultPipe = 'ok'; return next(); } catch (err) { @@ -438,7 +457,7 @@ export class AdminMWs { public static async resetIndexes(req: Express.Request, res: Response, next: NextFunction) { try { - await ObjectManagerRepository.getInstance().IndexingManager.reset(); + await ObjectManagerRepository.getInstance().IndexingTaskManager.reset(); req.resultPipe = 'ok'; return next(); } catch (err) { diff --git a/backend/middlewares/GalleryMWs.ts b/backend/middlewares/GalleryMWs.ts index 80f5fe3..61154d1 100644 --- a/backend/middlewares/GalleryMWs.ts +++ b/backend/middlewares/GalleryMWs.ts @@ -222,7 +222,7 @@ export class GalleryMWs { } public static async autocomplete(req: Request, res: Response, next: NextFunction) { - if (Config.Client.Search.autocompleteEnabled === false) { + if (Config.Client.Search.AutoComplete.enabled === false) { return next(); } if (!(req.params.text)) { diff --git a/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts b/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts index 5a1eecb..0c0b0e7 100644 --- a/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts +++ b/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts @@ -65,7 +65,7 @@ export class ThumbnailGeneratorMWs { } } catch (error) { - return next(new ErrorDTO(ErrorCodes.SERVER_ERROR, 'error during postprocessing result', error.toString())); + return next(new ErrorDTO(ErrorCodes.SERVER_ERROR, 'error during postprocessing result (adding thumbnail info)', error.toString())); } diff --git a/backend/model/ObjectManagerRepository.ts b/backend/model/ObjectManagerRepository.ts index 87a1e0e..c66b12d 100644 --- a/backend/model/ObjectManagerRepository.ts +++ b/backend/model/ObjectManagerRepository.ts @@ -4,7 +4,9 @@ import {ISearchManager} from './interfaces/ISearchManager'; import {SQLConnection} from './sql/SQLConnection'; import {ISharingManager} from './interfaces/ISharingManager'; import {Logger} from '../Logger'; +import {IIndexingTaskManager} from './interfaces/IIndexingTaskManager'; import {IIndexingManager} from './interfaces/IIndexingManager'; +import {IPersonManager} from './interfaces/IPersonManager'; export class ObjectManagerRepository { @@ -15,6 +17,16 @@ export class ObjectManagerRepository { private _searchManager: ISearchManager; private _sharingManager: ISharingManager; private _indexingManager: IIndexingManager; + private _indexingTaskManager: IIndexingTaskManager; + private _personManager: IPersonManager; + + get PersonManager(): IPersonManager { + return this._personManager; + } + + set PersonManager(value: IPersonManager) { + this._personManager = value; + } get IndexingManager(): IIndexingManager { return this._indexingManager; @@ -24,19 +36,14 @@ export class ObjectManagerRepository { this._indexingManager = value; } - public static getInstance() { - if (this._instance === null) { - this._instance = new ObjectManagerRepository(); - } - return this._instance; + get IndexingTaskManager(): IIndexingTaskManager { + return this._indexingTaskManager; } - public static async reset() { - await SQLConnection.close(); - this._instance = null; + set IndexingTaskManager(value: IIndexingTaskManager) { + this._indexingTaskManager = value; } - get GalleryManager(): IGalleryManager { return this._galleryManager; } @@ -69,18 +76,34 @@ export class ObjectManagerRepository { this._sharingManager = value; } + public static getInstance() { + if (this._instance === null) { + this._instance = new ObjectManagerRepository(); + } + return this._instance; + } + + public static async reset() { + await SQLConnection.close(); + this._instance = null; + } + public static async InitMemoryManagers() { await ObjectManagerRepository.reset(); const GalleryManager = require('./memory/GalleryManager').GalleryManager; const UserManager = require('./memory/UserManager').UserManager; const SearchManager = require('./memory/SearchManager').SearchManager; const SharingManager = require('./memory/SharingManager').SharingManager; + const IndexingTaskManager = require('./memory/IndexingTaskManager').IndexingTaskManager; const IndexingManager = require('./memory/IndexingManager').IndexingManager; + const PersonManager = require('./memory/PersonManager').PersonManager; ObjectManagerRepository.getInstance().GalleryManager = new GalleryManager(); ObjectManagerRepository.getInstance().UserManager = new UserManager(); ObjectManagerRepository.getInstance().SearchManager = new SearchManager(); ObjectManagerRepository.getInstance().SharingManager = new SharingManager(); + ObjectManagerRepository.getInstance().IndexingTaskManager = new IndexingTaskManager(); ObjectManagerRepository.getInstance().IndexingManager = new IndexingManager(); + ObjectManagerRepository.getInstance().PersonManager = new PersonManager(); } public static async InitSQLManagers() { @@ -90,12 +113,16 @@ export class ObjectManagerRepository { const UserManager = require('./sql/UserManager').UserManager; const SearchManager = require('./sql/SearchManager').SearchManager; const SharingManager = require('./sql/SharingManager').SharingManager; + const IndexingTaskManager = require('./sql/IndexingTaskManager').IndexingTaskManager; const IndexingManager = require('./sql/IndexingManager').IndexingManager; + const PersonManager = require('./sql/PersonManager').PersonManager; ObjectManagerRepository.getInstance().GalleryManager = new GalleryManager(); ObjectManagerRepository.getInstance().UserManager = new UserManager(); ObjectManagerRepository.getInstance().SearchManager = new SearchManager(); ObjectManagerRepository.getInstance().SharingManager = new SharingManager(); + ObjectManagerRepository.getInstance().IndexingTaskManager = new IndexingTaskManager(); ObjectManagerRepository.getInstance().IndexingManager = new IndexingManager(); + ObjectManagerRepository.getInstance().PersonManager = new PersonManager(); Logger.debug('SQL DB inited'); } diff --git a/backend/model/interfaces/IIndexingManager.ts b/backend/model/interfaces/IIndexingManager.ts index c741e9a..869b9b8 100644 --- a/backend/model/interfaces/IIndexingManager.ts +++ b/backend/model/interfaces/IIndexingManager.ts @@ -1,11 +1,5 @@ -import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO'; +import {DirectoryDTO} from '../../../common/entities/DirectoryDTO'; export interface IIndexingManager { - startIndexing(createThumbnails?: boolean): void; - - getProgress(): IndexingProgressDTO; - - cancelIndexing(): void; - - reset(): Promise; + indexDirectory(relativeDirectoryName: string): Promise; } diff --git a/backend/model/interfaces/IIndexingTaskManager.ts b/backend/model/interfaces/IIndexingTaskManager.ts new file mode 100644 index 0000000..af4eea1 --- /dev/null +++ b/backend/model/interfaces/IIndexingTaskManager.ts @@ -0,0 +1,11 @@ +import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO'; + +export interface IIndexingTaskManager { + startIndexing(createThumbnails?: boolean): void; + + getProgress(): IndexingProgressDTO; + + cancelIndexing(): void; + + reset(): Promise; +} diff --git a/backend/model/interfaces/IPersonManager.ts b/backend/model/interfaces/IPersonManager.ts new file mode 100644 index 0000000..dd23adf --- /dev/null +++ b/backend/model/interfaces/IPersonManager.ts @@ -0,0 +1,7 @@ +import {PersonEntry} from '../sql/enitites/PersonEntry'; + +export interface IPersonManager { + get(name: string): Promise; + + saveAll(names: string[]): Promise; +} diff --git a/backend/model/memory/IndexingManager.ts b/backend/model/memory/IndexingManager.ts index 6bd861e..71b1d88 100644 --- a/backend/model/memory/IndexingManager.ts +++ b/backend/model/memory/IndexingManager.ts @@ -1,21 +1,11 @@ import {IIndexingManager} from '../interfaces/IIndexingManager'; -import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO'; +import {DirectoryDTO} from '../../../common/entities/DirectoryDTO'; export class IndexingManager implements IIndexingManager { - startIndexing(): void { + indexDirectory(relativeDirectoryName: string): Promise { throw new Error('not supported by memory DB'); } - getProgress(): IndexingProgressDTO { - throw new Error('not supported by memory DB'); - } - cancelIndexing(): void { - throw new Error('not supported by memory DB'); - } - - reset(): Promise { - throw new Error('Method not implemented.'); - } } diff --git a/backend/model/memory/IndexingTaskManager.ts b/backend/model/memory/IndexingTaskManager.ts new file mode 100644 index 0000000..9b9880e --- /dev/null +++ b/backend/model/memory/IndexingTaskManager.ts @@ -0,0 +1,21 @@ +import {IIndexingTaskManager} from '../interfaces/IIndexingTaskManager'; +import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO'; + +export class IndexingTaskManager implements IIndexingTaskManager { + + startIndexing(): void { + throw new Error('not supported by memory DB'); + } + + getProgress(): IndexingProgressDTO { + throw new Error('not supported by memory DB'); + } + + cancelIndexing(): void { + throw new Error('not supported by memory DB'); + } + + reset(): Promise { + throw new Error('Method not implemented.'); + } +} diff --git a/backend/model/memory/PersonManager.ts b/backend/model/memory/PersonManager.ts new file mode 100644 index 0000000..b8c1265 --- /dev/null +++ b/backend/model/memory/PersonManager.ts @@ -0,0 +1,11 @@ +import {IPersonManager} from '../interfaces/IPersonManager'; + +export class IndexingTaskManager implements IPersonManager { + get(name: string): Promise { + throw new Error('not supported by memory DB'); + } + + saveAll(names: string[]): Promise { + throw new Error('not supported by memory DB'); + } +} diff --git a/backend/model/sql/GalleryManager.ts b/backend/model/sql/GalleryManager.ts index 752eb13..146ea9e 100644 --- a/backend/model/sql/GalleryManager.ts +++ b/backend/model/sql/GalleryManager.ts @@ -4,80 +4,26 @@ import * as path from 'path'; import * as fs from 'fs'; import {DirectoryEntity} from './enitites/DirectoryEntity'; import {SQLConnection} from './SQLConnection'; -import {DiskManager} from '../DiskManger'; import {PhotoEntity} from './enitites/PhotoEntity'; -import {Utils} from '../../../common/Utils'; import {ProjectPath} from '../../ProjectPath'; import {Config} from '../../../common/config/private/Config'; import {ISQLGalleryManager} from './IGalleryManager'; import {DatabaseType, ReIndexingSensitivity} from '../../../common/config/private/IPrivateConfig'; import {PhotoDTO} from '../../../common/entities/PhotoDTO'; import {OrientationType} from '../../../common/entities/RandomQueryDTO'; -import {Brackets, Connection, Transaction, TransactionRepository, Repository} from 'typeorm'; +import {Brackets, Connection} from 'typeorm'; import {MediaEntity} from './enitites/MediaEntity'; -import {MediaDTO} from '../../../common/entities/MediaDTO'; import {VideoEntity} from './enitites/VideoEntity'; -import {FileEntity} from './enitites/FileEntity'; -import {FileDTO} from '../../../common/entities/FileDTO'; -import {NotificationManager} from '../NotifocationManager'; import {DiskMangerWorker} from '../threading/DiskMangerWorker'; import {Logger} from '../../Logger'; +import {FaceRegionEntry} from './enitites/FaceRegionEntry'; +import {ObjectManagerRepository} from '../ObjectManagerRepository'; +import {DuplicatesDTO} from '../../../common/entities/DuplicatesDTO'; const LOG_TAG = '[GalleryManager]'; export class GalleryManager implements IGalleryManager, ISQLGalleryManager { - private savingQueue: DirectoryDTO[] = []; - private isSaving = false; - - protected async selectParentDir(connection: Connection, directoryName: string, directoryParent: string): Promise { - const query = connection - .getRepository(DirectoryEntity) - .createQueryBuilder('directory') - .where('directory.name = :name AND directory.path = :path', { - name: directoryName, - path: directoryParent - }) - .leftJoinAndSelect('directory.directories', 'directories') - .leftJoinAndSelect('directory.media', 'media'); - - if (Config.Client.MetaFile.enabled === true) { - query.leftJoinAndSelect('directory.metaFile', 'metaFile'); - } - - return await query.getOne(); - } - - protected async fillParentDir(connection: Connection, dir: DirectoryEntity): Promise { - if (dir.media) { - for (let i = 0; i < dir.media.length; i++) { - dir.media[i].directory = dir; - dir.media[i].readyThumbnails = []; - dir.media[i].readyIcon = false; - } - } - if (dir.directories) { - for (let i = 0; i < dir.directories.length; i++) { - dir.directories[i].media = await connection - .getRepository(MediaEntity) - .createQueryBuilder('media') - .where('media.directory = :dir', { - dir: dir.directories[i].id - }) - .orderBy('media.metadata.creationDate', 'ASC') - .limit(Config.Server.indexing.folderPreviewSize) - .getMany(); - dir.directories[i].isPartial = true; - - for (let j = 0; j < dir.directories[i].media.length; j++) { - dir.directories[i].media[j].directory = dir.directories[i]; - dir.directories[i].media[j].readyThumbnails = []; - dir.directories[i].media[j].readyIcon = false; - } - } - } - } - public async listDirectory(relativeDirectoryName: string, knownLastModified?: number, @@ -110,7 +56,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { if (dir.lastModified !== lastModified) { Logger.silly(LOG_TAG, 'Reindexing reason: lastModified mismatch: known: ' + dir.lastModified + ', current:' + lastModified); - return this.indexDirectory(relativeDirectoryName); + return ObjectManagerRepository.getInstance().IndexingManager.indexDirectory(relativeDirectoryName); } @@ -121,8 +67,8 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { // on the fly reindexing Logger.silly(LOG_TAG, 'lazy reindexing reason: cache timeout: lastScanned: ' - + (Date.now() - dir.lastScanned) + ', cachedFolderTimeout:' + Config.Server.indexing.cachedFolderTimeout); - this.indexDirectory(relativeDirectoryName).catch((err) => { + + (Date.now() - dir.lastScanned) + ' ms ago, cachedFolderTimeout:' + Config.Server.indexing.cachedFolderTimeout); + ObjectManagerRepository.getInstance().IndexingManager.indexDirectory(relativeDirectoryName).catch((err) => { console.error(err); }); } @@ -132,33 +78,11 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { // never scanned (deep indexed), do it and return with it Logger.silly(LOG_TAG, 'Reindexing reason: never scanned'); - return this.indexDirectory(relativeDirectoryName); + return ObjectManagerRepository.getInstance().IndexingManager.indexDirectory(relativeDirectoryName); } - - public indexDirectory(relativeDirectoryName: string): Promise { - return new Promise(async (resolve, reject) => { - try { - const scannedDirectory = await DiskManager.scanDirectory(relativeDirectoryName); - - // returning with the result - scannedDirectory.media.forEach(p => p.readyThumbnails = []); - resolve(scannedDirectory); - - this.queueForSave(scannedDirectory).catch(console.error); - - } catch (error) { - NotificationManager.warning('Unknown indexing error for: ' + relativeDirectoryName, error.toString()); - console.error(error); - return reject(error); - } - - }); - } - - public async getRandomPhoto(queryFilter: RandomQuery): Promise { const connection = await SQLConnection.getConnection(); const photosRepository = connection.getRepository(PhotoEntity); @@ -216,164 +140,6 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { } - // Todo fix it, once typeorm support connection pools ofr sqlite - protected async queueForSave(scannedDirectory: DirectoryDTO) { - if (this.savingQueue.findIndex(dir => dir.name === scannedDirectory.name && - dir.path === scannedDirectory.path && - dir.lastModified === scannedDirectory.lastModified && - dir.lastScanned === scannedDirectory.lastScanned && - (dir.media || dir.media.length) === (scannedDirectory.media || scannedDirectory.media.length) && - (dir.metaFile || dir.metaFile.length) === (scannedDirectory.metaFile || scannedDirectory.metaFile.length)) !== -1) { - return; - } - this.savingQueue.push(scannedDirectory); - while (this.isSaving === false && this.savingQueue.length > 0) { - await this.saveToDB(this.savingQueue[0]); - this.savingQueue.shift(); - } - - } - - protected async saveToDB(scannedDirectory: DirectoryDTO) { - this.isSaving = true; - try { - const connection = await SQLConnection.getConnection(); - - // saving to db - const directoryRepository = connection.getRepository(DirectoryEntity); - const mediaRepository = connection.getRepository(MediaEntity); - const fileRepository = connection.getRepository(FileEntity); - - - let currentDir: DirectoryEntity = await directoryRepository.createQueryBuilder('directory') - .where('directory.name = :name AND directory.path = :path', { - name: scannedDirectory.name, - path: scannedDirectory.path - }).getOne(); - if (!!currentDir) {// Updated parent dir (if it was in the DB previously) - currentDir.lastModified = scannedDirectory.lastModified; - currentDir.lastScanned = scannedDirectory.lastScanned; - currentDir.mediaCount = scannedDirectory.mediaCount; - currentDir = await directoryRepository.save(currentDir); - } else { - currentDir = await directoryRepository.save(scannedDirectory); - } - - - // save subdirectories - const childDirectories = await directoryRepository.createQueryBuilder('directory') - .where('directory.parent = :dir', { - dir: currentDir.id - }).getMany(); - - for (let i = 0; i < scannedDirectory.directories.length; i++) { - // Was this child Dir already indexed before? - let directory: DirectoryEntity = null; - for (let j = 0; j < childDirectories.length; j++) { - if (childDirectories[j].name === scannedDirectory.directories[i].name) { - directory = childDirectories[j]; - childDirectories.splice(j, 1); - break; - } - } - - if (directory != null) { // update existing directory - if (!directory.parent || !directory.parent.id) { // set parent if not set yet - directory.parent = currentDir; - delete directory.media; - await directoryRepository.save(directory); - } - } else { // dir does not exists yet - scannedDirectory.directories[i].parent = currentDir; - (scannedDirectory.directories[i]).lastScanned = null; // new child dir, not fully scanned yet - const d = await directoryRepository.save(scannedDirectory.directories[i]); - for (let j = 0; j < scannedDirectory.directories[i].media.length; j++) { - scannedDirectory.directories[i].media[j].directory = d; - } - - await this.saveMedia(connection, scannedDirectory.directories[i].media); - } - } - - // Remove child Dirs that are not anymore in the parent dir - await directoryRepository.remove(childDirectories, {chunk: Math.max(Math.ceil(childDirectories.length / 500), 1)}); - - // save media - const indexedMedia = await mediaRepository.createQueryBuilder('media') - .where('media.directory = :dir', { - dir: currentDir.id - }).getMany(); - - - const mediaToSave = []; - for (let i = 0; i < scannedDirectory.media.length; i++) { - let media: MediaDTO = null; - for (let j = 0; j < indexedMedia.length; j++) { - if (indexedMedia[j].name === scannedDirectory.media[i].name) { - media = indexedMedia[j]; - indexedMedia.splice(j, 1); - break; - } - } - if (media == null) { // not in DB yet - scannedDirectory.media[i].directory = null; - media = Utils.clone(scannedDirectory.media[i]); - scannedDirectory.media[i].directory = scannedDirectory; - media.directory = currentDir; - mediaToSave.push(media); - } else if (!Utils.equalsFilter(media.metadata, scannedDirectory.media[i].metadata)) { - media.metadata = scannedDirectory.media[i].metadata; - mediaToSave.push(media); - } - } - await this.saveMedia(connection, mediaToSave); - await mediaRepository.remove(indexedMedia); - - - // save files - const indexedMetaFiles = await fileRepository.createQueryBuilder('file') - .where('file.directory = :dir', { - dir: currentDir.id - }).getMany(); - - - const metaFilesToSave = []; - for (let i = 0; i < scannedDirectory.metaFile.length; i++) { - let metaFile: FileDTO = null; - for (let j = 0; j < indexedMetaFiles.length; j++) { - if (indexedMetaFiles[j].name === scannedDirectory.metaFile[i].name) { - metaFile = indexedMetaFiles[j]; - indexedMetaFiles.splice(j, 1); - break; - } - } - if (metaFile == null) { // not in DB yet - scannedDirectory.metaFile[i].directory = null; - metaFile = Utils.clone(scannedDirectory.metaFile[i]); - scannedDirectory.metaFile[i].directory = scannedDirectory; - metaFile.directory = currentDir; - metaFilesToSave.push(metaFile); - } - } - await fileRepository.save(metaFilesToSave, {chunk: Math.max(Math.ceil(metaFilesToSave.length / 500), 1)}); - await fileRepository.remove(indexedMetaFiles, {chunk: Math.max(Math.ceil(indexedMetaFiles.length / 500), 1)}); - } catch (e) { - throw e; - } finally { - this.isSaving = false; - } - } - - protected async saveMedia(connection: Connection, mediaList: MediaDTO[]): Promise { - const chunked = Utils.chunkArrays(mediaList, 100); - let list: MediaEntity[] = []; - for (let i = 0; i < chunked.length; i++) { - list = list.concat(await connection.getRepository(PhotoEntity).save(chunked[i].filter(m => MediaDTO.isPhoto(m)))); - list = list.concat(await connection.getRepository(VideoEntity).save(chunked[i].filter(m => MediaDTO.isVideo(m)))); - } - return list; - } - async countDirectories(): Promise { const connection = await SQLConnection.getConnection(); return await connection.getRepository(DirectoryEntity) @@ -404,5 +170,144 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { .getCount(); } + public async getPossibleDuplicates() { + const connection = await SQLConnection.getConnection(); + const mediaRepository = connection.getRepository(MediaEntity); + + let duplicates = await mediaRepository.createQueryBuilder('media') + .innerJoin(query => query.from(MediaEntity, 'innerMedia') + .select(['innerMedia.name as name', 'innerMedia.metadata.fileSize as fileSize', 'count(*)']) + .groupBy('innerMedia.name, innerMedia.metadata.fileSize') + .having('count(*)>1'), + 'innerMedia', + 'media.name=innerMedia.name AND media.metadata.fileSize = innerMedia.fileSize') + .innerJoinAndSelect('media.directory', 'directory') + .orderBy('media.name, media.metadata.fileSize') + .limit(Config.Server.duplicates.listingLimit).getMany(); + + + const duplicateParis: DuplicatesDTO[] = []; + const processDuplicates = (duplicateList: MediaEntity[], + equalFn: (a: MediaEntity, b: MediaEntity) => boolean, + checkDuplicates: boolean = false) => { + let i = duplicateList.length - 1; + while (i >= 0) { + const list = [duplicateList[i]]; + let j = i - 1; + while (j >= 0 && equalFn(duplicateList[i], duplicateList[j])) { + list.push(duplicateList[j]); + j--; + } + i = j; + // if we cut the select list with the SQL LIMIT, filter unpaired media + if (list.length < 2) { + continue; + } + if (checkDuplicates) { + // ad to group if one already existed + const foundDuplicates = duplicateParis.find(dp => + !!dp.media.find(m => + !!list.find(lm => lm.id === m.id))); + if (foundDuplicates) { + list.forEach(lm => { + if (!!foundDuplicates.media.find(m => m.id === lm.id)) { + return; + } + foundDuplicates.media.push(lm); + }); + continue; + } + } + + duplicateParis.push({media: list}); + } + }; + + processDuplicates(duplicates, + (a, b) => a.name === b.name && + a.metadata.fileSize === b.metadata.fileSize); + + + duplicates = await mediaRepository.createQueryBuilder('media') + .innerJoin(query => query.from(MediaEntity, 'innerMedia') + .select(['innerMedia.metadata.creationDate as creationDate', 'innerMedia.metadata.fileSize as fileSize', 'count(*)']) + .groupBy('innerMedia.metadata.creationDate, innerMedia.metadata.fileSize') + .having('count(*)>1'), + 'innerMedia', + 'media.metadata.creationDate=innerMedia.creationDate AND media.metadata.fileSize = innerMedia.fileSize') + .innerJoinAndSelect('media.directory', 'directory') + .orderBy('media.metadata.creationDate, media.metadata.fileSize') + .limit(Config.Server.duplicates.listingLimit).getMany(); + + processDuplicates(duplicates, + (a, b) => a.metadata.creationDate === b.metadata.creationDate && + a.metadata.fileSize === b.metadata.fileSize, true); + + return duplicateParis; + + } + + protected async selectParentDir(connection: Connection, directoryName: string, directoryParent: string): Promise { + const query = connection + .getRepository(DirectoryEntity) + .createQueryBuilder('directory') + .where('directory.name = :name AND directory.path = :path', { + name: directoryName, + path: directoryParent + }) + .leftJoinAndSelect('directory.directories', 'directories') + .leftJoinAndSelect('directory.media', 'media'); + + if (Config.Client.MetaFile.enabled === true) { + query.leftJoinAndSelect('directory.metaFile', 'metaFile'); + } + + return await query.getOne(); + } + + protected async fillParentDir(connection: Connection, dir: DirectoryEntity): Promise { + if (dir.media) { + const indexedFaces = await connection.getRepository(FaceRegionEntry) + .createQueryBuilder('face') + .leftJoinAndSelect('face.media', 'media') + .where('media.directory = :directory', { + directory: dir.id + }) + .leftJoinAndSelect('face.person', 'person') + .select(['face.id', 'face.box.x', + 'face.box.y', 'face.box.width', 'face.box.height', + 'media.id', 'person.name', 'person.id']) + .getMany(); + for (let i = 0; i < dir.media.length; i++) { + dir.media[i].directory = dir; + dir.media[i].readyThumbnails = []; + dir.media[i].readyIcon = false; + (dir.media[i]).metadata.faces = indexedFaces + .filter(fe => fe.media.id === dir.media[i].id) + .map(f => ({box: f.box, name: f.person.name})); + } + + } + if (dir.directories) { + for (let i = 0; i < dir.directories.length; i++) { + dir.directories[i].media = await connection + .getRepository(MediaEntity) + .createQueryBuilder('media') + .where('media.directory = :dir', { + dir: dir.directories[i].id + }) + .orderBy('media.metadata.creationDate', 'ASC') + .limit(Config.Server.indexing.folderPreviewSize) + .getMany(); + dir.directories[i].isPartial = true; + + for (let j = 0; j < dir.directories[i].media.length; j++) { + dir.directories[i].media[j].directory = dir.directories[i]; + dir.directories[i].media[j].readyThumbnails = []; + dir.directories[i].media[j].readyIcon = false; + } + } + } + } } diff --git a/backend/model/sql/IGalleryManager.ts b/backend/model/sql/IGalleryManager.ts index 8667c35..5dd5fa8 100644 --- a/backend/model/sql/IGalleryManager.ts +++ b/backend/model/sql/IGalleryManager.ts @@ -1,13 +1,12 @@ import {DirectoryDTO} from '../../../common/entities/DirectoryDTO'; import {IGalleryManager} from '../interfaces/IGalleryManager'; +import {DuplicatesDTO} from '../../../common/entities/DuplicatesDTO'; export interface ISQLGalleryManager extends IGalleryManager { listDirectory(relativeDirectoryName: string, knownLastModified?: number, knownLastScanned?: number): Promise; - indexDirectory(relativeDirectoryName: string): Promise; - countDirectories(): Promise; countPhotos(): Promise; @@ -15,4 +14,6 @@ export interface ISQLGalleryManager extends IGalleryManager { countVideos(): Promise; countMediaSize(): Promise; + + getPossibleDuplicates(): Promise; } diff --git a/backend/model/sql/IndexingManager.ts b/backend/model/sql/IndexingManager.ts index 21fb483..1c32db4 100644 --- a/backend/model/sql/IndexingManager.ts +++ b/backend/model/sql/IndexingManager.ts @@ -1,119 +1,325 @@ -import {IIndexingManager} from '../interfaces/IIndexingManager'; -import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO'; -import {ObjectManagerRepository} from '../ObjectManagerRepository'; -import {ISQLGalleryManager} from './IGalleryManager'; -import * as path from 'path'; -import * as fs from 'fs'; -import {SQLConnection} from './SQLConnection'; +import {DirectoryDTO} from '../../../common/entities/DirectoryDTO'; import {DirectoryEntity} from './enitites/DirectoryEntity'; -import {Logger} from '../../Logger'; -import {RendererInput, ThumbnailSourceType, ThumbnailWorker} from '../threading/ThumbnailWorker'; -import {Config} from '../../../common/config/private/Config'; +import {SQLConnection} from './SQLConnection'; +import {DiskManager} from '../DiskManger'; +import {PhotoEntity} from './enitites/PhotoEntity'; +import {Utils} from '../../../common/Utils'; +import {FaceRegion, PhotoMetadata} from '../../../common/entities/PhotoDTO'; +import {Connection, Repository} from 'typeorm'; +import {MediaEntity} from './enitites/MediaEntity'; import {MediaDTO} from '../../../common/entities/MediaDTO'; -import {ProjectPath} from '../../ProjectPath'; -import {ThumbnailGeneratorMWs} from '../../middlewares/thumbnail/ThumbnailGeneratorMWs'; +import {VideoEntity} from './enitites/VideoEntity'; +import {FileEntity} from './enitites/FileEntity'; +import {FileDTO} from '../../../common/entities/FileDTO'; +import {NotificationManager} from '../NotifocationManager'; +import {FaceRegionEntry} from './enitites/FaceRegionEntry'; +import {ObjectManagerRepository} from '../ObjectManagerRepository'; const LOG_TAG = '[IndexingManager]'; -export class IndexingManager implements IIndexingManager { - directoriesToIndex: string[] = []; - indexingProgress: IndexingProgressDTO = null; - enabled = false; - private indexNewDirectory = async (createThumbnails: boolean = false) => { - if (this.directoriesToIndex.length === 0) { - this.indexingProgress = null; - if (global.gc) { - global.gc(); - } - return; - } - const directory = this.directoriesToIndex.shift(); - this.indexingProgress.current = directory; - this.indexingProgress.left = this.directoriesToIndex.length; - const scanned = await (ObjectManagerRepository.getInstance().GalleryManager).indexDirectory(directory); - if (this.enabled === false) { - return; - } - this.indexingProgress.indexed++; - this.indexingProgress.time.current = Date.now(); - for (let i = 0; i < scanned.directories.length; i++) { - this.directoriesToIndex.push(path.join(scanned.directories[i].path, scanned.directories[i].name)); - } - if (createThumbnails) { - for (let i = 0; i < scanned.media.length; i++) { - try { - const media = scanned.media[i]; - const mPath = path.join(ProjectPath.ImageFolder, media.directory.path, media.directory.name, media.name); - const thPath = path.join(ProjectPath.ThumbnailFolder, - ThumbnailGeneratorMWs.generateThumbnailName(mPath, Config.Client.Thumbnail.thumbnailSizes[0])); - if (fs.existsSync(thPath)) { // skip existing thumbnails - continue; - } - await ThumbnailWorker.render({ - type: MediaDTO.isVideo(media) ? ThumbnailSourceType.Video : ThumbnailSourceType.Image, - mediaPath: mPath, - size: Config.Client.Thumbnail.thumbnailSizes[0], - thPath: thPath, - makeSquare: false, - qualityPriority: Config.Server.thumbnail.qualityPriority - }, Config.Server.thumbnail.processingLibrary); - } catch (e) { - console.error(e); - Logger.error(LOG_TAG, 'Error during indexing job: ' + e.toString()); - } +export class IndexingManager { + + private savingQueue: DirectoryDTO[] = []; + private isSaving = false; + + public indexDirectory(relativeDirectoryName: string): Promise { + return new Promise(async (resolve, reject) => { + try { + const scannedDirectory = await DiskManager.scanDirectory(relativeDirectoryName); + + // returning with the result + scannedDirectory.media.forEach(p => p.readyThumbnails = []); + resolve(scannedDirectory); + + this.queueForSave(scannedDirectory).catch(console.error); + + } catch (error) { + NotificationManager.warning('Unknown indexing error for: ' + relativeDirectoryName, error.toString()); + console.error(error); + return reject(error); } - } - process.nextTick(() => { - this.indexNewDirectory(createThumbnails); }); - }; + } + + // Todo fix it, once typeorm support connection pools ofr sqlite + protected async queueForSave(scannedDirectory: DirectoryDTO) { + if (this.savingQueue.findIndex(dir => dir.name === scannedDirectory.name && + dir.path === scannedDirectory.path && + dir.lastModified === scannedDirectory.lastModified && + dir.lastScanned === scannedDirectory.lastScanned && + (dir.media || dir.media.length) === (scannedDirectory.media || scannedDirectory.media.length) && + (dir.metaFile || dir.metaFile.length) === (scannedDirectory.metaFile || scannedDirectory.metaFile.length)) !== -1) { + return; + } + this.savingQueue.push(scannedDirectory); + while (this.isSaving === false && this.savingQueue.length > 0) { + await this.saveToDB(this.savingQueue[0]); + this.savingQueue.shift(); + } + + } + + protected async saveParentDir(connection: Connection, scannedDirectory: DirectoryDTO): Promise { + const directoryRepository = connection.getRepository(DirectoryEntity); + + const currentDir: DirectoryEntity = await directoryRepository.createQueryBuilder('directory') + .where('directory.name = :name AND directory.path = :path', { + name: scannedDirectory.name, + path: scannedDirectory.path + }).getOne(); + if (!!currentDir) {// Updated parent dir (if it was in the DB previously) + currentDir.lastModified = scannedDirectory.lastModified; + currentDir.lastScanned = scannedDirectory.lastScanned; + currentDir.mediaCount = scannedDirectory.mediaCount; + await directoryRepository.save(currentDir); + return currentDir.id; - startIndexing(createThumbnails: boolean = false): void { - if (this.directoriesToIndex.length === 0 && this.enabled === false) { - Logger.info(LOG_TAG, 'Starting indexing'); - this.indexingProgress = { - indexed: 0, - left: 0, - current: '', - time: { - start: Date.now(), - current: Date.now() - } - }; - this.directoriesToIndex.push('/'); - this.enabled = true; - this.indexNewDirectory(createThumbnails); } else { - Logger.info(LOG_TAG, 'Already indexing..'); + return (await directoryRepository.insert({ + mediaCount: scannedDirectory.mediaCount, + lastModified: scannedDirectory.lastModified, + lastScanned: scannedDirectory.lastScanned, + name: scannedDirectory.name, + path: scannedDirectory.path + })).identifiers[0].id; } } - getProgress(): IndexingProgressDTO { - return this.indexingProgress; + protected async saveChildDirs(connection: Connection, currentDirId: number, scannedDirectory: DirectoryDTO) { + const directoryRepository = connection.getRepository(DirectoryEntity); + // TODO: fix when first opened directory is not root + // save subdirectories + const childDirectories = await directoryRepository.createQueryBuilder('directory') + .where('directory.parent = :dir', { + dir: currentDirId + }).getMany(); + + for (let i = 0; i < scannedDirectory.directories.length; i++) { + // Was this child Dir already indexed before? + let directory: DirectoryEntity = null; + for (let j = 0; j < childDirectories.length; j++) { + if (childDirectories[j].name === scannedDirectory.directories[i].name) { + directory = childDirectories[j]; + childDirectories.splice(j, 1); + break; + } + } + + if (directory != null) { // update existing directory + if (!directory.parent || !directory.parent.id) { // set parent if not set yet + directory.parent = {id: currentDirId}; + delete directory.media; + await directoryRepository.save(directory); + } + } else { // dir does not exists yet + scannedDirectory.directories[i].parent = {id: currentDirId}; + (scannedDirectory.directories[i]).lastScanned = null; // new child dir, not fully scanned yet + const d = await directoryRepository.insert(scannedDirectory.directories[i]); + + await this.saveMedia(connection, d.identifiers[0].id, scannedDirectory.directories[i].media); + } + } + + // Remove child Dirs that are not anymore in the parent dir + await directoryRepository.remove(childDirectories, {chunk: Math.max(Math.ceil(childDirectories.length / 500), 1)}); + } - cancelIndexing(): void { - Logger.info(LOG_TAG, 'Canceling indexing'); - this.directoriesToIndex = []; - this.indexingProgress = null; - this.enabled = false; - if (global.gc) { - global.gc(); + protected async saveMetaFiles(connection: Connection, currentDirID: number, scannedDirectory: DirectoryDTO) { + const fileRepository = connection.getRepository(FileEntity); + // save files + const indexedMetaFiles = await fileRepository.createQueryBuilder('file') + .where('file.directory = :dir', { + dir: currentDirID + }).getMany(); + + + const metaFilesToSave = []; + for (let i = 0; i < scannedDirectory.metaFile.length; i++) { + let metaFile: FileDTO = null; + for (let j = 0; j < indexedMetaFiles.length; j++) { + if (indexedMetaFiles[j].name === scannedDirectory.metaFile[i].name) { + metaFile = indexedMetaFiles[j]; + indexedMetaFiles.splice(j, 1); + break; + } + } + if (metaFile == null) { // not in DB yet + scannedDirectory.metaFile[i].directory = null; + metaFile = Utils.clone(scannedDirectory.metaFile[i]); + scannedDirectory.metaFile[i].directory = scannedDirectory; + metaFile.directory = {id: currentDirID}; + metaFilesToSave.push(metaFile); + } + } + await fileRepository.save(metaFilesToSave, {chunk: Math.max(Math.ceil(metaFilesToSave.length / 500), 1)}); + await fileRepository.remove(indexedMetaFiles, {chunk: Math.max(Math.ceil(indexedMetaFiles.length / 500), 1)}); + } + + protected async saveMedia(connection: Connection, parentDirId: number, media: MediaDTO[]) { + const mediaRepository = connection.getRepository(MediaEntity); + const photoRepository = connection.getRepository(PhotoEntity); + const videoRepository = connection.getRepository(VideoEntity); + // save media + let indexedMedia = (await mediaRepository.createQueryBuilder('media') + .where('media.directory = :dir', { + dir: parentDirId + }) + .getMany()); + + const mediaChange: any = { + saveP: [], + saveV: [], + insertP: [], + insertV: [] + }; + const facesPerPhoto: { faces: FaceRegionEntry[], mediaName: string }[] = []; + for (let i = 0; i < media.length; i++) { + let mediaItem: MediaEntity = null; + for (let j = 0; j < indexedMedia.length; j++) { + if (indexedMedia[j].name === media[i].name) { + mediaItem = indexedMedia[j]; + indexedMedia.splice(j, 1); + break; + } + } + + const scannedFaces = (media[i].metadata).faces || []; + delete (media[i].metadata).faces; + + // let mediaItemId: number = null; + if (mediaItem == null) { // not in DB yet + media[i].directory = null; + mediaItem = Utils.clone(media[i]); + mediaItem.directory = {id: parentDirId}; + (MediaDTO.isPhoto(mediaItem) ? mediaChange.insertP : mediaChange.insertV).push(mediaItem); + } else { + delete (mediaItem.metadata).faces; + if (!Utils.equalsFilter(mediaItem.metadata, media[i].metadata)) { + mediaItem.metadata = media[i].metadata; + (MediaDTO.isPhoto(mediaItem) ? mediaChange.saveP : mediaChange.saveV).push(mediaItem); + + } + } + + facesPerPhoto.push({faces: scannedFaces as FaceRegionEntry[], mediaName: mediaItem.name}); + } + + await this.saveChunk(photoRepository, mediaChange.saveP, 100); + await this.saveChunk(videoRepository, mediaChange.saveV, 100); + await this.saveChunk(photoRepository, mediaChange.insertP, 100); + await this.saveChunk(videoRepository, mediaChange.insertV, 100); + + indexedMedia = (await mediaRepository.createQueryBuilder('media') + .where('media.directory = :dir', { + dir: parentDirId + }) + .select(['media.name', 'media.id']) + .getMany()); + + const faces: FaceRegionEntry[] = []; + facesPerPhoto.forEach(group => { + const mIndex = indexedMedia.findIndex(m => m.name === group.mediaName); + group.faces.forEach((sf: FaceRegionEntry) => sf.media = {id: indexedMedia[mIndex].id}); + + faces.push(...group.faces); + indexedMedia.splice(mIndex, 1); + }); + + await this.saveFaces(connection, parentDirId, faces); + await mediaRepository.remove(indexedMedia); + } + + protected async saveFaces(connection: Connection, parentDirId: number, scannedFaces: FaceRegion[]) { + const faceRepository = connection.getRepository(FaceRegionEntry); + + const persons: string[] = []; + + for (let i = 0; i < scannedFaces.length; i++) { + if (persons.indexOf(scannedFaces[i].name) === -1) { + persons.push(scannedFaces[i].name); + } + } + await ObjectManagerRepository.getInstance().PersonManager.saveAll(persons); + + + const indexedFaces = await faceRepository.createQueryBuilder('face') + .leftJoin('face.media', 'media') + .where('media.directory = :directory', { + directory: parentDirId + }) + .leftJoinAndSelect('face.person', 'person') + .getMany(); + + + const faceToInsert = []; + for (let i = 0; i < scannedFaces.length; i++) { + let face: FaceRegionEntry = null; + for (let j = 0; j < indexedFaces.length; j++) { + if (indexedFaces[j].box.height === scannedFaces[i].box.height && + indexedFaces[j].box.width === scannedFaces[i].box.width && + indexedFaces[j].box.x === scannedFaces[i].box.x && + indexedFaces[j].box.y === scannedFaces[i].box.y && + indexedFaces[j].person.name === scannedFaces[i].name) { + face = indexedFaces[j]; + indexedFaces.splice(j, 1); + break; + } + } + + if (face == null) { + (scannedFaces[i]).person = await ObjectManagerRepository.getInstance().PersonManager.get(scannedFaces[i].name); + faceToInsert.push(scannedFaces[i]); + } + } + if (faceToInsert.length > 0) { + await this.insertChunk(faceRepository, faceToInsert, 100); + } + await faceRepository.remove(indexedFaces, {chunk: Math.max(Math.ceil(indexedFaces.length / 500), 1)}); + + } + + protected async saveToDB(scannedDirectory: DirectoryDTO): Promise { + this.isSaving = true; + try { + const connection = await SQLConnection.getConnection(); + const currentDirId: number = await this.saveParentDir(connection, scannedDirectory); + await this.saveChildDirs(connection, currentDirId, scannedDirectory); + await this.saveMedia(connection, currentDirId, scannedDirectory.media); + await this.saveMetaFiles(connection, currentDirId, scannedDirectory); + } catch (e) { + throw e; + } finally { + this.isSaving = false; } } - async reset(): Promise { - Logger.info(LOG_TAG, 'Resetting DB'); - this.directoriesToIndex = []; - this.indexingProgress = null; - this.enabled = false; - const connection = await SQLConnection.getConnection(); - return connection - .getRepository(DirectoryEntity) - .createQueryBuilder('directory') - .delete() - .execute().then(() => { - }); + private async saveChunk(repository: Repository, entities: T[], size: number): Promise { + if (entities.length === 0) { + return []; + } + if (entities.length < size) { + return await repository.save(entities); + } + let list: T[] = []; + for (let i = 0; i < entities.length / size; i++) { + list = list.concat(await repository.save(entities.slice(i * size, (i + 1) * size))); + } + return list; + } + + private async insertChunk(repository: Repository, entities: T[], size: number): Promise { + if (entities.length === 0) { + return []; + } + if (entities.length < size) { + return (await repository.insert(entities)).identifiers.map((i: any) => i.id); + } + let list: number[] = []; + for (let i = 0; i < entities.length / size; i++) { + list = list.concat((await repository.insert(entities.slice(i * size, (i + 1) * size))).identifiers.map(ids => ids.id)); + } + return list; } } diff --git a/backend/model/sql/IndexingTaskManager.ts b/backend/model/sql/IndexingTaskManager.ts new file mode 100644 index 0000000..ea88b6d --- /dev/null +++ b/backend/model/sql/IndexingTaskManager.ts @@ -0,0 +1,118 @@ +import {IIndexingTaskManager} from '../interfaces/IIndexingTaskManager'; +import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO'; +import {ObjectManagerRepository} from '../ObjectManagerRepository'; +import * as path from 'path'; +import * as fs from 'fs'; +import {SQLConnection} from './SQLConnection'; +import {DirectoryEntity} from './enitites/DirectoryEntity'; +import {Logger} from '../../Logger'; +import {RendererInput, ThumbnailSourceType, ThumbnailWorker} from '../threading/ThumbnailWorker'; +import {Config} from '../../../common/config/private/Config'; +import {MediaDTO} from '../../../common/entities/MediaDTO'; +import {ProjectPath} from '../../ProjectPath'; +import {ThumbnailGeneratorMWs} from '../../middlewares/thumbnail/ThumbnailGeneratorMWs'; + +const LOG_TAG = '[IndexingTaskManager]'; + +export class IndexingTaskManager implements IIndexingTaskManager { + directoriesToIndex: string[] = []; + indexingProgress: IndexingProgressDTO = null; + enabled = false; + private indexNewDirectory = async (createThumbnails: boolean = false) => { + if (this.directoriesToIndex.length === 0) { + this.indexingProgress = null; + if (global.gc) { + global.gc(); + } + return; + } + const directory = this.directoriesToIndex.shift(); + this.indexingProgress.current = directory; + this.indexingProgress.left = this.directoriesToIndex.length; + const scanned = await ObjectManagerRepository.getInstance().IndexingManager.indexDirectory(directory); + if (this.enabled === false) { + return; + } + this.indexingProgress.indexed++; + this.indexingProgress.time.current = Date.now(); + for (let i = 0; i < scanned.directories.length; i++) { + this.directoriesToIndex.push(path.join(scanned.directories[i].path, scanned.directories[i].name)); + } + if (createThumbnails) { + for (let i = 0; i < scanned.media.length; i++) { + try { + const media = scanned.media[i]; + const mPath = path.join(ProjectPath.ImageFolder, media.directory.path, media.directory.name, media.name); + const thPath = path.join(ProjectPath.ThumbnailFolder, + ThumbnailGeneratorMWs.generateThumbnailName(mPath, Config.Client.Thumbnail.thumbnailSizes[0])); + if (fs.existsSync(thPath)) { // skip existing thumbnails + continue; + } + await ThumbnailWorker.render({ + type: MediaDTO.isVideo(media) ? ThumbnailSourceType.Video : ThumbnailSourceType.Image, + mediaPath: mPath, + size: Config.Client.Thumbnail.thumbnailSizes[0], + thPath: thPath, + makeSquare: false, + qualityPriority: Config.Server.thumbnail.qualityPriority + }, Config.Server.thumbnail.processingLibrary); + } catch (e) { + console.error(e); + Logger.error(LOG_TAG, 'Error during indexing job: ' + e.toString()); + } + } + + } + process.nextTick(() => { + this.indexNewDirectory(createThumbnails).catch(console.error); + }); + }; + + startIndexing(createThumbnails: boolean = false): void { + if (this.directoriesToIndex.length === 0 && this.enabled === false) { + Logger.info(LOG_TAG, 'Starting indexing'); + this.indexingProgress = { + indexed: 0, + left: 0, + current: '', + time: { + start: Date.now(), + current: Date.now() + } + }; + this.directoriesToIndex.push('/'); + this.enabled = true; + this.indexNewDirectory(createThumbnails).catch(console.error); + } else { + Logger.info(LOG_TAG, 'Already indexing..'); + } + } + + getProgress(): IndexingProgressDTO { + return this.indexingProgress; + } + + cancelIndexing(): void { + Logger.info(LOG_TAG, 'Canceling indexing'); + this.directoriesToIndex = []; + this.indexingProgress = null; + this.enabled = false; + if (global.gc) { + global.gc(); + } + } + + async reset(): Promise { + Logger.info(LOG_TAG, 'Resetting DB'); + this.directoriesToIndex = []; + this.indexingProgress = null; + this.enabled = false; + const connection = await SQLConnection.getConnection(); + return connection + .getRepository(DirectoryEntity) + .createQueryBuilder('directory') + .delete() + .execute().then(() => { + }); + } +} diff --git a/backend/model/sql/PersonManager.ts b/backend/model/sql/PersonManager.ts new file mode 100644 index 0000000..e13c8f4 --- /dev/null +++ b/backend/model/sql/PersonManager.ts @@ -0,0 +1,50 @@ +import {IPersonManager} from '../interfaces/IPersonManager'; +import {PersonEntry} from './enitites/PersonEntry'; +import {SQLConnection} from './SQLConnection'; + +const LOG_TAG = '[PersonManager]'; + +export class PersonManager implements IPersonManager { + + persons: PersonEntry[] = []; + + async get(name: string): Promise { + + let person = this.persons.find(p => p.name === name); + if (!person) { + const connection = await SQLConnection.getConnection(); + const personRepository = connection.getRepository(PersonEntry); + person = await personRepository.findOne({name: name}); + if (!person) { + person = await personRepository.save({name: name}); + } + this.persons.push(person); + } + return person; + } + + + async saveAll(names: string[]): Promise { + const toSave: { name: string }[] = []; + const connection = await SQLConnection.getConnection(); + const personRepository = connection.getRepository(PersonEntry); + this.persons = await personRepository.find(); + + for (let i = 0; i < names.length; i++) { + + const person = this.persons.find(p => p.name === names[i]); + if (!person) { + toSave.push({name: names[i]}); + } + } + + if (toSave.length > 0) { + for (let i = 0; i < toSave.length / 200; i++) { + await personRepository.insert(toSave.slice(i * 200, (i + 1) * 200)); + } + this.persons = await personRepository.find(); + } + + } + +} diff --git a/backend/model/sql/SQLConnection.ts b/backend/model/sql/SQLConnection.ts index 0afc79f..4ae051a 100644 --- a/backend/model/sql/SQLConnection.ts +++ b/backend/model/sql/SQLConnection.ts @@ -15,16 +15,19 @@ import {MediaEntity} from './enitites/MediaEntity'; import {VideoEntity} from './enitites/VideoEntity'; import {DataStructureVersion} from '../../../common/DataStructureVersion'; import {FileEntity} from './enitites/FileEntity'; +import {FaceRegionEntry} from './enitites/FaceRegionEntry'; +import {PersonEntry} from './enitites/PersonEntry'; +import {Utils} from '../../../common/Utils'; export class SQLConnection { + private static connection: Connection = null; + constructor() { } - private static connection: Connection = null; - public static async getConnection(): Promise { if (this.connection == null) { const options: any = this.getDriver(Config.Server.database); @@ -32,6 +35,8 @@ export class SQLConnection { options.entities = [ UserEntity, FileEntity, + FaceRegionEntry, + PersonEntry, MediaEntity, PhotoEntity, VideoEntity, @@ -40,8 +45,10 @@ export class SQLConnection { VersionEntity ]; options.synchronize = false; - // options.logging = 'all'; - this.connection = await createConnection(options); + // options.logging = 'all'; + + + this.connection = await this.createConnection(options); await SQLConnection.schemeSync(this.connection); } return this.connection; @@ -57,6 +64,8 @@ export class SQLConnection { options.entities = [ UserEntity, FileEntity, + FaceRegionEntry, + PersonEntry, MediaEntity, PhotoEntity, VideoEntity, @@ -66,7 +75,7 @@ export class SQLConnection { ]; options.synchronize = false; // options.logging = "all"; - const conn = await createConnection(options); + const conn = await this.createConnection(options); await SQLConnection.schemeSync(conn); await conn.close(); return true; @@ -86,6 +95,38 @@ export class SQLConnection { } + public static async close() { + try { + if (this.connection != null) { + await this.connection.close(); + this.connection = null; + } + } catch (err) { + console.error(err); + } + } + + private static async createConnection(options: ConnectionOptions) { + if (options.type === 'sqlite') { + return await createConnection(options); + } + try { + return await createConnection(options); + } catch (e) { + if (e.sqlMessage === 'Unknown database \'' + options.database + '\'') { + Logger.debug('creating database: ' + options.database); + const tmpOption = Utils.clone(options); + // @ts-ignore + delete tmpOption.database; + const tmpConn = await createConnection(tmpOption); + await tmpConn.query('CREATE DATABASE IF NOT EXISTS ' + options.database); + await tmpConn.close(); + return await createConnection(options); + } + throw e; + } + } + private static async schemeSync(connection: Connection) { let version = null; try { @@ -139,16 +180,5 @@ export class SQLConnection { return driver; } - public static async close() { - try { - if (this.connection != null) { - await this.connection.close(); - this.connection = null; - } - } catch (err) { - console.error(err); - } - } - } diff --git a/backend/model/sql/SearchManager.ts b/backend/model/sql/SearchManager.ts index b2c44de..12d7650 100644 --- a/backend/model/sql/SearchManager.ts +++ b/backend/model/sql/SearchManager.ts @@ -6,6 +6,10 @@ import {PhotoEntity} from './enitites/PhotoEntity'; import {DirectoryEntity} from './enitites/DirectoryEntity'; import {MediaEntity} from './enitites/MediaEntity'; import {VideoEntity} from './enitites/VideoEntity'; +import {PersonEntry} from './enitites/PersonEntry'; +import {FaceRegionEntry} from './enitites/FaceRegionEntry'; +import {SelectQueryBuilder} from 'typeorm'; +import {Config} from '../../../common/config/private/Config'; export class SearchManager implements ISearchManager { @@ -22,14 +26,14 @@ export class SearchManager implements ISearchManager { return a; } - async autocomplete(text: string): Promise> { + async autocomplete(text: string): Promise { const connection = await SQLConnection.getConnection(); let result: AutoCompleteItem[] = []; const photoRepository = connection.getRepository(PhotoEntity); const videoRepository = connection.getRepository(VideoEntity); - const mediaRepository = connection.getRepository(MediaEntity); + const personRepository = connection.getRepository(PersonEntry); const directoryRepository = connection.getRepository(DirectoryEntity); @@ -37,7 +41,7 @@ export class SearchManager implements ISearchManager { .createQueryBuilder('photo') .select('DISTINCT(photo.metadata.keywords)') .where('photo.metadata.keywords LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) - .limit(5) + .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory) .getRawMany()) .map(r => >(r.metadataKeywords).split(',')) .forEach(keywords => { @@ -45,6 +49,15 @@ export class SearchManager implements ISearchManager { .filter(k => k.toLowerCase().indexOf(text.toLowerCase()) !== -1), SearchTypes.keyword)); }); + result = result.concat(this.encapsulateAutoComplete((await personRepository + .createQueryBuilder('person') + .select('DISTINCT(person.name)') + .where('person.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) + .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory) + .orderBy('person.name') + .getRawMany()) + .map(r => r.name), SearchTypes.person)); + (await photoRepository .createQueryBuilder('photo') .select('photo.metadata.positionData.country as country, ' + @@ -53,7 +66,7 @@ export class SearchManager implements ISearchManager { .orWhere('photo.metadata.positionData.state LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) .orWhere('photo.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) .groupBy('photo.metadata.positionData.country, photo.metadata.positionData.state, photo.metadata.positionData.city') - .limit(5) + .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory) .getRawMany()) .filter(pm => !!pm) .map(pm => >[pm.city || '', pm.country || '', pm.state || '']) @@ -66,7 +79,7 @@ export class SearchManager implements ISearchManager { .createQueryBuilder('media') .select('DISTINCT(media.name)') .where('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) - .limit(5) + .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory) .getRawMany()) .map(r => r.name), SearchTypes.photo)); @@ -75,7 +88,7 @@ export class SearchManager implements ISearchManager { .createQueryBuilder('media') .select('DISTINCT(media.metadata.caption) as caption') .where('media.metadata.caption LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) - .limit(5) + .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory) .getRawMany()) .map(r => r.caption), SearchTypes.photo)); @@ -84,7 +97,7 @@ export class SearchManager implements ISearchManager { .createQueryBuilder('media') .select('DISTINCT(media.name)') .where('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) - .limit(5) + .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory) .getRawMany()) .map(r => r.name), SearchTypes.video)); @@ -92,7 +105,7 @@ export class SearchManager implements ISearchManager { .createQueryBuilder('dir') .select('DISTINCT(dir.name)') .where('dir.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) - .limit(5) + .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory) .getRawMany()) .map(r => r.name), SearchTypes.directory)); @@ -112,44 +125,60 @@ export class SearchManager implements ISearchManager { resultOverflow: false }; - let repostiroy = connection.getRepository(MediaEntity); + let usedEntity = MediaEntity; if (searchType === SearchTypes.photo) { - repostiroy = connection.getRepository(PhotoEntity); + usedEntity = PhotoEntity; } else if (searchType === SearchTypes.video) { - repostiroy = connection.getRepository(VideoEntity); + usedEntity = VideoEntity; } - const query = repostiroy.createQueryBuilder('media') - .innerJoinAndSelect('media.directory', 'directory') - .orderBy('media.metadata.creationDate', 'ASC'); + const query = await connection.getRepository(usedEntity).createQueryBuilder('media') + .innerJoin(q => { + const subQuery = q.from(usedEntity, 'media') + .select('distinct media.id') + .limit(2000); - if (!searchType || searchType === SearchTypes.directory) { - query.orWhere('directory.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}); - } + if (!searchType || searchType === SearchTypes.directory) { + subQuery.leftJoin('media.directory', 'directory') + .orWhere('directory.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}); + } - if (!searchType || searchType === SearchTypes.photo || searchType === SearchTypes.video) { - query.orWhere('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}); - } + if (!searchType || searchType === SearchTypes.photo || searchType === SearchTypes.video) { + subQuery.orWhere('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}); + } - if (!searchType || searchType === SearchTypes.photo) { - query.orWhere('media.metadata.caption LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}); - } + if (!searchType || searchType === SearchTypes.photo) { + subQuery.orWhere('media.metadata.caption LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}); + } + if (!searchType || searchType === SearchTypes.person) { + subQuery + .leftJoin('media.metadata.faces', 'faces') + .leftJoin('faces.person', 'person') + .orWhere('person.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}); + } - if (!searchType || searchType === SearchTypes.position) { - query.orWhere('media.metadata.positionData.country LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) - .orWhere('media.metadata.positionData.state LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) - .orWhere('media.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}); + if (!searchType || searchType === SearchTypes.position) { + subQuery.orWhere('media.metadata.positionData.country LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) + .orWhere('media.metadata.positionData.state LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) + .orWhere('media.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}); - } - if (!searchType || searchType === SearchTypes.keyword) { - query.orWhere('media.metadata.keywords LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}); - } + } + if (!searchType || searchType === SearchTypes.keyword) { + subQuery.orWhere('media.metadata.keywords LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}); + } - result.media = await query - .limit(2001) - .getMany(); + return subQuery; + }, + 'innerMedia', + 'media.id=innerMedia.id') + .leftJoinAndSelect('media.directory', 'directory') + .leftJoinAndSelect('media.metadata.faces', 'faces') + .leftJoinAndSelect('faces.person', 'person'); + + + result.media = await this.loadMediaWithFaces(query); if (result.media.length > 2000) { result.resultOverflow = true; @@ -181,19 +210,29 @@ export class SearchManager implements ISearchManager { resultOverflow: false }; - result.media = await connection - .getRepository(MediaEntity) - .createQueryBuilder('media') - .orderBy('media.metadata.creationDate', 'ASC') - .where('media.metadata.keywords LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) - .orWhere('media.metadata.positionData.country LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) - .orWhere('media.metadata.positionData.state LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) - .orWhere('media.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) - .orWhere('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) - .orWhere('media.metadata.caption LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) - .innerJoinAndSelect('media.directory', 'directory') - .limit(10) - .getMany(); + const query = await connection.getRepository(MediaEntity).createQueryBuilder('media') + .innerJoin(q => q.from(MediaEntity, 'media') + .select('distinct media.id') + .limit(10) + .leftJoin('media.directory', 'directory') + .leftJoin('media.metadata.faces', 'faces') + .leftJoin('faces.person', 'person') + .where('media.metadata.keywords LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) + .orWhere('media.metadata.positionData.country LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) + .orWhere('media.metadata.positionData.state LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) + .orWhere('media.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) + .orWhere('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) + .orWhere('media.metadata.caption LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) + .orWhere('person.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) + , + 'innerMedia', + 'media.id=innerMedia.id') + .leftJoinAndSelect('media.directory', 'directory') + .leftJoinAndSelect('media.metadata.faces', 'faces') + .leftJoinAndSelect('faces.person', 'person'); + + + result.media = await this.loadMediaWithFaces(query); result.directories = await connection @@ -213,4 +252,29 @@ export class SearchManager implements ISearchManager { }); return res; } + + private async loadMediaWithFaces(query: SelectQueryBuilder) { + const rawAndEntities = await query.orderBy('media.id').getRawAndEntities(); + const media: MediaEntity[] = rawAndEntities.entities; + + // console.log(rawAndEntities.raw); + let rawIndex = 0; + for (let i = 0; i < media.length; i++) { + if (rawAndEntities.raw[rawIndex].faces_id === null || + rawAndEntities.raw[rawIndex].media_id !== media[i].id) { + delete media[i].metadata.faces; + continue; + } + media[i].metadata.faces = []; + + while (rawAndEntities.raw[rawIndex].media_id === media[i].id) { + media[i].metadata.faces.push(FaceRegionEntry.fromRawToDTO(rawAndEntities.raw[rawIndex])); + rawIndex++; + if (rawIndex >= rawAndEntities.raw.length) { + return media; + } + } + } + return media; + } } diff --git a/backend/model/sql/enitites/DirectoryEntity.ts b/backend/model/sql/enitites/DirectoryEntity.ts index e43c633..717b53c 100644 --- a/backend/model/sql/enitites/DirectoryEntity.ts +++ b/backend/model/sql/enitites/DirectoryEntity.ts @@ -1,4 +1,4 @@ -import {Column, Entity, ManyToOne, OneToMany, PrimaryGeneratedColumn, Unique, Index} from 'typeorm'; +import {Column, Entity, Index, ManyToOne, OneToMany, PrimaryGeneratedColumn, Unique} from 'typeorm'; import {DirectoryDTO} from '../../../../common/entities/DirectoryDTO'; import {MediaEntity} from './MediaEntity'; import {FileEntity} from './FileEntity'; @@ -8,7 +8,7 @@ import {FileEntity} from './FileEntity'; export class DirectoryEntity implements DirectoryDTO { @Index() - @PrimaryGeneratedColumn() + @PrimaryGeneratedColumn({unsigned: true}) id: number; @Index() @@ -22,18 +22,28 @@ export class DirectoryEntity implements DirectoryDTO { /** * last time the directory was modified (from outside, eg.: a new media was added) */ - @Column('bigint') + @Column('bigint', { + unsigned: true, transformer: { + from: v => parseInt(v, 10), + to: v => v + } + }) public lastModified: number; /** * Last time the directory was fully scanned, not only for a few media to create a preview */ - @Column({type: 'bigint', nullable: true}) + @Column({ + type: 'bigint', nullable: true, unsigned: true, transformer: { + from: v => parseInt(v, 10), + to: v => v + } + }) public lastScanned: number; isPartial?: boolean; - @Column('smallint') + @Column('smallint', {unsigned: true}) mediaCount: number; @Index() diff --git a/backend/model/sql/enitites/FaceRegionEntry.ts b/backend/model/sql/enitites/FaceRegionEntry.ts new file mode 100644 index 0000000..411be7f --- /dev/null +++ b/backend/model/sql/enitites/FaceRegionEntry.ts @@ -0,0 +1,55 @@ +import {FaceRegion, FaceRegionBox} from '../../../../common/entities/PhotoDTO'; +import {Column, Entity, ManyToOne, PrimaryGeneratedColumn} from 'typeorm'; +import {PersonEntry} from './PersonEntry'; +import {MediaEntity} from './MediaEntity'; + +export class FaceRegionBoxEntry implements FaceRegionBox { + @Column('int') + height: number; + @Column('int') + width: number; + @Column('int') + x: number; + @Column('int') + y: number; +} + +/** + * This is a switching table between media and persons + */ +@Entity() +export class FaceRegionEntry { + + @PrimaryGeneratedColumn({unsigned: true}) + id: number; + + @Column(type => FaceRegionBoxEntry) + box: FaceRegionBoxEntry; + + // @PrimaryColumn('int') + @ManyToOne(type => MediaEntity, media => media.metadata.faces, {onDelete: 'CASCADE', nullable: false}) + media: MediaEntity; + + // @PrimaryColumn('int') + @ManyToOne(type => PersonEntry, person => person.faces, {onDelete: 'CASCADE', nullable: false}) + person: PersonEntry; + + name: string; + + public static fromRawToDTO(raw: { + faces_id: number, + faces_mediaId: number, + faces_personId: number, + faces_boxHeight: number, + faces_boxWidth: number, + faces_boxX: number, + faces_boxY: number, + person_id: number, + person_name: string + }): FaceRegion { + return { + box: {width: raw.faces_boxWidth, height: raw.faces_boxHeight, x: raw.faces_boxX, y: raw.faces_boxY}, + name: raw.person_name + }; + } +} diff --git a/backend/model/sql/enitites/FileEntity.ts b/backend/model/sql/enitites/FileEntity.ts index 0d19efd..b185cfe 100644 --- a/backend/model/sql/enitites/FileEntity.ts +++ b/backend/model/sql/enitites/FileEntity.ts @@ -7,7 +7,7 @@ import {FileDTO} from '../../../../common/entities/FileDTO'; export class FileEntity implements FileDTO { @Index() - @PrimaryGeneratedColumn() + @PrimaryGeneratedColumn({unsigned: true}) id: number; @Column('text') diff --git a/backend/model/sql/enitites/MediaEntity.ts b/backend/model/sql/enitites/MediaEntity.ts index 2a1aa00..10d3693 100644 --- a/backend/model/sql/enitites/MediaEntity.ts +++ b/backend/model/sql/enitites/MediaEntity.ts @@ -1,8 +1,9 @@ -import {Column, Entity, ManyToOne, PrimaryGeneratedColumn, TableInheritance, Unique, Index} from 'typeorm'; +import {Column, Entity, Index, ManyToOne, OneToMany, PrimaryGeneratedColumn, TableInheritance, Unique} from 'typeorm'; import {DirectoryEntity} from './DirectoryEntity'; import {MediaDimension, MediaDTO, MediaMetadata} from '../../../../common/entities/MediaDTO'; import {OrientationTypes} from 'ts-exif-parser'; import {CameraMetadataEntity, PositionMetaDataEntity} from './PhotoEntity'; +import {FaceRegionEntry} from './FaceRegionEntry'; export class MediaDimensionEntity implements MediaDimension { @@ -21,13 +22,17 @@ export class MediaMetadataEntity implements MediaMetadata { @Column(type => MediaDimensionEntity) size: MediaDimensionEntity; - @Column('bigint') + @Column('bigint', { + unsigned: true, transformer: { + from: v => parseInt(v, 10), + to: v => v + } + }) creationDate: number; - @Column('int') + @Column('int', {unsigned: true}) fileSize: number; - @Column('simple-array') keywords: string[]; @@ -37,27 +42,30 @@ export class MediaMetadataEntity implements MediaMetadata { @Column(type => PositionMetaDataEntity) positionData: PositionMetaDataEntity; - @Column('tinyint', {default: OrientationTypes.TOP_LEFT}) + @Column('tinyint', {unsigned: true, default: OrientationTypes.TOP_LEFT}) orientation: OrientationTypes; - @Column('int') + @OneToMany(type => FaceRegionEntry, faceRegion => faceRegion.media) + faces: FaceRegionEntry[]; + + @Column('int', {unsigned: true}) bitRate: number; - @Column('bigint') + @Column('int', {unsigned: true}) duration: number; } // TODO: fix inheritance once its working in typeorm @Entity() @Unique(['name', 'directory']) -@TableInheritance({column: {type: 'varchar', name: 'type'}}) +@TableInheritance({column: {type: 'varchar', name: 'type', length: 32}}) export abstract class MediaEntity implements MediaDTO { @Index() - @PrimaryGeneratedColumn() + @PrimaryGeneratedColumn({unsigned: true}) id: number; - @Column('text') + @Column() name: string; @Index() diff --git a/backend/model/sql/enitites/PersonEntry.ts b/backend/model/sql/enitites/PersonEntry.ts new file mode 100644 index 0000000..2826453 --- /dev/null +++ b/backend/model/sql/enitites/PersonEntry.ts @@ -0,0 +1,17 @@ +import {Column, Entity, OneToMany, PrimaryGeneratedColumn, Unique, Index} from 'typeorm'; +import {FaceRegionEntry} from './FaceRegionEntry'; + + +@Entity() +@Unique(['name']) +export class PersonEntry { + @Index() + @PrimaryGeneratedColumn({unsigned: true}) + id: number; + + @Column() + name: string; + + @OneToMany(type => FaceRegionEntry, faceRegion => faceRegion.person) + public faces: FaceRegionEntry[]; +} diff --git a/backend/model/sql/enitites/PhotoEntity.ts b/backend/model/sql/enitites/PhotoEntity.ts index fa9be6f..4fe4915 100644 --- a/backend/model/sql/enitites/PhotoEntity.ts +++ b/backend/model/sql/enitites/PhotoEntity.ts @@ -1,6 +1,13 @@ import {Column, Entity, ChildEntity, Unique} from 'typeorm'; -import {CameraMetadata, GPSMetadata, PhotoDTO, PhotoMetadata, PositionMetaData} from '../../../../common/entities/PhotoDTO'; -import {OrientationTypes} from 'ts-exif-parser'; +import { + CameraMetadata, + FaceRegion, + FaceRegionBox, + GPSMetadata, + PhotoDTO, + PhotoMetadata, + PositionMetaData +} from '../../../../common/entities/PhotoDTO'; import {MediaEntity, MediaMetadataEntity} from './MediaEntity'; export class CameraMetadataEntity implements CameraMetadata { @@ -38,6 +45,7 @@ export class GPSMetadataEntity implements GPSMetadata { altitude: number; } + export class PositionMetaDataEntity implements PositionMetaData { @Column(type => GPSMetadataEntity) @@ -75,5 +83,4 @@ export class PhotoMetadataEntity extends MediaMetadataEntity implements PhotoMet export class PhotoEntity extends MediaEntity implements PhotoDTO { @Column(type => PhotoMetadataEntity) metadata: PhotoMetadataEntity; - } diff --git a/backend/model/sql/enitites/SharingEntity.ts b/backend/model/sql/enitites/SharingEntity.ts index 2031235..7af3d2b 100644 --- a/backend/model/sql/enitites/SharingEntity.ts +++ b/backend/model/sql/enitites/SharingEntity.ts @@ -5,7 +5,7 @@ import {UserDTO} from '../../../../common/entities/UserDTO'; @Entity() export class SharingEntity implements SharingDTO { - @PrimaryGeneratedColumn() + @PrimaryGeneratedColumn({unsigned: true}) id: number; @Column() @@ -17,10 +17,20 @@ export class SharingEntity implements SharingDTO { @Column({type: 'text', nullable: true}) password: string; - @Column() + @Column('bigint', { + unsigned: true, transformer: { + from: v => parseInt(v, 10), + to: v => v + } + }) expires: number; - @Column() + @Column('bigint', { + unsigned: true, transformer: { + from: v => parseInt(v, 10), + to: v => v + } + }) timeStamp: number; @Column() diff --git a/backend/model/threading/MetadataLoader.ts b/backend/model/threading/MetadataLoader.ts index 16e506e..14a5eb9 100644 --- a/backend/model/threading/MetadataLoader.ts +++ b/backend/model/threading/MetadataLoader.ts @@ -1,14 +1,22 @@ import {VideoMetadata} from '../../../common/entities/VideoDTO'; -import {PhotoMetadata} from '../../../common/entities/PhotoDTO'; +import {FaceRegion, PhotoMetadata} from '../../../common/entities/PhotoDTO'; import {Config} from '../../../common/config/private/Config'; import {Logger} from '../../Logger'; import * as fs from 'fs'; import * as sizeOf from 'image-size'; -import {OrientationTypes, ExifParserFactory} from 'ts-exif-parser'; +import {ExifParserFactory, OrientationTypes} from 'ts-exif-parser'; import {IptcParser} from 'ts-node-iptc'; import {FFmpegFactory} from '../FFmpegFactory'; import {FfprobeData} from 'fluent-ffmpeg'; +// TODO: fix up different metadata loaders +// @ts-ignore +global.DataView = require('jdataview'); +// @ts-ignore +global.DOMParser = require('xmldom').DOMParser; +// @ts-ignore +const ExifReader = require('exifreader'); + const LOG_TAG = '[MetadataLoader]'; const ffmpeg = FFmpegFactory.get(); @@ -112,7 +120,7 @@ export class MetadataLoader { } if (exif.tags.CreateDate || exif.tags.DateTimeOriginal || exif.tags.ModifyDate) { - metadata.creationDate = exif.tags.CreateDate || exif.tags.DateTimeOriginal || exif.tags.ModifyDate; + metadata.creationDate = (exif.tags.CreateDate || exif.tags.DateTimeOriginal || exif.tags.ModifyDate) * 1000; } if (exif.tags.Orientation) { @@ -139,24 +147,72 @@ export class MetadataLoader { try { const iptcData = IptcParser.parse(data); - if (iptcData.country_or_primary_location_name || iptcData.province_or_state || iptcData.city) { + if (iptcData.country_or_primary_location_name) { metadata.positionData = metadata.positionData || {}; metadata.positionData.country = iptcData.country_or_primary_location_name.replace(/\0/g, '').trim(); + } + if (iptcData.province_or_state) { + metadata.positionData = metadata.positionData || {}; metadata.positionData.state = iptcData.province_or_state.replace(/\0/g, '').trim(); + } + if (iptcData.city) { + metadata.positionData = metadata.positionData || {}; metadata.positionData.city = iptcData.city.replace(/\0/g, '').trim(); } if (iptcData.caption) { metadata.caption = iptcData.caption.replace(/\0/g, '').trim(); } metadata.keywords = iptcData.keywords || []; + metadata.creationDate = (iptcData.date_time ? iptcData.date_time.getTime() : metadata.creationDate); } catch (err) { - // Logger.debug(LOG_TAG, 'Error parsing iptc data', fullPath, err); + // Logger.debug(LOG_TAG, 'Error parsing iptc data', fullPath, err); } metadata.creationDate = metadata.creationDate || 0; + + try { + const ret = ExifReader.load(data); + const faces: FaceRegion[] = []; + if (ret.Regions && ret.Regions.value.RegionList && ret.Regions.value.RegionList.value) { + for (let i = 0; i < ret.Regions.value.RegionList.value.length; i++) { + if (!ret.Regions.value.RegionList.value[i].value || + !ret.Regions.value.RegionList.value[i].value['rdf:Description'] || + !ret.Regions.value.RegionList.value[i].value['rdf:Description'].value || + !ret.Regions.value.RegionList.value[i].value['rdf:Description'].value['mwg-rs:Area']) { + continue; + } + const region = ret.Regions.value.RegionList.value[i].value['rdf:Description']; + const regionBox = ret.Regions.value.RegionList.value[i].value['rdf:Description'].value['mwg-rs:Area'].attributes; + if (region.attributes['mwg-rs:Type'] !== 'Face' || + !region.attributes['mwg-rs:Name']) { + continue; + } + const name = region.attributes['mwg-rs:Name']; + const box = { + width: Math.round(regionBox['stArea:w'] * metadata.size.width), + height: Math.round(regionBox['stArea:h'] * metadata.size.height), + x: Math.round(regionBox['stArea:x'] * metadata.size.width), + y: Math.round(regionBox['stArea:y'] * metadata.size.height) + }; + faces.push({name: name, box: box}); + } + } + if (faces.length > 0) { + metadata.faces = faces; // save faces + // remove faces from keywords + metadata.faces.forEach(f => { + const index = metadata.keywords.indexOf(f.name); + if (index !== -1) { + metadata.keywords.splice(index, 1); + } + }); + } + } catch (err) { + } + return resolve(metadata); } catch (err) { return reject({file: fullPath, error: err}); diff --git a/backend/routes/AdminRouter.ts b/backend/routes/AdminRouter.ts index 5993c66..ea6b165 100644 --- a/backend/routes/AdminRouter.ts +++ b/backend/routes/AdminRouter.ts @@ -8,6 +8,7 @@ export class AdminRouter { public static route(app: Express) { this.addGetStatistic(app); + this.addGetDuplicates(app); this.addIndexGallery(app); this.addSettings(app); } @@ -20,6 +21,14 @@ export class AdminRouter { RenderingMWs.renderResult ); } + private static addGetDuplicates(app: Express) { + app.get('/api/admin/duplicates', + AuthenticationMWs.authenticate, + AuthenticationMWs.authorise(UserRoles.Admin), + AdminMWs.getDuplicates, + RenderingMWs.renderResult + ); + } private static addIndexGallery(app: Express) { app.get('/api/admin/indexes/job/progress', diff --git a/backend/routes/GalleryRouter.ts b/backend/routes/GalleryRouter.ts index fdb4c06..96f023a 100644 --- a/backend/routes/GalleryRouter.ts +++ b/backend/routes/GalleryRouter.ts @@ -10,6 +10,7 @@ export class GalleryRouter { public static route(app: Express) { this.addGetImageIcon(app); + this.addGetVideoIcon(app); this.addGetImageThumbnail(app); this.addGetVideoThumbnail(app); this.addGetImage(app); @@ -92,6 +93,17 @@ export class GalleryRouter { ); } + + private static addGetVideoIcon(app: Express) { + app.get('/api/gallery/content/:mediaPath(*\.(mp4|ogg|ogv|webm))/icon', + AuthenticationMWs.authenticate, + // TODO: authorize path + GalleryMWs.loadFile, + ThumbnailGeneratorMWs.generateIconFactory(ThumbnailSourceType.Video), + RenderingMWs.renderFile + ); + } + private static addGetImageIcon(app: Express) { app.get('/api/gallery/content/:mediaPath(*\.(jpg|jpeg|jpe|webp|png|gif|svg))/icon', AuthenticationMWs.authenticate, diff --git a/backend/routes/PublicRouter.ts b/backend/routes/PublicRouter.ts index bc54b91..9ac4595 100644 --- a/backend/routes/PublicRouter.ts +++ b/backend/routes/PublicRouter.ts @@ -79,7 +79,7 @@ export class PublicRouter { }); - app.get(['/', '/login', '/gallery*', '/share*', '/admin', '/search*'], + app.get(['/', '/login', '/gallery*', '/share*', '/admin', '/duplicates', '/search*'], AuthenticationMWs.tryAuthenticate, setLocale, renderIndex diff --git a/benchmark/Benchmarks.ts b/benchmark/Benchmarks.ts new file mode 100644 index 0000000..6fe0411 --- /dev/null +++ b/benchmark/Benchmarks.ts @@ -0,0 +1,137 @@ +import {SQLConnection} from '../backend/model/sql/SQLConnection'; +import {Config} from '../common/config/private/Config'; +import {DatabaseType, ReIndexingSensitivity} from '../common/config/private/IPrivateConfig'; +import {ObjectManagerRepository} from '../backend/model/ObjectManagerRepository'; +import {DiskMangerWorker} from '../backend/model/threading/DiskMangerWorker'; +import {IndexingManager} from '../backend/model/sql/IndexingManager'; +import {SearchManager} from '../backend/model/sql/SearchManager'; +import * as fs from 'fs'; +import {SearchTypes} from '../common/entities/AutoCompleteItem'; +import {Utils} from '../common/Utils'; +import {GalleryManager} from '../backend/model/sql/GalleryManager'; +import {DirectoryDTO} from '../common/entities/DirectoryDTO'; + +export interface BenchmarkResult { + duration: number; + directories?: number; + media?: number; + items?: number; +} + +export class BMIndexingManager extends IndexingManager { + + public async saveToDB(scannedDirectory: DirectoryDTO): Promise { + return super.saveToDB(scannedDirectory); + } +} + +export class Benchmarks { + + constructor(public RUNS: number, public dbPath: string) { + + } + + async bmSaveDirectory(): Promise { + await this.resetDB(); + const dir = await DiskMangerWorker.scanDirectory('./'); + const im = new BMIndexingManager(); + return await this.benchmark(() => im.saveToDB(dir), () => this.resetDB()); + } + + async bmScanDirectory(): Promise { + return await this.benchmark(() => DiskMangerWorker.scanDirectory('./')); + } + + async bmListDirectory(): Promise { + const gm = new GalleryManager(); + await this.setupDB(); + Config.Server.indexing.reIndexingSensitivity = ReIndexingSensitivity.low; + return await this.benchmark(() => gm.listDirectory('./')); + } + + async bmAllSearch(text: string): Promise<{ result: BenchmarkResult, searchType: SearchTypes }[]> { + await this.setupDB(); + const types = Utils.enumToArray(SearchTypes).map(a => a.key).concat([null]); + const results: { result: BenchmarkResult, searchType: SearchTypes }[] = []; + const sm = new SearchManager(); + for (let i = 0; i < types.length; i++) { + results.push({result: await this.benchmark(() => sm.search(text, types[i])), searchType: types[i]}); + } + return results; + } + + async bmInstantSearch(text: string): Promise { + await this.setupDB(); + const sm = new SearchManager(); + return await this.benchmark(() => sm.instantSearch(text)); + } + + async bmAutocomplete(text: string): Promise { + await this.setupDB(); + const sm = new SearchManager(); + return await this.benchmark(() => sm.autocomplete(text)); + } + + + private async benchmark(fn: () => Promise<{ media: any[], directories: any[] } | any[] | void>, + beforeEach: () => Promise = null, + afterEach: () => Promise = null) { + const scanned = await fn(); + const start = process.hrtime(); + let skip = 0; + for (let i = 0; i < this.RUNS; i++) { + if (beforeEach) { + const startSkip = process.hrtime(); + await beforeEach(); + const endSkip = process.hrtime(startSkip); + skip += (endSkip[0] * 1000 + endSkip[1] / 1000000); + } + await fn(); + if (afterEach) { + const startSkip = process.hrtime(); + await afterEach(); + const endSkip = process.hrtime(startSkip); + skip += (endSkip[0] * 1000 + endSkip[1] / 1000000); + } + } + const end = process.hrtime(start); + const duration = (end[0] * 1000 + end[1] / 1000000) / this.RUNS; + + if (!scanned) { + return { + duration: duration + }; + } + + if (Array.isArray(scanned)) { + return { + duration: duration, + items: scanned.length + }; + } + + return { + duration: duration, + media: scanned.media.length, + directories: scanned.directories.length + }; + } + + private resetDB = async () => { + await SQLConnection.close(); + if (fs.existsSync(this.dbPath)) { + fs.unlinkSync(this.dbPath); + } + Config.Server.database.type = DatabaseType.sqlite; + Config.Server.database.sqlite.storage = this.dbPath; + await ObjectManagerRepository.InitSQLManagers(); + }; + + private async setupDB() { + const im = new BMIndexingManager(); + await this.resetDB(); + const dir = await DiskMangerWorker.scanDirectory('./'); + await im.saveToDB(dir); + } + +} diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000..27b1c43 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,22 @@ +# PiGallery2 performance benchmark results + +These results are created mostly for development, but I'm making them public for curious users. + +## PiGallery2 v1.5.8, 26.01.2019 + +**System**: Intel(R) Core(TM) i7-6700HQ CPU @ 2.60GHz, 16GB Ram, SHDD: 1TB, 5400 rpm +**Gallery**: directories: 0 media: 341, faces: 39 +| action | action details | average time | details | +|:------:|:--------------:|:------------:|:-------:| +| Scanning directory | | 2486.5ms | media: 341, directories:0 | +| Saving directory | | 780.0ms | - | +| Listing Directory | | 31.5ms | media: 341, directories:0 | +| searching | `a` as `directory` | 2.9ms | - | +| searching | `a` as `person` | 7.3ms | media: 39, directories:0 | +| searching | `a` as `keyword` | 30.8ms | media: 339, directories:0 | +| searching | `a` as `position` | 25.7ms | media: 282, directories:0 | +| searching | `a` as `photo` | 2.8ms | - | +| searching | `a` as `video` | 2.6ms | - | +| searching | `a` as `any` | 33.0ms | media: 339, directories:0 | +| instant search | `a` | 6.1ms | media: 10, directories:0 | +| auto complete | `a` | 5.4ms | items: 10 | diff --git a/benchmark/index.ts b/benchmark/index.ts new file mode 100644 index 0000000..b2aaf47 --- /dev/null +++ b/benchmark/index.ts @@ -0,0 +1,77 @@ +import {Config} from '../common/config/private/Config'; +import * as path from 'path'; +import {ProjectPath} from '../backend/ProjectPath'; +import {BenchmarkResult, Benchmarks} from './Benchmarks'; +import {SearchTypes} from '../common/entities/AutoCompleteItem'; +import {Utils} from '../common/Utils'; +import {DiskMangerWorker} from '../backend/model/threading/DiskMangerWorker'; + +const config: { path: string, system: string } = require(path.join(__dirname, 'config.json')); +Config.Server.imagesFolder = config.path; +const dbPath = path.join(__dirname, 'test.db'); +ProjectPath.reset(); +const RUNS = 50; + +let resultsText = ''; +const printLine = (text: string) => { + resultsText += text + '\n'; +}; + +const printHeader = async () => { + const dt = new Date(); + printLine('## PiGallery2 v' + require('./../package.json').version + + ', ' + Utils.zeroPrefix(dt.getDate(), 2) + + '.' + Utils.zeroPrefix(dt.getMonth() + 1, 2) + + '.' + dt.getFullYear()); + printLine('**System**: ' + config.system); + const dir = await DiskMangerWorker.scanDirectory('./'); + printLine('**Gallery**: directories: ' + + dir.directories.length + + ' media: ' + dir.media.length + + // @ts-ignore + ', faces: ' + dir.media.reduce((p, c) => p + (c.metadata.faces || []).length, 0)); +}; + + +const printTableHeader = () => { + printLine('| action | action details | average time | details |'); + printLine('|:------:|:--------------:|:------------:|:-------:|'); +}; +const printResult = (result: BenchmarkResult, action: string, actionDetails: string = '') => { + console.log('benchmarked: ' + action); + let details = '-'; + if (result.items) { + details = 'items: ' + result.items; + } + if (result.media) { + details = 'media: ' + result.media + ', directories:' + result.directories; + } + printLine('| ' + action + ' | ' + actionDetails + + ' | ' + (result.duration).toFixed(1) + 'ms | ' + details + ' |'); +}; + +const run = async () => { + const start = Date.now(); + const bm = new Benchmarks(RUNS, dbPath); + + // header + await printHeader(); + printTableHeader(); + printResult(await bm.bmScanDirectory(), 'Scanning directory'); + printResult(await bm.bmSaveDirectory(), 'Saving directory'); + printResult(await bm.bmListDirectory(), 'Listing Directory'); + (await bm.bmAllSearch('a')).forEach(res => { + if (res.searchType !== null) { + printResult(res.result, 'searching', '`a` as `' + SearchTypes[res.searchType] + '`'); + } else { + printResult(res.result, 'searching', '`a` as `any`'); + } + }); + printResult(await bm.bmInstantSearch('a'), 'instant search', '`a`'); + printResult(await bm.bmAutocomplete('a'), 'auto complete', '`a`'); + console.log(resultsText); + console.log('run for : ' + ((Date.now() - start)).toFixed(1) + 'ms'); +}; + +run(); + diff --git a/common/DataStructureVersion.ts b/common/DataStructureVersion.ts index 9313616..950d63a 100644 --- a/common/DataStructureVersion.ts +++ b/common/DataStructureVersion.ts @@ -1 +1 @@ -export const DataStructureVersion = 7; +export const DataStructureVersion = 9; diff --git a/common/Utils.ts b/common/Utils.ts index de5a8dc..f33c68d 100644 --- a/common/Utils.ts +++ b/common/Utils.ts @@ -90,14 +90,14 @@ export class Utils { continue; } - const part = args[i].replace('\\', '/'); + const part = args[i].replace(new RegExp('\\\\', 'g'), '/'); if (part === '/' || part === './') { continue; } url += part + '/'; } - url = url.replace('//', '/'); + url = url.replace(new RegExp('/+', 'g'), '/'); if (url.trim() === '') { url = './'; diff --git a/common/config/private/IPrivateConfig.ts b/common/config/private/IPrivateConfig.ts index c135669..7c23360 100644 --- a/common/config/private/IPrivateConfig.ts +++ b/common/config/private/IPrivateConfig.ts @@ -56,6 +56,10 @@ export interface ThreadingConfig { thumbnailThreads: number; } +export interface DuplicatesConfig { + listingLimit: number; // maximum number of duplicates to list +} + export interface ServerConfig { port: number; host: string; @@ -67,6 +71,7 @@ export interface ServerConfig { sessionTimeout: number; indexing: IndexingConfig; photoMetadataSize: number; + duplicates: DuplicatesConfig; } export interface IPrivateConfig { diff --git a/common/config/private/PrivateConfigClass.ts b/common/config/private/PrivateConfigClass.ts index ca3e908..0f5dfa4 100644 --- a/common/config/private/PrivateConfigClass.ts +++ b/common/config/private/PrivateConfigClass.ts @@ -45,6 +45,9 @@ export class PrivateConfigClass extends PublicConfigClass implements IPrivateCon folderPreviewSize: 2, cachedFolderTimeout: 1000 * 60 * 60, reIndexingSensitivity: ReIndexingSensitivity.low + }, + duplicates: { + listingLimit: 1000 } }; private ConfigLoader: any; @@ -60,7 +63,11 @@ export class PrivateConfigClass extends PublicConfigClass implements IPrivateCon public load() { ConfigLoader.loadBackendConfig(this, path.join(__dirname, './../../../config.json'), - [['PORT', 'Server-port']]); + [['PORT', 'Server-port'], + ['MYSQL_HOST', 'Server-database-mysql-host'], + ['MYSQL_PASSWORD', 'Server-database-mysql-password'], + ['MYSQL_USERNAME', 'Server-database-mysql-username'], + ['MYSQL_DATABASE', 'Server-database-mysql-database']]); if (Utils.enumToArray(UserRoles).map(r => r.key).indexOf(this.Client.unAuthenticatedUserRole) === -1) { throw new Error('Unknown user role for Client.unAuthenticatedUserRole, found: ' + this.Client.unAuthenticatedUserRole); diff --git a/common/config/public/ConfigClass.ts b/common/config/public/ConfigClass.ts index 16761a5..cce0061 100644 --- a/common/config/public/ConfigClass.ts +++ b/common/config/public/ConfigClass.ts @@ -7,14 +7,19 @@ export module ClientConfig { OpenStreetMap, Mapbox, Custom } + export interface AutoCompleteConfig { + enabled: boolean; + maxItemsPerCategory: number; + cacheTimeout: number; + } + export interface SearchConfig { enabled: boolean; instantSearchEnabled: boolean; - autocompleteEnabled: boolean; InstantSearchTimeout: number; - autocompleteCacheTimeout: number; instantSearchCacheTimeout: number; searchCacheTimeout: number; + AutoComplete: AutoCompleteConfig; } export interface SharingConfig { @@ -95,11 +100,14 @@ export class PublicConfigClass { Search: { enabled: true, instantSearchEnabled: true, - autocompleteEnabled: true, InstantSearchTimeout: 3000, - autocompleteCacheTimeout: 1000 * 60 * 60, searchCacheTimeout: 1000 * 60 * 60, - instantSearchCacheTimeout: 1000 * 60 * 60 + instantSearchCacheTimeout: 1000 * 60 * 60, + AutoComplete: { + enabled: true, + cacheTimeout: 1000 * 60 * 60, + maxItemsPerCategory: 5 + } }, Sharing: { enabled: true, diff --git a/common/entities/AutoCompleteItem.ts b/common/entities/AutoCompleteItem.ts index 9fe6e5a..f61323d 100644 --- a/common/entities/AutoCompleteItem.ts +++ b/common/entities/AutoCompleteItem.ts @@ -1,9 +1,10 @@ export enum SearchTypes { directory = 1, - keyword = 2, - position = 3, - photo = 4, - video = 5 + person = 2, + keyword = 3, + position = 5, + photo = 6, + video = 7 } export class AutoCompleteItem { diff --git a/common/entities/DuplicatesDTO.ts b/common/entities/DuplicatesDTO.ts new file mode 100644 index 0000000..66dd769 --- /dev/null +++ b/common/entities/DuplicatesDTO.ts @@ -0,0 +1,5 @@ +import {MediaDTO} from './MediaDTO'; + +export interface DuplicatesDTO { + media: MediaDTO[]; +} diff --git a/common/entities/Error.ts b/common/entities/Error.ts index d64cbba..6a84c06 100644 --- a/common/entities/Error.ts +++ b/common/entities/Error.ts @@ -23,4 +23,8 @@ export enum ErrorCodes { export class ErrorDTO { constructor(public code: ErrorCodes, public message?: string, public details?: any) { } + + toString(): string { + return '[' + ErrorCodes[this.code] + '] ' + this.message + (this.details ? this.details.toString() : ''); + } } diff --git a/common/entities/PhotoDTO.ts b/common/entities/PhotoDTO.ts index 0eb994b..7f605b3 100644 --- a/common/entities/PhotoDTO.ts +++ b/common/entities/PhotoDTO.ts @@ -1,6 +1,6 @@ import {DirectoryDTO} from './DirectoryDTO'; import {OrientationTypes} from 'ts-exif-parser'; -import {MediaDTO, MediaMetadata, MediaDimension} from './MediaDTO'; +import {MediaDimension, MediaDTO, MediaMetadata} from './MediaDTO'; export interface PhotoDTO extends MediaDTO { id: number; @@ -11,6 +11,18 @@ export interface PhotoDTO extends MediaDTO { readyIcon: boolean; } +export interface FaceRegionBox { + width: number; + height: number; + x: number; + y: number; +} + +export interface FaceRegion { + name: string; + box: FaceRegionBox; +} + export interface PhotoMetadata extends MediaMetadata { caption?: string; keywords?: string[]; @@ -20,6 +32,7 @@ export interface PhotoMetadata extends MediaMetadata { size: MediaDimension; creationDate: number; fileSize: number; + faces?: FaceRegion[]; } diff --git a/demo/images/IMG_5910.jpg b/demo/images/IMG_5910.jpg index 852d0a6..2ca467a 100644 Binary files a/demo/images/IMG_5910.jpg and b/demo/images/IMG_5910.jpg differ diff --git a/demo/images/IMG_6253.jpg b/demo/images/IMG_6253.jpg index 507ac5a..9cc007c 100644 Binary files a/demo/images/IMG_6253.jpg and b/demo/images/IMG_6253.jpg differ diff --git a/demo/images/IMG_6297.jpg b/demo/images/IMG_6297.jpg index 605fbe8..72d6d28 100644 Binary files a/demo/images/IMG_6297.jpg and b/demo/images/IMG_6297.jpg differ diff --git a/demo/images/IMG_9398-2.jpg b/demo/images/IMG_9398-2.jpg index 8865b28..44cbc0d 100644 Binary files a/demo/images/IMG_9398-2.jpg and b/demo/images/IMG_9398-2.jpg differ diff --git a/demo/images/IMG_9516.jpg b/demo/images/IMG_9516.jpg index 1fd0193..3c3255c 100644 Binary files a/demo/images/IMG_9516.jpg and b/demo/images/IMG_9516.jpg differ diff --git a/frontend/app/admin/admin.component.html b/frontend/app/admin/admin.component.html index dbe3dc1..bf70ef0 100644 --- a/frontend/app/admin/admin.component.html +++ b/frontend/app/admin/admin.component.html @@ -50,7 +50,8 @@ - + `, - + template: `` }) export class AppComponent implements OnInit, OnDestroy { diff --git a/frontend/app/app.module.ts b/frontend/app/app.module.ts index 45694f8..afe0b6b 100644 --- a/frontend/app/app.module.ts +++ b/frontend/app/app.module.ts @@ -1,10 +1,4 @@ -import { - Injectable, - LOCALE_ID, - NgModule, - TRANSLATIONS, - TRANSLATIONS_FORMAT -} from '@angular/core'; +import {Injectable, LOCALE_ID, NgModule, TRANSLATIONS, TRANSLATIONS_FORMAT} from '@angular/core'; import {BrowserModule, HAMMER_GESTURE_CONFIG, HammerGestureConfig} from '@angular/platform-browser'; import {FormsModule} from '@angular/forms'; import {AppComponent} from './app.component'; @@ -52,6 +46,7 @@ import {MapSettingsComponent} from './settings/map/map.settings.component'; import {TooltipModule} from 'ngx-bootstrap/tooltip'; import {BsDropdownModule} from 'ngx-bootstrap/dropdown'; import {CollapseModule} from 'ngx-bootstrap/collapse'; +import {PopoverModule} from 'ngx-bootstrap/popover'; import {ThumbnailSettingsComponent} from './settings/thumbnail/thumbanil.settings.component'; import {SearchSettingsComponent} from './settings/search/search.settings.component'; import {SettingsService} from './settings/settings.service'; @@ -75,6 +70,9 @@ import {MapService} from './gallery/map/map.service'; import {MetaFileSettingsComponent} from './settings/metafiles/metafile.settings.component'; import {ThumbnailLoaderService} from './gallery/thumbnailLoader.service'; import {FileSizePipe} from './pipes/FileSizePipe'; +import {DuplicateService} from './duplicates/duplicates.service'; +import {DuplicateComponent} from './duplicates/duplicates.component'; +import {DuplicatesPhotoComponent} from './duplicates/photo/photo.duplicates.component'; @Injectable() @@ -124,6 +122,7 @@ export function translationsFactory(locale: string) { ToastrModule.forRoot(), ModalModule.forRoot(), CollapseModule.forRoot(), + PopoverModule.forRoot(), BsDropdownModule.forRoot(), SlimLoadingBarModule.forRoot(), BsDatepickerModule.forRoot(), @@ -165,6 +164,8 @@ export function translationsFactory(locale: string) { BasicSettingsComponent, OtherSettingsComponent, IndexingSettingsComponent, + DuplicateComponent, + DuplicatesPhotoComponent, StringifyRole, IconizeSortingMethod, StringifySortingMethod, @@ -190,6 +191,7 @@ export function translationsFactory(locale: string) { SettingsService, OverlayService, QueryService, + DuplicateService, { provide: TRANSLATIONS, useFactory: translationsFactory, diff --git a/frontend/app/app.routing.ts b/frontend/app/app.routing.ts index 9f039cb..6155b8e 100644 --- a/frontend/app/app.routing.ts +++ b/frontend/app/app.routing.ts @@ -5,6 +5,7 @@ import {GalleryComponent} from './gallery/gallery.component'; import {AdminComponent} from './admin/admin.component'; import {ShareLoginComponent} from './sharelogin/share-login.component'; import {QueryParams} from '../../common/QueryParams'; +import {DuplicateComponent} from './duplicates/duplicates.component'; export function galleryMatcherFunction( segments: UrlSegment[]): UrlMatchResult | null { @@ -50,6 +51,10 @@ const ROUTES: Routes = [ path: 'admin', component: AdminComponent }, + { + path: 'duplicates', + component: DuplicateComponent + }, { matcher: galleryMatcherFunction, component: GalleryComponent diff --git a/frontend/app/duplicates/duplicates.component.css b/frontend/app/duplicates/duplicates.component.css new file mode 100644 index 0000000..6d4eb79 --- /dev/null +++ b/frontend/app/duplicates/duplicates.component.css @@ -0,0 +1,16 @@ +.card{ + margin: 8px 0; +} + +.row{ + margin: 5px 0; + cursor: pointer; +} + +.row:hover{ + background-color: #f8f9fa; +} + +a{ + color: #212529; +} diff --git a/frontend/app/duplicates/duplicates.component.html b/frontend/app/duplicates/duplicates.component.html new file mode 100644 index 0000000..c9dcd8e --- /dev/null +++ b/frontend/app/duplicates/duplicates.component.html @@ -0,0 +1,38 @@ + + +
+ + + + + + + loading + +
+ diff --git a/frontend/app/duplicates/duplicates.component.ts b/frontend/app/duplicates/duplicates.component.ts new file mode 100644 index 0000000..79c4b2d --- /dev/null +++ b/frontend/app/duplicates/duplicates.component.ts @@ -0,0 +1,132 @@ +import {Component, HostListener, OnDestroy} from '@angular/core'; +import {DuplicateService} from './duplicates.service'; +import {Utils} from '../../../common/Utils'; +import {QueryService} from '../model/query.service'; +import {DuplicatesDTO} from '../../../common/entities/DuplicatesDTO'; +import {DirectoryDTO} from '../../../common/entities/DirectoryDTO'; +import {Subscription} from 'rxjs'; +import {Config} from '../../../common/config/public/Config'; +import {PageHelper} from '../model/page.helper'; + +interface GroupedDuplicate { + name: string; + duplicates: DuplicatesDTO[]; +} + +@Component({ + selector: 'app-duplicate', + templateUrl: './duplicates.component.html', + styleUrls: ['./duplicates.component.css'] +}) +export class DuplicateComponent implements OnDestroy { + + directoryGroups: GroupedDuplicate[] = null; + renderedDirGroups: GroupedDuplicate[] = null; + renderedIndex = { + group: -1, + pairs: 0 + }; + subscription: Subscription; + renderTimer: number = null; + duplicateCount = { + pairs: 0, + photos: 0 + }; + + constructor(public _duplicateService: DuplicateService, + public queryService: QueryService) { + this._duplicateService.getDuplicates().catch(console.error); + this.subscription = this._duplicateService.duplicates.subscribe((duplicates: DuplicatesDTO[]) => { + this.directoryGroups = []; + this.renderedIndex = {group: -1, pairs: 0}; + this.renderedDirGroups = []; + this.duplicateCount = { + pairs: 0, + photos: 0 + }; + if (duplicates === null) { + return; + } + this.duplicateCount.photos = duplicates.reduce((prev: number, curr) => prev + curr.media.length, 0); + this.duplicateCount.pairs = duplicates.length; + + const getMostFrequentDir = (dupls: DuplicatesDTO[]) => { + if (dupls.length === 0) { + return null; + } + const dirFrequency: { [key: number]: { count: number, dir: DirectoryDTO } } = {}; + dupls.forEach(d => d.media.forEach(m => { + dirFrequency[m.directory.id] = dirFrequency[m.directory.id] || {dir: m.directory, count: 0}; + dirFrequency[m.directory.id].count++; + })); + let max: { count: number, dir: DirectoryDTO } = {count: -1, dir: null}; + for (const freq of Object.values(dirFrequency)) { + if (max.count <= freq.count) { + max = freq; + } + } + return max.dir; + }; + + while (duplicates.length > 0) { + const dir = getMostFrequentDir(duplicates); + const group = duplicates.filter(d => d.media.find(m => m.directory.id === dir.id)); + duplicates = duplicates.filter(d => !d.media.find(m => m.directory.id === dir.id)); + this.directoryGroups.push({name: this.getDirectoryPath(dir) + ' (' + group.length + ')', duplicates: group}); + } + this.renderMore(); + }); + } + + ngOnDestroy(): void { + if (this.subscription) { + this.subscription.unsubscribe(); + this.subscription = null; + } + } + + getDirectoryPath(directory: DirectoryDTO) { + return Utils.concatUrls(directory.path, directory.name); + } + + renderMore = () => { + if (this.renderTimer !== null) { + clearTimeout(this.renderTimer); + this.renderTimer = null; + } + + if (this.renderedIndex.group === this.directoryGroups.length - 1 && + this.renderedIndex.pairs >= + this.directoryGroups[this.renderedIndex.group].duplicates.length) { + return; + } + if (this.shouldRenderMore()) { + if (this.renderedDirGroups.length === 0 || + this.renderedIndex.pairs >= + this.directoryGroups[this.renderedIndex.group].duplicates.length) { + this.renderedDirGroups.push({ + name: this.directoryGroups[++this.renderedIndex.group].name, + duplicates: [] + }); + this.renderedIndex.pairs = 0; + } + this.renderedDirGroups[this.renderedDirGroups.length - 1].duplicates + .push(this.directoryGroups[this.renderedIndex.group].duplicates[this.renderedIndex.pairs++]); + + this.renderTimer = window.setTimeout(this.renderMore, 0); + } + }; + + + @HostListener('window:scroll') + onScroll() { + this.renderMore(); + } + + private shouldRenderMore(): boolean { + return Config.Client.Other.enableOnScrollRendering === false || + PageHelper.ScrollY >= PageHelper.MaxScrollY * 0.7 + || (document.body.clientHeight) * 0.85 < window.innerHeight; + } +} + diff --git a/frontend/app/duplicates/duplicates.service.ts b/frontend/app/duplicates/duplicates.service.ts new file mode 100644 index 0000000..7543689 --- /dev/null +++ b/frontend/app/duplicates/duplicates.service.ts @@ -0,0 +1,20 @@ +import {Injectable} from '@angular/core'; +import {NetworkService} from '../model/network/network.service'; +import {DuplicatesDTO} from '../../../common/entities/DuplicatesDTO'; +import {BehaviorSubject} from 'rxjs'; + + +@Injectable() +export class DuplicateService { + + public duplicates: BehaviorSubject; + + constructor(private networkService: NetworkService) { + this.duplicates = new BehaviorSubject(null); + } + + public async getDuplicates() { + this.duplicates.next(await this.networkService.getJson('/admin/duplicates')); + } + +} diff --git a/frontend/app/duplicates/photo/photo.duplicates.component.css b/frontend/app/duplicates/photo/photo.duplicates.component.css new file mode 100644 index 0000000..b299679 --- /dev/null +++ b/frontend/app/duplicates/photo/photo.duplicates.component.css @@ -0,0 +1,13 @@ +.icon { + height: 30px; +} + +.big-icon { + height: 60px; +} + +.photo-container { + width: inherit; + height: inherit; + overflow: hidden; +} diff --git a/frontend/app/duplicates/photo/photo.duplicates.component.html b/frontend/app/duplicates/photo/photo.duplicates.component.html new file mode 100644 index 0000000..f6091df --- /dev/null +++ b/frontend/app/duplicates/photo/photo.duplicates.component.html @@ -0,0 +1,15 @@ +
+ + {{media.name}} + + + {{media.name}} +
diff --git a/frontend/app/duplicates/photo/photo.duplicates.component.ts b/frontend/app/duplicates/photo/photo.duplicates.component.ts new file mode 100644 index 0000000..6d0abe9 --- /dev/null +++ b/frontend/app/duplicates/photo/photo.duplicates.component.ts @@ -0,0 +1,40 @@ +import {Component, Input, OnDestroy, OnInit} from '@angular/core'; +import {MediaDTO} from '../../../../common/entities/MediaDTO'; +import {Media} from '../../gallery/Media'; +import {IconThumbnail, Thumbnail, ThumbnailManagerService} from '../../gallery/thumbnailManager.service'; +import {PhotoDTO} from '../../../../common/entities/PhotoDTO'; +import {OrientationTypes} from 'ts-exif-parser'; +import {MediaIcon} from '../../gallery/MediaIcon'; + +@Component({ + selector: 'app-duplicates-photo', + templateUrl: './photo.duplicates.component.html', + styleUrls: ['./photo.duplicates.component.css'] +}) +export class DuplicatesPhotoComponent implements OnInit, OnDestroy { + @Input() media: MediaDTO; + + thumbnail: IconThumbnail; + + + constructor(private thumbnailService: ThumbnailManagerService) { + } + + get Orientation() { + if (!this.media) { + return OrientationTypes.TOP_LEFT; + } + return (this.media).metadata.orientation || OrientationTypes.TOP_LEFT; + } + + ngOnInit() { + this.thumbnail = this.thumbnailService.getIcon(new MediaIcon(this.media)); + + } + + ngOnDestroy() { + this.thumbnail.destroy(); + } + +} + diff --git a/frontend/app/frame/frame.component.html b/frontend/app/frame/frame.component.html index 1ad69de..9b9492c 100644 --- a/frontend/app/frame/frame.component.html +++ b/frontend/app/frame/frame.component.html @@ -22,23 +22,31 @@
    • -
    +
    -
    - +
    + #{{keyword}} - #{{keyword}} - , + [routerLink]="['/search', keyword.value, {type: SearchTypes[keyword.type]}]" [ngSwitch]="keyword.type"> + #{{keyword.value}} + + #{{keyword.value}} + ,
    diff --git a/frontend/app/gallery/grid/photo/photo.grid.gallery.component.ts b/frontend/app/gallery/grid/photo/photo.grid.gallery.component.ts index 8af6b46..43e56ed 100644 --- a/frontend/app/gallery/grid/photo/photo.grid.gallery.component.ts +++ b/frontend/app/gallery/grid/photo/photo.grid.gallery.component.ts @@ -5,9 +5,8 @@ import {SearchTypes} from '../../../../../common/entities/AutoCompleteItem'; import {RouterLink} from '@angular/router'; import {Thumbnail, ThumbnailManagerService} from '../../thumbnailManager.service'; import {Config} from '../../../../../common/config/public/Config'; -import {AnimationBuilder} from '@angular/animations'; import {PageHelper} from '../../../model/page.helper'; -import {PhotoDTO} from '../../../../../common/entities/PhotoDTO'; +import {PhotoDTO, PhotoMetadata} from '../../../../../common/entities/PhotoDTO'; @Component({ selector: 'app-gallery-grid-photo', @@ -22,6 +21,7 @@ export class GalleryPhotoComponent implements IRenderable, OnInit, OnDestroy { @ViewChild('photoContainer') container: ElementRef; thumbnail: Thumbnail; + keywords: { value: string, type: SearchTypes }[] = null; infoBar = { marginTop: 0, visible: false, @@ -34,17 +34,40 @@ export class GalleryPhotoComponent implements IRenderable, OnInit, OnDestroy { wasInView: boolean = null; - constructor(private thumbnailService: ThumbnailManagerService, - private _animationBuilder: AnimationBuilder) { + constructor(private thumbnailService: ThumbnailManagerService) { this.SearchTypes = SearchTypes; this.searchEnabled = Config.Client.Search.enabled; } - ngOnInit() { - this.thumbnail = this.thumbnailService.getThumbnail(this.gridPhoto); + get ScrollListener(): boolean { + return !this.thumbnail.Available && !this.thumbnail.Error; } + get Title(): string { + if (Config.Client.Other.captionFirstNaming === false) { + return this.gridPhoto.media.name; + } + if ((this.gridPhoto.media).metadata.caption) { + if ((this.gridPhoto.media).metadata.caption.length > 20) { + return (this.gridPhoto.media).metadata.caption.substring(0, 17) + '...'; + } + return (this.gridPhoto.media).metadata.caption; + } + return this.gridPhoto.media.name; + } + + ngOnInit() { + this.thumbnail = this.thumbnailService.getThumbnail(this.gridPhoto); + const metadata = this.gridPhoto.media.metadata as PhotoMetadata; + if ((metadata.keywords && metadata.keywords.length > 0) || + (metadata.faces && metadata.faces.length > 0)) { + this.keywords = (metadata.faces || []).map(f => ({value: f.name, type: SearchTypes.person})) + .concat((metadata.keywords || []).map(k => ({value: k, type: SearchTypes.keyword}))); + } + + } + ngOnDestroy() { this.thumbnail.destroy(); @@ -53,16 +76,11 @@ export class GalleryPhotoComponent implements IRenderable, OnInit, OnDestroy { } } - isInView(): boolean { return PageHelper.ScrollY < this.container.nativeElement.offsetTop + this.container.nativeElement.clientHeight && PageHelper.ScrollY + window.innerHeight > this.container.nativeElement.offsetTop; } - get ScrollListener(): boolean { - return !this.thumbnail.Available && !this.thumbnail.Error; - } - onScroll() { if (this.thumbnail.Available === true || this.thumbnail.Error === true) { return; @@ -74,7 +92,6 @@ export class GalleryPhotoComponent implements IRenderable, OnInit, OnDestroy { } } - getPositionText(): string { if (!this.gridPhoto || !this.gridPhoto.isPhoto()) { return ''; @@ -84,7 +101,6 @@ export class GalleryPhotoComponent implements IRenderable, OnInit, OnDestroy { (this.gridPhoto.media).metadata.positionData.country; } - mouseOver() { this.infoBar.visible = true; if (this.animationTimer != null) { @@ -124,19 +140,6 @@ export class GalleryPhotoComponent implements IRenderable, OnInit, OnDestroy { } - get Title(): string { - if (Config.Client.Other.captionFirstNaming === false) { - return this.gridPhoto.media.name; - } - if ((this.gridPhoto.media).metadata.caption) { - if ((this.gridPhoto.media).metadata.caption.length > 20) { - return (this.gridPhoto.media).metadata.caption.substring(0, 17) + '...'; - } - return (this.gridPhoto.media).metadata.caption; - } - return this.gridPhoto.media.name; - } - /* onImageLoad() { this.loading.show = false; diff --git a/frontend/app/gallery/navigator/navigator.gallery.component.html b/frontend/app/gallery/navigator/navigator.gallery.component.html index 9a45dee..c2056bc 100644 --- a/frontend/app/gallery/navigator/navigator.gallery.component.html +++ b/frontend/app/gallery/navigator/navigator.gallery.component.html @@ -15,6 +15,7 @@ + {{searchResult.searchText}} diff --git a/frontend/app/gallery/search/search.gallery.component.html b/frontend/app/gallery/search/search.gallery.component.html index 40de478..2c85a97 100644 --- a/frontend/app/gallery/search/search.gallery.component.html +++ b/frontend/app/gallery/search/search.gallery.component.html @@ -25,6 +25,7 @@ + {{item.preText}}{{item.highLightText}}{{item.postText}} diff --git a/frontend/app/gallery/search/search.gallery.component.ts b/frontend/app/gallery/search/search.gallery.component.ts index ea5d620..cc2645b 100644 --- a/frontend/app/gallery/search/search.gallery.component.ts +++ b/frontend/app/gallery/search/search.gallery.component.ts @@ -54,7 +54,7 @@ export class GallerySearchComponent implements OnDestroy { const searchText = (event.target).value.trim(); - if (Config.Client.Search.autocompleteEnabled && + if (Config.Client.Search.AutoComplete.enabled && this.cache.lastAutocomplete !== searchText) { this.cache.lastAutocomplete = searchText; this.autocomplete(searchText).catch(console.error); @@ -92,7 +92,7 @@ export class GallerySearchComponent implements OnDestroy { } private async autocomplete(searchText: string) { - if (!Config.Client.Search.autocompleteEnabled) { + if (!Config.Client.Search.AutoComplete.enabled) { return; } if (searchText.trim() === '.') { diff --git a/frontend/app/model/notification.service.ts b/frontend/app/model/notification.service.ts index 87947b7..48699cd 100644 --- a/frontend/app/model/notification.service.ts +++ b/frontend/app/model/notification.service.ts @@ -1,4 +1,4 @@ -import {Injectable, ViewContainerRef} from '@angular/core'; +import {Injectable} from '@angular/core'; import {ToastrService} from 'ngx-toastr'; import {NetworkService} from './network/network.service'; import {AuthenticationService} from './network/authentication.service'; diff --git a/frontend/app/model/page.helper.ts b/frontend/app/model/page.helper.ts index 1decc2a..9f3da06 100644 --- a/frontend/app/model/page.helper.ts +++ b/frontend/app/model/page.helper.ts @@ -11,14 +11,20 @@ export class PageHelper { return this.supportPageOffset ? window.pageYOffset : this.isCSS1Compat ? document.documentElement.scrollTop : document.body.scrollTop; } - public static get ScrollX(): number { - return this.supportPageOffset ? window.pageXOffset : this.isCSS1Compat ? document.documentElement.scrollLeft : document.body.scrollLeft; - } - public static set ScrollY(value: number) { window.scrollTo(this.ScrollX, value); } + public static get MaxScrollY(): number { + return Math.max(document.body.scrollHeight, document.body.offsetHeight, + document.documentElement.clientHeight, document.documentElement.scrollHeight, + document.documentElement.offsetHeight) - window.innerHeight; + } + + public static get ScrollX(): number { + return this.supportPageOffset ? window.pageXOffset : this.isCSS1Compat ? document.documentElement.scrollLeft : document.body.scrollLeft; + } + public static showScrollY() { PageHelper.body.style.overflowY = 'scroll'; } diff --git a/frontend/app/model/query.service.ts b/frontend/app/model/query.service.ts index 0be440d9..224a88b 100644 --- a/frontend/app/model/query.service.ts +++ b/frontend/app/model/query.service.ts @@ -1,6 +1,5 @@ import {Injectable} from '@angular/core'; import {ShareService} from '../gallery/share.service'; -import {PhotoDTO} from '../../../common/entities/PhotoDTO'; import {MediaDTO} from '../../../common/entities/MediaDTO'; import {QueryParams} from '../../../common/QueryParams'; import {Utils} from '../../../common/Utils'; diff --git a/frontend/app/pipes/FileSizePipe.ts b/frontend/app/pipes/FileSizePipe.ts index de9e996..d187f5f 100644 --- a/frontend/app/pipes/FileSizePipe.ts +++ b/frontend/app/pipes/FileSizePipe.ts @@ -1,5 +1,4 @@ import {Pipe, PipeTransform} from '@angular/core'; -import {I18n} from '@ngx-translate/i18n-polyfill'; @Pipe({name: 'fileSize'}) diff --git a/frontend/app/settings/indexing/indexing.settings.component.ts b/frontend/app/settings/indexing/indexing.settings.component.ts index 53484bf..6bae8fa 100644 --- a/frontend/app/settings/indexing/indexing.settings.component.ts +++ b/frontend/app/settings/indexing/indexing.settings.component.ts @@ -4,7 +4,7 @@ import {AuthenticationService} from '../../model/network/authentication.service' import {NavigationService} from '../../model/navigation.service'; import {NotificationService} from '../../model/notification.service'; import {ErrorDTO} from '../../../../common/entities/Error'; -import {Observable, interval} from 'rxjs'; +import {interval, Observable} from 'rxjs'; import {IndexingConfig, ReIndexingSensitivity} from '../../../../common/config/private/IPrivateConfig'; import {SettingsComponent} from '../_abstract/abstract.settings.component'; import {Utils} from '../../../../common/Utils'; @@ -23,15 +23,46 @@ export class IndexingSettingsComponent extends SettingsComponent = null; + + constructor(_authService: AuthenticationService, + _navigation: NavigationService, + _settingsService: IndexingSettingsService, + notification: NotificationService, + i18n: I18n) { + + super(i18n('Indexing'), + _authService, + _navigation, + _settingsService, + notification, + i18n, + s => s.Server.indexing); + + } + + get TimeLeft() { + const prg = this._settingsService.progress.value; + return (prg.time.current - prg.time.start) / prg.indexed * prg.left; + } + + get TimeElapsed() { + const prg = this._settingsService.progress.value; + return (prg.time.current - prg.time.start); + } + updateProgress = async () => { try { + const wasRunning = this._settingsService.progress.value !== null; await (this._settingsService).getProgress(); + if (wasRunning && this._settingsService.progress.value === null) { + this.notification.success(this.i18n('Folder indexed'), this.i18n('Success')); + } } catch (err) { if (this.subscription.timer != null) { this.subscription.timer.unsubscribe(); @@ -50,22 +81,6 @@ export class IndexingSettingsComponent extends SettingsComponent_settingsService, - notification, - i18n, - s => s.Server.indexing); - - } - async ngOnInit() { super.ngOnInit(); this.types = Utils @@ -105,7 +120,7 @@ export class IndexingSettingsComponent extends SettingsComponentthis._settingsService).cancel(); - this.notification.success(this.i18n('Folder indexed'), this.i18n('Success')); + this._settingsService.progress.next(null); + this.notification.info(this.i18n('Folder indexing interrupted')); this.inProgress = false; return true; } catch (err) { diff --git a/frontend/app/settings/settings.service.ts b/frontend/app/settings/settings.service.ts index a2e85cf..8a3b37a 100644 --- a/frontend/app/settings/settings.service.ts +++ b/frontend/app/settings/settings.service.ts @@ -15,12 +15,15 @@ export class SettingsService { Client: { Search: { enabled: true, - autocompleteEnabled: true, + AutoComplete: { + enabled: true, + cacheTimeout: 1000 * 60 * 60, + maxItemsPerCategory: 5 + }, instantSearchEnabled: true, InstantSearchTimeout: 0, searchCacheTimeout: 1000 * 60 * 60, instantSearchCacheTimeout: 1000 * 60 * 60, - autocompleteCacheTimeout: 1000 * 60 * 60 }, Thumbnail: { concurrentThumbnailGenerations: null, @@ -71,8 +74,8 @@ export class SettingsService { updateTimeout: 2000 }, imagesFolder: '', - port: 80, - host: '0.0.0.0', + port: 80, + host: '0.0.0.0', thumbnail: { folder: '', qualityPriority: true, @@ -88,7 +91,10 @@ export class SettingsService { folderPreviewSize: 0, reIndexingSensitivity: ReIndexingSensitivity.medium }, - photoMetadataSize: 512 * 1024 + photoMetadataSize: 512 * 1024, + duplicates: { + listingLimit: 1000 + } } }); } diff --git a/gulpfile.js b/gulpfile.js deleted file mode 100644 index a1d23df..0000000 --- a/gulpfile.js +++ /dev/null @@ -1,195 +0,0 @@ -var ts = require('gulp-typescript'); -var gulp = require('gulp'); -var zip = require('gulp-zip'); -var fs = require('fs'); -var runSequence = require('run-sequence'); -var jsonModify = require('gulp-json-modify'); -var exec = require('child_process').exec; - -var translationFolder = "translate"; -var tsBackendProject = ts.createProject('tsconfig.json'); -gulp.task('build-backend', function () { - return gulp.src([ - "common/**/*.ts", - "backend/**/*.ts"], {base: "."}) - .pipe(tsBackendProject()) - .js - .pipe(gulp.dest("./release")) - -}); -var createFrontendTask = function (type, script) { - gulp.task(type, function (cb) { - exec(script, function (err, stdout, stderr) { - console.log(stdout); - console.log(stderr); - cb(err); - }); - }); -}; - -gulp.task('build-frontend', function (done) { - var languages = getLanguages().filter(function (l) { - return l !== "en"; - }); - var tasks = []; - createFrontendTask('build-frontend-release default', - "ng build --aot --prod --output-path=./release/dist --no-progress --i18n-locale=en" + - " --i18n-format xlf --i18n-file frontend/" + translationFolder + "/messages.en.xlf" + - " --i18n-missing-translation warning"); - tasks.push('build-frontend-release default'); - for (var i = 0; i < languages.length; i++) { - createFrontendTask('build-frontend-release ' + languages[i], - "ng build --aot --prod --output-path=./release/dist/" + languages[i] + - " --no-progress --i18n-locale=" + languages[i] + - " --i18n-format xlf --i18n-file frontend/" + translationFolder + "/messages." + languages[i] + ".xlf" + - " --i18n-missing-translation warning"); - tasks.push('build-frontend-release ' + languages[i]); - } - tasks.push(function () { - done(); - }); - - runSequence.apply(this, tasks); - -}); - -gulp.task('copy-static', function () { - return gulp.src([ - "backend/model/diagnostics/blank.jpg", - "README.md", - "LICENSE"], {base: "."}) - .pipe(gulp.dest('./release')); -}); - -gulp.task('copy-package', function () { - return gulp.src([ - "package.json"], {base: "."}) - .pipe(jsonModify({ - key: 'devDependencies', - value: {} - })) - .pipe(jsonModify({ - key: 'scripts', - value: {"start": "node ./backend/index.js"} - })) - .pipe(gulp.dest('./release')); -}); - - -gulp.task('zip-release', function () { - return gulp.src(['release/**/*'], {base: "./release"}) - .pipe(zip('pigallery2.zip')) - .pipe(gulp.dest('.')); -}); - -gulp.task('build-release', function (done) { - runSequence('build-frontend', 'build-backend', 'copy-static', 'copy-package', 'zip-release', function () { - done(); - }); -}); - -var getLanguages = function () { - if (!fs.existsSync("./frontend/" + translationFolder)) { - return []; - } - var dirCont = fs.readdirSync("./frontend/" + translationFolder); - var files = dirCont.filter(function (elm) { - return elm.match(/.*\.[a-zA-Z]+\.(xlf)/ig); - }); - return files.map(function (f) { - return f.split(".")[1] - }); -}; - -var simpleBuild = function (isProd) { - return function (done) { - var languages = getLanguages().filter(function (l) { - return l !== "en"; - }); - var tasks = []; - var cmd = "ng build --aot "; - if (isProd) { - cmd += " --prod --no-extract-licenses " - } - createFrontendTask('build-frontend default', cmd + "--output-path=./dist --no-progress --no-progress --i18n-locale en" + - " --i18n-format=xlf --i18n-file=frontend/" + translationFolder + "/messages.en.xlf" + " --i18n-missing-translation warning"); - tasks.push('build-frontend default'); - for (var i = 0; i < languages.length; i++) { - createFrontendTask('build-frontend ' + languages[i], cmd + "--output-path=./dist/" + languages[i] + " --no-progress --i18n-locale " + languages[i] + - " --i18n-format=xlf --i18n-file=frontend/" + translationFolder + "/messages." + languages[i] + ".xlf" + " --i18n-missing-translation warning"); - tasks.push('build-frontend ' + languages[i]); - } - tasks.push(function () { - done(); - }); - - runSequence.apply(this, tasks); - }; -}; - -gulp.task("extract-locale", function (cb) { - console.log("creating source translation file: locale.source.xlf"); - exec('ng xi18n --out-file=./../locale.source.xlf --i18n-format=xlf --i18n-locale=en', {maxBuffer: 1024 * 1024}, function (err, stdout, stderr) { - console.log(stdout); - console.log(stderr); - if (err) { - return cb(err); - } - exec('ngx-extractor -i frontend/**/*.ts -f xlf --out-file locale.source.xlf', function (err, stdout, stderr) { - console.log(stdout); - console.log(stderr); - cb(err); - }); - }); -}); - -var translate = function (list, cb) { - var localsStr = '"[\\"' + list.join('\\",\\"') + '\\"]"'; - exec('xlf-google-translate --source-lang="en" --source-file="./locale.source.xlf" --destination-folder="./frontend/"' + - translationFolder + ' --destination-languages=' + localsStr, function (err, stdout, stderr) { - console.log(stdout); - console.log(stderr); - cb(err); - }); -}; - -gulp.task("update-translation-only", function (cb) { - translate(getLanguages(), cb) -}); - -gulp.task("update-translation", function (done) { - runSequence('extract-locale', 'update-translation-only', function () { - done(); - }); -}); - - -gulp.task("add-translation-only", function (cb) { - var languages = getLanguages(); - var lng = null; - for (var i = 0; i < process.argv.length - 1; i++) { - if (process.argv[i] === "add-translation") { - lng = process.argv[i + 1].replace("--", ""); - } - } - if (lng == null) { - console.error("Error: set language with '--' e.g: npm run add-translation -- --en"); - return cb(); - } - if (languages.indexOf(lng) !== -1) { - console.error("Error: language already exists, can't add. These language(s) already exist(s): " + languages); - return cb(); - } - translate([lng], cb) -}); - - -gulp.task("add-translation", function (done) { - runSequence('extract-locale', 'add-translation-only', function () { - done(); - }); -}); - - -gulp.task('build-dev', simpleBuild(false)); -gulp.task('build-prod', simpleBuild(true)); diff --git a/gulpfile.ts b/gulpfile.ts new file mode 100644 index 0000000..26398bf --- /dev/null +++ b/gulpfile.ts @@ -0,0 +1,181 @@ +import * as gulp from 'gulp'; +import * as fs from 'fs'; +import * as zip from 'gulp-zip'; +import * as ts from 'gulp-typescript'; +// @ts-ignore +import * as jsonModify from 'gulp-json-modify'; + + +const exec = require('child_process').exec; + +const translationFolder = 'translate'; +const tsBackendProject = ts.createProject('tsconfig.json'); + +gulp.task('build-backend', function () { + return gulp.src([ + 'common/**/*.ts', + 'backend/**/*.ts'], {base: '.'}) + .pipe(tsBackendProject()) + .js + .pipe(gulp.dest('./release')); + +}); + +const handleError = (cb: (err: any) => void) => { + return (err: any, stdout: string, stderr: string) => { + console.log(stdout); + console.log(stderr); + cb(err); + }; +}; + +const createFrontendTask = (type: string, script: string) => { + gulp.task(type, (cb) => { + exec(script, handleError(cb)); + }); +}; + +const getLanguages = () => { + if (!fs.existsSync('./frontend/' + translationFolder)) { + return []; + } + const dirCont = fs.readdirSync('./frontend/' + translationFolder); + const files: string[] = dirCont.filter((elm) => { + return elm.match(/.*\.[a-zA-Z]+\.(xlf)/ig); + }); + return files.map((f: string) => { + return f.split('.')[1]; + }); +}; + +gulp.task('build-frontend', (() => { + const languages = getLanguages().filter((l) => { + return l !== 'en'; + }); + const tasks = []; + createFrontendTask('build-frontend-release default', + 'ng build --aot --prod --output-path=./release/dist --no-progress --i18n-locale=en' + + ' --i18n-format xlf --i18n-file frontend/' + translationFolder + '/messages.en.xlf' + + ' --i18n-missing-translation warning'); + tasks.push('build-frontend-release default'); + for (let i = 0; i < languages.length; i++) { + createFrontendTask('build-frontend-release ' + languages[i], + 'ng build --aot --prod --output-path=./release/dist/' + languages[i] + + ' --no-progress --i18n-locale=' + languages[i] + + ' --i18n-format xlf --i18n-file frontend/' + translationFolder + '/messages.' + languages[i] + '.xlf' + + ' --i18n-missing-translation warning'); + tasks.push('build-frontend-release ' + languages[i]); + } + return gulp.series(...tasks); +})()); + +gulp.task('copy-static', function () { + return gulp.src([ + 'backend/model/diagnostics/blank.jpg', + 'README.md', + 'LICENSE'], {base: '.'}) + .pipe(gulp.dest('./release')); +}); + +gulp.task('copy-package', function () { + return gulp.src([ + 'package.json'], {base: '.'}) + .pipe(jsonModify({ + key: 'devDependencies', + value: {} + })) + .pipe(jsonModify({ + key: 'scripts', + value: {'start': 'node ./backend/index.js'} + })) + .pipe(gulp.dest('./release')); +}); + + +gulp.task('zip-release', function () { + return gulp.src(['release/**/*'], {base: './release'}) + .pipe(zip('pigallery2.zip')) + .pipe(gulp.dest('.')); +}); + +gulp.task('build-release', gulp.series('build-frontend', 'build-backend', 'copy-static', 'copy-package', 'zip-release')); + + +const simpleBuild = (isProd: boolean) => { + const languages = getLanguages().filter(function (l) { + return l !== 'en'; + }); + const tasks = []; + let cmd = 'ng build --aot '; + if (isProd) { + cmd += ' --prod --no-extract-licenses '; + } + createFrontendTask('build-frontend default', cmd + '--output-path=./dist --no-progress --no-progress --i18n-locale en' + + ' --i18n-format=xlf --i18n-file=frontend/' + translationFolder + '/messages.en.xlf' + ' --i18n-missing-translation warning'); + tasks.push('build-frontend default'); + if (!process.env.CI) { // don't build languages if running in CI + for (let i = 0; i < languages.length; i++) { + createFrontendTask('build-frontend ' + languages[i], cmd + + '--output-path=./dist/' + languages[i] + + ' --no-progress --i18n-locale ' + languages[i] + + ' --i18n-format=xlf --i18n-file=frontend/' + translationFolder + + '/messages.' + languages[i] + '.xlf' + ' --i18n-missing-translation warning'); + tasks.push('build-frontend ' + languages[i]); + } + } + return gulp.series(...tasks); +}; + +gulp.task('extract-locale', (cb) => { + console.log('creating source translation file: locale.source.xlf'); + exec('ng xi18n --out-file=./../locale.source.xlf --i18n-format=xlf --i18n-locale=en', + {maxBuffer: 1024 * 1024}, (error: any, stdOut: string, stdErr: string) => { + console.log(stdOut); + console.log(stdErr); + if (error) { + return cb(error); + } + exec('ngx-extractor -i frontend/**/*.ts -f xlf --out-file locale.source.xlf', + handleError(cb)); + }); +}); + +const translate = (list: any[], cb: (err: any) => void) => { + const localsStr = '"[\\"' + list.join('\\",\\"') + '\\"]"'; + exec('xlf-google-translate --source-lang="en" --source-file="./locale.source.xlf" --destination-folder="./frontend/"' + + translationFolder + ' --destination-languages=' + localsStr, + handleError(cb)); +}; + +gulp.task('update-translation-only', function (cb) { + translate(getLanguages(), cb); +}); + +gulp.task('update-translation', gulp.series('extract-locale', 'update-translation-only')); + + +gulp.task('add-translation-only', (cb) => { + const languages = getLanguages(); + let lng = null; + for (let i = 0; i < process.argv.length - 1; i++) { + if (process.argv[i] === 'add-translation') { + lng = process.argv[i + 1].replace('--', ''); + } + } + if (lng == null) { + console.error('Error: set language with \'--\' e.g: npm run add-translation -- --en'); + return cb(); + } + if (languages.indexOf(lng) !== -1) { + console.error('Error: language already exists, can\'t add. These language(s) already exist(s): ' + languages); + return cb(); + } + translate([lng], cb); +}); + + +gulp.task('add-translation', gulp.series('extract-locale', 'add-translation-only')); + + +gulp.task('build-dev', simpleBuild(false)); +gulp.task('build-prod', simpleBuild(true)); diff --git a/package.json b/package.json index 269a7cf..fb23df4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pigallery2", - "version": "1.5.6", + "version": "1.5.8", "description": "This is a photo gallery optimised for running low resource servers (especially on raspberry pi)", "author": "Patrik J. Braun", "homepage": "https://github.com/bpatrik/PiGallery2", @@ -30,55 +30,60 @@ "cookie-parser": "1.4.3", "cookie-session": "2.0.0-beta.3", "ejs": "2.6.1", + "exifreader": "2.6.0", "express": "4.16.4", "fluent-ffmpeg": "2.1.2", - "image-size": "0.6.3", + "image-size": "0.7.1", + "jdataview": "2.5.0", "jimp": "0.6.0", "locale": "0.1.0", - "reflect-metadata": "0.1.12", - "sqlite3": "4.0.4", + "reflect-metadata": "0.1.13", + "sqlite3": "4.0.6", "ts-exif-parser": "0.1.4", "ts-node-iptc": "1.0.11", "typeconfig": "1.0.7", - "typeorm": "0.2.9", - "winston": "2.4.2" + "typeorm": "0.2.12", + "winston": "2.4.4", + "xmldom": "0.1.27" }, "devDependencies": { - "@angular-devkit/build-angular": "0.11.4", - "@angular-devkit/build-optimizer": "0.11.4", - "@angular/animations": "7.1.4", - "@angular/cli": "7.1.4", - "@angular/common": "7.1.4", - "@angular/compiler": "7.1.4", - "@angular/compiler-cli": "7.1.4", - "@angular/core": "7.1.4", - "@angular/forms": "7.1.4", - "@angular/http": "7.1.4", - "@angular/language-service": "7.1.4", - "@angular/platform-browser": "7.1.4", - "@angular/platform-browser-dynamic": "7.1.4", - "@angular/router": "7.1.4", + "@angular-devkit/build-angular": "0.12.3", + "@angular-devkit/build-optimizer": "0.12.3", + "@angular/animations": "7.2.2", + "@angular/cli": "7.2.3", + "@angular/common": "7.2.2", + "@angular/compiler": "7.2.2", + "@angular/compiler-cli": "7.2.2", + "@angular/core": "7.2.2", + "@angular/forms": "7.2.2", + "@angular/http": "7.2.2", + "@angular/language-service": "7.2.2", + "@angular/platform-browser": "7.2.2", + "@angular/platform-browser-dynamic": "7.2.2", + "@angular/router": "7.2.2", "@ngx-translate/i18n-polyfill": "1.0.0", "@types/bcryptjs": "2.4.2", "@types/chai": "4.1.7", "@types/cookie-parser": "1.4.1", "@types/cookie-session": "2.0.36", "@types/ejs": "2.6.1", - "@types/express": "4.16.0", + "@types/express": "4.16.1", "@types/fluent-ffmpeg": "2.1.9", "@types/gm": "1.18.2", - "@types/image-size": "0.0.29", - "@types/jasmine": "3.3.5", + "@types/gulp": "^4.0.5", + "@types/gulp-zip": "^4.0.0", + "@types/image-size": "0.7.0", + "@types/jasmine": "3.3.8", "@types/node": "10.12.18", - "@types/sharp": "0.21.0", - "@types/winston": "2.3.9", + "@types/sharp": "0.21.2", + "@types/winston": "2.4.4", "@yaga/leaflet-ng2": "1.0.0", "bootstrap": "4.1.3", "chai": "4.2.0", "codelyzer": "4.5.0", - "core-js": "2.6.1", + "core-js": "2.6.3", "ejs-loader": "0.3.1", - "gulp": "3.9.1", + "gulp": "4.0.0", "gulp-json-modify": "1.0.2", "gulp-typescript": "5.0.0", "gulp-zip": "4.2.0", @@ -86,8 +91,8 @@ "intl": "1.2.5", "jasmine-core": "3.3.0", "jasmine-spec-reporter": "4.2.1", - "jw-bootstrap-switch-ng2": "2.0.2", - "karma": "3.1.4", + "jw-bootstrap-switch-ng2": "2.0.4", + "karma": "4.0.0", "karma-chrome-launcher": "2.2.0", "karma-cli": "2.0.0", "karma-coverage-istanbul-reporter": "2.0.4", @@ -103,18 +108,18 @@ "ngx-clipboard": "11.1.9", "ngx-toastr": "9.1.1", "open-iconic": "1.1.1", - "protractor": "5.4.1", - "remap-istanbul": "0.12.0", - "rimraf": "2.6.2", + "protractor": "5.4.2", + "remap-istanbul": "0.13.0", + "rimraf": "2.6.3", "run-sequence": "2.2.1", "rxjs": "6.3.3", "rxjs-compat": "^6.3.3", "ts-helpers": "1.1.2", - "ts-node": "7.0.1", - "tslint": "5.12.0", - "typescript": "3.1.6", + "ts-node": "8.0.2", + "tslint": "5.12.1", + "typescript": "3.2.4", "xlf-google-translate": "1.0.0-beta.13", - "zone.js": "0.8.26" + "zone.js": "0.8.29" }, "resolutions": { "natives": "1.1.3" @@ -122,10 +127,10 @@ "optionalDependencies": { "@ffmpeg-installer/ffmpeg": "1.0.17", "@ffprobe-installer/ffprobe": "1.0.9", - "bcrypt": "3.0.2", + "bcrypt": "3.0.3", "gm": "1.23.1", "mysql": "2.16.0", - "sharp": "0.21.1" + "sharp": "0.21.3" }, "engines": { "node": ">= 6.9 <11.0" diff --git a/test/backend/SQLTestHelper.ts b/test/backend/SQLTestHelper.ts new file mode 100644 index 0000000..147ed1a --- /dev/null +++ b/test/backend/SQLTestHelper.ts @@ -0,0 +1,107 @@ +import {Config} from '../../common/config/private/Config'; +import {DatabaseType} from '../../common/config/private/IPrivateConfig'; +import * as fs from 'fs'; +import * as path from 'path'; +import {SQLConnection} from '../../backend/model/sql/SQLConnection'; + +declare let describe: any; +const savedDescribe = describe; + +export class SQLTestHelper { + + static enable = { + sqlite: true, + mysql: true + }; + public static readonly savedDescribe = savedDescribe; + tempDir: string; + dbPath: string; + + constructor(public dbType: DatabaseType) { + this.tempDir = path.resolve(__dirname, './tmp'); + this.dbPath = path.resolve(__dirname, './tmp', 'test.db'); + + } + + static describe(name: string, tests: (helper?: SQLTestHelper) => void) { + savedDescribe(name, async () => { + if (SQLTestHelper.enable.sqlite) { + const helper = new SQLTestHelper(DatabaseType.sqlite); + savedDescribe('sqlite', () => { + return tests(helper); + }); + } + if (SQLTestHelper.enable.mysql) { + const helper = new SQLTestHelper(DatabaseType.mysql); + savedDescribe('mysql', function () { + this.timeout(99999999); + // @ts-ignore + return tests(helper); + }); + } + }); + } + + public async initDB() { + if (this.dbType === DatabaseType.sqlite) { + await this.initSQLite(); + } else { + await this.initMySQL(); + } + } + + + public async clearDB() { + if (this.dbType === DatabaseType.sqlite) { + await this.clearUpSQLite(); + } else { + await this.clearUpMysql(); + } + } + + private async initSQLite() { + await this.resetSQLite(); + + Config.Server.database.type = DatabaseType.sqlite; + Config.Server.database.sqlite.storage = this.dbPath; + } + + private async initMySQL() { + Config.Server.database.type = DatabaseType.mysql; + Config.Server.database.mysql.database = 'pigallery2_test'; + + await this.resetMySQL(); + } + + private async resetSQLite() { + await SQLConnection.close(); + + if (fs.existsSync(this.dbPath)) { + fs.unlinkSync(this.dbPath); + } + if (fs.existsSync(this.tempDir)) { + fs.rmdirSync(this.tempDir); + } + } + + private async resetMySQL() { + Config.Server.database.type = DatabaseType.mysql; + Config.Server.database.mysql.database = 'pigallery2_test'; + const conn = await SQLConnection.getConnection(); + await conn.query('DROP DATABASE IF EXISTS ' + conn.options.database); + await conn.query('CREATE DATABASE IF NOT EXISTS ' + conn.options.database); + await SQLConnection.close(); + } + + private async clearUpMysql() { + Config.Server.database.type = DatabaseType.mysql; + Config.Server.database.mysql.database = 'pigallery2_test'; + const conn = await SQLConnection.getConnection(); + await conn.query('DROP DATABASE IF EXISTS ' + conn.options.database); + await SQLConnection.close(); + } + + private async clearUpSQLite() { + return this.resetSQLite(); + } +} diff --git a/test/backend/integration/model/sql/typeorm.ts b/test/backend/integration/model/sql/typeorm.ts index 679fb1b..c159efc 100644 --- a/test/backend/integration/model/sql/typeorm.ts +++ b/test/backend/integration/model/sql/typeorm.ts @@ -16,7 +16,6 @@ import { PositionMetaDataEntity } from '../../../../../backend/model/sql/enitites/PhotoEntity'; import {MediaDimensionEntity} from '../../../../../backend/model/sql/enitites/MediaEntity'; -import {DataStructureVersion} from '../../../../../common/DataStructureVersion'; import {VersionEntity} from '../../../../../backend/model/sql/enitites/VersionEntity'; describe('Typeorm integration', () => { diff --git a/test/backend/unit/assets/old_photo.jpg b/test/backend/unit/assets/old_photo.jpg new file mode 100644 index 0000000..0f257df Binary files /dev/null and b/test/backend/unit/assets/old_photo.jpg differ diff --git a/test/backend/unit/assets/old_photo.json b/test/backend/unit/assets/old_photo.json new file mode 100644 index 0000000..249c328 --- /dev/null +++ b/test/backend/unit/assets/old_photo.json @@ -0,0 +1,17 @@ +{ + "cameraData": { + "ISO": 200, + "exposure": 0.008, + "fStop": 2.8, + "focalLength": 9.4, + "make": "FUJIFILM", + "model": "FinePix F601 ZOOM" + }, + "creationDate": 1126455782000, + "fileSize": 2582, + "orientation": 1, + "size": { + "height": 5, + "width": 7 + } +} diff --git a/test/backend/unit/assets/test image öüóőúéáű-.,.jpg b/test/backend/unit/assets/test image öüóőúéáű-.,.jpg index 4f838c4..f0fa299 100644 Binary files a/test/backend/unit/assets/test image öüóőúéáű-.,.jpg and b/test/backend/unit/assets/test image öüóőúéáű-.,.jpg differ diff --git a/test/backend/unit/assets/test image öüóőúéáű-.,.json b/test/backend/unit/assets/test image öüóőúéáű-.,.json new file mode 100644 index 0000000..dd0bd2c --- /dev/null +++ b/test/backend/unit/assets/test image öüóőúéáű-.,.json @@ -0,0 +1,54 @@ +{ + "cameraData": { + "ISO": 3200, + "exposure": 0.00125, + "fStop": 5.6, + "focalLength": 85, + "lens": "EF-S15-85mm f/3.5-5.6 IS USM", + "make": "Canon", + "model": "óüöúőűáé ÓÜÖÚŐŰÁÉ" + }, + "caption": "Test caption", + "creationDate": 1434018566000, + "faces": [ + { + "box": { + "height": 2, + "width": 2, + "x": 8, + "y": 4 + }, + "name": "squirrel" + }, + { + "box": { + "height": 3, + "width": 2, + "x": 5, + "y": 5 + }, + "name": "special_chars űáéúőóüío?._:" + } + ], + "fileSize": 39424, + "keywords": [ + "Berkley", + "USA", + "űáéúőóüö ŰÁÉÚŐÓÜÖ" + ], + "orientation": 1, + "positionData": { + "GPSData": { + "altitude": 90, + "latitude": 37.871093333333334, + "longitude": -122.25678 + }, + "city": "test city őúéáűóöí-.,)(=", + "country": "test country őúéáűóöí-.,)(=/%!+\"'", + "state": "test state őúéáűóöí-.,)(" + }, + "size": { + "height": 10, + "width": 14 + } +} diff --git a/test/backend/unit/model/sql/GalleryManager.ts b/test/backend/unit/model/sql/GalleryManager.ts index 5a4e6b0..6774d71 100644 --- a/test/backend/unit/model/sql/GalleryManager.ts +++ b/test/backend/unit/model/sql/GalleryManager.ts @@ -1,284 +1,58 @@ import {expect} from 'chai'; -import * as fs from 'fs'; -import * as path from 'path'; -import {Config} from '../../../../../common/config/private/Config'; -import {DatabaseType, ReIndexingSensitivity} from '../../../../../common/config/private/IPrivateConfig'; -import {SQLConnection} from '../../../../../backend/model/sql/SQLConnection'; -import {GalleryManager} from '../../../../../backend/model/sql/GalleryManager'; -import {DirectoryDTO} from '../../../../../common/entities/DirectoryDTO'; import {TestHelper} from './TestHelper'; -import {Connection} from 'typeorm'; -import {DirectoryEntity} from '../../../../../backend/model/sql/enitites/DirectoryEntity'; +import {SQLTestHelper} from '../../../SQLTestHelper'; +import {GalleryManager} from '../../../../../backend/model/sql/GalleryManager'; +import {IndexingManager} from '../../../../../backend/model/sql/IndexingManager'; +import {DirectoryDTO} from '../../../../../common/entities/DirectoryDTO'; import {Utils} from '../../../../../common/Utils'; -import {MediaDTO} from '../../../../../common/entities/MediaDTO'; -import {FileDTO} from '../../../../../common/entities/FileDTO'; +import {ObjectManagerRepository} from '../../../../../backend/model/ObjectManagerRepository'; +import {PersonManager} from '../../../../../backend/model/sql/PersonManager'; +import {MediaEntity} from '../../../../../backend/model/sql/enitites/MediaEntity'; -class GalleryManagerTest extends GalleryManager { +class IndexingManagerTest extends IndexingManager { - - public async selectParentDir(connection: Connection, directoryName: string, directoryParent: string): Promise { - return super.selectParentDir(connection, directoryName, directoryParent); - } - - public async fillParentDir(connection: Connection, dir: DirectoryEntity): Promise { - return super.fillParentDir(connection, dir); - } - - public async saveToDB(scannedDirectory: DirectoryDTO) { + public async saveToDB(scannedDirectory: DirectoryDTO): Promise { return super.saveToDB(scannedDirectory); } - - public async queueForSave(scannedDirectory: DirectoryDTO): Promise { - return super.queueForSave(scannedDirectory); - } } -describe('GalleryManager', () => { +// to help WebStorm to handle the test cases +declare let describe: any; +declare const after: any; +describe = SQLTestHelper.describe; +describe('GalleryManager', (sqlHelper: SQLTestHelper) => { - const tempDir = path.join(__dirname, '../../tmp'); - const dbPath = path.join(tempDir, 'test.db'); - - - const setUpSqlDB = async () => { - if (fs.existsSync(dbPath)) { - fs.unlinkSync(dbPath); - } - if (!fs.existsSync(tempDir)) { - fs.mkdirSync(tempDir); - } - - Config.Server.database.type = DatabaseType.sqlite; - Config.Server.database.sqlite.storage = dbPath; - - }; - - const tearDownSqlDB = async () => { - await SQLConnection.close(); - if (fs.existsSync(dbPath)) { - fs.unlinkSync(dbPath); - } - if (fs.existsSync(tempDir)) { - fs.rmdirSync(tempDir); - } - }; beforeEach(async () => { - await setUpSqlDB(); + await sqlHelper.initDB(); + ObjectManagerRepository.getInstance().PersonManager = new PersonManager(); }); - afterEach(async () => { - await tearDownSqlDB(); + + after(async () => { + await sqlHelper.clearDB(); }); - const removeIds = (dir: DirectoryDTO) => { - delete dir.id; - dir.media.forEach((media: MediaDTO) => { - delete media.id; - }); - if (dir.metaFile) { - if (dir.metaFile.length === 0) { - delete dir.metaFile; - } else { - dir.metaFile.forEach((file: FileDTO) => { - delete file.id; - }); - } - } - if (dir.directories) { - dir.directories.forEach((directory: DirectoryDTO) => { - removeIds(directory); - }); - } - }; - - it('should save parent directory', async () => { - const gm = new GalleryManagerTest(); + it('should get random photo', async () => { + const gm = new GalleryManager(); + const im = new IndexingManagerTest(); const parent = TestHelper.getRandomizedDirectoryEntry(); const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1'); - const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2'); - const gpx = TestHelper.getRandomizedGPXEntry(parent, 'GPX1'); - const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir'); - const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1'); - const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2'); - - + expect(await gm.getRandomPhoto({})).to.not.exist; DirectoryDTO.removeReferences(parent); - await gm.saveToDB(Utils.clone(parent)); + await im.saveToDB(Utils.clone(parent)); - const conn = await SQLConnection.getConnection(); - const selected = await gm.selectParentDir(conn, parent.name, parent.path); - await gm.fillParentDir(conn, selected); - - DirectoryDTO.removeReferences(selected); - removeIds(selected); - subDir.isPartial = true; - delete subDir.directories; - delete subDir.metaFile; - expect(Utils.clone(Utils.removeNullOrEmptyObj(selected))) - .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent))); - }); - - it('should skip meta files', async () => { - const gm = new GalleryManagerTest(); - const parent = TestHelper.getRandomizedDirectoryEntry(); - const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1'); - const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2'); - const gpx = TestHelper.getRandomizedGPXEntry(parent, 'GPX1'); - DirectoryDTO.removeReferences(parent); - Config.Client.MetaFile.enabled = true; - await gm.saveToDB(Utils.clone(parent)); - - Config.Client.MetaFile.enabled = false; - const conn = await SQLConnection.getConnection(); - const selected = await gm.selectParentDir(conn, parent.name, parent.path); - await gm.fillParentDir(conn, selected); - - delete parent.metaFile; - DirectoryDTO.removeReferences(selected); - removeIds(selected); - expect(Utils.clone(Utils.removeNullOrEmptyObj(selected))) - .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent))); - }); - - it('should update sub directory', async () => { - const gm = new GalleryManagerTest(); - - const parent = TestHelper.getRandomizedDirectoryEntry(); - parent.name = 'parent'; - const p1 = TestHelper.getRandomizedPhotoEntry(parent); - const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir'); - subDir.name = 'subDir'; - const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1'); - - DirectoryDTO.removeReferences(parent); - await gm.saveToDB(Utils.clone(parent)); - - const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2'); - const sp3 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto3'); - - DirectoryDTO.removeReferences(subDir); - await gm.saveToDB(Utils.clone(subDir)); - - const conn = await SQLConnection.getConnection(); - const selected = await gm.selectParentDir(conn, subDir.name, subDir.path); - await gm.fillParentDir(conn, selected); - - // subDir.isPartial = true; - // delete subDir.directories; - DirectoryDTO.removeReferences(selected); - delete subDir.parent; - delete subDir.metaFile; - removeIds(selected); - // selected.directories[0].parent = selected; - expect(Utils.clone(Utils.removeNullOrEmptyObj(selected))) - .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(subDir))); - }); - - it('should avoid race condition', async () => { - const conn = await SQLConnection.getConnection(); - const gm = new GalleryManagerTest(); - Config.Client.MetaFile.enabled = true; - const parent = TestHelper.getRandomizedDirectoryEntry(); - const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1'); - const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2'); - const gpx = TestHelper.getRandomizedGPXEntry(parent, 'GPX1'); - const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir'); - const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1'); - const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2'); - - - DirectoryDTO.removeReferences(parent); - const s1 = gm.queueForSave(Utils.clone(parent)); - const s2 = gm.queueForSave(Utils.clone(parent)); - const s3 = gm.queueForSave(Utils.clone(parent)); - - await Promise.all([s1, s2, s3]); - - const selected = await gm.selectParentDir(conn, parent.name, parent.path); - await gm.fillParentDir(conn, selected); - - DirectoryDTO.removeReferences(selected); - removeIds(selected); - subDir.isPartial = true; - delete subDir.directories; - delete subDir.metaFile; - expect(Utils.clone(Utils.removeNullOrEmptyObj(selected))) - .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent))); - }); - - - (it('should save 1500 photos', async () => { - const conn = await SQLConnection.getConnection(); - const gm = new GalleryManagerTest(); - Config.Client.MetaFile.enabled = true; - const parent = TestHelper.getRandomizedDirectoryEntry(); - DirectoryDTO.removeReferences(parent); - await gm.saveToDB(Utils.clone(parent)); - const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir'); - for (let i = 0; i < 1500; i++) { - TestHelper.getRandomizedPhotoEntry(subDir, 'p' + i); - } - - DirectoryDTO.removeReferences(parent); - await gm.saveToDB(subDir); - - - const selected = await gm.selectParentDir(conn, subDir.name, subDir.path); - expect(selected.media.length).to.deep.equal(subDir.media.length); - })).timeout(20000); - - describe('Test listDirectory', () => { - const statSync = fs.statSync; - let dirTime = 0; - const indexedTime = { - lastScanned: 0, - lastModified: 0 - }; - - beforeEach(() => { - dirTime = 0; - indexedTime.lastModified = 0; - indexedTime.lastScanned = 0; - }); - - afterEach(() => { - // @ts-ignore - fs.statSync = statSync; - }); - - it('with re indexing severity low', async () => { - Config.Server.indexing.reIndexingSensitivity = ReIndexingSensitivity.low; - - // @ts-ignore - fs.statSync = () => ({ctime: new Date(dirTime), mtime: new Date(dirTime)}); - const gm = new GalleryManagerTest(); - gm.selectParentDir = (connection: Connection, directoryName: string, directoryParent: string) => { - return Promise.resolve(indexedTime); - }; - gm.fillParentDir = (connection: Connection, dir: DirectoryEntity) => { - return Promise.resolve(); - }; - - gm.indexDirectory = (...args) => { - return Promise.resolve('indexing'); - }; - - indexedTime.lastScanned = null; - expect(await gm.listDirectory('./')).to.be.equal('indexing'); - indexedTime.lastModified = 0; - dirTime = 1; - expect(await gm.listDirectory('./')).to.be.equal('indexing'); - indexedTime.lastScanned = 10; - indexedTime.lastModified = 1; - dirTime = 1; - expect(await gm.listDirectory('./')).to.be.equal(indexedTime); - expect(await gm.listDirectory('./', 1, 10)) - .to.be.equal(null); - - - }); + delete p1.metadata.faces; + delete p1.directory; + delete p1.id; + const found: MediaEntity = await gm.getRandomPhoto({}); + delete found.metadata.bitRate; + delete found.metadata.duration; + delete found.directory; + delete found.id; + expect(Utils.clone(found)).to.be.deep.equal(Utils.clone(p1)); }); }); diff --git a/test/backend/unit/model/sql/IndexingManager.ts b/test/backend/unit/model/sql/IndexingManager.ts new file mode 100644 index 0000000..1467e85 --- /dev/null +++ b/test/backend/unit/model/sql/IndexingManager.ts @@ -0,0 +1,281 @@ +import {expect} from 'chai'; +import * as fs from 'fs'; +import {Config} from '../../../../../common/config/private/Config'; +import {ReIndexingSensitivity} from '../../../../../common/config/private/IPrivateConfig'; +import {SQLConnection} from '../../../../../backend/model/sql/SQLConnection'; +import {GalleryManager} from '../../../../../backend/model/sql/GalleryManager'; +import {DirectoryDTO} from '../../../../../common/entities/DirectoryDTO'; +import {TestHelper} from './TestHelper'; +import {Connection} from 'typeorm'; +import {DirectoryEntity} from '../../../../../backend/model/sql/enitites/DirectoryEntity'; +import {Utils} from '../../../../../common/Utils'; +import {MediaDTO} from '../../../../../common/entities/MediaDTO'; +import {FileDTO} from '../../../../../common/entities/FileDTO'; +import {IndexingManager} from '../../../../../backend/model/sql/IndexingManager'; +import {ObjectManagerRepository} from '../../../../../backend/model/ObjectManagerRepository'; +import {PersonManager} from '../../../../../backend/model/sql/PersonManager'; +import {SQLTestHelper} from '../../../SQLTestHelper'; + +class GalleryManagerTest extends GalleryManager { + + + public async selectParentDir(connection: Connection, directoryName: string, directoryParent: string): Promise { + return super.selectParentDir(connection, directoryName, directoryParent); + } + + public async fillParentDir(connection: Connection, dir: DirectoryEntity): Promise { + return super.fillParentDir(connection, dir); + } + +} + +class IndexingManagerTest extends IndexingManager { + + + public async queueForSave(scannedDirectory: DirectoryDTO): Promise { + return super.queueForSave(scannedDirectory); + } + + public async saveToDB(scannedDirectory: DirectoryDTO): Promise { + return super.saveToDB(scannedDirectory); + } +} + +// to help WebStorm to handle the test cases +declare let describe: any; +declare const after: any; +describe = SQLTestHelper.describe; + +describe('IndexingManager', (sqlHelper: SQLTestHelper) => { + + + beforeEach(async () => { + await sqlHelper.initDB(); + ObjectManagerRepository.getInstance().PersonManager = new PersonManager(); + }); + + + after(async () => { + await sqlHelper.clearDB(); + }); + + const removeIds = (dir: DirectoryDTO) => { + delete dir.id; + dir.media.forEach((media: MediaDTO) => { + delete media.id; + }); + if (dir.metaFile) { + if (dir.metaFile.length === 0) { + delete dir.metaFile; + } else { + dir.metaFile.forEach((file: FileDTO) => { + delete file.id; + }); + } + } + if (dir.directories) { + dir.directories.forEach((directory: DirectoryDTO) => { + removeIds(directory); + }); + } + }; + + it('should save parent directory', async () => { + const gm = new GalleryManagerTest(); + const im = new IndexingManagerTest(); + + const parent = TestHelper.getRandomizedDirectoryEntry(); + const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1'); + const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2'); + const gpx = TestHelper.getRandomizedGPXEntry(parent, 'GPX1'); + const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir'); + const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1', 0); + const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2', 0); + + + DirectoryDTO.removeReferences(parent); + await im.saveToDB(Utils.clone(parent)); + + const conn = await SQLConnection.getConnection(); + const selected = await gm.selectParentDir(conn, parent.name, parent.path); + await gm.fillParentDir(conn, selected); + + DirectoryDTO.removeReferences(selected); + removeIds(selected); + subDir.isPartial = true; + delete subDir.directories; + delete subDir.metaFile; + expect(Utils.clone(Utils.removeNullOrEmptyObj(selected))) + .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent))); + }); + + it('should skip meta files', async () => { + const gm = new GalleryManagerTest(); + const im = new IndexingManagerTest(); + const parent = TestHelper.getRandomizedDirectoryEntry(); + const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1'); + const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2'); + const gpx = TestHelper.getRandomizedGPXEntry(parent, 'GPX1'); + DirectoryDTO.removeReferences(parent); + Config.Client.MetaFile.enabled = true; + await im.saveToDB(Utils.clone(parent)); + + Config.Client.MetaFile.enabled = false; + const conn = await SQLConnection.getConnection(); + const selected = await gm.selectParentDir(conn, parent.name, parent.path); + await gm.fillParentDir(conn, selected); + + delete parent.metaFile; + DirectoryDTO.removeReferences(selected); + removeIds(selected); + expect(Utils.clone(Utils.removeNullOrEmptyObj(selected))) + .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent))); + }); + + it('should update sub directory', async () => { + const gm = new GalleryManagerTest(); + const im = new IndexingManagerTest(); + + const parent = TestHelper.getRandomizedDirectoryEntry(); + parent.name = 'parent'; + const p1 = TestHelper.getRandomizedPhotoEntry(parent); + const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir'); + subDir.name = 'subDir'; + const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1'); + + DirectoryDTO.removeReferences(parent); + await im.saveToDB(Utils.clone(parent)); + + const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2'); + const sp3 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto3'); + + DirectoryDTO.removeReferences(subDir); + await im.saveToDB(Utils.clone(subDir)); + + const conn = await SQLConnection.getConnection(); + const selected = await gm.selectParentDir(conn, subDir.name, subDir.path); + await gm.fillParentDir(conn, selected); + + // subDir.isPartial = true; + // delete subDir.directories; + DirectoryDTO.removeReferences(selected); + delete subDir.parent; + delete subDir.metaFile; + removeIds(selected); + // selected.directories[0].parent = selected; + expect(Utils.clone(Utils.removeNullOrEmptyObj(selected))) + .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(subDir))); + }); + + it('should avoid race condition', async () => { + const conn = await SQLConnection.getConnection(); + const gm = new GalleryManagerTest(); + const im = new IndexingManagerTest(); + Config.Client.MetaFile.enabled = true; + const parent = TestHelper.getRandomizedDirectoryEntry(); + const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1'); + const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2'); + const gpx = TestHelper.getRandomizedGPXEntry(parent, 'GPX1'); + const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir'); + const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1', 1); + const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2', 1); + + + DirectoryDTO.removeReferences(parent); + const s1 = im.queueForSave(Utils.clone(parent)); + const s2 = im.queueForSave(Utils.clone(parent)); + const s3 = im.queueForSave(Utils.clone(parent)); + + await Promise.all([s1, s2, s3]); + + const selected = await gm.selectParentDir(conn, parent.name, parent.path); + await gm.fillParentDir(conn, selected); + + DirectoryDTO.removeReferences(selected); + removeIds(selected); + subDir.isPartial = true; + delete subDir.directories; + delete subDir.metaFile; + delete sp1.metadata.faces; + delete sp2.metadata.faces; + expect(Utils.clone(Utils.removeNullOrEmptyObj(selected))) + .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent))); + }); + + + (it('should save 1500 photos', async () => { + const conn = await SQLConnection.getConnection(); + const gm = new GalleryManagerTest(); + const im = new IndexingManagerTest(); + Config.Client.MetaFile.enabled = true; + const parent = TestHelper.getRandomizedDirectoryEntry(); + DirectoryDTO.removeReferences(parent); + await im.saveToDB(Utils.clone(parent)); + const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir'); + for (let i = 0; i < 1500; i++) { + TestHelper.getRandomizedPhotoEntry(subDir, 'p' + i); + } + + DirectoryDTO.removeReferences(parent); + await im.saveToDB(subDir); + + + const selected = await gm.selectParentDir(conn, subDir.name, subDir.path); + expect(selected.media.length).to.deep.equal(subDir.media.length); + }) as any).timeout(40000); + + SQLTestHelper.savedDescribe('Test listDirectory', () => { + const statSync = fs.statSync; + let dirTime = 0; + const indexedTime = { + lastScanned: 0, + lastModified: 0 + }; + + beforeEach(() => { + dirTime = 0; + + ObjectManagerRepository.getInstance().IndexingManager = new IndexingManagerTest(); + indexedTime.lastModified = 0; + indexedTime.lastScanned = 0; + }); + + afterEach(() => { + // @ts-ignore + fs.statSync = statSync; + }); + + it('with re indexing severity low', async () => { + Config.Server.indexing.reIndexingSensitivity = ReIndexingSensitivity.low; + + // @ts-ignore + fs.statSync = () => ({ctime: new Date(dirTime), mtime: new Date(dirTime)}); + const gm = new GalleryManagerTest(); + gm.selectParentDir = (connection: Connection, directoryName: string, directoryParent: string) => { + return Promise.resolve(indexedTime); + }; + gm.fillParentDir = (connection: Connection, dir: DirectoryEntity) => { + return Promise.resolve(); + }; + + ObjectManagerRepository.getInstance().IndexingManager.indexDirectory = (...args) => { + return Promise.resolve('indexing'); + }; + + indexedTime.lastScanned = null; + expect(await gm.listDirectory('./')).to.be.equal('indexing'); + indexedTime.lastModified = 0; + dirTime = 1; + expect(await gm.listDirectory('./')).to.be.equal('indexing'); + indexedTime.lastScanned = 10; + indexedTime.lastModified = 1; + dirTime = 1; + expect(await gm.listDirectory('./')).to.be.equal(indexedTime); + expect(await gm.listDirectory('./', 1, 10)) + .to.be.equal(null); + + + }); + }); + +}); diff --git a/test/backend/unit/model/sql/SearchManager.ts b/test/backend/unit/model/sql/SearchManager.ts index cefccda..a92c362 100644 --- a/test/backend/unit/model/sql/SearchManager.ts +++ b/test/backend/unit/model/sql/SearchManager.ts @@ -1,16 +1,6 @@ import {expect} from 'chai'; -import * as fs from 'fs'; -import * as path from 'path'; -import {Config} from '../../../../../common/config/private/Config'; -import {DatabaseType} from '../../../../../common/config/private/IPrivateConfig'; import {SQLConnection} from '../../../../../backend/model/sql/SQLConnection'; -import { - CameraMetadataEntity, - GPSMetadataEntity, - PhotoEntity, - PhotoMetadataEntity, - PositionMetaDataEntity -} from '../../../../../backend/model/sql/enitites/PhotoEntity'; +import {PhotoEntity} from '../../../../../backend/model/sql/enitites/PhotoEntity'; import {SearchManager} from '../../../../../backend/model/sql/SearchManager'; import {AutoCompleteItem, SearchTypes} from '../../../../../common/entities/AutoCompleteItem'; import {SearchResultDTO} from '../../../../../common/entities/SearchResultDTO'; @@ -18,64 +8,72 @@ import {DirectoryEntity} from '../../../../../backend/model/sql/enitites/Directo import {Utils} from '../../../../../common/Utils'; import {TestHelper} from './TestHelper'; import {VideoEntity} from '../../../../../backend/model/sql/enitites/VideoEntity'; +import {PersonEntry} from '../../../../../backend/model/sql/enitites/PersonEntry'; +import {FaceRegionEntry} from '../../../../../backend/model/sql/enitites/FaceRegionEntry'; +import {PhotoDTO} from '../../../../../common/entities/PhotoDTO'; +import {SQLTestHelper} from '../../../SQLTestHelper'; +import {Config} from '../../../../../common/config/private/Config'; -describe('SearchManager', () => { +// to help WebStorm to handle the test cases +declare let describe: any; +declare const after: any; +describe = SQLTestHelper.describe; - - const tempDir = path.join(__dirname, '../../tmp'); - const dbPath = path.join(tempDir, 'test.db'); +describe('SearchManager', (sqlHelper: SQLTestHelper) => { const dir = TestHelper.getDirectoryEntry(); const p = TestHelper.getPhotoEntry1(dir); const p2 = TestHelper.getPhotoEntry2(dir); + const p_faceLess = TestHelper.getPhotoEntry2(dir); + delete p_faceLess.metadata.faces; + p_faceLess.name = 'fl'; const v = TestHelper.getVideoEntry1(dir); const setUpSqlDB = async () => { - if (fs.existsSync(dbPath)) { - fs.unlinkSync(dbPath); - } - if (!fs.existsSync(tempDir)) { - fs.mkdirSync(tempDir); - } - - Config.Server.database.type = DatabaseType.sqlite; - Config.Server.database.sqlite.storage = dbPath; + await sqlHelper.initDB(); + const savePhoto = async (photo: PhotoDTO) => { + const savedPhoto = await pr.save(photo); + if (!photo.metadata.faces) { + return; + } + for (let i = 0; i < photo.metadata.faces.length; i++) { + const face = photo.metadata.faces[i]; + const person = await conn.getRepository(PersonEntry).save({name: face.name}); + await conn.getRepository(FaceRegionEntry).save({box: face.box, person: person, media: savedPhoto}); + } + }; const conn = await SQLConnection.getConnection(); const pr = conn.getRepository(PhotoEntity); await conn.getRepository(DirectoryEntity).save(p.directory); - await pr.save(p); - await pr.save(p2); + await savePhoto(p); + await savePhoto(p2); + await savePhoto(p_faceLess); + await conn.getRepository(VideoEntity).save(v); await SQLConnection.close(); }; - const tearDownSqlDB = async () => { - await SQLConnection.close(); - if (fs.existsSync(dbPath)) { - fs.unlinkSync(dbPath); - } - if (fs.existsSync(tempDir)) { - fs.rmdirSync(tempDir); - } - }; beforeEach(async () => { await setUpSqlDB(); }); - afterEach(async () => { - await tearDownSqlDB(); - }); + after(async () => { + await sqlHelper.clearDB(); + }); it('should get autocomplete', async () => { const sm = new SearchManager(); const cmp = (a: AutoCompleteItem, b: AutoCompleteItem) => { + if (a.text === b.text) { + return a.type - b.type; + } return a.text.localeCompare(b.text); }; @@ -87,11 +85,19 @@ describe('SearchManager', () => { new AutoCompleteItem('wars dir', SearchTypes.directory)]); expect((await sm.autocomplete('arch'))).eql([new AutoCompleteItem('Research City', SearchTypes.position)]); + + Config.Client.Search.AutoComplete.maxItemsPerCategory = 99999; expect((await sm.autocomplete('a')).sort(cmp)).eql([ new AutoCompleteItem('Boba Fett', SearchTypes.keyword), + new AutoCompleteItem('Boba Fett', SearchTypes.person), new AutoCompleteItem('star wars', SearchTypes.keyword), new AutoCompleteItem('Anakin', SearchTypes.keyword), + new AutoCompleteItem('Anakin Skywalker', SearchTypes.person), + new AutoCompleteItem('Luke Skywalker', SearchTypes.person), + new AutoCompleteItem('Han Solo', SearchTypes.person), new AutoCompleteItem('death star', SearchTypes.keyword), + new AutoCompleteItem('Padmé Amidala', SearchTypes.person), + new AutoCompleteItem('Obivan Kenobi', SearchTypes.person), new AutoCompleteItem('Padmé Amidala', SearchTypes.keyword), new AutoCompleteItem('Natalie Portman', SearchTypes.keyword), new AutoCompleteItem('Han Solo\'s dice', SearchTypes.photo), @@ -100,10 +106,24 @@ describe('SearchManager', () => { new AutoCompleteItem('wars dir', SearchTypes.directory), new AutoCompleteItem('Research City', SearchTypes.position)].sort(cmp)); + Config.Client.Search.AutoComplete.maxItemsPerCategory = 1; + expect((await sm.autocomplete('a')).sort(cmp)).eql([ + new AutoCompleteItem('Anakin', SearchTypes.keyword), + new AutoCompleteItem('star wars', SearchTypes.keyword), + new AutoCompleteItem('death star', SearchTypes.keyword), + new AutoCompleteItem('Anakin Skywalker', SearchTypes.person), + new AutoCompleteItem('Han Solo\'s dice', SearchTypes.photo), + new AutoCompleteItem('Kamino', SearchTypes.position), + new AutoCompleteItem('Research City', SearchTypes.position), + new AutoCompleteItem('wars dir', SearchTypes.directory), + new AutoCompleteItem('Boba Fett', SearchTypes.keyword)].sort(cmp)); + Config.Client.Search.AutoComplete.maxItemsPerCategory = 5; + expect((await sm.autocomplete('sw')).sort(cmp)).to.deep.equal([new AutoCompleteItem('sw1', SearchTypes.photo), new AutoCompleteItem('sw2', SearchTypes.photo), new AutoCompleteItem(v.name, SearchTypes.video)].sort(cmp)); expect((await sm.autocomplete(v.name)).sort(cmp)).to.deep.equal([new AutoCompleteItem(v.name, SearchTypes.video)]); + }); @@ -120,6 +140,15 @@ describe('SearchManager', () => { resultOverflow: false })); + expect(Utils.clone(await sm.search('Boba', null))).to.deep.equal(Utils.clone({ + searchText: 'Boba', + searchType: null, + directories: [], + media: [p], + metaFile: [], + resultOverflow: false + })); + expect(Utils.clone(await sm.search('Tatooine', SearchTypes.position))).to.deep.equal(Utils.clone({ searchText: 'Tatooine', searchType: SearchTypes.position, @@ -133,7 +162,7 @@ describe('SearchManager', () => { searchText: 'ortm', searchType: SearchTypes.keyword, directories: [], - media: [p2], + media: [p2, p_faceLess], metaFile: [], resultOverflow: false })); @@ -142,7 +171,7 @@ describe('SearchManager', () => { searchText: 'ortm', searchType: SearchTypes.keyword, directories: [], - media: [p2], + media: [p2, p_faceLess], metaFile: [], resultOverflow: false })); @@ -151,7 +180,7 @@ describe('SearchManager', () => { searchText: 'wa', searchType: SearchTypes.keyword, directories: [dir], - media: [p, p2], + media: [p, p2, p_faceLess], metaFile: [], resultOverflow: false })); @@ -165,6 +194,15 @@ describe('SearchManager', () => { resultOverflow: false })); + expect(Utils.clone(await sm.search('sw', SearchTypes.video))).to.deep.equal(Utils.clone({ + searchText: 'sw', + searchType: SearchTypes.video, + directories: [], + media: [v], + metaFile: [], + resultOverflow: false + })); + expect(Utils.clone(await sm.search('han', SearchTypes.keyword))).to.deep.equal(Utils.clone({ searchText: 'han', searchType: SearchTypes.keyword, @@ -173,6 +211,15 @@ describe('SearchManager', () => { metaFile: [], resultOverflow: false })); + + expect(Utils.clone(await sm.search('Boba', SearchTypes.person))).to.deep.equal(Utils.clone({ + searchText: 'Boba', + searchType: SearchTypes.person, + directories: [], + media: [p], + metaFile: [], + resultOverflow: false + })); }); @@ -198,23 +245,16 @@ describe('SearchManager', () => { expect(Utils.clone(await sm.instantSearch('ortm'))).to.deep.equal(Utils.clone({ searchText: 'ortm', directories: [], - media: [p2], + media: [p2, p_faceLess], metaFile: [], resultOverflow: false })); - expect(Utils.clone(await sm.instantSearch('ortm'))).to.deep.equal(Utils.clone({ - searchText: 'ortm', - directories: [], - media: [p2], - metaFile: [], - resultOverflow: false - })); expect(Utils.clone(await sm.instantSearch('wa'))).to.deep.equal(Utils.clone({ searchText: 'wa', directories: [dir], - media: [p, p2], + media: [p, p2, p_faceLess], metaFile: [], resultOverflow: false })); @@ -226,6 +266,13 @@ describe('SearchManager', () => { metaFile: [], resultOverflow: false })); + expect(Utils.clone(await sm.instantSearch('Boba'))).to.deep.equal(Utils.clone({ + searchText: 'Boba', + directories: [], + media: [p], + metaFile: [], + resultOverflow: false + })); }); diff --git a/test/backend/unit/model/sql/SharingManager.ts b/test/backend/unit/model/sql/SharingManager.ts index d11b72b..c866a4a 100644 --- a/test/backend/unit/model/sql/SharingManager.ts +++ b/test/backend/unit/model/sql/SharingManager.ts @@ -1,32 +1,23 @@ import {expect} from 'chai'; -import * as fs from 'fs'; -import * as path from 'path'; -import {Config} from '../../../../../common/config/private/Config'; -import {DatabaseType} from '../../../../../common/config/private/IPrivateConfig'; import {SQLConnection} from '../../../../../backend/model/sql/SQLConnection'; import {SharingManager} from '../../../../../backend/model/sql/SharingManager'; import {SharingDTO} from '../../../../../common/entities/SharingDTO'; import {UserEntity} from '../../../../../backend/model/sql/enitites/UserEntity'; import {UserDTO, UserRoles} from '../../../../../common/entities/UserDTO'; +import {SQLTestHelper} from '../../../SQLTestHelper'; -describe('SharingManager', () => { +// to help WebStorm to handle the test cases +declare let describe: any; +declare const after: any; +describe = SQLTestHelper.describe; +describe('SharingManager', (sqlHelper: SQLTestHelper) => { - const tempDir = path.join(__dirname, '../../tmp'); - const dbPath = path.join(tempDir, 'test.db'); let creator: UserDTO = null; const setUpSqlDB = async () => { - if (fs.existsSync(dbPath)) { - fs.unlinkSync(dbPath); - } - if (!fs.existsSync(tempDir)) { - fs.mkdirSync(tempDir); - } - - Config.Server.database.type = DatabaseType.sqlite; - Config.Server.database.sqlite.storage = dbPath; + await sqlHelper.initDB(); const conn = await SQLConnection.getConnection(); @@ -41,22 +32,13 @@ describe('SharingManager', () => { await SQLConnection.close(); }; - const teardownUpSqlDB = async () => { - await SQLConnection.close(); - if (fs.existsSync(dbPath)) { - fs.unlinkSync(dbPath); - } - if (fs.existsSync(tempDir)) { - fs.rmdirSync(tempDir); - } - }; beforeEach(async () => { await setUpSqlDB(); }); - afterEach(async () => { - await teardownUpSqlDB(); + after(async () => { + await sqlHelper.clearDB(); }); diff --git a/test/backend/unit/model/sql/TestHelper.ts b/test/backend/unit/model/sql/TestHelper.ts index 623e3fc..05c985e 100644 --- a/test/backend/unit/model/sql/TestHelper.ts +++ b/test/backend/unit/model/sql/TestHelper.ts @@ -1,7 +1,8 @@ import {MediaDimensionEntity} from '../../../../../backend/model/sql/enitites/MediaEntity'; import { CameraMetadataEntity, - GPSMetadataEntity, PhotoEntity, + GPSMetadataEntity, + PhotoEntity, PhotoMetadataEntity, PositionMetaDataEntity } from '../../../../../backend/model/sql/enitites/PhotoEntity'; @@ -9,9 +10,8 @@ import * as path from 'path'; import {OrientationTypes} from 'ts-exif-parser'; import {DirectoryEntity} from '../../../../../backend/model/sql/enitites/DirectoryEntity'; import {VideoEntity, VideoMetadataEntity} from '../../../../../backend/model/sql/enitites/VideoEntity'; -import {FileEntity} from '../../../../../backend/model/sql/enitites/FileEntity'; import {MediaDimension} from '../../../../../common/entities/MediaDTO'; -import {CameraMetadata, GPSMetadata, PhotoDTO, PhotoMetadata, PositionMetaData} from '../../../../../common/entities/PhotoDTO'; +import {CameraMetadata, FaceRegion, GPSMetadata, PhotoDTO, PhotoMetadata, PositionMetaData} from '../../../../../common/entities/PhotoDTO'; import {DirectoryDTO} from '../../../../../common/entities/DirectoryDTO'; import {FileDTO} from '../../../../../common/entities/FileDTO'; @@ -104,6 +104,20 @@ export class TestHelper { p.metadata.positionData.city = 'Mos Eisley'; p.metadata.positionData.country = 'Tatooine'; p.name = 'sw1'; + + p.metadata.faces = [{ + box: {height: 10, width: 10, x: 10, y: 10}, + name: 'Boba Fett' + }, { + box: {height: 10, width: 10, x: 101, y: 101}, + name: 'Luke Skywalker' + }, { + box: {height: 10, width: 10, x: 101, y: 101}, + name: 'Han Solo' + }, { + box: {height: 10, width: 10, x: 101, y: 101}, + name: 'Unkle Ben' + }]; return p; } @@ -121,6 +135,16 @@ export class TestHelper { p.metadata.positionData.state = 'Research City'; p.metadata.positionData.country = 'Kamino'; p.name = 'sw2'; + p.metadata.faces = [{ + box: {height: 10, width: 10, x: 10, y: 10}, + name: 'Padmé Amidala' + }, { + box: {height: 10, width: 10, x: 101, y: 101}, + name: 'Anakin Skywalker' + }, { + box: {height: 10, width: 10, x: 101, y: 101}, + name: 'Obivan Kenobi' + }]; return p; } @@ -157,14 +181,37 @@ export class TestHelper { return d; } - public static getRandomizedPhotoEntry(dir: DirectoryDTO, forceStr: string = null) { + + public static getRandomizedFace(media: PhotoDTO, forceStr: string = null) { + const rndStr = () => { + return forceStr + '_' + Math.random().toString(36).substring(7); + }; + + const rndInt = (max = 5000) => { + return Math.floor(Math.random() * max); + }; + + const f: FaceRegion = { + name: rndStr() + '.jpg', + box: { + x: rndInt(), + y: rndInt(), + width: rndInt(), + height: rndInt() + } + }; + media.metadata.faces = (media.metadata.faces || []); + media.metadata.faces.push(f); + return f; + } + + public static getRandomizedPhotoEntry(dir: DirectoryDTO, forceStr: string = null, faces: number = 2): PhotoDTO { const rndStr = () => { return forceStr + '_' + Math.random().toString(36).substring(7); }; - const rndInt = (max = 5000) => { return Math.floor(Math.random() * max); }; @@ -211,10 +258,14 @@ export class TestHelper { name: rndStr() + '.jpg', directory: dir, metadata: m, - readyThumbnails: null, + readyThumbnails: [], readyIcon: false }; + for (let i = 0; i < faces; i++) { + this.getRandomizedFace(d, 'Person ' + i); + } + dir.media.push(d); return d; } diff --git a/test/backend/unit/model/threading/DiskMangerWorker.spec.ts b/test/backend/unit/model/threading/DiskMangerWorker.spec.ts index 9c541d9..b045bb9 100644 --- a/test/backend/unit/model/threading/DiskMangerWorker.spec.ts +++ b/test/backend/unit/model/threading/DiskMangerWorker.spec.ts @@ -3,7 +3,7 @@ import {DiskMangerWorker} from '../../../../../backend/model/threading/DiskMange import * as path from 'path'; import {Config} from '../../../../../common/config/private/Config'; import {ProjectPath} from '../../../../../backend/ProjectPath'; -import {PhotoDTO} from '../../../../../common/entities/PhotoDTO'; +import {Utils} from '../../../../../common/Utils'; describe('DiskMangerWorker', () => { @@ -11,34 +11,10 @@ describe('DiskMangerWorker', () => { Config.Server.imagesFolder = path.join(__dirname, '/../../assets'); ProjectPath.ImageFolder = path.join(__dirname, '/../../assets'); const dir = await DiskMangerWorker.scanDirectory('/'); - expect(dir.media.length).to.be.equals(2); - expect(dir.media[0].name).to.be.equals('test image öüóőúéáű-.,.jpg'); - expect((dir.media[0]).metadata.keywords).to.deep.equals(['Berkley', 'USA', 'űáéúőóüö ŰÁÉÚŐÓÜÖ']); - expect(dir.media[0].metadata.fileSize).to.deep.equals(62786); - expect(dir.media[0].metadata.size).to.deep.equals({width: 140, height: 93}); - expect((dir.media[0]).metadata.cameraData).to.deep.equals({ - ISO: 3200, - model: 'óüöúőűáé ÓÜÖÚŐŰÁÉ', - make: 'Canon', - fStop: 5.6, - exposure: 0.00125, - focalLength: 85, - lens: 'EF-S15-85mm f/3.5-5.6 IS USM' - }); - - expect((dir.media[0]).metadata.positionData).to.deep.equals({ - GPSData: { - latitude: 37.871093333333334, - longitude: -122.25678, - altitude: 102.4498997995992 - }, - country: 'mmóüöúőűáé ÓÜÖÚŐŰÁÉmm-.,|\\mm', - state: 'óüöúőűáé ÓÜÖÚŐŰÁ', - city: 'óüöúőűáé ÓÜÖÚŐŰÁ' - }); - - expect(dir.media[0].metadata.creationDate).to.be.equals(1434018566000); - + expect(dir.media.length).to.be.equals(3); + const expected = require(path.join(__dirname, '/../../assets/test image öüóőúéáű-.,.json')); + expect(Utils.clone(dir.media[1].name)).to.be.deep.equal('test image öüóőúéáű-.,.jpg'); + expect(Utils.clone(dir.media[1].metadata)).to.be.deep.equal(expected); }); }); diff --git a/test/backend/unit/model/threading/MetaDataLoader.spec.ts b/test/backend/unit/model/threading/MetaDataLoader.spec.ts index a5ddaf5..468fa9f 100644 --- a/test/backend/unit/model/threading/MetaDataLoader.spec.ts +++ b/test/backend/unit/model/threading/MetaDataLoader.spec.ts @@ -20,36 +20,15 @@ describe('MetadataLoader', () => { it('should load jpg', async () => { const data = await MetadataLoader.loadPhotoMetadata(path.join(__dirname, '/../../assets/test image öüóőúéáű-.,.jpg')); - expect(Utils.clone(data)).to.be.deep.equal(Utils.clone({ - size: {width: 140, height: 93}, - orientation: 1, - caption: 'Test caption', - creationDate: 1434018566000, - fileSize: 62786, - cameraData: - { - ISO: 3200, - model: 'óüöúőűáé ÓÜÖÚŐŰÁÉ', - make: 'Canon', - fStop: 5.6, - exposure: 0.00125, - focalLength: 85, - lens: 'EF-S15-85mm f/3.5-5.6 IS USM' - }, - positionData: - { - GPSData: - { - latitude: 37.871093333333334, - longitude: -122.25678, - altitude: 102.4498997995992 - }, - country: 'mmóüöúőűáé ÓÜÖÚŐŰÁÉmm-.,|\\mm', - state: 'óüöúőűáé ÓÜÖÚŐŰÁ', - city: 'óüöúőűáé ÓÜÖÚŐŰÁ' - }, - keywords: ['Berkley', 'USA', 'űáéúőóüö ŰÁÉÚŐÓÜÖ'] - })); + const expected = require(path.join(__dirname, '/../../assets/test image öüóőúéáű-.,.json')); + expect(Utils.clone(data)).to.be.deep.equal(expected); + }); + + + it('should load jpg 2', async () => { + const data = await MetadataLoader.loadPhotoMetadata(path.join(__dirname, '/../../assets/old_photo.jpg')); + const expected = require(path.join(__dirname, '/../../assets/old_photo.json')); + expect(Utils.clone(data)).to.be.deep.equal(expected); }); }); diff --git a/test/common/unit/Utils.spec.ts b/test/common/unit/Utils.spec.ts new file mode 100644 index 0000000..6895a56 --- /dev/null +++ b/test/common/unit/Utils.spec.ts @@ -0,0 +1,18 @@ +import {expect} from 'chai'; +import {Utils} from '../../../common/Utils'; + +describe('Utils', () => { + it('should concat urls', () => { + expect(Utils.concatUrls('abc', 'cde')).to.be.equal('abc/cde'); + expect(Utils.concatUrls('abc/', 'cde')).to.be.equal('abc/cde'); + expect(Utils.concatUrls('abc\\', 'cde')).to.be.equal('abc/cde'); + expect(Utils.concatUrls('abc/', 'cde/')).to.be.equal('abc/cde'); + expect(Utils.concatUrls('./abc\\', 'cde/')).to.be.equal('./abc/cde'); + expect(Utils.concatUrls('abc/', '\\cde/')).to.be.equal('abc/cde'); + expect(Utils.concatUrls('abc\\', '\\cde/')).to.be.equal('abc/cde'); + expect(Utils.concatUrls('abc\\', '/cde/')).to.be.equal('abc/cde'); + expect(Utils.concatUrls('abc/', '/cde/')).to.be.equal('abc/cde'); + expect(Utils.concatUrls('abc\\/', '/cde/')).to.be.equal('abc/cde'); + expect(Utils.concatUrls('abc\\/', '/cde/', 'fgh')).to.be.equal('abc/cde/fgh'); + }); +});