Merge pull request #43 from bpatrik/develop

Develop

.travis.yml

@@ -3,7 +3,10 @@ language: node_js
 node_js:
   - '10'
   - '11'
-
+env:
+  - MYSQL_HOST='localhost' MYSQL_PASSWORD='' MYSQL_USERNAME='root' MYSQL_DATABASE='pigallery2_test'
+services:
+  - mysql
 addons:
   chrome: stable
 before_install:

backend/middlewares/AdminMWs.ts

@@ -42,6 +42,25 @@ export class AdminMWs {
     }
   }
 
+
+  public static async getDuplicates(req: Request, res: Response, next: NextFunction) {
+    if (Config.Server.database.type === DatabaseType.memory) {
+      return next(new ErrorDTO(ErrorCodes.GENERAL_ERROR, 'Statistic is only available for indexed content'));
+    }
+
+
+    const galleryManager = <ISQLGalleryManager>ObjectManagerRepository.getInstance().GalleryManager;
+    try {
+      req.resultPipe =  await galleryManager.getPossibleDuplicates();
+      return next();
+    } catch (err) {
+      if (err instanceof Error) {
+        return next(new ErrorDTO(ErrorCodes.GENERAL_ERROR, 'Error while getting duplicates: ' + err.toString(), err));
+      }
+      return next(new ErrorDTO(ErrorCodes.GENERAL_ERROR, 'Error while getting duplicates', err));
+    }
+  }
+
   public static async updateDatabaseSettings(req: Request, res: Response, next: NextFunction) {
 
     if ((typeof req.body === 'undefined') || (typeof req.body.settings === 'undefined')) {
@@ -399,7 +418,7 @@ export class AdminMWs {
   public static startIndexing(req: Request, res: Response, next: NextFunction) {
     try {
       const createThumbnails: boolean = (<IndexingDTO>req.body).createThumbnails || false;
-      ObjectManagerRepository.getInstance().IndexingManager.startIndexing(createThumbnails);
+      ObjectManagerRepository.getInstance().IndexingTaskManager.startIndexing(createThumbnails);
       req.resultPipe = 'ok';
       return next();
     } catch (err) {
@@ -413,7 +432,7 @@ export class AdminMWs {
 
   public static getIndexingProgress(req: Request, res: Response, next: NextFunction) {
     try {
-      req.resultPipe = ObjectManagerRepository.getInstance().IndexingManager.getProgress();
+      req.resultPipe = ObjectManagerRepository.getInstance().IndexingTaskManager.getProgress();
       return next();
     } catch (err) {
       if (err instanceof Error) {
@@ -425,7 +444,7 @@ export class AdminMWs {
 
   public static cancelIndexing(req: Request, res: Response, next: NextFunction) {
     try {
-      ObjectManagerRepository.getInstance().IndexingManager.cancelIndexing();
+      ObjectManagerRepository.getInstance().IndexingTaskManager.cancelIndexing();
       req.resultPipe = 'ok';
       return next();
     } catch (err) {
@@ -438,7 +457,7 @@ export class AdminMWs {
 
   public static async resetIndexes(req: Express.Request, res: Response, next: NextFunction) {
     try {
-      await ObjectManagerRepository.getInstance().IndexingManager.reset();
+      await ObjectManagerRepository.getInstance().IndexingTaskManager.reset();
       req.resultPipe = 'ok';
       return next();
     } catch (err) {

backend/middlewares/GalleryMWs.ts

@@ -222,7 +222,7 @@ export class GalleryMWs {
   }
 
   public static async autocomplete(req: Request, res: Response, next: NextFunction) {
-    if (Config.Client.Search.autocompleteEnabled === false) {
+    if (Config.Client.Search.AutoComplete.enabled === false) {
       return next();
     }
     if (!(req.params.text)) {

backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts

@@ -65,7 +65,7 @@ export class ThumbnailGeneratorMWs {
       }
 
     } catch (error) {
-      return next(new ErrorDTO(ErrorCodes.SERVER_ERROR, 'error during postprocessing result', error.toString()));
+      return next(new ErrorDTO(ErrorCodes.SERVER_ERROR, 'error during postprocessing result (adding thumbnail info)', error.toString()));
 
     }
 

backend/model/ObjectManagerRepository.ts

@@ -4,7 +4,9 @@ import {ISearchManager} from './interfaces/ISearchManager';
 import {SQLConnection} from './sql/SQLConnection';
 import {ISharingManager} from './interfaces/ISharingManager';
 import {Logger} from '../Logger';
+import {IIndexingTaskManager} from './interfaces/IIndexingTaskManager';
 import {IIndexingManager} from './interfaces/IIndexingManager';
+import {IPersonManager} from './interfaces/IPersonManager';
 
 export class ObjectManagerRepository {
 
@@ -15,6 +17,16 @@ export class ObjectManagerRepository {
   private _searchManager: ISearchManager;
   private _sharingManager: ISharingManager;
   private _indexingManager: IIndexingManager;
+  private _indexingTaskManager: IIndexingTaskManager;
+  private _personManager: IPersonManager;
+
+  get PersonManager(): IPersonManager {
+    return this._personManager;
+  }
+
+  set PersonManager(value: IPersonManager) {
+    this._personManager = value;
+  }
 
   get IndexingManager(): IIndexingManager {
     return this._indexingManager;
@@ -24,19 +36,14 @@ export class ObjectManagerRepository {
     this._indexingManager = value;
   }
 
-  public static getInstance() {
-    if (this._instance === null) {
-      this._instance = new ObjectManagerRepository();
-    }
-    return this._instance;
+  get IndexingTaskManager(): IIndexingTaskManager {
+    return this._indexingTaskManager;
   }
 
-  public static async reset() {
-    await SQLConnection.close();
-    this._instance = null;
+  set IndexingTaskManager(value: IIndexingTaskManager) {
+    this._indexingTaskManager = value;
   }
 
-
   get GalleryManager(): IGalleryManager {
     return this._galleryManager;
   }
@@ -69,18 +76,34 @@ export class ObjectManagerRepository {
     this._sharingManager = value;
   }
 
+  public static getInstance() {
+    if (this._instance === null) {
+      this._instance = new ObjectManagerRepository();
+    }
+    return this._instance;
+  }
+
+  public static async reset() {
+    await SQLConnection.close();
+    this._instance = null;
+  }
+
   public static async InitMemoryManagers() {
     await ObjectManagerRepository.reset();
     const GalleryManager = require('./memory/GalleryManager').GalleryManager;
     const UserManager = require('./memory/UserManager').UserManager;
     const SearchManager = require('./memory/SearchManager').SearchManager;
     const SharingManager = require('./memory/SharingManager').SharingManager;
+    const IndexingTaskManager = require('./memory/IndexingTaskManager').IndexingTaskManager;
     const IndexingManager = require('./memory/IndexingManager').IndexingManager;
+    const PersonManager = require('./memory/PersonManager').PersonManager;
     ObjectManagerRepository.getInstance().GalleryManager = new GalleryManager();
     ObjectManagerRepository.getInstance().UserManager = new UserManager();
     ObjectManagerRepository.getInstance().SearchManager = new SearchManager();
     ObjectManagerRepository.getInstance().SharingManager = new SharingManager();
+    ObjectManagerRepository.getInstance().IndexingTaskManager = new IndexingTaskManager();
     ObjectManagerRepository.getInstance().IndexingManager = new IndexingManager();
+    ObjectManagerRepository.getInstance().PersonManager = new PersonManager();
   }
 
   public static async InitSQLManagers() {
@@ -90,12 +113,16 @@ export class ObjectManagerRepository {
     const UserManager = require('./sql/UserManager').UserManager;
     const SearchManager = require('./sql/SearchManager').SearchManager;
     const SharingManager = require('./sql/SharingManager').SharingManager;
+    const IndexingTaskManager = require('./sql/IndexingTaskManager').IndexingTaskManager;
     const IndexingManager = require('./sql/IndexingManager').IndexingManager;
+    const PersonManager = require('./sql/PersonManager').PersonManager;
     ObjectManagerRepository.getInstance().GalleryManager = new GalleryManager();
     ObjectManagerRepository.getInstance().UserManager = new UserManager();
     ObjectManagerRepository.getInstance().SearchManager = new SearchManager();
     ObjectManagerRepository.getInstance().SharingManager = new SharingManager();
+    ObjectManagerRepository.getInstance().IndexingTaskManager = new IndexingTaskManager();
     ObjectManagerRepository.getInstance().IndexingManager = new IndexingManager();
+    ObjectManagerRepository.getInstance().PersonManager = new PersonManager();
     Logger.debug('SQL DB inited');
   }
 

backend/model/interfaces/IIndexingManager.ts

@@ -1,11 +1,5 @@
-import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO';
+import {DirectoryDTO} from '../../../common/entities/DirectoryDTO';
 
 export interface IIndexingManager {
-  startIndexing(createThumbnails?: boolean): void;
-
-  getProgress(): IndexingProgressDTO;
-
-  cancelIndexing(): void;
-
-  reset(): Promise<void>;
+  indexDirectory(relativeDirectoryName: string): Promise<DirectoryDTO>;
 }

backend/model/interfaces/IIndexingTaskManager.ts

@@ -0,0 +1,11 @@
+import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO';
+
+export interface IIndexingTaskManager {
+  startIndexing(createThumbnails?: boolean): void;
+
+  getProgress(): IndexingProgressDTO;
+
+  cancelIndexing(): void;
+
+  reset(): Promise<void>;
+}

backend/model/interfaces/IPersonManager.ts

@@ -0,0 +1,7 @@
+import {PersonEntry} from '../sql/enitites/PersonEntry';
+
+export interface IPersonManager {
+  get(name: string): Promise<PersonEntry>;
+
+  saveAll(names: string[]): Promise<void>;
+}

backend/model/memory/IndexingManager.ts

@@ -1,21 +1,11 @@
 import {IIndexingManager} from '../interfaces/IIndexingManager';
-import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO';
+import {DirectoryDTO} from '../../../common/entities/DirectoryDTO';
 
 export class IndexingManager implements IIndexingManager {
 
-  startIndexing(): void {
+  indexDirectory(relativeDirectoryName: string): Promise<DirectoryDTO> {
     throw new Error('not supported by memory DB');
   }
 
-  getProgress(): IndexingProgressDTO {
-    throw new Error('not supported by memory DB');
-  }
 
-  cancelIndexing(): void {
-    throw new Error('not supported by memory DB');
-  }
-
-  reset(): Promise<void> {
-    throw new Error('Method not implemented.');
-  }
 }

backend/model/memory/IndexingTaskManager.ts

@@ -0,0 +1,21 @@
+import {IIndexingTaskManager} from '../interfaces/IIndexingTaskManager';
+import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO';
+
+export class IndexingTaskManager implements IIndexingTaskManager {
+
+  startIndexing(): void {
+    throw new Error('not supported by memory DB');
+  }
+
+  getProgress(): IndexingProgressDTO {
+    throw new Error('not supported by memory DB');
+  }
+
+  cancelIndexing(): void {
+    throw new Error('not supported by memory DB');
+  }
+
+  reset(): Promise<void> {
+    throw new Error('Method not implemented.');
+  }
+}

backend/model/memory/PersonManager.ts

@@ -0,0 +1,11 @@
+import {IPersonManager} from '../interfaces/IPersonManager';
+
+export class IndexingTaskManager implements IPersonManager {
+  get(name: string): Promise<any> {
+    throw new Error('not supported by memory DB');
+  }
+
+  saveAll(names: string[]): Promise<void> {
+    throw new Error('not supported by memory DB');
+  }
+}

backend/model/sql/GalleryManager.ts

@@ -4,80 +4,26 @@ import * as path from 'path';
 import * as fs from 'fs';
 import {DirectoryEntity} from './enitites/DirectoryEntity';
 import {SQLConnection} from './SQLConnection';
-import {DiskManager} from '../DiskManger';
 import {PhotoEntity} from './enitites/PhotoEntity';
-import {Utils} from '../../../common/Utils';
 import {ProjectPath} from '../../ProjectPath';
 import {Config} from '../../../common/config/private/Config';
 import {ISQLGalleryManager} from './IGalleryManager';
 import {DatabaseType, ReIndexingSensitivity} from '../../../common/config/private/IPrivateConfig';
 import {PhotoDTO} from '../../../common/entities/PhotoDTO';
 import {OrientationType} from '../../../common/entities/RandomQueryDTO';
-import {Brackets, Connection, Transaction, TransactionRepository, Repository} from 'typeorm';
+import {Brackets, Connection} from 'typeorm';
 import {MediaEntity} from './enitites/MediaEntity';
-import {MediaDTO} from '../../../common/entities/MediaDTO';
 import {VideoEntity} from './enitites/VideoEntity';
-import {FileEntity} from './enitites/FileEntity';
-import {FileDTO} from '../../../common/entities/FileDTO';
-import {NotificationManager} from '../NotifocationManager';
 import {DiskMangerWorker} from '../threading/DiskMangerWorker';
 import {Logger} from '../../Logger';
+import {FaceRegionEntry} from './enitites/FaceRegionEntry';
+import {ObjectManagerRepository} from '../ObjectManagerRepository';
+import {DuplicatesDTO} from '../../../common/entities/DuplicatesDTO';
 
 const LOG_TAG = '[GalleryManager]';
 
 export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
 
-  private savingQueue: DirectoryDTO[] = [];
-  private isSaving = false;
-
-  protected async selectParentDir(connection: Connection, directoryName: string, directoryParent: string): Promise<DirectoryEntity> {
-    const query = connection
-      .getRepository(DirectoryEntity)
-      .createQueryBuilder('directory')
-      .where('directory.name = :name AND directory.path = :path', {
-        name: directoryName,
-        path: directoryParent
-      })
-      .leftJoinAndSelect('directory.directories', 'directories')
-      .leftJoinAndSelect('directory.media', 'media');
-
-    if (Config.Client.MetaFile.enabled === true) {
-      query.leftJoinAndSelect('directory.metaFile', 'metaFile');
-    }
-
-    return await query.getOne();
-  }
-
-  protected async fillParentDir(connection: Connection, dir: DirectoryEntity): Promise<void> {
-    if (dir.media) {
-      for (let i = 0; i < dir.media.length; i++) {
-        dir.media[i].directory = dir;
-        dir.media[i].readyThumbnails = [];
-        dir.media[i].readyIcon = false;
-      }
-    }
-    if (dir.directories) {
-      for (let i = 0; i < dir.directories.length; i++) {
-        dir.directories[i].media = await connection
-          .getRepository(MediaEntity)
-          .createQueryBuilder('media')
-          .where('media.directory = :dir', {
-            dir: dir.directories[i].id
-          })
-          .orderBy('media.metadata.creationDate', 'ASC')
-          .limit(Config.Server.indexing.folderPreviewSize)
-          .getMany();
-        dir.directories[i].isPartial = true;
-
-        for (let j = 0; j < dir.directories[i].media.length; j++) {
-          dir.directories[i].media[j].directory = dir.directories[i];
-          dir.directories[i].media[j].readyThumbnails = [];
-          dir.directories[i].media[j].readyIcon = false;
-        }
-      }
-    }
-  }
-
 
   public async listDirectory(relativeDirectoryName: string,
                              knownLastModified?: number,
@@ -110,7 +56,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
       if (dir.lastModified !== lastModified) {
         Logger.silly(LOG_TAG, 'Reindexing reason: lastModified mismatch: known: '
           + dir.lastModified + ', current:' + lastModified);
-        return this.indexDirectory(relativeDirectoryName);
+        return ObjectManagerRepository.getInstance().IndexingManager.indexDirectory(relativeDirectoryName);
       }
 
 
@@ -121,8 +67,8 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
         // on the fly reindexing
 
         Logger.silly(LOG_TAG, 'lazy reindexing reason: cache timeout: lastScanned: '
-          + (Date.now() - dir.lastScanned) + ', cachedFolderTimeout:' + Config.Server.indexing.cachedFolderTimeout);
-        this.indexDirectory(relativeDirectoryName).catch((err) => {
+          + (Date.now() - dir.lastScanned) + ' ms ago, cachedFolderTimeout:' + Config.Server.indexing.cachedFolderTimeout);
+        ObjectManagerRepository.getInstance().IndexingManager.indexDirectory(relativeDirectoryName).catch((err) => {
           console.error(err);
         });
       }
@@ -132,33 +78,11 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
 
     // never scanned (deep indexed), do it and return with it
     Logger.silly(LOG_TAG, 'Reindexing reason: never scanned');
-    return this.indexDirectory(relativeDirectoryName);
+    return ObjectManagerRepository.getInstance().IndexingManager.indexDirectory(relativeDirectoryName);
 
 
   }
 
-
-  public indexDirectory(relativeDirectoryName: string): Promise<DirectoryDTO> {
-    return new Promise(async (resolve, reject) => {
-      try {
-        const scannedDirectory = await DiskManager.scanDirectory(relativeDirectoryName);
-
-        // returning with the result
-        scannedDirectory.media.forEach(p => p.readyThumbnails = []);
-        resolve(scannedDirectory);
-
-        this.queueForSave(scannedDirectory).catch(console.error);
-
-      } catch (error) {
-        NotificationManager.warning('Unknown indexing error for: ' + relativeDirectoryName, error.toString());
-        console.error(error);
-        return reject(error);
-      }
-
-    });
-  }
-
-
   public async getRandomPhoto(queryFilter: RandomQuery): Promise<PhotoDTO> {
     const connection = await SQLConnection.getConnection();
     const photosRepository = connection.getRepository(PhotoEntity);
@@ -216,164 +140,6 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
 
   }
 
-  // Todo fix it, once typeorm support connection pools ofr sqlite
-  protected async queueForSave(scannedDirectory: DirectoryDTO) {
-    if (this.savingQueue.findIndex(dir => dir.name === scannedDirectory.name &&
-      dir.path === scannedDirectory.path &&
-      dir.lastModified === scannedDirectory.lastModified &&
-      dir.lastScanned === scannedDirectory.lastScanned &&
-      (dir.media || dir.media.length) === (scannedDirectory.media || scannedDirectory.media.length) &&
-      (dir.metaFile || dir.metaFile.length) === (scannedDirectory.metaFile || scannedDirectory.metaFile.length)) !== -1) {
-      return;
-    }
-    this.savingQueue.push(scannedDirectory);
-    while (this.isSaving === false && this.savingQueue.length > 0) {
-      await this.saveToDB(this.savingQueue[0]);
-      this.savingQueue.shift();
-    }
-
-  }
-
-  protected async saveToDB(scannedDirectory: DirectoryDTO) {
-    this.isSaving = true;
-    try {
-      const connection = await SQLConnection.getConnection();
-
-      // saving to db
-      const directoryRepository = connection.getRepository(DirectoryEntity);
-      const mediaRepository = connection.getRepository(MediaEntity);
-      const fileRepository = connection.getRepository(FileEntity);
-
-
-      let currentDir: DirectoryEntity = await directoryRepository.createQueryBuilder('directory')
-        .where('directory.name = :name AND directory.path = :path', {
-          name: scannedDirectory.name,
-          path: scannedDirectory.path
-        }).getOne();
-      if (!!currentDir) {// Updated parent dir (if it was in the DB previously)
-        currentDir.lastModified = scannedDirectory.lastModified;
-        currentDir.lastScanned = scannedDirectory.lastScanned;
-        currentDir.mediaCount = scannedDirectory.mediaCount;
-        currentDir = await directoryRepository.save(currentDir);
-      } else {
-        currentDir = await directoryRepository.save(<DirectoryEntity>scannedDirectory);
-      }
-
-
-      // save subdirectories
-      const childDirectories = await directoryRepository.createQueryBuilder('directory')
-        .where('directory.parent = :dir', {
-          dir: currentDir.id
-        }).getMany();
-
-      for (let i = 0; i < scannedDirectory.directories.length; i++) {
-        // Was this child Dir already indexed before?
-        let directory: DirectoryEntity = null;
-        for (let j = 0; j < childDirectories.length; j++) {
-          if (childDirectories[j].name === scannedDirectory.directories[i].name) {
-            directory = childDirectories[j];
-            childDirectories.splice(j, 1);
-            break;
-          }
-        }
-
-        if (directory != null) { // update existing directory
-          if (!directory.parent || !directory.parent.id) { // set parent if not set yet
-            directory.parent = currentDir;
-            delete directory.media;
-            await directoryRepository.save(directory);
-          }
-        } else { // dir does not exists yet
-          scannedDirectory.directories[i].parent = currentDir;
-          (<DirectoryEntity>scannedDirectory.directories[i]).lastScanned = null; // new child dir, not fully scanned yet
-          const d = await directoryRepository.save(<DirectoryEntity>scannedDirectory.directories[i]);
-          for (let j = 0; j < scannedDirectory.directories[i].media.length; j++) {
-            scannedDirectory.directories[i].media[j].directory = d;
-          }
-
-          await this.saveMedia(connection, scannedDirectory.directories[i].media);
-        }
-      }
-
-      // Remove child Dirs that are not anymore in the parent dir
-      await directoryRepository.remove(childDirectories, {chunk: Math.max(Math.ceil(childDirectories.length / 500), 1)});
-
-      // save media
-      const indexedMedia = await mediaRepository.createQueryBuilder('media')
-        .where('media.directory = :dir', {
-          dir: currentDir.id
-        }).getMany();
-
-
-      const mediaToSave = [];
-      for (let i = 0; i < scannedDirectory.media.length; i++) {
-        let media: MediaDTO = null;
-        for (let j = 0; j < indexedMedia.length; j++) {
-          if (indexedMedia[j].name === scannedDirectory.media[i].name) {
-            media = indexedMedia[j];
-            indexedMedia.splice(j, 1);
-            break;
-          }
-        }
-        if (media == null) { // not in DB yet
-          scannedDirectory.media[i].directory = null;
-          media = Utils.clone(scannedDirectory.media[i]);
-          scannedDirectory.media[i].directory = scannedDirectory;
-          media.directory = currentDir;
-          mediaToSave.push(media);
-        } else if (!Utils.equalsFilter(media.metadata, scannedDirectory.media[i].metadata)) {
-          media.metadata = scannedDirectory.media[i].metadata;
-          mediaToSave.push(media);
-        }
-      }
-      await this.saveMedia(connection, mediaToSave);
-      await mediaRepository.remove(indexedMedia);
-
-
-      // save files
-      const indexedMetaFiles = await fileRepository.createQueryBuilder('file')
-        .where('file.directory = :dir', {
-          dir: currentDir.id
-        }).getMany();
-
-
-      const metaFilesToSave = [];
-      for (let i = 0; i < scannedDirectory.metaFile.length; i++) {
-        let metaFile: FileDTO = null;
-        for (let j = 0; j < indexedMetaFiles.length; j++) {
-          if (indexedMetaFiles[j].name === scannedDirectory.metaFile[i].name) {
-            metaFile = indexedMetaFiles[j];
-            indexedMetaFiles.splice(j, 1);
-            break;
-          }
-        }
-        if (metaFile == null) { // not in DB yet
-          scannedDirectory.metaFile[i].directory = null;
-          metaFile = Utils.clone(scannedDirectory.metaFile[i]);
-          scannedDirectory.metaFile[i].directory = scannedDirectory;
-          metaFile.directory = currentDir;
-          metaFilesToSave.push(metaFile);
-        }
-      }
-      await fileRepository.save(metaFilesToSave, {chunk: Math.max(Math.ceil(metaFilesToSave.length / 500), 1)});
-      await fileRepository.remove(indexedMetaFiles, {chunk: Math.max(Math.ceil(indexedMetaFiles.length / 500), 1)});
-    } catch (e) {
-      throw e;
-    } finally {
-      this.isSaving = false;
-    }
-  }
-
-  protected async saveMedia(connection: Connection, mediaList: MediaDTO[]): Promise<MediaEntity[]> {
-    const chunked = Utils.chunkArrays(mediaList, 100);
-    let list: MediaEntity[] = [];
-    for (let i = 0; i < chunked.length; i++) {
-      list = list.concat(await connection.getRepository(PhotoEntity).save(<PhotoEntity[]>chunked[i].filter(m => MediaDTO.isPhoto(m))));
-      list = list.concat(await connection.getRepository(VideoEntity).save(<VideoEntity[]>chunked[i].filter(m => MediaDTO.isVideo(m))));
-    }
-    return list;
-  }
-
   async countDirectories(): Promise<number> {
     const connection = await SQLConnection.getConnection();
     return await connection.getRepository(DirectoryEntity)
@@ -404,5 +170,144 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
       .getCount();
   }
 
+  public async getPossibleDuplicates() {
+    const connection = await SQLConnection.getConnection();
+    const mediaRepository = connection.getRepository(MediaEntity);
+
+    let duplicates = await mediaRepository.createQueryBuilder('media')
+      .innerJoin(query => query.from(MediaEntity, 'innerMedia')
+          .select(['innerMedia.name as name', 'innerMedia.metadata.fileSize as fileSize', 'count(*)'])
+          .groupBy('innerMedia.name, innerMedia.metadata.fileSize')
+          .having('count(*)>1'),
+        'innerMedia',
+        'media.name=innerMedia.name AND media.metadata.fileSize = innerMedia.fileSize')
+      .innerJoinAndSelect('media.directory', 'directory')
+      .orderBy('media.name, media.metadata.fileSize')
+      .limit(Config.Server.duplicates.listingLimit).getMany();
+
+
+    const duplicateParis: DuplicatesDTO[] = [];
+    const processDuplicates = (duplicateList: MediaEntity[],
+                               equalFn: (a: MediaEntity, b: MediaEntity) => boolean,
+                               checkDuplicates: boolean = false) => {
+      let i = duplicateList.length - 1;
+      while (i >= 0) {
+        const list = [duplicateList[i]];
+        let j = i - 1;
+        while (j >= 0 && equalFn(duplicateList[i], duplicateList[j])) {
+          list.push(duplicateList[j]);
+          j--;
+        }
+        i = j;
+        // if we cut the select list with the SQL LIMIT, filter unpaired media
+        if (list.length < 2) {
+          continue;
+        }
+        if (checkDuplicates) {
+          // ad to group if one already existed
+          const foundDuplicates = duplicateParis.find(dp =>
+            !!dp.media.find(m =>
+              !!list.find(lm => lm.id === m.id)));
+          if (foundDuplicates) {
+            list.forEach(lm => {
+              if (!!foundDuplicates.media.find(m => m.id === lm.id)) {
+                return;
+              }
+              foundDuplicates.media.push(lm);
+            });
+            continue;
+          }
+        }
+
+        duplicateParis.push({media: list});
+      }
+    };
+
+    processDuplicates(duplicates,
+      (a, b) => a.name === b.name &&
+        a.metadata.fileSize === b.metadata.fileSize);
+
+
+    duplicates = await mediaRepository.createQueryBuilder('media')
+      .innerJoin(query => query.from(MediaEntity, 'innerMedia')
+          .select(['innerMedia.metadata.creationDate as creationDate', 'innerMedia.metadata.fileSize as fileSize', 'count(*)'])
+          .groupBy('innerMedia.metadata.creationDate, innerMedia.metadata.fileSize')
+          .having('count(*)>1'),
+        'innerMedia',
+        'media.metadata.creationDate=innerMedia.creationDate AND media.metadata.fileSize = innerMedia.fileSize')
+      .innerJoinAndSelect('media.directory', 'directory')
+      .orderBy('media.metadata.creationDate, media.metadata.fileSize')
+      .limit(Config.Server.duplicates.listingLimit).getMany();
+
+    processDuplicates(duplicates,
+      (a, b) => a.metadata.creationDate === b.metadata.creationDate &&
+        a.metadata.fileSize === b.metadata.fileSize, true);
+
+    return duplicateParis;
+
+  }
+
+  protected async selectParentDir(connection: Connection, directoryName: string, directoryParent: string): Promise<DirectoryEntity> {
+    const query = connection
+      .getRepository(DirectoryEntity)
+      .createQueryBuilder('directory')
+      .where('directory.name = :name AND directory.path = :path', {
+        name: directoryName,
+        path: directoryParent
+      })
+      .leftJoinAndSelect('directory.directories', 'directories')
+      .leftJoinAndSelect('directory.media', 'media');
+
+    if (Config.Client.MetaFile.enabled === true) {
+      query.leftJoinAndSelect('directory.metaFile', 'metaFile');
+    }
+
+    return await query.getOne();
+  }
+
+  protected async fillParentDir(connection: Connection, dir: DirectoryEntity): Promise<void> {
+    if (dir.media) {
+      const indexedFaces = await connection.getRepository(FaceRegionEntry)
+        .createQueryBuilder('face')
+        .leftJoinAndSelect('face.media', 'media')
+        .where('media.directory = :directory', {
+          directory: dir.id
+        })
+        .leftJoinAndSelect('face.person', 'person')
+        .select(['face.id', 'face.box.x',
+          'face.box.y', 'face.box.width', 'face.box.height',
+          'media.id', 'person.name', 'person.id'])
+        .getMany();
+      for (let i = 0; i < dir.media.length; i++) {
+        dir.media[i].directory = dir;
+        dir.media[i].readyThumbnails = [];
+        dir.media[i].readyIcon = false;
+        (<PhotoDTO>dir.media[i]).metadata.faces = indexedFaces
+          .filter(fe => fe.media.id === dir.media[i].id)
+          .map(f => ({box: f.box, name: f.person.name}));
+      }
+
+    }
+    if (dir.directories) {
+      for (let i = 0; i < dir.directories.length; i++) {
+        dir.directories[i].media = await connection
+          .getRepository(MediaEntity)
+          .createQueryBuilder('media')
+          .where('media.directory = :dir', {
+            dir: dir.directories[i].id
+          })
+          .orderBy('media.metadata.creationDate', 'ASC')
+          .limit(Config.Server.indexing.folderPreviewSize)
+          .getMany();
+        dir.directories[i].isPartial = true;
+
+        for (let j = 0; j < dir.directories[i].media.length; j++) {
+          dir.directories[i].media[j].directory = dir.directories[i];
+          dir.directories[i].media[j].readyThumbnails = [];
+          dir.directories[i].media[j].readyIcon = false;
+        }
+      }
+    }
+  }
 
 }

backend/model/sql/IGalleryManager.ts

@@ -1,13 +1,12 @@
 import {DirectoryDTO} from '../../../common/entities/DirectoryDTO';
 import {IGalleryManager} from '../interfaces/IGalleryManager';
+import {DuplicatesDTO} from '../../../common/entities/DuplicatesDTO';
 
 export interface ISQLGalleryManager extends IGalleryManager {
   listDirectory(relativeDirectoryName: string,
                 knownLastModified?: number,
                 knownLastScanned?: number): Promise<DirectoryDTO>;
 
-  indexDirectory(relativeDirectoryName: string): Promise<DirectoryDTO>;
-
   countDirectories(): Promise<number>;
 
   countPhotos(): Promise<number>;
@@ -15,4 +14,6 @@ export interface ISQLGalleryManager extends IGalleryManager {
   countVideos(): Promise<number>;
 
   countMediaSize(): Promise<number>;
+
+  getPossibleDuplicates(): Promise<DuplicatesDTO[]>;
 }

backend/model/sql/IndexingManager.ts

@@ -1,119 +1,325 @@
-import {IIndexingManager} from '../interfaces/IIndexingManager';
-import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO';
-import {ObjectManagerRepository} from '../ObjectManagerRepository';
-import {ISQLGalleryManager} from './IGalleryManager';
-import * as path from 'path';
-import * as fs from 'fs';
-import {SQLConnection} from './SQLConnection';
+import {DirectoryDTO} from '../../../common/entities/DirectoryDTO';
 import {DirectoryEntity} from './enitites/DirectoryEntity';
-import {Logger} from '../../Logger';
-import {RendererInput, ThumbnailSourceType, ThumbnailWorker} from '../threading/ThumbnailWorker';
-import {Config} from '../../../common/config/private/Config';
+import {SQLConnection} from './SQLConnection';
+import {DiskManager} from '../DiskManger';
+import {PhotoEntity} from './enitites/PhotoEntity';
+import {Utils} from '../../../common/Utils';
+import {FaceRegion, PhotoMetadata} from '../../../common/entities/PhotoDTO';
+import {Connection,  Repository} from 'typeorm';
+import {MediaEntity} from './enitites/MediaEntity';
 import {MediaDTO} from '../../../common/entities/MediaDTO';
-import {ProjectPath} from '../../ProjectPath';
-import {ThumbnailGeneratorMWs} from '../../middlewares/thumbnail/ThumbnailGeneratorMWs';
+import {VideoEntity} from './enitites/VideoEntity';
+import {FileEntity} from './enitites/FileEntity';
+import {FileDTO} from '../../../common/entities/FileDTO';
+import {NotificationManager} from '../NotifocationManager';
+import {FaceRegionEntry} from './enitites/FaceRegionEntry';
+import {ObjectManagerRepository} from '../ObjectManagerRepository';
 
 const LOG_TAG = '[IndexingManager]';
 
-export class IndexingManager implements IIndexingManager {
-  directoriesToIndex: string[] = [];
-  indexingProgress: IndexingProgressDTO = null;
-  enabled = false;
-  private indexNewDirectory = async (createThumbnails: boolean = false) => {
-    if (this.directoriesToIndex.length === 0) {
-      this.indexingProgress = null;
-      if (global.gc) {
-        global.gc();
-      }
-      return;
-    }
-    const directory = this.directoriesToIndex.shift();
-    this.indexingProgress.current = directory;
-    this.indexingProgress.left = this.directoriesToIndex.length;
-    const scanned = await (<ISQLGalleryManager>ObjectManagerRepository.getInstance().GalleryManager).indexDirectory(directory);
-    if (this.enabled === false) {
-      return;
-    }
-    this.indexingProgress.indexed++;
-    this.indexingProgress.time.current = Date.now();
-    for (let i = 0; i < scanned.directories.length; i++) {
-      this.directoriesToIndex.push(path.join(scanned.directories[i].path, scanned.directories[i].name));
-    }
-    if (createThumbnails) {
-      for (let i = 0; i < scanned.media.length; i++) {
+export class IndexingManager {
+
+  private savingQueue: DirectoryDTO[] = [];
+  private isSaving = false;
+
+  public indexDirectory(relativeDirectoryName: string): Promise<DirectoryDTO> {
+    return new Promise(async (resolve, reject) => {
       try {
-          const media = scanned.media[i];
-          const mPath = path.join(ProjectPath.ImageFolder, media.directory.path, media.directory.name, media.name);
-          const thPath = path.join(ProjectPath.ThumbnailFolder,
-            ThumbnailGeneratorMWs.generateThumbnailName(mPath, Config.Client.Thumbnail.thumbnailSizes[0]));
-          if (fs.existsSync(thPath)) { // skip existing thumbnails
-            continue;
-          }
-          await ThumbnailWorker.render(<RendererInput>{
-            type: MediaDTO.isVideo(media) ? ThumbnailSourceType.Video : ThumbnailSourceType.Image,
-            mediaPath: mPath,
-            size: Config.Client.Thumbnail.thumbnailSizes[0],
-            thPath: thPath,
-            makeSquare: false,
-            qualityPriority: Config.Server.thumbnail.qualityPriority
-          }, Config.Server.thumbnail.processingLibrary);
-        } catch (e) {
-          console.error(e);
-          Logger.error(LOG_TAG, 'Error during indexing job: ' + e.toString());
-        }
+        const scannedDirectory = await DiskManager.scanDirectory(relativeDirectoryName);
+
+        // returning with the result
+        scannedDirectory.media.forEach(p => p.readyThumbnails = []);
+        resolve(scannedDirectory);
+
+        this.queueForSave(scannedDirectory).catch(console.error);
+
+      } catch (error) {
+        NotificationManager.warning('Unknown indexing error for: ' + relativeDirectoryName, error.toString());
+        console.error(error);
+        return reject(error);
       }
 
-    }
-    process.nextTick(() => {
-      this.indexNewDirectory(createThumbnails);
     });
-  };
-
-  startIndexing(createThumbnails: boolean = false): void {
-    if (this.directoriesToIndex.length === 0 && this.enabled === false) {
-      Logger.info(LOG_TAG, 'Starting indexing');
-      this.indexingProgress = {
-        indexed: 0,
-        left: 0,
-        current: '',
-        time: {
-          start: Date.now(),
-          current: Date.now()
   }
-      };
-      this.directoriesToIndex.push('/');
-      this.enabled = true;
-      this.indexNewDirectory(createThumbnails);
+
+  // Todo fix it, once typeorm support connection pools ofr sqlite
+  protected async queueForSave(scannedDirectory: DirectoryDTO) {
+    if (this.savingQueue.findIndex(dir => dir.name === scannedDirectory.name &&
+      dir.path === scannedDirectory.path &&
+      dir.lastModified === scannedDirectory.lastModified &&
+      dir.lastScanned === scannedDirectory.lastScanned &&
+      (dir.media || dir.media.length) === (scannedDirectory.media || scannedDirectory.media.length) &&
+      (dir.metaFile || dir.metaFile.length) === (scannedDirectory.metaFile || scannedDirectory.metaFile.length)) !== -1) {
+      return;
+    }
+    this.savingQueue.push(scannedDirectory);
+    while (this.isSaving === false && this.savingQueue.length > 0) {
+      await this.saveToDB(this.savingQueue[0]);
+      this.savingQueue.shift();
+    }
+
+  }
+
+  protected async saveParentDir(connection: Connection, scannedDirectory: DirectoryDTO): Promise<number> {
+    const directoryRepository = connection.getRepository(DirectoryEntity);
+
+    const currentDir: DirectoryEntity = await directoryRepository.createQueryBuilder('directory')
+      .where('directory.name = :name AND directory.path = :path', {
+        name: scannedDirectory.name,
+        path: scannedDirectory.path
+      }).getOne();
+    if (!!currentDir) {// Updated parent dir (if it was in the DB previously)
+      currentDir.lastModified = scannedDirectory.lastModified;
+      currentDir.lastScanned = scannedDirectory.lastScanned;
+      currentDir.mediaCount = scannedDirectory.mediaCount;
+      await directoryRepository.save(currentDir);
+      return currentDir.id;
+
     } else {
-      Logger.info(LOG_TAG, 'Already indexing..');
+      return (await directoryRepository.insert(<DirectoryEntity>{
+        mediaCount: scannedDirectory.mediaCount,
+        lastModified: scannedDirectory.lastModified,
+        lastScanned: scannedDirectory.lastScanned,
+        name: scannedDirectory.name,
+        path: scannedDirectory.path
+      })).identifiers[0].id;
     }
   }
 
-  getProgress(): IndexingProgressDTO {
-    return this.indexingProgress;
-  }
+  protected async saveChildDirs(connection: Connection, currentDirId: number, scannedDirectory: DirectoryDTO) {
+    const directoryRepository = connection.getRepository(DirectoryEntity);
+    // TODO: fix when first opened directory is not root
+    // save subdirectories
+    const childDirectories = await directoryRepository.createQueryBuilder('directory')
+      .where('directory.parent = :dir', {
+        dir: currentDirId
+      }).getMany();
 
-  cancelIndexing(): void {
-    Logger.info(LOG_TAG, 'Canceling indexing');
-    this.directoriesToIndex = [];
-    this.indexingProgress = null;
-    this.enabled = false;
-    if (global.gc) {
-      global.gc();
+    for (let i = 0; i < scannedDirectory.directories.length; i++) {
+      // Was this child Dir already indexed before?
+      let directory: DirectoryEntity = null;
+      for (let j = 0; j < childDirectories.length; j++) {
+        if (childDirectories[j].name === scannedDirectory.directories[i].name) {
+          directory = childDirectories[j];
+          childDirectories.splice(j, 1);
+          break;
         }
       }
 
-  async reset(): Promise<void> {
-    Logger.info(LOG_TAG, 'Resetting DB');
-    this.directoriesToIndex = [];
-    this.indexingProgress = null;
-    this.enabled = false;
-    const connection = await SQLConnection.getConnection();
-    return connection
-      .getRepository(DirectoryEntity)
-      .createQueryBuilder('directory')
-      .delete()
-      .execute().then(() => {
+      if (directory != null) { // update existing directory
+        if (!directory.parent || !directory.parent.id) { // set parent if not set yet
+          directory.parent = <any>{id: currentDirId};
+          delete directory.media;
+          await directoryRepository.save(directory);
+        }
+      } else { // dir does not exists yet
+        scannedDirectory.directories[i].parent = <any>{id: currentDirId};
+        (<DirectoryEntity>scannedDirectory.directories[i]).lastScanned = null; // new child dir, not fully scanned yet
+        const d = await directoryRepository.insert(<DirectoryEntity>scannedDirectory.directories[i]);
+
+        await this.saveMedia(connection, d.identifiers[0].id, scannedDirectory.directories[i].media);
+      }
+    }
+
+    // Remove child Dirs that are not anymore in the parent dir
+    await directoryRepository.remove(childDirectories, {chunk: Math.max(Math.ceil(childDirectories.length / 500), 1)});
+
+  }
+
+  protected async saveMetaFiles(connection: Connection, currentDirID: number, scannedDirectory: DirectoryDTO) {
+    const fileRepository = connection.getRepository(FileEntity);
+    // save files
+    const indexedMetaFiles = await fileRepository.createQueryBuilder('file')
+      .where('file.directory = :dir', {
+        dir: currentDirID
+      }).getMany();
+
+
+    const metaFilesToSave = [];
+    for (let i = 0; i < scannedDirectory.metaFile.length; i++) {
+      let metaFile: FileDTO = null;
+      for (let j = 0; j < indexedMetaFiles.length; j++) {
+        if (indexedMetaFiles[j].name === scannedDirectory.metaFile[i].name) {
+          metaFile = indexedMetaFiles[j];
+          indexedMetaFiles.splice(j, 1);
+          break;
+        }
+      }
+      if (metaFile == null) { // not in DB yet
+        scannedDirectory.metaFile[i].directory = null;
+        metaFile = Utils.clone(scannedDirectory.metaFile[i]);
+        scannedDirectory.metaFile[i].directory = scannedDirectory;
+        metaFile.directory = <any>{id: currentDirID};
+        metaFilesToSave.push(metaFile);
+      }
+    }
+    await fileRepository.save(metaFilesToSave, {chunk: Math.max(Math.ceil(metaFilesToSave.length / 500), 1)});
+    await fileRepository.remove(indexedMetaFiles, {chunk: Math.max(Math.ceil(indexedMetaFiles.length / 500), 1)});
+  }
+
+  protected async saveMedia(connection: Connection, parentDirId: number, media: MediaDTO[]) {
+    const mediaRepository = connection.getRepository(MediaEntity);
+    const photoRepository = connection.getRepository(PhotoEntity);
+    const videoRepository = connection.getRepository(VideoEntity);
+    // save media
+    let indexedMedia = (await mediaRepository.createQueryBuilder('media')
+      .where('media.directory = :dir', {
+        dir: parentDirId
+      })
+      .getMany());
+
+    const mediaChange: any = {
+      saveP: [],
+      saveV: [],
+      insertP: [],
+      insertV: []
+    };
+    const facesPerPhoto: { faces: FaceRegionEntry[], mediaName: string }[] = [];
+    for (let i = 0; i < media.length; i++) {
+      let mediaItem: MediaEntity = null;
+      for (let j = 0; j < indexedMedia.length; j++) {
+        if (indexedMedia[j].name === media[i].name) {
+          mediaItem = indexedMedia[j];
+          indexedMedia.splice(j, 1);
+          break;
+        }
+      }
+
+      const scannedFaces = (<PhotoMetadata>media[i].metadata).faces || [];
+      delete (<PhotoMetadata>media[i].metadata).faces;
+
+      // let mediaItemId: number = null;
+      if (mediaItem == null) { // not in DB yet
+        media[i].directory = null;
+        mediaItem = <any>Utils.clone(media[i]);
+        mediaItem.directory = <any>{id: parentDirId};
+        (MediaDTO.isPhoto(mediaItem) ? mediaChange.insertP : mediaChange.insertV).push(mediaItem);
+      } else {
+        delete (<PhotoMetadata>mediaItem.metadata).faces;
+        if (!Utils.equalsFilter(mediaItem.metadata, media[i].metadata)) {
+          mediaItem.metadata = <any>media[i].metadata;
+          (MediaDTO.isPhoto(mediaItem) ? mediaChange.saveP : mediaChange.saveV).push(mediaItem);
+
+        }
+      }
+
+      facesPerPhoto.push({faces: scannedFaces as FaceRegionEntry[], mediaName: mediaItem.name});
+    }
+
+    await this.saveChunk(photoRepository, mediaChange.saveP, 100);
+    await this.saveChunk(videoRepository, mediaChange.saveV, 100);
+    await this.saveChunk(photoRepository, mediaChange.insertP, 100);
+    await this.saveChunk(videoRepository, mediaChange.insertV, 100);
+
+    indexedMedia = (await mediaRepository.createQueryBuilder('media')
+      .where('media.directory = :dir', {
+        dir: parentDirId
+      })
+      .select(['media.name', 'media.id'])
+      .getMany());
+
+    const faces: FaceRegionEntry[] = [];
+    facesPerPhoto.forEach(group => {
+      const mIndex = indexedMedia.findIndex(m => m.name === group.mediaName);
+      group.faces.forEach((sf: FaceRegionEntry) => sf.media = <any>{id: indexedMedia[mIndex].id});
+
+      faces.push(...group.faces);
+      indexedMedia.splice(mIndex, 1);
     });
+
+    await this.saveFaces(connection, parentDirId, faces);
+    await mediaRepository.remove(indexedMedia);
+  }
+
+  protected async saveFaces(connection: Connection, parentDirId: number, scannedFaces: FaceRegion[]) {
+    const faceRepository = connection.getRepository(FaceRegionEntry);
+
+    const persons: string[] = [];
+
+    for (let i = 0; i < scannedFaces.length; i++) {
+      if (persons.indexOf(scannedFaces[i].name) === -1) {
+        persons.push(scannedFaces[i].name);
+      }
+    }
+    await ObjectManagerRepository.getInstance().PersonManager.saveAll(persons);
+
+
+    const indexedFaces = await faceRepository.createQueryBuilder('face')
+      .leftJoin('face.media', 'media')
+      .where('media.directory = :directory', {
+        directory: parentDirId
+      })
+      .leftJoinAndSelect('face.person', 'person')
+      .getMany();
+
+
+    const faceToInsert = [];
+    for (let i = 0; i < scannedFaces.length; i++) {
+      let face: FaceRegionEntry = null;
+      for (let j = 0; j < indexedFaces.length; j++) {
+        if (indexedFaces[j].box.height === scannedFaces[i].box.height &&
+          indexedFaces[j].box.width === scannedFaces[i].box.width &&
+          indexedFaces[j].box.x === scannedFaces[i].box.x &&
+          indexedFaces[j].box.y === scannedFaces[i].box.y &&
+          indexedFaces[j].person.name === scannedFaces[i].name) {
+          face = indexedFaces[j];
+          indexedFaces.splice(j, 1);
+          break;
+        }
+      }
+
+      if (face == null) {
+        (<FaceRegionEntry>scannedFaces[i]).person = await ObjectManagerRepository.getInstance().PersonManager.get(scannedFaces[i].name);
+        faceToInsert.push(scannedFaces[i]);
+      }
+    }
+    if (faceToInsert.length > 0) {
+      await this.insertChunk(faceRepository, faceToInsert, 100);
+    }
+    await faceRepository.remove(indexedFaces, {chunk: Math.max(Math.ceil(indexedFaces.length / 500), 1)});
+
+  }
+
+  protected async saveToDB(scannedDirectory: DirectoryDTO): Promise<void> {
+    this.isSaving = true;
+    try {
+      const connection = await SQLConnection.getConnection();
+      const currentDirId: number = await this.saveParentDir(connection, scannedDirectory);
+      await this.saveChildDirs(connection, currentDirId, scannedDirectory);
+      await this.saveMedia(connection, currentDirId, scannedDirectory.media);
+      await this.saveMetaFiles(connection, currentDirId, scannedDirectory);
+    } catch (e) {
+      throw e;
+    } finally {
+      this.isSaving = false;
+    }
+  }
+
+  private async saveChunk<T>(repository: Repository<any>, entities: T[], size: number): Promise<T[]> {
+    if (entities.length === 0) {
+      return [];
+    }
+    if (entities.length < size) {
+      return await repository.save(entities);
+    }
+    let list: T[] = [];
+    for (let i = 0; i < entities.length / size; i++) {
+      list = list.concat(await repository.save(entities.slice(i * size, (i + 1) * size)));
+    }
+    return list;
+  }
+
+  private async insertChunk<T>(repository: Repository<any>, entities: T[], size: number): Promise<number[]> {
+    if (entities.length === 0) {
+      return [];
+    }
+    if (entities.length < size) {
+      return (await repository.insert(entities)).identifiers.map((i: any) => i.id);
+    }
+    let list: number[] = [];
+    for (let i = 0; i < entities.length / size; i++) {
+      list = list.concat((await repository.insert(entities.slice(i * size, (i + 1) * size))).identifiers.map(ids => ids.id));
+    }
+    return list;
   }
 }

backend/model/sql/IndexingTaskManager.ts

@@ -0,0 +1,118 @@
+import {IIndexingTaskManager} from '../interfaces/IIndexingTaskManager';
+import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO';
+import {ObjectManagerRepository} from '../ObjectManagerRepository';
+import * as path from 'path';
+import * as fs from 'fs';
+import {SQLConnection} from './SQLConnection';
+import {DirectoryEntity} from './enitites/DirectoryEntity';
+import {Logger} from '../../Logger';
+import {RendererInput, ThumbnailSourceType, ThumbnailWorker} from '../threading/ThumbnailWorker';
+import {Config} from '../../../common/config/private/Config';
+import {MediaDTO} from '../../../common/entities/MediaDTO';
+import {ProjectPath} from '../../ProjectPath';
+import {ThumbnailGeneratorMWs} from '../../middlewares/thumbnail/ThumbnailGeneratorMWs';
+
+const LOG_TAG = '[IndexingTaskManager]';
+
+export class IndexingTaskManager implements IIndexingTaskManager {
+  directoriesToIndex: string[] = [];
+  indexingProgress: IndexingProgressDTO = null;
+  enabled = false;
+  private indexNewDirectory = async (createThumbnails: boolean = false) => {
+    if (this.directoriesToIndex.length === 0) {
+      this.indexingProgress = null;
+      if (global.gc) {
+        global.gc();
+      }
+      return;
+    }
+    const directory = this.directoriesToIndex.shift();
+    this.indexingProgress.current = directory;
+    this.indexingProgress.left = this.directoriesToIndex.length;
+    const scanned = await ObjectManagerRepository.getInstance().IndexingManager.indexDirectory(directory);
+    if (this.enabled === false) {
+      return;
+    }
+    this.indexingProgress.indexed++;
+    this.indexingProgress.time.current = Date.now();
+    for (let i = 0; i < scanned.directories.length; i++) {
+      this.directoriesToIndex.push(path.join(scanned.directories[i].path, scanned.directories[i].name));
+    }
+    if (createThumbnails) {
+      for (let i = 0; i < scanned.media.length; i++) {
+        try {
+          const media = scanned.media[i];
+          const mPath = path.join(ProjectPath.ImageFolder, media.directory.path, media.directory.name, media.name);
+          const thPath = path.join(ProjectPath.ThumbnailFolder,
+            ThumbnailGeneratorMWs.generateThumbnailName(mPath, Config.Client.Thumbnail.thumbnailSizes[0]));
+          if (fs.existsSync(thPath)) { // skip existing thumbnails
+            continue;
+          }
+          await ThumbnailWorker.render(<RendererInput>{
+            type: MediaDTO.isVideo(media) ? ThumbnailSourceType.Video : ThumbnailSourceType.Image,
+            mediaPath: mPath,
+            size: Config.Client.Thumbnail.thumbnailSizes[0],
+            thPath: thPath,
+            makeSquare: false,
+            qualityPriority: Config.Server.thumbnail.qualityPriority
+          }, Config.Server.thumbnail.processingLibrary);
+        } catch (e) {
+          console.error(e);
+          Logger.error(LOG_TAG, 'Error during indexing job: ' + e.toString());
+        }
+      }
+
+    }
+    process.nextTick(() => {
+      this.indexNewDirectory(createThumbnails).catch(console.error);
+    });
+  };
+
+  startIndexing(createThumbnails: boolean = false): void {
+    if (this.directoriesToIndex.length === 0 && this.enabled === false) {
+      Logger.info(LOG_TAG, 'Starting indexing');
+      this.indexingProgress = {
+        indexed: 0,
+        left: 0,
+        current: '',
+        time: {
+          start: Date.now(),
+          current: Date.now()
+        }
+      };
+      this.directoriesToIndex.push('/');
+      this.enabled = true;
+      this.indexNewDirectory(createThumbnails).catch(console.error);
+    } else {
+      Logger.info(LOG_TAG, 'Already indexing..');
+    }
+  }
+
+  getProgress(): IndexingProgressDTO {
+    return this.indexingProgress;
+  }
+
+  cancelIndexing(): void {
+    Logger.info(LOG_TAG, 'Canceling indexing');
+    this.directoriesToIndex = [];
+    this.indexingProgress = null;
+    this.enabled = false;
+    if (global.gc) {
+      global.gc();
+    }
+  }
+
+  async reset(): Promise<void> {
+    Logger.info(LOG_TAG, 'Resetting DB');
+    this.directoriesToIndex = [];
+    this.indexingProgress = null;
+    this.enabled = false;
+    const connection = await SQLConnection.getConnection();
+    return connection
+      .getRepository(DirectoryEntity)
+      .createQueryBuilder('directory')
+      .delete()
+      .execute().then(() => {
+      });
+  }
+}

backend/model/sql/PersonManager.ts

@@ -0,0 +1,50 @@
+import {IPersonManager} from '../interfaces/IPersonManager';
+import {PersonEntry} from './enitites/PersonEntry';
+import {SQLConnection} from './SQLConnection';
+
+const LOG_TAG = '[PersonManager]';
+
+export class PersonManager implements IPersonManager {
+
+  persons: PersonEntry[] = [];
+
+  async get(name: string): Promise<PersonEntry> {
+
+    let person = this.persons.find(p => p.name === name);
+    if (!person) {
+      const connection = await SQLConnection.getConnection();
+      const personRepository = connection.getRepository(PersonEntry);
+      person = await personRepository.findOne({name: name});
+      if (!person) {
+        person = await personRepository.save(<PersonEntry>{name: name});
+      }
+      this.persons.push(person);
+    }
+    return person;
+  }
+
+
+  async saveAll(names: string[]): Promise<void> {
+    const toSave: { name: string }[] = [];
+    const connection = await SQLConnection.getConnection();
+    const personRepository = connection.getRepository(PersonEntry);
+    this.persons = await personRepository.find();
+
+    for (let i = 0; i < names.length; i++) {
+
+      const person = this.persons.find(p => p.name === names[i]);
+      if (!person) {
+        toSave.push({name: names[i]});
+      }
+    }
+
+    if (toSave.length > 0) {
+      for (let i = 0; i < toSave.length / 200; i++) {
+        await personRepository.insert(toSave.slice(i * 200, (i + 1) * 200));
+      }
+       this.persons = await personRepository.find();
+    }
+
+  }
+
+}

backend/model/sql/SQLConnection.ts

@@ -15,16 +15,19 @@ import {MediaEntity} from './enitites/MediaEntity';
 import {VideoEntity} from './enitites/VideoEntity';
 import {DataStructureVersion} from '../../../common/DataStructureVersion';
 import {FileEntity} from './enitites/FileEntity';
+import {FaceRegionEntry} from './enitites/FaceRegionEntry';
+import {PersonEntry} from './enitites/PersonEntry';
+import {Utils} from '../../../common/Utils';
 
 
 export class SQLConnection {
 
 
+  private static connection: Connection = null;
+
   constructor() {
   }
 
-  private static connection: Connection = null;
-
   public static async getConnection(): Promise<Connection> {
     if (this.connection == null) {
       const options: any = this.getDriver(Config.Server.database);
@@ -32,6 +35,8 @@ export class SQLConnection {
       options.entities = [
         UserEntity,
         FileEntity,
+        FaceRegionEntry,
+        PersonEntry,
         MediaEntity,
         PhotoEntity,
         VideoEntity,
@@ -41,7 +46,9 @@ export class SQLConnection {
       ];
       options.synchronize = false;
       //  options.logging = 'all';
-      this.connection = await createConnection(options);
+
+
+      this.connection = await this.createConnection(options);
       await SQLConnection.schemeSync(this.connection);
     }
     return this.connection;
@@ -57,6 +64,8 @@ export class SQLConnection {
     options.entities = [
       UserEntity,
       FileEntity,
+      FaceRegionEntry,
+      PersonEntry,
       MediaEntity,
       PhotoEntity,
       VideoEntity,
@@ -66,7 +75,7 @@ export class SQLConnection {
     ];
     options.synchronize = false;
     // options.logging = "all";
-    const conn = await createConnection(options);
+    const conn = await this.createConnection(options);
     await SQLConnection.schemeSync(conn);
     await conn.close();
     return true;
@@ -86,6 +95,38 @@ export class SQLConnection {
 
   }
 
+  public static async close() {
+    try {
+      if (this.connection != null) {
+        await this.connection.close();
+        this.connection = null;
+      }
+    } catch (err) {
+      console.error(err);
+    }
+  }
+
+  private static async createConnection(options: ConnectionOptions) {
+    if (options.type === 'sqlite') {
+      return await createConnection(options);
+    }
+    try {
+      return await createConnection(options);
+    } catch (e) {
+      if (e.sqlMessage === 'Unknown database \'' + options.database + '\'') {
+        Logger.debug('creating database: ' + options.database);
+        const tmpOption = Utils.clone(options);
+        // @ts-ignore
+        delete tmpOption.database;
+        const tmpConn = await createConnection(tmpOption);
+        await tmpConn.query('CREATE DATABASE IF NOT EXISTS ' + options.database);
+        await tmpConn.close();
+        return await createConnection(options);
+      }
+      throw e;
+    }
+  }
+
   private static async schemeSync(connection: Connection) {
     let version = null;
     try {
@@ -139,16 +180,5 @@ export class SQLConnection {
     return driver;
   }
 
-  public static async close() {
-    try {
-      if (this.connection != null) {
-        await this.connection.close();
-        this.connection = null;
-      }
-    } catch (err) {
-      console.error(err);
-    }
-  }
-
 
 }

backend/model/sql/SearchManager.ts

@@ -6,6 +6,10 @@ import {PhotoEntity} from './enitites/PhotoEntity';
 import {DirectoryEntity} from './enitites/DirectoryEntity';
 import {MediaEntity} from './enitites/MediaEntity';
 import {VideoEntity} from './enitites/VideoEntity';
+import {PersonEntry} from './enitites/PersonEntry';
+import {FaceRegionEntry} from './enitites/FaceRegionEntry';
+import {SelectQueryBuilder} from 'typeorm';
+import {Config} from '../../../common/config/private/Config';
 
 export class SearchManager implements ISearchManager {
 
@@ -22,14 +26,14 @@ export class SearchManager implements ISearchManager {
     return a;
   }
 
-  async autocomplete(text: string): Promise<Array<AutoCompleteItem>> {
+  async autocomplete(text: string): Promise<AutoCompleteItem[]> {
 
     const connection = await SQLConnection.getConnection();
 
     let result: AutoCompleteItem[] = [];
     const photoRepository = connection.getRepository(PhotoEntity);
     const videoRepository = connection.getRepository(VideoEntity);
-    const mediaRepository = connection.getRepository(MediaEntity);
+    const personRepository = connection.getRepository(PersonEntry);
     const directoryRepository = connection.getRepository(DirectoryEntity);
 
 
@@ -37,7 +41,7 @@ export class SearchManager implements ISearchManager {
       .createQueryBuilder('photo')
       .select('DISTINCT(photo.metadata.keywords)')
       .where('photo.metadata.keywords LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
-      .limit(5)
+      .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
       .getRawMany())
       .map(r => <Array<string>>(<string>r.metadataKeywords).split(','))
       .forEach(keywords => {
@@ -45,6 +49,15 @@ export class SearchManager implements ISearchManager {
           .filter(k => k.toLowerCase().indexOf(text.toLowerCase()) !== -1), SearchTypes.keyword));
       });
 
+    result = result.concat(this.encapsulateAutoComplete((await personRepository
+      .createQueryBuilder('person')
+      .select('DISTINCT(person.name)')
+      .where('person.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
+      .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
+      .orderBy('person.name')
+      .getRawMany())
+      .map(r => r.name), SearchTypes.person));
+
     (await photoRepository
       .createQueryBuilder('photo')
       .select('photo.metadata.positionData.country as country, ' +
@@ -53,7 +66,7 @@ export class SearchManager implements ISearchManager {
       .orWhere('photo.metadata.positionData.state LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
       .orWhere('photo.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
       .groupBy('photo.metadata.positionData.country, photo.metadata.positionData.state, photo.metadata.positionData.city')
-      .limit(5)
+      .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
       .getRawMany())
       .filter(pm => !!pm)
       .map(pm => <Array<string>>[pm.city || '', pm.country || '', pm.state || ''])
@@ -66,7 +79,7 @@ export class SearchManager implements ISearchManager {
       .createQueryBuilder('media')
       .select('DISTINCT(media.name)')
       .where('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
-      .limit(5)
+      .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
       .getRawMany())
       .map(r => r.name), SearchTypes.photo));
 
@@ -75,7 +88,7 @@ export class SearchManager implements ISearchManager {
       .createQueryBuilder('media')
       .select('DISTINCT(media.metadata.caption) as caption')
       .where('media.metadata.caption LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
-      .limit(5)
+      .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
       .getRawMany())
       .map(r => r.caption), SearchTypes.photo));
 
@@ -84,7 +97,7 @@ export class SearchManager implements ISearchManager {
       .createQueryBuilder('media')
       .select('DISTINCT(media.name)')
       .where('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
-      .limit(5)
+      .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
       .getRawMany())
       .map(r => r.name), SearchTypes.video));
 
@@ -92,7 +105,7 @@ export class SearchManager implements ISearchManager {
       .createQueryBuilder('dir')
       .select('DISTINCT(dir.name)')
       .where('dir.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
-      .limit(5)
+      .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
       .getRawMany())
       .map(r => r.name), SearchTypes.directory));
 
@@ -112,44 +125,60 @@ export class SearchManager implements ISearchManager {
       resultOverflow: false
     };
 
-    let repostiroy = connection.getRepository(MediaEntity);
+    let usedEntity = MediaEntity;
 
     if (searchType === SearchTypes.photo) {
-      repostiroy = connection.getRepository(PhotoEntity);
+      usedEntity = PhotoEntity;
     } else if (searchType === SearchTypes.video) {
-      repostiroy = connection.getRepository(VideoEntity);
+      usedEntity = VideoEntity;
     }
 
-    const query = repostiroy.createQueryBuilder('media')
-      .innerJoinAndSelect('media.directory', 'directory')
-      .orderBy('media.metadata.creationDate', 'ASC');
+    const query = await connection.getRepository(usedEntity).createQueryBuilder('media')
+      .innerJoin(q => {
+          const subQuery = q.from(usedEntity, 'media')
+            .select('distinct media.id')
+            .limit(2000);
 
 
           if (!searchType || searchType === SearchTypes.directory) {
-      query.orWhere('directory.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'});
+            subQuery.leftJoin('media.directory', 'directory')
+              .orWhere('directory.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'});
           }
 
           if (!searchType || searchType === SearchTypes.photo || searchType === SearchTypes.video) {
-      query.orWhere('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'});
+            subQuery.orWhere('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'});
           }
 
           if (!searchType || searchType === SearchTypes.photo) {
-      query.orWhere('media.metadata.caption LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'});
+            subQuery.orWhere('media.metadata.caption LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'});
+          }
+          if (!searchType || searchType === SearchTypes.person) {
+            subQuery
+              .leftJoin('media.metadata.faces', 'faces')
+              .leftJoin('faces.person', 'person')
+              .orWhere('person.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'});
           }
 
           if (!searchType || searchType === SearchTypes.position) {
-      query.orWhere('media.metadata.positionData.country LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
+            subQuery.orWhere('media.metadata.positionData.country LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
               .orWhere('media.metadata.positionData.state LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
               .orWhere('media.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'});
 
           }
           if (!searchType || searchType === SearchTypes.keyword) {
-      query.orWhere('media.metadata.keywords LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'});
+            subQuery.orWhere('media.metadata.keywords LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'});
           }
 
-    result.media = await query
-      .limit(2001)
-      .getMany();
+          return subQuery;
+        },
+        'innerMedia',
+        'media.id=innerMedia.id')
+      .leftJoinAndSelect('media.directory', 'directory')
+      .leftJoinAndSelect('media.metadata.faces', 'faces')
+      .leftJoinAndSelect('faces.person', 'person');
+
+
+    result.media = await this.loadMediaWithFaces(query);
 
     if (result.media.length > 2000) {
       result.resultOverflow = true;
@@ -181,19 +210,29 @@ export class SearchManager implements ISearchManager {
       resultOverflow: false
     };
 
-    result.media = await connection
-      .getRepository(MediaEntity)
-      .createQueryBuilder('media')
-      .orderBy('media.metadata.creationDate', 'ASC')
+    const query = await connection.getRepository(MediaEntity).createQueryBuilder('media')
+      .innerJoin(q => q.from(MediaEntity, 'media')
+          .select('distinct media.id')
+          .limit(10)
+          .leftJoin('media.directory', 'directory')
+          .leftJoin('media.metadata.faces', 'faces')
+          .leftJoin('faces.person', 'person')
           .where('media.metadata.keywords LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
           .orWhere('media.metadata.positionData.country LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
           .orWhere('media.metadata.positionData.state LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
           .orWhere('media.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
           .orWhere('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
           .orWhere('media.metadata.caption LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
-      .innerJoinAndSelect('media.directory', 'directory')
-      .limit(10)
-      .getMany();
+          .orWhere('person.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
+        ,
+        'innerMedia',
+        'media.id=innerMedia.id')
+      .leftJoinAndSelect('media.directory', 'directory')
+      .leftJoinAndSelect('media.metadata.faces', 'faces')
+      .leftJoinAndSelect('faces.person', 'person');
+
+
+    result.media = await this.loadMediaWithFaces(query);
 
 
     result.directories = await connection
@@ -213,4 +252,29 @@ export class SearchManager implements ISearchManager {
     });
     return res;
   }
+
+  private async loadMediaWithFaces(query: SelectQueryBuilder<MediaEntity>) {
+    const rawAndEntities = await query.orderBy('media.id').getRawAndEntities();
+    const media: MediaEntity[] = rawAndEntities.entities;
+
+    //  console.log(rawAndEntities.raw);
+    let rawIndex = 0;
+    for (let i = 0; i < media.length; i++) {
+      if (rawAndEntities.raw[rawIndex].faces_id === null ||
+        rawAndEntities.raw[rawIndex].media_id !== media[i].id) {
+        delete media[i].metadata.faces;
+        continue;
+      }
+      media[i].metadata.faces = [];
+
+      while (rawAndEntities.raw[rawIndex].media_id === media[i].id) {
+        media[i].metadata.faces.push(<any>FaceRegionEntry.fromRawToDTO(rawAndEntities.raw[rawIndex]));
+        rawIndex++;
+        if (rawIndex >= rawAndEntities.raw.length) {
+          return media;
+        }
+      }
+    }
+    return media;
+  }
 }

backend/model/sql/enitites/DirectoryEntity.ts

@@ -1,4 +1,4 @@
-import {Column, Entity, ManyToOne, OneToMany, PrimaryGeneratedColumn, Unique, Index} from 'typeorm';
+import {Column, Entity, Index, ManyToOne, OneToMany, PrimaryGeneratedColumn, Unique} from 'typeorm';
 import {DirectoryDTO} from '../../../../common/entities/DirectoryDTO';
 import {MediaEntity} from './MediaEntity';
 import {FileEntity} from './FileEntity';
@@ -8,7 +8,7 @@ import {FileEntity} from './FileEntity';
 export class DirectoryEntity implements DirectoryDTO {
 
   @Index()
-  @PrimaryGeneratedColumn()
+  @PrimaryGeneratedColumn({unsigned: true})
   id: number;
 
   @Index()
@@ -22,18 +22,28 @@ export class DirectoryEntity implements DirectoryDTO {
   /**
    * last time the directory was modified (from outside, eg.: a new media was added)
    */
-  @Column('bigint')
+  @Column('bigint', {
+    unsigned: true, transformer: {
+      from: v => parseInt(v, 10),
+      to: v => v
+    }
+  })
   public lastModified: number;
 
   /**
    * Last time the directory was fully scanned, not only for a few media to create a preview
    */
-  @Column({type: 'bigint', nullable: true})
+  @Column({
+    type: 'bigint', nullable: true, unsigned: true, transformer: {
+      from: v => parseInt(v, 10),
+      to: v => v
+    }
+  })
   public lastScanned: number;
 
   isPartial?: boolean;
 
-  @Column('smallint')
+  @Column('smallint', {unsigned: true})
   mediaCount: number;
 
   @Index()

backend/model/sql/enitites/FaceRegionEntry.ts

@@ -0,0 +1,55 @@
+import {FaceRegion, FaceRegionBox} from '../../../../common/entities/PhotoDTO';
+import {Column, Entity, ManyToOne, PrimaryGeneratedColumn} from 'typeorm';
+import {PersonEntry} from './PersonEntry';
+import {MediaEntity} from './MediaEntity';
+
+export class FaceRegionBoxEntry implements FaceRegionBox {
+  @Column('int')
+  height: number;
+  @Column('int')
+  width: number;
+  @Column('int')
+  x: number;
+  @Column('int')
+  y: number;
+}
+
+/**
+ * This is a switching table between media and persons
+ */
+@Entity()
+export class FaceRegionEntry {
+
+  @PrimaryGeneratedColumn({unsigned: true})
+  id: number;
+
+  @Column(type => FaceRegionBoxEntry)
+  box: FaceRegionBoxEntry;
+
+  // @PrimaryColumn('int')
+  @ManyToOne(type => MediaEntity, media => media.metadata.faces, {onDelete: 'CASCADE', nullable: false})
+  media: MediaEntity;
+
+  // @PrimaryColumn('int')
+  @ManyToOne(type => PersonEntry, person => person.faces, {onDelete: 'CASCADE', nullable: false})
+  person: PersonEntry;
+
+  name: string;
+
+  public static fromRawToDTO(raw: {
+    faces_id: number,
+    faces_mediaId: number,
+    faces_personId: number,
+    faces_boxHeight: number,
+    faces_boxWidth: number,
+    faces_boxX: number,
+    faces_boxY: number,
+    person_id: number,
+    person_name: string
+  }): FaceRegion {
+    return {
+      box: {width: raw.faces_boxWidth, height: raw.faces_boxHeight, x: raw.faces_boxX, y: raw.faces_boxY},
+      name: raw.person_name
+    };
+  }
+}

backend/model/sql/enitites/FileEntity.ts

@@ -7,7 +7,7 @@ import {FileDTO} from '../../../../common/entities/FileDTO';
 export class FileEntity implements FileDTO {
 
   @Index()
-  @PrimaryGeneratedColumn()
+  @PrimaryGeneratedColumn({unsigned: true})
   id: number;
 
   @Column('text')

backend/model/sql/enitites/MediaEntity.ts

@@ -1,8 +1,9 @@
-import {Column, Entity, ManyToOne, PrimaryGeneratedColumn, TableInheritance, Unique, Index} from 'typeorm';
+import {Column, Entity, Index, ManyToOne, OneToMany, PrimaryGeneratedColumn, TableInheritance, Unique} from 'typeorm';
 import {DirectoryEntity} from './DirectoryEntity';
 import {MediaDimension, MediaDTO, MediaMetadata} from '../../../../common/entities/MediaDTO';
 import {OrientationTypes} from 'ts-exif-parser';
 import {CameraMetadataEntity, PositionMetaDataEntity} from './PhotoEntity';
+import {FaceRegionEntry} from './FaceRegionEntry';
 
 export class MediaDimensionEntity implements MediaDimension {
 
@@ -21,13 +22,17 @@ export class MediaMetadataEntity implements MediaMetadata {
   @Column(type => MediaDimensionEntity)
   size: MediaDimensionEntity;
 
-  @Column('bigint')
+  @Column('bigint', {
+    unsigned: true, transformer: {
+      from: v => parseInt(v, 10),
+      to: v => v
+    }
+  })
   creationDate: number;
 
-  @Column('int')
+  @Column('int', {unsigned: true})
   fileSize: number;
 
-
   @Column('simple-array')
   keywords: string[];
 
@@ -37,27 +42,30 @@ export class MediaMetadataEntity implements MediaMetadata {
   @Column(type => PositionMetaDataEntity)
   positionData: PositionMetaDataEntity;
 
-  @Column('tinyint', {default: OrientationTypes.TOP_LEFT})
+  @Column('tinyint', {unsigned: true, default: OrientationTypes.TOP_LEFT})
   orientation: OrientationTypes;
 
-  @Column('int')
+  @OneToMany(type => FaceRegionEntry, faceRegion => faceRegion.media)
+  faces: FaceRegionEntry[];
+
+  @Column('int', {unsigned: true})
   bitRate: number;
 
-  @Column('bigint')
+  @Column('int', {unsigned: true})
   duration: number;
 }
 
 // TODO: fix inheritance once its working in typeorm
 @Entity()
 @Unique(['name', 'directory'])
-@TableInheritance({column: {type: 'varchar', name: 'type'}})
+@TableInheritance({column: {type: 'varchar', name: 'type', length: 32}})
 export abstract class MediaEntity implements MediaDTO {
 
   @Index()
-  @PrimaryGeneratedColumn()
+  @PrimaryGeneratedColumn({unsigned: true})
   id: number;
 
-  @Column('text')
+  @Column()
   name: string;
 
   @Index()

backend/model/sql/enitites/PersonEntry.ts

@@ -0,0 +1,17 @@
+import {Column, Entity,  OneToMany, PrimaryGeneratedColumn,  Unique, Index} from 'typeorm';
+import {FaceRegionEntry} from './FaceRegionEntry';
+
+
+@Entity()
+@Unique(['name'])
+export class PersonEntry {
+  @Index()
+  @PrimaryGeneratedColumn({unsigned: true})
+  id: number;
+
+  @Column()
+  name: string;
+
+  @OneToMany(type => FaceRegionEntry, faceRegion => faceRegion.person)
+  public faces: FaceRegionEntry[];
+}

backend/model/sql/enitites/PhotoEntity.ts

@@ -1,6 +1,13 @@
 import {Column, Entity, ChildEntity, Unique} from 'typeorm';
-import {CameraMetadata, GPSMetadata, PhotoDTO, PhotoMetadata, PositionMetaData} from '../../../../common/entities/PhotoDTO';
-import {OrientationTypes} from 'ts-exif-parser';
+import {
+  CameraMetadata,
+  FaceRegion,
+  FaceRegionBox,
+  GPSMetadata,
+  PhotoDTO,
+  PhotoMetadata,
+  PositionMetaData
+} from '../../../../common/entities/PhotoDTO';
 import {MediaEntity, MediaMetadataEntity} from './MediaEntity';
 
 export class CameraMetadataEntity implements CameraMetadata {
@@ -38,6 +45,7 @@ export class GPSMetadataEntity implements GPSMetadata {
   altitude: number;
 }
 
+
 export class PositionMetaDataEntity implements PositionMetaData {
 
   @Column(type => GPSMetadataEntity)
@@ -75,5 +83,4 @@ export class PhotoMetadataEntity extends MediaMetadataEntity implements PhotoMet
 export class PhotoEntity extends MediaEntity implements PhotoDTO {
   @Column(type => PhotoMetadataEntity)
   metadata: PhotoMetadataEntity;
-
 }

backend/model/sql/enitites/SharingEntity.ts

@@ -5,7 +5,7 @@ import {UserDTO} from '../../../../common/entities/UserDTO';
 
 @Entity()
 export class SharingEntity implements SharingDTO {
-  @PrimaryGeneratedColumn()
+  @PrimaryGeneratedColumn({unsigned: true})
   id: number;
 
   @Column()
@@ -17,10 +17,20 @@ export class SharingEntity implements SharingDTO {
   @Column({type: 'text', nullable: true})
   password: string;
 
-  @Column()
+  @Column('bigint', {
+    unsigned: true, transformer: {
+      from: v => parseInt(v, 10),
+      to: v => v
+    }
+  })
   expires: number;
 
-  @Column()
+  @Column('bigint', {
+    unsigned: true, transformer: {
+      from: v => parseInt(v, 10),
+      to: v => v
+    }
+  })
   timeStamp: number;
 
   @Column()

backend/model/threading/MetadataLoader.ts

@@ -1,14 +1,22 @@
 import {VideoMetadata} from '../../../common/entities/VideoDTO';
-import {PhotoMetadata} from '../../../common/entities/PhotoDTO';
+import {FaceRegion, PhotoMetadata} from '../../../common/entities/PhotoDTO';
 import {Config} from '../../../common/config/private/Config';
 import {Logger} from '../../Logger';
 import * as fs from 'fs';
 import * as sizeOf from 'image-size';
-import {OrientationTypes, ExifParserFactory} from 'ts-exif-parser';
+import {ExifParserFactory, OrientationTypes} from 'ts-exif-parser';
 import {IptcParser} from 'ts-node-iptc';
 import {FFmpegFactory} from '../FFmpegFactory';
 import {FfprobeData} from 'fluent-ffmpeg';
 
+// TODO: fix up different metadata loaders
+// @ts-ignore
+global.DataView = require('jdataview');
+// @ts-ignore
+global.DOMParser = require('xmldom').DOMParser;
+// @ts-ignore
+const ExifReader = require('exifreader');
+
 const LOG_TAG = '[MetadataLoader]';
 const ffmpeg = FFmpegFactory.get();
 
@@ -112,7 +120,7 @@ export class MetadataLoader {
               }
 
               if (exif.tags.CreateDate || exif.tags.DateTimeOriginal || exif.tags.ModifyDate) {
-                metadata.creationDate = exif.tags.CreateDate || exif.tags.DateTimeOriginal || exif.tags.ModifyDate;
+                metadata.creationDate = (exif.tags.CreateDate || exif.tags.DateTimeOriginal || exif.tags.ModifyDate) * 1000;
               }
 
               if (exif.tags.Orientation) {
@@ -139,16 +147,23 @@ export class MetadataLoader {
 
             try {
               const iptcData = IptcParser.parse(data);
-              if (iptcData.country_or_primary_location_name || iptcData.province_or_state || iptcData.city) {
+              if (iptcData.country_or_primary_location_name) {
                 metadata.positionData = metadata.positionData || {};
                 metadata.positionData.country = iptcData.country_or_primary_location_name.replace(/\0/g, '').trim();
+              }
+              if (iptcData.province_or_state) {
+                metadata.positionData = metadata.positionData || {};
                 metadata.positionData.state = iptcData.province_or_state.replace(/\0/g, '').trim();
+              }
+              if (iptcData.city) {
+                metadata.positionData = metadata.positionData || {};
                 metadata.positionData.city = iptcData.city.replace(/\0/g, '').trim();
               }
               if (iptcData.caption) {
                 metadata.caption = iptcData.caption.replace(/\0/g, '').trim();
               }
               metadata.keywords = iptcData.keywords || [];
+
               metadata.creationDate = <number>(iptcData.date_time ? iptcData.date_time.getTime() : metadata.creationDate);
 
             } catch (err) {
@@ -157,6 +172,47 @@ export class MetadataLoader {
 
             metadata.creationDate = metadata.creationDate || 0;
 
+
+            try {
+              const ret = ExifReader.load(data);
+              const faces: FaceRegion[] = [];
+              if (ret.Regions && ret.Regions.value.RegionList && ret.Regions.value.RegionList.value) {
+                for (let i = 0; i < ret.Regions.value.RegionList.value.length; i++) {
+                  if (!ret.Regions.value.RegionList.value[i].value ||
+                    !ret.Regions.value.RegionList.value[i].value['rdf:Description'] ||
+                    !ret.Regions.value.RegionList.value[i].value['rdf:Description'].value ||
+                    !ret.Regions.value.RegionList.value[i].value['rdf:Description'].value['mwg-rs:Area']) {
+                    continue;
+                  }
+                  const region = ret.Regions.value.RegionList.value[i].value['rdf:Description'];
+                  const regionBox = ret.Regions.value.RegionList.value[i].value['rdf:Description'].value['mwg-rs:Area'].attributes;
+                  if (region.attributes['mwg-rs:Type'] !== 'Face' ||
+                    !region.attributes['mwg-rs:Name']) {
+                    continue;
+                  }
+                  const name = region.attributes['mwg-rs:Name'];
+                  const box = {
+                    width: Math.round(regionBox['stArea:w'] * metadata.size.width),
+                    height: Math.round(regionBox['stArea:h'] * metadata.size.height),
+                    x: Math.round(regionBox['stArea:x'] * metadata.size.width),
+                    y: Math.round(regionBox['stArea:y'] * metadata.size.height)
+                  };
+                  faces.push({name: name, box: box});
+                }
+              }
+              if (faces.length > 0) {
+                metadata.faces = faces; // save faces
+                // remove faces from keywords
+                metadata.faces.forEach(f => {
+                  const index = metadata.keywords.indexOf(f.name);
+                  if (index !== -1) {
+                    metadata.keywords.splice(index, 1);
+                  }
+                });
+              }
+            } catch (err) {
+            }
+
             return resolve(metadata);
           } catch (err) {
             return reject({file: fullPath, error: err});

backend/routes/AdminRouter.ts

@@ -8,6 +8,7 @@ export class AdminRouter {
   public static route(app: Express) {
 
     this.addGetStatistic(app);
+    this.addGetDuplicates(app);
     this.addIndexGallery(app);
     this.addSettings(app);
   }
@@ -20,6 +21,14 @@ export class AdminRouter {
       RenderingMWs.renderResult
     );
   }
+  private static addGetDuplicates(app: Express) {
+    app.get('/api/admin/duplicates',
+      AuthenticationMWs.authenticate,
+      AuthenticationMWs.authorise(UserRoles.Admin),
+      AdminMWs.getDuplicates,
+      RenderingMWs.renderResult
+    );
+  }
 
   private static addIndexGallery(app: Express) {
     app.get('/api/admin/indexes/job/progress',

backend/routes/GalleryRouter.ts

@@ -10,6 +10,7 @@ export class GalleryRouter {
   public static route(app: Express) {
 
     this.addGetImageIcon(app);
+    this.addGetVideoIcon(app);
     this.addGetImageThumbnail(app);
     this.addGetVideoThumbnail(app);
     this.addGetImage(app);
@@ -92,6 +93,17 @@ export class GalleryRouter {
     );
   }
 
+
+  private static addGetVideoIcon(app: Express) {
+    app.get('/api/gallery/content/:mediaPath(*\.(mp4|ogg|ogv|webm))/icon',
+      AuthenticationMWs.authenticate,
+      // TODO: authorize path
+      GalleryMWs.loadFile,
+      ThumbnailGeneratorMWs.generateIconFactory(ThumbnailSourceType.Video),
+      RenderingMWs.renderFile
+    );
+  }
+
   private static addGetImageIcon(app: Express) {
     app.get('/api/gallery/content/:mediaPath(*\.(jpg|jpeg|jpe|webp|png|gif|svg))/icon',
       AuthenticationMWs.authenticate,

backend/routes/PublicRouter.ts

@@ -79,7 +79,7 @@ export class PublicRouter {
       });
 
 
-    app.get(['/', '/login', '/gallery*', '/share*', '/admin', '/search*'],
+      app.get(['/', '/login', '/gallery*', '/share*', '/admin', '/duplicates', '/search*'],
       AuthenticationMWs.tryAuthenticate,
       setLocale,
       renderIndex

benchmark/Benchmarks.ts

@@ -0,0 +1,137 @@
+import {SQLConnection} from '../backend/model/sql/SQLConnection';
+import {Config} from '../common/config/private/Config';
+import {DatabaseType, ReIndexingSensitivity} from '../common/config/private/IPrivateConfig';
+import {ObjectManagerRepository} from '../backend/model/ObjectManagerRepository';
+import {DiskMangerWorker} from '../backend/model/threading/DiskMangerWorker';
+import {IndexingManager} from '../backend/model/sql/IndexingManager';
+import {SearchManager} from '../backend/model/sql/SearchManager';
+import * as fs from 'fs';
+import {SearchTypes} from '../common/entities/AutoCompleteItem';
+import {Utils} from '../common/Utils';
+import {GalleryManager} from '../backend/model/sql/GalleryManager';
+import {DirectoryDTO} from '../common/entities/DirectoryDTO';
+
+export interface BenchmarkResult {
+  duration: number;
+  directories?: number;
+  media?: number;
+  items?: number;
+}
+
+export class BMIndexingManager extends IndexingManager {
+
+  public async saveToDB(scannedDirectory: DirectoryDTO): Promise<void> {
+    return super.saveToDB(scannedDirectory);
+  }
+}
+
+export class Benchmarks {
+
+  constructor(public RUNS: number, public dbPath: string) {
+
+  }
+
+  async bmSaveDirectory(): Promise<BenchmarkResult> {
+    await this.resetDB();
+    const dir = await DiskMangerWorker.scanDirectory('./');
+    const im = new BMIndexingManager();
+    return await this.benchmark(() => im.saveToDB(dir), () => this.resetDB());
+  }
+
+  async bmScanDirectory(): Promise<BenchmarkResult> {
+    return await this.benchmark(() => DiskMangerWorker.scanDirectory('./'));
+  }
+
+  async bmListDirectory(): Promise<BenchmarkResult> {
+    const gm = new GalleryManager();
+    await this.setupDB();
+    Config.Server.indexing.reIndexingSensitivity = ReIndexingSensitivity.low;
+    return await this.benchmark(() => gm.listDirectory('./'));
+  }
+
+  async bmAllSearch(text: string): Promise<{ result: BenchmarkResult, searchType: SearchTypes }[]> {
+    await this.setupDB();
+    const types = Utils.enumToArray(SearchTypes).map(a => a.key).concat([null]);
+    const results: { result: BenchmarkResult, searchType: SearchTypes }[] = [];
+    const sm = new SearchManager();
+    for (let i = 0; i < types.length; i++) {
+      results.push({result: await this.benchmark(() => sm.search(text, types[i])), searchType: types[i]});
+    }
+    return results;
+  }
+
+  async bmInstantSearch(text: string): Promise<BenchmarkResult> {
+    await this.setupDB();
+    const sm = new SearchManager();
+    return await this.benchmark(() => sm.instantSearch(text));
+  }
+
+  async bmAutocomplete(text: string): Promise<BenchmarkResult> {
+    await this.setupDB();
+    const sm = new SearchManager();
+    return await this.benchmark(() => sm.autocomplete(text));
+  }
+
+
+  private async benchmark(fn: () => Promise<{ media: any[], directories: any[] } | any[] | void>,
+                          beforeEach: () => Promise<any> = null,
+                          afterEach: () => Promise<any> = null) {
+    const scanned = await fn();
+    const start = process.hrtime();
+    let skip = 0;
+    for (let i = 0; i < this.RUNS; i++) {
+      if (beforeEach) {
+        const startSkip = process.hrtime();
+        await beforeEach();
+        const endSkip = process.hrtime(startSkip);
+        skip += (endSkip[0] * 1000 + endSkip[1] / 1000000);
+      }
+      await fn();
+      if (afterEach) {
+        const startSkip = process.hrtime();
+        await afterEach();
+        const endSkip = process.hrtime(startSkip);
+        skip += (endSkip[0] * 1000 + endSkip[1] / 1000000);
+      }
+    }
+    const end = process.hrtime(start);
+    const duration = (end[0] * 1000 + end[1] / 1000000) / this.RUNS;
+
+    if (!scanned) {
+      return {
+        duration: duration
+      };
+    }
+
+    if (Array.isArray(scanned)) {
+      return {
+        duration: duration,
+        items: scanned.length
+      };
+    }
+
+    return {
+      duration: duration,
+      media: scanned.media.length,
+      directories: scanned.directories.length
+    };
+  }
+
+  private resetDB = async () => {
+    await SQLConnection.close();
+    if (fs.existsSync(this.dbPath)) {
+      fs.unlinkSync(this.dbPath);
+    }
+    Config.Server.database.type = DatabaseType.sqlite;
+    Config.Server.database.sqlite.storage = this.dbPath;
+    await ObjectManagerRepository.InitSQLManagers();
+  };
+
+  private async setupDB() {
+    const im = new BMIndexingManager();
+    await this.resetDB();
+    const dir = await DiskMangerWorker.scanDirectory('./');
+    await im.saveToDB(dir);
+  }
+
+}

benchmark/README.md

@@ -0,0 +1,22 @@
+# PiGallery2 performance benchmark results
+
+These results are created mostly for development, but I'm making them public for curious users.
+
+## PiGallery2 v1.5.8, 26.01.2019
+
+**System**: Intel(R) Core(TM) i7-6700HQ CPU @ 2.60GHz, 16GB Ram, SHDD: 1TB, 5400 rpm
+**Gallery**: directories: 0 media: 341, faces: 39
+| action | action details | average time | details |
+|:------:|:--------------:|:------------:|:-------:|
+| Scanning directory |  | 2486.5ms | media: 341, directories:0 |
+| Saving directory |  | 780.0ms | - |
+| Listing Directory |  | 31.5ms | media: 341, directories:0 |
+| searching | `a` as `directory` | 2.9ms | - |
+| searching | `a` as `person` | 7.3ms | media: 39, directories:0 |
+| searching | `a` as `keyword` | 30.8ms | media: 339, directories:0 |
+| searching | `a` as `position` | 25.7ms | media: 282, directories:0 |
+| searching | `a` as `photo` | 2.8ms | - |
+| searching | `a` as `video` | 2.6ms | - |
+| searching | `a` as `any` | 33.0ms | media: 339, directories:0 |
+| instant search | `a` | 6.1ms | media: 10, directories:0 |
+| auto complete | `a` | 5.4ms | items: 10 | 

benchmark/index.ts

@@ -0,0 +1,77 @@
+import {Config} from '../common/config/private/Config';
+import * as path from 'path';
+import {ProjectPath} from '../backend/ProjectPath';
+import {BenchmarkResult, Benchmarks} from './Benchmarks';
+import {SearchTypes} from '../common/entities/AutoCompleteItem';
+import {Utils} from '../common/Utils';
+import {DiskMangerWorker} from '../backend/model/threading/DiskMangerWorker';
+
+const config: { path: string, system: string } = require(path.join(__dirname, 'config.json'));
+Config.Server.imagesFolder = config.path;
+const dbPath = path.join(__dirname, 'test.db');
+ProjectPath.reset();
+const RUNS = 50;
+
+let resultsText = '';
+const printLine = (text: string) => {
+  resultsText += text + '\n';
+};
+
+const printHeader = async () => {
+  const dt = new Date();
+  printLine('## PiGallery2 v' + require('./../package.json').version +
+    ', ' + Utils.zeroPrefix(dt.getDate(), 2) +
+    '.' + Utils.zeroPrefix(dt.getMonth() + 1, 2) +
+    '.' + dt.getFullYear());
+  printLine('**System**: ' + config.system);
+  const dir = await DiskMangerWorker.scanDirectory('./');
+  printLine('**Gallery**: directories: ' +
+    dir.directories.length +
+    ' media: ' + dir.media.length +
+    // @ts-ignore
+    ', faces: ' + dir.media.reduce((p, c) => p + (c.metadata.faces || []).length, 0));
+};
+
+
+const printTableHeader = () => {
+  printLine('| action | action details | average time | details |');
+  printLine('|:------:|:--------------:|:------------:|:-------:|');
+};
+const printResult = (result: BenchmarkResult, action: string, actionDetails: string = '') => {
+  console.log('benchmarked: ' + action);
+  let details = '-';
+  if (result.items) {
+    details = 'items: ' + result.items;
+  }
+  if (result.media) {
+    details = 'media: ' + result.media + ', directories:' + result.directories;
+  }
+  printLine('| ' + action + ' | ' + actionDetails +
+    ' | ' + (result.duration).toFixed(1) + 'ms | ' + details + ' |');
+};
+
+const run = async () => {
+  const start = Date.now();
+  const bm = new Benchmarks(RUNS, dbPath);
+
+  // header
+  await printHeader();
+  printTableHeader();
+  printResult(await bm.bmScanDirectory(), 'Scanning directory');
+  printResult(await bm.bmSaveDirectory(), 'Saving directory');
+  printResult(await bm.bmListDirectory(), 'Listing Directory');
+  (await bm.bmAllSearch('a')).forEach(res => {
+    if (res.searchType !== null) {
+      printResult(res.result, 'searching', '`a` as `' + SearchTypes[res.searchType] + '`');
+    } else {
+      printResult(res.result, 'searching', '`a` as `any`');
+    }
+  });
+  printResult(await bm.bmInstantSearch('a'), 'instant search', '`a`');
+  printResult(await bm.bmAutocomplete('a'), 'auto complete', '`a`');
+  console.log(resultsText);
+  console.log('run for : ' + ((Date.now() - start)).toFixed(1) + 'ms');
+};
+
+run();
+

common/DataStructureVersion.ts

@@ -1 +1 @@
-export const DataStructureVersion = 7;
+export const DataStructureVersion = 9;

common/Utils.ts

@@ -90,14 +90,14 @@ export class Utils {
         continue;
       }
 
-      const part = args[i].replace('\\', '/');
+      const part = args[i].replace(new RegExp('\\\\', 'g'), '/');
       if (part === '/' || part === './') {
         continue;
       }
 
       url += part + '/';
     }
-    url = url.replace('//', '/');
+    url = url.replace(new RegExp('/+', 'g'), '/');
 
     if (url.trim() === '') {
       url = './';

common/config/private/IPrivateConfig.ts

@@ -56,6 +56,10 @@ export interface ThreadingConfig {
   thumbnailThreads: number;
 }
 
+export interface DuplicatesConfig {
+  listingLimit: number; // maximum number of duplicates to list
+}
+
 export interface ServerConfig {
   port: number;
   host: string;
@@ -67,6 +71,7 @@ export interface ServerConfig {
   sessionTimeout: number;
   indexing: IndexingConfig;
   photoMetadataSize: number;
+  duplicates: DuplicatesConfig;
 }
 
 export interface IPrivateConfig {

common/config/private/PrivateConfigClass.ts

@@ -45,6 +45,9 @@ export class PrivateConfigClass extends PublicConfigClass implements IPrivateCon
       folderPreviewSize: 2,
       cachedFolderTimeout: 1000 * 60 * 60,
       reIndexingSensitivity: ReIndexingSensitivity.low
+    },
+    duplicates: {
+      listingLimit: 1000
     }
   };
   private ConfigLoader: any;
@@ -60,7 +63,11 @@ export class PrivateConfigClass extends PublicConfigClass implements IPrivateCon
   public load() {
     ConfigLoader.loadBackendConfig(this,
       path.join(__dirname, './../../../config.json'),
-      [['PORT', 'Server-port']]);
+      [['PORT', 'Server-port'],
+        ['MYSQL_HOST', 'Server-database-mysql-host'],
+        ['MYSQL_PASSWORD', 'Server-database-mysql-password'],
+        ['MYSQL_USERNAME', 'Server-database-mysql-username'],
+        ['MYSQL_DATABASE', 'Server-database-mysql-database']]);
 
     if (Utils.enumToArray(UserRoles).map(r => r.key).indexOf(this.Client.unAuthenticatedUserRole) === -1) {
       throw new Error('Unknown user role for Client.unAuthenticatedUserRole, found: ' + this.Client.unAuthenticatedUserRole);

common/config/public/ConfigClass.ts

@@ -7,14 +7,19 @@ export module ClientConfig {
     OpenStreetMap, Mapbox, Custom
   }
 
+  export interface AutoCompleteConfig {
+    enabled: boolean;
+    maxItemsPerCategory: number;
+    cacheTimeout: number;
+  }
+
   export interface SearchConfig {
     enabled: boolean;
     instantSearchEnabled: boolean;
-    autocompleteEnabled: boolean;
     InstantSearchTimeout: number;
-    autocompleteCacheTimeout: number;
     instantSearchCacheTimeout: number;
     searchCacheTimeout: number;
+    AutoComplete: AutoCompleteConfig;
   }
 
   export interface SharingConfig {
@@ -95,11 +100,14 @@ export class PublicConfigClass {
     Search: {
       enabled: true,
       instantSearchEnabled: true,
-      autocompleteEnabled: true,
       InstantSearchTimeout: 3000,
-      autocompleteCacheTimeout: 1000 * 60 * 60,
       searchCacheTimeout: 1000 * 60 * 60,
-      instantSearchCacheTimeout: 1000 * 60 * 60
+      instantSearchCacheTimeout: 1000 * 60 * 60,
+      AutoComplete: {
+        enabled: true,
+        cacheTimeout: 1000 * 60 * 60,
+        maxItemsPerCategory: 5
+      }
     },
     Sharing: {
       enabled: true,

common/entities/AutoCompleteItem.ts

@@ -1,9 +1,10 @@
 export enum SearchTypes {
   directory = 1,
-  keyword = 2,
-  position = 3,
-  photo = 4,
-  video = 5
+  person = 2,
+  keyword = 3,
+  position = 5,
+  photo = 6,
+  video = 7
 }
 
 export class AutoCompleteItem {

common/entities/DuplicatesDTO.ts

@@ -0,0 +1,5 @@
+import {MediaDTO} from './MediaDTO';
+
+export interface DuplicatesDTO {
+  media: MediaDTO[];
+}

common/entities/Error.ts

@@ -23,4 +23,8 @@ export enum ErrorCodes {
 export class ErrorDTO {
   constructor(public code: ErrorCodes, public message?: string, public details?: any) {
   }
+
+  toString(): string {
+    return '[' + ErrorCodes[this.code] + '] ' + this.message + (this.details ? this.details.toString() : '');
+  }
 }

common/entities/PhotoDTO.ts

@@ -1,6 +1,6 @@
 import {DirectoryDTO} from './DirectoryDTO';
 import {OrientationTypes} from 'ts-exif-parser';
-import {MediaDTO, MediaMetadata, MediaDimension} from './MediaDTO';
+import {MediaDimension, MediaDTO, MediaMetadata} from './MediaDTO';
 
 export interface PhotoDTO extends MediaDTO {
   id: number;
@@ -11,6 +11,18 @@ export interface PhotoDTO extends MediaDTO {
   readyIcon: boolean;
 }
 
+export interface FaceRegionBox {
+  width: number;
+  height: number;
+  x: number;
+  y: number;
+}
+
+export interface FaceRegion {
+  name: string;
+  box: FaceRegionBox;
+}
+
 export interface PhotoMetadata extends MediaMetadata {
   caption?: string;
   keywords?: string[];
@@ -20,6 +32,7 @@ export interface PhotoMetadata extends MediaMetadata {
   size: MediaDimension;
   creationDate: number;
   fileSize: number;
+  faces?: FaceRegion[];
 }
 
 

frontend/app/admin/admin.component.html

@@ -50,7 +50,8 @@
     <app-settings-share #share [hidden]="!share.hasAvailableSettings"
                         [simplifiedMode]="simplifiedMode"></app-settings-share>
     <app-settings-map #map [hidden]="!map.hasAvailableSettings" [simplifiedMode]="simplifiedMode"></app-settings-map>
-    <app-settings-meta-file #metaFile [hidden]="!metaFile.hasAvailableSettings" [simplifiedMode]="simplifiedMode"></app-settings-meta-file>
+    <app-settings-meta-file #metaFile [hidden]="!metaFile.hasAvailableSettings"
+                            [simplifiedMode]="simplifiedMode"></app-settings-meta-file>
     <app-settings-other #other [hidden]="!other.hasAvailableSettings"
                         [simplifiedMode]="simplifiedMode"></app-settings-other>
     <app-settings-random-photo #random [hidden]="!random.hasAvailableSettings"

frontend/app/app.component.ts

@@ -1,4 +1,4 @@
-import {Component, OnDestroy, OnInit, ViewContainerRef} from '@angular/core';
+import {Component, OnDestroy, OnInit} from '@angular/core';
 import {AuthenticationService} from './model/network/authentication.service';
 import {UserDTO} from '../../common/entities/UserDTO';
 import {Router} from '@angular/router';
@@ -10,9 +10,7 @@ import {Subscription} from 'rxjs';
 
 @Component({
   selector: 'app-pi-gallery2',
-  template: `
-    <router-outlet></router-outlet>`,
-
+  template: `<router-outlet></router-outlet>`
 })
 export class AppComponent implements OnInit, OnDestroy {
 

frontend/app/app.module.ts

@@ -1,10 +1,4 @@
-import {
-  Injectable,
-  LOCALE_ID,
-  NgModule,
-  TRANSLATIONS,
-  TRANSLATIONS_FORMAT
-} from '@angular/core';
+import {Injectable, LOCALE_ID, NgModule, TRANSLATIONS, TRANSLATIONS_FORMAT} from '@angular/core';
 import {BrowserModule, HAMMER_GESTURE_CONFIG, HammerGestureConfig} from '@angular/platform-browser';
 import {FormsModule} from '@angular/forms';
 import {AppComponent} from './app.component';
@@ -52,6 +46,7 @@ import {MapSettingsComponent} from './settings/map/map.settings.component';
 import {TooltipModule} from 'ngx-bootstrap/tooltip';
 import {BsDropdownModule} from 'ngx-bootstrap/dropdown';
 import {CollapseModule} from 'ngx-bootstrap/collapse';
+import {PopoverModule} from 'ngx-bootstrap/popover';
 import {ThumbnailSettingsComponent} from './settings/thumbnail/thumbanil.settings.component';
 import {SearchSettingsComponent} from './settings/search/search.settings.component';
 import {SettingsService} from './settings/settings.service';
@@ -75,6 +70,9 @@ import {MapService} from './gallery/map/map.service';
 import {MetaFileSettingsComponent} from './settings/metafiles/metafile.settings.component';
 import {ThumbnailLoaderService} from './gallery/thumbnailLoader.service';
 import {FileSizePipe} from './pipes/FileSizePipe';
+import {DuplicateService} from './duplicates/duplicates.service';
+import {DuplicateComponent} from './duplicates/duplicates.component';
+import {DuplicatesPhotoComponent} from './duplicates/photo/photo.duplicates.component';
 
 
 @Injectable()
@@ -124,6 +122,7 @@ export function translationsFactory(locale: string) {
     ToastrModule.forRoot(),
     ModalModule.forRoot(),
     CollapseModule.forRoot(),
+    PopoverModule.forRoot(),
     BsDropdownModule.forRoot(),
     SlimLoadingBarModule.forRoot(),
     BsDatepickerModule.forRoot(),
@@ -165,6 +164,8 @@ export function translationsFactory(locale: string) {
     BasicSettingsComponent,
     OtherSettingsComponent,
     IndexingSettingsComponent,
+    DuplicateComponent,
+    DuplicatesPhotoComponent,
     StringifyRole,
     IconizeSortingMethod,
     StringifySortingMethod,
@@ -190,6 +191,7 @@ export function translationsFactory(locale: string) {
     SettingsService,
     OverlayService,
     QueryService,
+    DuplicateService,
     {
       provide: TRANSLATIONS,
       useFactory: translationsFactory,

frontend/app/app.routing.ts

@@ -5,6 +5,7 @@ import {GalleryComponent} from './gallery/gallery.component';
 import {AdminComponent} from './admin/admin.component';
 import {ShareLoginComponent} from './sharelogin/share-login.component';
 import {QueryParams} from '../../common/QueryParams';
+import {DuplicateComponent} from './duplicates/duplicates.component';
 
 export function galleryMatcherFunction(
   segments: UrlSegment[]): UrlMatchResult | null {
@@ -50,6 +51,10 @@ const ROUTES: Routes = [
     path: 'admin',
     component: AdminComponent
   },
+  {
+    path: 'duplicates',
+    component: DuplicateComponent
+  },
   {
     matcher: galleryMatcherFunction,
     component: GalleryComponent

frontend/app/duplicates/duplicates.component.css

@@ -0,0 +1,16 @@
+.card{
+  margin: 8px 0;
+}
+
+.row{
+  margin: 5px 0;
+  cursor: pointer;
+}
+
+.row:hover{
+  background-color: #f8f9fa;
+}
+
+a{
+  color: #212529;
+}

frontend/app/duplicates/duplicates.component.html

@@ -0,0 +1,38 @@
+<app-frame>
+
+  <div body class="container">
+    <ng-template [ngIf]="renderedDirGroups">
+      <div class="alert alert-secondary" role="alert" *ngIf=" duplicateCount.photos >0" i18n>
+        Listing <strong>{{duplicateCount.pairs}}</strong> duplicates ({{duplicateCount.photos}} photos).
+      </div>
+      <div class="alert alert-secondary" role="alert" *ngIf=" duplicateCount.photos ==0" i18n>
+        No duplicates found
+      </div>
+      <div *ngFor="let group of renderedDirGroups">
+        <strong>{{group.name}}</strong>
+        <div *ngFor="let pairs of group.duplicates" class="card">
+          <div class="card-body">
+            <a *ngFor="let media of pairs.media"
+               class="row"
+               [routerLink]="['/gallery', getDirectoryPath(media.directory)]"
+               [queryParams]="queryService.getParams()">
+              <app-duplicates-photo class="col-1" [media]="media"></app-duplicates-photo>
+              <div class="col-6">
+                /{{getDirectoryPath(media.directory)}}/<strong>{{media.name}}</strong>
+              </div>
+              <div class="col-2">
+                {{media.metadata.fileSize | fileSize}}
+              </div>
+              <div class="col-3" [title]="media.metadata.creationDate">
+                {{media.metadata.creationDate | date}}, {{media.metadata.creationDate | date:'mediumTime'}}
+              </div>
+            </a>
+          </div>
+        </div>
+      </div>
+    </ng-template>
+    <ng-template [ngIf]="!renderedDirGroups">
+      loading
+    </ng-template>
+  </div>
+</app-frame>

frontend/app/duplicates/duplicates.component.ts

@@ -0,0 +1,132 @@
+import {Component, HostListener, OnDestroy} from '@angular/core';
+import {DuplicateService} from './duplicates.service';
+import {Utils} from '../../../common/Utils';
+import {QueryService} from '../model/query.service';
+import {DuplicatesDTO} from '../../../common/entities/DuplicatesDTO';
+import {DirectoryDTO} from '../../../common/entities/DirectoryDTO';
+import {Subscription} from 'rxjs';
+import {Config} from '../../../common/config/public/Config';
+import {PageHelper} from '../model/page.helper';
+
+interface GroupedDuplicate {
+  name: string;
+  duplicates: DuplicatesDTO[];
+}
+
+@Component({
+  selector: 'app-duplicate',
+  templateUrl: './duplicates.component.html',
+  styleUrls: ['./duplicates.component.css']
+})
+export class DuplicateComponent implements OnDestroy {
+
+  directoryGroups: GroupedDuplicate[] = null;
+  renderedDirGroups: GroupedDuplicate[] = null;
+  renderedIndex = {
+    group: -1,
+    pairs: 0
+  };
+  subscription: Subscription;
+  renderTimer: number = null;
+  duplicateCount = {
+    pairs: 0,
+    photos: 0
+  };
+
+  constructor(public _duplicateService: DuplicateService,
+              public queryService: QueryService) {
+    this._duplicateService.getDuplicates().catch(console.error);
+    this.subscription = this._duplicateService.duplicates.subscribe((duplicates: DuplicatesDTO[]) => {
+      this.directoryGroups = [];
+      this.renderedIndex = {group: -1, pairs: 0};
+      this.renderedDirGroups = [];
+      this.duplicateCount = {
+        pairs: 0,
+        photos: 0
+      };
+      if (duplicates === null) {
+        return;
+      }
+      this.duplicateCount.photos = duplicates.reduce((prev: number, curr) => prev + curr.media.length, 0);
+      this.duplicateCount.pairs = duplicates.length;
+
+      const getMostFrequentDir = (dupls: DuplicatesDTO[]) => {
+        if (dupls.length === 0) {
+          return null;
+        }
+        const dirFrequency: { [key: number]: { count: number, dir: DirectoryDTO } } = {};
+        dupls.forEach(d => d.media.forEach(m => {
+          dirFrequency[m.directory.id] = dirFrequency[m.directory.id] || {dir: m.directory, count: 0};
+          dirFrequency[m.directory.id].count++;
+        }));
+        let max: { count: number, dir: DirectoryDTO } = {count: -1, dir: null};
+        for (const freq of Object.values(dirFrequency)) {
+          if (max.count <= freq.count) {
+            max = freq;
+          }
+        }
+        return max.dir;
+      };
+
+      while (duplicates.length > 0) {
+        const dir = getMostFrequentDir(duplicates);
+        const group = duplicates.filter(d => d.media.find(m => m.directory.id === dir.id));
+        duplicates = duplicates.filter(d => !d.media.find(m => m.directory.id === dir.id));
+        this.directoryGroups.push({name: this.getDirectoryPath(dir) + ' (' + group.length + ')', duplicates: group});
+      }
+      this.renderMore();
+    });
+  }
+
+  ngOnDestroy(): void {
+    if (this.subscription) {
+      this.subscription.unsubscribe();
+      this.subscription = null;
+    }
+  }
+
+  getDirectoryPath(directory: DirectoryDTO) {
+    return Utils.concatUrls(directory.path, directory.name);
+  }
+
+  renderMore = () => {
+    if (this.renderTimer !== null) {
+      clearTimeout(this.renderTimer);
+      this.renderTimer = null;
+    }
+
+    if (this.renderedIndex.group === this.directoryGroups.length - 1 &&
+      this.renderedIndex.pairs >=
+      this.directoryGroups[this.renderedIndex.group].duplicates.length) {
+      return;
+    }
+    if (this.shouldRenderMore()) {
+      if (this.renderedDirGroups.length === 0 ||
+        this.renderedIndex.pairs >=
+        this.directoryGroups[this.renderedIndex.group].duplicates.length) {
+        this.renderedDirGroups.push({
+          name: this.directoryGroups[++this.renderedIndex.group].name,
+          duplicates: []
+        });
+        this.renderedIndex.pairs = 0;
+      }
+      this.renderedDirGroups[this.renderedDirGroups.length - 1].duplicates
+        .push(this.directoryGroups[this.renderedIndex.group].duplicates[this.renderedIndex.pairs++]);
+
+      this.renderTimer = window.setTimeout(this.renderMore, 0);
+    }
+  };
+
+
+  @HostListener('window:scroll')
+  onScroll() {
+    this.renderMore();
+  }
+
+  private shouldRenderMore(): boolean {
+    return Config.Client.Other.enableOnScrollRendering === false ||
+      PageHelper.ScrollY >= PageHelper.MaxScrollY * 0.7
+      || (document.body.clientHeight) * 0.85 < window.innerHeight;
+  }
+}
+

frontend/app/duplicates/duplicates.service.ts

@@ -0,0 +1,20 @@
+import {Injectable} from '@angular/core';
+import {NetworkService} from '../model/network/network.service';
+import {DuplicatesDTO} from '../../../common/entities/DuplicatesDTO';
+import {BehaviorSubject} from 'rxjs';
+
+
+@Injectable()
+export class DuplicateService {
+
+  public duplicates: BehaviorSubject<DuplicatesDTO[]>;
+
+  constructor(private networkService: NetworkService) {
+    this.duplicates = new BehaviorSubject<DuplicatesDTO[]>(null);
+  }
+
+  public async getDuplicates() {
+    this.duplicates.next(await this.networkService.getJson<DuplicatesDTO[]>('/admin/duplicates'));
+  }
+
+}

frontend/app/duplicates/photo/photo.duplicates.component.css

@@ -0,0 +1,13 @@
+.icon {
+  height: 30px;
+}
+
+.big-icon {
+  height: 60px;
+}
+
+.photo-container {
+  width: inherit;
+  height: inherit;
+  overflow: hidden;
+}

frontend/app/duplicates/photo/photo.duplicates.component.html

@@ -0,0 +1,15 @@
+<div class="photo-container">
+  <ng-template #popTemplate>
+    <img alt="{{media.name}}"
+         class="big-icon"
+         [src]="thumbnail.Src | fixOrientation:Orientation | async"
+         *ngIf="thumbnail.Available">
+  </ng-template>
+
+  <img alt="{{media.name}}"
+       class="icon"
+       [popover]="popTemplate"
+       triggers="mouseenter:mouseleave"
+       [src]="thumbnail.Src | fixOrientation:Orientation | async"
+       *ngIf="thumbnail.Available">
+</div>

frontend/app/duplicates/photo/photo.duplicates.component.ts

@@ -0,0 +1,40 @@
+import {Component, Input, OnDestroy, OnInit} from '@angular/core';
+import {MediaDTO} from '../../../../common/entities/MediaDTO';
+import {Media} from '../../gallery/Media';
+import {IconThumbnail, Thumbnail, ThumbnailManagerService} from '../../gallery/thumbnailManager.service';
+import {PhotoDTO} from '../../../../common/entities/PhotoDTO';
+import {OrientationTypes} from 'ts-exif-parser';
+import {MediaIcon} from '../../gallery/MediaIcon';
+
+@Component({
+  selector: 'app-duplicates-photo',
+  templateUrl: './photo.duplicates.component.html',
+  styleUrls: ['./photo.duplicates.component.css']
+})
+export class DuplicatesPhotoComponent implements OnInit, OnDestroy {
+  @Input() media: MediaDTO;
+
+  thumbnail: IconThumbnail;
+
+
+  constructor(private thumbnailService: ThumbnailManagerService) {
+  }
+
+  get Orientation() {
+    if (!this.media) {
+      return OrientationTypes.TOP_LEFT;
+    }
+    return (<PhotoDTO>this.media).metadata.orientation || OrientationTypes.TOP_LEFT;
+  }
+
+  ngOnInit() {
+    this.thumbnail = this.thumbnailService.getIcon(new MediaIcon(this.media));
+
+  }
+
+  ngOnDestroy() {
+    this.thumbnail.destroy();
+  }
+
+}
+

frontend/app/frame/frame.component.html

@@ -28,17 +28,25 @@
                 type="button" class="btn btn-dark dropdown-toggle"
                 aria-controls="dropdown-basic">
           <span class="oi oi-menu"></span>
-          <span *ngIf="isAdmin() && notificationService.notifications.length>0" class="navbar-badge badge badge-warning">{{notificationService.notifications.length}}</span>
+          <span *ngIf="isAdmin() && notificationService.notifications.length>0"
+                class="navbar-badge badge badge-warning">{{notificationService.notifications.length}}</span>
 
         </button>
         <ul id="dropdown-basic" *dropdownMenu
             class="dropdown-menu dropdown-menu-right"
             role="menu" aria-labelledby="button-basic">
           <ng-content select="[navbar-menu]"></ng-content>
+          <li role="menuitem" *ngIf="isAdmin()">
+            <a class="dropdown-item" href="#" [routerLink]="['/duplicates']">
+              <span class="oi oi-layers"></span>
+              <ng-container i18n>duplicates</ng-container>
+            </a>
+          </li>
           <li role="menuitem" *ngIf="isAdmin()">
             <a class="dropdown-item" href="#" [routerLink]="['/admin']">
               <span class="oi oi-wrench"></span>
-              <span *ngIf="notificationService.notifications.length>0" class="badge">{{notificationService.notifications.length}}</span>
+              <span *ngIf="notificationService.notifications.length>0"
+                    class="badge">{{notificationService.notifications.length}}</span>
               <ng-container i18n>Settings</ng-container>
             </a>
           </li>

frontend/app/frame/frame.component.ts

@@ -1,11 +1,10 @@
-import {Component, ElementRef, ViewChild, ViewEncapsulation} from '@angular/core';
+import {Component, ViewEncapsulation} from '@angular/core';
 import {RouterLink} from '@angular/router';
 import {AuthenticationService} from '../model/network/authentication.service';
 import {UserDTO, UserRoles} from '../../../common/entities/UserDTO';
 import {Config} from '../../../common/config/public/Config';
 import {BehaviorSubject} from 'rxjs';
 import {NotificationService} from '../model/notification.service';
-import {ShareService} from '../gallery/share.service';
 import {QueryService} from '../model/query.service';
 
 @Component({

frontend/app/gallery/Media.ts

@@ -1,4 +1,3 @@
-import {PhotoDTO} from '../../../common/entities/PhotoDTO';
 import {Utils} from '../../../common/Utils';
 import {MediaIcon} from './MediaIcon';
 import {Config} from '../../../common/config/public/Config';

frontend/app/gallery/MediaIcon.ts

@@ -1,4 +1,3 @@
-import {PhotoDTO} from '../../../common/entities/PhotoDTO';
 import {Utils} from '../../../common/Utils';
 import {Config} from '../../../common/config/public/Config';
 import {MediaDTO} from '../../../common/entities/MediaDTO';

frontend/app/gallery/cache.gallery.service.ts

@@ -77,7 +77,7 @@ export class GalleryCacheService {
     const tmp = localStorage.getItem(key);
     if (tmp != null) {
       const value: CacheItem<AutoCompleteItem[]> = JSON.parse(tmp);
-      if (value.timestamp < Date.now() - Config.Client.Search.autocompleteCacheTimeout) {
+      if (value.timestamp < Date.now() - Config.Client.Search.AutoComplete.cacheTimeout) {
         localStorage.removeItem(key);
         return null;
       }

frontend/app/gallery/grid/grid.gallery.component.ts

@@ -292,7 +292,6 @@ export class GalleryGridComponent implements OnChanges, OnInit, AfterViewInit, O
     return Config.Client.Other.enableOnScrollRendering === false ||
       PageHelper.ScrollY >= (document.body.clientHeight + offset - window.innerHeight) * 0.7
       || (document.body.clientHeight + offset) * 0.85 < window.innerHeight;
-
   }
 
 

frontend/app/gallery/grid/photo/photo.grid.gallery.component.css

@@ -111,3 +111,7 @@ a {
 .photo-container:hover .video-indicator {
   background-color: rgba(0, 0, 0, 0.8);
 }
+
+.photo-keywords .oi-person{
+  margin-right: 2px;
+}

frontend/app/gallery/grid/photo/photo.grid.gallery.component.html

@@ -36,11 +36,17 @@
       </ng-template>
     </div>
 
-    <div class="photo-keywords" *ngIf="gridPhoto.media.metadata.keywords && gridPhoto.media.metadata.keywords.length">
-      <ng-template ngFor let-keyword [ngForOf]="gridPhoto.media.metadata.keywords" let-last="last">
+    <div class="photo-keywords" *ngIf="keywords">
+      <ng-template ngFor let-keyword [ngForOf]="keywords" let-last="last">
         <a *ngIf="searchEnabled"
-           [routerLink]="['/search', keyword, {type: SearchTypes[SearchTypes.keyword]}]">#{{keyword}}</a>
-        <span *ngIf="!searchEnabled">#{{keyword}}</span>
+           [routerLink]="['/search', keyword.value, {type: SearchTypes[keyword.type]}]" [ngSwitch]="keyword.type">
+          <ng-template [ngSwitchCase]="SearchTypes.keyword">#</ng-template><!--
+       --><ng-template [ngSwitchCase]="SearchTypes.person"><span class="oi oi-person"></span></ng-template><!--
+       -->{{keyword.value}}</a>
+        <span *ngIf="!searchEnabled" [ngSwitch]="keyword.type">
+          <ng-template [ngSwitchCase]="SearchTypes.keyword">#</ng-template><!--
+       --><ng-template [ngSwitchCase]="SearchTypes.person"><span class="oi oi-person"></span></ng-template><!--
+       -->{{keyword.value}}</span>
         <ng-template [ngIf]="!last">,</ng-template>
       </ng-template>
 

frontend/app/gallery/grid/photo/photo.grid.gallery.component.ts

@@ -5,9 +5,8 @@ import {SearchTypes} from '../../../../../common/entities/AutoCompleteItem';
 import {RouterLink} from '@angular/router';
 import {Thumbnail, ThumbnailManagerService} from '../../thumbnailManager.service';
 import {Config} from '../../../../../common/config/public/Config';
-import {AnimationBuilder} from '@angular/animations';
 import {PageHelper} from '../../../model/page.helper';
-import {PhotoDTO} from '../../../../../common/entities/PhotoDTO';
+import {PhotoDTO, PhotoMetadata} from '../../../../../common/entities/PhotoDTO';
 
 @Component({
   selector: 'app-gallery-grid-photo',
@@ -22,6 +21,7 @@ export class GalleryPhotoComponent implements IRenderable, OnInit, OnDestroy {
   @ViewChild('photoContainer') container: ElementRef;
 
   thumbnail: Thumbnail;
+  keywords: { value: string, type: SearchTypes }[] = null;
   infoBar = {
     marginTop: 0,
     visible: false,
@@ -34,17 +34,40 @@ export class GalleryPhotoComponent implements IRenderable, OnInit, OnDestroy {
 
   wasInView: boolean = null;
 
-  constructor(private thumbnailService: ThumbnailManagerService,
-              private _animationBuilder: AnimationBuilder) {
+  constructor(private thumbnailService: ThumbnailManagerService) {
     this.SearchTypes = SearchTypes;
     this.searchEnabled = Config.Client.Search.enabled;
   }
 
-  ngOnInit() {
-    this.thumbnail = this.thumbnailService.getThumbnail(this.gridPhoto);
+  get ScrollListener(): boolean {
+    return !this.thumbnail.Available && !this.thumbnail.Error;
   }
 
 
+  get Title(): string {
+    if (Config.Client.Other.captionFirstNaming === false) {
+      return this.gridPhoto.media.name;
+    }
+    if ((<PhotoDTO>this.gridPhoto.media).metadata.caption) {
+      if ((<PhotoDTO>this.gridPhoto.media).metadata.caption.length > 20) {
+        return (<PhotoDTO>this.gridPhoto.media).metadata.caption.substring(0, 17) + '...';
+      }
+      return (<PhotoDTO>this.gridPhoto.media).metadata.caption;
+    }
+    return this.gridPhoto.media.name;
+  }
+
+  ngOnInit() {
+    this.thumbnail = this.thumbnailService.getThumbnail(this.gridPhoto);
+    const metadata = this.gridPhoto.media.metadata as PhotoMetadata;
+    if ((metadata.keywords && metadata.keywords.length > 0) ||
+      (metadata.faces && metadata.faces.length > 0)) {
+      this.keywords = (metadata.faces || []).map(f => ({value: f.name, type: SearchTypes.person}))
+        .concat((metadata.keywords || []).map(k => ({value: k, type: SearchTypes.keyword})));
+    }
+
+  }
+
   ngOnDestroy() {
     this.thumbnail.destroy();
 
@@ -53,16 +76,11 @@ export class GalleryPhotoComponent implements IRenderable, OnInit, OnDestroy {
     }
   }
 
-
   isInView(): boolean {
     return PageHelper.ScrollY < this.container.nativeElement.offsetTop + this.container.nativeElement.clientHeight
       && PageHelper.ScrollY + window.innerHeight > this.container.nativeElement.offsetTop;
   }
 
-  get ScrollListener(): boolean {
-    return !this.thumbnail.Available && !this.thumbnail.Error;
-  }
-
   onScroll() {
     if (this.thumbnail.Available === true || this.thumbnail.Error === true) {
       return;
@@ -74,7 +92,6 @@ export class GalleryPhotoComponent implements IRenderable, OnInit, OnDestroy {
     }
   }
 
-
   getPositionText(): string {
     if (!this.gridPhoto || !this.gridPhoto.isPhoto()) {
       return '';
@@ -84,7 +101,6 @@ export class GalleryPhotoComponent implements IRenderable, OnInit, OnDestroy {
       (<PhotoDTO>this.gridPhoto.media).metadata.positionData.country;
   }
 
-
   mouseOver() {
     this.infoBar.visible = true;
     if (this.animationTimer != null) {
@@ -124,19 +140,6 @@ export class GalleryPhotoComponent implements IRenderable, OnInit, OnDestroy {
 
   }
 
-  get Title(): string {
-    if (Config.Client.Other.captionFirstNaming === false) {
-      return this.gridPhoto.media.name;
-    }
-    if ((<PhotoDTO>this.gridPhoto.media).metadata.caption) {
-      if ((<PhotoDTO>this.gridPhoto.media).metadata.caption.length > 20) {
-        return (<PhotoDTO>this.gridPhoto.media).metadata.caption.substring(0, 17) + '...';
-      }
-      return (<PhotoDTO>this.gridPhoto.media).metadata.caption;
-    }
-    return this.gridPhoto.media.name;
-  }
-
   /*
    onImageLoad() {
    this.loading.show = false;

frontend/app/gallery/navigator/navigator.gallery.component.html

@@ -15,6 +15,7 @@
                       <span *ngSwitchCase="SearchTypes.video" class="oi oi-video"></span>
                       <span *ngSwitchCase="SearchTypes.directory" class="oi oi-folder"></span>
                       <span *ngSwitchCase="SearchTypes.keyword" class="oi oi-tag"></span>
+                      <span *ngSwitchCase="SearchTypes.person" class="oi oi-person"></span>
                       <span *ngSwitchCase="SearchTypes.position" class="oi oi-map-marker"></span>
                     </span>
       <strong> {{searchResult.searchText}}</strong>

frontend/app/gallery/search/search.gallery.component.html

@@ -25,6 +25,7 @@
                       <span *ngSwitchCase="SearchTypes.video" class="oi oi-video"></span>
                       <span *ngSwitchCase="SearchTypes.directory" class="oi oi-folder"></span>
                       <span *ngSwitchCase="SearchTypes.keyword" class="oi oi-tag"></span>
+                      <span *ngSwitchCase="SearchTypes.person" class="oi oi-person"></span>
                       <span *ngSwitchCase="SearchTypes.position" class="oi oi-map-marker"></span>
                     </span>
           {{item.preText}}<strong>{{item.highLightText}}</strong>{{item.postText}}

frontend/app/gallery/search/search.gallery.component.ts

@@ -54,7 +54,7 @@ export class GallerySearchComponent implements OnDestroy {
 
     const searchText = (<HTMLInputElement>event.target).value.trim();
 
-    if (Config.Client.Search.autocompleteEnabled &&
+    if (Config.Client.Search.AutoComplete.enabled &&
       this.cache.lastAutocomplete !== searchText) {
       this.cache.lastAutocomplete = searchText;
       this.autocomplete(searchText).catch(console.error);
@@ -92,7 +92,7 @@ export class GallerySearchComponent implements OnDestroy {
   }
 
   private async autocomplete(searchText: string) {
-    if (!Config.Client.Search.autocompleteEnabled) {
+    if (!Config.Client.Search.AutoComplete.enabled) {
       return;
     }
     if (searchText.trim() === '.') {

frontend/app/model/notification.service.ts

@@ -1,4 +1,4 @@
-import {Injectable, ViewContainerRef} from '@angular/core';
+import {Injectable} from '@angular/core';
 import {ToastrService} from 'ngx-toastr';
 import {NetworkService} from './network/network.service';
 import {AuthenticationService} from './network/authentication.service';

frontend/app/model/page.helper.ts

@@ -11,14 +11,20 @@ export class PageHelper {
     return this.supportPageOffset ? window.pageYOffset : this.isCSS1Compat ? document.documentElement.scrollTop : document.body.scrollTop;
   }
 
-  public static get ScrollX(): number {
-    return this.supportPageOffset ? window.pageXOffset : this.isCSS1Compat ? document.documentElement.scrollLeft : document.body.scrollLeft;
-  }
-
   public static set ScrollY(value: number) {
     window.scrollTo(this.ScrollX, value);
   }
 
+  public static get MaxScrollY(): number {
+    return Math.max(document.body.scrollHeight, document.body.offsetHeight,
+      document.documentElement.clientHeight, document.documentElement.scrollHeight,
+      document.documentElement.offsetHeight) - window.innerHeight;
+  }
+
+  public static get ScrollX(): number {
+    return this.supportPageOffset ? window.pageXOffset : this.isCSS1Compat ? document.documentElement.scrollLeft : document.body.scrollLeft;
+  }
+
   public static showScrollY() {
     PageHelper.body.style.overflowY = 'scroll';
   }

frontend/app/model/query.service.ts

@@ -1,6 +1,5 @@
 import {Injectable} from '@angular/core';
 import {ShareService} from '../gallery/share.service';
-import {PhotoDTO} from '../../../common/entities/PhotoDTO';
 import {MediaDTO} from '../../../common/entities/MediaDTO';
 import {QueryParams} from '../../../common/QueryParams';
 import {Utils} from '../../../common/Utils';

frontend/app/pipes/FileSizePipe.ts

@@ -1,5 +1,4 @@
 import {Pipe, PipeTransform} from '@angular/core';
-import {I18n} from '@ngx-translate/i18n-polyfill';
 
 
 @Pipe({name: 'fileSize'})

frontend/app/settings/indexing/indexing.settings.component.ts

@@ -4,7 +4,7 @@ import {AuthenticationService} from '../../model/network/authentication.service'
 import {NavigationService} from '../../model/navigation.service';
 import {NotificationService} from '../../model/notification.service';
 import {ErrorDTO} from '../../../../common/entities/Error';
-import {Observable, interval} from 'rxjs';
+import {interval, Observable} from 'rxjs';
 import {IndexingConfig, ReIndexingSensitivity} from '../../../../common/config/private/IPrivateConfig';
 import {SettingsComponent} from '../_abstract/abstract.settings.component';
 import {Utils} from '../../../../common/Utils';
@@ -23,15 +23,46 @@ export class IndexingSettingsComponent extends SettingsComponent<IndexingConfig,
 
 
   types: { key: number; value: string }[] = [];
+  statistic: StatisticDTO;
   private subscription: { timer: any, settings: any } = {
     timer: null,
     settings: null
   };
-  statistic: StatisticDTO;
   private $counter: Observable<number> = null;
+
+  constructor(_authService: AuthenticationService,
+              _navigation: NavigationService,
+              _settingsService: IndexingSettingsService,
+              notification: NotificationService,
+              i18n: I18n) {
+
+    super(i18n('Indexing'),
+      _authService,
+      _navigation,
+      <any>_settingsService,
+      notification,
+      i18n,
+      s => s.Server.indexing);
+
+  }
+
+  get TimeLeft() {
+    const prg = this._settingsService.progress.value;
+    return (prg.time.current - prg.time.start) / prg.indexed * prg.left;
+  }
+
+  get TimeElapsed() {
+    const prg = this._settingsService.progress.value;
+    return (prg.time.current - prg.time.start);
+  }
+
   updateProgress = async () => {
     try {
+      const wasRunning = this._settingsService.progress.value !== null;
       await (<IndexingSettingsService>this._settingsService).getProgress();
+      if (wasRunning && this._settingsService.progress.value === null) {
+        this.notification.success(this.i18n('Folder indexed'), this.i18n('Success'));
+      }
     } catch (err) {
       if (this.subscription.timer != null) {
         this.subscription.timer.unsubscribe();
@@ -50,22 +81,6 @@ export class IndexingSettingsComponent extends SettingsComponent<IndexingConfig,
     }
   };
 
-  constructor(_authService: AuthenticationService,
-              _navigation: NavigationService,
-              _settingsService: IndexingSettingsService,
-              notification: NotificationService,
-              i18n: I18n) {
-
-    super(i18n('Indexing'),
-      _authService,
-      _navigation,
-      <any>_settingsService,
-      notification,
-      i18n,
-      s => s.Server.indexing);
-
-  }
-
   async ngOnInit() {
     super.ngOnInit();
     this.types = Utils
@@ -105,7 +120,7 @@ export class IndexingSettingsComponent extends SettingsComponent<IndexingConfig,
     try {
       await this._settingsService.index(createThumbnails);
       this.updateProgress();
-      this.notification.success(this.i18n('Folder indexed'), this.i18n('Success'));
+      this.notification.info(this.i18n('Folder indexing started'));
       this.inProgress = false;
       return true;
     } catch (err) {
@@ -119,21 +134,13 @@ export class IndexingSettingsComponent extends SettingsComponent<IndexingConfig,
     return false;
   }
 
-  get TimeLeft() {
-    const prg = this._settingsService.progress.value;
-    return (prg.time.current - prg.time.start) / prg.indexed * prg.left;
-  }
-  get TimeElapsed() {
-    const prg = this._settingsService.progress.value;
-    return (prg.time.current - prg.time.start);
-  }
-
   async cancelIndexing() {
     this.inProgress = true;
     this.error = '';
     try {
       await (<IndexingSettingsService>this._settingsService).cancel();
-      this.notification.success(this.i18n('Folder indexed'), this.i18n('Success'));
+      this._settingsService.progress.next(null);
+      this.notification.info(this.i18n('Folder indexing interrupted'));
       this.inProgress = false;
       return true;
     } catch (err) {

frontend/app/settings/settings.service.ts

@@ -15,12 +15,15 @@ export class SettingsService {
       Client: {
         Search: {
           enabled: true,
-          autocompleteEnabled: true,
+          AutoComplete: {
+            enabled: true,
+            cacheTimeout: 1000 * 60 * 60,
+            maxItemsPerCategory: 5
+          },
           instantSearchEnabled: true,
           InstantSearchTimeout: 0,
           searchCacheTimeout: 1000 * 60 * 60,
           instantSearchCacheTimeout: 1000 * 60 * 60,
-          autocompleteCacheTimeout: 1000 * 60 * 60
         },
         Thumbnail: {
           concurrentThumbnailGenerations: null,
@@ -88,7 +91,10 @@ export class SettingsService {
           folderPreviewSize: 0,
           reIndexingSensitivity: ReIndexingSensitivity.medium
         },
-        photoMetadataSize: 512 * 1024
+        photoMetadataSize: 512 * 1024,
+        duplicates: {
+          listingLimit: 1000
+        }
       }
     });
   }

gulpfile.js

@@ -1,195 +0,0 @@
-var ts = require('gulp-typescript');
-var gulp = require('gulp');
-var zip = require('gulp-zip');
-var fs = require('fs');
-var runSequence = require('run-sequence');
-var jsonModify = require('gulp-json-modify');
-var exec = require('child_process').exec;
-
-var translationFolder = "translate";
-var tsBackendProject = ts.createProject('tsconfig.json');
-gulp.task('build-backend', function () {
-  return gulp.src([
-    "common/**/*.ts",
-    "backend/**/*.ts"], {base: "."})
-    .pipe(tsBackendProject())
-    .js
-    .pipe(gulp.dest("./release"))
-
-});
-var createFrontendTask = function (type, script) {
-  gulp.task(type, function (cb) {
-    exec(script, function (err, stdout, stderr) {
-      console.log(stdout);
-      console.log(stderr);
-      cb(err);
-    });
-  });
-};
-
-gulp.task('build-frontend', function (done) {
-  var languages = getLanguages().filter(function (l) {
-    return l !== "en";
-  });
-  var tasks = [];
-  createFrontendTask('build-frontend-release default',
-    "ng build --aot --prod --output-path=./release/dist --no-progress --i18n-locale=en" +
-    " --i18n-format xlf --i18n-file frontend/" + translationFolder + "/messages.en.xlf" +
-    " --i18n-missing-translation warning");
-  tasks.push('build-frontend-release default');
-  for (var i = 0; i < languages.length; i++) {
-    createFrontendTask('build-frontend-release ' + languages[i],
-      "ng build --aot --prod --output-path=./release/dist/" + languages[i] +
-      " --no-progress --i18n-locale=" + languages[i] +
-      " --i18n-format xlf --i18n-file frontend/" + translationFolder + "/messages." + languages[i] + ".xlf" +
-      " --i18n-missing-translation warning");
-    tasks.push('build-frontend-release ' + languages[i]);
-  }
-  tasks.push(function () {
-    done();
-  });
-
-  runSequence.apply(this, tasks);
-
-});
-
-gulp.task('copy-static', function () {
-   return gulp.src([
-    "backend/model/diagnostics/blank.jpg",
-    "README.md",
-    "LICENSE"], {base: "."})
-    .pipe(gulp.dest('./release'));
-});
-
-gulp.task('copy-package', function () {
-  return gulp.src([
-    "package.json"], {base: "."})
-    .pipe(jsonModify({
-      key: 'devDependencies',
-      value: {}
-    }))
-    .pipe(jsonModify({
-      key: 'scripts',
-      value: {"start": "node ./backend/index.js"}
-    }))
-    .pipe(gulp.dest('./release'));
-});
-
-
-gulp.task('zip-release', function () {
-  return gulp.src(['release/**/*'], {base: "./release"})
-    .pipe(zip('pigallery2.zip'))
-    .pipe(gulp.dest('.'));
-});
-
-gulp.task('build-release', function (done) {
-  runSequence('build-frontend', 'build-backend', 'copy-static', 'copy-package', 'zip-release', function () {
-    done();
-  });
-});
-
-var getLanguages = function () {
-  if (!fs.existsSync("./frontend/" + translationFolder)) {
-    return [];
-  }
-  var dirCont = fs.readdirSync("./frontend/" + translationFolder);
-  var files = dirCont.filter(function (elm) {
-    return elm.match(/.*\.[a-zA-Z]+\.(xlf)/ig);
-  });
-  return files.map(function (f) {
-    return f.split(".")[1]
-  });
-};
-
-var simpleBuild = function (isProd) {
-  return function (done) {
-    var languages = getLanguages().filter(function (l) {
-      return l !== "en";
-    });
-    var tasks = [];
-    var cmd = "ng build --aot ";
-    if (isProd) {
-      cmd += " --prod --no-extract-licenses "
-    }
-    createFrontendTask('build-frontend default', cmd + "--output-path=./dist --no-progress --no-progress --i18n-locale en" +
-      " --i18n-format=xlf --i18n-file=frontend/" + translationFolder + "/messages.en.xlf" + " --i18n-missing-translation warning");
-    tasks.push('build-frontend default');
-    for (var i = 0; i < languages.length; i++) {
-      createFrontendTask('build-frontend ' + languages[i], cmd + "--output-path=./dist/" + languages[i] + " --no-progress --i18n-locale " + languages[i] +
-        " --i18n-format=xlf --i18n-file=frontend/" + translationFolder + "/messages." + languages[i] + ".xlf" + " --i18n-missing-translation warning");
-      tasks.push('build-frontend ' + languages[i]);
-    }
-    tasks.push(function () {
-      done();
-    });
-
-    runSequence.apply(this, tasks);
-  };
-};
-
-gulp.task("extract-locale", function (cb) {
-  console.log("creating source translation file:  locale.source.xlf");
-  exec('ng xi18n --out-file=./../locale.source.xlf  --i18n-format=xlf --i18n-locale=en', {maxBuffer: 1024 * 1024}, function (err, stdout, stderr) {
-    console.log(stdout);
-    console.log(stderr);
-    if (err) {
-      return cb(err);
-    }
-    exec('ngx-extractor -i frontend/**/*.ts -f xlf --out-file locale.source.xlf', function (err, stdout, stderr) {
-      console.log(stdout);
-      console.log(stderr);
-      cb(err);
-    });
-  });
-});
-
-var translate = function (list, cb) {
-  var localsStr = '"[\\"' + list.join('\\",\\"') + '\\"]"';
-  exec('xlf-google-translate --source-lang="en" --source-file="./locale.source.xlf" --destination-folder="./frontend/"' +
-    translationFolder + ' --destination-languages=' + localsStr, function (err, stdout, stderr) {
-    console.log(stdout);
-    console.log(stderr);
-    cb(err);
-  });
-};
-
-gulp.task("update-translation-only", function (cb) {
-  translate(getLanguages(), cb)
-});
-
-gulp.task("update-translation", function (done) {
-  runSequence('extract-locale', 'update-translation-only', function () {
-    done();
-  });
-});
-
-
-gulp.task("add-translation-only", function (cb) {
-  var languages = getLanguages();
-  var lng = null;
-  for (var i = 0; i < process.argv.length - 1; i++) {
-    if (process.argv[i] === "add-translation") {
-      lng = process.argv[i + 1].replace("--", "");
-    }
-  }
-  if (lng == null) {
-    console.error("Error: set language with '--' e.g: npm run add-translation -- --en");
-    return cb();
-  }
-  if (languages.indexOf(lng) !== -1) {
-    console.error("Error: language already exists, can't add. These language(s) already exist(s): " + languages);
-    return cb();
-  }
-  translate([lng], cb)
-});
-
-
-gulp.task("add-translation", function (done) {
-  runSequence('extract-locale', 'add-translation-only', function () {
-    done();
-  });
-});
-
-
-gulp.task('build-dev', simpleBuild(false));
-gulp.task('build-prod', simpleBuild(true));

gulpfile.ts

@@ -0,0 +1,181 @@
+import * as gulp from 'gulp';
+import * as fs from 'fs';
+import * as zip from 'gulp-zip';
+import * as ts from 'gulp-typescript';
+// @ts-ignore
+import * as jsonModify from 'gulp-json-modify';
+
+
+const exec = require('child_process').exec;
+
+const translationFolder = 'translate';
+const tsBackendProject = ts.createProject('tsconfig.json');
+
+gulp.task('build-backend', function () {
+  return gulp.src([
+    'common/**/*.ts',
+    'backend/**/*.ts'], {base: '.'})
+    .pipe(tsBackendProject())
+    .js
+    .pipe(gulp.dest('./release'));
+
+});
+
+const handleError = (cb: (err: any) => void) => {
+  return (err: any, stdout: string, stderr: string) => {
+    console.log(stdout);
+    console.log(stderr);
+    cb(err);
+  };
+};
+
+const createFrontendTask = (type: string, script: string) => {
+  gulp.task(type, (cb) => {
+    exec(script, handleError(cb));
+  });
+};
+
+const getLanguages = () => {
+  if (!fs.existsSync('./frontend/' + translationFolder)) {
+    return [];
+  }
+  const dirCont = fs.readdirSync('./frontend/' + translationFolder);
+  const files: string[] = dirCont.filter((elm) => {
+    return elm.match(/.*\.[a-zA-Z]+\.(xlf)/ig);
+  });
+  return files.map((f: string) => {
+    return f.split('.')[1];
+  });
+};
+
+gulp.task('build-frontend', (() => {
+  const languages = getLanguages().filter((l) => {
+    return l !== 'en';
+  });
+  const tasks = [];
+  createFrontendTask('build-frontend-release default',
+    'ng build --aot --prod --output-path=./release/dist --no-progress --i18n-locale=en' +
+    ' --i18n-format xlf --i18n-file frontend/' + translationFolder + '/messages.en.xlf' +
+    ' --i18n-missing-translation warning');
+  tasks.push('build-frontend-release default');
+  for (let i = 0; i < languages.length; i++) {
+    createFrontendTask('build-frontend-release ' + languages[i],
+      'ng build --aot --prod --output-path=./release/dist/' + languages[i] +
+      ' --no-progress --i18n-locale=' + languages[i] +
+      ' --i18n-format xlf --i18n-file frontend/' + translationFolder + '/messages.' + languages[i] + '.xlf' +
+      ' --i18n-missing-translation warning');
+    tasks.push('build-frontend-release ' + languages[i]);
+  }
+  return gulp.series(...tasks);
+})());
+
+gulp.task('copy-static', function () {
+  return gulp.src([
+    'backend/model/diagnostics/blank.jpg',
+    'README.md',
+    'LICENSE'], {base: '.'})
+    .pipe(gulp.dest('./release'));
+});
+
+gulp.task('copy-package', function () {
+  return gulp.src([
+    'package.json'], {base: '.'})
+    .pipe(jsonModify({
+      key: 'devDependencies',
+      value: {}
+    }))
+    .pipe(jsonModify({
+      key: 'scripts',
+      value: {'start': 'node ./backend/index.js'}
+    }))
+    .pipe(gulp.dest('./release'));
+});
+
+
+gulp.task('zip-release', function () {
+  return gulp.src(['release/**/*'], {base: './release'})
+    .pipe(zip('pigallery2.zip'))
+    .pipe(gulp.dest('.'));
+});
+
+gulp.task('build-release', gulp.series('build-frontend', 'build-backend', 'copy-static', 'copy-package', 'zip-release'));
+
+
+const simpleBuild = (isProd: boolean) => {
+  const languages = getLanguages().filter(function (l) {
+    return l !== 'en';
+  });
+  const tasks = [];
+  let cmd = 'ng build --aot ';
+  if (isProd) {
+    cmd += ' --prod --no-extract-licenses ';
+  }
+  createFrontendTask('build-frontend default', cmd + '--output-path=./dist --no-progress --no-progress --i18n-locale en' +
+    ' --i18n-format=xlf --i18n-file=frontend/' + translationFolder + '/messages.en.xlf' + ' --i18n-missing-translation warning');
+  tasks.push('build-frontend default');
+  if (!process.env.CI) { // don't build languages if running in CI
+    for (let i = 0; i < languages.length; i++) {
+      createFrontendTask('build-frontend ' + languages[i], cmd +
+        '--output-path=./dist/' + languages[i] +
+        ' --no-progress --i18n-locale ' + languages[i] +
+        ' --i18n-format=xlf --i18n-file=frontend/' + translationFolder +
+        '/messages.' + languages[i] + '.xlf' + ' --i18n-missing-translation warning');
+      tasks.push('build-frontend ' + languages[i]);
+    }
+  }
+  return gulp.series(...tasks);
+};
+
+gulp.task('extract-locale', (cb) => {
+  console.log('creating source translation file:  locale.source.xlf');
+  exec('ng xi18n --out-file=./../locale.source.xlf  --i18n-format=xlf --i18n-locale=en',
+    {maxBuffer: 1024 * 1024}, (error: any, stdOut: string, stdErr: string) => {
+      console.log(stdOut);
+      console.log(stdErr);
+      if (error) {
+        return cb(error);
+      }
+      exec('ngx-extractor -i frontend/**/*.ts -f xlf --out-file locale.source.xlf',
+        handleError(cb));
+    });
+});
+
+const translate = (list: any[], cb: (err: any) => void) => {
+  const localsStr = '"[\\"' + list.join('\\",\\"') + '\\"]"';
+  exec('xlf-google-translate --source-lang="en" --source-file="./locale.source.xlf" --destination-folder="./frontend/"' +
+    translationFolder + ' --destination-languages=' + localsStr,
+    handleError(cb));
+};
+
+gulp.task('update-translation-only', function (cb) {
+  translate(getLanguages(), cb);
+});
+
+gulp.task('update-translation', gulp.series('extract-locale', 'update-translation-only'));
+
+
+gulp.task('add-translation-only', (cb) => {
+  const languages = getLanguages();
+  let lng = null;
+  for (let i = 0; i < process.argv.length - 1; i++) {
+    if (process.argv[i] === 'add-translation') {
+      lng = process.argv[i + 1].replace('--', '');
+    }
+  }
+  if (lng == null) {
+    console.error('Error: set language with \'--\' e.g: npm run add-translation -- --en');
+    return cb();
+  }
+  if (languages.indexOf(lng) !== -1) {
+    console.error('Error: language already exists, can\'t add. These language(s) already exist(s): ' + languages);
+    return cb();
+  }
+  translate([lng], cb);
+});
+
+
+gulp.task('add-translation', gulp.series('extract-locale', 'add-translation-only'));
+
+
+gulp.task('build-dev', simpleBuild(false));
+gulp.task('build-prod', simpleBuild(true));

package.json

@@ -1,6 +1,6 @@
 {
   "name": "pigallery2",
-  "version": "1.5.6",
+  "version": "1.5.8",
   "description": "This is a photo gallery optimised for running low resource servers (especially on raspberry pi)",
   "author": "Patrik J. Braun",
   "homepage": "https://github.com/bpatrik/PiGallery2",
@@ -30,55 +30,60 @@
     "cookie-parser": "1.4.3",
     "cookie-session": "2.0.0-beta.3",
     "ejs": "2.6.1",
+    "exifreader": "2.6.0",
     "express": "4.16.4",
     "fluent-ffmpeg": "2.1.2",
-    "image-size": "0.6.3",
+    "image-size": "0.7.1",
+    "jdataview": "2.5.0",
     "jimp": "0.6.0",
     "locale": "0.1.0",
-    "reflect-metadata": "0.1.12",
-    "sqlite3": "4.0.4",
+    "reflect-metadata": "0.1.13",
+    "sqlite3": "4.0.6",
     "ts-exif-parser": "0.1.4",
     "ts-node-iptc": "1.0.11",
     "typeconfig": "1.0.7",
-    "typeorm": "0.2.9",
-    "winston": "2.4.2"
+    "typeorm": "0.2.12",
+    "winston": "2.4.4",
+    "xmldom": "0.1.27"
   },
   "devDependencies": {
-    "@angular-devkit/build-angular": "0.11.4",
-    "@angular-devkit/build-optimizer": "0.11.4",
-    "@angular/animations": "7.1.4",
-    "@angular/cli": "7.1.4",
-    "@angular/common": "7.1.4",
-    "@angular/compiler": "7.1.4",
-    "@angular/compiler-cli": "7.1.4",
-    "@angular/core": "7.1.4",
-    "@angular/forms": "7.1.4",
-    "@angular/http": "7.1.4",
-    "@angular/language-service": "7.1.4",
-    "@angular/platform-browser": "7.1.4",
-    "@angular/platform-browser-dynamic": "7.1.4",
-    "@angular/router": "7.1.4",
+    "@angular-devkit/build-angular": "0.12.3",
+    "@angular-devkit/build-optimizer": "0.12.3",
+    "@angular/animations": "7.2.2",
+    "@angular/cli": "7.2.3",
+    "@angular/common": "7.2.2",
+    "@angular/compiler": "7.2.2",
+    "@angular/compiler-cli": "7.2.2",
+    "@angular/core": "7.2.2",
+    "@angular/forms": "7.2.2",
+    "@angular/http": "7.2.2",
+    "@angular/language-service": "7.2.2",
+    "@angular/platform-browser": "7.2.2",
+    "@angular/platform-browser-dynamic": "7.2.2",
+    "@angular/router": "7.2.2",
     "@ngx-translate/i18n-polyfill": "1.0.0",
     "@types/bcryptjs": "2.4.2",
     "@types/chai": "4.1.7",
     "@types/cookie-parser": "1.4.1",
     "@types/cookie-session": "2.0.36",
     "@types/ejs": "2.6.1",
-    "@types/express": "4.16.0",
+    "@types/express": "4.16.1",
     "@types/fluent-ffmpeg": "2.1.9",
     "@types/gm": "1.18.2",
-    "@types/image-size": "0.0.29",
-    "@types/jasmine": "3.3.5",
+    "@types/gulp": "^4.0.5",
+    "@types/gulp-zip": "^4.0.0",
+    "@types/image-size": "0.7.0",
+    "@types/jasmine": "3.3.8",
     "@types/node": "10.12.18",
-    "@types/sharp": "0.21.0",
-    "@types/winston": "2.3.9",
+    "@types/sharp": "0.21.2",
+    "@types/winston": "2.4.4",
     "@yaga/leaflet-ng2": "1.0.0",
     "bootstrap": "4.1.3",
     "chai": "4.2.0",
     "codelyzer": "4.5.0",
-    "core-js": "2.6.1",
+    "core-js": "2.6.3",
     "ejs-loader": "0.3.1",
-    "gulp": "3.9.1",
+    "gulp": "4.0.0",
     "gulp-json-modify": "1.0.2",
     "gulp-typescript": "5.0.0",
     "gulp-zip": "4.2.0",
@@ -86,8 +91,8 @@
     "intl": "1.2.5",
     "jasmine-core": "3.3.0",
     "jasmine-spec-reporter": "4.2.1",
-    "jw-bootstrap-switch-ng2": "2.0.2",
-    "karma": "3.1.4",
+    "jw-bootstrap-switch-ng2": "2.0.4",
+    "karma": "4.0.0",
     "karma-chrome-launcher": "2.2.0",
     "karma-cli": "2.0.0",
     "karma-coverage-istanbul-reporter": "2.0.4",
@@ -103,18 +108,18 @@
     "ngx-clipboard": "11.1.9",
     "ngx-toastr": "9.1.1",
     "open-iconic": "1.1.1",
-    "protractor": "5.4.1",
-    "remap-istanbul": "0.12.0",
-    "rimraf": "2.6.2",
+    "protractor": "5.4.2",
+    "remap-istanbul": "0.13.0",
+    "rimraf": "2.6.3",
     "run-sequence": "2.2.1",
     "rxjs": "6.3.3",
     "rxjs-compat": "^6.3.3",
     "ts-helpers": "1.1.2",
-    "ts-node": "7.0.1",
-    "tslint": "5.12.0",
-    "typescript": "3.1.6",
+    "ts-node": "8.0.2",
+    "tslint": "5.12.1",
+    "typescript": "3.2.4",
     "xlf-google-translate": "1.0.0-beta.13",
-    "zone.js": "0.8.26"
+    "zone.js": "0.8.29"
   },
   "resolutions": {
     "natives": "1.1.3"
@@ -122,10 +127,10 @@
   "optionalDependencies": {
     "@ffmpeg-installer/ffmpeg": "1.0.17",
     "@ffprobe-installer/ffprobe": "1.0.9",
-    "bcrypt": "3.0.2",
+    "bcrypt": "3.0.3",
     "gm": "1.23.1",
     "mysql": "2.16.0",
-    "sharp": "0.21.1"
+    "sharp": "0.21.3"
   },
   "engines": {
     "node": ">= 6.9 <11.0"

test/backend/SQLTestHelper.ts

@@ -0,0 +1,107 @@
+import {Config} from '../../common/config/private/Config';
+import {DatabaseType} from '../../common/config/private/IPrivateConfig';
+import * as fs from 'fs';
+import * as path from 'path';
+import {SQLConnection} from '../../backend/model/sql/SQLConnection';
+
+declare let describe: any;
+const savedDescribe = describe;
+
+export class SQLTestHelper {
+
+  static enable = {
+    sqlite: true,
+    mysql: true
+  };
+  public static readonly savedDescribe = savedDescribe;
+  tempDir: string;
+  dbPath: string;
+
+  constructor(public dbType: DatabaseType) {
+    this.tempDir = path.resolve(__dirname, './tmp');
+    this.dbPath = path.resolve(__dirname, './tmp', 'test.db');
+
+  }
+
+  static describe(name: string, tests: (helper?: SQLTestHelper) => void) {
+    savedDescribe(name, async () => {
+      if (SQLTestHelper.enable.sqlite) {
+        const helper = new SQLTestHelper(DatabaseType.sqlite);
+        savedDescribe('sqlite', () => {
+          return tests(helper);
+        });
+      }
+      if (SQLTestHelper.enable.mysql) {
+        const helper = new SQLTestHelper(DatabaseType.mysql);
+        savedDescribe('mysql', function () {
+          this.timeout(99999999);
+          // @ts-ignore
+          return tests(helper);
+        });
+      }
+    });
+  }
+
+  public async initDB() {
+    if (this.dbType === DatabaseType.sqlite) {
+      await this.initSQLite();
+    } else {
+      await this.initMySQL();
+    }
+  }
+
+
+  public async clearDB() {
+    if (this.dbType === DatabaseType.sqlite) {
+      await this.clearUpSQLite();
+    } else {
+      await this.clearUpMysql();
+    }
+  }
+
+  private async initSQLite() {
+    await this.resetSQLite();
+
+    Config.Server.database.type = DatabaseType.sqlite;
+    Config.Server.database.sqlite.storage = this.dbPath;
+  }
+
+  private async initMySQL() {
+    Config.Server.database.type = DatabaseType.mysql;
+    Config.Server.database.mysql.database = 'pigallery2_test';
+
+    await this.resetMySQL();
+  }
+
+  private async resetSQLite() {
+    await SQLConnection.close();
+
+    if (fs.existsSync(this.dbPath)) {
+      fs.unlinkSync(this.dbPath);
+    }
+    if (fs.existsSync(this.tempDir)) {
+      fs.rmdirSync(this.tempDir);
+    }
+  }
+
+  private async resetMySQL() {
+    Config.Server.database.type = DatabaseType.mysql;
+    Config.Server.database.mysql.database = 'pigallery2_test';
+    const conn = await SQLConnection.getConnection();
+    await conn.query('DROP DATABASE IF EXISTS ' + conn.options.database);
+    await conn.query('CREATE DATABASE IF NOT EXISTS ' + conn.options.database);
+    await SQLConnection.close();
+  }
+
+  private async clearUpMysql() {
+    Config.Server.database.type = DatabaseType.mysql;
+    Config.Server.database.mysql.database = 'pigallery2_test';
+    const conn = await SQLConnection.getConnection();
+    await conn.query('DROP DATABASE IF EXISTS ' + conn.options.database);
+    await SQLConnection.close();
+  }
+
+  private async clearUpSQLite() {
+    return this.resetSQLite();
+  }
+}

test/backend/integration/model/sql/typeorm.ts

@@ -16,7 +16,6 @@ import {
   PositionMetaDataEntity
 } from '../../../../../backend/model/sql/enitites/PhotoEntity';
 import {MediaDimensionEntity} from '../../../../../backend/model/sql/enitites/MediaEntity';
-import {DataStructureVersion} from '../../../../../common/DataStructureVersion';
 import {VersionEntity} from '../../../../../backend/model/sql/enitites/VersionEntity';
 
 describe('Typeorm integration', () => {

test/backend/unit/assets/old_photo.json

@@ -0,0 +1,17 @@
+{
+  "cameraData": {
+    "ISO": 200,
+    "exposure": 0.008,
+    "fStop": 2.8,
+    "focalLength": 9.4,
+    "make": "FUJIFILM",
+    "model": "FinePix F601 ZOOM"
+  },
+  "creationDate": 1126455782000,
+  "fileSize": 2582,
+  "orientation": 1,
+  "size": {
+    "height": 5,
+    "width": 7
+  }
+}

test/backend/unit/assets/test image öüóőúéáű-.,.json

@@ -0,0 +1,54 @@
+{
+  "cameraData": {
+    "ISO": 3200,
+    "exposure": 0.00125,
+    "fStop": 5.6,
+    "focalLength": 85,
+    "lens": "EF-S15-85mm f/3.5-5.6 IS USM",
+    "make": "Canon",
+    "model": "óüöúőűáé ÓÜÖÚŐŰÁÉ"
+  },
+  "caption": "Test caption",
+  "creationDate": 1434018566000,
+  "faces": [
+    {
+      "box": {
+        "height": 2,
+        "width": 2,
+        "x": 8,
+        "y": 4
+      },
+      "name": "squirrel"
+    },
+    {
+      "box": {
+        "height": 3,
+        "width": 2,
+        "x": 5,
+        "y": 5
+      },
+      "name": "special_chars űáéúőóüío?._:"
+    }
+  ],
+  "fileSize": 39424,
+  "keywords": [
+    "Berkley",
+    "USA",
+    "űáéúőóüö ŰÁÉÚŐÓÜÖ"
+  ],
+  "orientation": 1,
+  "positionData": {
+    "GPSData": {
+      "altitude": 90,
+      "latitude": 37.871093333333334,
+      "longitude": -122.25678
+    },
+    "city": "test city őúéáűóöí-.,)(=",
+    "country": "test country őúéáűóöí-.,)(=/%!+\"'",
+    "state": "test state őúéáűóöí-.,)("
+  },
+  "size": {
+    "height": 10,
+    "width": 14
+  }
+}

test/backend/unit/model/sql/GalleryManager.ts

@@ -1,284 +1,58 @@
 import {expect} from 'chai';
-import * as fs from 'fs';
-import * as path from 'path';
-import {Config} from '../../../../../common/config/private/Config';
-import {DatabaseType, ReIndexingSensitivity} from '../../../../../common/config/private/IPrivateConfig';
-import {SQLConnection} from '../../../../../backend/model/sql/SQLConnection';
-import {GalleryManager} from '../../../../../backend/model/sql/GalleryManager';
-import {DirectoryDTO} from '../../../../../common/entities/DirectoryDTO';
 import {TestHelper} from './TestHelper';
-import {Connection} from 'typeorm';
-import {DirectoryEntity} from '../../../../../backend/model/sql/enitites/DirectoryEntity';
+import {SQLTestHelper} from '../../../SQLTestHelper';
+import {GalleryManager} from '../../../../../backend/model/sql/GalleryManager';
+import {IndexingManager} from '../../../../../backend/model/sql/IndexingManager';
+import {DirectoryDTO} from '../../../../../common/entities/DirectoryDTO';
 import {Utils} from '../../../../../common/Utils';
-import {MediaDTO} from '../../../../../common/entities/MediaDTO';
-import {FileDTO} from '../../../../../common/entities/FileDTO';
+import {ObjectManagerRepository} from '../../../../../backend/model/ObjectManagerRepository';
+import {PersonManager} from '../../../../../backend/model/sql/PersonManager';
+import {MediaEntity} from '../../../../../backend/model/sql/enitites/MediaEntity';
 
-class GalleryManagerTest extends GalleryManager {
+class IndexingManagerTest extends IndexingManager {
 
-
-  public async selectParentDir(connection: Connection, directoryName: string, directoryParent: string): Promise<DirectoryEntity> {
-    return super.selectParentDir(connection, directoryName, directoryParent);
-  }
-
-  public async fillParentDir(connection: Connection, dir: DirectoryEntity): Promise<void> {
-    return super.fillParentDir(connection, dir);
-  }
-
-  public async saveToDB(scannedDirectory: DirectoryDTO) {
+  public async saveToDB(scannedDirectory: DirectoryDTO): Promise<void> {
     return super.saveToDB(scannedDirectory);
   }
-
-  public async queueForSave(scannedDirectory: DirectoryDTO): Promise<void> {
-    return super.queueForSave(scannedDirectory);
-  }
 }
 
-describe('GalleryManager', () => {
+// to help WebStorm to handle the test cases
+declare let describe: any;
+declare const after: any;
+describe = SQLTestHelper.describe;
 
+describe('GalleryManager', (sqlHelper: SQLTestHelper) => {
 
-  const tempDir = path.join(__dirname, '../../tmp');
-  const dbPath = path.join(tempDir, 'test.db');
-
-
-  const setUpSqlDB = async () => {
-    if (fs.existsSync(dbPath)) {
-      fs.unlinkSync(dbPath);
-    }
-    if (!fs.existsSync(tempDir)) {
-      fs.mkdirSync(tempDir);
-    }
-
-    Config.Server.database.type = DatabaseType.sqlite;
-    Config.Server.database.sqlite.storage = dbPath;
-
-  };
-
-  const tearDownSqlDB = async () => {
-    await SQLConnection.close();
-    if (fs.existsSync(dbPath)) {
-      fs.unlinkSync(dbPath);
-    }
-    if (fs.existsSync(tempDir)) {
-      fs.rmdirSync(tempDir);
-    }
-  };
 
   beforeEach(async () => {
-    await setUpSqlDB();
+    await sqlHelper.initDB();
+    ObjectManagerRepository.getInstance().PersonManager = new PersonManager();
   });
 
-  afterEach(async () => {
-    await tearDownSqlDB();
+
+  after(async () => {
+    await sqlHelper.clearDB();
   });
 
-  const removeIds = (dir: DirectoryDTO) => {
-    delete dir.id;
-    dir.media.forEach((media: MediaDTO) => {
-      delete media.id;
-    });
-    if (dir.metaFile) {
-      if (dir.metaFile.length === 0) {
-        delete dir.metaFile;
-      } else {
-        dir.metaFile.forEach((file: FileDTO) => {
-          delete file.id;
-        });
-      }
-    }
-    if (dir.directories) {
-      dir.directories.forEach((directory: DirectoryDTO) => {
-        removeIds(directory);
-      });
-    }
-  };
-
-  it('should save parent directory', async () => {
-    const gm = new GalleryManagerTest();
+  it('should get random photo', async () => {
+    const gm = new GalleryManager();
+    const im = new IndexingManagerTest();
 
     const parent = TestHelper.getRandomizedDirectoryEntry();
     const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
-    const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2');
-    const gpx = TestHelper.getRandomizedGPXEntry(parent, 'GPX1');
-    const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir');
-    const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1');
-    const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2');
-
-
+    expect(await gm.getRandomPhoto({})).to.not.exist;
     DirectoryDTO.removeReferences(parent);
-    await gm.saveToDB(Utils.clone(parent));
+    await im.saveToDB(Utils.clone(parent));
 
-    const conn = await SQLConnection.getConnection();
-    const selected = await gm.selectParentDir(conn, parent.name, parent.path);
-    await gm.fillParentDir(conn, selected);
-
-    DirectoryDTO.removeReferences(selected);
-    removeIds(selected);
-    subDir.isPartial = true;
-    delete subDir.directories;
-    delete subDir.metaFile;
-    expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
-      .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent)));
-  });
-
-  it('should skip meta files', async () => {
-    const gm = new GalleryManagerTest();
-    const parent = TestHelper.getRandomizedDirectoryEntry();
-    const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
-    const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2');
-    const gpx = TestHelper.getRandomizedGPXEntry(parent, 'GPX1');
-    DirectoryDTO.removeReferences(parent);
-    Config.Client.MetaFile.enabled = true;
-    await gm.saveToDB(Utils.clone(parent));
-
-    Config.Client.MetaFile.enabled = false;
-    const conn = await SQLConnection.getConnection();
-    const selected = await gm.selectParentDir(conn, parent.name, parent.path);
-    await gm.fillParentDir(conn, selected);
-
-    delete parent.metaFile;
-    DirectoryDTO.removeReferences(selected);
-    removeIds(selected);
-    expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
-      .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent)));
-  });
-
-  it('should update sub directory', async () => {
-    const gm = new GalleryManagerTest();
-
-    const parent = TestHelper.getRandomizedDirectoryEntry();
-    parent.name = 'parent';
-    const p1 = TestHelper.getRandomizedPhotoEntry(parent);
-    const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir');
-    subDir.name = 'subDir';
-    const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1');
-
-    DirectoryDTO.removeReferences(parent);
-    await gm.saveToDB(Utils.clone(parent));
-
-    const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2');
-    const sp3 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto3');
-
-    DirectoryDTO.removeReferences(subDir);
-    await gm.saveToDB(Utils.clone(subDir));
-
-    const conn = await SQLConnection.getConnection();
-    const selected = await gm.selectParentDir(conn, subDir.name, subDir.path);
-    await gm.fillParentDir(conn, selected);
-
-    // subDir.isPartial = true;
-    //  delete subDir.directories;
-    DirectoryDTO.removeReferences(selected);
-    delete subDir.parent;
-    delete subDir.metaFile;
-    removeIds(selected);
-    // selected.directories[0].parent = selected;
-    expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
-      .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(subDir)));
-  });
-
-  it('should avoid race condition', async () => {
-    const conn = await SQLConnection.getConnection();
-    const gm = new GalleryManagerTest();
-    Config.Client.MetaFile.enabled = true;
-    const parent = TestHelper.getRandomizedDirectoryEntry();
-    const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
-    const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2');
-    const gpx = TestHelper.getRandomizedGPXEntry(parent, 'GPX1');
-    const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir');
-    const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1');
-    const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2');
-
-
-    DirectoryDTO.removeReferences(parent);
-    const s1 = gm.queueForSave(Utils.clone(parent));
-    const s2 = gm.queueForSave(Utils.clone(parent));
-    const s3 = gm.queueForSave(Utils.clone(parent));
-
-    await Promise.all([s1, s2, s3]);
-
-    const selected = await gm.selectParentDir(conn, parent.name, parent.path);
-    await gm.fillParentDir(conn, selected);
-
-    DirectoryDTO.removeReferences(selected);
-    removeIds(selected);
-    subDir.isPartial = true;
-    delete subDir.directories;
-    delete subDir.metaFile;
-    expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
-      .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent)));
-  });
-
-
-  (<any>it('should save 1500 photos', async () => {
-    const conn = await SQLConnection.getConnection();
-    const gm = new GalleryManagerTest();
-    Config.Client.MetaFile.enabled = true;
-    const parent = TestHelper.getRandomizedDirectoryEntry();
-    DirectoryDTO.removeReferences(parent);
-    await gm.saveToDB(Utils.clone(parent));
-    const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir');
-    for (let i = 0; i < 1500; i++) {
-      TestHelper.getRandomizedPhotoEntry(subDir, 'p' + i);
-    }
-
-    DirectoryDTO.removeReferences(parent);
-    await gm.saveToDB(subDir);
-
-
-    const selected = await gm.selectParentDir(conn, subDir.name, subDir.path);
-    expect(selected.media.length).to.deep.equal(subDir.media.length);
-  })).timeout(20000);
-
-  describe('Test listDirectory', () => {
-    const statSync = fs.statSync;
-    let dirTime = 0;
-    const indexedTime = {
-      lastScanned: 0,
-      lastModified: 0
-    };
-
-    beforeEach(() => {
-      dirTime = 0;
-      indexedTime.lastModified = 0;
-      indexedTime.lastScanned = 0;
-    });
-
-    afterEach(() => {
-      // @ts-ignore
-      fs.statSync = statSync;
-    });
-
-    it('with re indexing severity low', async () => {
-      Config.Server.indexing.reIndexingSensitivity = ReIndexingSensitivity.low;
-
-      // @ts-ignore
-      fs.statSync = () => ({ctime: new Date(dirTime), mtime: new Date(dirTime)});
-      const gm = new GalleryManagerTest();
-      gm.selectParentDir = (connection: Connection, directoryName: string, directoryParent: string) => {
-        return Promise.resolve(<any>indexedTime);
-      };
-      gm.fillParentDir = (connection: Connection, dir: DirectoryEntity) => {
-        return Promise.resolve();
-      };
-
-      gm.indexDirectory = (...args) => {
-        return <any>Promise.resolve('indexing');
-      };
-
-      indexedTime.lastScanned = null;
-      expect(await gm.listDirectory('./')).to.be.equal('indexing');
-      indexedTime.lastModified = 0;
-      dirTime = 1;
-      expect(await gm.listDirectory('./')).to.be.equal('indexing');
-      indexedTime.lastScanned = 10;
-      indexedTime.lastModified = 1;
-      dirTime = 1;
-      expect(await gm.listDirectory('./')).to.be.equal(indexedTime);
-      expect(await gm.listDirectory('./', 1, 10))
-        .to.be.equal(null);
-
-
-    });
+    delete p1.metadata.faces;
+    delete p1.directory;
+    delete p1.id;
+    const found: MediaEntity = <any>await gm.getRandomPhoto({});
+    delete found.metadata.bitRate;
+    delete found.metadata.duration;
+    delete found.directory;
+    delete found.id;
+    expect(Utils.clone(found)).to.be.deep.equal(Utils.clone(p1));
   });
 
 });

test/backend/unit/model/sql/IndexingManager.ts

@@ -0,0 +1,281 @@
+import {expect} from 'chai';
+import * as fs from 'fs';
+import {Config} from '../../../../../common/config/private/Config';
+import {ReIndexingSensitivity} from '../../../../../common/config/private/IPrivateConfig';
+import {SQLConnection} from '../../../../../backend/model/sql/SQLConnection';
+import {GalleryManager} from '../../../../../backend/model/sql/GalleryManager';
+import {DirectoryDTO} from '../../../../../common/entities/DirectoryDTO';
+import {TestHelper} from './TestHelper';
+import {Connection} from 'typeorm';
+import {DirectoryEntity} from '../../../../../backend/model/sql/enitites/DirectoryEntity';
+import {Utils} from '../../../../../common/Utils';
+import {MediaDTO} from '../../../../../common/entities/MediaDTO';
+import {FileDTO} from '../../../../../common/entities/FileDTO';
+import {IndexingManager} from '../../../../../backend/model/sql/IndexingManager';
+import {ObjectManagerRepository} from '../../../../../backend/model/ObjectManagerRepository';
+import {PersonManager} from '../../../../../backend/model/sql/PersonManager';
+import {SQLTestHelper} from '../../../SQLTestHelper';
+
+class GalleryManagerTest extends GalleryManager {
+
+
+  public async selectParentDir(connection: Connection, directoryName: string, directoryParent: string): Promise<DirectoryEntity> {
+    return super.selectParentDir(connection, directoryName, directoryParent);
+  }
+
+  public async fillParentDir(connection: Connection, dir: DirectoryEntity): Promise<void> {
+    return super.fillParentDir(connection, dir);
+  }
+
+}
+
+class IndexingManagerTest extends IndexingManager {
+
+
+  public async queueForSave(scannedDirectory: DirectoryDTO): Promise<void> {
+    return super.queueForSave(scannedDirectory);
+  }
+
+  public async saveToDB(scannedDirectory: DirectoryDTO): Promise<void> {
+    return super.saveToDB(scannedDirectory);
+  }
+}
+
+// to help WebStorm to handle the test cases
+declare let describe: any;
+declare const after: any;
+describe = SQLTestHelper.describe;
+
+describe('IndexingManager', (sqlHelper: SQLTestHelper) => {
+
+
+  beforeEach(async () => {
+    await sqlHelper.initDB();
+    ObjectManagerRepository.getInstance().PersonManager = new PersonManager();
+  });
+
+
+  after(async () => {
+    await sqlHelper.clearDB();
+  });
+
+  const removeIds = (dir: DirectoryDTO) => {
+    delete dir.id;
+    dir.media.forEach((media: MediaDTO) => {
+      delete media.id;
+    });
+    if (dir.metaFile) {
+      if (dir.metaFile.length === 0) {
+        delete dir.metaFile;
+      } else {
+        dir.metaFile.forEach((file: FileDTO) => {
+          delete file.id;
+        });
+      }
+    }
+    if (dir.directories) {
+      dir.directories.forEach((directory: DirectoryDTO) => {
+        removeIds(directory);
+      });
+    }
+  };
+
+  it('should save parent directory', async () => {
+    const gm = new GalleryManagerTest();
+    const im = new IndexingManagerTest();
+
+    const parent = TestHelper.getRandomizedDirectoryEntry();
+    const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
+    const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2');
+    const gpx = TestHelper.getRandomizedGPXEntry(parent, 'GPX1');
+    const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir');
+    const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1', 0);
+    const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2', 0);
+
+
+    DirectoryDTO.removeReferences(parent);
+    await im.saveToDB(Utils.clone(parent));
+
+    const conn = await SQLConnection.getConnection();
+    const selected = await gm.selectParentDir(conn, parent.name, parent.path);
+    await gm.fillParentDir(conn, selected);
+
+    DirectoryDTO.removeReferences(selected);
+    removeIds(selected);
+    subDir.isPartial = true;
+    delete subDir.directories;
+    delete subDir.metaFile;
+    expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
+      .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent)));
+  });
+
+  it('should skip meta files', async () => {
+    const gm = new GalleryManagerTest();
+    const im = new IndexingManagerTest();
+    const parent = TestHelper.getRandomizedDirectoryEntry();
+    const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
+    const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2');
+    const gpx = TestHelper.getRandomizedGPXEntry(parent, 'GPX1');
+    DirectoryDTO.removeReferences(parent);
+    Config.Client.MetaFile.enabled = true;
+    await im.saveToDB(Utils.clone(parent));
+
+    Config.Client.MetaFile.enabled = false;
+    const conn = await SQLConnection.getConnection();
+    const selected = await gm.selectParentDir(conn, parent.name, parent.path);
+    await gm.fillParentDir(conn, selected);
+
+    delete parent.metaFile;
+    DirectoryDTO.removeReferences(selected);
+    removeIds(selected);
+    expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
+      .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent)));
+  });
+
+  it('should update sub directory', async () => {
+    const gm = new GalleryManagerTest();
+    const im = new IndexingManagerTest();
+
+    const parent = TestHelper.getRandomizedDirectoryEntry();
+    parent.name = 'parent';
+    const p1 = TestHelper.getRandomizedPhotoEntry(parent);
+    const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir');
+    subDir.name = 'subDir';
+    const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1');
+
+    DirectoryDTO.removeReferences(parent);
+    await im.saveToDB(Utils.clone(parent));
+
+    const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2');
+    const sp3 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto3');
+
+    DirectoryDTO.removeReferences(subDir);
+    await im.saveToDB(Utils.clone(subDir));
+
+    const conn = await SQLConnection.getConnection();
+    const selected = await gm.selectParentDir(conn, subDir.name, subDir.path);
+    await gm.fillParentDir(conn, selected);
+
+    // subDir.isPartial = true;
+    //  delete subDir.directories;
+    DirectoryDTO.removeReferences(selected);
+    delete subDir.parent;
+    delete subDir.metaFile;
+    removeIds(selected);
+    // selected.directories[0].parent = selected;
+    expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
+      .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(subDir)));
+  });
+
+  it('should avoid race condition', async () => {
+    const conn = await SQLConnection.getConnection();
+    const gm = new GalleryManagerTest();
+    const im = new IndexingManagerTest();
+    Config.Client.MetaFile.enabled = true;
+    const parent = TestHelper.getRandomizedDirectoryEntry();
+    const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
+    const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2');
+    const gpx = TestHelper.getRandomizedGPXEntry(parent, 'GPX1');
+    const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir');
+    const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1', 1);
+    const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2', 1);
+
+
+    DirectoryDTO.removeReferences(parent);
+    const s1 = im.queueForSave(Utils.clone(parent));
+    const s2 = im.queueForSave(Utils.clone(parent));
+    const s3 = im.queueForSave(Utils.clone(parent));
+
+    await Promise.all([s1, s2, s3]);
+
+    const selected = await gm.selectParentDir(conn, parent.name, parent.path);
+    await gm.fillParentDir(conn, selected);
+
+    DirectoryDTO.removeReferences(selected);
+    removeIds(selected);
+    subDir.isPartial = true;
+    delete subDir.directories;
+    delete subDir.metaFile;
+    delete sp1.metadata.faces;
+    delete sp2.metadata.faces;
+    expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
+      .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent)));
+  });
+
+
+  (it('should save 1500 photos', async () => {
+    const conn = await SQLConnection.getConnection();
+    const gm = new GalleryManagerTest();
+    const im = new IndexingManagerTest();
+    Config.Client.MetaFile.enabled = true;
+    const parent = TestHelper.getRandomizedDirectoryEntry();
+    DirectoryDTO.removeReferences(parent);
+    await im.saveToDB(Utils.clone(parent));
+    const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir');
+    for (let i = 0; i < 1500; i++) {
+      TestHelper.getRandomizedPhotoEntry(subDir, 'p' + i);
+    }
+
+    DirectoryDTO.removeReferences(parent);
+    await im.saveToDB(subDir);
+
+
+    const selected = await gm.selectParentDir(conn, subDir.name, subDir.path);
+    expect(selected.media.length).to.deep.equal(subDir.media.length);
+  }) as any).timeout(40000);
+
+  SQLTestHelper.savedDescribe('Test listDirectory', () => {
+    const statSync = fs.statSync;
+    let dirTime = 0;
+    const indexedTime = {
+      lastScanned: 0,
+      lastModified: 0
+    };
+
+    beforeEach(() => {
+      dirTime = 0;
+
+      ObjectManagerRepository.getInstance().IndexingManager = new IndexingManagerTest();
+      indexedTime.lastModified = 0;
+      indexedTime.lastScanned = 0;
+    });
+
+    afterEach(() => {
+      // @ts-ignore
+      fs.statSync = statSync;
+    });
+
+    it('with re indexing severity low', async () => {
+      Config.Server.indexing.reIndexingSensitivity = ReIndexingSensitivity.low;
+
+      // @ts-ignore
+      fs.statSync = () => ({ctime: new Date(dirTime), mtime: new Date(dirTime)});
+      const gm = new GalleryManagerTest();
+      gm.selectParentDir = (connection: Connection, directoryName: string, directoryParent: string) => {
+        return Promise.resolve(<any>indexedTime);
+      };
+      gm.fillParentDir = (connection: Connection, dir: DirectoryEntity) => {
+        return Promise.resolve();
+      };
+
+      ObjectManagerRepository.getInstance().IndexingManager.indexDirectory = (...args) => {
+        return <any>Promise.resolve('indexing');
+      };
+
+      indexedTime.lastScanned = null;
+      expect(await gm.listDirectory('./')).to.be.equal('indexing');
+      indexedTime.lastModified = 0;
+      dirTime = 1;
+      expect(await gm.listDirectory('./')).to.be.equal('indexing');
+      indexedTime.lastScanned = 10;
+      indexedTime.lastModified = 1;
+      dirTime = 1;
+      expect(await gm.listDirectory('./')).to.be.equal(indexedTime);
+      expect(await gm.listDirectory('./', 1, 10))
+        .to.be.equal(null);
+
+
+    });
+  });
+
+});

test/backend/unit/model/sql/SearchManager.ts

@@ -1,16 +1,6 @@
 import {expect} from 'chai';
-import * as fs from 'fs';
-import * as path from 'path';
-import {Config} from '../../../../../common/config/private/Config';
-import {DatabaseType} from '../../../../../common/config/private/IPrivateConfig';
 import {SQLConnection} from '../../../../../backend/model/sql/SQLConnection';
-import {
-  CameraMetadataEntity,
-  GPSMetadataEntity,
-  PhotoEntity,
-  PhotoMetadataEntity,
-  PositionMetaDataEntity
-} from '../../../../../backend/model/sql/enitites/PhotoEntity';
+import {PhotoEntity} from '../../../../../backend/model/sql/enitites/PhotoEntity';
 import {SearchManager} from '../../../../../backend/model/sql/SearchManager';
 import {AutoCompleteItem, SearchTypes} from '../../../../../common/entities/AutoCompleteItem';
 import {SearchResultDTO} from '../../../../../common/entities/SearchResultDTO';
@@ -18,64 +8,72 @@ import {DirectoryEntity} from '../../../../../backend/model/sql/enitites/Directo
 import {Utils} from '../../../../../common/Utils';
 import {TestHelper} from './TestHelper';
 import {VideoEntity} from '../../../../../backend/model/sql/enitites/VideoEntity';
+import {PersonEntry} from '../../../../../backend/model/sql/enitites/PersonEntry';
+import {FaceRegionEntry} from '../../../../../backend/model/sql/enitites/FaceRegionEntry';
+import {PhotoDTO} from '../../../../../common/entities/PhotoDTO';
+import {SQLTestHelper} from '../../../SQLTestHelper';
+import {Config} from '../../../../../common/config/private/Config';
 
-describe('SearchManager', () => {
+// to help WebStorm to handle the test cases
+declare let describe: any;
+declare const after: any;
+describe = SQLTestHelper.describe;
 
-
-  const tempDir = path.join(__dirname, '../../tmp');
-  const dbPath = path.join(tempDir, 'test.db');
+describe('SearchManager', (sqlHelper: SQLTestHelper) => {
 
   const dir = TestHelper.getDirectoryEntry();
   const p = TestHelper.getPhotoEntry1(dir);
   const p2 = TestHelper.getPhotoEntry2(dir);
+  const p_faceLess = TestHelper.getPhotoEntry2(dir);
+  delete p_faceLess.metadata.faces;
+  p_faceLess.name = 'fl';
   const v = TestHelper.getVideoEntry1(dir);
 
   const setUpSqlDB = async () => {
-    if (fs.existsSync(dbPath)) {
-      fs.unlinkSync(dbPath);
-    }
-    if (!fs.existsSync(tempDir)) {
-      fs.mkdirSync(tempDir);
-    }
-
-    Config.Server.database.type = DatabaseType.sqlite;
-    Config.Server.database.sqlite.storage = dbPath;
+    await sqlHelper.initDB();
 
+    const savePhoto = async (photo: PhotoDTO) => {
+      const savedPhoto = await pr.save(photo);
+      if (!photo.metadata.faces) {
+        return;
+      }
+      for (let i = 0; i < photo.metadata.faces.length; i++) {
+        const face = photo.metadata.faces[i];
+        const person = await conn.getRepository(PersonEntry).save({name: face.name});
+        await conn.getRepository(FaceRegionEntry).save({box: face.box, person: person, media: savedPhoto});
+      }
+    };
     const conn = await SQLConnection.getConnection();
 
     const pr = conn.getRepository(PhotoEntity);
 
     await conn.getRepository(DirectoryEntity).save(p.directory);
-    await pr.save(p);
-    await pr.save(p2);
+    await savePhoto(p);
+    await savePhoto(p2);
+    await savePhoto(p_faceLess);
+
     await conn.getRepository(VideoEntity).save(v);
 
     await SQLConnection.close();
   };
 
-  const tearDownSqlDB = async () => {
-    await SQLConnection.close();
-    if (fs.existsSync(dbPath)) {
-      fs.unlinkSync(dbPath);
-    }
-    if (fs.existsSync(tempDir)) {
-      fs.rmdirSync(tempDir);
-    }
-  };
 
   beforeEach(async () => {
     await setUpSqlDB();
   });
 
-  afterEach(async () => {
-    await tearDownSqlDB();
-  });
 
+  after(async () => {
+    await sqlHelper.clearDB();
+  });
 
   it('should get autocomplete', async () => {
     const sm = new SearchManager();
 
     const cmp = (a: AutoCompleteItem, b: AutoCompleteItem) => {
+      if (a.text === b.text) {
+        return a.type - b.type;
+      }
       return a.text.localeCompare(b.text);
     };
 
@@ -87,11 +85,19 @@ describe('SearchManager', () => {
       new AutoCompleteItem('wars dir', SearchTypes.directory)]);
 
     expect((await sm.autocomplete('arch'))).eql([new AutoCompleteItem('Research City', SearchTypes.position)]);
+
+    Config.Client.Search.AutoComplete.maxItemsPerCategory = 99999;
     expect((await sm.autocomplete('a')).sort(cmp)).eql([
       new AutoCompleteItem('Boba Fett', SearchTypes.keyword),
+      new AutoCompleteItem('Boba Fett', SearchTypes.person),
       new AutoCompleteItem('star wars', SearchTypes.keyword),
       new AutoCompleteItem('Anakin', SearchTypes.keyword),
+      new AutoCompleteItem('Anakin Skywalker', SearchTypes.person),
+      new AutoCompleteItem('Luke Skywalker', SearchTypes.person),
+      new AutoCompleteItem('Han Solo', SearchTypes.person),
       new AutoCompleteItem('death star', SearchTypes.keyword),
+      new AutoCompleteItem('Padmé Amidala', SearchTypes.person),
+      new AutoCompleteItem('Obivan Kenobi', SearchTypes.person),
       new AutoCompleteItem('Padmé Amidala', SearchTypes.keyword),
       new AutoCompleteItem('Natalie Portman', SearchTypes.keyword),
       new AutoCompleteItem('Han Solo\'s dice', SearchTypes.photo),
@@ -100,10 +106,24 @@ describe('SearchManager', () => {
       new AutoCompleteItem('wars dir', SearchTypes.directory),
       new AutoCompleteItem('Research City', SearchTypes.position)].sort(cmp));
 
+    Config.Client.Search.AutoComplete.maxItemsPerCategory = 1;
+    expect((await sm.autocomplete('a')).sort(cmp)).eql([
+      new AutoCompleteItem('Anakin', SearchTypes.keyword),
+      new AutoCompleteItem('star wars', SearchTypes.keyword),
+      new AutoCompleteItem('death star', SearchTypes.keyword),
+      new AutoCompleteItem('Anakin Skywalker', SearchTypes.person),
+      new AutoCompleteItem('Han Solo\'s dice', SearchTypes.photo),
+      new AutoCompleteItem('Kamino', SearchTypes.position),
+      new AutoCompleteItem('Research City', SearchTypes.position),
+      new AutoCompleteItem('wars dir', SearchTypes.directory),
+      new AutoCompleteItem('Boba Fett', SearchTypes.keyword)].sort(cmp));
+    Config.Client.Search.AutoComplete.maxItemsPerCategory = 5;
+
     expect((await sm.autocomplete('sw')).sort(cmp)).to.deep.equal([new AutoCompleteItem('sw1', SearchTypes.photo),
       new AutoCompleteItem('sw2', SearchTypes.photo), new AutoCompleteItem(v.name, SearchTypes.video)].sort(cmp));
 
     expect((await sm.autocomplete(v.name)).sort(cmp)).to.deep.equal([new AutoCompleteItem(v.name, SearchTypes.video)]);
+
   });
 
 
@@ -120,6 +140,15 @@ describe('SearchManager', () => {
       resultOverflow: false
     }));
 
+    expect(Utils.clone(await sm.search('Boba', null))).to.deep.equal(Utils.clone(<SearchResultDTO>{
+      searchText: 'Boba',
+      searchType: null,
+      directories: [],
+      media: [p],
+      metaFile: [],
+      resultOverflow: false
+    }));
+
     expect(Utils.clone(await sm.search('Tatooine', SearchTypes.position))).to.deep.equal(Utils.clone(<SearchResultDTO>{
       searchText: 'Tatooine',
       searchType: SearchTypes.position,
@@ -133,7 +162,7 @@ describe('SearchManager', () => {
       searchText: 'ortm',
       searchType: SearchTypes.keyword,
       directories: [],
-      media: [p2],
+      media: [p2, p_faceLess],
       metaFile: [],
       resultOverflow: false
     }));
@@ -142,7 +171,7 @@ describe('SearchManager', () => {
       searchText: 'ortm',
       searchType: SearchTypes.keyword,
       directories: [],
-      media: [p2],
+      media: [p2, p_faceLess],
       metaFile: [],
       resultOverflow: false
     }));
@@ -151,7 +180,7 @@ describe('SearchManager', () => {
       searchText: 'wa',
       searchType: SearchTypes.keyword,
       directories: [dir],
-      media: [p, p2],
+      media: [p, p2, p_faceLess],
       metaFile: [],
       resultOverflow: false
     }));
@@ -165,6 +194,15 @@ describe('SearchManager', () => {
       resultOverflow: false
     }));
 
+    expect(Utils.clone(await sm.search('sw', SearchTypes.video))).to.deep.equal(Utils.clone(<SearchResultDTO>{
+      searchText: 'sw',
+      searchType: SearchTypes.video,
+      directories: [],
+      media: [v],
+      metaFile: [],
+      resultOverflow: false
+    }));
+
     expect(Utils.clone(await sm.search('han', SearchTypes.keyword))).to.deep.equal(Utils.clone(<SearchResultDTO>{
       searchText: 'han',
       searchType: SearchTypes.keyword,
@@ -173,6 +211,15 @@ describe('SearchManager', () => {
       metaFile: [],
       resultOverflow: false
     }));
+
+    expect(Utils.clone(await sm.search('Boba', SearchTypes.person))).to.deep.equal(Utils.clone(<SearchResultDTO>{
+      searchText: 'Boba',
+      searchType: SearchTypes.person,
+      directories: [],
+      media: [p],
+      metaFile: [],
+      resultOverflow: false
+    }));
   });
 
 
@@ -198,23 +245,16 @@ describe('SearchManager', () => {
     expect(Utils.clone(await sm.instantSearch('ortm'))).to.deep.equal(Utils.clone({
       searchText: 'ortm',
       directories: [],
-      media: [p2],
+      media: [p2, p_faceLess],
       metaFile: [],
       resultOverflow: false
     }));
 
-    expect(Utils.clone(await sm.instantSearch('ortm'))).to.deep.equal(Utils.clone({
-      searchText: 'ortm',
-      directories: [],
-      media: [p2],
-      metaFile: [],
-      resultOverflow: false
-    }));
 
     expect(Utils.clone(await sm.instantSearch('wa'))).to.deep.equal(Utils.clone({
       searchText: 'wa',
       directories: [dir],
-      media: [p, p2],
+      media: [p, p2, p_faceLess],
       metaFile: [],
       resultOverflow: false
     }));
@@ -226,6 +266,13 @@ describe('SearchManager', () => {
       metaFile: [],
       resultOverflow: false
     }));
+    expect(Utils.clone(await sm.instantSearch('Boba'))).to.deep.equal(Utils.clone({
+      searchText: 'Boba',
+      directories: [],
+      media: [p],
+      metaFile: [],
+      resultOverflow: false
+    }));
   });
 
 

test/backend/unit/model/sql/SharingManager.ts

@@ -1,32 +1,23 @@
 import {expect} from 'chai';
-import * as fs from 'fs';
-import * as path from 'path';
-import {Config} from '../../../../../common/config/private/Config';
-import {DatabaseType} from '../../../../../common/config/private/IPrivateConfig';
 import {SQLConnection} from '../../../../../backend/model/sql/SQLConnection';
 import {SharingManager} from '../../../../../backend/model/sql/SharingManager';
 import {SharingDTO} from '../../../../../common/entities/SharingDTO';
 import {UserEntity} from '../../../../../backend/model/sql/enitites/UserEntity';
 import {UserDTO, UserRoles} from '../../../../../common/entities/UserDTO';
+import {SQLTestHelper} from '../../../SQLTestHelper';
 
-describe('SharingManager', () => {
+// to help WebStorm to handle the test cases
+declare let describe: any;
+declare const after: any;
+describe = SQLTestHelper.describe;
 
+describe('SharingManager', (sqlHelper: SQLTestHelper) => {
 
-  const tempDir = path.join(__dirname, '../../tmp');
-  const dbPath = path.join(tempDir, 'test.db');
 
   let creator: UserDTO = null;
 
   const setUpSqlDB = async () => {
-    if (fs.existsSync(dbPath)) {
-      fs.unlinkSync(dbPath);
-    }
-    if (!fs.existsSync(tempDir)) {
-      fs.mkdirSync(tempDir);
-    }
-
-    Config.Server.database.type = DatabaseType.sqlite;
-    Config.Server.database.sqlite.storage = dbPath;
+    await sqlHelper.initDB();
 
     const conn = await SQLConnection.getConnection();
 
@@ -41,22 +32,13 @@ describe('SharingManager', () => {
     await SQLConnection.close();
   };
 
-  const teardownUpSqlDB = async () => {
-    await SQLConnection.close();
-    if (fs.existsSync(dbPath)) {
-      fs.unlinkSync(dbPath);
-    }
-    if (fs.existsSync(tempDir)) {
-      fs.rmdirSync(tempDir);
-    }
-  };
 
   beforeEach(async () => {
     await setUpSqlDB();
   });
 
-  afterEach(async () => {
-    await teardownUpSqlDB();
+  after(async () => {
+    await sqlHelper.clearDB();
   });
 
 

test/backend/unit/model/sql/TestHelper.ts

@@ -1,7 +1,8 @@
 import {MediaDimensionEntity} from '../../../../../backend/model/sql/enitites/MediaEntity';
 import {
   CameraMetadataEntity,
-  GPSMetadataEntity, PhotoEntity,
+  GPSMetadataEntity,
+  PhotoEntity,
   PhotoMetadataEntity,
   PositionMetaDataEntity
 } from '../../../../../backend/model/sql/enitites/PhotoEntity';
@@ -9,9 +10,8 @@ import * as path from 'path';
 import {OrientationTypes} from 'ts-exif-parser';
 import {DirectoryEntity} from '../../../../../backend/model/sql/enitites/DirectoryEntity';
 import {VideoEntity, VideoMetadataEntity} from '../../../../../backend/model/sql/enitites/VideoEntity';
-import {FileEntity} from '../../../../../backend/model/sql/enitites/FileEntity';
 import {MediaDimension} from '../../../../../common/entities/MediaDTO';
-import {CameraMetadata, GPSMetadata, PhotoDTO, PhotoMetadata, PositionMetaData} from '../../../../../common/entities/PhotoDTO';
+import {CameraMetadata, FaceRegion, GPSMetadata, PhotoDTO, PhotoMetadata, PositionMetaData} from '../../../../../common/entities/PhotoDTO';
 import {DirectoryDTO} from '../../../../../common/entities/DirectoryDTO';
 import {FileDTO} from '../../../../../common/entities/FileDTO';
 
@@ -104,6 +104,20 @@ export class TestHelper {
     p.metadata.positionData.city = 'Mos Eisley';
     p.metadata.positionData.country = 'Tatooine';
     p.name = 'sw1';
+
+    p.metadata.faces = [<any>{
+      box: {height: 10, width: 10, x: 10, y: 10},
+      name: 'Boba Fett'
+    }, <any>{
+      box: {height: 10, width: 10, x: 101, y: 101},
+      name: 'Luke Skywalker'
+    }, <any>{
+      box: {height: 10, width: 10, x: 101, y: 101},
+      name: 'Han Solo'
+    }, <any>{
+      box: {height: 10, width: 10, x: 101, y: 101},
+      name: 'Unkle Ben'
+    }];
     return p;
   }
 
@@ -121,6 +135,16 @@ export class TestHelper {
     p.metadata.positionData.state = 'Research City';
     p.metadata.positionData.country = 'Kamino';
     p.name = 'sw2';
+    p.metadata.faces = [<any>{
+      box: {height: 10, width: 10, x: 10, y: 10},
+      name: 'Padmé Amidala'
+    }, <any>{
+      box: {height: 10, width: 10, x: 101, y: 101},
+      name: 'Anakin Skywalker'
+    }, <any>{
+      box: {height: 10, width: 10, x: 101, y: 101},
+      name: 'Obivan Kenobi'
+    }];
     return p;
   }
 
@@ -157,14 +181,37 @@ export class TestHelper {
     return d;
   }
 
-  public static getRandomizedPhotoEntry(dir: DirectoryDTO, forceStr: string = null) {
+
+  public static getRandomizedFace(media: PhotoDTO, forceStr: string = null) {
+    const rndStr = () => {
+      return forceStr + '_' + Math.random().toString(36).substring(7);
+    };
+
+    const rndInt = (max = 5000) => {
+      return Math.floor(Math.random() * max);
+    };
+
+    const f: FaceRegion = {
+      name: rndStr() + '.jpg',
+      box: {
+        x: rndInt(),
+        y: rndInt(),
+        width: rndInt(),
+        height: rndInt()
+      }
+    };
+    media.metadata.faces = (media.metadata.faces || []);
+    media.metadata.faces.push(f);
+    return f;
+  }
+
+  public static getRandomizedPhotoEntry(dir: DirectoryDTO, forceStr: string = null, faces: number = 2): PhotoDTO {
 
 
     const rndStr = () => {
       return forceStr + '_' + Math.random().toString(36).substring(7);
     };
 
-
     const rndInt = (max = 5000) => {
       return Math.floor(Math.random() * max);
     };
@@ -211,10 +258,14 @@ export class TestHelper {
       name: rndStr() + '.jpg',
       directory: dir,
       metadata: m,
-      readyThumbnails: null,
+      readyThumbnails: [],
       readyIcon: false
     };
 
+    for (let i = 0; i < faces; i++) {
+      this.getRandomizedFace(d, 'Person ' + i);
+    }
+
     dir.media.push(d);
     return d;
   }

test/backend/unit/model/threading/DiskMangerWorker.spec.ts

@@ -3,7 +3,7 @@ import {DiskMangerWorker} from '../../../../../backend/model/threading/DiskMange
 import * as path from 'path';
 import {Config} from '../../../../../common/config/private/Config';
 import {ProjectPath} from '../../../../../backend/ProjectPath';
-import {PhotoDTO} from '../../../../../common/entities/PhotoDTO';
+import {Utils} from '../../../../../common/Utils';
 
 describe('DiskMangerWorker', () => {
 
@@ -11,34 +11,10 @@ describe('DiskMangerWorker', () => {
     Config.Server.imagesFolder = path.join(__dirname, '/../../assets');
     ProjectPath.ImageFolder = path.join(__dirname, '/../../assets');
     const dir = await DiskMangerWorker.scanDirectory('/');
-    expect(dir.media.length).to.be.equals(2);
-    expect(dir.media[0].name).to.be.equals('test image öüóőúéáű-.,.jpg');
-    expect((<PhotoDTO>dir.media[0]).metadata.keywords).to.deep.equals(['Berkley', 'USA', 'űáéúőóüö ŰÁÉÚŐÓÜÖ']);
-    expect(dir.media[0].metadata.fileSize).to.deep.equals(62786);
-    expect(dir.media[0].metadata.size).to.deep.equals({width: 140, height: 93});
-    expect((<PhotoDTO>dir.media[0]).metadata.cameraData).to.deep.equals({
-      ISO: 3200,
-      model: 'óüöúőűáé ÓÜÖÚŐŰÁÉ',
-      make: 'Canon',
-      fStop: 5.6,
-      exposure: 0.00125,
-      focalLength: 85,
-      lens: 'EF-S15-85mm f/3.5-5.6 IS USM'
-    });
-
-    expect((<PhotoDTO>dir.media[0]).metadata.positionData).to.deep.equals({
-      GPSData: {
-        latitude: 37.871093333333334,
-        longitude: -122.25678,
-        altitude: 102.4498997995992
-      },
-      country: 'mmóüöúőűáé ÓÜÖÚŐŰÁÉmm-.,|\\mm',
-      state: 'óüöúőűáé ÓÜÖÚŐŰÁ',
-      city: 'óüöúőűáé ÓÜÖÚŐŰÁ'
-    });
-
-    expect(dir.media[0].metadata.creationDate).to.be.equals(1434018566000);
-
+    expect(dir.media.length).to.be.equals(3);
+    const expected = require(path.join(__dirname, '/../../assets/test image öüóőúéáű-.,.json'));
+    expect(Utils.clone(dir.media[1].name)).to.be.deep.equal('test image öüóőúéáű-.,.jpg');
+    expect(Utils.clone(dir.media[1].metadata)).to.be.deep.equal(expected);
   });
 
 });

test/backend/unit/model/threading/MetaDataLoader.spec.ts

@@ -20,36 +20,15 @@ describe('MetadataLoader', () => {
 
   it('should load jpg', async () => {
     const data = await MetadataLoader.loadPhotoMetadata(path.join(__dirname, '/../../assets/test image öüóőúéáű-.,.jpg'));
-    expect(Utils.clone(data)).to.be.deep.equal(Utils.clone({
-      size: {width: 140, height: 93},
-      orientation: 1,
-      caption: 'Test caption',
-      creationDate: 1434018566000,
-      fileSize: 62786,
-      cameraData:
-        {
-          ISO: 3200,
-          model: 'óüöúőűáé ÓÜÖÚŐŰÁÉ',
-          make: 'Canon',
-          fStop: 5.6,
-          exposure: 0.00125,
-          focalLength: 85,
-          lens: 'EF-S15-85mm f/3.5-5.6 IS USM'
-        },
-      positionData:
-        {
-          GPSData:
-            {
-              latitude: 37.871093333333334,
-              longitude: -122.25678,
-              altitude: 102.4498997995992
-            },
-          country: 'mmóüöúőűáé ÓÜÖÚŐŰÁÉmm-.,|\\mm',
-          state: 'óüöúőűáé ÓÜÖÚŐŰÁ',
-          city: 'óüöúőűáé ÓÜÖÚŐŰÁ'
-        },
-      keywords: ['Berkley', 'USA', 'űáéúőóüö ŰÁÉÚŐÓÜÖ']
-    }));
+    const expected = require(path.join(__dirname, '/../../assets/test image öüóőúéáű-.,.json'));
+    expect(Utils.clone(data)).to.be.deep.equal(expected);
+  });
+
+
+  it('should load jpg 2', async () => {
+    const data = await MetadataLoader.loadPhotoMetadata(path.join(__dirname, '/../../assets/old_photo.jpg'));
+    const expected = require(path.join(__dirname, '/../../assets/old_photo.json'));
+    expect(Utils.clone(data)).to.be.deep.equal(expected);
   });
 
 });

test/common/unit/Utils.spec.ts

@@ -0,0 +1,18 @@
+import {expect} from 'chai';
+import {Utils} from '../../../common/Utils';
+
+describe('Utils', () => {
+  it('should concat urls', () => {
+    expect(Utils.concatUrls('abc', 'cde')).to.be.equal('abc/cde');
+    expect(Utils.concatUrls('abc/', 'cde')).to.be.equal('abc/cde');
+    expect(Utils.concatUrls('abc\\', 'cde')).to.be.equal('abc/cde');
+    expect(Utils.concatUrls('abc/', 'cde/')).to.be.equal('abc/cde');
+    expect(Utils.concatUrls('./abc\\', 'cde/')).to.be.equal('./abc/cde');
+    expect(Utils.concatUrls('abc/', '\\cde/')).to.be.equal('abc/cde');
+    expect(Utils.concatUrls('abc\\', '\\cde/')).to.be.equal('abc/cde');
+    expect(Utils.concatUrls('abc\\', '/cde/')).to.be.equal('abc/cde');
+    expect(Utils.concatUrls('abc/', '/cde/')).to.be.equal('abc/cde');
+    expect(Utils.concatUrls('abc\\/', '/cde/')).to.be.equal('abc/cde');
+    expect(Utils.concatUrls('abc\\/', '/cde/', 'fgh')).to.be.equal('abc/cde/fgh');
+  });
+});