improving duplicate finding and UI

backend/model/sql/GalleryManager.ts

@@ -181,50 +181,67 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
           .having('count(*)>1'),
         'innerMedia',
         'media.name=innerMedia.name AND media.metadata.fileSize = innerMedia.fileSize')
-      .innerJoinAndSelect('media.directory', 'directory').getMany();
+      .innerJoinAndSelect('media.directory', 'directory')
+      .orderBy('media.name, media.metadata.fileSize')
+      .limit(Config.Server.duplicates.listingLimit).getMany();
+
 
     const duplicateParis: DuplicatesDTO[] = [];
-    let i = duplicates.length - 1;
+    const processDuplicates = (duplicateList: MediaEntity[],
-    while (i >= 0) {
+                               equalFn: (a: MediaEntity, b: MediaEntity) => boolean,
-      const list = [duplicates[i]];
+                               checkDuplicates: boolean = false) => {
-      let j = i - 1;
+      let i = duplicateList.length - 1;
-      while (j >= 0 && duplicates[i].name === duplicates[j].name && duplicates[i].metadata.fileSize === duplicates[j].metadata.fileSize) {
+      while (i >= 0) {
-        list.push(duplicates[j]);
+        const list = [duplicateList[i]];
-        j--;
+        let j = i - 1;
+        while (j >= 0 && equalFn(duplicateList[i], duplicateList[j])) {
+          list.push(duplicateList[j]);
+          j--;
+        }
+        i = j;
+        // if we cut the select list with the SQL LIMIT, filter unpaired media
+        if (list.length < 2) {
+          continue;
+        }
+        if (checkDuplicates) {
+          // ad to group if one already existed
+          const foundDuplicates = duplicateParis.find(dp =>
+            !!dp.media.find(m =>
+              !!list.find(lm => lm.id === m.id)));
+          if (foundDuplicates) {
+            list.forEach(lm => {
+              if (!!foundDuplicates.media.find(m => m.id === lm.id)) {
+                return;
+              }
+              foundDuplicates.media.push(lm);
+            });
+            continue;
+          }
+        }
+
+        duplicateParis.push({media: list});
       }
-      i = j;
+    };
-      duplicateParis.push({media: list});
+
-    }
+    processDuplicates(duplicates,
+      (a, b) => a.name === b.name &&
+        a.metadata.fileSize === b.metadata.fileSize);
 
 
     duplicates = await mediaRepository.createQueryBuilder('media')
       .innerJoin(query => query.from(MediaEntity, 'innerMedia')
           .select(['innerMedia.metadata.creationDate as creationDate', 'innerMedia.metadata.fileSize as fileSize', 'count(*)'])
-          .groupBy('innerMedia.name, innerMedia.metadata.fileSize')
+          .groupBy('innerMedia.metadata.creationDate, innerMedia.metadata.fileSize')
           .having('count(*)>1'),
         'innerMedia',
         'media.metadata.creationDate=innerMedia.creationDate AND media.metadata.fileSize = innerMedia.fileSize')
-      .innerJoinAndSelect('media.directory', 'directory').getMany();
+      .innerJoinAndSelect('media.directory', 'directory')
-
+      .orderBy('media.metadata.creationDate, media.metadata.fileSize')
-    i = duplicates.length - 1;
+      .limit(Config.Server.duplicates.listingLimit).getMany();
-    while (i >= 0) {
-      const list = [duplicates[i]];
-      let j = i - 1;
-      while (j >= 0 && duplicates[i].metadata.creationDate === duplicates[j].metadata.creationDate && duplicates[i].metadata.fileSize === duplicates[j].metadata.fileSize) {
-        list.push(duplicates[j]);
-        j--;
-      }
-      i = j;
-      if (list.filter(paired =>
-        !!duplicateParis.find(dp =>
-          !!dp.media.find(m =>
-            m.id === paired.id))).length === list.length) {
-        continue;
-      }
-
-      duplicateParis.push({media: list});
-    }
 
+    processDuplicates(duplicates,
+      (a, b) => a.metadata.creationDate === b.metadata.creationDate &&
+        a.metadata.fileSize === b.metadata.fileSize, true);
 
     return duplicateParis;
 

common/config/private/IPrivateConfig.ts

@@ -56,6 +56,10 @@ export interface ThreadingConfig {
   thumbnailThreads: number;
 }
 
+export interface DuplicatesConfig {
+  listingLimit: number; // maximum number of duplicates to list
+}
+
 export interface ServerConfig {
   port: number;
   host: string;
@@ -67,6 +71,7 @@ export interface ServerConfig {
   sessionTimeout: number;
   indexing: IndexingConfig;
   photoMetadataSize: number;
+  duplicates: DuplicatesConfig;
 }
 
 export interface IPrivateConfig {

common/config/private/PrivateConfigClass.ts

@@ -45,6 +45,9 @@ export class PrivateConfigClass extends PublicConfigClass implements IPrivateCon
       folderPreviewSize: 2,
       cachedFolderTimeout: 1000 * 60 * 60,
       reIndexingSensitivity: ReIndexingSensitivity.low
+    },
+    duplicates: {
+      listingLimit: 1000
     }
   };
   private ConfigLoader: any;

frontend/app/duplicates/duplicates.component.html

@@ -1,31 +1,34 @@
 <app-frame>
 
   <div body class="container">
-    <ng-template [ngIf]="_duplicateService.duplicates.value">
+    <ng-template [ngIf]="renderedDirGroups">
-      <div *ngFor="let pairs of _duplicateService.duplicates.value" class="card">
+      <div *ngFor="let group of renderedDirGroups">
-        <div class="card-body">
+        <strong>{{group.name}}</strong>
-          <div *ngFor="let media of pairs.media"
+        <div *ngFor="let pairs of group.duplicates" class="card">
-               class="row"
+          <div class="card-body">
-               [routerLink]="['/gallery', getDirectoryPath(media)]"
+            <div *ngFor="let media of pairs.media"
-               [queryParams]="queryService.getParams()">
+                 class="row"
-            <app-duplicates-photo class="col-1" [media]="media"></app-duplicates-photo>
+                 [routerLink]="['/gallery', getDirectoryPath(media.directory)]"
-            <div class="col-5">
+                 [queryParams]="queryService.getParams()">
-              /{{getDirectoryPath(media)}}/<span class="same-data">{{media.name}}</span>
+              <app-duplicates-photo class="col-1" [media]="media"></app-duplicates-photo>
-            </div>
+              <div class="col-5">
-            <div class="col-2">
+                /{{getDirectoryPath(media.directory)}}/<span class="same-data">{{media.name}}</span>
-              <span class="same-data">{{media.metadata.fileSize | fileSize}}</span>
+              </div>
-            </div>
+              <div class="col-2">
-            <div class="col-2" [title]="media.metadata.creationDate">
+                <span class="same-data">{{media.metadata.fileSize | fileSize}}</span>
-              {{media.metadata.creationDate | date}}
+              </div>
-            </div>
+              <div class="col-2" [title]="media.metadata.creationDate">
-            <div class="col-2">
+                {{media.metadata.creationDate | date}}
-              {{media.metadata.size.width}}x{{media.metadata.size.height}}
+              </div>
+              <div class="col-2">
+                {{media.metadata.size.width}}x{{media.metadata.size.height}}
+              </div>
             </div>
           </div>
         </div>
       </div>
     </ng-template>
-    <ng-template [ngIf]="!_duplicateService.duplicates.value">
+    <ng-template [ngIf]="!renderedDirGroups">
       loading
     </ng-template>
   </div>

frontend/app/duplicates/duplicates.component.ts

@@ -1,22 +1,121 @@
-import {Component} from '@angular/core';
+import {Component, HostListener, OnDestroy} from '@angular/core';
 import {DuplicateService} from './duplicates.service';
-import {MediaDTO} from '../../../common/entities/MediaDTO';
 import {Utils} from '../../../common/Utils';
 import {QueryService} from '../model/query.service';
+import {DuplicatesDTO} from '../../../common/entities/DuplicatesDTO';
+import {DirectoryDTO} from '../../../common/entities/DirectoryDTO';
+import {Subscription} from 'rxjs';
+import {Config} from '../../../common/config/public/Config';
+import {PageHelper} from '../model/page.helper';
+
+interface GroupedDuplicate {
+  name: string;
+  duplicates: DuplicatesDTO[];
+}
 
 @Component({
   selector: 'app-duplicate',
   templateUrl: './duplicates.component.html',
   styleUrls: ['./duplicates.component.css']
 })
-export class DuplicateComponent {
+export class DuplicateComponent implements OnDestroy {
+
+  directoryGroups: GroupedDuplicate[] = null;
+  renderedDirGroups: GroupedDuplicate[] = null;
+  renderedIndex = {
+    group: -1,
+    pairs: 0
+  };
+  subscription: Subscription;
+  renderTimer: number = null;
+
   constructor(public _duplicateService: DuplicateService,
               public queryService: QueryService) {
     this._duplicateService.getDuplicates().catch(console.error);
+    this.subscription = this._duplicateService.duplicates.subscribe((duplicates: DuplicatesDTO[]) => {
+      this.directoryGroups = [];
+      this.renderedIndex = {group: -1, pairs: 0};
+      this.renderedDirGroups = [];
+      if (duplicates === null) {
+        return;
+      }
+      const getMostFrequentDir = (dupls: DuplicatesDTO[]) => {
+        if (dupls.length === 0) {
+          return null;
+        }
+        const dirFrequency: { [key: number]: { count: number, dir: DirectoryDTO } } = {};
+        dupls.forEach(d => d.media.forEach(m => {
+          dirFrequency[m.directory.id] = dirFrequency[m.directory.id] || {dir: m.directory, count: 0};
+          dirFrequency[m.directory.id].count++;
+        }));
+        let max: { count: number, dir: DirectoryDTO } = {count: -1, dir: null};
+        for (const freq of Object.values(dirFrequency)) {
+          if (max.count <= freq.count) {
+            max = freq;
+          }
+        }
+        return max.dir;
+      };
+
+      while (duplicates.length > 0) {
+        const dir = getMostFrequentDir(duplicates);
+        const group = duplicates.filter(d => d.media.find(m => m.directory.id === dir.id));
+        duplicates = duplicates.filter(d => !d.media.find(m => m.directory.id === dir.id));
+        this.directoryGroups.push({name: this.getDirectoryPath(dir) + ' (' + group.length + ')', duplicates: group});
+      }
+      this.renderMore();
+    });
   }
 
-  getDirectoryPath(media: MediaDTO) {
+  ngOnDestroy(): void {
-    return Utils.concatUrls(media.directory.path, media.directory.name);
+    if (this.subscription) {
+      this.subscription.unsubscribe();
+      this.subscription = null;
+    }
+  }
+
+  getDirectoryPath(directory: DirectoryDTO) {
+    return Utils.concatUrls(directory.path, directory.name);
+  }
+
+  renderMore = () => {
+    if (this.renderTimer !== null) {
+      clearTimeout(this.renderTimer);
+      this.renderTimer = null;
+    }
+
+    if (this.renderedIndex.group === this.directoryGroups.length - 1 &&
+      this.renderedIndex.pairs >=
+      this.directoryGroups[this.renderedIndex.group].duplicates.length) {
+      return;
+    }
+    if (this.shouldRenderMore()) {
+      if (this.renderedDirGroups.length === 0 ||
+        this.renderedIndex.pairs >=
+        this.directoryGroups[this.renderedIndex.group].duplicates.length) {
+        this.renderedDirGroups.push({
+          name: this.directoryGroups[++this.renderedIndex.group].name,
+          duplicates: []
+        });
+        this.renderedIndex.pairs = 0;
+      }
+      this.renderedDirGroups[this.renderedDirGroups.length - 1].duplicates
+        .push(this.directoryGroups[this.renderedIndex.group].duplicates[this.renderedIndex.pairs++]);
+
+      this.renderTimer = window.setTimeout(this.renderMore, 0);
+    }
+  };
+
+
+  @HostListener('window:scroll')
+  onScroll() {
+    this.renderMore();
+  }
+
+  private shouldRenderMore(): boolean {
+    return Config.Client.Other.enableOnScrollRendering === false ||
+      PageHelper.ScrollY >= PageHelper.MaxScrollY * 0.7
+      || (document.body.clientHeight) * 0.85 < window.innerHeight;
   }
 }
 

frontend/app/gallery/grid/grid.gallery.component.ts

@@ -292,7 +292,6 @@ export class GalleryGridComponent implements OnChanges, OnInit, AfterViewInit, O
     return Config.Client.Other.enableOnScrollRendering === false ||
       PageHelper.ScrollY >= (document.body.clientHeight + offset - window.innerHeight) * 0.7
       || (document.body.clientHeight + offset) * 0.85 < window.innerHeight;
-
   }
 
 

frontend/app/model/page.helper.ts

@@ -11,14 +11,20 @@ export class PageHelper {
     return this.supportPageOffset ? window.pageYOffset : this.isCSS1Compat ? document.documentElement.scrollTop : document.body.scrollTop;
   }
 
-  public static get ScrollX(): number {
-    return this.supportPageOffset ? window.pageXOffset : this.isCSS1Compat ? document.documentElement.scrollLeft : document.body.scrollLeft;
-  }
-
   public static set ScrollY(value: number) {
     window.scrollTo(this.ScrollX, value);
   }
 
+  public static get MaxScrollY(): number {
+    return Math.max(document.body.scrollHeight, document.body.offsetHeight,
+      document.documentElement.clientHeight, document.documentElement.scrollHeight,
+      document.documentElement.offsetHeight) - window.innerHeight;
+  }
+
+  public static get ScrollX(): number {
+    return this.supportPageOffset ? window.pageXOffset : this.isCSS1Compat ? document.documentElement.scrollLeft : document.body.scrollLeft;
+  }
+
   public static showScrollY() {
     PageHelper.body.style.overflowY = 'scroll';
   }

frontend/app/settings/settings.service.ts

@@ -71,8 +71,8 @@ export class SettingsService {
           updateTimeout: 2000
         },
         imagesFolder: '',
-	port: 80,
+        port: 80,
-	host: '0.0.0.0',
+        host: '0.0.0.0',
         thumbnail: {
           folder: '',
           qualityPriority: true,
@@ -88,7 +88,10 @@ export class SettingsService {
           folderPreviewSize: 0,
           reIndexingSensitivity: ReIndexingSensitivity.medium
         },
-        photoMetadataSize: 512 * 1024
+        photoMetadataSize: 512 * 1024,
+        duplicates: {
+          listingLimit: 1000
+        }
       }
     });
   }