diff --git a/backend/model/sql/GalleryManager.ts b/backend/model/sql/GalleryManager.ts index 0112ddf..8846236 100644 --- a/backend/model/sql/GalleryManager.ts +++ b/backend/model/sql/GalleryManager.ts @@ -275,7 +275,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { delete directory.media; await directoryRepository.save(directory); } - } else { + } else { //dir does not exists yet scannedDirectory.directories[i].parent = currentDir; (scannedDirectory.directories[i]).lastScanned = null; // new child dir, not fully scanned yet const d = await directoryRepository.save(scannedDirectory.directories[i]); @@ -288,7 +288,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { } // Remove child Dirs that are not anymore in the parent dir - await directoryRepository.remove(childDirectories); + await directoryRepository.remove(childDirectories, {chunk: Math.max(Math.ceil(childDirectories.length / 500), 1)}); // save media const indexedMedia = await mediaRepository.createQueryBuilder('media') @@ -321,6 +321,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { await this.saveMedia(connection, mediaToSave); await mediaRepository.remove(indexedMedia); + // save files const indexedMetaFiles = await fileRepository.createQueryBuilder('file') .where('file.directory = :dir', { @@ -346,8 +347,8 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { metaFilesToSave.push(metaFile); } } - await fileRepository.save(metaFilesToSave); - await fileRepository.remove(indexedMetaFiles); + await fileRepository.save(metaFilesToSave, {chunk: Math.max(Math.ceil(metaFilesToSave.length / 500), 1)}); + await fileRepository.remove(indexedMetaFiles, {chunk: Math.max(Math.ceil(indexedMetaFiles.length / 500), 1)}); } catch (e) { throw e; } finally { diff --git a/backend/model/sql/enitites/PhotoEntity.ts b/backend/model/sql/enitites/PhotoEntity.ts index ede2fc3..fa9be6f 100644 --- a/backend/model/sql/enitites/PhotoEntity.ts +++ b/backend/model/sql/enitites/PhotoEntity.ts @@ -12,7 +12,7 @@ export class CameraMetadataEntity implements CameraMetadata { model: string; @Column('text', {nullable: true}) - maker: string; + make: string; @Column('int', {nullable: true}) fStop: number; @@ -55,19 +55,19 @@ export class PositionMetaDataEntity implements PositionMetaData { export class PhotoMetadataEntity extends MediaMetadataEntity implements PhotoMetadata { -/* - @Column('simple-array') - keywords: string[]; + /* + @Column('simple-array') + keywords: string[]; - @Column(type => CameraMetadataEntity) - cameraData: CameraMetadataEntity; + @Column(type => CameraMetadataEntity) + cameraData: CameraMetadataEntity; - @Column(type => PositionMetaDataEntity) - positionData: PositionMetaDataEntity; + @Column(type => PositionMetaDataEntity) + positionData: PositionMetaDataEntity; - @Column('tinyint', {default: OrientationTypes.TOP_LEFT}) - orientation: OrientationTypes; -*/ + @Column('tinyint', {default: OrientationTypes.TOP_LEFT}) + orientation: OrientationTypes; + */ } @@ -75,4 +75,5 @@ export class PhotoMetadataEntity extends MediaMetadataEntity implements PhotoMet export class PhotoEntity extends MediaEntity implements PhotoDTO { @Column(type => PhotoMetadataEntity) metadata: PhotoMetadataEntity; + } diff --git a/backend/model/threading/DiskMangerWorker.ts b/backend/model/threading/DiskMangerWorker.ts index 4f91386..7a327ba 100644 --- a/backend/model/threading/DiskMangerWorker.ts +++ b/backend/model/threading/DiskMangerWorker.ts @@ -209,7 +209,7 @@ export class DiskMangerWorker { try { const exif = ExifParserFactory.create(data).parse(); - metadata.cameraData = { + metadata.cameraData = { ISO: exif.tags.ISO, model: exif.tags.Model, make: exif.tags.Make, @@ -220,7 +220,7 @@ export class DiskMangerWorker { }; if (!isNaN(exif.tags.GPSLatitude) || exif.tags.GPSLongitude || exif.tags.GPSAltitude) { metadata.positionData = metadata.positionData || {}; - metadata.positionData.GPSData = { + metadata.positionData.GPSData = { latitude: exif.tags.GPSLatitude, longitude: exif.tags.GPSLongitude, altitude: exif.tags.GPSAltitude @@ -236,15 +236,15 @@ export class DiskMangerWorker { } if (exif.imageSize) { - metadata.size = {width: exif.imageSize.width, height: exif.imageSize.height}; + metadata.size = {width: exif.imageSize.width, height: exif.imageSize.height}; } else if (exif.tags.RelatedImageWidth && exif.tags.RelatedImageHeight) { - metadata.size = {width: exif.tags.RelatedImageWidth, height: exif.tags.RelatedImageHeight}; + metadata.size = {width: exif.tags.RelatedImageWidth, height: exif.tags.RelatedImageHeight}; } else { - metadata.size = {width: 1, height: 1}; + metadata.size = {width: 1, height: 1}; } } catch (err) { Logger.debug(LOG_TAG, 'Error parsing exif', fullPath, err); - metadata.size = {width: 1, height: 1}; + metadata.size = {width: 1, height: 1}; } try { diff --git a/common/entities/DirectoryDTO.ts b/common/entities/DirectoryDTO.ts index 4f89af5..5027831 100644 --- a/common/entities/DirectoryDTO.ts +++ b/common/entities/DirectoryDTO.ts @@ -36,9 +36,11 @@ export module DirectoryDTO { }; export const removeReferences = (dir: DirectoryDTO): void => { - dir.media.forEach((media: MediaDTO) => { - media.directory = null; - }); + if (dir.media) { + dir.media.forEach((media: MediaDTO) => { + media.directory = null; + }); + } if (dir.metaFile) { dir.metaFile.forEach((file: FileDTO) => { file.directory = null; diff --git a/test/backend/integration/model/sql/typeorm.ts b/test/backend/integration/model/sql/typeorm.ts index c257039..7c8673c 100644 --- a/test/backend/integration/model/sql/typeorm.ts +++ b/test/backend/integration/model/sql/typeorm.ts @@ -83,7 +83,7 @@ describe('Typeorm integration', () => { const cd = new CameraMetadataEntity(); cd.ISO = 100; cd.model = '60D'; - cd.maker = 'Canon'; + cd.make = 'Canon'; cd.fStop = 1; cd.exposure = 1; cd.focalLength = 1; diff --git a/test/backend/unit/model/sql/GalleryManager.ts b/test/backend/unit/model/sql/GalleryManager.ts index a7198fe..40aa5f6 100644 --- a/test/backend/unit/model/sql/GalleryManager.ts +++ b/test/backend/unit/model/sql/GalleryManager.ts @@ -12,9 +12,6 @@ import {DirectoryEntity} from '../../../../../backend/model/sql/enitites/Directo import {Utils} from '../../../../../common/Utils'; import {MediaDTO} from '../../../../../common/entities/MediaDTO'; import {FileDTO} from '../../../../../common/entities/FileDTO'; -import {PhotoEntity} from '../../../../../backend/model/sql/enitites/PhotoEntity'; -import {FileEntity} from '../../../../../backend/model/sql/enitites/FileEntity'; - class GalleryManagerTest extends GalleryManager { @@ -202,9 +199,6 @@ describe('GalleryManager', () => { const selected = await gm.selectParentDir(conn, parent.name, parent.path); await gm.fillParentDir(conn, selected); - const query = conn.getRepository(FileEntity).createQueryBuilder('photo'); - query.innerJoinAndSelect('photo.directory', 'directory'); - console.log((await query.getMany())); DirectoryDTO.removeReferences(selected); removeIds(selected); subDir.isPartial = true; @@ -214,4 +208,25 @@ describe('GalleryManager', () => { .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent))); }); + + (it('should save 1500 photos', async () => { + const conn = await SQLConnection.getConnection(); + const gm = new GalleryManagerTest(); + Config.Client.MetaFile.enabled = true; + const parent = TestHelper.getRandomizedDirectoryEntry(); + DirectoryDTO.removeReferences(parent); + await gm.saveToDB(Utils.clone(parent)); + const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir'); + for (let i = 0; i < 1500; i++) { + TestHelper.getRandomizedPhotoEntry(subDir, 'p' + i); + } + + DirectoryDTO.removeReferences(parent); + await gm.saveToDB(subDir); + + + const selected = await gm.selectParentDir(conn, subDir.name, subDir.path); + expect(selected.media.length).to.deep.equal(subDir.media.length); + })).timeout(20000); + }); diff --git a/test/backend/unit/model/sql/TestHelper.ts b/test/backend/unit/model/sql/TestHelper.ts index f102427..c36f68c 100644 --- a/test/backend/unit/model/sql/TestHelper.ts +++ b/test/backend/unit/model/sql/TestHelper.ts @@ -10,6 +10,10 @@ import {OrientationTypes} from 'ts-exif-parser'; import {DirectoryEntity} from '../../../../../backend/model/sql/enitites/DirectoryEntity'; import {VideoEntity, VideoMetadataEntity} from '../../../../../backend/model/sql/enitites/VideoEntity'; import {FileEntity} from '../../../../../backend/model/sql/enitites/FileEntity'; +import {MediaDimension} from '../../../../../common/entities/MediaDTO'; +import {CameraMetadata, GPSMetadata, PhotoDTO, PhotoMetadata, PositionMetaData} from '../../../../../common/entities/PhotoDTO'; +import {DirectoryDTO} from '../../../../../common/entities/DirectoryDTO'; +import {FileDTO} from '../../../../../common/entities/FileDTO'; export class TestHelper { @@ -40,7 +44,7 @@ export class TestHelper { const cd = new CameraMetadataEntity(); cd.ISO = 100; cd.model = '60D'; - cd.maker = 'Canon'; + cd.make = 'Canon'; cd.fStop = 1; cd.exposure = 1; cd.focalLength = 1; @@ -119,35 +123,38 @@ export class TestHelper { } - public static getRandomizedDirectoryEntry(parent: DirectoryEntity = null, forceStr: string = null) { + public static getRandomizedDirectoryEntry(parent: DirectoryDTO = null, forceStr: string = null) { - const dir = new DirectoryEntity(); - dir.name = forceStr || Math.random().toString(36).substring(7); - dir.path = '.'; + const dir: DirectoryDTO = { + id: null, + name: forceStr || Math.random().toString(36).substring(7), + path: '.', + directories: [], + metaFile: [], + media: [], + lastModified: Date.now(), + lastScanned: null, + parent: null + }; if (parent !== null) { dir.path = path.join(parent.path, parent.name); parent.directories.push(dir); } - dir.directories = []; - dir.metaFile = []; - dir.media = []; - dir.lastModified = Date.now(); - dir.lastScanned = null; - return dir; } - public static getRandomizedGPXEntry(dir: DirectoryEntity, forceStr: string = null): FileEntity { - const d = new FileEntity(); - d.name = forceStr + '_' + Math.random().toString(36).substring(7) + '.gpx'; - d.directory = dir; - + public static getRandomizedGPXEntry(dir: DirectoryDTO, forceStr: string = null): FileDTO { + const d: FileDTO = { + id: null, + name: forceStr + '_' + Math.random().toString(36).substring(7) + '.gpx', + directory: dir + }; dir.metaFile.push(d); return d; } - public static getRandomizedPhotoEntry(dir: DirectoryEntity, forceStr: string = null) { + public static getRandomizedPhotoEntry(dir: DirectoryDTO, forceStr: string = null) { const rndStr = () => { @@ -159,44 +166,51 @@ export class TestHelper { return Math.floor(Math.random() * max); }; - const sd = new MediaDimensionEntity(); - sd.height = rndInt(); - sd.width = rndInt(); - const gps = new GPSMetadataEntity(); - gps.altitude = rndInt(1000); - gps.latitude = rndInt(1000); - gps.longitude = rndInt(1000); - const pd = new PositionMetaDataEntity(); - pd.city = rndStr(); - pd.country = rndStr(); - pd.state = rndStr(); - pd.GPSData = gps; - const cd = new CameraMetadataEntity(); - cd.ISO = rndInt(500); - cd.model = rndStr(); - cd.maker = rndStr(); - cd.fStop = rndInt(10); - cd.exposure = rndInt(10); - cd.focalLength = rndInt(10); - cd.lens = rndStr(); - const m = new PhotoMetadataEntity(); - m.keywords = [rndStr(), rndStr()]; - m.cameraData = cd; - m.positionData = pd; - m.size = sd; - m.creationDate = Date.now(); - m.fileSize = rndInt(10000); - m.orientation = OrientationTypes.TOP_LEFT; + const sd: MediaDimension = { + height: rndInt(), + width: rndInt(), + }; - // TODO: remove when typeorm is fixed - m.duration = null; - m.bitRate = null; + const gps: GPSMetadata = { + altitude: rndInt(1000), + latitude: rndInt(1000), + longitude: rndInt(1000) + }; + const pd: PositionMetaData = { + city: rndStr(), + country: rndStr(), + state: rndStr(), + GPSData: gps + }; + const cd: CameraMetadata = { + ISO: rndInt(500), + model: rndStr(), + make: rndStr(), + fStop: rndInt(10), + exposure: rndInt(10), + focalLength: rndInt(10), + lens: rndStr() + }; + const m: PhotoMetadata = { + keywords: [rndStr(), rndStr()], + cameraData: cd, + positionData: pd, + size: sd, + creationDate: Date.now(), + fileSize: rndInt(10000), + orientation: OrientationTypes.TOP_LEFT, + caption: rndStr() + }; - const d = new PhotoEntity(); - d.name = rndStr() + '.jpg'; - d.directory = dir; - d.metadata = m; + const d: PhotoDTO = { + id: null, + name: rndStr() + '.jpg', + directory: dir, + metadata: m, + readyThumbnails: null, + readyIcon: false + }; dir.media.push(d); return d;