remove separate entity, handle re-running with different threshold, s…

immich-app · Apr 27, 2024 · c87dfe4 · c87dfe4
1 parent 3f4c81e
commit c87dfe4
Show file tree

Hide file tree

Showing 12 changed files with 24 additions and 62 deletions.
diff --git a/server/src/cores/system-config.core.ts b/server/src/cores/system-config.core.ts
@@ -72,7 +72,7 @@ export const defaults = Object.freeze<SystemConfig>({
     clip: {
       enabled: true,
       modelName: 'ViT-B-32__openai',
-      duplicateThreshold: 0.03,
+      duplicateThreshold: 0.02,
     },
     facialRecognition: {
       enabled: true,

diff --git a/server/src/entities/asset.entity.ts b/server/src/entities/asset.entity.ts
@@ -169,7 +169,7 @@ export class AssetEntity {
   @OneToOne(() => AssetJobStatusEntity, (jobStatus) => jobStatus.asset, { nullable: true })
   jobStatus?: AssetJobStatusEntity;
 
-  @Column({ nullable: true })
+  @Column({ type: 'uuid', nullable: true })
   duplicateId!: string | null;
 }
 

diff --git a/server/src/entities/index.ts b/server/src/entities/index.ts
@@ -22,15 +22,13 @@ import { SystemConfigEntity } from 'src/entities/system-config.entity';
 import { SystemMetadataEntity } from 'src/entities/system-metadata.entity';
 import { TagEntity } from 'src/entities/tag.entity';
 import { UserEntity } from 'src/entities/user.entity';
-import { AssetDuplicateEntity } from 'src/entities/asset-duplicate.entity';
 
 export const entities = [
   ActivityEntity,
   AlbumEntity,
   AlbumUserEntity,
   APIKeyEntity,
   AssetEntity,
-  AssetDuplicateEntity,
   AssetStackEntity,
   AssetFaceEntity,
   AssetJobStatusEntity,

diff --git a/server/src/entities/system-config.entity.ts b/server/src/entities/system-config.entity.ts
@@ -49,7 +49,7 @@ export const SystemConfigKey = {
   JOB_METADATA_EXTRACTION_CONCURRENCY: 'job.metadataExtraction.concurrency',
   JOB_VIDEO_CONVERSION_CONCURRENCY: 'job.videoConversion.concurrency',
   JOB_FACE_DETECTION_CONCURRENCY: 'job.faceDetection.concurrency',
-  JOB_SMART_SEARCH_CONCURRENCY = 'job.smartSearch.concurrency',
+  JOB_SMART_SEARCH_CONCURRENCY: 'job.smartSearch.concurrency',
   JOB_BACKGROUND_TASK_CONCURRENCY: 'job.backgroundTask.concurrency',
   JOB_SEARCH_CONCURRENCY: 'job.search.concurrency',
   JOB_SIDECAR_CONCURRENCY: 'job.sidecar.concurrency',

diff --git a/server/src/interfaces/asset-duplicate.interface.ts b/server/src/interfaces/asset-duplicate.interface.ts
diff --git a/server/src/migrations/1711989989911-AddAssetDuplicateColumns.ts b/server/src/migrations/1711989989911-AddAssetDuplicateColumns.ts
@@ -0,0 +1,14 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class CreateAssetDuplicateColumns1711989989911 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE assets ADD COLUMN "duplicateId" uuid`);
+    await queryRunner.query(`ALTER TABLE asset_job_status ADD COLUMN "duplicatesDetectedAt" timestamptz`);
+    await queryRunner.query(`CREATE INDEX "IDX_assets_duplicateId" ON assets ("duplicateId")`);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE assets DROP COLUMN "duplicateId"`);
+    await queryRunner.query(`ALTER TABLE asset_job_status DROP COLUMN "duplicatesDetectedAt"`);
+  }
+}
diff --git a/server/src/migrations/1711989989911-CreateAssetDuplicateTable.ts b/server/src/migrations/1711989989911-CreateAssetDuplicateTable.ts
diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts
@@ -354,7 +354,7 @@ export class AssetRepository implements IAssetRepository {
 
       case WithoutProperty.DUPLICATE: {
         where = {
-          resizePath: Not(IsNull()),
+          previewPath: Not(IsNull()),
           isVisible: true,
           smartSearch: true,
           jobStatus: {
@@ -698,7 +698,7 @@ export class AssetRepository implements IAssetRepository {
     }
 
     if (options.isDuplicate !== undefined) {
-      builder = builder.andWhere(`asset.duplicateId ${options.isDuplicate ? 'IS NOT NULL' : 'IS NULL'}`);
+      builder.andWhere(`asset.duplicateId ${options.isDuplicate ? 'IS NOT NULL' : 'IS NULL'}`);
     }
 
     if (options.personId !== undefined) {

diff --git a/server/src/repositories/index.ts b/server/src/repositories/index.ts
@@ -3,7 +3,6 @@ import { IActivityRepository } from 'src/interfaces/activity.interface';
 import { IAlbumUserRepository } from 'src/interfaces/album-user.interface';
 import { IAlbumRepository } from 'src/interfaces/album.interface';
 import { IKeyRepository } from 'src/interfaces/api-key.interface';
-import { IAssetDuplicateRepository } from 'src/interfaces/asset-duplicate.interface';
 import { IAssetStackRepository } from 'src/interfaces/asset-stack.interface';
 import { IAssetRepositoryV1 } from 'src/interfaces/asset-v1.interface';
 import { IAssetRepository } from 'src/interfaces/asset.interface';
@@ -63,14 +62,12 @@ import { SystemConfigRepository } from 'src/repositories/system-config.repositor
 import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
 import { TagRepository } from 'src/repositories/tag.repository';
 import { UserRepository } from 'src/repositories/user.repository';
-import { AssetDuplicateRepository } from 'src/repositories/asset-duplicate.repository';
 
 export const repositories = [
   { provide: IActivityRepository, useClass: ActivityRepository },
   { provide: IAccessRepository, useClass: AccessRepository },
   { provide: IAlbumRepository, useClass: AlbumRepository },
   { provide: IAlbumUserRepository, useClass: AlbumUserRepository },
-  { provide: IAssetDuplicateRepository, useClass: AssetDuplicateRepository },
   { provide: IAssetRepository, useClass: AssetRepository },
   { provide: IAssetRepositoryV1, useClass: AssetRepositoryV1 },
   { provide: IAssetStackRepository, useClass: AssetStackRepository },

diff --git a/server/src/repositories/search.repository.ts b/server/src/repositories/search.repository.ts
@@ -173,7 +173,7 @@ export class SearchRepository implements ISearchRepository {
       .andWhere('asset.isVisible = :isVisible')
       .orderBy('search.embedding <=> :embedding')
       .limit(64)
-      .setParameters({ assetId, embedding, isVisible: true, userIds });
+      .setParameters({ assetId, embedding: asVector(embedding), isVisible: true, userIds });
 
     const builder = this.assetRepository.manager
       .createQueryBuilder()

diff --git a/server/src/services/search.service.ts b/server/src/services/search.service.ts
@@ -191,11 +191,6 @@ export class SearchService {
       return JobStatus.SKIPPED;
     }
 
-    if (asset.duplicateId) {
-      this.logger.debug(`Asset ${id} already has a duplicateId, skipping`);
-      return JobStatus.SKIPPED;
-    }
-
     if (!asset.previewPath) {
       this.logger.warn(`Asset ${id} is missing preview image`);
       return JobStatus.FAILED;
@@ -225,6 +220,9 @@ export class SearchService {
 
       assetIds.push(...duplicateAssets.map((duplicate) => duplicate.assetId));
       await this.assetRepository.updateAll(assetIds, { duplicateId });
+    } else if (asset.duplicateId) {
+      this.logger.debug(`No duplicates found for asset ${asset.id}, removing duplicateId`);
+      await this.assetRepository.updateAll(assetIds, { duplicateId: null });
     }
 
     const duplicatesDetectedAt = new Date();

diff --git a/server/src/services/system-config.service.spec.ts b/server/src/services/system-config.service.spec.ts
@@ -77,7 +77,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
     clip: {
       enabled: true,
       modelName: 'ViT-B-32__openai',
-      duplicateThreshold: 0.03,
+      duplicateThreshold: 0.02,
     },
     facialRecognition: {
       enabled: true,