From b23d5f56e0b8314f9f98e24671c0077347c9aa66 Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Tue, 20 Aug 2024 20:35:59 -0400 Subject: [PATCH] refresh faces handle non-ml faces --- mobile/openapi/lib/model/asset_job_name.dart | 9 +- mobile/openapi/lib/model/job_command_dto.dart | 19 +++- open-api/immich-openapi-specs.json | 6 +- open-api/typescript-sdk/src/fetch-client.ts | 5 +- server/src/dtos/asset.dto.ts | 3 +- server/src/dtos/job.dto.ts | 2 +- server/src/interfaces/person.interface.ts | 7 +- server/src/queries/metadata.repository.sql | 56 ------------ server/src/repositories/person.repository.ts | 37 ++++++-- server/src/services/asset.service.ts | 7 +- server/src/services/metadata.service.spec.ts | 16 ++-- server/src/services/metadata.service.ts | 34 ++++--- server/src/services/person.service.ts | 91 +++++++++++++++---- .../repositories/person.repository.mock.ts | 2 +- .../admin-page/jobs/job-tile-button.svelte | 7 +- .../admin-page/jobs/job-tile.svelte | 40 ++++++-- .../admin-page/jobs/jobs-panel.svelte | 38 +++++--- .../asset-viewer/asset-viewer-nav-bar.svelte | 6 ++ web/src/lib/i18n/en.json | 6 +- web/src/lib/utils.ts | 5 +- 20 files changed, 250 insertions(+), 146 deletions(-) delete mode 100644 server/src/queries/metadata.repository.sql diff --git a/mobile/openapi/lib/model/asset_job_name.dart b/mobile/openapi/lib/model/asset_job_name.dart index a5b42f4ee52eb..11e0555b868d4 100644 --- a/mobile/openapi/lib/model/asset_job_name.dart +++ b/mobile/openapi/lib/model/asset_job_name.dart @@ -23,14 +23,16 @@ class AssetJobName { String toJson() => value; - static const regenerateThumbnail = AssetJobName._(r'regenerate-thumbnail'); + static const refreshFaces = AssetJobName._(r'refresh-faces'); static const refreshMetadata = AssetJobName._(r'refresh-metadata'); + static const regenerateThumbnail = AssetJobName._(r'regenerate-thumbnail'); static const transcodeVideo = AssetJobName._(r'transcode-video'); /// List of all possible values in this [enum][AssetJobName]. static const values = [ - regenerateThumbnail, + refreshFaces, refreshMetadata, + regenerateThumbnail, transcodeVideo, ]; @@ -70,8 +72,9 @@ class AssetJobNameTypeTransformer { AssetJobName? decode(dynamic data, {bool allowNull = true}) { if (data != null) { switch (data) { - case r'regenerate-thumbnail': return AssetJobName.regenerateThumbnail; + case r'refresh-faces': return AssetJobName.refreshFaces; case r'refresh-metadata': return AssetJobName.refreshMetadata; + case r'regenerate-thumbnail': return AssetJobName.regenerateThumbnail; case r'transcode-video': return AssetJobName.transcodeVideo; default: if (!allowNull) { diff --git a/mobile/openapi/lib/model/job_command_dto.dart b/mobile/openapi/lib/model/job_command_dto.dart index 5c56715644f22..08b1bfdad3468 100644 --- a/mobile/openapi/lib/model/job_command_dto.dart +++ b/mobile/openapi/lib/model/job_command_dto.dart @@ -14,12 +14,18 @@ class JobCommandDto { /// Returns a new [JobCommandDto] instance. JobCommandDto({ required this.command, - required this.force, + this.force, }); JobCommand command; - bool force; + /// + /// Please note: This property should have been non-nullable! Since the specification file + /// does not include a default value (using the "default:" property), however, the generated + /// source code must fall back to having a nullable type. + /// Consider adding a "default:" property in the specification file to hide this note. + /// + bool? force; @override bool operator ==(Object other) => identical(this, other) || other is JobCommandDto && @@ -30,7 +36,7 @@ class JobCommandDto { int get hashCode => // ignore: unnecessary_parenthesis (command.hashCode) + - (force.hashCode); + (force == null ? 0 : force!.hashCode); @override String toString() => 'JobCommandDto[command=$command, force=$force]'; @@ -38,7 +44,11 @@ class JobCommandDto { Map toJson() { final json = {}; json[r'command'] = this.command; + if (this.force != null) { json[r'force'] = this.force; + } else { + // json[r'force'] = null; + } return json; } @@ -51,7 +61,7 @@ class JobCommandDto { return JobCommandDto( command: JobCommand.fromJson(json[r'command'])!, - force: mapValueOfType(json, r'force')!, + force: mapValueOfType(json, r'force'), ); } return null; @@ -100,7 +110,6 @@ class JobCommandDto { /// The list of required keys that must be present in a JSON. static const requiredKeys = { 'command', - 'force', }; } diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 2325f24ee59d4..b01992a71b01f 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -8170,8 +8170,9 @@ }, "AssetJobName": { "enum": [ - "regenerate-thumbnail", + "refresh-faces", "refresh-metadata", + "regenerate-thumbnail", "transcode-video" ], "type": "string" @@ -9230,8 +9231,7 @@ } }, "required": [ - "command", - "force" + "command" ], "type": "object" }, diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index 43777552c59bf..07797a80666dd 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -548,7 +548,7 @@ export type AllJobStatusResponseDto = { }; export type JobCommandDto = { command: JobCommand; - force: boolean; + force?: boolean; }; export type LibraryResponseDto = { assetCount: number; @@ -3350,8 +3350,9 @@ export enum Reason { UnsupportedFormat = "unsupported-format" } export enum AssetJobName { - RegenerateThumbnail = "regenerate-thumbnail", + RefreshFaces = "refresh-faces", RefreshMetadata = "refresh-metadata", + RegenerateThumbnail = "regenerate-thumbnail", TranscodeVideo = "transcode-video" } export enum AssetMediaSize { diff --git a/server/src/dtos/asset.dto.ts b/server/src/dtos/asset.dto.ts index 5a2fdb51200d7..608209afcd3fd 100644 --- a/server/src/dtos/asset.dto.ts +++ b/server/src/dtos/asset.dto.ts @@ -89,8 +89,9 @@ export class AssetIdsDto { } export enum AssetJobName { - REGENERATE_THUMBNAIL = 'regenerate-thumbnail', + REFRESH_FACES = 'refresh-faces', REFRESH_METADATA = 'refresh-metadata', + REGENERATE_THUMBNAIL = 'regenerate-thumbnail', TRANSCODE_VIDEO = 'transcode-video', } diff --git a/server/src/dtos/job.dto.ts b/server/src/dtos/job.dto.ts index b7d8cf59bf55a..f2b0e2d1cfad3 100644 --- a/server/src/dtos/job.dto.ts +++ b/server/src/dtos/job.dto.ts @@ -17,7 +17,7 @@ export class JobCommandDto { command!: JobCommand; @ValidateBoolean({ optional: true }) - force!: boolean; + force?: boolean; } export class JobCountsDto { diff --git a/server/src/interfaces/person.interface.ts b/server/src/interfaces/person.interface.ts index fc6a389f3cc06..b932ef4feea8a 100644 --- a/server/src/interfaces/person.interface.ts +++ b/server/src/interfaces/person.interface.ts @@ -1,5 +1,6 @@ import { AssetFaceEntity } from 'src/entities/asset-face.entity'; import { AssetEntity } from 'src/entities/asset.entity'; +import { FaceSearchEntity } from 'src/entities/face-search.entity'; import { PersonEntity } from 'src/entities/person.entity'; import { Paginated, PaginationOptions } from 'src/utils/pagination'; import { FindManyOptions, FindOptionsRelations, FindOptionsSelect } from 'typeorm'; @@ -59,7 +60,11 @@ export interface IPersonRepository { delete(entities: PersonEntity[]): Promise; deleteAll(): Promise; deleteAllFaces(options: DeleteAllFacesOptions): Promise; - replaceFaces(assetId: string, entities: Partial[], sourceType?: string): Promise; + refreshFaces( + facesToAdd: Partial[], + faceIdsToRemove: string[], + embeddingsToAdd?: FaceSearchEntity[], + ): Promise; getAllFaces(pagination: PaginationOptions, options?: FindManyOptions): Paginated; getFaceById(id: string): Promise; getFaceByIdWithAssets( diff --git a/server/src/queries/metadata.repository.sql b/server/src/queries/metadata.repository.sql deleted file mode 100644 index 077b4644b824d..0000000000000 --- a/server/src/queries/metadata.repository.sql +++ /dev/null @@ -1,56 +0,0 @@ --- NOTE: This file is auto generated by ./sql-generator - --- MetadataRepository.getCountries -SELECT DISTINCT - ON ("exif"."country") "exif"."country" AS "country" -FROM - "exif" "exif" - LEFT JOIN "assets" "asset" ON "asset"."id" = "exif"."assetId" - AND ("asset"."deletedAt" IS NULL) -WHERE - "asset"."ownerId" = $1 - --- MetadataRepository.getStates -SELECT DISTINCT - ON ("exif"."state") "exif"."state" AS "state" -FROM - "exif" "exif" - LEFT JOIN "assets" "asset" ON "asset"."id" = "exif"."assetId" - AND ("asset"."deletedAt" IS NULL) -WHERE - "asset"."ownerId" = $1 - AND "exif"."country" = $2 - --- MetadataRepository.getCities -SELECT DISTINCT - ON ("exif"."city") "exif"."city" AS "city" -FROM - "exif" "exif" - LEFT JOIN "assets" "asset" ON "asset"."id" = "exif"."assetId" - AND ("asset"."deletedAt" IS NULL) -WHERE - "asset"."ownerId" = $1 - AND "exif"."country" = $2 - AND "exif"."state" = $3 - --- MetadataRepository.getCameraMakes -SELECT DISTINCT - ON ("exif"."make") "exif"."make" AS "make" -FROM - "exif" "exif" - LEFT JOIN "assets" "asset" ON "asset"."id" = "exif"."assetId" - AND ("asset"."deletedAt" IS NULL) -WHERE - "asset"."ownerId" = $1 - AND "exif"."model" = $2 - --- MetadataRepository.getCameraModels -SELECT DISTINCT - ON ("exif"."model") "exif"."model" AS "model" -FROM - "exif" "exif" - LEFT JOIN "assets" "asset" ON "asset"."id" = "exif"."assetId" - AND ("asset"."deletedAt" IS NULL) -WHERE - "asset"."ownerId" = $1 - AND "exif"."make" = $2 diff --git a/server/src/repositories/person.repository.ts b/server/src/repositories/person.repository.ts index 1290df740e62d..07be589a01642 100644 --- a/server/src/repositories/person.repository.ts +++ b/server/src/repositories/person.repository.ts @@ -5,6 +5,7 @@ import { ChunkedArray, DummyValue, GenerateSql } from 'src/decorators'; import { AssetFaceEntity } from 'src/entities/asset-face.entity'; import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity'; import { AssetEntity } from 'src/entities/asset.entity'; +import { FaceSearchEntity } from 'src/entities/face-search.entity'; import { PersonEntity } from 'src/entities/person.entity'; import { SourceType } from 'src/enum'; import { @@ -30,6 +31,7 @@ export class PersonRepository implements IPersonRepository { @InjectRepository(AssetEntity) private assetRepository: Repository, @InjectRepository(PersonEntity) private personRepository: Repository, @InjectRepository(AssetFaceEntity) private assetFaceRepository: Repository, + @InjectRepository(FaceSearchEntity) private faceSearchRepository: Repository, @InjectRepository(AssetJobStatusEntity) private jobStatusRepository: Repository, ) {} @@ -289,12 +291,35 @@ export class PersonRepository implements IPersonRepository { return res.map((row) => row.id); } - async replaceFaces(assetId: string, entities: AssetFaceEntity[], sourceType: string): Promise { - return this.dataSource.transaction(async (manager) => { - await manager.delete(AssetFaceEntity, { assetId, sourceType }); - const assetFaces = await manager.save(AssetFaceEntity, entities); - return assetFaces.map(({ id }) => id); - }); + async refreshFaces( + facesToAdd: Partial[], + faceIdsToRemove: string[], + embeddingsToAdd?: FaceSearchEntity[], + ): Promise { + const faceDeleteQuery = this.assetFaceRepository + .createQueryBuilder() + .delete() + .where('id = any(:faceIdsToRemove)', { faceIdsToRemove }); + + if (facesToAdd.length === 0) { + await faceDeleteQuery.execute(); + return; + } + + const faceInsertQuery = this.assetFaceRepository.createQueryBuilder().insert().values(facesToAdd); + if (!embeddingsToAdd || embeddingsToAdd.length === 0) { + await faceInsertQuery.addCommonTableExpression(faceDeleteQuery, 'deleted').execute(); + return; + } + + await this.faceSearchRepository + .createQueryBuilder() + .addCommonTableExpression(faceDeleteQuery, 'deleted') + .addCommonTableExpression(faceInsertQuery, 'added') + .insert() + .values(embeddingsToAdd) + .orIgnore() + .execute(); } async update(entities: Partial[]): Promise { diff --git a/server/src/services/asset.service.ts b/server/src/services/asset.service.ts index bfd3a0c4d26b5..62a219f97dbb7 100644 --- a/server/src/services/asset.service.ts +++ b/server/src/services/asset.service.ts @@ -113,9 +113,9 @@ export class AssetService { id, { exifInfo: true, - tags: true, sharedLinks: true, smartInfo: true, + tags: true, owner: true, faces: { person: true, @@ -298,6 +298,11 @@ export class AssetService { for (const id of dto.assetIds) { switch (dto.name) { + case AssetJobName.REFRESH_FACES: { + jobs.push({ name: JobName.FACE_DETECTION, data: { id } }); + break; + } + case AssetJobName.REFRESH_METADATA: { jobs.push({ name: JobName.METADATA_EXTRACTION, data: { id } }); break; diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.ts index 52f6609772b9d..dfd0ce52fc083 100644 --- a/server/src/services/metadata.service.spec.ts +++ b/server/src/services/metadata.service.spec.ts @@ -990,11 +990,11 @@ describe(MetadataService.name, () => { metadataMock.readTags.mockResolvedValue(metadataStub.withFaceNoName); personMock.getDistinctNames.mockResolvedValue([]); personMock.create.mockResolvedValue([]); - personMock.replaceFaces.mockResolvedValue([]); + personMock.refreshFaces.mockResolvedValue([]); personMock.update.mockResolvedValue([]); await sut.handleMetadataExtraction({ id: assetStub.image.id }); expect(personMock.create).toHaveBeenCalledWith([]); - expect(personMock.replaceFaces).toHaveBeenCalledWith(assetStub.primaryImage.id, [], SourceType.EXIF); + expect(personMock.refreshFaces).toHaveBeenCalledWith(assetStub.primaryImage.id, [], SourceType.EXIF); expect(personMock.update).toHaveBeenCalledWith([]); }); @@ -1004,11 +1004,11 @@ describe(MetadataService.name, () => { metadataMock.readTags.mockResolvedValue(metadataStub.withFaceEmptyName); personMock.getDistinctNames.mockResolvedValue([]); personMock.create.mockResolvedValue([]); - personMock.replaceFaces.mockResolvedValue([]); + personMock.refreshFaces.mockResolvedValue([]); personMock.update.mockResolvedValue([]); await sut.handleMetadataExtraction({ id: assetStub.image.id }); expect(personMock.create).toHaveBeenCalledWith([]); - expect(personMock.replaceFaces).toHaveBeenCalledWith(assetStub.primaryImage.id, [], SourceType.EXIF); + expect(personMock.refreshFaces).toHaveBeenCalledWith(assetStub.primaryImage.id, [], SourceType.EXIF); expect(personMock.update).toHaveBeenCalledWith([]); }); @@ -1018,13 +1018,13 @@ describe(MetadataService.name, () => { metadataMock.readTags.mockResolvedValue(metadataStub.withFace); personMock.getDistinctNames.mockResolvedValue([]); personMock.create.mockResolvedValue([personStub.withName]); - personMock.replaceFaces.mockResolvedValue(['face-asset-uuid']); + personMock.refreshFaces.mockResolvedValue(['face-asset-uuid']); personMock.update.mockResolvedValue([personStub.withName]); await sut.handleMetadataExtraction({ id: assetStub.primaryImage.id }); expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.primaryImage.id]); expect(personMock.getDistinctNames).toHaveBeenCalledWith(assetStub.primaryImage.ownerId, { withHidden: true }); expect(personMock.create).toHaveBeenCalledWith([expect.objectContaining({ name: personStub.withName.name })]); - expect(personMock.replaceFaces).toHaveBeenCalledWith( + expect(personMock.refreshFaces).toHaveBeenCalledWith( assetStub.primaryImage.id, [ { @@ -1057,13 +1057,13 @@ describe(MetadataService.name, () => { metadataMock.readTags.mockResolvedValue(metadataStub.withFace); personMock.getDistinctNames.mockResolvedValue([{ id: personStub.withName.id, name: personStub.withName.name }]); personMock.create.mockResolvedValue([]); - personMock.replaceFaces.mockResolvedValue(['face-asset-uuid']); + personMock.refreshFaces.mockResolvedValue(['face-asset-uuid']); personMock.update.mockResolvedValue([personStub.withName]); await sut.handleMetadataExtraction({ id: assetStub.primaryImage.id }); expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.primaryImage.id]); expect(personMock.getDistinctNames).toHaveBeenCalledWith(assetStub.primaryImage.ownerId, { withHidden: true }); expect(personMock.create).toHaveBeenCalledWith([]); - expect(personMock.replaceFaces).toHaveBeenCalledWith( + expect(personMock.refreshFaces).toHaveBeenCalledWith( assetStub.primaryImage.id, [ { diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts index 58e7b994480ac..97cb3b1a5c6a0 100644 --- a/server/src/services/metadata.service.ts +++ b/server/src/services/metadata.service.ts @@ -219,7 +219,7 @@ export class MetadataService { async handleMetadataExtraction({ id }: IEntityJob): Promise { const { metadata } = await this.configCore.getConfig({ withCache: true }); - const [asset] = await this.assetRepository.getByIds([id]); + const [asset] = await this.assetRepository.getByIds([id], { faces: { person: false } }); if (!asset) { return JobStatus.FAILED; } @@ -522,7 +522,7 @@ export class MetadataService { return; } - const discoveredFaces: Partial[] = []; + const facesToAdd: Partial[] = []; const existingNames = await this.personRepository.getDistinctNames(asset.ownerId, { withHidden: true }); const existingNameMap = new Map(existingNames.map(({ id, name }) => [name.toLowerCase(), id])); const missing: Partial[] = []; @@ -550,7 +550,7 @@ export class MetadataService { sourceType: SourceType.EXIF, }; - discoveredFaces.push(face); + facesToAdd.push(face); if (!existingNameMap.has(loweredName)) { missing.push({ id: personId, ownerId: asset.ownerId, name: region.Name }); missingWithFaceAsset.push({ id: personId, faceAssetId: face.id }); @@ -559,21 +559,29 @@ export class MetadataService { if (missing.length > 0) { this.logger.debug(`Creating missing persons: ${missing.map((p) => `${p.name}/${p.id}`)}`); + const newPersons = await this.personRepository.create(missing); + const jobs = newPersons.map( + (person) => ({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: person.id } }) as const, + ); + await this.jobRepository.queueAll(jobs); } - const newPersons = await this.personRepository.create(missing); + const facesToRemove = asset.faces.filter((face) => face.sourceType === SourceType.EXIF).map((face) => face.id); + if (facesToRemove.length > 0) { + this.logger.debug(`Removing ${facesToRemove.length} faces for asset ${asset.id}`); + } - const faceIds = await this.personRepository.replaceFaces(asset.id, discoveredFaces, SourceType.EXIF); - this.logger.debug(`Created ${faceIds.length} faces for asset ${asset.id}`); + if (facesToAdd.length > 0) { + this.logger.debug(`Creating ${facesToAdd} faces from metadata for asset ${asset.id}`); + } - await this.personRepository.update(missingWithFaceAsset); + if (facesToRemove.length > 0 || facesToAdd.length > 0) { + await this.personRepository.refreshFaces(facesToAdd, facesToRemove); + } - await this.jobRepository.queueAll( - newPersons.map((person) => ({ - name: JobName.GENERATE_PERSON_THUMBNAIL, - data: { id: person.id }, - })), - ); + if (missingWithFaceAsset.length > 0) { + await this.personRepository.update(missingWithFaceAsset); + } } private async exifData( diff --git a/server/src/services/person.service.ts b/server/src/services/person.service.ts index c4b5df5719352..36364c8b050eb 100644 --- a/server/src/services/person.service.ts +++ b/server/src/services/person.service.ts @@ -23,6 +23,7 @@ import { } from 'src/dtos/person.dto'; import { AssetFaceEntity } from 'src/entities/asset-face.entity'; import { AssetEntity } from 'src/entities/asset.entity'; +import { FaceSearchEntity } from 'src/entities/face-search.entity'; import { PersonPathType } from 'src/entities/move.entity'; import { PersonEntity } from 'src/entities/person.entity'; import { AssetType, Permission, SourceType, SystemMetadataKey } from 'src/enum'; @@ -301,14 +302,14 @@ export class PersonService { } const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => { - return force - ? this.assetRepository.getAll(pagination, { + return force === false + ? this.assetRepository.getWithout(pagination, WithoutProperty.FACES) + : this.assetRepository.getAll(pagination, { orderDirection: 'DESC', withFaces: true, withArchived: true, isVisible: true, - }) - : this.assetRepository.getWithout(pagination, WithoutProperty.FACES); + }); }); for await (const assets of assetPagination) { @@ -317,6 +318,10 @@ export class PersonService { ); } + if (force === undefined) { + await this.jobRepository.queue({ name: JobName.PERSON_CLEANUP }); + } + return JobStatus.SUCCESS; } @@ -335,11 +340,11 @@ export class PersonService { }; const [asset] = await this.assetRepository.getByIds([id], relations); const { previewFile } = getAssetFiles(asset.files); - if (!asset || !previewFile || asset.faces?.length > 0) { + if (!asset || !previewFile) { return JobStatus.FAILED; } - if (!asset.isVisible || asset.faces.length > 0) { + if (!asset.isVisible) { return JobStatus.SKIPPED; } @@ -348,29 +353,63 @@ export class PersonService { previewFile.path, machineLearning.facialRecognition, ); - this.logger.debug(`${faces.length} faces detected in ${previewFile.path}`); - if (faces.length > 0) { - await this.jobRepository.queue({ name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false } }); - const mappedFaces: Partial[] = []; - for (const face of faces) { + const facesToAdd: Partial[] = []; + const embeddingsToAdd: FaceSearchEntity[] = []; + const facesToRemove = new Map(); + for (const face of asset.faces) { + facesToRemove.set(face.id, face); + } + + const heightScale = imageHeight / (asset.faces[0]?.imageHeight || 1); + const widthScale = imageWidth / (asset.faces[0]?.imageWidth || 1); + for (const { boundingBox, embedding } of faces) { + const scaledBox = { + x1: boundingBox.x1 * widthScale, + y1: boundingBox.y1 * heightScale, + x2: boundingBox.x2 * widthScale, + y2: boundingBox.y2 * heightScale, + }; + const match = asset.faces.find((face) => this.iou(face, scaledBox) > 0.5); + + if (match) { + const existing = facesToRemove.get(match.id)!; + facesToRemove.delete(existing.id); + if (existing.sourceType !== SourceType.MACHINE_LEARNING) { + embeddingsToAdd.push({ faceId: existing.id, embedding }); + } + } else { const faceId = this.cryptoRepository.randomUUID(); - mappedFaces.push({ + facesToAdd.push({ id: faceId, assetId: asset.id, imageHeight, imageWidth, - boundingBoxX1: face.boundingBox.x1, - boundingBoxY1: face.boundingBox.y1, - boundingBoxX2: face.boundingBox.x2, - boundingBoxY2: face.boundingBox.y2, - faceSearch: { faceId, embedding: face.embedding }, + boundingBoxX1: boundingBox.x1, + boundingBoxY1: boundingBox.y1, + boundingBoxX2: boundingBox.x2, + boundingBoxY2: boundingBox.y2, }); + embeddingsToAdd.push({ faceId, embedding }); } + } + const faceIdsToRemove = [...facesToRemove.values()].map((face) => face.id); + + if (facesToAdd.length > 0 || faceIdsToRemove.length > 0) { + await this.repository.refreshFaces(facesToAdd, faceIdsToRemove, embeddingsToAdd); + } - const faceIds = await this.repository.createFaces(mappedFaces); - await this.jobRepository.queueAll(faceIds.map((id) => ({ name: JobName.FACIAL_RECOGNITION, data: { id } }))); + if (faceIdsToRemove.length > 0) { + this.logger.log(`Removed ${faceIdsToRemove.length} faces below detection threshold in asset ${id}`); + } + + if (facesToAdd.length > 0) { + this.logger.log(`Detected ${facesToAdd.length} new faces in asset ${id}`); + await this.jobRepository.queueAll([ + { name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false } }, + ...facesToAdd.map((face) => ({ name: JobName.FACIAL_RECOGNITION, data: { id: face.id! } }) as const), + ]); } await this.assetRepository.upsertJobStatus({ @@ -381,6 +420,20 @@ export class PersonService { return JobStatus.SUCCESS; } + private iou(face: AssetFaceEntity, newBox: BoundingBox): number { + const x1 = Math.max(face.boundingBoxX1, newBox.x1); + const y1 = Math.max(face.boundingBoxY1, newBox.y1); + const x2 = Math.min(face.boundingBoxX2, newBox.x2); + const y2 = Math.min(face.boundingBoxY2, newBox.y2); + + const intersection = Math.max(0, x2 - x1) * Math.max(0, y2 - y1); + const area1 = (face.boundingBoxX2 - face.boundingBoxX1) * (face.boundingBoxY2 - face.boundingBoxY1); + const area2 = (newBox.x2 - newBox.x1) * (newBox.y2 - newBox.y1); + const union = area1 + area2 - intersection; + + return intersection / union; + } + async handleQueueRecognizeFaces({ force, nightly }: INightlyJob): Promise { const { machineLearning } = await this.configCore.getConfig({ withCache: false }); if (!isFacialRecognitionEnabled(machineLearning)) { diff --git a/server/test/repositories/person.repository.mock.ts b/server/test/repositories/person.repository.mock.ts index 6547a543390a0..e91d283e399cc 100644 --- a/server/test/repositories/person.repository.mock.ts +++ b/server/test/repositories/person.repository.mock.ts @@ -25,7 +25,7 @@ export const newPersonRepositoryMock = (): Mocked => { reassignFaces: vitest.fn(), createFaces: vitest.fn(), - replaceFaces: vitest.fn(), + refreshFaces: vitest.fn(), getFaces: vitest.fn(), reassignFace: vitest.fn(), getFaceById: vitest.fn(), diff --git a/web/src/lib/components/admin-page/jobs/job-tile-button.svelte b/web/src/lib/components/admin-page/jobs/job-tile-button.svelte index 0aa90ed4d8ae3..69d3706230de9 100644 --- a/web/src/lib/components/admin-page/jobs/job-tile-button.svelte +++ b/web/src/lib/components/admin-page/jobs/job-tile-button.svelte @@ -1,5 +1,5 @@
- {#each jobList as [jobName, { title, subtitle, description, disabled, allText, missingText, allowForceCommand, icon, handleCommand: handleCommandOverride }]} + {#each jobList as [jobName, { title, subtitle, description, disabled, allText, refreshText, missingText, allowForceCommand, icon, handleCommand: handleCommandOverride }]} {@const { jobCounts, queueStatus } = jobs[jobName]} (handleCommandOverride || handleCommand)(jobName, detail)} diff --git a/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte b/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte index db216641d5c2f..d82382250503d 100644 --- a/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte +++ b/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte @@ -34,6 +34,7 @@ mdiContentCopy, mdiDatabaseRefreshOutline, mdiDotsVertical, + mdiHeadSyncOutline, mdiImageRefreshOutline, mdiImageSearch, mdiMagnifyMinusOutline, @@ -167,6 +168,11 @@ /> {/if}
+ onRunJob(AssetJobName.RefreshFaces)} + text={$getAssetJobName(AssetJobName.RefreshFaces)} + /> onRunJob(AssetJobName.RefreshMetadata)} diff --git a/web/src/lib/i18n/en.json b/web/src/lib/i18n/en.json index b8f0032836665..b3639830ef050 100644 --- a/web/src/lib/i18n/en.json +++ b/web/src/lib/i18n/en.json @@ -47,8 +47,8 @@ "external_library_created_at": "External library (created on {date})", "external_library_management": "External Library Management", "face_detection": "Face detection", - "face_detection_description": "Detect the faces in assets using machine learning. For videos, only the thumbnail is considered. \"All\" (re-)processes all assets. \"Missing\" queues assets that haven't been processed yet. Detected faces will be queued for Facial Recognition after Face Detection is complete, grouping them into existing or new people.", - "facial_recognition_job_description": "Group detected faces into people. This step runs after Face Detection is complete. \"All\" (re-)clusters all faces. \"Missing\" queues faces that don't have a person assigned.", + "face_detection_description": "Detect the faces in assets using machine learning. For videos, only the thumbnail is considered. \"Refresh\" (re-)processes all assets. \"Reset\" additionally clears all current face data. \"Missing\" queues assets that haven't been processed yet. Detected faces will be queued for Facial Recognition after Face Detection is complete, grouping them into existing or new people.", + "facial_recognition_job_description": "Group detected faces into people. This step runs after Face Detection is complete. \"Reset\" (re-)clusters all faces. \"Missing\" queues faces that don't have a person assigned.", "failed_job_command": "Command {command} failed for job: {job}", "force_delete_user_warning": "WARNING: This will immediately remove the user and all assets. This cannot be undone and the files cannot be recovered.", "forcing_refresh_library_files": "Forcing refresh of all library files", @@ -1000,11 +1000,13 @@ "recent_searches": "Recent searches", "refresh": "Refresh", "refresh_encoded_videos": "Refresh encoded videos", + "refresh_faces": "Refresh faces", "refresh_metadata": "Refresh metadata", "refresh_thumbnails": "Refresh thumbnails", "refreshed": "Refreshed", "refreshes_every_file": "Refreshes every file", "refreshing_encoded_video": "Refreshing encoded video", + "refreshing_faces": "Refreshing faces", "refreshing_metadata": "Refreshing metadata", "regenerating_thumbnails": "Regenerating thumbnails", "remove": "Remove", diff --git a/web/src/lib/utils.ts b/web/src/lib/utils.ts index 395d9796f4fb3..40fd66d8f9bf8 100644 --- a/web/src/lib/utils.ts +++ b/web/src/lib/utils.ts @@ -20,7 +20,7 @@ import { type PersonResponseDto, type SharedLinkResponseDto, } from '@immich/sdk'; -import { mdiCogRefreshOutline, mdiDatabaseRefreshOutline, mdiImageRefreshOutline } from '@mdi/js'; +import { mdiCogRefreshOutline, mdiDatabaseRefreshOutline, mdiHeadSyncOutline, mdiImageRefreshOutline } from '@mdi/js'; import { sortBy } from 'lodash-es'; import { init, register, t } from 'svelte-i18n'; import { derived, get } from 'svelte/store'; @@ -212,6 +212,7 @@ export const getPeopleThumbnailUrl = (person: PersonResponseDto, updatedAt?: str export const getAssetJobName = derived(t, ($t) => { return (job: AssetJobName) => { const names: Record = { + [AssetJobName.RefreshFaces]: $t('refresh_faces'), [AssetJobName.RefreshMetadata]: $t('refresh_metadata'), [AssetJobName.RegenerateThumbnail]: $t('refresh_thumbnails'), [AssetJobName.TranscodeVideo]: $t('refresh_encoded_videos'), @@ -224,6 +225,7 @@ export const getAssetJobName = derived(t, ($t) => { export const getAssetJobMessage = derived(t, ($t) => { return (job: AssetJobName) => { const messages: Record = { + [AssetJobName.RefreshFaces]: $t('refreshing_faces'), [AssetJobName.RefreshMetadata]: $t('refreshing_metadata'), [AssetJobName.RegenerateThumbnail]: $t('regenerating_thumbnails'), [AssetJobName.TranscodeVideo]: $t('refreshing_encoded_video'), @@ -235,6 +237,7 @@ export const getAssetJobMessage = derived(t, ($t) => { export const getAssetJobIcon = (job: AssetJobName) => { const names: Record = { + [AssetJobName.RefreshFaces]: mdiHeadSyncOutline, [AssetJobName.RefreshMetadata]: mdiDatabaseRefreshOutline, [AssetJobName.RegenerateThumbnail]: mdiImageRefreshOutline, [AssetJobName.TranscodeVideo]: mdiCogRefreshOutline,