Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(server): refresh face detection #12335

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
+250 −146
Draft

feat(server): refresh face detection #12335

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions mobile/openapi/lib/model/asset_job_name.dart
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

19 changes: 14 additions & 5 deletions mobile/openapi/lib/model/job_command_dto.dart
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 3 additions & 3 deletions open-api/immich-openapi-specs.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8170,8 +8170,9 @@
},
"AssetJobName": {
"enum": [
"regenerate-thumbnail",
"refresh-faces",
"refresh-metadata",
"regenerate-thumbnail",
"transcode-video"
],
"type": "string"
Expand Down Expand Up @@ -9230,8 +9231,7 @@
}
},
"required": [
"command",
"force"
"command"
],
"type": "object"
},
Expand Down
5 changes: 3 additions & 2 deletions open-api/typescript-sdk/src/fetch-client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -548,7 +548,7 @@ export type AllJobStatusResponseDto = {
};
export type JobCommandDto = {
command: JobCommand;
force: boolean;
force?: boolean;
};
export type LibraryResponseDto = {
assetCount: number;
Expand Down Expand Up @@ -3350,8 +3350,9 @@ export enum Reason {
UnsupportedFormat = "unsupported-format"
}
export enum AssetJobName {
RegenerateThumbnail = "regenerate-thumbnail",
RefreshFaces = "refresh-faces",
RefreshMetadata = "refresh-metadata",
RegenerateThumbnail = "regenerate-thumbnail",
TranscodeVideo = "transcode-video"
}
export enum AssetMediaSize {
Expand Down
3 changes: 2 additions & 1 deletion server/src/dtos/asset.dto.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,9 @@ export class AssetIdsDto {
}

export enum AssetJobName {
REGENERATE_THUMBNAIL = 'regenerate-thumbnail',
REFRESH_FACES = 'refresh-faces',
REFRESH_METADATA = 'refresh-metadata',
REGENERATE_THUMBNAIL = 'regenerate-thumbnail',
TRANSCODE_VIDEO = 'transcode-video',
}

Expand Down
2 changes: 1 addition & 1 deletion server/src/dtos/job.dto.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ export class JobCommandDto {
command!: JobCommand;

@ValidateBoolean({ optional: true })
force!: boolean;
force?: boolean;
}

export class JobCountsDto {
Expand Down
7 changes: 6 additions & 1 deletion server/src/interfaces/person.interface.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
import { AssetEntity } from 'src/entities/asset.entity';
import { FaceSearchEntity } from 'src/entities/face-search.entity';
import { PersonEntity } from 'src/entities/person.entity';
import { Paginated, PaginationOptions } from 'src/utils/pagination';
import { FindManyOptions, FindOptionsRelations, FindOptionsSelect } from 'typeorm';
Expand Down Expand Up @@ -59,7 +60,11 @@ export interface IPersonRepository {
delete(entities: PersonEntity[]): Promise<void>;
deleteAll(): Promise<void>;
deleteAllFaces(options: DeleteAllFacesOptions): Promise<void>;
replaceFaces(assetId: string, entities: Partial<AssetFaceEntity>[], sourceType?: string): Promise<string[]>;
refreshFaces(
facesToAdd: Partial<AssetFaceEntity>[],
faceIdsToRemove: string[],
embeddingsToAdd?: FaceSearchEntity[],
): Promise<void>;
getAllFaces(pagination: PaginationOptions, options?: FindManyOptions<AssetFaceEntity>): Paginated<AssetFaceEntity>;
getFaceById(id: string): Promise<AssetFaceEntity>;
getFaceByIdWithAssets(
Expand Down
56 changes: 0 additions & 56 deletions server/src/queries/metadata.repository.sql
View file
Open in desktop

This file was deleted.

37 changes: 31 additions & 6 deletions server/src/repositories/person.repository.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { ChunkedArray, DummyValue, GenerateSql } from 'src/decorators';
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity';
import { AssetEntity } from 'src/entities/asset.entity';
import { FaceSearchEntity } from 'src/entities/face-search.entity';
import { PersonEntity } from 'src/entities/person.entity';
import { SourceType } from 'src/enum';
import {
Expand All @@ -30,6 +31,7 @@ export class PersonRepository implements IPersonRepository {
@InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>,
@InjectRepository(PersonEntity) private personRepository: Repository<PersonEntity>,
@InjectRepository(AssetFaceEntity) private assetFaceRepository: Repository<AssetFaceEntity>,
@InjectRepository(FaceSearchEntity) private faceSearchRepository: Repository<FaceSearchEntity>,
@InjectRepository(AssetJobStatusEntity) private jobStatusRepository: Repository<AssetJobStatusEntity>,
) {}

Expand Down Expand Up @@ -289,12 +291,35 @@ export class PersonRepository implements IPersonRepository {
return res.map((row) => row.id);
}

async replaceFaces(assetId: string, entities: AssetFaceEntity[], sourceType: string): Promise<string[]> {
return this.dataSource.transaction(async (manager) => {
await manager.delete(AssetFaceEntity, { assetId, sourceType });
const assetFaces = await manager.save(AssetFaceEntity, entities);
return assetFaces.map(({ id }) => id);
});
async refreshFaces(
facesToAdd: Partial<AssetFaceEntity>[],
faceIdsToRemove: string[],
embeddingsToAdd?: FaceSearchEntity[],
): Promise<void> {
const faceDeleteQuery = this.assetFaceRepository
.createQueryBuilder()
.delete()
.where('id = any(:faceIdsToRemove)', { faceIdsToRemove });

if (facesToAdd.length === 0) {
await faceDeleteQuery.execute();
return;
}

const faceInsertQuery = this.assetFaceRepository.createQueryBuilder().insert().values(facesToAdd);
if (!embeddingsToAdd || embeddingsToAdd.length === 0) {
await faceInsertQuery.addCommonTableExpression(faceDeleteQuery, 'deleted').execute();
return;
}

await this.faceSearchRepository
.createQueryBuilder()
.addCommonTableExpression(faceDeleteQuery, 'deleted')
.addCommonTableExpression(faceInsertQuery, 'added')
.insert()
.values(embeddingsToAdd)
.orIgnore()
.execute();
}

async update(entities: Partial<PersonEntity>[]): Promise<PersonEntity[]> {
Expand Down
7 changes: 6 additions & 1 deletion server/src/services/asset.service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,9 +113,9 @@ export class AssetService {
id,
{
exifInfo: true,
tags: true,
sharedLinks: true,
smartInfo: true,
tags: true,
owner: true,
faces: {
person: true,
Expand Down Expand Up @@ -298,6 +298,11 @@ export class AssetService {

for (const id of dto.assetIds) {
switch (dto.name) {
case AssetJobName.REFRESH_FACES: {
jobs.push({ name: JobName.FACE_DETECTION, data: { id } });
break;
}

case AssetJobName.REFRESH_METADATA: {
jobs.push({ name: JobName.METADATA_EXTRACTION, data: { id } });
break;
Expand Down
16 changes: 8 additions & 8 deletions server/src/services/metadata.service.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,7 @@
it('should handle an asset that could not be found', async () => {
await expect(sut.handleMetadataExtraction({ id: assetStub.image.id })).resolves.toBe(JobStatus.FAILED);

expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);

Check failure on line 290 in server/src/services/metadata.service.spec.ts

View workflow job for this annotation

GitHub Actions / Test & Lint Server

src/services/metadata.service.spec.ts > MetadataService > handleMetadataExtraction > should handle an asset that could not be found 

AssertionError: expected "spy" to be called with arguments: [ [ 'asset-id' ] ]

Received: 

  1st spy call:

  Array [
    Array [
      "asset-id",
    ],
+   Object {
+     "faces": Object {
+       "person": false,
+     },
+   },
  ]


Number of calls: 1

 ❯ src/services/metadata.service.spec.ts:290:34
expect(assetMock.upsertExif).not.toHaveBeenCalled();
expect(assetMock.update).not.toHaveBeenCalled();
});
Expand All @@ -305,7 +305,7 @@
});

await sut.handleMetadataExtraction({ id: assetStub.image.id });
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.sidecar.id]);

Check failure on line 308 in server/src/services/metadata.service.spec.ts

View workflow job for this annotation

GitHub Actions / Test & Lint Server

src/services/metadata.service.spec.ts > MetadataService > handleMetadataExtraction > should handle a date in a sidecar file 

AssertionError: expected "spy" to be called with arguments: [ [ 'asset-id' ] ]

Received: 

  1st spy call:

  Array [
    Array [
      "asset-id",
    ],
+   Object {
+     "faces": Object {
+       "person": false,
+     },
+   },
  ]


Number of calls: 1

 ❯ src/services/metadata.service.spec.ts:308:34
expect(assetMock.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ dateTimeOriginal: sidecarDate }));
expect(assetMock.update).toHaveBeenCalledWith({
id: assetStub.image.id,
Expand All @@ -320,7 +320,7 @@
metadataMock.readTags.mockResolvedValue({ ISO: [160] as any });

await sut.handleMetadataExtraction({ id: assetStub.image.id });
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);

Check failure on line 323 in server/src/services/metadata.service.spec.ts

View workflow job for this annotation

GitHub Actions / Test & Lint Server

src/services/metadata.service.spec.ts > MetadataService > handleMetadataExtraction > should handle lists of numbers 

AssertionError: expected "spy" to be called with arguments: [ [ 'asset-id' ] ]

Received: 

  1st spy call:

  Array [
    Array [
      "asset-id",
    ],
+   Object {
+     "faces": Object {
+       "person": false,
+     },
+   },
  ]


Number of calls: 1

 ❯ src/services/metadata.service.spec.ts:323:34
expect(assetMock.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ iso: 160 }));
expect(assetMock.update).toHaveBeenCalledWith({
id: assetStub.image.id,
Expand All @@ -340,7 +340,7 @@
});

await sut.handleMetadataExtraction({ id: assetStub.image.id });
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);

Check failure on line 343 in server/src/services/metadata.service.spec.ts

View workflow job for this annotation

GitHub Actions / Test & Lint Server

src/services/metadata.service.spec.ts > MetadataService > handleMetadataExtraction > should apply reverse geocoding 

AssertionError: expected "spy" to be called with arguments: [ [ 'asset-id' ] ]

Received: 

  1st spy call:

  Array [
    Array [
      "asset-id",
    ],
+   Object {
+     "faces": Object {
+       "person": false,
+     },
+   },
  ]


Number of calls: 1

 ❯ src/services/metadata.service.spec.ts:343:34
expect(assetMock.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({ city: 'City', state: 'State', country: 'Country' }),
);
Expand All @@ -360,7 +360,7 @@
});

await sut.handleMetadataExtraction({ id: assetStub.image.id });
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);

Check failure on line 363 in server/src/services/metadata.service.spec.ts

View workflow job for this annotation

GitHub Actions / Test & Lint Server

src/services/metadata.service.spec.ts > MetadataService > handleMetadataExtraction > should discard latitude and longitude on null island 

AssertionError: expected "spy" to be called with arguments: [ [ 'asset-id' ] ]

Received: 

  1st spy call:

  Array [
    Array [
      "asset-id",
    ],
+   Object {
+     "faces": Object {
+       "person": false,
+     },
+   },
  ]


Number of calls: 1

 ❯ src/services/metadata.service.spec.ts:363:34
expect(assetMock.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ latitude: null, longitude: null }));
});

Expand Down Expand Up @@ -510,7 +510,7 @@
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);

await sut.handleMetadataExtraction({ id: assetStub.livePhotoMotionAsset.id });
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoMotionAsset.id]);

Check failure on line 513 in server/src/services/metadata.service.spec.ts

View workflow job for this annotation

GitHub Actions / Test & Lint Server

src/services/metadata.service.spec.ts > MetadataService > handleMetadataExtraction > should not apply motion photos if asset is video 

AssertionError: expected "spy" to be called with arguments: [ [ 'live-photo-motion-asset' ] ]

Received: 

  1st spy call:

  Array [
    Array [
      "live-photo-motion-asset",
    ],
+   Object {
+     "faces": Object {
+       "person": false,
+     },
+   },
  ]


Number of calls: 1

 ❯ src/services/metadata.service.spec.ts:513:34
expect(storageMock.writeFile).not.toHaveBeenCalled();
expect(jobMock.queue).not.toHaveBeenCalled();
expect(jobMock.queueAll).not.toHaveBeenCalled();
Expand All @@ -526,7 +526,7 @@

await sut.handleMetadataExtraction({ id: assetStub.video.id });

expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.video.id]);

Check failure on line 529 in server/src/services/metadata.service.spec.ts

View workflow job for this annotation

GitHub Actions / Test & Lint Server

src/services/metadata.service.spec.ts > MetadataService > handleMetadataExtraction > should extract the correct video orientation 

AssertionError: expected "spy" to be called with arguments: [ [ 'asset-id' ] ]

Received: 

  1st spy call:

  Array [
    Array [
      "asset-id",
    ],
+   Object {
+     "faces": Object {
+       "person": false,
+     },
+   },
  ]


Number of calls: 1

 ❯ src/services/metadata.service.spec.ts:529:34
expect(assetMock.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({ orientation: Orientation.Rotate270CW }),
);
Expand Down Expand Up @@ -554,7 +554,7 @@
assetStub.livePhotoWithOriginalFileName.originalPath,
'MotionPhotoVideo',
);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoWithOriginalFileName.id]);

Check failure on line 557 in server/src/services/metadata.service.spec.ts

View workflow job for this annotation

GitHub Actions / Test & Lint Server

src/services/metadata.service.spec.ts > MetadataService > handleMetadataExtraction > should extract the MotionPhotoVideo tag from Samsung HEIC motion photos 

AssertionError: expected "spy" to be called with arguments: [ [ 'live-photo-still-asset' ] ]

Received: 

  1st spy call:

  Array [
    Array [
      "live-photo-still-asset",
    ],
+   Object {
+     "faces": Object {
+       "person": false,
+     },
+   },
  ]


Number of calls: 1

 ❯ src/services/metadata.service.spec.ts:557:34
expect(assetMock.create).toHaveBeenCalledWith({
checksum: expect.any(Buffer),
deviceAssetId: 'NONE',
Expand Down Expand Up @@ -597,7 +597,7 @@
assetStub.livePhotoWithOriginalFileName.originalPath,
'EmbeddedVideoFile',
);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoWithOriginalFileName.id]);

Check failure on line 600 in server/src/services/metadata.service.spec.ts

View workflow job for this annotation

GitHub Actions / Test & Lint Server

src/services/metadata.service.spec.ts > MetadataService > handleMetadataExtraction > should extract the EmbeddedVideo tag from Samsung JPEG motion photos 

AssertionError: expected "spy" to be called with arguments: [ [ 'live-photo-still-asset' ] ]

Received: 

  1st spy call:

  Array [
    Array [
      "live-photo-still-asset",
    ],
+   Object {
+     "faces": Object {
+       "person": false,
+     },
+   },
  ]


Number of calls: 1

 ❯ src/services/metadata.service.spec.ts:600:34
expect(assetMock.create).toHaveBeenCalledWith({
checksum: expect.any(Buffer),
deviceAssetId: 'NONE',
Expand Down Expand Up @@ -637,7 +637,7 @@
storageMock.readFile.mockResolvedValue(video);

await sut.handleMetadataExtraction({ id: assetStub.livePhotoWithOriginalFileName.id });
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoWithOriginalFileName.id]);

Check failure on line 640 in server/src/services/metadata.service.spec.ts

View workflow job for this annotation

GitHub Actions / Test & Lint Server

src/services/metadata.service.spec.ts > MetadataService > handleMetadataExtraction > should extract the motion photo video from the XMP directory entry 

AssertionError: expected "spy" to be called with arguments: [ [ 'live-photo-still-asset' ] ]

Received: 

  1st spy call:

  Array [
    Array [
      "live-photo-still-asset",
    ],
+   Object {
+     "faces": Object {
+       "person": false,
+     },
+   },
  ]


Number of calls: 1

 ❯ src/services/metadata.service.spec.ts:640:34
expect(storageMock.readFile).toHaveBeenCalledWith(
assetStub.livePhotoWithOriginalFileName.originalPath,
expect.any(Object),
Expand Down Expand Up @@ -990,11 +990,11 @@
metadataMock.readTags.mockResolvedValue(metadataStub.withFaceNoName);
personMock.getDistinctNames.mockResolvedValue([]);
personMock.create.mockResolvedValue([]);
personMock.replaceFaces.mockResolvedValue([]);
personMock.refreshFaces.mockResolvedValue([]);
personMock.update.mockResolvedValue([]);
await sut.handleMetadataExtraction({ id: assetStub.image.id });
expect(personMock.create).toHaveBeenCalledWith([]);
expect(personMock.replaceFaces).toHaveBeenCalledWith(assetStub.primaryImage.id, [], SourceType.EXIF);
expect(personMock.refreshFaces).toHaveBeenCalledWith(assetStub.primaryImage.id, [], SourceType.EXIF);
expect(personMock.update).toHaveBeenCalledWith([]);
});

Expand All @@ -1004,11 +1004,11 @@
metadataMock.readTags.mockResolvedValue(metadataStub.withFaceEmptyName);
personMock.getDistinctNames.mockResolvedValue([]);
personMock.create.mockResolvedValue([]);
personMock.replaceFaces.mockResolvedValue([]);
personMock.refreshFaces.mockResolvedValue([]);
personMock.update.mockResolvedValue([]);
await sut.handleMetadataExtraction({ id: assetStub.image.id });
expect(personMock.create).toHaveBeenCalledWith([]);
expect(personMock.replaceFaces).toHaveBeenCalledWith(assetStub.primaryImage.id, [], SourceType.EXIF);
expect(personMock.refreshFaces).toHaveBeenCalledWith(assetStub.primaryImage.id, [], SourceType.EXIF);
expect(personMock.update).toHaveBeenCalledWith([]);
});

Expand All @@ -1018,13 +1018,13 @@
metadataMock.readTags.mockResolvedValue(metadataStub.withFace);
personMock.getDistinctNames.mockResolvedValue([]);
personMock.create.mockResolvedValue([personStub.withName]);
personMock.replaceFaces.mockResolvedValue(['face-asset-uuid']);
personMock.refreshFaces.mockResolvedValue(['face-asset-uuid']);
personMock.update.mockResolvedValue([personStub.withName]);
await sut.handleMetadataExtraction({ id: assetStub.primaryImage.id });
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.primaryImage.id]);
expect(personMock.getDistinctNames).toHaveBeenCalledWith(assetStub.primaryImage.ownerId, { withHidden: true });
expect(personMock.create).toHaveBeenCalledWith([expect.objectContaining({ name: personStub.withName.name })]);
expect(personMock.replaceFaces).toHaveBeenCalledWith(
expect(personMock.refreshFaces).toHaveBeenCalledWith(
assetStub.primaryImage.id,
[
{
Expand Down Expand Up @@ -1057,13 +1057,13 @@
metadataMock.readTags.mockResolvedValue(metadataStub.withFace);
personMock.getDistinctNames.mockResolvedValue([{ id: personStub.withName.id, name: personStub.withName.name }]);
personMock.create.mockResolvedValue([]);
personMock.replaceFaces.mockResolvedValue(['face-asset-uuid']);
personMock.refreshFaces.mockResolvedValue(['face-asset-uuid']);
personMock.update.mockResolvedValue([personStub.withName]);
await sut.handleMetadataExtraction({ id: assetStub.primaryImage.id });
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.primaryImage.id]);
expect(personMock.getDistinctNames).toHaveBeenCalledWith(assetStub.primaryImage.ownerId, { withHidden: true });
expect(personMock.create).toHaveBeenCalledWith([]);
expect(personMock.replaceFaces).toHaveBeenCalledWith(
expect(personMock.refreshFaces).toHaveBeenCalledWith(
assetStub.primaryImage.id,
[
{
Expand Down
Loading
Loading