From d5eedbab0de04b189379dd9839ef256aab482a20 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Sat, 2 Mar 2024 16:26:54 -0800 Subject: [PATCH] Merge --- src-ui/src/app/app.module.ts | 4 + .../merge-confirm-dialog.component.html | 50 +++++++++++++ .../merge-confirm-dialog.component.scss | 3 + .../merge-confirm-dialog.component.spec.ts | 73 +++++++++++++++++++ .../merge-confirm-dialog.component.ts | 51 +++++++++++++ .../bulk-editor/bulk-editor.component.html | 3 + .../bulk-editor/bulk-editor.component.spec.ts | 42 ++++++++++- .../bulk-editor/bulk-editor.component.ts | 40 +++++++++- src/documents/bulk_edit.py | 69 ++++++++++++++++++ src/documents/serialisers.py | 3 + src/documents/tests/test_api_bulk_edit.py | 23 ++++++ 11 files changed, 357 insertions(+), 4 deletions(-) create mode 100644 src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.html create mode 100644 src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.scss create mode 100644 src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.spec.ts create mode 100644 src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.ts diff --git a/src-ui/src/app/app.module.ts b/src-ui/src/app/app.module.ts index 857bf2873..26c890a79 100644 --- a/src-ui/src/app/app.module.ts +++ b/src-ui/src/app/app.module.ts @@ -117,6 +117,7 @@ import { MonetaryComponent } from './components/common/input/monetary/monetary.c import { SystemStatusDialogComponent } from './components/common/system-status-dialog/system-status-dialog.component' import { NgxFilesizeModule } from 'ngx-filesize' import { RotateConfirmDialogComponent } from './components/common/confirm-dialog/rotate-confirm-dialog/rotate-confirm-dialog.component' +import { MergeConfirmDialogComponent } from './components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component' import { airplane, archive, @@ -177,6 +178,7 @@ import { hddStack, house, infoCircle, + journals, link, listTask, listUl, @@ -269,6 +271,7 @@ const icons = { hddStack, house, infoCircle, + journals, link, listTask, listUl, @@ -464,6 +467,7 @@ function initializeApp(settings: SettingsService) { MonetaryComponent, SystemStatusDialogComponent, RotateConfirmDialogComponent, + MergeConfirmDialogComponent, ], imports: [ BrowserModule, diff --git a/src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.html b/src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.html new file mode 100644 index 000000000..5dc902283 --- /dev/null +++ b/src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.html @@ -0,0 +1,50 @@ +
+ + +
+
+

{{message}}

+
+ + +
+
+ + +
+

Note that only PDFs will be included.

+
+
+ + + @if (alternativeBtnCaption) { + + } +
diff --git a/src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.scss b/src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.scss new file mode 100644 index 000000000..c780e5a35 --- /dev/null +++ b/src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.scss @@ -0,0 +1,3 @@ +.list-group-item { + cursor: move; +} diff --git a/src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.spec.ts b/src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.spec.ts new file mode 100644 index 000000000..8b9bf3898 --- /dev/null +++ b/src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.spec.ts @@ -0,0 +1,73 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing' +import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap' +import { MergeConfirmDialogComponent } from './merge-confirm-dialog.component' +import { HttpClientTestingModule } from '@angular/common/http/testing' +import { NgxBootstrapIconsModule, allIcons } from 'ngx-bootstrap-icons' +import { of } from 'rxjs' +import { DocumentService } from 'src/app/services/rest/document.service' +import { FormsModule, ReactiveFormsModule } from '@angular/forms' + +describe('MergeConfirmDialogComponent', () => { + let component: MergeConfirmDialogComponent + let fixture: ComponentFixture + let documentService: DocumentService + + beforeEach(async () => { + await TestBed.configureTestingModule({ + declarations: [MergeConfirmDialogComponent], + providers: [NgbActiveModal], + imports: [ + HttpClientTestingModule, + NgxBootstrapIconsModule.pick(allIcons), + ReactiveFormsModule, + FormsModule, + ], + }).compileComponents() + + fixture = TestBed.createComponent(MergeConfirmDialogComponent) + documentService = TestBed.inject(DocumentService) + component = fixture.componentInstance + fixture.detectChanges() + }) + + it('should fetch documents on ngOnInit', () => { + const documents = [ + { id: 1, name: 'Document 1' }, + { id: 2, name: 'Document 2' }, + { id: 3, name: 'Document 3' }, + ] + jest.spyOn(documentService, 'getCachedMany').mockReturnValue(of(documents)) + + component.ngOnInit() + + expect(component.documents).toEqual(documents) + expect(documentService.getCachedMany).toHaveBeenCalledWith( + component.documentIDs + ) + }) + + it('should move documentIDs on drop', () => { + component.documentIDs = [1, 2, 3] + const event = { + previousIndex: 1, + currentIndex: 2, + } + + component.onDrop(event as any) + + expect(component.documentIDs).toEqual([1, 3, 2]) + }) + + it('should get document by ID', () => { + const documents = [ + { id: 1, name: 'Document 1' }, + { id: 2, name: 'Document 2' }, + { id: 3, name: 'Document 3' }, + ] + jest.spyOn(documentService, 'getCachedMany').mockReturnValue(of(documents)) + + component.ngOnInit() + + expect(component.getDocument(2)).toEqual({ id: 2, name: 'Document 2' }) + }) +}) diff --git a/src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.ts b/src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.ts new file mode 100644 index 000000000..fd52459e0 --- /dev/null +++ b/src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.ts @@ -0,0 +1,51 @@ +import { Component, OnInit } from '@angular/core' +import { ConfirmDialogComponent } from '../confirm-dialog.component' +import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap' +import { DocumentService } from 'src/app/services/rest/document.service' +import { CdkDragDrop, moveItemInArray } from '@angular/cdk/drag-drop' +import { Subject, takeUntil } from 'rxjs' +import { Document } from 'src/app/data/document' + +@Component({ + selector: 'pngx-merge-confirm-dialog', + templateUrl: './merge-confirm-dialog.component.html', + styleUrl: './merge-confirm-dialog.component.scss', +}) +export class MergeConfirmDialogComponent + extends ConfirmDialogComponent + implements OnInit +{ + public documentIDs: number[] = [] + private _documents: Document[] = [] + get documents(): Document[] { + return this._documents + } + + public metadataDocumentID: number = -1 + + private unsubscribeNotifier: Subject = new Subject() + + constructor( + activeModal: NgbActiveModal, + private documentService: DocumentService + ) { + super(activeModal) + } + + ngOnInit() { + this.documentService + .getCachedMany(this.documentIDs) + .pipe(takeUntil(this.unsubscribeNotifier)) + .subscribe((documents) => { + this._documents = documents + }) + } + + onDrop(event: CdkDragDrop) { + moveItemInArray(this.documentIDs, event.previousIndex, event.currentIndex) + } + + getDocument(documentID: number): Document { + return this.documents.find((d) => d.id === documentID) + } +} diff --git a/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.html b/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.html index 9a6b7b10b..dd3fec5a4 100644 --- a/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.html +++ b/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.html @@ -94,6 +94,9 @@ + diff --git a/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.spec.ts b/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.spec.ts index 623a21d5b..e38138df1 100644 --- a/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.spec.ts +++ b/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.spec.ts @@ -54,6 +54,7 @@ import { DocumentTypeEditDialogComponent } from '../../common/edit-dialog/docume import { StoragePathEditDialogComponent } from '../../common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component' import { IsNumberPipe } from 'src/app/pipes/is-number.pipe' import { RotateConfirmDialogComponent } from '../../common/confirm-dialog/rotate-confirm-dialog/rotate-confirm-dialog.component' +import { MergeConfirmDialogComponent } from '../../common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component' const selectionData: SelectionData = { selected_tags: [ @@ -100,6 +101,8 @@ describe('BulkEditorComponent', () => { PermissionsUserComponent, SwitchComponent, RotateConfirmDialogComponent, + IsNumberPipe, + MergeConfirmDialogComponent, ], providers: [ PermissionsService, @@ -834,7 +837,6 @@ describe('BulkEditorComponent', () => { jest .spyOn(permissionsService, 'currentUserHasObjectPermissions') .mockReturnValue(true) - component.showConfirmationDialogs = true fixture.detectChanges() component.rotateSelected() expect(modal).not.toBeUndefined() @@ -857,6 +859,44 @@ describe('BulkEditorComponent', () => { ) // listAllFilteredIds }) + it('should support merge', () => { + let modal: NgbModalRef + modalService.activeInstances.subscribe((m) => (modal = m[0])) + jest.spyOn(permissionsService, 'currentUserCan').mockReturnValue(true) + jest + .spyOn(documentListViewService, 'documents', 'get') + .mockReturnValue([{ id: 3 }, { id: 4 }]) + jest + .spyOn(documentService, 'getCachedMany') + .mockReturnValue(of([{ id: 3 }, { id: 4 }])) + jest + .spyOn(documentListViewService, 'selected', 'get') + .mockReturnValue(new Set([3, 4])) + jest + .spyOn(permissionsService, 'currentUserHasObjectPermissions') + .mockReturnValue(true) + fixture.detectChanges() + component.mergeSelected() + expect(modal).not.toBeUndefined() + modal.componentInstance.metadataDocumentID = 3 + modal.componentInstance.confirm() + let req = httpTestingController.expectOne( + `${environment.apiBaseUrl}documents/bulk_edit/` + ) + req.flush(true) + expect(req.request.body).toEqual({ + documents: [3, 4], + method: 'merge', + parameters: { metadata_document_id: 3 }, + }) + httpTestingController.match( + `${environment.apiBaseUrl}documents/?page=1&page_size=50&ordering=-created&truncate_content=true` + ) // list reload + httpTestingController.match( + `${environment.apiBaseUrl}documents/?page=1&page_size=100000&fields=id` + ) // listAllFilteredIds + }) + it('should support bulk download with archive, originals or both and file formatting', () => { jest.spyOn(permissionsService, 'currentUserCan').mockReturnValue(true) jest diff --git a/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.ts b/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.ts index cee054d9d..46a4980a6 100644 --- a/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.ts +++ b/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.ts @@ -6,7 +6,7 @@ import { TagService } from 'src/app/services/rest/tag.service' import { CorrespondentService } from 'src/app/services/rest/correspondent.service' import { DocumentTypeService } from 'src/app/services/rest/document-type.service' import { DocumentListViewService } from 'src/app/services/document-list-view.service' -import { NgbModal } from '@ng-bootstrap/ng-bootstrap' +import { NgbModal, NgbModalRef } from '@ng-bootstrap/ng-bootstrap' import { DocumentService, SelectionDataItem, @@ -40,6 +40,7 @@ import { TagEditDialogComponent } from '../../common/edit-dialog/tag-edit-dialog import { DocumentTypeEditDialogComponent } from '../../common/edit-dialog/document-type-edit-dialog/document-type-edit-dialog.component' import { StoragePathEditDialogComponent } from '../../common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component' import { RotateConfirmDialogComponent } from '../../common/confirm-dialog/rotate-confirm-dialog/rotate-confirm-dialog.component' +import { MergeConfirmDialogComponent } from '../../common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component' @Component({ selector: 'pngx-bulk-editor', @@ -193,12 +194,21 @@ export class BulkEditorComponent this.unsubscribeNotifier.complete() } - private executeBulkOperation(modal, method: string, args) { + private executeBulkOperation( + modal: NgbModalRef, + method: string, + args: any, + overrideDocumentIDs?: number[] + ) { if (modal) { modal.componentInstance.buttonsEnabled = false } this.documentService - .bulkEdit(Array.from(this.list.selected), method, args) + .bulkEdit( + overrideDocumentIDs ?? Array.from(this.list.selected), + method, + args + ) .pipe(first()) .subscribe({ next: () => { @@ -663,4 +673,28 @@ export class BulkEditorComponent }) }) } + + mergeSelected() { + let modal = this.modalService.open(MergeConfirmDialogComponent, { + backdrop: 'static', + }) + const mergeDialog = modal.componentInstance as MergeConfirmDialogComponent + mergeDialog.title = $localize`Merge confirm` + mergeDialog.messageBold = $localize`This operation will merge ${this.list.selected.size} selected documents into a new document.` + mergeDialog.btnCaption = $localize`Proceed` + mergeDialog.documentIDs = Array.from(this.list.selected) + mergeDialog.confirmClicked + .pipe(takeUntil(this.unsubscribeNotifier)) + .subscribe(() => { + const args = {} + if (mergeDialog.metadataDocumentID > -1) { + args['metadata_document_id'] = mergeDialog.metadataDocumentID + } + mergeDialog.buttonsEnabled = false + this.executeBulkOperation(modal, 'merge', args, mergeDialog.documentIDs) + this.toastService.showInfo( + $localize`Merged document will be queued for consumption.` + ) + }) + } } diff --git a/src/documents/bulk_edit.py b/src/documents/bulk_edit.py index 5bd33c69a..8f4cb05d1 100644 --- a/src/documents/bulk_edit.py +++ b/src/documents/bulk_edit.py @@ -2,15 +2,20 @@ import hashlib import itertools import logging import os +from typing import Optional from django.db.models import Q +from documents.data_models import ConsumableDocument +from documents.data_models import DocumentMetadataOverrides +from documents.data_models import DocumentSource from documents.models import Correspondent from documents.models import Document from documents.models import DocumentType from documents.models import StoragePath from documents.permissions import set_permissions_for_object from documents.tasks import bulk_update_documents +from documents.tasks import consume_file from documents.tasks import update_document_archive_file from paperless import settings @@ -179,3 +184,67 @@ def rotate(doc_ids: list[int], degrees: int): bulk_update_documents.delay(document_ids=affected_docs) return "OK" + + +def merge(doc_ids: list[int], metadata_document_id: Optional[int] = None): + qs = Document.objects.filter(id__in=doc_ids) + import pikepdf + + merged_pdf = pikepdf.new() + # use doc_ids to preserve order + for doc_id in doc_ids: + doc = qs.get(id=doc_id) + if doc is None: + continue + path = os.path.join(settings.ORIGINALS_DIR, str(doc.filename)) + try: + with pikepdf.open(path, allow_overwriting_input=True) as pdf: + merged_pdf.pages.extend(pdf.pages) + except Exception as e: + logger.exception( + f"Error merging document {doc.id}, it will not be included in the merge", + e, + ) + + filepath = os.path.join( + settings.CONSUMPTION_DIR, + f"merged_{('_'.join([str(doc_id) for doc_id in doc_ids]))[:100]}.pdf", + ) + merged_pdf.save(filepath) + + overrides = DocumentMetadataOverrides() + + if metadata_document_id: + metadata_document = qs.get(id=metadata_document_id) + if metadata_document is not None: + overrides.title = metadata_document.title + " (merged)" + overrides.correspondent_id = ( + metadata_document.correspondent.pk + if metadata_document.correspondent + else None + ) + overrides.document_type_id = ( + metadata_document.document_type.pk + if metadata_document.document_type + else None + ) + overrides.storage_path_id = ( + metadata_document.storage_path.pk + if metadata_document.storage_path + else None + ) + overrides.tag_ids = list( + metadata_document.tags.values_list("id", flat=True), + ) + # Include owner and permissions? + + logger.info("Adding merged document to the task queue.") + consume_file.delay( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=filepath, + ), + overrides, + ) + + return "OK" diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 972d1c8e5..5e9b3edbb 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -870,6 +870,7 @@ class BulkEditSerializer(DocumentListSerializer, SetPermissionsMixin): "redo_ocr", "set_permissions", "rotate", + "merge", ], label="Method", write_only=True, @@ -909,6 +910,8 @@ class BulkEditSerializer(DocumentListSerializer, SetPermissionsMixin): return bulk_edit.set_permissions elif method == "rotate": return bulk_edit.rotate + elif method == "merge": + return bulk_edit.merge else: raise serializers.ValidationError("Unsupported method.") diff --git a/src/documents/tests/test_api_bulk_edit.py b/src/documents/tests/test_api_bulk_edit.py index bbd2485af..e1af6830e 100644 --- a/src/documents/tests/test_api_bulk_edit.py +++ b/src/documents/tests/test_api_bulk_edit.py @@ -836,3 +836,26 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) m.assert_not_called() + + @mock.patch("documents.serialisers.bulk_edit.merge") + def test_merge(self, m): + m.return_value = "OK" + + response = self.client.post( + "/api/documents/bulk_edit/", + json.dumps( + { + "documents": [self.doc2.id, self.doc3.id], + "method": "merge", + "parameters": {"metadata_document_id": self.doc3.id}, + }, + ), + content_type="application/json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + m.assert_called_once() + args, kwargs = m.call_args + self.assertCountEqual(args[0], [self.doc2.id, self.doc3.id]) + self.assertEqual(kwargs["metadata_document_id"], self.doc3.id)