Merge

src-ui/src/app/app.module.ts

@@ -117,6 +117,7 @@ import { MonetaryComponent } from './components/common/input/monetary/monetary.c
 import { SystemStatusDialogComponent } from './components/common/system-status-dialog/system-status-dialog.component'
 import { NgxFilesizeModule } from 'ngx-filesize'
 import { RotateConfirmDialogComponent } from './components/common/confirm-dialog/rotate-confirm-dialog/rotate-confirm-dialog.component'
+import { MergeConfirmDialogComponent } from './components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component'
 import {
   airplane,
   archive,
@@ -177,6 +178,7 @@ import {
   hddStack,
   house,
   infoCircle,
+  journals,
   link,
   listTask,
   listUl,
@@ -269,6 +271,7 @@ const icons = {
   hddStack,
   house,
   infoCircle,
+  journals,
   link,
   listTask,
   listUl,
@@ -464,6 +467,7 @@ function initializeApp(settings: SettingsService) {
     MonetaryComponent,
     SystemStatusDialogComponent,
     RotateConfirmDialogComponent,
+    MergeConfirmDialogComponent,
   ],
   imports: [
     BrowserModule,

src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.html

@@ -0,0 +1,50 @@
+<div class="modal-header">
+    <h4 class="modal-title" id="modal-basic-title">{{title}}</h4>
+    <button type="button" class="btn-close" aria-label="Close" (click)="cancel()">
+    </button>
+  </div>
+  <div class="modal-body">
+    <p>{{message}}</p>
+    <div class="form-group">
+        <label class="form-label" for="metadataDocumentID" i18n>Documents:</label>
+        <ul class="list-group"
+            cdkDropList
+            (cdkDropListDropped)="onDrop($event)">
+            @for (documentID of documentIDs; track documentID) {
+                <li class="list-group-item" cdkDrag>
+                    <i-bs name="grip-vertical" class="me-2"></i-bs>
+                    {{getDocument(documentID)?.title}}
+                </li>
+            }
+        </ul>
+        </div>
+    <div class="form-group mt-4">
+        <label class="form-label" for="metadataDocumentID" i18n>Use metadata from:</label>
+        <select class="form-select" [(ngModel)]="metadataDocumentID">
+            <option [ngValue]="-1" i18n>Regenerate all metadata</option>
+            @for (document of documents; track document.id) {
+              <option [ngValue]="document.id">{{document.title}}</option>
+            }
+        </select>
+    </div>
+    <p class="small text-muted fst-italic mt-4" i18n>Note that only PDFs will be included.</p>
+  </div>
+  <div class="modal-footer">
+    <button type="button" class="btn" [class]="cancelBtnClass" (click)="cancel()" [disabled]="!buttonsEnabled">
+          <span class="d-inline-block" style="padding-bottom: 1px;">{{cancelBtnCaption}}</span>
+        </button>
+    <button type="button" class="btn" [class]="btnClass" (click)="confirm()" [disabled]="!confirmButtonEnabled || !buttonsEnabled">
+      <span>
+        {{btnCaption}}
+        <span class="visually-hidden">{{ seconds | number: '1.0-0' }} seconds</span>
+      </span>
+      @if (!confirmButtonEnabled) {
+        <ngb-progressbar style="height: 1px;" type="dark" [max]="secondsTotal" [value]="seconds"></ngb-progressbar>
+      }
+    </button>
+    @if (alternativeBtnCaption) {
+      <button type="button" class="btn" [class]="alternativeBtnClass" (click)="alternative()" [disabled]="!alternativeButtonEnabled || !buttonsEnabled">
+        {{alternativeBtnCaption}}
+      </button>
+    }
+  </div>

src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.scss

@@ -0,0 +1,3 @@
+.list-group-item {
+    cursor: move;
+}

src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.spec.ts

@@ -0,0 +1,73 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing'
+import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
+import { MergeConfirmDialogComponent } from './merge-confirm-dialog.component'
+import { HttpClientTestingModule } from '@angular/common/http/testing'
+import { NgxBootstrapIconsModule, allIcons } from 'ngx-bootstrap-icons'
+import { of } from 'rxjs'
+import { DocumentService } from 'src/app/services/rest/document.service'
+import { FormsModule, ReactiveFormsModule } from '@angular/forms'
+
+describe('MergeConfirmDialogComponent', () => {
+  let component: MergeConfirmDialogComponent
+  let fixture: ComponentFixture<MergeConfirmDialogComponent>
+  let documentService: DocumentService
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [MergeConfirmDialogComponent],
+      providers: [NgbActiveModal],
+      imports: [
+        HttpClientTestingModule,
+        NgxBootstrapIconsModule.pick(allIcons),
+        ReactiveFormsModule,
+        FormsModule,
+      ],
+    }).compileComponents()
+
+    fixture = TestBed.createComponent(MergeConfirmDialogComponent)
+    documentService = TestBed.inject(DocumentService)
+    component = fixture.componentInstance
+    fixture.detectChanges()
+  })
+
+  it('should fetch documents on ngOnInit', () => {
+    const documents = [
+      { id: 1, name: 'Document 1' },
+      { id: 2, name: 'Document 2' },
+      { id: 3, name: 'Document 3' },
+    ]
+    jest.spyOn(documentService, 'getCachedMany').mockReturnValue(of(documents))
+
+    component.ngOnInit()
+
+    expect(component.documents).toEqual(documents)
+    expect(documentService.getCachedMany).toHaveBeenCalledWith(
+      component.documentIDs
+    )
+  })
+
+  it('should move documentIDs on drop', () => {
+    component.documentIDs = [1, 2, 3]
+    const event = {
+      previousIndex: 1,
+      currentIndex: 2,
+    }
+
+    component.onDrop(event as any)
+
+    expect(component.documentIDs).toEqual([1, 3, 2])
+  })
+
+  it('should get document by ID', () => {
+    const documents = [
+      { id: 1, name: 'Document 1' },
+      { id: 2, name: 'Document 2' },
+      { id: 3, name: 'Document 3' },
+    ]
+    jest.spyOn(documentService, 'getCachedMany').mockReturnValue(of(documents))
+
+    component.ngOnInit()
+
+    expect(component.getDocument(2)).toEqual({ id: 2, name: 'Document 2' })
+  })
+})

src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.ts

@@ -0,0 +1,51 @@
+import { Component, OnInit } from '@angular/core'
+import { ConfirmDialogComponent } from '../confirm-dialog.component'
+import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
+import { DocumentService } from 'src/app/services/rest/document.service'
+import { CdkDragDrop, moveItemInArray } from '@angular/cdk/drag-drop'
+import { Subject, takeUntil } from 'rxjs'
+import { Document } from 'src/app/data/document'
+
+@Component({
+  selector: 'pngx-merge-confirm-dialog',
+  templateUrl: './merge-confirm-dialog.component.html',
+  styleUrl: './merge-confirm-dialog.component.scss',
+})
+export class MergeConfirmDialogComponent
+  extends ConfirmDialogComponent
+  implements OnInit
+{
+  public documentIDs: number[] = []
+  private _documents: Document[] = []
+  get documents(): Document[] {
+    return this._documents
+  }
+
+  public metadataDocumentID: number = -1
+
+  private unsubscribeNotifier: Subject<any> = new Subject()
+
+  constructor(
+    activeModal: NgbActiveModal,
+    private documentService: DocumentService
+  ) {
+    super(activeModal)
+  }
+
+  ngOnInit() {
+    this.documentService
+      .getCachedMany(this.documentIDs)
+      .pipe(takeUntil(this.unsubscribeNotifier))
+      .subscribe((documents) => {
+        this._documents = documents
+      })
+  }
+
+  onDrop(event: CdkDragDrop<number[]>) {
+    moveItemInArray(this.documentIDs, event.previousIndex, event.currentIndex)
+  }
+
+  getDocument(documentID: number): Document {
+    return this.documents.find((d) => d.id === documentID)
+  }
+}

src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.html

@@ -94,6 +94,9 @@
                 <button ngbDropdownItem (click)="rotateSelected()" [disabled]="!userCanEditAll">
                   <i-bs name="arrow-clockwise"></i-bs>&nbsp;<ng-container i18n>Rotate</ng-container>
                 </button>
+                <button ngbDropdownItem (click)="mergeSelected()" [disabled]="!userCanEditAll || list.selected.size < 2">
+                  <i-bs name="journals"></i-bs>&nbsp;<ng-container i18n>Merge</ng-container>
+                </button>
               </div>
             </div>
           </div>

src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.spec.ts

@@ -54,6 +54,7 @@ import { DocumentTypeEditDialogComponent } from '../../common/edit-dialog/docume
 import { StoragePathEditDialogComponent } from '../../common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component'
 import { IsNumberPipe } from 'src/app/pipes/is-number.pipe'
 import { RotateConfirmDialogComponent } from '../../common/confirm-dialog/rotate-confirm-dialog/rotate-confirm-dialog.component'
+import { MergeConfirmDialogComponent } from '../../common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component'
 
 const selectionData: SelectionData = {
   selected_tags: [
@@ -100,6 +101,8 @@ describe('BulkEditorComponent', () => {
         PermissionsUserComponent,
         SwitchComponent,
         RotateConfirmDialogComponent,
+        IsNumberPipe,
+        MergeConfirmDialogComponent,
       ],
       providers: [
         PermissionsService,
@@ -834,7 +837,6 @@ describe('BulkEditorComponent', () => {
     jest
       .spyOn(permissionsService, 'currentUserHasObjectPermissions')
       .mockReturnValue(true)
-    component.showConfirmationDialogs = true
     fixture.detectChanges()
     component.rotateSelected()
     expect(modal).not.toBeUndefined()
@@ -857,6 +859,44 @@ describe('BulkEditorComponent', () => {
     ) // listAllFilteredIds
   })
 
+  it('should support merge', () => {
+    let modal: NgbModalRef
+    modalService.activeInstances.subscribe((m) => (modal = m[0]))
+    jest.spyOn(permissionsService, 'currentUserCan').mockReturnValue(true)
+    jest
+      .spyOn(documentListViewService, 'documents', 'get')
+      .mockReturnValue([{ id: 3 }, { id: 4 }])
+    jest
+      .spyOn(documentService, 'getCachedMany')
+      .mockReturnValue(of([{ id: 3 }, { id: 4 }]))
+    jest
+      .spyOn(documentListViewService, 'selected', 'get')
+      .mockReturnValue(new Set([3, 4]))
+    jest
+      .spyOn(permissionsService, 'currentUserHasObjectPermissions')
+      .mockReturnValue(true)
+    fixture.detectChanges()
+    component.mergeSelected()
+    expect(modal).not.toBeUndefined()
+    modal.componentInstance.metadataDocumentID = 3
+    modal.componentInstance.confirm()
+    let req = httpTestingController.expectOne(
+      `${environment.apiBaseUrl}documents/bulk_edit/`
+    )
+    req.flush(true)
+    expect(req.request.body).toEqual({
+      documents: [3, 4],
+      method: 'merge',
+      parameters: { metadata_document_id: 3 },
+    })
+    httpTestingController.match(
+      `${environment.apiBaseUrl}documents/?page=1&page_size=50&ordering=-created&truncate_content=true`
+    ) // list reload
+    httpTestingController.match(
+      `${environment.apiBaseUrl}documents/?page=1&page_size=100000&fields=id`
+    ) // listAllFilteredIds
+  })
+
   it('should support bulk download with archive, originals or both and file formatting', () => {
     jest.spyOn(permissionsService, 'currentUserCan').mockReturnValue(true)
     jest

src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.ts

@@ -6,7 +6,7 @@ import { TagService } from 'src/app/services/rest/tag.service'
 import { CorrespondentService } from 'src/app/services/rest/correspondent.service'
 import { DocumentTypeService } from 'src/app/services/rest/document-type.service'
 import { DocumentListViewService } from 'src/app/services/document-list-view.service'
-import { NgbModal } from '@ng-bootstrap/ng-bootstrap'
+import { NgbModal, NgbModalRef } from '@ng-bootstrap/ng-bootstrap'
 import {
   DocumentService,
   SelectionDataItem,
@@ -40,6 +40,7 @@ import { TagEditDialogComponent } from '../../common/edit-dialog/tag-edit-dialog
 import { DocumentTypeEditDialogComponent } from '../../common/edit-dialog/document-type-edit-dialog/document-type-edit-dialog.component'
 import { StoragePathEditDialogComponent } from '../../common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component'
 import { RotateConfirmDialogComponent } from '../../common/confirm-dialog/rotate-confirm-dialog/rotate-confirm-dialog.component'
+import { MergeConfirmDialogComponent } from '../../common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component'
 
 @Component({
   selector: 'pngx-bulk-editor',
@@ -193,12 +194,21 @@ export class BulkEditorComponent
     this.unsubscribeNotifier.complete()
   }
 
-  private executeBulkOperation(modal, method: string, args) {
+  private executeBulkOperation(
+    modal: NgbModalRef,
+    method: string,
+    args: any,
+    overrideDocumentIDs?: number[]
+  ) {
     if (modal) {
       modal.componentInstance.buttonsEnabled = false
     }
     this.documentService
-      .bulkEdit(Array.from(this.list.selected), method, args)
+      .bulkEdit(
+        overrideDocumentIDs ?? Array.from(this.list.selected),
+        method,
+        args
+      )
       .pipe(first())
       .subscribe({
         next: () => {
@@ -663,4 +673,28 @@ export class BulkEditorComponent
         })
       })
   }
+
+  mergeSelected() {
+    let modal = this.modalService.open(MergeConfirmDialogComponent, {
+      backdrop: 'static',
+    })
+    const mergeDialog = modal.componentInstance as MergeConfirmDialogComponent
+    mergeDialog.title = $localize`Merge confirm`
+    mergeDialog.messageBold = $localize`This operation will merge ${this.list.selected.size} selected documents into a new document.`
+    mergeDialog.btnCaption = $localize`Proceed`
+    mergeDialog.documentIDs = Array.from(this.list.selected)
+    mergeDialog.confirmClicked
+      .pipe(takeUntil(this.unsubscribeNotifier))
+      .subscribe(() => {
+        const args = {}
+        if (mergeDialog.metadataDocumentID > -1) {
+          args['metadata_document_id'] = mergeDialog.metadataDocumentID
+        }
+        mergeDialog.buttonsEnabled = false
+        this.executeBulkOperation(modal, 'merge', args, mergeDialog.documentIDs)
+        this.toastService.showInfo(
+          $localize`Merged document will be queued for consumption.`
+        )
+      })
+  }
 }

src/documents/bulk_edit.py

@@ -2,15 +2,20 @@ import hashlib
 import itertools
 import logging
 import os
+from typing import Optional
 
 from django.db.models import Q
 
+from documents.data_models import ConsumableDocument
+from documents.data_models import DocumentMetadataOverrides
+from documents.data_models import DocumentSource
 from documents.models import Correspondent
 from documents.models import Document
 from documents.models import DocumentType
 from documents.models import StoragePath
 from documents.permissions import set_permissions_for_object
 from documents.tasks import bulk_update_documents
+from documents.tasks import consume_file
 from documents.tasks import update_document_archive_file
 from paperless import settings
 
@@ -179,3 +184,67 @@ def rotate(doc_ids: list[int], degrees: int):
         bulk_update_documents.delay(document_ids=affected_docs)
 
     return "OK"
+
+
+def merge(doc_ids: list[int], metadata_document_id: Optional[int] = None):
+    qs = Document.objects.filter(id__in=doc_ids)
+    import pikepdf
+
+    merged_pdf = pikepdf.new()
+    # use doc_ids to preserve order
+    for doc_id in doc_ids:
+        doc = qs.get(id=doc_id)
+        if doc is None:
+            continue
+        path = os.path.join(settings.ORIGINALS_DIR, str(doc.filename))
+        try:
+            with pikepdf.open(path, allow_overwriting_input=True) as pdf:
+                merged_pdf.pages.extend(pdf.pages)
+        except Exception as e:
+            logger.exception(
+                f"Error merging document {doc.id}, it will not be included in the merge",
+                e,
+            )
+
+    filepath = os.path.join(
+        settings.CONSUMPTION_DIR,
+        f"merged_{('_'.join([str(doc_id) for doc_id in doc_ids]))[:100]}.pdf",
+    )
+    merged_pdf.save(filepath)
+
+    overrides = DocumentMetadataOverrides()
+
+    if metadata_document_id:
+        metadata_document = qs.get(id=metadata_document_id)
+        if metadata_document is not None:
+            overrides.title = metadata_document.title + " (merged)"
+            overrides.correspondent_id = (
+                metadata_document.correspondent.pk
+                if metadata_document.correspondent
+                else None
+            )
+            overrides.document_type_id = (
+                metadata_document.document_type.pk
+                if metadata_document.document_type
+                else None
+            )
+            overrides.storage_path_id = (
+                metadata_document.storage_path.pk
+                if metadata_document.storage_path
+                else None
+            )
+            overrides.tag_ids = list(
+                metadata_document.tags.values_list("id", flat=True),
+            )
+            # Include owner and permissions?
+
+    logger.info("Adding merged document to the task queue.")
+    consume_file.delay(
+        ConsumableDocument(
+            source=DocumentSource.ConsumeFolder,
+            original_file=filepath,
+        ),
+        overrides,
+    )
+
+    return "OK"

src/documents/serialisers.py

@@ -870,6 +870,7 @@ class BulkEditSerializer(DocumentListSerializer, SetPermissionsMixin):
             "redo_ocr",
             "set_permissions",
             "rotate",
+            "merge",
         ],
         label="Method",
         write_only=True,
@@ -909,6 +910,8 @@ class BulkEditSerializer(DocumentListSerializer, SetPermissionsMixin):
             return bulk_edit.set_permissions
         elif method == "rotate":
             return bulk_edit.rotate
+        elif method == "merge":
+            return bulk_edit.merge
         else:
             raise serializers.ValidationError("Unsupported method.")
 

src/documents/tests/test_api_bulk_edit.py

@@ -836,3 +836,26 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
 
         m.assert_not_called()
+
+    @mock.patch("documents.serialisers.bulk_edit.merge")
+    def test_merge(self, m):
+        m.return_value = "OK"
+
+        response = self.client.post(
+            "/api/documents/bulk_edit/",
+            json.dumps(
+                {
+                    "documents": [self.doc2.id, self.doc3.id],
+                    "method": "merge",
+                    "parameters": {"metadata_document_id": self.doc3.id},
+                },
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+        m.assert_called_once()
+        args, kwargs = m.call_args
+        self.assertCountEqual(args[0], [self.doc2.id, self.doc3.id])
+        self.assertEqual(kwargs["metadata_document_id"], self.doc3.id)