lgcosta/paperless-ngx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge

Browse Source
This commit is contained in:
shamoon 2024-03-02 16:26:54 -08:00
parent c9dd407cbe
commit d5eedbab0d
11 changed files with 357 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src-ui/src/app/app.module.ts
View File

@ -117,6 +117,7 @@ import { MonetaryComponent } from './components/common/input/monetary/monetary.c
import { SystemStatusDialogComponent } from './components/common/system-status-dialog/system-status-dialog.component'
import { NgxFilesizeModule } from 'ngx-filesize'
import { RotateConfirmDialogComponent } from './components/common/confirm-dialog/rotate-confirm-dialog/rotate-confirm-dialog.component'
import { MergeConfirmDialogComponent } from './components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component'
import {
airplane,
archive,
@ -177,6 +178,7 @@ import {
hddStack,
house,
infoCircle,
journals,
link,
listTask,
listUl,
@ -269,6 +271,7 @@ const icons = {
hddStack,
house,
infoCircle,
journals,
link,
listTask,
listUl,
@ -464,6 +467,7 @@ function initializeApp(settings: SettingsService) {
MonetaryComponent,
SystemStatusDialogComponent,
RotateConfirmDialogComponent,
MergeConfirmDialogComponent,
],
imports: [
BrowserModule,

50
src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.html Normal file
View File

@ -0,0 +1,50 @@
<div class="modal-header">
<h4 class="modal-title" id="modal-basic-title">{{title}}</h4>
<button type="button" class="btn-close" aria-label="Close" (click)="cancel()">
</button>
</div>
<div class="modal-body">
<p>{{message}}</p>
<div class="form-group">
<label class="form-label" for="metadataDocumentID" i18n>Documents:</label>
<ul class="list-group"
cdkDropList
(cdkDropListDropped)="onDrop($event)">
@for (documentID of documentIDs; track documentID) {
<li class="list-group-item" cdkDrag>
<i-bs name="grip-vertical" class="me-2"></i-bs>
{{getDocument(documentID)?.title}}
</li>
}
</ul>
</div>
<div class="form-group mt-4">
<label class="form-label" for="metadataDocumentID" i18n>Use metadata from:</label>
<select class="form-select" [(ngModel)]="metadataDocumentID">
<option [ngValue]="-1" i18n>Regenerate all metadata</option>
@for (document of documents; track document.id) {
<option [ngValue]="document.id">{{document.title}}</option>
}
</select>
</div>
<p class="small text-muted fst-italic mt-4" i18n>Note that only PDFs will be included.</p>
</div>
<div class="modal-footer">
<button type="button" class="btn" [class]="cancelBtnClass" (click)="cancel()" [disabled]="!buttonsEnabled">
<span class="d-inline-block" style="padding-bottom: 1px;">{{cancelBtnCaption}}</span>
</button>
<button type="button" class="btn" [class]="btnClass" (click)="confirm()" [disabled]="!confirmButtonEnabled || !buttonsEnabled">
<span>
{{btnCaption}}
<span class="visually-hidden">{{ seconds | number: '1.0-0' }} seconds</span>
</span>
@if (!confirmButtonEnabled) {
<ngb-progressbar style="height: 1px;" type="dark" [max]="secondsTotal" [value]="seconds"></ngb-progressbar>
}
</button>
@if (alternativeBtnCaption) {
<button type="button" class="btn" [class]="alternativeBtnClass" (click)="alternative()" [disabled]="!alternativeButtonEnabled || !buttonsEnabled">
{{alternativeBtnCaption}}
</button>
}
</div>

3
src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.scss Normal file
View File

@ -0,0 +1,3 @@
.list-group-item {
cursor: move;
}

73
src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.spec.ts Normal file
View File

@ -0,0 +1,73 @@
import { ComponentFixture, TestBed } from '@angular/core/testing'
import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
import { MergeConfirmDialogComponent } from './merge-confirm-dialog.component'
import { HttpClientTestingModule } from '@angular/common/http/testing'
import { NgxBootstrapIconsModule, allIcons } from 'ngx-bootstrap-icons'
import { of } from 'rxjs'
import { DocumentService } from 'src/app/services/rest/document.service'
import { FormsModule, ReactiveFormsModule } from '@angular/forms'
describe('MergeConfirmDialogComponent', () => {
let component: MergeConfirmDialogComponent
let fixture: ComponentFixture<MergeConfirmDialogComponent>
let documentService: DocumentService
beforeEach(async () => {
await TestBed.configureTestingModule({
declarations: [MergeConfirmDialogComponent],
providers: [NgbActiveModal],
imports: [
HttpClientTestingModule,
NgxBootstrapIconsModule.pick(allIcons),
ReactiveFormsModule,
FormsModule,
],
}).compileComponents()
fixture = TestBed.createComponent(MergeConfirmDialogComponent)
documentService = TestBed.inject(DocumentService)
component = fixture.componentInstance
fixture.detectChanges()
})
it('should fetch documents on ngOnInit', () => {
const documents = [
{ id: 1, name: 'Document 1' },
{ id: 2, name: 'Document 2' },
{ id: 3, name: 'Document 3' },
]
jest.spyOn(documentService, 'getCachedMany').mockReturnValue(of(documents))
component.ngOnInit()
expect(component.documents).toEqual(documents)
expect(documentService.getCachedMany).toHaveBeenCalledWith(
component.documentIDs
)
})
it('should move documentIDs on drop', () => {
component.documentIDs = [1, 2, 3]
const event = {
previousIndex: 1,
currentIndex: 2,
}
component.onDrop(event as any)
expect(component.documentIDs).toEqual([1, 3, 2])
})
it('should get document by ID', () => {
const documents = [
{ id: 1, name: 'Document 1' },
{ id: 2, name: 'Document 2' },
{ id: 3, name: 'Document 3' },
]
jest.spyOn(documentService, 'getCachedMany').mockReturnValue(of(documents))
component.ngOnInit()
expect(component.getDocument(2)).toEqual({ id: 2, name: 'Document 2' })
})
})

51
src-ui/src/app/components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component.ts Normal file
View File

@ -0,0 +1,51 @@
import { Component, OnInit } from '@angular/core'
import { ConfirmDialogComponent } from '../confirm-dialog.component'
import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
import { DocumentService } from 'src/app/services/rest/document.service'
import { CdkDragDrop, moveItemInArray } from '@angular/cdk/drag-drop'
import { Subject, takeUntil } from 'rxjs'
import { Document } from 'src/app/data/document'
@Component({
selector: 'pngx-merge-confirm-dialog',
templateUrl: './merge-confirm-dialog.component.html',
styleUrl: './merge-confirm-dialog.component.scss',
})
export class MergeConfirmDialogComponent
extends ConfirmDialogComponent
implements OnInit
{
public documentIDs: number[] = []
private _documents: Document[] = []
get documents(): Document[] {
return this._documents
}
public metadataDocumentID: number = -1
private unsubscribeNotifier: Subject<any> = new Subject()
constructor(
activeModal: NgbActiveModal,
private documentService: DocumentService
) {
super(activeModal)
}
ngOnInit() {
this.documentService
.getCachedMany(this.documentIDs)
.pipe(takeUntil(this.unsubscribeNotifier))
.subscribe((documents) => {
this._documents = documents
})
}
onDrop(event: CdkDragDrop<number[]>) {
moveItemInArray(this.documentIDs, event.previousIndex, event.currentIndex)
}
getDocument(documentID: number): Document {
return this.documents.find((d) => d.id === documentID)
}
}

3
src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.html
View File

@ -94,6 +94,9 @@
<button ngbDropdownItem (click)="rotateSelected()" [disabled]="!userCanEditAll">
<i-bs name="arrow-clockwise"></i-bs>&nbsp;<ng-container i18n>Rotate</ng-container>
</button>
<button ngbDropdownItem (click)="mergeSelected()" [disabled]="!userCanEditAll || list.selected.size < 2">
<i-bs name="journals"></i-bs>&nbsp;<ng-container i18n>Merge</ng-container>
</button>
</div>
</div>
</div>

42
src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.spec.ts
View File

@ -54,6 +54,7 @@ import { DocumentTypeEditDialogComponent } from '../../common/edit-dialog/docume
import { StoragePathEditDialogComponent } from '../../common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component'
import { IsNumberPipe } from 'src/app/pipes/is-number.pipe'
import { RotateConfirmDialogComponent } from '../../common/confirm-dialog/rotate-confirm-dialog/rotate-confirm-dialog.component'
import { MergeConfirmDialogComponent } from '../../common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component'
const selectionData: SelectionData = {
selected_tags: [
@ -100,6 +101,8 @@ describe('BulkEditorComponent', () => {
PermissionsUserComponent,
SwitchComponent,
RotateConfirmDialogComponent,
IsNumberPipe,
MergeConfirmDialogComponent,
],
providers: [
PermissionsService,
@ -834,7 +837,6 @@ describe('BulkEditorComponent', () => {
jest
.spyOn(permissionsService, 'currentUserHasObjectPermissions')
.mockReturnValue(true)
component.showConfirmationDialogs = true
fixture.detectChanges()
component.rotateSelected()
expect(modal).not.toBeUndefined()
@ -857,6 +859,44 @@ describe('BulkEditorComponent', () => {
) // listAllFilteredIds
})
it('should support merge', () => {
let modal: NgbModalRef
modalService.activeInstances.subscribe((m) => (modal = m[0]))
jest.spyOn(permissionsService, 'currentUserCan').mockReturnValue(true)
jest
.spyOn(documentListViewService, 'documents', 'get')
.mockReturnValue([{ id: 3 }, { id: 4 }])
jest
.spyOn(documentService, 'getCachedMany')
.mockReturnValue(of([{ id: 3 }, { id: 4 }]))
jest
.spyOn(documentListViewService, 'selected', 'get')
.mockReturnValue(new Set([3, 4]))
jest
.spyOn(permissionsService, 'currentUserHasObjectPermissions')
.mockReturnValue(true)
fixture.detectChanges()
component.mergeSelected()
expect(modal).not.toBeUndefined()
modal.componentInstance.metadataDocumentID = 3
modal.componentInstance.confirm()
let req = httpTestingController.expectOne(
`${environment.apiBaseUrl}documents/bulk_edit/`
)
req.flush(true)
expect(req.request.body).toEqual({
documents: [3, 4],
method: 'merge',
parameters: { metadata_document_id: 3 },
})
httpTestingController.match(
`${environment.apiBaseUrl}documents/?page=1&page_size=50&ordering=-created&truncate_content=true`
) // list reload
httpTestingController.match(
`${environment.apiBaseUrl}documents/?page=1&page_size=100000&fields=id`
) // listAllFilteredIds
})
it('should support bulk download with archive, originals or both and file formatting', () => {
jest.spyOn(permissionsService, 'currentUserCan').mockReturnValue(true)
jest

40
src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.ts
View File

@ -6,7 +6,7 @@ import { TagService } from 'src/app/services/rest/tag.service'
import { CorrespondentService } from 'src/app/services/rest/correspondent.service'
import { DocumentTypeService } from 'src/app/services/rest/document-type.service'
import { DocumentListViewService } from 'src/app/services/document-list-view.service'
import { NgbModal } from '@ng-bootstrap/ng-bootstrap'
import { NgbModal, NgbModalRef } from '@ng-bootstrap/ng-bootstrap'
import {
DocumentService,
SelectionDataItem,
@ -40,6 +40,7 @@ import { TagEditDialogComponent } from '../../common/edit-dialog/tag-edit-dialog
import { DocumentTypeEditDialogComponent } from '../../common/edit-dialog/document-type-edit-dialog/document-type-edit-dialog.component'
import { StoragePathEditDialogComponent } from '../../common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component'
import { RotateConfirmDialogComponent } from '../../common/confirm-dialog/rotate-confirm-dialog/rotate-confirm-dialog.component'
import { MergeConfirmDialogComponent } from '../../common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component'
@Component({
selector: 'pngx-bulk-editor',
@ -193,12 +194,21 @@ export class BulkEditorComponent
this.unsubscribeNotifier.complete()
}
private executeBulkOperation(modal, method: string, args) {
private executeBulkOperation(
modal: NgbModalRef,
method: string,
args: any,
overrideDocumentIDs?: number[]
) {
if (modal) {
modal.componentInstance.buttonsEnabled = false
}
this.documentService
.bulkEdit(Array.from(this.list.selected), method, args)
.bulkEdit(
overrideDocumentIDs ?? Array.from(this.list.selected),
method,
args
)
.pipe(first())
.subscribe({
next: () => {
@ -663,4 +673,28 @@ export class BulkEditorComponent
})
})
}
mergeSelected() {
let modal = this.modalService.open(MergeConfirmDialogComponent, {
backdrop: 'static',
})
const mergeDialog = modal.componentInstance as MergeConfirmDialogComponent
mergeDialog.title = $localize`Merge confirm`
mergeDialog.messageBold = $localize`This operation will merge ${this.list.selected.size} selected documents into a new document.`
mergeDialog.btnCaption = $localize`Proceed`
mergeDialog.documentIDs = Array.from(this.list.selected)
mergeDialog.confirmClicked
.pipe(takeUntil(this.unsubscribeNotifier))
.subscribe(() => {
const args = {}
if (mergeDialog.metadataDocumentID > -1) {
args['metadata_document_id'] = mergeDialog.metadataDocumentID
}
mergeDialog.buttonsEnabled = false
this.executeBulkOperation(modal, 'merge', args, mergeDialog.documentIDs)
this.toastService.showInfo(
$localize`Merged document will be queued for consumption.`
)
})
}
}

69
src/documents/bulk_edit.py
View File

@ -2,15 +2,20 @@ import hashlib
import itertools
import logging
import os
from typing import Optional
from django.db.models import Q
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides
from documents.data_models import DocumentSource
from documents.models import Correspondent
from documents.models import Document
from documents.models import DocumentType
from documents.models import StoragePath
from documents.permissions import set_permissions_for_object
from documents.tasks import bulk_update_documents
from documents.tasks import consume_file
from documents.tasks import update_document_archive_file
from paperless import settings
@ -179,3 +184,67 @@ def rotate(doc_ids: list[int], degrees: int):
bulk_update_documents.delay(document_ids=affected_docs)
return "OK"
def merge(doc_ids: list[int], metadata_document_id: Optional[int] = None):
qs = Document.objects.filter(id__in=doc_ids)
import pikepdf
merged_pdf = pikepdf.new()
# use doc_ids to preserve order
for doc_id in doc_ids:
doc = qs.get(id=doc_id)
if doc is None:
continue
path = os.path.join(settings.ORIGINALS_DIR, str(doc.filename))
try:
with pikepdf.open(path, allow_overwriting_input=True) as pdf:
merged_pdf.pages.extend(pdf.pages)
except Exception as e:
logger.exception(
f"Error merging document {doc.id}, it will not be included in the merge",
e,
)
filepath = os.path.join(
settings.CONSUMPTION_DIR,
f"merged_{('_'.join([str(doc_id) for doc_id in doc_ids]))[:100]}.pdf",
)
merged_pdf.save(filepath)
overrides = DocumentMetadataOverrides()
if metadata_document_id:
metadata_document = qs.get(id=metadata_document_id)
if metadata_document is not None:
overrides.title = metadata_document.title + " (merged)"
overrides.correspondent_id = (
metadata_document.correspondent.pk
if metadata_document.correspondent
else None
)
overrides.document_type_id = (
metadata_document.document_type.pk
if metadata_document.document_type
else None
)
overrides.storage_path_id = (
metadata_document.storage_path.pk
if metadata_document.storage_path
else None
)
overrides.tag_ids = list(
metadata_document.tags.values_list("id", flat=True),
)
# Include owner and permissions?
logger.info("Adding merged document to the task queue.")
consume_file.delay(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=filepath,
),
overrides,
)
return "OK"

3
src/documents/serialisers.py
View File

@ -870,6 +870,7 @@ class BulkEditSerializer(DocumentListSerializer, SetPermissionsMixin):
"redo_ocr",
"set_permissions",
"rotate",
"merge",
],
label="Method",
write_only=True,
@ -909,6 +910,8 @@ class BulkEditSerializer(DocumentListSerializer, SetPermissionsMixin):
return bulk_edit.set_permissions
elif method == "rotate":
return bulk_edit.rotate
elif method == "merge":
return bulk_edit.merge
else:
raise serializers.ValidationError("Unsupported method.")

23
src/documents/tests/test_api_bulk_edit.py
View File

@ -836,3 +836,26 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
m.assert_not_called()
@mock.patch("documents.serialisers.bulk_edit.merge")
def test_merge(self, m):
m.return_value = "OK"
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id, self.doc3.id],
"method": "merge",
"parameters": {"metadata_document_id": self.doc3.id},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
args, kwargs = m.call_args
self.assertCountEqual(args[0], [self.doc2.id, self.doc3.id])
self.assertEqual(kwargs["metadata_document_id"], self.doc3.id)