lgcosta/paperless-ngx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Split

Browse Source
This commit is contained in:
shamoon 2024-03-03 12:23:43 -08:00
parent d5eedbab0d
commit 713598d86b
12 changed files with 468 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src-ui/src/app/app.module.ts
View File

@ -118,6 +118,7 @@ import { SystemStatusDialogComponent } from './components/common/system-status-d
import { NgxFilesizeModule } from 'ngx-filesize'
import { RotateConfirmDialogComponent } from './components/common/confirm-dialog/rotate-confirm-dialog/rotate-confirm-dialog.component'
import { MergeConfirmDialogComponent } from './components/common/confirm-dialog/merge-confirm-dialog/merge-confirm-dialog.component'
import { SplitConfirmDialogComponent } from './components/common/confirm-dialog/split-confirm-dialog/split-confirm-dialog.component'
import {
airplane,
archive,
@ -193,6 +194,7 @@ import {
plus,
plusCircle,
questionCircle,
scissors,
search,
slashCircle,
sliders2Vertical,
@ -286,6 +288,7 @@ const icons = {
plus,
plusCircle,
questionCircle,
scissors,
search,
slashCircle,
sliders2Vertical,
@ -468,6 +471,7 @@ function initializeApp(settings: SettingsService) {
SystemStatusDialogComponent,
RotateConfirmDialogComponent,
MergeConfirmDialogComponent,
SplitConfirmDialogComponent,
],
imports: [
BrowserModule,

55
src-ui/src/app/components/common/confirm-dialog/split-confirm-dialog/split-confirm-dialog.component.html Normal file
View File

@ -0,0 +1,55 @@
<div class="modal-header">
<h4 class="modal-title" id="modal-basic-title">{{title}}</h4>
<button type="button" class="btn-close" aria-label="Close" (click)="cancel()">
</button>
</div>
<div class="modal-body">
<p>{{message}}</p>
<div class="row mb-2">
<div class="col-6">
<div class="input-group input-group-sm">
<div class="input-group-text" i18n>Page</div>
<input class="form-control" type="number" min="1" [(ngModel)]="page" />
<div class="input-group-text" i18n>of {{totalPages}}</div>
</div>
<div class="pdf-viewer-container w-100 mt-3">
<pngx-pdf-viewer [src]="pdfSrc" [(page)]="page"
[original-size]="false"
[zoom]="1"
zoom-scale="page-fit"
(after-load-complete)="pdfPreviewLoaded($event)">
</pngx-pdf-viewer>
</div>
</div>
<div class="col-6">
<div class="d-grid">
<button class="btn btn-sm btn-primary" (click)="addSplit()">
<i-bs name="plus-circle"></i-bs>&nbsp;
<span i18n>Add Split</span>
</button>
</div>
<ul class="list-group mt-3">
@for (pageStr of pagesString.split(','); track pageStr; let i = $index) {
<li class="list-group-item">
{{pageStr}}
@if (pagesString.split(',').length > 1) {
&nbsp;
<button class="btn btn-sm btn-danger" (click)="removeSplit(i)">
<i-bs name="trash"></i-bs>
</button>
}
</li>
}
</ul>
</div>
</div>
</div>
<div class="modal-footer">
<button type="button" class="btn" [class]="cancelBtnClass" (click)="cancel()" [disabled]="!buttonsEnabled">
<span class="d-inline-block" style="padding-bottom: 1px;">{{cancelBtnCaption}}</span>
</button>
<button type="button" class="btn" [class]="btnClass" (click)="confirm()" [disabled]="!confirmButtonEnabled || !buttonsEnabled">
{{btnCaption}}
</button>
</div>

9
src-ui/src/app/components/common/confirm-dialog/split-confirm-dialog/split-confirm-dialog.component.scss Normal file
View File

@ -0,0 +1,9 @@
.pdf-viewer-container {
background-color: gray;
height: 300px;
pngx-pdf-viewer {
width: 100%;
height: 100%;
}
}

81
src-ui/src/app/components/common/confirm-dialog/split-confirm-dialog/split-confirm-dialog.component.spec.ts Normal file
View File

@ -0,0 +1,81 @@
import { ComponentFixture, TestBed } from '@angular/core/testing'
import { SplitConfirmDialogComponent } from './split-confirm-dialog.component'
import { HttpClientTestingModule } from '@angular/common/http/testing'
import { ReactiveFormsModule, FormsModule } from '@angular/forms'
import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
import { NgxBootstrapIconsModule, allIcons } from 'ngx-bootstrap-icons'
import { DocumentService } from 'src/app/services/rest/document.service'
import { PdfViewerComponent } from '../../pdf-viewer/pdf-viewer.component'
describe('SplitConfirmDialogComponent', () => {
let component: SplitConfirmDialogComponent
let fixture: ComponentFixture<SplitConfirmDialogComponent>
let documentService: DocumentService
beforeEach(async () => {
await TestBed.configureTestingModule({
declarations: [SplitConfirmDialogComponent, PdfViewerComponent],
providers: [NgbActiveModal],
imports: [
HttpClientTestingModule,
NgxBootstrapIconsModule.pick(allIcons),
ReactiveFormsModule,
FormsModule,
],
}).compileComponents()
fixture = TestBed.createComponent(SplitConfirmDialogComponent)
documentService = TestBed.inject(DocumentService)
component = fixture.componentInstance
fixture.detectChanges()
})
it('should update pagesString when pages are added', () => {
component.totalPages = 5
component.page = 2
component.addSplit()
expect(component.pagesString).toEqual('1-2,3-5')
component.page = 4
component.addSplit()
expect(component.pagesString).toEqual('1-2,3-4,5')
})
it('should update pagesString when pages are removed', () => {
component.totalPages = 5
component.page = 2
component.addSplit()
component.page = 4
component.addSplit()
expect(component.pagesString).toEqual('1-2,3-4,5')
component.removeSplit(0)
expect(component.pagesString).toEqual('1-4,5')
})
it('should enable confirm button when pages are added', () => {
component.totalPages = 5
component.page = 2
component.addSplit()
expect(component.confirmButtonEnabled).toBeTruthy()
})
it('should disable confirm button when all pages are removed', () => {
component.totalPages = 5
component.page = 2
component.addSplit()
component.removeSplit(0)
expect(component.confirmButtonEnabled).toBeFalsy()
})
it('should not add split if page is the last page', () => {
component.totalPages = 5
component.page = 5
component.addSplit()
expect(component.pagesString).toEqual('1-5')
})
it('should update totalPages when pdf is loaded', () => {
component.pdfPreviewLoaded({ numPages: 5 } as any)
expect(component.totalPages).toEqual(5)
})
})

66
src-ui/src/app/components/common/confirm-dialog/split-confirm-dialog/split-confirm-dialog.component.ts Normal file
View File

@ -0,0 +1,66 @@
import { Component } from '@angular/core'
import { ConfirmDialogComponent } from '../confirm-dialog.component'
import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
import { DocumentService } from 'src/app/services/rest/document.service'
import { PDFDocumentProxy } from '../../pdf-viewer/typings'
@Component({
selector: 'pngx-split-confirm-dialog',
templateUrl: './split-confirm-dialog.component.html',
styleUrl: './split-confirm-dialog.component.scss',
})
export class SplitConfirmDialogComponent extends ConfirmDialogComponent {
public get pagesString(): string {
let pagesStr = ''
let lastPage = 1
for (let i = 1; i <= this.totalPages; i++) {
if (this.pages.has(i) || i === this.totalPages) {
if (lastPage === i) {
pagesStr += `${i},`
lastPage = Math.min(i + 1, this.totalPages)
} else {
pagesStr += `${lastPage}-${i},`
lastPage = Math.min(i + 1, this.totalPages)
}
}
}
return pagesStr.replace(/,$/, '')
}
private pages: Set<number> = new Set()
public documentID: number
public page: number = 1
public totalPages: number
public get pdfSrc(): string {
return this.documentService.getPreviewUrl(this.documentID)
}
constructor(
activeModal: NgbActiveModal,
private documentService: DocumentService
) {
super(activeModal)
this.confirmButtonEnabled = this.pages.size > 0
}
pdfPreviewLoaded(pdf: PDFDocumentProxy) {
this.totalPages = pdf.numPages
}
addSplit() {
if (this.page === this.totalPages) return
this.pages.add(this.page)
this.pages = new Set(Array.from(this.pages).sort())
this.confirmButtonEnabled = this.pages.size > 0
}
removeSplit(i: number) {
let page = Array.from(this.pages)[Math.min(i, this.pages.size - 1)]
this.pages.delete(page)
this.confirmButtonEnabled = this.pages.size > 0
}
}

8
src-ui/src/app/components/document-detail/document-detail.component.html
View File

@ -44,11 +44,15 @@
</button>
<div ngbDropdownMenu aria-labelledby="actionsDropdown" class="shadow">
<button ngbDropdownItem (click)="redoOcr()" [disabled]="!userCanEdit">
<i-bs width="1em" height="1em" name="arrow-counterclockwise"></i-bs><span class="ps-1" i18n>Redo OCR</span>
<i-bs width="1em" height="1em" name="arrow-counterclockwise"></i-bs>&nbsp;<span i18n>Redo OCR</span>
</button>
<button ngbDropdownItem (click)="moreLike()">
<i-bs width="1em" height="1em" name="diagram-3"></i-bs><span class="ps-1" i18n>More like this</span>
<i-bs width="1em" height="1em" name="diagram-3"></i-bs>&nbsp;<span i18n>More like this</span>
</button>
<button ngbDropdownItem (click)="splitDocument()" [disabled]="contentRenderType !== ContentRenderType.PDF">
<i-bs width="1em" height="1em" name="scissors"></i-bs>&nbsp;<span i18n>Split</span>
</button>
</div>
</div>

27
src-ui/src/app/components/document-detail/document-detail.component.spec.ts
View File

@ -1070,6 +1070,33 @@ describe('DocumentDetailComponent', () => {
).not.toBeUndefined()
})
it('should support split', () => {
let modal: NgbModalRef
modalService.activeInstances.subscribe((m) => (modal = m[0]))
initNormally()
component.splitDocument()
expect(modal).not.toBeUndefined()
modal.componentInstance.documentID = doc.id
modal.componentInstance.totalPages = 5
modal.componentInstance.page = 2
modal.componentInstance.addSplit()
modal.componentInstance.confirm()
let req = httpTestingController.expectOne(
`${environment.apiBaseUrl}documents/bulk_edit/`
)
expect(req.request.body).toEqual({
documents: [doc.id],
method: 'split',
parameters: { pages: '1-2,3-5' },
})
req.error(new ProgressEvent('failed'))
modal.componentInstance.confirm()
req = httpTestingController.expectOne(
`${environment.apiBaseUrl}documents/bulk_edit/`
)
req.flush(true)
})
function initNormally() {
jest
.spyOn(activatedRoute, 'paramMap', 'get')

38
src-ui/src/app/components/document-detail/document-detail.component.ts
View File

@ -67,6 +67,7 @@ import { CustomField, CustomFieldDataType } from 'src/app/data/custom-field'
import { CustomFieldInstance } from 'src/app/data/custom-field-instance'
import { CustomFieldsService } from 'src/app/services/rest/custom-fields.service'
import { PDFDocumentProxy } from '../common/pdf-viewer/typings'
import { SplitConfirmDialogComponent } from '../common/confirm-dialog/split-confirm-dialog/split-confirm-dialog.component'
enum DocumentDetailNavIDs {
Details = 1,
@ -1040,4 +1041,41 @@ export class DocumentDetailComponent
this.updateFormForCustomFields(true)
this.documentForm.updateValueAndValidity()
}
splitDocument() {
let modal = this.modalService.open(SplitConfirmDialogComponent, {
backdrop: 'static',
})
modal.componentInstance.title = $localize`Split confirm`
modal.componentInstance.messageBold = $localize`This operation will split the selected document(s) into new documents.`
modal.componentInstance.btnCaption = $localize`Proceed`
modal.componentInstance.documentID = this.document.id
modal.componentInstance.confirmClicked
.pipe(takeUntil(this.unsubscribeNotifier))
.subscribe(() => {
modal.componentInstance.buttonsEnabled = false
this.documentsService
.bulkEdit([this.document.id], 'split', {
pages: modal.componentInstance.pagesString,
})
.pipe(first(), takeUntil(this.unsubscribeNotifier))
.subscribe({
next: () => {
this.toastService.showInfo(
$localize`Split operation will begin in the background.`
)
modal.close()
},
error: (error) => {
if (modal) {
modal.componentInstance.buttonsEnabled = true
}
this.toastService.showError(
$localize`Error executing split operation`,
error
)
},
})
})
}
}

63
src/documents/bulk_edit.py
View File

@ -212,31 +212,13 @@ def merge(doc_ids: list[int], metadata_document_id: Optional[int] = None):
)
merged_pdf.save(filepath)
overrides = DocumentMetadataOverrides()
if metadata_document_id:
metadata_document = qs.get(id=metadata_document_id)
if metadata_document is not None:
overrides = DocumentMetadataOverrides.from_document(metadata_document)
overrides.title = metadata_document.title + " (merged)"
overrides.correspondent_id = (
metadata_document.correspondent.pk
if metadata_document.correspondent
else None
)
overrides.document_type_id = (
metadata_document.document_type.pk
if metadata_document.document_type
else None
)
overrides.storage_path_id = (
metadata_document.storage_path.pk
if metadata_document.storage_path
else None
)
overrides.tag_ids = list(
metadata_document.tags.values_list("id", flat=True),
)
# Include owner and permissions?
else:
overrides = DocumentMetadataOverrides()
logger.info("Adding merged document to the task queue.")
consume_file.delay(
@ -248,3 +230,42 @@ def merge(doc_ids: list[int], metadata_document_id: Optional[int] = None):
)
return "OK"
def split(doc_ids: list[int], pages: list[list[int]]):
logger.info(
f"Attempting to split document {doc_ids[0]} into {len(pages)} documents",
)
doc = Document.objects.get(id=doc_ids[0])
import pikepdf
path = os.path.join(settings.ORIGINALS_DIR, str(doc.filename))
try:
with pikepdf.open(path, allow_overwriting_input=True) as pdf:
for idx, split_doc in enumerate(pages):
dst = pikepdf.new()
for page in split_doc:
dst.pages.append(pdf.pages[page - 1])
filepath = os.path.join(
settings.CONSUMPTION_DIR,
f"{doc.filename}_{split_doc[0]}-{split_doc[-1]}.pdf",
)
dst.save(filepath)
overrides = DocumentMetadataOverrides().from_document(doc)
overrides.title = f"{doc.title} (split {idx + 1})"
logger.info(
f"Adding split document with pages {split_doc} to the task queue.",
)
consume_file.delay(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=filepath,
),
overrides,
)
except Exception as e:
logger.exception(f"Error splitting document {doc.id}", e)
return "OK"

36
src/documents/data_models.py
View File

@ -5,6 +5,8 @@ from pathlib import Path
from typing import Optional
import magic
from guardian.shortcuts import get_groups_with_perms
from guardian.shortcuts import get_users_with_perms
@dataclasses.dataclass
@ -88,6 +90,40 @@ class DocumentMetadataOverrides:
return self
@staticmethod
def from_document(doc) -> "DocumentMetadataOverrides":
"""
Fills in the overrides from a document object
"""
overrides = DocumentMetadataOverrides()
overrides.title = doc.title
overrides.correspondent_id = doc.correspondent.id if doc.correspondent else None
overrides.document_type_id = doc.document_type.id if doc.document_type else None
overrides.storage_path_id = doc.storage_path.id if doc.storage_path else None
overrides.owner_id = doc.owner.id if doc.owner else None
overrides.tag_ids = list(doc.tags.values_list("id", flat=True))
overrides.view_users = get_users_with_perms(
doc,
only_with_perms_in=["view_document"],
).values_list("id", flat=True)
overrides.view_groups = get_groups_with_perms(
doc,
only_with_perms_in=["view_document"],
).values_list("id", flat=True)
overrides.change_users = get_users_with_perms(
doc,
only_with_perms_in=["change_document"],
).values_list("id", flat=True)
overrides.change_groups = get_groups_with_perms(
doc,
only_with_perms_in=["change_document"],
).values_list("id", flat=True)
overrides.custom_field_ids = list(
doc.custom_fields.values_list("id", flat=True),
)
return overrides
class DocumentSource(IntEnum):
"""

32
src/documents/serialisers.py
View File

@ -871,6 +871,7 @@ class BulkEditSerializer(DocumentListSerializer, SetPermissionsMixin):
"set_permissions",
"rotate",
"merge",
"split",
],
label="Method",
write_only=True,
@ -912,6 +913,8 @@ class BulkEditSerializer(DocumentListSerializer, SetPermissionsMixin):
return bulk_edit.rotate
elif method == "merge":
return bulk_edit.merge
elif method == "split":
return bulk_edit.split
else:
raise serializers.ValidationError("Unsupported method.")
@ -1000,6 +1003,29 @@ class BulkEditSerializer(DocumentListSerializer, SetPermissionsMixin):
except ValueError:
raise serializers.ValidationError("invalid rotation degrees")
def _validate_parameters_split(self, parameters):
if "pages" not in parameters:
raise serializers.ValidationError("pages not specified")
try:
pages = []
docs = parameters["pages"].split(",")
for doc in docs:
if "-" in doc:
pages.append(
[
x
for x in range(
int(doc.split("-")[0]),
int(doc.split("-")[1]) + 1,
)
],
)
else:
pages.append([int(doc)])
parameters["pages"] = pages
except ValueError:
raise serializers.ValidationError("invalid pages specified")
def validate(self, attrs):
method = attrs["method"]
parameters = attrs["parameters"]
@ -1018,6 +1044,12 @@ class BulkEditSerializer(DocumentListSerializer, SetPermissionsMixin):
self._validate_parameters_set_permissions(parameters)
elif method == bulk_edit.rotate:
self._validate_parameters_rotate(parameters)
elif method == bulk_edit.split:
if len(attrs["documents"]) > 1:
raise serializers.ValidationError(
"Split method only supports one document",
)
self._validate_parameters_split(parameters)
return attrs

72
src/documents/tests/test_api_bulk_edit.py
View File

@ -859,3 +859,75 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
args, kwargs = m.call_args
self.assertCountEqual(args[0], [self.doc2.id, self.doc3.id])
self.assertEqual(kwargs["metadata_document_id"], self.doc3.id)
@mock.patch("documents.serialisers.bulk_edit.split")
def test_split(self, m):
m.return_value = "OK"
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "split",
"parameters": {"pages": "1,2-4,5-6,7"},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
args, kwargs = m.call_args
self.assertCountEqual(args[0], [self.doc2.id])
self.assertEqual(kwargs["pages"], [[1], [2, 3, 4], [5, 6], [7]])
def test_split_invalid_params(self):
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "split",
"parameters": {}, # pages not specified
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"pages not specified", response.content)
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "split",
"parameters": {"pages": "1:7"}, # wrong format
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"invalid pages specified", response.content)
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [
self.doc1.id,
self.doc2.id,
], # only one document supported
"method": "split",
"parameters": {"pages": "1-2,3-7"}, # wrong format
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"Split method only supports one document", response.content)