lgcosta/paperless-ngx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'dev' into pr/5190

Browse Source
This commit is contained in:
shamoon 2024-02-06 23:37:03 -08:00
commit ba7104d609
26 changed files with 742 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
.ruff.toml
View File

@ -1,7 +1,29 @@
# https://beta.ruff.rs/docs/settings/
# https://beta.ruff.rs/docs/rules/
extend-select = ["I", "W", "UP", "COM", "DJ", "EXE", "ISC", "ICN", "G201", "INP", "PIE", "RSE", "SIM", "TID", "PLC", "PLE", "RUF"]
# TODO PTH
# https://docs.astral.sh/ruff/settings/
# https://docs.astral.sh/ruff/rules/
extend-select = [
"W", # https://docs.astral.sh/ruff/rules/#pycodestyle-e-w
"I", # https://docs.astral.sh/ruff/rules/#isort-i
"UP", # https://docs.astral.sh/ruff/rules/#pyupgrade-up
"COM", # https://docs.astral.sh/ruff/rules/#flake8-commas-com
"DJ", # https://docs.astral.sh/ruff/rules/#flake8-django-dj
"EXE", # https://docs.astral.sh/ruff/rules/#flake8-executable-exe
"ISC", # https://docs.astral.sh/ruff/rules/#flake8-implicit-str-concat-isc
"ICN", # https://docs.astral.sh/ruff/rules/#flake8-import-conventions-icn
"G201", # https://docs.astral.sh/ruff/rules/#flake8-logging-format-g
"INP", # https://docs.astral.sh/ruff/rules/#flake8-no-pep420-inp
"PIE", # https://docs.astral.sh/ruff/rules/#flake8-pie-pie
"Q", # https://docs.astral.sh/ruff/rules/#flake8-quotes-q
"RSE", # https://docs.astral.sh/ruff/rules/#flake8-raise-rse
"T20", # https://docs.astral.sh/ruff/rules/#flake8-print-t20
"SIM", # https://docs.astral.sh/ruff/rules/#flake8-simplify-sim
"TID", # https://docs.astral.sh/ruff/rules/#flake8-tidy-imports-tid
"TCH", # https://docs.astral.sh/ruff/rules/#flake8-type-checking-tch
"PLC", # https://docs.astral.sh/ruff/rules/#pylint-pl
"PLE", # https://docs.astral.sh/ruff/rules/#pylint-pl
"RUF", # https://docs.astral.sh/ruff/rules/#ruff-specific-rules-ruf
"FLY", # https://docs.astral.sh/ruff/rules/#flynt-fly
]
# TODO PTH https://docs.astral.sh/ruff/rules/#flake8-use-pathlib-pth
ignore = ["DJ001", "SIM105", "RUF012"]
fix = true
line-length = 88
@ -13,9 +35,9 @@ show-fixes = true
[per-file-ignores]
".github/scripts/*.py" = ["E501", "INP001", "SIM117"]
"docker/wait-for-redis.py" = ["INP001"]
"docker/wait-for-redis.py" = ["INP001", "T201"]
"*/tests/*.py" = ["E501", "SIM117"]
"*/migrations/*.py" = ["E501", "SIM"]
"*/migrations/*.py" = ["E501", "SIM", "T201"]
"src/paperless_tesseract/tests/test_parser.py" = ["RUF001"]
"src/documents/models.py" = ["SIM115"]

49
docs/configuration.md
View File

@ -1211,6 +1211,55 @@ combination with PAPERLESS_CONSUMER_BARCODE_UPSCALE bigger than 1.0.
Defaults to "300"
#### [`PAPERLESS_CONSUMER_ENABLE_TAG_BARCODE=<bool>`](#PAPERLESS_CONSUMER_ENABLE_TAG_BARCODE) {#PAPERLESS_CONSUMER_ENABLE_TAG_BARCODE}
: Enables the detection of barcodes in the scanned document and
assigns or creates tags if a properly formatted barcode is detected.
The barcode must match one of the (configurable) regular expressions.
If the barcode text contains ',' (comma), it is split into multiple
barcodes which are individually processed for tagging.
Matching is case insensitive.
Defaults to false.
#### [`PAPERLESS_CONSUMER_TAG_BARCODE_MAPPING=<json dict>`](#PAPERLESS_CONSUMER_TAG_BARCODE_MAPPING) {#PAPERLESS_CONSUMER_TAG_BARCODE_MAPPING}
: Defines a dictionary of filter regex and substitute expressions.
Syntax: {"<regex>": "<substitute>" [,...]]}
A barcode is considered for tagging if the barcode text matches
at least one of the provided <regex> pattern.
If a match is found, the <substitute> rule is applied. This allows very
versatile reformatting and mapping of barcode pattern to tag values.
If a tag is not found it will be created.
Defaults to:
{"TAG:(.*)": "\\g<1>"} which defines
- a regex TAG:(.*) which includes barcodes beginning with TAG:
followed by any text that gets stored into match group #1 and
- a substitute \\g<1> that replaces the original barcode text
by the content in match group #1.
Consequently, the tag is the barcode text without its TAG: prefix.
More examples:
{"ASN12.*": "JOHN", "ASN13.*": "SMITH"} for example maps
- ASN12nnnn barcodes to the tag JOHN and
- ASN13nnnn barcodes to the tag SMITH.
{"T-J": "JOHN", "T-S": "SMITH", "T-D": "DOE"} directly maps
- T-J barcodes to the tag JOHN,
- T-S barcodes to the tag SMITH and
- T-D barcodes to the tag DOE.
Please refer to the Python regex documentation for more information.
## Audit Trail
#### [`PAPERLESS_AUDIT_LOG_ENABLED=<bool>`](#PAPERLESS_AUDIT_LOG_ENABLED) {#PAPERLESS_AUDIT_LOG_ENABLED}

2
paperless.conf.example
View File

@ -68,6 +68,8 @@
#PAPERLESS_CONSUMER_BARCODE_STRING=PATCHT
#PAPERLESS_CONSUMER_BARCODE_UPSCALE=0.0
#PAPERLESS_CONSUMER_BARCODE_DPI=300
#PAPERLESS_CONSUMER_ENABLE_TAG_BARCODE=false
#PAPERLESS_CONSUMER_TAG_BARCODE_MAPPING={"TAG:(.*)": "\\g<1>"}
#PAPERLESS_CONSUMER_ENABLE_COLLATE_DOUBLE_SIDED=false
#PAPERLESS_CONSUMER_COLLATE_DOUBLE_SIDED_SUBDIR_NAME=double-sided
#PAPERLESS_CONSUMER_COLLATE_DOUBLE_SIDED_TIFF_SUPPORT=false

8
src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.html
View File

@ -45,11 +45,19 @@
</div>
}
@if (editing) {
@if ((selectionModel.itemsSorted | filter: filterText).length === 0 && createRef !== undefined) {
<button class="list-group-item list-group-item-action bg-light" (click)="createClicked()" [disabled]="disabled">
<small class="ms-2"><ng-container i18n>Create</ng-container> "{{filterText}}"</small>
<i-bs width="1.5em" height="1em" name="plus"></i-bs>
</button>
}
@if ((selectionModel.itemsSorted | filter: filterText).length > 0) {
<button class="list-group-item list-group-item-action bg-light" (click)="applyClicked()" [disabled]="!modelIsDirty || disabled">
<small class="ms-2" [ngClass]="{'fw-bold': modelIsDirty}" i18n>Apply</small>
<i-bs width="1.5em" height="1em" name="arrow-right"></i-bs>
</button>
}
}
@if (!editing && manyToOne) {
<div class="list-group-item list-group-item-note pt-1 pb-2">
<small i18n>Click again to exclude items.</small>

42
src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.spec.ts
View File

@ -500,4 +500,46 @@ describe('FilterableDropdownComponent & FilterableDropdownSelectionModel', () =>
selectionModel.apply()
expect(selectionModel.itemsSorted).toEqual([nullItem, items[1], items[0]])
})
it('should set support create, keep open model and call createRef method', fakeAsync(() => {
component.items = items
component.icon = 'tag-fill'
component.selectionModel = selectionModel
fixture.nativeElement
.querySelector('button')
.dispatchEvent(new MouseEvent('click')) // open
fixture.detectChanges()
tick(100)
component.filterText = 'Test Filter Text'
component.createRef = jest.fn()
component.createClicked()
expect(component.creating).toBeTruthy()
expect(component.createRef).toHaveBeenCalledWith('Test Filter Text')
const openSpy = jest.spyOn(component.dropdown, 'open')
component.dropdownOpenChange(false)
expect(openSpy).toHaveBeenCalled() // should keep open
}))
it('should call create on enter inside filter field if 0 items remain while editing', fakeAsync(() => {
component.items = items
component.icon = 'tag-fill'
component.editing = true
component.createRef = jest.fn()
const createSpy = jest.spyOn(component, 'createClicked')
expect(component.selectionModel.getSelectedItems()).toEqual([])
fixture.nativeElement
.querySelector('button')
.dispatchEvent(new MouseEvent('click')) // open
fixture.detectChanges()
tick(100)
component.filterText = 'FooBar'
fixture.detectChanges()
component.listFilterTextInput.nativeElement.dispatchEvent(
new KeyboardEvent('keyup', { key: 'Enter' })
)
expect(component.selectionModel.getSelectedItems()).toEqual([])
tick(300)
expect(createSpy).toHaveBeenCalled()
}))
})

17
src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.ts
View File

@ -398,6 +398,11 @@ export class FilterableDropdownComponent {
@Input()
disabled = false
@Input()
createRef: (name) => void
creating: boolean = false
@Output()
apply = new EventEmitter<ChangedItems>()
@ -437,6 +442,11 @@ export class FilterableDropdownComponent {
}
}
createClicked() {
this.creating = true
this.createRef(this.filterText)
}
dropdownOpenChange(open: boolean): void {
if (open) {
setTimeout(() => {
@ -447,6 +457,10 @@ export class FilterableDropdownComponent {
this.modelIsDirty = false
}
this.opened.next(this)
} else {
if (this.creating) {
this.dropdown.open()
this.creating = false
} else {
this.filterText = ''
if (this.applyOnClose && this.selectionModel.isDirty()) {
@ -454,6 +468,7 @@ export class FilterableDropdownComponent {
}
}
}
}
listFilterEnter(): void {
let filtered = this.filterPipe.transform(this.items, this.filterText)
@ -466,6 +481,8 @@ export class FilterableDropdownComponent {
this.dropdown.close()
}
}, 200)
} else if (filtered.length == 0 && this.createRef) {
this.createClicked()
}
}

4
src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.html
View File

@ -25,6 +25,7 @@
[editing]="true"
[manyToOne]="true"
[applyOnClose]="applyOnClose"
[createRef]="createTag.bind(this)"
(opened)="openTagsDropdown()"
[(selectionModel)]="tagSelectionModel"
[documentCounts]="tagDocumentCounts"
@ -38,6 +39,7 @@
[disabled]="!userCanEditAll"
[editing]="true"
[applyOnClose]="applyOnClose"
[createRef]="createCorrespondent.bind(this)"
(opened)="openCorrespondentDropdown()"
[(selectionModel)]="correspondentSelectionModel"
[documentCounts]="correspondentDocumentCounts"
@ -51,6 +53,7 @@
[disabled]="!userCanEditAll"
[editing]="true"
[applyOnClose]="applyOnClose"
[createRef]="createDocumentType.bind(this)"
(opened)="openDocumentTypeDropdown()"
[(selectionModel)]="documentTypeSelectionModel"
[documentCounts]="documentTypeDocumentCounts"
@ -64,6 +67,7 @@
[disabled]="!userCanEditAll"
[editing]="true"
[applyOnClose]="applyOnClose"
[createRef]="createStoragePath.bind(this)"
(opened)="openStoragePathDropdown()"
[(selectionModel)]="storagePathsSelectionModel"
[documentCounts]="storagePathDocumentCounts"

194
src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.spec.ts
View File

@ -42,6 +42,16 @@ import { NgSelectModule } from '@ng-select/ng-select'
import { GroupService } from 'src/app/services/rest/group.service'
import { NgxBootstrapIconsModule, allIcons } from 'ngx-bootstrap-icons'
import { SwitchComponent } from '../../common/input/switch/switch.component'
import { EditDialogMode } from '../../common/edit-dialog/edit-dialog.component'
import { TagEditDialogComponent } from '../../common/edit-dialog/tag-edit-dialog/tag-edit-dialog.component'
import { Results } from 'src/app/data/results'
import { Tag } from 'src/app/data/tag'
import { Correspondent } from 'src/app/data/correspondent'
import { DocumentType } from 'src/app/data/document-type'
import { StoragePath } from 'src/app/data/storage-path'
import { CorrespondentEditDialogComponent } from '../../common/edit-dialog/correspondent-edit-dialog/correspondent-edit-dialog.component'
import { DocumentTypeEditDialogComponent } from '../../common/edit-dialog/document-type-edit-dialog/document-type-edit-dialog.component'
import { StoragePathEditDialogComponent } from '../../common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component'
const selectionData: SelectionData = {
selected_tags: [
@ -65,6 +75,10 @@ describe('BulkEditorComponent', () => {
let documentService: DocumentService
let toastService: ToastService
let modalService: NgbModal
let tagService: TagService
let correspondentsService: CorrespondentService
let documentTypeService: DocumentTypeService
let storagePathService: StoragePathService
let httpTestingController: HttpTestingController
beforeEach(async () => {
@ -165,6 +179,10 @@ describe('BulkEditorComponent', () => {
documentService = TestBed.inject(DocumentService)
toastService = TestBed.inject(ToastService)
modalService = TestBed.inject(NgbModal)
tagService = TestBed.inject(TagService)
correspondentsService = TestBed.inject(CorrespondentService)
documentTypeService = TestBed.inject(DocumentTypeService)
storagePathService = TestBed.inject(StoragePathService)
httpTestingController = TestBed.inject(HttpTestingController)
fixture = TestBed.createComponent(BulkEditorComponent)
@ -902,4 +920,180 @@ describe('BulkEditorComponent', () => {
`${environment.apiBaseUrl}documents/storage_paths/`
)
})
it('should support create new tag', () => {
const name = 'New Tag'
const newTag = { id: 101, name: 'New Tag' }
const tags: Results<Tag> = {
results: [
{ id: 1, name: 'Tag 1' },
{ id: 2, name: 'Tag 2' },
],
count: 2,
all: [1, 2],
}
const modalInstance = {
componentInstance: {
dialogMode: EditDialogMode.CREATE,
object: { name },
succeeded: of(newTag),
},
}
const tagListAllSpy = jest.spyOn(tagService, 'listAll')
tagListAllSpy.mockReturnValue(of(tags))
const tagSelectionModelToggleSpy = jest.spyOn(
component.tagSelectionModel,
'toggle'
)
const modalServiceOpenSpy = jest.spyOn(modalService, 'open')
modalServiceOpenSpy.mockReturnValue(modalInstance as any)
component.createTag(name)
expect(modalServiceOpenSpy).toHaveBeenCalledWith(TagEditDialogComponent, {
backdrop: 'static',
})
expect(tagListAllSpy).toHaveBeenCalled()
expect(tagSelectionModelToggleSpy).toHaveBeenCalledWith(newTag.id)
expect(component.tags).toEqual(tags.results)
})
it('should support create new correspondent', () => {
const name = 'New Correspondent'
const newCorrespondent = { id: 101, name: 'New Correspondent' }
const correspondents: Results<Correspondent> = {
results: [
{ id: 1, name: 'Correspondent 1' },
{ id: 2, name: 'Correspondent 2' },
],
count: 2,
all: [1, 2],
}
const modalInstance = {
componentInstance: {
dialogMode: EditDialogMode.CREATE,
object: { name },
succeeded: of(newCorrespondent),
},
}
const correspondentsListAllSpy = jest.spyOn(
correspondentsService,
'listAll'
)
correspondentsListAllSpy.mockReturnValue(of(correspondents))
const correspondentSelectionModelToggleSpy = jest.spyOn(
component.correspondentSelectionModel,
'toggle'
)
const modalServiceOpenSpy = jest.spyOn(modalService, 'open')
modalServiceOpenSpy.mockReturnValue(modalInstance as any)
component.createCorrespondent(name)
expect(modalServiceOpenSpy).toHaveBeenCalledWith(
CorrespondentEditDialogComponent,
{ backdrop: 'static' }
)
expect(correspondentsListAllSpy).toHaveBeenCalled()
expect(correspondentSelectionModelToggleSpy).toHaveBeenCalledWith(
newCorrespondent.id
)
expect(component.correspondents).toEqual(correspondents.results)
})
it('should support create new document type', () => {
const name = 'New Document Type'
const newDocumentType = { id: 101, name: 'New Document Type' }
const documentTypes: Results<DocumentType> = {
results: [
{ id: 1, name: 'Document Type 1' },
{ id: 2, name: 'Document Type 2' },
],
count: 2,
all: [1, 2],
}
const modalInstance = {
componentInstance: {
dialogMode: EditDialogMode.CREATE,
object: { name },
succeeded: of(newDocumentType),
},
}
const documentTypesListAllSpy = jest.spyOn(documentTypeService, 'listAll')
documentTypesListAllSpy.mockReturnValue(of(documentTypes))
const documentTypeSelectionModelToggleSpy = jest.spyOn(
component.documentTypeSelectionModel,
'toggle'
)
const modalServiceOpenSpy = jest.spyOn(modalService, 'open')
modalServiceOpenSpy.mockReturnValue(modalInstance as any)
component.createDocumentType(name)
expect(modalServiceOpenSpy).toHaveBeenCalledWith(
DocumentTypeEditDialogComponent,
{ backdrop: 'static' }
)
expect(documentTypesListAllSpy).toHaveBeenCalled()
expect(documentTypeSelectionModelToggleSpy).toHaveBeenCalledWith(
newDocumentType.id
)
expect(component.documentTypes).toEqual(documentTypes.results)
})
it('should support create new storage path', () => {
const name = 'New Storage Path'
const newStoragePath = { id: 101, name: 'New Storage Path' }
const storagePaths: Results<StoragePath> = {
results: [
{ id: 1, name: 'Storage Path 1' },
{ id: 2, name: 'Storage Path 2' },
],
count: 2,
all: [1, 2],
}
const modalInstance = {
componentInstance: {
dialogMode: EditDialogMode.CREATE,
object: { name },
succeeded: of(newStoragePath),
},
}
const storagePathsListAllSpy = jest.spyOn(storagePathService, 'listAll')
storagePathsListAllSpy.mockReturnValue(of(storagePaths))
const storagePathsSelectionModelToggleSpy = jest.spyOn(
component.storagePathsSelectionModel,
'toggle'
)
const modalServiceOpenSpy = jest.spyOn(modalService, 'open')
modalServiceOpenSpy.mockReturnValue(modalInstance as any)
component.createStoragePath(name)
expect(modalServiceOpenSpy).toHaveBeenCalledWith(
StoragePathEditDialogComponent,
{ backdrop: 'static' }
)
expect(storagePathsListAllSpy).toHaveBeenCalled()
expect(storagePathsSelectionModelToggleSpy).toHaveBeenCalledWith(
newStoragePath.id
)
expect(component.storagePaths).toEqual(storagePaths.results)
})
})

93
src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.ts
View File

@ -33,7 +33,12 @@ import {
PermissionType,
} from 'src/app/services/permissions.service'
import { FormControl, FormGroup } from '@angular/forms'
import { first, Subject, takeUntil } from 'rxjs'
import { first, map, Subject, switchMap, takeUntil } from 'rxjs'
import { CorrespondentEditDialogComponent } from '../../common/edit-dialog/correspondent-edit-dialog/correspondent-edit-dialog.component'
import { EditDialogMode } from '../../common/edit-dialog/edit-dialog.component'
import { TagEditDialogComponent } from '../../common/edit-dialog/tag-edit-dialog/tag-edit-dialog.component'
import { DocumentTypeEditDialogComponent } from '../../common/edit-dialog/document-type-edit-dialog/document-type-edit-dialog.component'
import { StoragePathEditDialogComponent } from '../../common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component'
@Component({
selector: 'pngx-bulk-editor',
@ -479,6 +484,92 @@ export class BulkEditorComponent
}
}
createTag(name: string) {
let modal = this.modalService.open(TagEditDialogComponent, {
backdrop: 'static',
})
modal.componentInstance.dialogMode = EditDialogMode.CREATE
modal.componentInstance.object = { name }
modal.componentInstance.succeeded
.pipe(
switchMap((newTag) => {
return this.tagService
.listAll()
.pipe(map((tags) => ({ newTag, tags })))
})
)
.pipe(takeUntil(this.unsubscribeNotifier))
.subscribe(({ newTag, tags }) => {
this.tags = tags.results
this.tagSelectionModel.toggle(newTag.id)
})
}
createCorrespondent(name: string) {
let modal = this.modalService.open(CorrespondentEditDialogComponent, {
backdrop: 'static',
})
modal.componentInstance.dialogMode = EditDialogMode.CREATE
modal.componentInstance.object = { name }
modal.componentInstance.succeeded
.pipe(
switchMap((newCorrespondent) => {
return this.correspondentService
.listAll()
.pipe(
map((correspondents) => ({ newCorrespondent, correspondents }))
)
})
)
.pipe(takeUntil(this.unsubscribeNotifier))
.subscribe(({ newCorrespondent, correspondents }) => {
this.correspondents = correspondents.results
this.correspondentSelectionModel.toggle(newCorrespondent.id)
})
}
createDocumentType(name: string) {
let modal = this.modalService.open(DocumentTypeEditDialogComponent, {
backdrop: 'static',
})
modal.componentInstance.dialogMode = EditDialogMode.CREATE
modal.componentInstance.object = { name }
modal.componentInstance.succeeded
.pipe(
switchMap((newDocumentType) => {
return this.documentTypeService
.listAll()
.pipe(map((documentTypes) => ({ newDocumentType, documentTypes })))
})
)
.pipe(takeUntil(this.unsubscribeNotifier))
.subscribe(({ newDocumentType, documentTypes }) => {
this.documentTypes = documentTypes.results
this.documentTypeSelectionModel.toggle(newDocumentType.id)
})
}
createStoragePath(name: string) {
let modal = this.modalService.open(StoragePathEditDialogComponent, {
backdrop: 'static',
})
modal.componentInstance.dialogMode = EditDialogMode.CREATE
modal.componentInstance.object = { name }
modal.componentInstance.succeeded
.pipe(
switchMap((newStoragePath) => {
return this.storagePathService
.listAll()
.pipe(map((storagePaths) => ({ newStoragePath, storagePaths })))
})
)
.pipe(takeUntil(this.unsubscribeNotifier))
.subscribe(({ newStoragePath, storagePaths }) => {
this.storagePaths = storagePaths.results
this.storagePathsSelectionModel.toggle(newStoragePath.id)
})
}
applyDelete() {
let modal = this.modalService.open(ConfirmDialogComponent, {
backdrop: 'static',

14
src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html
View File

@ -25,7 +25,7 @@
@if (notesEnabled && document.notes.length) {
<a routerLink="/documents/{{document.id}}/notes" class="document-card-notes py-2 px-1">
<span class="badge rounded-pill bg-light border text-primary">
<i-bs width="0.9rem" height="0.9rem" class="ms-1 me-1" name="chat-left-text"></i-bs>
<i-bs width="1.2em" height="1.2em" class="ms-1 me-1" name="chat-left-text"></i-bs>
{{document.notes.length}}</span>
</a>
}
@ -43,14 +43,14 @@
@if (document.document_type) {
<button type="button" class="list-group-item list-group-item-action bg-transparent ps-0 p-1 border-0" title="Toggle document type filter" i18n-title
(click)="clickDocumentType.emit(document.document_type);$event.stopPropagation()">
<i-bs width="0.9rem" height="0.9rem" class="me-2 text-muted" name="file-earmark"></i-bs>
<i-bs width="1em" height="1em" class="me-2 text-muted" name="file-earmark"></i-bs>
<small>{{(document.document_type$ | async)?.name ?? privateName}}</small>
</button>
}
@if (document.storage_path) {
<button type="button" class="list-group-item list-group-item-action bg-transparent ps-0 p-1 border-0" title="Toggle storage path filter" i18n-title
(click)="clickStoragePath.emit(document.storage_path);$event.stopPropagation()">
<i-bs width="0.9rem" height="0.9rem" class="me-2 text-muted" name="folder"></i-bs>
<i-bs width="1em" height="1em" class="me-2 text-muted" name="folder"></i-bs>
<small>{{(document.storage_path$ | async)?.name ?? privateName}}</small>
</button>
}
@ -63,25 +63,25 @@
</div>
</ng-template>
<div class="ps-0 p-1" placement="top" [ngbTooltip]="dateTooltip">
<i-bs width="0.9rem" height="0.9rem" class="me-2 text-muted" name="calendar-event"></i-bs>
<i-bs width="1em" height="1em" class="me-2 text-muted" name="calendar-event"></i-bs>
<small>{{document.created_date | customDate:'mediumDate'}}</small>
</div>
</div>
@if (document.archive_serial_number | isNumber) {
<div class="ps-0 p-1">
<i-bs width="0.9rem" height="0.9rem" class="me-2 text-muted" name="upc-scan"></i-bs>
<i-bs width="1em" height="1em" class="me-2 text-muted" name="upc-scan"></i-bs>
<small>#{{document.archive_serial_number}}</small>
</div>
}
@if (document.owner && document.owner !== settingsService.currentUser.id) {
<div class="ps-0 p-1">
<i-bs width="0.9rem" height="0.9rem" class="me-2 text-muted" name="person-fill-lock"></i-bs>
<i-bs width="1em" height="1em" class="me-2 text-muted" name="person-fill-lock"></i-bs>
<small>{{document.owner | username}}</small>
</div>
}
@if (document.is_shared_by_requester) {
<div class="ps-0 p-1">
<i-bs width="0.9rem" height="0.9rem" class="me-2 text-muted" name="people-fill"></i-bs>
<i-bs width="1em" height="1em" class="me-2 text-muted" name="people-fill"></i-bs>
<small i18n>Shared</small>
</div>
}

2
src-ui/src/app/components/document-list/document-list.component.html
View File

@ -232,7 +232,7 @@
@if (d.notes.length) {
<a routerLink="/documents/{{d.id}}/notes" class="btn btn-sm p-0">
<span class="badge rounded-pill bg-light border text-primary">
<i-bs width="0.9rem" height="0.9rem" class="ms-1 me-1" name="chat-left-text"></i-bs>
<i-bs width="1.2em" height="1.2em" class="ms-1 me-1" name="chat-left-text"></i-bs>
{{d.notes.length}}</span>
</a>
}

62
src/documents/barcodes.py
View File

@ -14,6 +14,7 @@ from PIL import Image
from documents.converters import convert_from_tiff_to_pdf
from documents.data_models import ConsumableDocument
from documents.models import Tag
from documents.plugins.base import ConsumeTaskPlugin
from documents.plugins.base import StopConsumeTaskError
from documents.plugins.helpers import ProgressStatusOptions
@ -65,7 +66,9 @@ class BarcodePlugin(ConsumeTaskPlugin):
supported_mimes = {"application/pdf"}
return (
settings.CONSUMER_ENABLE_ASN_BARCODE or settings.CONSUMER_ENABLE_BARCODES
settings.CONSUMER_ENABLE_ASN_BARCODE
or settings.CONSUMER_ENABLE_BARCODES
or settings.CONSUMER_ENABLE_TAG_BARCODE
) and self.input_doc.mime_type in supported_mimes
def setup(self):
@ -90,6 +93,16 @@ class BarcodePlugin(ConsumeTaskPlugin):
logger.info(f"Found ASN in barcode: {located_asn}")
self.metadata.asn = located_asn
# try reading tags from barcodes
if settings.CONSUMER_ENABLE_TAG_BARCODE:
tags = self.tags
if tags is not None and len(tags) > 0:
if self.metadata.tag_ids:
self.metadata.tag_ids += tags
else:
self.metadata.tag_ids = tags
logger.info(f"Found tags in barcode: {tags}")
separator_pages = self.get_separation_pages()
if not separator_pages:
return "No pages to split on!"
@ -279,6 +292,53 @@ class BarcodePlugin(ConsumeTaskPlugin):
return asn
@property
def tags(self) -> Optional[list[int]]:
"""
Search the parsed barcodes for any tags.
Returns the detected tag ids (or empty list)
"""
tags = []
# Ensure the barcodes have been read
self.detect()
for x in self.barcodes:
tag_texts = x.value
for raw in tag_texts.split(","):
try:
tag = None
for regex in settings.CONSUMER_TAG_BARCODE_MAPPING:
if re.match(regex, raw, flags=re.IGNORECASE):
sub = settings.CONSUMER_TAG_BARCODE_MAPPING[regex]
tag = (
re.sub(regex, sub, raw, flags=re.IGNORECASE)
if sub
else raw
)
break
if tag:
tag = Tag.objects.get_or_create(
name__iexact=tag,
defaults={"name": tag},
)[0]
logger.debug(
f"Found Tag Barcode '{raw}', substituted "
f"to '{tag}' and mapped to "
f"tag #{tag.pk}.",
)
tags.append(tag.pk)
except Exception as e:
logger.error(
f"Failed to find or create TAG '{raw}' because: {e}",
)
return tags
def get_separation_pages(self) -> dict[int, bool]:
"""
Search the parsed barcodes for separators and returns a dict of page

1
src/documents/caching.py
View File

@ -90,7 +90,6 @@ def set_suggestions_cache(
"""
if classifier is not None:
doc_key = get_suggestion_cache_key(document_id)
print(classifier.last_auto_type_hash)
cache.set(
doc_key,
SuggestionCacheData(

7
src/documents/classifier.py
View File

@ -4,11 +4,14 @@ import pickle
import re
import warnings
from collections.abc import Iterator
from datetime import datetime
from hashlib import sha256
from pathlib import Path
from typing import TYPE_CHECKING
from typing import Optional
if TYPE_CHECKING:
from datetime import datetime
from pathlib import Path
from django.conf import settings
from django.core.cache import cache
from sklearn.exceptions import InconsistentVersionWarning

14
src/documents/management/commands/document_retagger.py
View File

@ -69,8 +69,6 @@ class Command(ProgressBarMixin, BaseCommand):
def handle(self, *args, **options):
self.handle_progress_bar_mixin(**options)
# Detect if we support color
color = self.style.ERROR("test") != "test"
if options["inbox_only"]:
queryset = Document.objects.filter(tags__is_inbox_tag=True)
@ -96,7 +94,8 @@ class Command(ProgressBarMixin, BaseCommand):
use_first=options["use_first"],
suggest=options["suggest"],
base_url=options["base_url"],
color=color,
stdout=self.stdout,
style_func=self.style,
)
if options["document_type"]:
@ -108,7 +107,8 @@ class Command(ProgressBarMixin, BaseCommand):
use_first=options["use_first"],
suggest=options["suggest"],
base_url=options["base_url"],
color=color,
stdout=self.stdout,
style_func=self.style,
)
if options["tags"]:
@ -119,7 +119,8 @@ class Command(ProgressBarMixin, BaseCommand):
replace=options["overwrite"],
suggest=options["suggest"],
base_url=options["base_url"],
color=color,
stdout=self.stdout,
style_func=self.style,
)
if options["storage_path"]:
set_storage_path(
@ -130,5 +131,6 @@ class Command(ProgressBarMixin, BaseCommand):
use_first=options["use_first"],
suggest=options["suggest"],
base_url=options["base_url"],
color=color,
stdout=self.stdout,
style_func=self.style,
)

2
src/documents/management/commands/document_thumbnails.py
View File

@ -19,7 +19,7 @@ def _process_document(doc_id):
if parser_class:
parser = parser_class(logging_group=None)
else:
print(f"{document} No parser for mime type {document.mime_type}")
print(f"{document} No parser for mime type {document.mime_type}") # noqa: T201
return
try:

2
src/documents/plugins/helpers.py
View File

@ -5,6 +5,8 @@ from typing import Union
from asgiref.sync import async_to_sync
from channels.layers import get_channel_layer
if TYPE_CHECKING:
from channels_redis.pubsub import RedisPubSubChannelLayer

124
src/documents/signals/handlers.py
View File

@ -18,7 +18,6 @@ from django.db import close_old_connections
from django.db import models
from django.db.models import Q
from django.dispatch import receiver
from django.utils import termcolors
from django.utils import timezone
from filelock import FileLock
@ -54,6 +53,26 @@ def add_inbox_tags(sender, document: Document, logging_group=None, **kwargs):
document.tags.add(*inbox_tags)
def _suggestion_printer(
stdout,
style_func,
suggestion_type: str,
document: Document,
selected: MatchingModel,
base_url: Optional[str] = None,
):
"""
Smaller helper to reduce duplication when just outputting suggestions to the console
"""
doc_str = str(document)
if base_url is not None:
stdout.write(style_func.SUCCESS(doc_str))
stdout.write(style_func.SUCCESS(f"{base_url}/documents/{document.pk}"))
else:
stdout.write(style_func.SUCCESS(f"{doc_str} [{document.pk}]"))
stdout.write(f"Suggest {suggestion_type}: {selected}")
def set_correspondent(
sender,
document: Document,
@ -63,7 +82,8 @@ def set_correspondent(
use_first=True,
suggest=False,
base_url=None,
color=False,
stdout=None,
style_func=None,
**kwargs,
):
if document.correspondent and not replace:
@ -90,23 +110,14 @@ def set_correspondent(
if selected or replace:
if suggest:
if base_url:
print(
termcolors.colorize(str(document), fg="green")
if color
else str(document),
_suggestion_printer(
stdout,
style_func,
"correspondent",
document,
selected,
base_url,
)
print(f"{base_url}/documents/{document.pk}")
else:
print(
(
termcolors.colorize(str(document), fg="green")
if color
else str(document)
)
+ f" [{document.pk}]",
)
print(f"Suggest correspondent {selected}")
else:
logger.info(
f"Assigning correspondent {selected} to {document}",
@ -126,7 +137,8 @@ def set_document_type(
use_first=True,
suggest=False,
base_url=None,
color=False,
stdout=None,
style_func=None,
**kwargs,
):
if document.document_type and not replace:
@ -154,23 +166,14 @@ def set_document_type(
if selected or replace:
if suggest:
if base_url:
print(
termcolors.colorize(str(document), fg="green")
if color
else str(document),
_suggestion_printer(
stdout,
style_func,
"document type",
document,
selected,
base_url,
)
print(f"{base_url}/documents/{document.pk}")
else:
print(
(
termcolors.colorize(str(document), fg="green")
if color
else str(document)
)
+ f" [{document.pk}]",
)
print(f"Suggest document type {selected}")
else:
logger.info(
f"Assigning document type {selected} to {document}",
@ -189,7 +192,8 @@ def set_tags(
replace=False,
suggest=False,
base_url=None,
color=False,
stdout=None,
style_func=None,
**kwargs,
):
if replace:
@ -212,26 +216,16 @@ def set_tags(
]
if not relevant_tags and not extra_tags:
return
doc_str = style_func.SUCCESS(str(document))
if base_url:
print(
termcolors.colorize(str(document), fg="green")
if color
else str(document),
)
print(f"{base_url}/documents/{document.pk}")
stdout.write(doc_str)
stdout.write(f"{base_url}/documents/{document.pk}")
else:
print(
(
termcolors.colorize(str(document), fg="green")
if color
else str(document)
)
+ f" [{document.pk}]",
)
stdout.write(doc_str + style_func.SUCCESS(f" [{document.pk}]"))
if relevant_tags:
print("Suggest tags: " + ", ".join([t.name for t in relevant_tags]))
stdout.write("Suggest tags: " + ", ".join([t.name for t in relevant_tags]))
if extra_tags:
print("Extra tags: " + ", ".join([t.name for t in extra_tags]))
stdout.write("Extra tags: " + ", ".join([t.name for t in extra_tags]))
else:
if not relevant_tags:
return
@ -254,7 +248,8 @@ def set_storage_path(
use_first=True,
suggest=False,
base_url=None,
color=False,
stdout=None,
style_func=None,
**kwargs,
):
if document.storage_path and not replace:
@ -285,23 +280,14 @@ def set_storage_path(
if selected or replace:
if suggest:
if base_url:
print(
termcolors.colorize(str(document), fg="green")
if color
else str(document),
_suggestion_printer(
stdout,
style_func,
"storage directory",
document,
selected,
base_url,
)
print(f"{base_url}/documents/{document.pk}")
else:
print(
(
termcolors.colorize(str(document), fg="green")
if color
else str(document)
)
+ f" [{document.pk}]",
)
print(f"Suggest storage directory {selected}")
else:
logger.info(
f"Assigning storage path {selected} to {document}",

2
src/documents/tests/test_api_bulk_download.py
View File

@ -246,8 +246,6 @@ class TestBulkDownload(DirectoriesMixin, APITestCase):
self.doc3.title = "Title 2 - Doc 3"
self.doc3.save()
print(self.doc3.archive_path)
print(self.doc3.archive_filename)
response = self.client.post(
self.ENDPOINT,

123
src/documents/tests/test_barcodes.py
View File

@ -14,6 +14,7 @@ from documents.barcodes import BarcodePlugin
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides
from documents.data_models import DocumentSource
from documents.models import Tag
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import DocumentConsumeDelayMixin
from documents.tests.utils import DummyProgressManager
@ -741,3 +742,125 @@ class TestBarcodeZxing(TestBarcode):
@override_settings(CONSUMER_BARCODE_SCANNER="ZXING")
class TestAsnBarcodesZxing(TestAsnBarcode):
pass
class TestTagBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, TestCase):
@contextmanager
def get_reader(self, filepath: Path) -> BarcodePlugin:
reader = BarcodePlugin(
ConsumableDocument(DocumentSource.ConsumeFolder, original_file=filepath),
DocumentMetadataOverrides(),
DummyProgressManager(filepath.name, None),
self.dirs.scratch_dir,
"task-id",
)
reader.setup()
yield reader
reader.cleanup()
@override_settings(CONSUMER_ENABLE_TAG_BARCODE=True)
def test_scan_file_without_matching_barcodes(self):
"""
GIVEN:
- PDF containing tag barcodes but none with matching prefix (default "TAG:")
WHEN:
- File is scanned for barcodes
THEN:
- No TAG has been created
"""
test_file = self.BARCODE_SAMPLE_DIR / "barcode-39-asn-custom-prefix.pdf"
with self.get_reader(test_file) as reader:
reader.run()
tags = reader.metadata.tag_ids
self.assertEqual(tags, None)
@override_settings(
CONSUMER_ENABLE_TAG_BARCODE=False,
CONSUMER_TAG_BARCODE_MAPPING={"CUSTOM-PREFIX-(.*)": "\\g<1>"},
)
def test_scan_file_with_matching_barcode_but_function_disabled(self):
"""
GIVEN:
- PDF containing a tag barcode with matching custom prefix
- The tag barcode functionality is disabled
WHEN:
- File is scanned for barcodes
THEN:
- No TAG has been created
"""
test_file = self.BARCODE_SAMPLE_DIR / "barcode-39-asn-custom-prefix.pdf"
with self.get_reader(test_file) as reader:
reader.run()
tags = reader.metadata.tag_ids
self.assertEqual(tags, None)
@override_settings(
CONSUMER_ENABLE_TAG_BARCODE=True,
CONSUMER_TAG_BARCODE_MAPPING={"CUSTOM-PREFIX-(.*)": "\\g<1>"},
)
def test_scan_file_for_tag_custom_prefix(self):
"""
GIVEN:
- PDF containing a tag barcode with custom prefix
- The barcode mapping accepts this prefix and removes it from the mapped tag value
- The created tag is the non-prefixed values
WHEN:
- File is scanned for barcodes
THEN:
- The TAG is located
- One TAG has been created
"""
test_file = self.BARCODE_SAMPLE_DIR / "barcode-39-asn-custom-prefix.pdf"
with self.get_reader(test_file) as reader:
reader.metadata.tag_ids = [99]
reader.run()
self.assertEqual(reader.pdf_file, test_file)
tags = reader.metadata.tag_ids
self.assertEqual(len(tags), 2)
self.assertEqual(tags[0], 99)
self.assertEqual(Tag.objects.get(name__iexact="00123").pk, tags[1])
@override_settings(
CONSUMER_ENABLE_TAG_BARCODE=True,
CONSUMER_TAG_BARCODE_MAPPING={"ASN(.*)": "\\g<1>"},
)
def test_scan_file_for_many_custom_tags(self):
"""
GIVEN:
- PDF containing multiple tag barcode with custom prefix
- The barcode mapping accepts this prefix and removes it from the mapped tag value
- The created tags are the non-prefixed values
WHEN:
- File is scanned for barcodes
THEN:
- The TAG is located
- File Tags have been created
"""
test_file = self.BARCODE_SAMPLE_DIR / "split-by-asn-1.pdf"
with self.get_reader(test_file) as reader:
reader.run()
tags = reader.metadata.tag_ids
self.assertEqual(len(tags), 5)
self.assertEqual(Tag.objects.get(name__iexact="00123").pk, tags[0])
self.assertEqual(Tag.objects.get(name__iexact="00124").pk, tags[1])
self.assertEqual(Tag.objects.get(name__iexact="00125").pk, tags[2])
self.assertEqual(Tag.objects.get(name__iexact="00126").pk, tags[3])
self.assertEqual(Tag.objects.get(name__iexact="00127").pk, tags[4])
@override_settings(
CONSUMER_ENABLE_TAG_BARCODE=True,
CONSUMER_TAG_BARCODE_MAPPING={"CUSTOM-PREFIX-(.*)": "\\g<3>"},
)
def test_scan_file_for_tag_raises_value_error(self):
"""
GIVEN:
- Any error occurs during tag barcode processing
THEN:
- The processing should be skipped and not break the import
"""
test_file = self.BARCODE_SAMPLE_DIR / "barcode-39-asn-custom-prefix.pdf"
with self.get_reader(test_file) as reader:
reader.run()
# expect error to be caught and logged only
tags = reader.metadata.tag_ids
self.assertEqual(tags, None)

8
src/documents/tests/test_management_consumer.py
View File

@ -88,10 +88,10 @@ class ConsumerThreadMixin(DocumentConsumeDelayMixin):
):
eq = filecmp.cmp(input_doc.original_file, self.sample_file, shallow=False)
if not eq:
print("Consumed an INVALID file.")
print("Consumed an INVALID file.") # noqa: T201
raise ConsumerError("Incomplete File READ FAILED")
else:
print("Consumed a perfectly valid file.")
print("Consumed a perfectly valid file.") # noqa: T201
def slow_write_file(self, target, incomplete=False):
with open(self.sample_file, "rb") as f:
@ -102,11 +102,11 @@ class ConsumerThreadMixin(DocumentConsumeDelayMixin):
with open(target, "wb") as f:
# this will take 2 seconds, since the file is about 20k.
print("Start writing file.")
print("Start writing file.") # noqa: T201
for b in chunked(1000, pdf_bytes):
f.write(b)
sleep(0.1)
print("file completed.")
print("file completed.") # noqa: T201
@override_settings(

2
src/documents/tests/test_management_fuzzy.py
View File

@ -196,7 +196,7 @@ class TestFuzzyMatchCommand(TestCase):
self.assertEqual(Document.objects.count(), 3)
stdout, _ = self.call_command("--delete")
print(stdout)
lines = [x.strip() for x in stdout.split("\n") if len(x.strip())]
self.assertEqual(len(lines), 3)
self.assertEqual(

5
src/documents/tests/test_workflows.py
View File

@ -1,16 +1,19 @@
from datetime import timedelta
from pathlib import Path
from typing import TYPE_CHECKING
from unittest import mock
from django.contrib.auth.models import Group
from django.contrib.auth.models import User
from django.db.models import QuerySet
from django.utils import timezone
from guardian.shortcuts import assign_perm
from guardian.shortcuts import get_groups_with_perms
from guardian.shortcuts import get_users_with_perms
from rest_framework.test import APITestCase
if TYPE_CHECKING:
from django.db.models import QuerySet
from documents import tasks
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentSource

1
src/documents/tests/utils.py
View File

@ -340,7 +340,6 @@ class DummyProgressManager:
def __init__(self, filename: str, task_id: Optional[str] = None) -> None:
self.filename = filename
self.task_id = task_id
print("hello world")
self.payloads = []
def __enter__(self):

6
src/paperless/auth.py
View File

@ -1,3 +1,5 @@
import logging
from django.conf import settings
from django.contrib import auth
from django.contrib.auth.middleware import PersistentRemoteUserMiddleware
@ -6,6 +8,8 @@ from django.http import HttpRequest
from django.utils.deprecation import MiddlewareMixin
from rest_framework import authentication
logger = logging.getLogger("paperless.auth")
class AutoLoginMiddleware(MiddlewareMixin):
def process_request(self, request: HttpRequest):
@ -35,7 +39,7 @@ class AngularApiAuthenticationOverride(authentication.BaseAuthentication):
and request.headers["Referer"].startswith("http://localhost:4200/")
):
user = User.objects.filter(is_staff=True).first()
print(f"Auto-Login with user {user}")
logger.debug(f"Auto-Login with user {user}")
return (user, None)
else:
return None

18
src/paperless/settings.py
View File

@ -796,6 +796,11 @@ CACHES = {
},
}
if DEBUG and os.getenv("PAPERLESS_CACHE_BACKEND") is None:
CACHES["default"][
"BACKEND"
] = "django.core.cache.backends.locmem.LocMemCache" # pragma: no cover
def default_threads_per_worker(task_workers) -> int:
# always leave one core open
@ -878,6 +883,19 @@ CONSUMER_BARCODE_UPSCALE: Final[float] = __get_float(
CONSUMER_BARCODE_DPI: Final[int] = __get_int("PAPERLESS_CONSUMER_BARCODE_DPI", 300)
CONSUMER_ENABLE_TAG_BARCODE: Final[bool] = __get_boolean(
"PAPERLESS_CONSUMER_ENABLE_TAG_BARCODE",
)
CONSUMER_TAG_BARCODE_MAPPING = dict(
json.loads(
os.getenv(
"PAPERLESS_CONSUMER_TAG_BARCODE_MAPPING",
'{"TAG:(.*)": "\\\\g<1>"}',
),
),
)
CONSUMER_ENABLE_COLLATE_DOUBLE_SIDED: Final[bool] = __get_boolean(
"PAPERLESS_CONSUMER_ENABLE_COLLATE_DOUBLE_SIDED",
)