lgcosta/paperless-ngx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support consumption template source

Browse Source
This commit is contained in:
shamoon 2023-09-16 11:08:16 -07:00
parent ed37f82c0f
commit 5a32b6fb9a
11 changed files with 609 additions and 456 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Pipfile
View File

@ -3,7 +3,6 @@ url = "https://pypi.python.org/simple"
verify_ssl = true
name = "pypi"
[packages]
dateparser = "~=1.1"
# WARNING: django does not use semver.
@ -51,6 +50,7 @@ pdf2image = "*"
flower = "*"
bleach = "*"
zxing-cpp = {version = "*", platform_machine = "== 'x86_64'"}
django-multiselectfield = "*"
[dev-packages]
# Linting

939
Pipfile.lock generated
View File

File diff suppressed because it is too large Load Diff

1
src-ui/src/app/components/common/edit-dialog/consumption-template-edit-dialog/consumption-template-edit-dialog.component.html
View File

@ -10,6 +10,7 @@
<pngx-input-text i18n-title title="Name" formControlName="name" [error]="error?.name"></pngx-input-text>
<pngx-input-number i18n-title title="Rule order" formControlName="order" [showAdd]="false" [error]="error?.order"></pngx-input-number>
<p class="small" i18n>Paperless-ngx will process mails that match <em>any</em> of the filters specified below.</p>
<pngx-input-select i18n-title title="Filter sources" [items]="sourceOptions" [multiple]="true" formControlName="sources" [error]="error?.filter_filename"></pngx-input-select>
<pngx-input-text i18n-title title="Filter filename" formControlName="filter_filename" i18n-hint hint="Apply template to documents that match this filename. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive." [error]="error?.filter_filename"></pngx-input-text>
<pngx-input-text i18n-title title="Filter path" formControlName="filter_path" i18n-hint hint="Apply template to documents that match this path. Wildcards specified as * are allowed. Case insensitive." [error]="error?.filter_path"></pngx-input-text>
</div>

32
src-ui/src/app/components/common/edit-dialog/consumption-template-edit-dialog/consumption-template-edit-dialog.component.ts
View File

@ -2,16 +2,12 @@ import { Component } from '@angular/core'
import { FormGroup, FormControl } from '@angular/forms'
import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
import { first } from 'rxjs'
import { PaperlessConsumptionTemplate } from 'src/app/data/paperless-consumption-template'
import {
DocumentSource,
PaperlessConsumptionTemplate,
} from 'src/app/data/paperless-consumption-template'
import { PaperlessCorrespondent } from 'src/app/data/paperless-correspondent'
import { PaperlessDocumentType } from 'src/app/data/paperless-document-type'
import {
MailFilterAttachmentType,
MailRuleConsumptionScope,
MailAction,
MailMetadataTitleOption,
MailMetadataCorrespondentOption,
} from 'src/app/data/paperless-mail-rule'
import { PaperlessStoragePath } from 'src/app/data/paperless-storage-path'
import { ConsumptionTemplateService } from 'src/app/services/rest/consumption-template.service'
import { CorrespondentService } from 'src/app/services/rest/correspondent.service'
@ -21,6 +17,21 @@ import { UserService } from 'src/app/services/rest/user.service'
import { SettingsService } from 'src/app/services/settings.service'
import { EditDialogComponent } from '../edit-dialog.component'
const SOURCE_OPTIONS = [
{
id: DocumentSource.ConsumeFolder,
name: $localize`Documents uploaded via consume folder`,
},
{
id: DocumentSource.ApiUpload,
name: $localize`Documents uploaded via api upload`,
},
{
id: DocumentSource.MailFetch,
name: $localize`Documents uploaded via mail fetch`,
},
]
@Component({
selector: 'pngx-consumption-template-edit-dialog',
templateUrl: './consumption-template-edit-dialog.component.html',
@ -74,6 +85,7 @@ export class ConsumptionTemplateEditDialogComponent extends EditDialogComponent<
filter_filename: new FormControl(null),
filter_path: new FormControl(null),
order: new FormControl(null),
sources: new FormControl([]),
assign_tags: new FormControl([]),
assign_owner: new FormControl(null),
assign_document_type: new FormControl(null),
@ -85,4 +97,8 @@ export class ConsumptionTemplateEditDialogComponent extends EditDialogComponent<
assign_change_groups: new FormControl(null),
})
}
get sourceOptions() {
return SOURCE_OPTIONS
}
}

11
src-ui/src/app/components/manage/consmption-templates-list/consmption-templates-list.component.spec.ts
View File

@ -9,7 +9,10 @@ import {
NgbModalModule,
} from '@ng-bootstrap/ng-bootstrap'
import { of, throwError } from 'rxjs'
import { PaperlessConsumptionTemplate } from 'src/app/data/paperless-consumption-template'
import {
DocumentSource,
PaperlessConsumptionTemplate,
} from 'src/app/data/paperless-consumption-template'
import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
import { ConsumptionTemplateService } from 'src/app/services/rest/consumption-template.service'
import { ToastService } from 'src/app/services/toast.service'
@ -24,6 +27,11 @@ const templates: PaperlessConsumptionTemplate[] = [
id: 0,
name: 'Template 1',
order: 0,
sources: [
DocumentSource.ConsumeFolder,
DocumentSource.ApiUpload,
DocumentSource.MailFetch,
],
filter_filename: 'foo',
filter_path: 'bar',
assign_tags: [1, 2, 3],
@ -32,6 +40,7 @@ const templates: PaperlessConsumptionTemplate[] = [
id: 1,
name: 'Template 2',
order: 1,
sources: [DocumentSource.MailFetch],
filter_filename: null,
filter_path: 'foo/bar',
assign_owner: 1,

8
src-ui/src/app/data/paperless-consumption-template.ts
View File

@ -1,10 +1,18 @@
import { ObjectWithPermissions } from './object-with-permissions'
export enum DocumentSource {
ConsumeFolder = 1,
ApiUpload = 2,
MailFetch = 3,
}
export interface PaperlessConsumptionTemplate extends ObjectWithPermissions {
name: string
order: number
sources: DocumentSource[]
filter_filename: string
filter_path: string

20
src/documents/consumer.py
View File

@ -21,6 +21,7 @@ from django.utils import timezone
from filelock import FileLock
from rest_framework.reverse import reverse
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides
from documents.permissions import set_permissions_for_object
from documents.utils import copy_basic_file_stats
@ -595,16 +596,25 @@ class Consumer(LoggingMixin):
def get_template_overrides(
self,
input_doc: Path,
input_doc: ConsumableDocument,
) -> DocumentMetadataOverrides:
overrides = DocumentMetadataOverrides()
for template in ConsumptionTemplate.objects.all():
template_overrides = DocumentMetadataOverrides()
if fnmatch(
input_doc.name.lower(),
template.filter_filename.lower(),
) or input_doc.match(template.filter_path):
if int(input_doc.source) in list(template.sources) and (
(
template.filter_filename is not None
and fnmatch(
input_doc.original_file.name.lower(),
template.filter_filename.lower(),
)
)
or (
template.filter_path is not None
and input_doc.original_file.match(template.filter_path)
)
):
self.log.info(f"Document matched consumption template {template.name}")
if template.assign_tags is not None:
template_overrides.tag_ids = [

17
src/documents/migrations/1039_consumptiontemplate.py
View File

@ -1,10 +1,13 @@
# Generated by Django 4.1.11 on 2023-09-16 05:01
# Generated by Django 4.1.11 on 2023-09-16 18:04
import django.db.models.deletion
import multiselectfield.db.fields
from django.conf import settings
from django.db import migrations
from django.db import models
# TODO: migrate permissions, upgrade mail rules?
class Migration(migrations.Migration):
dependencies = [
@ -31,6 +34,18 @@ class Migration(migrations.Migration):
models.CharField(max_length=256, unique=True, verbose_name="name"),
),
("order", models.IntegerField(default=0, verbose_name="order")),
(
"sources",
multiselectfield.db.fields.MultiSelectField(
choices=[
(1, "Consume Folder"),
(2, "Api Upload"),
(3, "Mail Fetch"),
],
default="1,2,3",
max_length=3,
),
),
(
"filter_path",
models.CharField(

15
src/documents/models.py
View File

@ -18,7 +18,9 @@ from django.core.validators import MinValueValidator
from django.db import models
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from multiselectfield import MultiSelectField
from documents.data_models import DocumentSource
from documents.parsers import get_default_file_extension
ALL_STATES = sorted(states.ALL_STATES)
@ -738,6 +740,13 @@ class ShareLink(models.Model):
return f"Share Link for {self.document.title}"
DOCUMENT_SOURCE = (
(int(DocumentSource.ConsumeFolder), _("Consume Folder")),
(int(DocumentSource.ApiUpload), _("Api Upload")),
(int(DocumentSource.MailFetch), _("Mail Fetch")),
)
class ConsumptionTemplate(ModelWithOwner):
class Meta:
verbose_name = _("consumption template")
@ -747,6 +756,12 @@ class ConsumptionTemplate(ModelWithOwner):
order = models.IntegerField(_("order"), default=0)
sources = MultiSelectField(
max_length=3,
choices=DOCUMENT_SOURCE,
default=f"{DocumentSource.ConsumeFolder},{DocumentSource.ApiUpload},{DocumentSource.MailFetch}",
)
filter_path = models.CharField(
_("filter path"),
max_length=256,

18
src/documents/serialisers.py
View File

@ -13,13 +13,16 @@ from django.utils.text import slugify
from django.utils.translation import gettext as _
from guardian.core import ObjectPermissionChecker
from guardian.shortcuts import get_users_with_perms
from rest_framework import fields
from rest_framework import serializers
from rest_framework.fields import SerializerMethodField
from documents.data_models import DocumentSource
from documents.permissions import get_groups_with_only_permission
from documents.permissions import set_permissions_for_object
from . import bulk_edit
from .models import DOCUMENT_SOURCE
from .models import ConsumptionTemplate
from .models import Correspondent
from .models import Document
@ -1039,17 +1042,28 @@ class BulkEditObjectPermissionsSerializer(serializers.Serializer, SetPermissions
class ConsumptionTemplateSerializer(OwnedObjectSerializer):
order = serializers.IntegerField(required=False)
sources = fields.MultipleChoiceField(
choices=DOCUMENT_SOURCE,
allow_empty=False,
default={
DocumentSource.ConsumeFolder,
DocumentSource.ApiUpload,
DocumentSource.MailFetch,
},
)
assign_correspondent = CorrespondentField(allow_null=True, required=False)
assign_tags = TagsField(many=True, allow_null=True, required=False)
assign_document_type = DocumentTypeField(allow_null=True, required=False)
assign_storage_path = StoragePathField(allow_null=True, required=False)
order = serializers.IntegerField(required=False)
class Meta:
model = ConsumptionTemplate
fields = [
"id",
"name",
"order",
"sources",
"filter_path",
"filter_filename",
"assign_tags",
@ -1061,7 +1075,6 @@ class ConsumptionTemplateSerializer(OwnedObjectSerializer):
"assign_view_groups",
"assign_change_users",
"assign_change_groups",
"order",
"owner",
"user_can_change",
"permissions",
@ -1081,6 +1094,7 @@ class ConsumptionTemplateSerializer(OwnedObjectSerializer):
# return mail_rule
# def validate(self, attrs):
# TODO: require path or filename filter
# if (
# attrs["action"] == ConsumptionTemplate.MailAction.TAG
# or attrs["action"] == ConsumptionTemplate.MailAction.MOVE

2
src/documents/tasks.py
View File

@ -155,7 +155,7 @@ def consume_file(
logger.info(f"Found ASN in barcode: {overrides.asn}")
template_overrides = Consumer().get_template_overrides(
input_doc=input_doc.original_file,
input_doc=input_doc,
)
overrides = merge_overrides(overridesA=overrides, overridesB=template_overrides)