lgcosta/paperless-ngx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add mail rule filter

Browse Source
This commit is contained in:
shamoon 2023-09-18 10:15:53 -07:00
parent 10f247f323
commit d37051fae1
12 changed files with 205 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
docs/usage.md
View File

@ -256,12 +256,12 @@ permissions can be granted to limit access to certain parts of the UI (and corre
## Consumption templates
Introduced in v2.0, consumption templates allow for finer control over what metadata (tags, doc types)
and permissions (owner, privileges) are assigned to documents during consumption. In general, templates
are applied sequentially (by sort order) but subsequent templates will never override an assignment from
a preceding template. The same is true for mail rules, e.g. if you set the correspondent in a mail rule
any subsequent consumption templates that are applied _will not_ overwrite this. The exception to this
is assignments that can be multiple e.g. tags and permissions which will be merged.
Consumption templates were introduced in v2.0 and allow for finer control over what metadata (tags, doc
types) and permissions (owner, privileges) are assigned to documents during consumption. In general,
templates are applied sequentially (by sort order) but subsequent templates will never override an
assignment from a preceding template. The same is true for mail rules, e.g. if you set the correspondent
in a mail rule any subsequent consumption templates that are applied _will not_ overwrite this. The
exception to this is assignments that can be multiple e.g. tags and permissions, which will be merged.
Consumption templates allow you to filter by:
@ -269,10 +269,12 @@ Consumption templates allow you to filter by:
- File name, including wildcards e.g. \*.pdf will apply to all pdfs
- File path, including wildcards. Note that enabling `PAPERLESS_CONSUMER_RECURSIVE` would allow, for
example, automatically assigning documents to different owners based on the upload directory.
- Mail rule. Choosing this option will force 'mail fetch' to be the template source.
!!! note
You must include a file name filter and / or a path filter. Use * for either to apply to all files.
You must include a file name filter, a path filter or a mail rule filter. Use * for either to apply
to all files.
Consumption templates can assign:

5
src-ui/src/app/components/common/edit-dialog/consumption-template-edit-dialog/consumption-template-edit-dialog.component.html
View File

@ -12,10 +12,11 @@
<p class="small" i18n>Paperless-ngx will process mails that match <em>all</em> of the filters specified below.</p>
<pngx-input-select i18n-title title="Filter sources" [items]="sourceOptions" [multiple]="true" formControlName="sources" [error]="error?.filter_filename"></pngx-input-select>
<pngx-input-text i18n-title title="Filter filename" formControlName="filter_filename" i18n-hint hint="Apply to documents that match this filename. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive." [error]="error?.filter_filename"></pngx-input-text>
<pngx-input-text i18n-title title="Filter path" formControlName="filter_path" i18n-hint hint="Apply to documents that match this path. Wildcards specified as * are allowed. Case insensitive. Also see PAPERLESS_CONSUMER_RECURSIVE" [error]="error?.filter_path"></pngx-input-text>
<pngx-input-text i18n-title title="Filter path" formControlName="filter_path" i18n-hint hint="Apply to documents that match this path. Wildcards specified as * are allowed. Case insensitive. Also see <a target='_blank' href='https://docs.paperless-ngx.com/configuration/#consume_config'>PAPERLESS_CONSUMER_RECURSIVE</a>" [error]="error?.filter_path"></pngx-input-text>
<pngx-input-select i18n-title title="Filter mail rule" [items]="mailRules" [allowNull]="true" formControlName="filter_mailrule" i18n-hint hint="Apply to documents consumed via this mail rule." [error]="error?.filter_mailrule"></pngx-input-select>
</div>
<div class="col">
<pngx-input-text i18n-title title="Assign title" formControlName="assign_title" i18n-hint hint="Assign a document title, can include some placeholders, see documentation." [error]="error?.assign_title"></pngx-input-text>
<pngx-input-text i18n-title title="Assign title" formControlName="assign_title" i18n-hint hint="Assign a document title, can include some placeholders, see <a target='_blank' href='https://docs.paperless-ngx.com/usage/#consumption-templates'>documentation</a>." [error]="error?.assign_title"></pngx-input-text>
<pngx-input-tags [allowCreate]="false" i18n-title title="Assign tags" formControlName="assign_tags"></pngx-input-tags>
<pngx-input-select i18n-title title="Assign document type" [items]="documentTypes" [allowNull]="true" formControlName="assign_document_type"></pngx-input-select>
<pngx-input-select i18n-title title="Assign correspondent" [items]="correspondents" [allowNull]="true" formControlName="assign_correspondent"></pngx-input-select>

10
src-ui/src/app/components/common/edit-dialog/consumption-template-edit-dialog/consumption-template-edit-dialog.component.ts
View File

@ -16,6 +16,8 @@ import { StoragePathService } from 'src/app/services/rest/storage-path.service'
import { UserService } from 'src/app/services/rest/user.service'
import { SettingsService } from 'src/app/services/settings.service'
import { EditDialogComponent } from '../edit-dialog.component'
import { MailRuleService } from 'src/app/services/rest/mail-rule.service'
import { PaperlessMailRule } from 'src/app/data/paperless-mail-rule'
export const DOCUMENT_SOURCE_OPTIONS = [
{
@ -42,6 +44,7 @@ export class ConsumptionTemplateEditDialogComponent extends EditDialogComponent<
correspondents: PaperlessCorrespondent[]
documentTypes: PaperlessDocumentType[]
storagePaths: PaperlessStoragePath[]
mailRules: PaperlessMailRule[]
constructor(
service: ConsumptionTemplateService,
@ -49,6 +52,7 @@ export class ConsumptionTemplateEditDialogComponent extends EditDialogComponent<
correspondentService: CorrespondentService,
documentTypeService: DocumentTypeService,
storagePathService: StoragePathService,
mailRuleService: MailRuleService,
userService: UserService,
settingsService: SettingsService
) {
@ -68,6 +72,11 @@ export class ConsumptionTemplateEditDialogComponent extends EditDialogComponent<
.listAll()
.pipe(first())
.subscribe((result) => (this.storagePaths = result.results))
mailRuleService
.listAll()
.pipe(first())
.subscribe((result) => (this.mailRules = result.results))
}
getCreateTitle() {
@ -84,6 +93,7 @@ export class ConsumptionTemplateEditDialogComponent extends EditDialogComponent<
account: new FormControl(null),
filter_filename: new FormControl(null),
filter_path: new FormControl(null),
filter_mailrule: new FormControl(null),
order: new FormControl(null),
sources: new FormControl([]),
assign_title: new FormControl(null),

4
src-ui/src/app/data/paperless-consumption-template.ts
View File

@ -15,7 +15,9 @@ export interface PaperlessConsumptionTemplate extends ObjectWithPermissions {
filter_filename: string
filter_path: string
filter_path?: string
filter_mailrule?: number // PaperlessMailRule.id
assign_title?: string

10
src/documents/consumer.py
View File

@ -600,13 +600,19 @@ class Consumer(LoggingMixin):
) -> DocumentMetadataOverrides:
"""
Match consumption templates to a document based on source and
file name filters or path filters, if specified
file name filters, path filters or mail rule filter if specified
"""
overrides = DocumentMetadataOverrides()
for template in ConsumptionTemplate.objects.all().order_by("order"):
template_overrides = DocumentMetadataOverrides()
if int(input_doc.source) in [int(x) for x in list(template.sources)] and (
if (
int(input_doc.source) in [int(x) for x in list(template.sources)]
and (
input_doc.mailrule_id is None
or input_doc.mailrule_id == template.filter_mailrule.pk
)
) and (
(
template.filter_filename is None
or len(template.filter_filename) == 0

1
src/documents/data_models.py
View File

@ -50,6 +50,7 @@ class ConsumableDocument:
source: DocumentSource
original_file: Path
mime_type: str = dataclasses.field(init=False, default=None)
mailrule_id: int = dataclasses.field(default=None)
def __post_init__(self):
"""

11
src/documents/migrations/1039_consumptiontemplate.py
View File

@ -48,6 +48,7 @@ class Migration(migrations.Migration):
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
("auth", "0012_alter_user_first_name_max_length"),
("documents", "1038_sharelink"),
("paperless_mail", "0021_alter_mailaccount_password"),
]
operations = [
@ -100,6 +101,16 @@ class Migration(migrations.Migration):
verbose_name="filter filename",
),
),
(
"filter_mailrule",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
to="paperless_mail.mailrule",
verbose_name="filter documents from this mail rule",
),
),
(
"assign_change_groups",
models.ManyToManyField(

10
src/documents/models.py
View File

@ -786,13 +786,21 @@ class ConsumptionTemplate(ModelWithOwner):
),
)
filter_mailrule = models.ForeignKey(
"paperless_mail.MailRule",
null=True,
blank=True,
on_delete=models.SET_NULL,
verbose_name=_("filter documents from this mail rule"),
)
assign_title = models.CharField(
_("assign title"),
max_length=256,
null=True,
blank=True,
help_text=_(
"Assign a document title, can include some placeholders,"
"Assign a document title, can include some placeholders, "
"see documentation.",
),
)

13
src/documents/serialisers.py
View File

@ -1066,6 +1066,7 @@ class ConsumptionTemplateSerializer(OwnedObjectSerializer):
"sources",
"filter_path",
"filter_filename",
"filter_mailrule",
"assign_title",
"assign_tags",
"assign_correspondent",
@ -1083,9 +1084,15 @@ class ConsumptionTemplateSerializer(OwnedObjectSerializer):
]
def validate(self, attrs):
if ("filter_filename" not in attrs or len(attrs["filter_filename"]) == 0) and (
"filter_path" not in attrs or len(attrs["filter_path"]) == 0
if ("filter_mailrule") in attrs:
attrs["sources"] = {int(DocumentSource.MailFetch)}
if (
("filter_mailrule" not in attrs)
and ("filter_filename" not in attrs or len(attrs["filter_filename"]) == 0)
and ("filter_path" not in attrs or len(attrs["filter_path"]) == 0)
):
raise serializers.ValidationError("File name or path filter are required")
raise serializers.ValidationError(
"File name, path or mail rule filter are required",
)
return attrs

54
src/documents/tests/test_api.py
View File

@ -47,6 +47,8 @@ from documents.models import Tag
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import DocumentConsumeDelayMixin
from paperless import version
from paperless_mail.models import MailAccount
from paperless_mail.models import MailRule
class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
@ -5428,3 +5430,55 @@ class TestApiConsumptionTemplates(DirectoriesMixin, APITestCase):
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertEqual(StoragePath.objects.count(), 1)
def test_api_create_consumption_template_with_mailrule(self):
"""
GIVEN:
- API request to create a consumption template with a mail rule but no MailFetch source
WHEN:
- API is called
THEN:
- Correct HTTP response
- New template is created with MailFetch as source
"""
account1 = MailAccount.objects.create(
name="Email1",
username="username1",
password="password1",
imap_server="server.example.com",
imap_port=443,
imap_security=MailAccount.ImapSecurity.SSL,
character_set="UTF-8",
)
rule1 = MailRule.objects.create(
name="Rule1",
account=account1,
folder="INBOX",
filter_from="from@example.com",
filter_to="someone@somewhere.com",
filter_subject="subject",
filter_body="body",
filter_attachment_filename="file.pdf",
maximum_age=30,
action=MailRule.MailAction.MARK_READ,
assign_title_from=MailRule.TitleSource.FROM_SUBJECT,
assign_correspondent_from=MailRule.CorrespondentSource.FROM_NOTHING,
order=0,
attachment_type=MailRule.AttachmentProcessing.ATTACHMENTS_ONLY,
)
response = self.client.post(
self.ENDPOINT,
json.dumps(
{
"name": "Template 2",
"order": 1,
"sources": [DocumentSource.ApiUpload],
"filter_mailrule": rule1.pk,
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
self.assertEqual(ConsumptionTemplate.objects.count(), 2)
ct = ConsumptionTemplate.objects.get(name="Template 2")
self.assertEqual(ct.sources, [int(DocumentSource.MailFetch).__str__()])

86
src/documents/tests/test_consumption_templates.py
View File

@ -16,6 +16,8 @@ from documents.models import StoragePath
from documents.models import Tag
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import FileSystemAssertsMixin
from paperless_mail.models import MailAccount
from paperless_mail.models import MailRule
@pytest.mark.django_db
@ -96,6 +98,90 @@ class TestConsumptionTemplates(DirectoriesMixin, FileSystemAssertsMixin, TestCas
self.assertEqual(overrides["override_change_groups"], [self.group1.pk])
self.assertEqual(overrides["override_title"], "Doc from {correspondent}")
@mock.patch("documents.consumer.Consumer.try_consume_file")
def test_consumption_template_match_mailrule(self, m):
"""
GIVEN:
- Existing consumption template
WHEN:
- File that matches is consumed via mail rule
THEN:
- Template overrides are applied
"""
account1 = MailAccount.objects.create(
name="Email1",
username="username1",
password="password1",
imap_server="server.example.com",
imap_port=443,
imap_security=MailAccount.ImapSecurity.SSL,
character_set="UTF-8",
)
rule1 = MailRule.objects.create(
name="Rule1",
account=account1,
folder="INBOX",
filter_from="from@example.com",
filter_to="someone@somewhere.com",
filter_subject="subject",
filter_body="body",
filter_attachment_filename="file.pdf",
maximum_age=30,
action=MailRule.MailAction.MARK_READ,
assign_title_from=MailRule.TitleSource.FROM_SUBJECT,
assign_correspondent_from=MailRule.CorrespondentSource.FROM_NOTHING,
order=0,
attachment_type=MailRule.AttachmentProcessing.ATTACHMENTS_ONLY,
)
ct = ConsumptionTemplate.objects.create(
name="Template 1",
order=0,
sources=f"{int(DocumentSource.ApiUpload)},{int(DocumentSource.ConsumeFolder)},{int(DocumentSource.MailFetch)}",
filter_mailrule=rule1,
assign_title="Doc from {correspondent}",
assign_correspondent=self.c,
assign_document_type=self.dt,
assign_storage_path=self.sp,
assign_owner=self.user2,
)
ct.assign_tags.add(self.t1)
ct.assign_tags.add(self.t2)
ct.assign_tags.add(self.t3)
ct.assign_view_users.add(self.user3.pk)
ct.assign_view_groups.add(self.group1.pk)
ct.assign_change_users.add(self.user3.pk)
ct.assign_change_groups.add(self.group1.pk)
ct.save()
self.assertEqual(ct.__str__(), "Template 1")
test_file = self.SAMPLE_DIR / "simple.pdf"
with mock.patch("documents.tasks.async_to_sync"):
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=test_file,
mailrule_id=rule1.pk,
),
None,
)
m.assert_called_once()
_, overrides = m.call_args
self.assertEqual(overrides["override_correspondent_id"], self.c.pk)
self.assertEqual(overrides["override_document_type_id"], self.dt.pk)
self.assertEqual(
overrides["override_tag_ids"],
[self.t1.pk, self.t2.pk, self.t3.pk],
)
self.assertEqual(overrides["override_storage_path_id"], self.sp.pk)
self.assertEqual(overrides["override_owner_id"], self.user2.pk)
self.assertEqual(overrides["override_view_users"], [self.user3.pk])
self.assertEqual(overrides["override_view_groups"], [self.group1.pk])
self.assertEqual(overrides["override_change_users"], [self.user3.pk])
self.assertEqual(overrides["override_change_groups"], [self.group1.pk])
self.assertEqual(overrides["override_title"], "Doc from {correspondent}")
@mock.patch("documents.consumer.Consumer.try_consume_file")
def test_consumption_template_match_multiple(self, m):
"""

1
src/paperless_mail/mail.py
View File

@ -690,6 +690,7 @@ class MailAccountHandler(LoggingMixin):
input_doc = ConsumableDocument(
source=DocumentSource.MailFetch,
original_file=temp_filename,
mailrule_id=rule.pk,
)
doc_overrides = DocumentMetadataOverrides(
title=title,