Add mail rule filter

docs/usage.md

@@ -256,12 +256,12 @@ permissions can be granted to limit access to certain parts of the UI (and corre
 
 ## Consumption templates
 
-Introduced in v2.0, consumption templates allow for finer control over what metadata (tags, doc types)
+Consumption templates were introduced in v2.0 and allow for finer control over what metadata (tags, doc
-and permissions (owner, privileges) are assigned to documents during consumption. In general, templates
+types) and permissions (owner, privileges) are assigned to documents during consumption. In general,
-are applied sequentially (by sort order) but subsequent templates will never override an assignment from
+templates are applied sequentially (by sort order) but subsequent templates will never override an
-a preceding template. The same is true for mail rules, e.g. if you set the correspondent in a mail rule
+assignment from a preceding template. The same is true for mail rules, e.g. if you set the correspondent
-any subsequent consumption templates that are applied _will not_ overwrite this. The exception to this
+in a mail rule any subsequent consumption templates that are applied _will not_ overwrite this. The
-is assignments that can be multiple e.g. tags and permissions which will be merged.
+exception to this is assignments that can be multiple e.g. tags and permissions, which will be merged.
 
 Consumption templates allow you to filter by:
 
@@ -269,10 +269,12 @@ Consumption templates allow you to filter by:
 - File name, including wildcards e.g. \*.pdf will apply to all pdfs
 - File path, including wildcards. Note that enabling `PAPERLESS_CONSUMER_RECURSIVE` would allow, for
   example, automatically assigning documents to different owners based on the upload directory.
+- Mail rule. Choosing this option will force 'mail fetch' to be the template source.
 
 !!! note
 
-    You must include a file name filter and / or a path filter. Use * for either to apply to all files.
+    You must include a file name filter, a path filter or a mail rule filter. Use * for either to apply
+    to all files.
 
 Consumption templates can assign:
 

src-ui/src/app/components/common/edit-dialog/consumption-template-edit-dialog/consumption-template-edit-dialog.component.html

@@ -12,10 +12,11 @@
           <p class="small" i18n>Paperless-ngx will process mails that match <em>all</em> of the filters specified below.</p>
           <pngx-input-select i18n-title title="Filter sources" [items]="sourceOptions" [multiple]="true" formControlName="sources" [error]="error?.filter_filename"></pngx-input-select>
           <pngx-input-text i18n-title title="Filter filename" formControlName="filter_filename" i18n-hint hint="Apply to documents that match this filename. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive." [error]="error?.filter_filename"></pngx-input-text>
-          <pngx-input-text i18n-title title="Filter path" formControlName="filter_path" i18n-hint hint="Apply to documents that match this path. Wildcards specified as * are allowed. Case insensitive. Also see PAPERLESS_CONSUMER_RECURSIVE" [error]="error?.filter_path"></pngx-input-text>
+          <pngx-input-text i18n-title title="Filter path" formControlName="filter_path" i18n-hint hint="Apply to documents that match this path. Wildcards specified as * are allowed. Case insensitive. Also see <a target='_blank' href='https://docs.paperless-ngx.com/configuration/#consume_config'>PAPERLESS_CONSUMER_RECURSIVE</a>" [error]="error?.filter_path"></pngx-input-text>
+          <pngx-input-select i18n-title title="Filter mail rule" [items]="mailRules" [allowNull]="true" formControlName="filter_mailrule" i18n-hint hint="Apply to documents consumed via this mail rule." [error]="error?.filter_mailrule"></pngx-input-select>
         </div>
         <div class="col">
-            <pngx-input-text i18n-title title="Assign title" formControlName="assign_title" i18n-hint hint="Assign a document title, can include some placeholders, see documentation." [error]="error?.assign_title"></pngx-input-text>
+            <pngx-input-text i18n-title title="Assign title" formControlName="assign_title" i18n-hint hint="Assign a document title, can include some placeholders, see <a target='_blank' href='https://docs.paperless-ngx.com/usage/#consumption-templates'>documentation</a>." [error]="error?.assign_title"></pngx-input-text>
             <pngx-input-tags [allowCreate]="false" i18n-title title="Assign tags" formControlName="assign_tags"></pngx-input-tags>
             <pngx-input-select i18n-title title="Assign document type" [items]="documentTypes" [allowNull]="true" formControlName="assign_document_type"></pngx-input-select>
             <pngx-input-select i18n-title title="Assign correspondent" [items]="correspondents" [allowNull]="true" formControlName="assign_correspondent"></pngx-input-select>

src-ui/src/app/components/common/edit-dialog/consumption-template-edit-dialog/consumption-template-edit-dialog.component.ts

@@ -16,6 +16,8 @@ import { StoragePathService } from 'src/app/services/rest/storage-path.service'
 import { UserService } from 'src/app/services/rest/user.service'
 import { SettingsService } from 'src/app/services/settings.service'
 import { EditDialogComponent } from '../edit-dialog.component'
+import { MailRuleService } from 'src/app/services/rest/mail-rule.service'
+import { PaperlessMailRule } from 'src/app/data/paperless-mail-rule'
 
 export const DOCUMENT_SOURCE_OPTIONS = [
   {
@@ -42,6 +44,7 @@ export class ConsumptionTemplateEditDialogComponent extends EditDialogComponent<
   correspondents: PaperlessCorrespondent[]
   documentTypes: PaperlessDocumentType[]
   storagePaths: PaperlessStoragePath[]
+  mailRules: PaperlessMailRule[]
 
   constructor(
     service: ConsumptionTemplateService,
@@ -49,6 +52,7 @@ export class ConsumptionTemplateEditDialogComponent extends EditDialogComponent<
     correspondentService: CorrespondentService,
     documentTypeService: DocumentTypeService,
     storagePathService: StoragePathService,
+    mailRuleService: MailRuleService,
     userService: UserService,
     settingsService: SettingsService
   ) {
@@ -68,6 +72,11 @@ export class ConsumptionTemplateEditDialogComponent extends EditDialogComponent<
       .listAll()
       .pipe(first())
       .subscribe((result) => (this.storagePaths = result.results))
+
+    mailRuleService
+      .listAll()
+      .pipe(first())
+      .subscribe((result) => (this.mailRules = result.results))
   }
 
   getCreateTitle() {
@@ -84,6 +93,7 @@ export class ConsumptionTemplateEditDialogComponent extends EditDialogComponent<
       account: new FormControl(null),
       filter_filename: new FormControl(null),
       filter_path: new FormControl(null),
+      filter_mailrule: new FormControl(null),
       order: new FormControl(null),
       sources: new FormControl([]),
       assign_title: new FormControl(null),

src-ui/src/app/data/paperless-consumption-template.ts

@@ -15,7 +15,9 @@ export interface PaperlessConsumptionTemplate extends ObjectWithPermissions {
 
   filter_filename: string
 
-  filter_path: string
+  filter_path?: string
+
+  filter_mailrule?: number // PaperlessMailRule.id
 
   assign_title?: string
 

src/documents/consumer.py

@@ -600,13 +600,19 @@ class Consumer(LoggingMixin):
     ) -> DocumentMetadataOverrides:
         """
         Match consumption templates to a document based on source and
-        file name filters or path filters, if specified
+        file name filters, path filters or mail rule filter if specified
         """
         overrides = DocumentMetadataOverrides()
         for template in ConsumptionTemplate.objects.all().order_by("order"):
             template_overrides = DocumentMetadataOverrides()
 
-            if int(input_doc.source) in [int(x) for x in list(template.sources)] and (
+            if (
+                int(input_doc.source) in [int(x) for x in list(template.sources)]
+                and (
+                    input_doc.mailrule_id is None
+                    or input_doc.mailrule_id == template.filter_mailrule.pk
+                )
+            ) and (
                 (
                     template.filter_filename is None
                     or len(template.filter_filename) == 0

src/documents/data_models.py

@@ -50,6 +50,7 @@ class ConsumableDocument:
     source: DocumentSource
     original_file: Path
     mime_type: str = dataclasses.field(init=False, default=None)
+    mailrule_id: int = dataclasses.field(default=None)
 
     def __post_init__(self):
         """

src/documents/migrations/1039_consumptiontemplate.py

@@ -48,6 +48,7 @@ class Migration(migrations.Migration):
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
         ("auth", "0012_alter_user_first_name_max_length"),
         ("documents", "1038_sharelink"),
+        ("paperless_mail", "0021_alter_mailaccount_password"),
     ]
 
     operations = [
@@ -100,6 +101,16 @@ class Migration(migrations.Migration):
                         verbose_name="filter filename",
                     ),
                 ),
+                (
+                    "filter_mailrule",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        to="paperless_mail.mailrule",
+                        verbose_name="filter documents from this mail rule",
+                    ),
+                ),
                 (
                     "assign_change_groups",
                     models.ManyToManyField(

src/documents/models.py

@@ -786,13 +786,21 @@ class ConsumptionTemplate(ModelWithOwner):
         ),
     )
 
+    filter_mailrule = models.ForeignKey(
+        "paperless_mail.MailRule",
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        verbose_name=_("filter documents from this mail rule"),
+    )
+
     assign_title = models.CharField(
         _("assign title"),
         max_length=256,
         null=True,
         blank=True,
         help_text=_(
-            "Assign a document title, can include some placeholders,"
+            "Assign a document title, can include some placeholders, "
             "see documentation.",
         ),
     )

src/documents/serialisers.py

@@ -1066,6 +1066,7 @@ class ConsumptionTemplateSerializer(OwnedObjectSerializer):
             "sources",
             "filter_path",
             "filter_filename",
+            "filter_mailrule",
             "assign_title",
             "assign_tags",
             "assign_correspondent",
@@ -1083,9 +1084,15 @@ class ConsumptionTemplateSerializer(OwnedObjectSerializer):
         ]
 
     def validate(self, attrs):
-        if ("filter_filename" not in attrs or len(attrs["filter_filename"]) == 0) and (
+        if ("filter_mailrule") in attrs:
-            "filter_path" not in attrs or len(attrs["filter_path"]) == 0
+            attrs["sources"] = {int(DocumentSource.MailFetch)}
+        if (
+            ("filter_mailrule" not in attrs)
+            and ("filter_filename" not in attrs or len(attrs["filter_filename"]) == 0)
+            and ("filter_path" not in attrs or len(attrs["filter_path"]) == 0)
         ):
-            raise serializers.ValidationError("File name or path filter are required")
+            raise serializers.ValidationError(
+                "File name, path or mail rule filter are required",
+            )
 
         return attrs

src/documents/tests/test_api.py

@@ -47,6 +47,8 @@ from documents.models import Tag
 from documents.tests.utils import DirectoriesMixin
 from documents.tests.utils import DocumentConsumeDelayMixin
 from paperless import version
+from paperless_mail.models import MailAccount
+from paperless_mail.models import MailRule
 
 
 class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
@@ -5428,3 +5430,55 @@ class TestApiConsumptionTemplates(DirectoriesMixin, APITestCase):
         )
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
         self.assertEqual(StoragePath.objects.count(), 1)
+
+    def test_api_create_consumption_template_with_mailrule(self):
+        """
+        GIVEN:
+            - API request to create a consumption template with a mail rule but no MailFetch source
+        WHEN:
+            - API is called
+        THEN:
+            - Correct HTTP response
+            - New template is created with MailFetch as source
+        """
+        account1 = MailAccount.objects.create(
+            name="Email1",
+            username="username1",
+            password="password1",
+            imap_server="server.example.com",
+            imap_port=443,
+            imap_security=MailAccount.ImapSecurity.SSL,
+            character_set="UTF-8",
+        )
+        rule1 = MailRule.objects.create(
+            name="Rule1",
+            account=account1,
+            folder="INBOX",
+            filter_from="from@example.com",
+            filter_to="someone@somewhere.com",
+            filter_subject="subject",
+            filter_body="body",
+            filter_attachment_filename="file.pdf",
+            maximum_age=30,
+            action=MailRule.MailAction.MARK_READ,
+            assign_title_from=MailRule.TitleSource.FROM_SUBJECT,
+            assign_correspondent_from=MailRule.CorrespondentSource.FROM_NOTHING,
+            order=0,
+            attachment_type=MailRule.AttachmentProcessing.ATTACHMENTS_ONLY,
+        )
+        response = self.client.post(
+            self.ENDPOINT,
+            json.dumps(
+                {
+                    "name": "Template 2",
+                    "order": 1,
+                    "sources": [DocumentSource.ApiUpload],
+                    "filter_mailrule": rule1.pk,
+                },
+            ),
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+        self.assertEqual(ConsumptionTemplate.objects.count(), 2)
+        ct = ConsumptionTemplate.objects.get(name="Template 2")
+        self.assertEqual(ct.sources, [int(DocumentSource.MailFetch).__str__()])

src/documents/tests/test_consumption_templates.py

@@ -16,6 +16,8 @@ from documents.models import StoragePath
 from documents.models import Tag
 from documents.tests.utils import DirectoriesMixin
 from documents.tests.utils import FileSystemAssertsMixin
+from paperless_mail.models import MailAccount
+from paperless_mail.models import MailRule
 
 
 @pytest.mark.django_db
@@ -96,6 +98,90 @@ class TestConsumptionTemplates(DirectoriesMixin, FileSystemAssertsMixin, TestCas
             self.assertEqual(overrides["override_change_groups"], [self.group1.pk])
             self.assertEqual(overrides["override_title"], "Doc from {correspondent}")
 
+    @mock.patch("documents.consumer.Consumer.try_consume_file")
+    def test_consumption_template_match_mailrule(self, m):
+        """
+        GIVEN:
+            - Existing consumption template
+        WHEN:
+            - File that matches is consumed via mail rule
+        THEN:
+            - Template overrides are applied
+        """
+        account1 = MailAccount.objects.create(
+            name="Email1",
+            username="username1",
+            password="password1",
+            imap_server="server.example.com",
+            imap_port=443,
+            imap_security=MailAccount.ImapSecurity.SSL,
+            character_set="UTF-8",
+        )
+        rule1 = MailRule.objects.create(
+            name="Rule1",
+            account=account1,
+            folder="INBOX",
+            filter_from="from@example.com",
+            filter_to="someone@somewhere.com",
+            filter_subject="subject",
+            filter_body="body",
+            filter_attachment_filename="file.pdf",
+            maximum_age=30,
+            action=MailRule.MailAction.MARK_READ,
+            assign_title_from=MailRule.TitleSource.FROM_SUBJECT,
+            assign_correspondent_from=MailRule.CorrespondentSource.FROM_NOTHING,
+            order=0,
+            attachment_type=MailRule.AttachmentProcessing.ATTACHMENTS_ONLY,
+        )
+        ct = ConsumptionTemplate.objects.create(
+            name="Template 1",
+            order=0,
+            sources=f"{int(DocumentSource.ApiUpload)},{int(DocumentSource.ConsumeFolder)},{int(DocumentSource.MailFetch)}",
+            filter_mailrule=rule1,
+            assign_title="Doc from {correspondent}",
+            assign_correspondent=self.c,
+            assign_document_type=self.dt,
+            assign_storage_path=self.sp,
+            assign_owner=self.user2,
+        )
+        ct.assign_tags.add(self.t1)
+        ct.assign_tags.add(self.t2)
+        ct.assign_tags.add(self.t3)
+        ct.assign_view_users.add(self.user3.pk)
+        ct.assign_view_groups.add(self.group1.pk)
+        ct.assign_change_users.add(self.user3.pk)
+        ct.assign_change_groups.add(self.group1.pk)
+        ct.save()
+
+        self.assertEqual(ct.__str__(), "Template 1")
+
+        test_file = self.SAMPLE_DIR / "simple.pdf"
+
+        with mock.patch("documents.tasks.async_to_sync"):
+            tasks.consume_file(
+                ConsumableDocument(
+                    source=DocumentSource.ConsumeFolder,
+                    original_file=test_file,
+                    mailrule_id=rule1.pk,
+                ),
+                None,
+            )
+            m.assert_called_once()
+            _, overrides = m.call_args
+            self.assertEqual(overrides["override_correspondent_id"], self.c.pk)
+            self.assertEqual(overrides["override_document_type_id"], self.dt.pk)
+            self.assertEqual(
+                overrides["override_tag_ids"],
+                [self.t1.pk, self.t2.pk, self.t3.pk],
+            )
+            self.assertEqual(overrides["override_storage_path_id"], self.sp.pk)
+            self.assertEqual(overrides["override_owner_id"], self.user2.pk)
+            self.assertEqual(overrides["override_view_users"], [self.user3.pk])
+            self.assertEqual(overrides["override_view_groups"], [self.group1.pk])
+            self.assertEqual(overrides["override_change_users"], [self.user3.pk])
+            self.assertEqual(overrides["override_change_groups"], [self.group1.pk])
+            self.assertEqual(overrides["override_title"], "Doc from {correspondent}")
+
     @mock.patch("documents.consumer.Consumer.try_consume_file")
     def test_consumption_template_match_multiple(self, m):
         """

src/paperless_mail/mail.py

@@ -690,6 +690,7 @@ class MailAccountHandler(LoggingMixin):
                 input_doc = ConsumableDocument(
                     source=DocumentSource.MailFetch,
                     original_file=temp_filename,
+                    mailrule_id=rule.pk,
                 )
                 doc_overrides = DocumentMetadataOverrides(
                     title=title,