diff --git a/src/documents/tests/test_api.py b/src/documents/tests/test_api.py index 9000c3c21..b82dc8f10 100644 --- a/src/documents/tests/test_api.py +++ b/src/documents/tests/test_api.py @@ -5769,7 +5769,7 @@ class TestApiConsumptionTemplates(DirectoriesMixin, APITestCase): filter_to="someone@somewhere.com", filter_subject="subject", filter_body="body", - filter_attachment_filename="file.pdf", + filter_attachment_filename_include="file.pdf", maximum_age=30, action=MailRule.MailAction.MARK_READ, assign_title_from=MailRule.TitleSource.FROM_SUBJECT, diff --git a/src/documents/tests/test_consumption_templates.py b/src/documents/tests/test_consumption_templates.py index 23cda7c1e..dd5d7b2af 100644 --- a/src/documents/tests/test_consumption_templates.py +++ b/src/documents/tests/test_consumption_templates.py @@ -54,7 +54,7 @@ class TestConsumptionTemplates(DirectoriesMixin, FileSystemAssertsMixin, TestCas filter_to="someone@somewhere.com", filter_subject="subject", filter_body="body", - filter_attachment_filename="file.pdf", + filter_attachment_filename_include="file.pdf", maximum_age=30, action=MailRule.MailAction.MARK_READ, assign_title_from=MailRule.TitleSource.NONE, diff --git a/src/paperless_mail/admin.py b/src/paperless_mail/admin.py index b2eed5ce3..a266b85ae 100644 --- a/src/paperless_mail/admin.py +++ b/src/paperless_mail/admin.py @@ -68,7 +68,8 @@ class MailRuleAdmin(GuardedModelAdmin): "filter_to", "filter_subject", "filter_body", - "filter_attachment_filename", + "filter_attachment_filename_include", + "filter_attachment_filename_exclude", "maximum_age", "consumption_scope", "attachment_type", diff --git a/src/paperless_mail/mail.py b/src/paperless_mail/mail.py index 2179b8c90..0b6ec316f 100644 --- a/src/paperless_mail/mail.py +++ b/src/paperless_mail/mail.py @@ -668,12 +668,29 @@ class MailAccountHandler(LoggingMixin): ) continue - if rule.filter_attachment_filename and not fnmatch( + if rule.filter_attachment_filename_include and not fnmatch( att.filename.lower(), - rule.filter_attachment_filename.lower(), + rule.filter_attachment_filename_include.lower(), ): # Force the filename and pattern to the lowercase # as this is system dependent otherwise + self.log.debug( + f"Rule {rule}: " + f"Skipping attachment {att.filename} " + f"does not match pattern {rule.filter_attachment_filename_include}", + ) + continue + elif rule.filter_attachment_filename_exclude and fnmatch( + att.filename.lower(), + rule.filter_attachment_filename_exclude.lower(), + ): + # Force the filename and pattern to the lowercase + # as this is system dependent otherwise + self.log.debug( + f"Rule {rule}: " + f"Skipping attachment {att.filename} " + f"does match pattern {rule.filter_attachment_filename_exclude}", + ) continue correspondent = self._get_correspondent(message, rule) diff --git a/src/paperless_mail/migrations/0023_remove_mailrule_filter_attachment_filename_and_more.py b/src/paperless_mail/migrations/0023_remove_mailrule_filter_attachment_filename_and_more.py new file mode 100644 index 000000000..17cb7d645 --- /dev/null +++ b/src/paperless_mail/migrations/0023_remove_mailrule_filter_attachment_filename_and_more.py @@ -0,0 +1,29 @@ +# Generated by Django 4.2.7 on 2023-11-28 17:47 + +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + dependencies = [ + ("paperless_mail", "0022_mailrule_assign_owner_from_rule_and_more"), + ] + + operations = [ + migrations.RenameField( + model_name="mailrule", + old_name="filter_attachment_filename", + new_name="filter_attachment_filename_include", + ), + migrations.AddField( + model_name="mailrule", + name="filter_attachment_filename_exclude", + field=models.CharField( + blank=True, + help_text="Do not consume documents which entirely match this filename if specified. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive.", + max_length=256, + null=True, + verbose_name="filter attachment filename exclusive", + ), + ), + ] diff --git a/src/paperless_mail/models.py b/src/paperless_mail/models.py index 6f1896f1f..29a0afbef 100644 --- a/src/paperless_mail/models.py +++ b/src/paperless_mail/models.py @@ -139,8 +139,8 @@ class MailRule(document_models.ModelWithOwner): blank=True, ) - filter_attachment_filename = models.CharField( - _("filter attachment filename"), + filter_attachment_filename_include = models.CharField( + _("filter attachment filename inclusive"), max_length=256, null=True, blank=True, @@ -151,6 +151,18 @@ class MailRule(document_models.ModelWithOwner): ), ) + filter_attachment_filename_exclude = models.CharField( + _("filter attachment filename exclusive"), + max_length=256, + null=True, + blank=True, + help_text=_( + "Do not consume documents which entirely match this " + "filename if specified. Wildcards such as *.pdf or " + "*invoice* are allowed. Case insensitive.", + ), + ) + maximum_age = models.PositiveIntegerField( _("maximum age"), default=30, diff --git a/src/paperless_mail/serialisers.py b/src/paperless_mail/serialisers.py index e5da4a7da..38ee9661e 100644 --- a/src/paperless_mail/serialisers.py +++ b/src/paperless_mail/serialisers.py @@ -79,7 +79,8 @@ class MailRuleSerializer(OwnedObjectSerializer): "filter_to", "filter_subject", "filter_body", - "filter_attachment_filename", + "filter_attachment_filename_include", + "filter_attachment_filename_exclude", "maximum_age", "action", "action_parameter", diff --git a/src/paperless_mail/tests/test_api.py b/src/paperless_mail/tests/test_api.py index a02057114..7e9bbfe84 100644 --- a/src/paperless_mail/tests/test_api.py +++ b/src/paperless_mail/tests/test_api.py @@ -377,7 +377,7 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase): filter_to="someone@somewhere.com", filter_subject="subject", filter_body="body", - filter_attachment_filename="file.pdf", + filter_attachment_filename_include="file.pdf", maximum_age=30, action=MailRule.MailAction.MARK_READ, assign_title_from=MailRule.TitleSource.FROM_SUBJECT, @@ -400,8 +400,8 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase): self.assertEqual(returned_rule1["filter_subject"], rule1.filter_subject) self.assertEqual(returned_rule1["filter_body"], rule1.filter_body) self.assertEqual( - returned_rule1["filter_attachment_filename"], - rule1.filter_attachment_filename, + returned_rule1["filter_attachment_filename_include"], + rule1.filter_attachment_filename_include, ) self.assertEqual(returned_rule1["maximum_age"], rule1.maximum_age) self.assertEqual(returned_rule1["action"], rule1.action) @@ -453,7 +453,7 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase): "filter_to": "aperson@aplace.com", "filter_subject": "subject", "filter_body": "body", - "filter_attachment_filename": "file.pdf", + "filter_attachment_filename_include": "file.pdf", "maximum_age": 30, "action": MailRule.MailAction.MARK_READ, "assign_title_from": MailRule.TitleSource.FROM_SUBJECT, @@ -488,8 +488,8 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase): self.assertEqual(returned_rule1["filter_subject"], rule1["filter_subject"]) self.assertEqual(returned_rule1["filter_body"], rule1["filter_body"]) self.assertEqual( - returned_rule1["filter_attachment_filename"], - rule1["filter_attachment_filename"], + returned_rule1["filter_attachment_filename_include"], + rule1["filter_attachment_filename_include"], ) self.assertEqual(returned_rule1["maximum_age"], rule1["maximum_age"]) self.assertEqual(returned_rule1["action"], rule1["action"]) @@ -545,7 +545,7 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase): filter_from="from@example.com", filter_subject="subject", filter_body="body", - filter_attachment_filename="file.pdf", + filter_attachment_filename_include="file.pdf", maximum_age=30, action=MailRule.MailAction.MARK_READ, assign_title_from=MailRule.TitleSource.FROM_SUBJECT, @@ -589,7 +589,7 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase): filter_from="from@example.com", filter_subject="subject", filter_body="body", - filter_attachment_filename="file.pdf", + filter_attachment_filename_include="file.pdf", maximum_age=30, action=MailRule.MailAction.MARK_READ, assign_title_from=MailRule.TitleSource.FROM_SUBJECT, diff --git a/src/paperless_mail/tests/test_mail.py b/src/paperless_mail/tests/test_mail.py index 48b1e1a7a..55983939a 100644 --- a/src/paperless_mail/tests/test_mail.py +++ b/src/paperless_mail/tests/test_mail.py @@ -526,6 +526,16 @@ class TestMail( ) def test_filename_filter(self): + """ + GIVEN: + - Email with multiple similar named attachments + - Rule with inclusive and exclusive filters + WHEN: + - Mail action filtering is checked + THEN: + - Mail action should not be performed for files excluded + - Mail action should be performed for files included + """ message = self.create_message( attachments=[ _AttachmentDef(filename="f1.pdf"), @@ -537,15 +547,67 @@ class TestMail( ], ) + @dataclasses.dataclass(frozen=True) + class FilterTestCase: + name: str + include_pattern: Optional[str] + exclude_pattern: Optional[str] + expected_matches: list[str] + tests = [ - ("*.pdf", ["f1.pdf", "f2.pdf", "f3.pdf", "file.PDf", "f1.Pdf"]), - ("f1.pdf", ["f1.pdf", "f1.Pdf"]), - ("*", ["f1.pdf", "f2.pdf", "f3.pdf", "f2.png", "file.PDf", "f1.Pdf"]), - ("*.png", ["f2.png"]), + FilterTestCase( + "PDF Wildcard", + include_pattern="*.pdf", + exclude_pattern=None, + expected_matches=["f1.pdf", "f2.pdf", "f3.pdf", "file.PDf", "f1.Pdf"], + ), + FilterTestCase( + "F1 PDF Only", + include_pattern="f1.pdf", + exclude_pattern=None, + expected_matches=["f1.pdf", "f1.Pdf"], + ), + FilterTestCase( + "All Files", + include_pattern="*", + exclude_pattern=None, + expected_matches=[ + "f1.pdf", + "f2.pdf", + "f3.pdf", + "f2.png", + "file.PDf", + "f1.Pdf", + ], + ), + FilterTestCase( + "PNG Only", + include_pattern="*.png", + exclude_pattern=None, + expected_matches=["f2.png"], + ), + FilterTestCase( + "PDF Files without f1", + include_pattern="*.pdf", + exclude_pattern="f1*", + expected_matches=["f2.pdf", "f3.pdf", "file.PDf"], + ), + FilterTestCase( + "All Files, no PNG", + include_pattern="*", + exclude_pattern="*.png", + expected_matches=[ + "f1.pdf", + "f2.pdf", + "f3.pdf", + "file.PDf", + "f1.Pdf", + ], + ), ] - for pattern, matches in tests: - with self.subTest(msg=pattern): + for test_case in tests: + with self.subTest(msg=test_case.name): self._queue_consumption_tasks_mock.reset_mock() account = MailAccount(name=str(uuid.uuid4())) account.save() @@ -553,14 +615,15 @@ class TestMail( name=str(uuid.uuid4()), assign_title_from=MailRule.TitleSource.FROM_FILENAME, account=account, - filter_attachment_filename=pattern, + filter_attachment_filename_include=test_case.include_pattern, + filter_attachment_filename_exclude=test_case.exclude_pattern, ) rule.save() self.mail_account_handler._handle_message(message, rule) self.assert_queue_consumption_tasks_call_args( [ - [{"override_filename": m} for m in matches], + [{"override_filename": m} for m in test_case.expected_matches], ], ) @@ -593,7 +656,7 @@ class TestMail( name=str(uuid.uuid4()), assign_title_from=MailRule.TitleSource.FROM_FILENAME, account=account, - filter_attachment_filename="*.pdf", + filter_attachment_filename_include="*.pdf", attachment_type=MailRule.AttachmentProcessing.EVERYTHING, action=MailRule.MailAction.DELETE, )