Initial implementation of consumption templates

src/documents/consumer.py

@@ -4,6 +4,7 @@ import os
 import tempfile
 import uuid
 from enum import Enum
+from fnmatch import fnmatch
 from pathlib import Path
 from subprocess import CompletedProcess
 from subprocess import run
@@ -20,6 +21,8 @@ from django.utils import timezone
 from filelock import FileLock
 from rest_framework.reverse import reverse
 
+from documents.data_models import DocumentMetadataOverrides
+from documents.permissions import set_permissions_for_object
 from documents.utils import copy_basic_file_stats
 from documents.utils import copy_file_with_basic_stats
 
@@ -27,10 +30,12 @@ from .classifier import load_classifier
 from .file_handling import create_source_path_directory
 from .file_handling import generate_unique_filename
 from .loggers import LoggingMixin
+from .models import ConsumptionTemplate
 from .models import Correspondent
 from .models import Document
 from .models import DocumentType
 from .models import FileInfo
+from .models import StoragePath
 from .models import Tag
 from .parsers import DocumentParser
 from .parsers import ParseError
@@ -319,10 +324,15 @@ class Consumer(LoggingMixin):
         override_correspondent_id=None,
         override_document_type_id=None,
         override_tag_ids=None,
+        override_storage_path_id=None,
         task_id=None,
         override_created=None,
         override_asn=None,
         override_owner_id=None,
+        override_view_users=None,
+        override_view_groups=None,
+        override_change_users=None,
+        override_change_groups=None,
     ) -> Document:
         """
         Return the document object if it was successfully created.
@@ -334,10 +344,15 @@ class Consumer(LoggingMixin):
         self.override_correspondent_id = override_correspondent_id
         self.override_document_type_id = override_document_type_id
         self.override_tag_ids = override_tag_ids
+        self.override_storage_path_id = override_storage_path_id
         self.task_id = task_id or str(uuid.uuid4())
         self.override_created = override_created
         self.override_asn = override_asn
         self.override_owner_id = override_owner_id
+        self.override_view_users = override_view_users
+        self.override_view_groups = override_view_groups
+        self.override_change_users = override_change_users
+        self.override_change_groups = override_change_groups
 
         self._send_progress(
             0,
@@ -578,6 +593,57 @@ class Consumer(LoggingMixin):
 
         return document
 
+    def get_template_overrides(
+        self,
+        input_doc: Path,
+    ) -> DocumentMetadataOverrides:
+        overrides = DocumentMetadataOverrides()
+        for template in ConsumptionTemplate.objects.all():
+            template_overrides = DocumentMetadataOverrides()
+
+            if fnmatch(
+                input_doc.name.lower(),
+                template.filter_filename.lower(),
+            ) or input_doc.match(template.filter_path):
+                self.log.info(f"Document matched consumption template {template.name}")
+                if template.assign_tags is not None:
+                    template_overrides.tag_ids = [
+                        tag.pk for tag in template.assign_tags.all()
+                    ]
+                if template.assign_correspondent is not None:
+                    template_overrides.correspondent_id = (
+                        template.assign_correspondent.pk
+                    )
+                if template.assign_document_type is not None:
+                    template_overrides.document_type_id = (
+                        template.assign_document_type.pk
+                    )
+                if template.assign_storage_path is not None:
+                    template_overrides.storage_path_id = template.assign_storage_path.pk
+                if template.assign_owner is not None:
+                    template_overrides.owner_id = template.assign_owner
+                if template.assign_view_users is not None:
+                    template_overrides.view_users = [
+                        user.pk for user in template.assign_view_users.all()
+                    ]
+                if template.assign_view_groups is not None:
+                    template_overrides.view_groups = [
+                        group.pk for group in template.assign_view_groups.all()
+                    ]
+                if template.assign_change_users is not None:
+                    template_overrides.change_users = [
+                        user.pk for user in template.assign_change_users.all()
+                    ]
+                if template.assign_change_groups is not None:
+                    template_overrides.change_groups = [
+                        group.pk for group in template.assign_change_groups.all()
+                    ]
+                overrides = merge_overrides(
+                    overridesA=overrides,
+                    overridesB=template_overrides,
+                )
+        return overrides
+
     def _store(
         self,
         text: str,
@@ -643,6 +709,11 @@ class Consumer(LoggingMixin):
             for tag_id in self.override_tag_ids:
                 document.tags.add(Tag.objects.get(pk=tag_id))
 
+        if self.override_storage_path_id:
+            document.storage_path = StoragePath.objects.get(
+                pk=self.override_storage_path_id,
+            )
+
         if self.override_asn:
             document.archive_serial_number = self.override_asn
 
@@ -651,6 +722,24 @@ class Consumer(LoggingMixin):
                 pk=self.override_owner_id,
             )
 
+        if (
+            self.override_view_users is not None
+            or self.override_view_groups is not None
+            or self.override_change_users is not None
+            or self.override_change_users is not None
+        ):
+            permissions = {
+                "view": {
+                    "users": self.override_view_users,
+                    "groups": self.override_view_groups,
+                },
+                "change": {
+                    "users": self.override_change_users,
+                    "groups": self.override_change_groups,
+                },
+            }
+            set_permissions_for_object(permissions=permissions, object=document)
+
     def _write(self, storage_type, source, target):
         with open(source, "rb") as read_file, open(target, "wb") as write_file:
             write_file.write(read_file.read())
@@ -695,3 +784,28 @@ class Consumer(LoggingMixin):
             self.log.warning("Script stderr:")
             for line in stderr_str:
                 self.log.warning(line)
+
+
+def merge_overrides(
+    overridesA: DocumentMetadataOverrides,
+    overridesB: DocumentMetadataOverrides,
+) -> DocumentMetadataOverrides:
+    if overridesA.tag_ids is None:
+        overridesA.tag_ids = overridesB.tag_ids
+    if overridesA.correspondent_id is None:
+        overridesA.correspondent_id = overridesB.correspondent_id
+    if overridesA.document_type_id is None:
+        overridesA.document_type_id = overridesB.document_type_id
+    if overridesA.storage_path_id is None:
+        overridesA.storage_path_id = overridesB.storage_path_id
+    if overridesA.owner_id is None:
+        overridesA.owner_id = overridesB.owner_id
+    if overridesA.view_users is None:
+        overridesA.view_users = overridesB.view_users
+    if overridesA.view_groups is None:
+        overridesA.view_groups = overridesB.view_groups
+    if overridesA.change_users is None:
+        overridesA.change_users = overridesB.change_users
+    if overridesA.change_groups is None:
+        overridesA.change_groups = overridesB.change_groups
+    return overridesA

src/documents/data_models.py

@@ -20,9 +20,14 @@ class DocumentMetadataOverrides:
     correspondent_id: Optional[int] = None
     document_type_id: Optional[int] = None
     tag_ids: Optional[list[int]] = None
+    storage_path_id: Optional[int] = None
     created: Optional[datetime.datetime] = None
     asn: Optional[int] = None
     owner_id: Optional[int] = None
+    view_users: Optional[list[int]] = None
+    view_groups: Optional[list[int]] = None
+    change_users: Optional[list[int]] = None
+    change_groups: Optional[list[int]] = None
 
 
 class DocumentSource(enum.IntEnum):

src/documents/migrations/1039_consumptiontemplate.py

@@ -0,0 +1,155 @@
+# Generated by Django 4.1.11 on 2023-09-16 05:01
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ("auth", "0012_alter_user_first_name_max_length"),
+        ("documents", "1038_sharelink"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ConsumptionTemplate",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "name",
+                    models.CharField(max_length=256, unique=True, verbose_name="name"),
+                ),
+                ("order", models.IntegerField(default=0, verbose_name="order")),
+                (
+                    "filter_path",
+                    models.CharField(
+                        blank=True,
+                        help_text="Only consume documents with a path that matches this if specified. Wildcards specified as * are allowed. Case insensitive.",
+                        max_length=256,
+                        null=True,
+                        verbose_name="filter path",
+                    ),
+                ),
+                (
+                    "filter_filename",
+                    models.CharField(
+                        blank=True,
+                        help_text="Only consume documents which entirely match this filename if specified. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive.",
+                        max_length=256,
+                        null=True,
+                        verbose_name="filter filename",
+                    ),
+                ),
+                (
+                    "assign_change_groups",
+                    models.ManyToManyField(
+                        blank=True,
+                        related_name="+",
+                        to="auth.group",
+                        verbose_name="grant change permissions to these groups",
+                    ),
+                ),
+                (
+                    "assign_change_users",
+                    models.ManyToManyField(
+                        blank=True,
+                        related_name="+",
+                        to=settings.AUTH_USER_MODEL,
+                        verbose_name="grant change permissions to these users",
+                    ),
+                ),
+                (
+                    "assign_correspondent",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        to="documents.correspondent",
+                        verbose_name="assign this correspondent",
+                    ),
+                ),
+                (
+                    "assign_document_type",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        to="documents.documenttype",
+                        verbose_name="assign this document type",
+                    ),
+                ),
+                (
+                    "assign_owner",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="+",
+                        to=settings.AUTH_USER_MODEL,
+                        verbose_name="assign this owner",
+                    ),
+                ),
+                (
+                    "assign_storage_path",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        to="documents.storagepath",
+                        verbose_name="assign this storage path",
+                    ),
+                ),
+                (
+                    "assign_tags",
+                    models.ManyToManyField(
+                        blank=True,
+                        to="documents.tag",
+                        verbose_name="assign this tag",
+                    ),
+                ),
+                (
+                    "assign_view_groups",
+                    models.ManyToManyField(
+                        blank=True,
+                        related_name="+",
+                        to="auth.group",
+                        verbose_name="grant view permissions to these groups",
+                    ),
+                ),
+                (
+                    "assign_view_users",
+                    models.ManyToManyField(
+                        blank=True,
+                        related_name="+",
+                        to=settings.AUTH_USER_MODEL,
+                        verbose_name="grant view permissions to these users",
+                    ),
+                ),
+                (
+                    "owner",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        to=settings.AUTH_USER_MODEL,
+                        verbose_name="owner",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "consumption template",
+                "verbose_name_plural": "consumption templates",
+            },
+        ),
+    ]

src/documents/models.py

@@ -11,6 +11,7 @@ import dateutil.parser
 import pathvalidate
 from celery import states
 from django.conf import settings
+from django.contrib.auth.models import Group
 from django.contrib.auth.models import User
 from django.core.validators import MaxValueValidator
 from django.core.validators import MinValueValidator
@@ -735,3 +736,107 @@ class ShareLink(models.Model):
 
     def __str__(self):
         return f"Share Link for {self.document.title}"
+
+
+class ConsumptionTemplate(ModelWithOwner):
+    class Meta:
+        verbose_name = _("consumption template")
+        verbose_name_plural = _("consumption templates")
+
+    name = models.CharField(_("name"), max_length=256, unique=True)
+
+    order = models.IntegerField(_("order"), default=0)
+
+    filter_path = models.CharField(
+        _("filter path"),
+        max_length=256,
+        null=True,
+        blank=True,
+        help_text=_(
+            "Only consume documents with a path that matches "
+            "this if specified. Wildcards specified as * are "
+            "allowed. Case insensitive.",
+        ),
+    )
+
+    filter_filename = models.CharField(
+        _("filter filename"),
+        max_length=256,
+        null=True,
+        blank=True,
+        help_text=_(
+            "Only consume documents which entirely match this "
+            "filename if specified. Wildcards such as *.pdf or "
+            "*invoice* are allowed. Case insensitive.",
+        ),
+    )
+
+    assign_tags = models.ManyToManyField(
+        Tag,
+        blank=True,
+        verbose_name=_("assign this tag"),
+    )
+
+    assign_document_type = models.ForeignKey(
+        DocumentType,
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        verbose_name=_("assign this document type"),
+    )
+
+    assign_correspondent = models.ForeignKey(
+        Correspondent,
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        verbose_name=_("assign this correspondent"),
+    )
+
+    assign_storage_path = models.ForeignKey(
+        StoragePath,
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        verbose_name=_("assign this storage path"),
+    )
+
+    assign_owner = models.ForeignKey(
+        User,
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        related_name="+",
+        verbose_name=_("assign this owner"),
+    )
+
+    assign_view_users = models.ManyToManyField(
+        User,
+        blank=True,
+        related_name="+",
+        verbose_name=_("grant view permissions to these users"),
+    )
+
+    assign_view_groups = models.ManyToManyField(
+        Group,
+        blank=True,
+        related_name="+",
+        verbose_name=_("grant view permissions to these groups"),
+    )
+
+    assign_change_users = models.ManyToManyField(
+        User,
+        blank=True,
+        related_name="+",
+        verbose_name=_("grant change permissions to these users"),
+    )
+
+    assign_change_groups = models.ManyToManyField(
+        Group,
+        blank=True,
+        related_name="+",
+        verbose_name=_("grant change permissions to these groups"),
+    )
+
+    def __str__(self):
+        return f"{self.name}"

src/documents/serialisers.py

@@ -20,6 +20,7 @@ from documents.permissions import get_groups_with_only_permission
 from documents.permissions import set_permissions_for_object
 
 from . import bulk_edit
+from .models import ConsumptionTemplate
 from .models import Correspondent
 from .models import Document
 from .models import DocumentType
@@ -1035,3 +1036,55 @@ class BulkEditObjectPermissionsSerializer(serializers.Serializer, SetPermissions
             self._validate_permissions(permissions)
 
         return attrs
+
+
+class ConsumptionTemplateSerializer(OwnedObjectSerializer):
+    assign_correspondent = CorrespondentField(allow_null=True, required=False)
+    assign_tags = TagsField(many=True, allow_null=True, required=False)
+    assign_document_type = DocumentTypeField(allow_null=True, required=False)
+    assign_storage_path = StoragePathField(allow_null=True, required=False)
+    order = serializers.IntegerField(required=False)
+
+    class Meta:
+        model = ConsumptionTemplate
+        fields = [
+            "id",
+            "name",
+            "filter_path",
+            "filter_filename",
+            "assign_tags",
+            "assign_correspondent",
+            "assign_document_type",
+            "assign_storage_path",
+            "assign_owner",
+            "assign_view_users",
+            "assign_view_groups",
+            "assign_change_users",
+            "assign_change_groups",
+            "order",
+            "owner",
+            "user_can_change",
+            "permissions",
+            "set_permissions",
+        ]
+
+    def update(self, instance, validated_data):
+        super().update(instance, validated_data)
+        return instance
+
+    # def create(self, validated_data):
+    #     if "assign_tags" in validated_data:
+    #         assign_tags = validated_data.pop("assign_tags")
+    #     mail_rule = super().create(validated_data)
+    #     if assign_tags:
+    #         mail_rule.assign_tags.set(assign_tags)
+    #     return mail_rule
+
+    # def validate(self, attrs):
+    #     if (
+    #         attrs["action"] == ConsumptionTemplate.MailAction.TAG
+    #         or attrs["action"] == ConsumptionTemplate.MailAction.MOVE
+    #     ) and attrs["action_parameter"] is None:
+    #         raise serializers.ValidationError("An action parameter is required.")
+
+    #     return attrs

src/documents/tasks.py

@@ -23,6 +23,7 @@ from documents.classifier import DocumentClassifier
 from documents.classifier import load_classifier
 from documents.consumer import Consumer
 from documents.consumer import ConsumerError
+from documents.consumer import merge_overrides
 from documents.data_models import ConsumableDocument
 from documents.data_models import DocumentMetadataOverrides
 from documents.double_sided import collate
@@ -153,6 +154,12 @@ def consume_file(
                 overrides.asn = reader.asn
                 logger.info(f"Found ASN in barcode: {overrides.asn}")
 
+    template_overrides = Consumer().get_template_overrides(
+        input_doc=input_doc.original_file,
+    )
+
+    overrides = merge_overrides(overridesA=overrides, overridesB=template_overrides)
+
     # continue with consumption if no barcode was found
     document = Consumer().try_consume_file(
         input_doc.original_file,
@@ -161,9 +168,14 @@ def consume_file(
         override_correspondent_id=overrides.correspondent_id,
         override_document_type_id=overrides.document_type_id,
         override_tag_ids=overrides.tag_ids,
+        override_storage_path_id=overrides.storage_path_id,
         override_created=overrides.created,
         override_asn=overrides.asn,
         override_owner_id=overrides.owner_id,
+        override_view_users=overrides.view_users,
+        override_view_groups=overrides.view_groups,
+        override_change_users=overrides.change_users,
+        override_change_groups=overrides.change_groups,
         task_id=self.request.id,
     )
 

src/documents/tests/test_management_exporter.py

@@ -153,7 +153,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
 
         manifest = self._do_export(use_filename_format=use_filename_format)
 
-        self.assertEqual(len(manifest), 154)
+        self.assertEqual(len(manifest), 159)
 
         # dont include consumer or AnonymousUser users
         self.assertEqual(
@@ -247,7 +247,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
             self.assertEqual(Document.objects.get(id=self.d4.id).title, "wow_dec")
             self.assertEqual(GroupObjectPermission.objects.count(), 1)
             self.assertEqual(UserObjectPermission.objects.count(), 1)
-            self.assertEqual(Permission.objects.count(), 112)
+            self.assertEqual(Permission.objects.count(), 116)
             messages = check_sanity()
             # everything is alright after the test
             self.assertEqual(len(messages), 0)
@@ -676,15 +676,15 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
             os.path.join(self.dirs.media_dir, "documents"),
         )
 
-        self.assertEqual(ContentType.objects.count(), 28)
+        self.assertEqual(ContentType.objects.count(), 29)
-        self.assertEqual(Permission.objects.count(), 112)
+        self.assertEqual(Permission.objects.count(), 116)
 
         manifest = self._do_export()
 
         with paperless_environment():
             self.assertEqual(
                 len(list(filter(lambda e: e["model"] == "auth.permission", manifest))),
-                112,
+                116,
             )
             # add 1 more to db to show objects are not re-created by import
             Permission.objects.create(
@@ -692,7 +692,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
                 codename="test_perm",
                 content_type_id=1,
             )
-            self.assertEqual(Permission.objects.count(), 113)
+            self.assertEqual(Permission.objects.count(), 117)
 
             # will cause an import error
             self.user.delete()
@@ -701,5 +701,5 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
             with self.assertRaises(IntegrityError):
                 call_command("document_importer", "--no-progress-bar", self.target)
 
-            self.assertEqual(ContentType.objects.count(), 28)
+            self.assertEqual(ContentType.objects.count(), 29)
-            self.assertEqual(Permission.objects.count(), 113)
+            self.assertEqual(Permission.objects.count(), 117)

src/documents/views.py

@@ -86,6 +86,7 @@ from .matching import match_correspondents
 from .matching import match_document_types
 from .matching import match_storage_paths
 from .matching import match_tags
+from .models import ConsumptionTemplate
 from .models import Correspondent
 from .models import Document
 from .models import DocumentType
@@ -101,6 +102,7 @@ from .serialisers import AcknowledgeTasksViewSerializer
 from .serialisers import BulkDownloadSerializer
 from .serialisers import BulkEditObjectPermissionsSerializer
 from .serialisers import BulkEditSerializer
+from .serialisers import ConsumptionTemplateSerializer
 from .serialisers import CorrespondentSerializer
 from .serialisers import DocumentListSerializer
 from .serialisers import DocumentSerializer
@@ -1248,3 +1250,13 @@ class BulkEditObjectPermissionsView(GenericAPIView, PassUserMixin):
             return HttpResponseBadRequest(
                 "Error performing bulk permissions edit, check logs for more detail.",
             )
+
+
+class ConsumptionTemplateViewSet(ModelViewSet, PassUserMixin):
+    model = ConsumptionTemplate
+
+    queryset = ConsumptionTemplate.objects.all().order_by("order")
+    serializer_class = ConsumptionTemplateSerializer
+    pagination_class = StandardPagination
+    permission_classes = (IsAuthenticated, PaperlessObjectPermissions)
+    filter_backends = (ObjectOwnedOrGrantedPermissionsFilter,)

src/paperless/urls.py

@@ -14,6 +14,7 @@ from documents.views import AcknowledgeTasksView
 from documents.views import BulkDownloadView
 from documents.views import BulkEditObjectPermissionsView
 from documents.views import BulkEditView
+from documents.views import ConsumptionTemplateViewSet
 from documents.views import CorrespondentViewSet
 from documents.views import DocumentTypeViewSet
 from documents.views import IndexView
@@ -53,6 +54,7 @@ api_router.register(r"groups", GroupViewSet, basename="groups")
 api_router.register(r"mail_accounts", MailAccountViewSet)
 api_router.register(r"mail_rules", MailRuleViewSet)
 api_router.register(r"share_links", ShareLinkViewSet)
+api_router.register(r"consumption_templates", ConsumptionTemplateViewSet)
 
 
 urlpatterns = [