diff --git a/src/documents/consumer.py b/src/documents/consumer.py index 3f83e0f50..42db79600 100644 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -4,6 +4,7 @@ import os import tempfile import uuid from enum import Enum +from fnmatch import fnmatch from pathlib import Path from subprocess import CompletedProcess from subprocess import run @@ -20,6 +21,8 @@ from django.utils import timezone from filelock import FileLock from rest_framework.reverse import reverse +from documents.data_models import DocumentMetadataOverrides +from documents.permissions import set_permissions_for_object from documents.utils import copy_basic_file_stats from documents.utils import copy_file_with_basic_stats @@ -27,10 +30,12 @@ from .classifier import load_classifier from .file_handling import create_source_path_directory from .file_handling import generate_unique_filename from .loggers import LoggingMixin +from .models import ConsumptionTemplate from .models import Correspondent from .models import Document from .models import DocumentType from .models import FileInfo +from .models import StoragePath from .models import Tag from .parsers import DocumentParser from .parsers import ParseError @@ -319,10 +324,15 @@ class Consumer(LoggingMixin): override_correspondent_id=None, override_document_type_id=None, override_tag_ids=None, + override_storage_path_id=None, task_id=None, override_created=None, override_asn=None, override_owner_id=None, + override_view_users=None, + override_view_groups=None, + override_change_users=None, + override_change_groups=None, ) -> Document: """ Return the document object if it was successfully created. @@ -334,10 +344,15 @@ class Consumer(LoggingMixin): self.override_correspondent_id = override_correspondent_id self.override_document_type_id = override_document_type_id self.override_tag_ids = override_tag_ids + self.override_storage_path_id = override_storage_path_id self.task_id = task_id or str(uuid.uuid4()) self.override_created = override_created self.override_asn = override_asn self.override_owner_id = override_owner_id + self.override_view_users = override_view_users + self.override_view_groups = override_view_groups + self.override_change_users = override_change_users + self.override_change_groups = override_change_groups self._send_progress( 0, @@ -578,6 +593,57 @@ class Consumer(LoggingMixin): return document + def get_template_overrides( + self, + input_doc: Path, + ) -> DocumentMetadataOverrides: + overrides = DocumentMetadataOverrides() + for template in ConsumptionTemplate.objects.all(): + template_overrides = DocumentMetadataOverrides() + + if fnmatch( + input_doc.name.lower(), + template.filter_filename.lower(), + ) or input_doc.match(template.filter_path): + self.log.info(f"Document matched consumption template {template.name}") + if template.assign_tags is not None: + template_overrides.tag_ids = [ + tag.pk for tag in template.assign_tags.all() + ] + if template.assign_correspondent is not None: + template_overrides.correspondent_id = ( + template.assign_correspondent.pk + ) + if template.assign_document_type is not None: + template_overrides.document_type_id = ( + template.assign_document_type.pk + ) + if template.assign_storage_path is not None: + template_overrides.storage_path_id = template.assign_storage_path.pk + if template.assign_owner is not None: + template_overrides.owner_id = template.assign_owner + if template.assign_view_users is not None: + template_overrides.view_users = [ + user.pk for user in template.assign_view_users.all() + ] + if template.assign_view_groups is not None: + template_overrides.view_groups = [ + group.pk for group in template.assign_view_groups.all() + ] + if template.assign_change_users is not None: + template_overrides.change_users = [ + user.pk for user in template.assign_change_users.all() + ] + if template.assign_change_groups is not None: + template_overrides.change_groups = [ + group.pk for group in template.assign_change_groups.all() + ] + overrides = merge_overrides( + overridesA=overrides, + overridesB=template_overrides, + ) + return overrides + def _store( self, text: str, @@ -643,6 +709,11 @@ class Consumer(LoggingMixin): for tag_id in self.override_tag_ids: document.tags.add(Tag.objects.get(pk=tag_id)) + if self.override_storage_path_id: + document.storage_path = StoragePath.objects.get( + pk=self.override_storage_path_id, + ) + if self.override_asn: document.archive_serial_number = self.override_asn @@ -651,6 +722,24 @@ class Consumer(LoggingMixin): pk=self.override_owner_id, ) + if ( + self.override_view_users is not None + or self.override_view_groups is not None + or self.override_change_users is not None + or self.override_change_users is not None + ): + permissions = { + "view": { + "users": self.override_view_users, + "groups": self.override_view_groups, + }, + "change": { + "users": self.override_change_users, + "groups": self.override_change_groups, + }, + } + set_permissions_for_object(permissions=permissions, object=document) + def _write(self, storage_type, source, target): with open(source, "rb") as read_file, open(target, "wb") as write_file: write_file.write(read_file.read()) @@ -695,3 +784,28 @@ class Consumer(LoggingMixin): self.log.warning("Script stderr:") for line in stderr_str: self.log.warning(line) + + +def merge_overrides( + overridesA: DocumentMetadataOverrides, + overridesB: DocumentMetadataOverrides, +) -> DocumentMetadataOverrides: + if overridesA.tag_ids is None: + overridesA.tag_ids = overridesB.tag_ids + if overridesA.correspondent_id is None: + overridesA.correspondent_id = overridesB.correspondent_id + if overridesA.document_type_id is None: + overridesA.document_type_id = overridesB.document_type_id + if overridesA.storage_path_id is None: + overridesA.storage_path_id = overridesB.storage_path_id + if overridesA.owner_id is None: + overridesA.owner_id = overridesB.owner_id + if overridesA.view_users is None: + overridesA.view_users = overridesB.view_users + if overridesA.view_groups is None: + overridesA.view_groups = overridesB.view_groups + if overridesA.change_users is None: + overridesA.change_users = overridesB.change_users + if overridesA.change_groups is None: + overridesA.change_groups = overridesB.change_groups + return overridesA diff --git a/src/documents/data_models.py b/src/documents/data_models.py index d8995287f..97548200f 100644 --- a/src/documents/data_models.py +++ b/src/documents/data_models.py @@ -20,9 +20,14 @@ class DocumentMetadataOverrides: correspondent_id: Optional[int] = None document_type_id: Optional[int] = None tag_ids: Optional[list[int]] = None + storage_path_id: Optional[int] = None created: Optional[datetime.datetime] = None asn: Optional[int] = None owner_id: Optional[int] = None + view_users: Optional[list[int]] = None + view_groups: Optional[list[int]] = None + change_users: Optional[list[int]] = None + change_groups: Optional[list[int]] = None class DocumentSource(enum.IntEnum): diff --git a/src/documents/migrations/1039_consumptiontemplate.py b/src/documents/migrations/1039_consumptiontemplate.py new file mode 100644 index 000000000..12968be20 --- /dev/null +++ b/src/documents/migrations/1039_consumptiontemplate.py @@ -0,0 +1,155 @@ +# Generated by Django 4.1.11 on 2023-09-16 05:01 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("auth", "0012_alter_user_first_name_max_length"), + ("documents", "1038_sharelink"), + ] + + operations = [ + migrations.CreateModel( + name="ConsumptionTemplate", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "name", + models.CharField(max_length=256, unique=True, verbose_name="name"), + ), + ("order", models.IntegerField(default=0, verbose_name="order")), + ( + "filter_path", + models.CharField( + blank=True, + help_text="Only consume documents with a path that matches this if specified. Wildcards specified as * are allowed. Case insensitive.", + max_length=256, + null=True, + verbose_name="filter path", + ), + ), + ( + "filter_filename", + models.CharField( + blank=True, + help_text="Only consume documents which entirely match this filename if specified. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive.", + max_length=256, + null=True, + verbose_name="filter filename", + ), + ), + ( + "assign_change_groups", + models.ManyToManyField( + blank=True, + related_name="+", + to="auth.group", + verbose_name="grant change permissions to these groups", + ), + ), + ( + "assign_change_users", + models.ManyToManyField( + blank=True, + related_name="+", + to=settings.AUTH_USER_MODEL, + verbose_name="grant change permissions to these users", + ), + ), + ( + "assign_correspondent", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="documents.correspondent", + verbose_name="assign this correspondent", + ), + ), + ( + "assign_document_type", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="documents.documenttype", + verbose_name="assign this document type", + ), + ), + ( + "assign_owner", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to=settings.AUTH_USER_MODEL, + verbose_name="assign this owner", + ), + ), + ( + "assign_storage_path", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="documents.storagepath", + verbose_name="assign this storage path", + ), + ), + ( + "assign_tags", + models.ManyToManyField( + blank=True, + to="documents.tag", + verbose_name="assign this tag", + ), + ), + ( + "assign_view_groups", + models.ManyToManyField( + blank=True, + related_name="+", + to="auth.group", + verbose_name="grant view permissions to these groups", + ), + ), + ( + "assign_view_users", + models.ManyToManyField( + blank=True, + related_name="+", + to=settings.AUTH_USER_MODEL, + verbose_name="grant view permissions to these users", + ), + ), + ( + "owner", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to=settings.AUTH_USER_MODEL, + verbose_name="owner", + ), + ), + ], + options={ + "verbose_name": "consumption template", + "verbose_name_plural": "consumption templates", + }, + ), + ] diff --git a/src/documents/models.py b/src/documents/models.py index b7c188d34..a273207d8 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -11,6 +11,7 @@ import dateutil.parser import pathvalidate from celery import states from django.conf import settings +from django.contrib.auth.models import Group from django.contrib.auth.models import User from django.core.validators import MaxValueValidator from django.core.validators import MinValueValidator @@ -735,3 +736,107 @@ class ShareLink(models.Model): def __str__(self): return f"Share Link for {self.document.title}" + + +class ConsumptionTemplate(ModelWithOwner): + class Meta: + verbose_name = _("consumption template") + verbose_name_plural = _("consumption templates") + + name = models.CharField(_("name"), max_length=256, unique=True) + + order = models.IntegerField(_("order"), default=0) + + filter_path = models.CharField( + _("filter path"), + max_length=256, + null=True, + blank=True, + help_text=_( + "Only consume documents with a path that matches " + "this if specified. Wildcards specified as * are " + "allowed. Case insensitive.", + ), + ) + + filter_filename = models.CharField( + _("filter filename"), + max_length=256, + null=True, + blank=True, + help_text=_( + "Only consume documents which entirely match this " + "filename if specified. Wildcards such as *.pdf or " + "*invoice* are allowed. Case insensitive.", + ), + ) + + assign_tags = models.ManyToManyField( + Tag, + blank=True, + verbose_name=_("assign this tag"), + ) + + assign_document_type = models.ForeignKey( + DocumentType, + null=True, + blank=True, + on_delete=models.SET_NULL, + verbose_name=_("assign this document type"), + ) + + assign_correspondent = models.ForeignKey( + Correspondent, + null=True, + blank=True, + on_delete=models.SET_NULL, + verbose_name=_("assign this correspondent"), + ) + + assign_storage_path = models.ForeignKey( + StoragePath, + null=True, + blank=True, + on_delete=models.SET_NULL, + verbose_name=_("assign this storage path"), + ) + + assign_owner = models.ForeignKey( + User, + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="+", + verbose_name=_("assign this owner"), + ) + + assign_view_users = models.ManyToManyField( + User, + blank=True, + related_name="+", + verbose_name=_("grant view permissions to these users"), + ) + + assign_view_groups = models.ManyToManyField( + Group, + blank=True, + related_name="+", + verbose_name=_("grant view permissions to these groups"), + ) + + assign_change_users = models.ManyToManyField( + User, + blank=True, + related_name="+", + verbose_name=_("grant change permissions to these users"), + ) + + assign_change_groups = models.ManyToManyField( + Group, + blank=True, + related_name="+", + verbose_name=_("grant change permissions to these groups"), + ) + + def __str__(self): + return f"{self.name}" diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 0f99d5dcc..44da5e21b 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -20,6 +20,7 @@ from documents.permissions import get_groups_with_only_permission from documents.permissions import set_permissions_for_object from . import bulk_edit +from .models import ConsumptionTemplate from .models import Correspondent from .models import Document from .models import DocumentType @@ -1035,3 +1036,55 @@ class BulkEditObjectPermissionsSerializer(serializers.Serializer, SetPermissions self._validate_permissions(permissions) return attrs + + +class ConsumptionTemplateSerializer(OwnedObjectSerializer): + assign_correspondent = CorrespondentField(allow_null=True, required=False) + assign_tags = TagsField(many=True, allow_null=True, required=False) + assign_document_type = DocumentTypeField(allow_null=True, required=False) + assign_storage_path = StoragePathField(allow_null=True, required=False) + order = serializers.IntegerField(required=False) + + class Meta: + model = ConsumptionTemplate + fields = [ + "id", + "name", + "filter_path", + "filter_filename", + "assign_tags", + "assign_correspondent", + "assign_document_type", + "assign_storage_path", + "assign_owner", + "assign_view_users", + "assign_view_groups", + "assign_change_users", + "assign_change_groups", + "order", + "owner", + "user_can_change", + "permissions", + "set_permissions", + ] + + def update(self, instance, validated_data): + super().update(instance, validated_data) + return instance + + # def create(self, validated_data): + # if "assign_tags" in validated_data: + # assign_tags = validated_data.pop("assign_tags") + # mail_rule = super().create(validated_data) + # if assign_tags: + # mail_rule.assign_tags.set(assign_tags) + # return mail_rule + + # def validate(self, attrs): + # if ( + # attrs["action"] == ConsumptionTemplate.MailAction.TAG + # or attrs["action"] == ConsumptionTemplate.MailAction.MOVE + # ) and attrs["action_parameter"] is None: + # raise serializers.ValidationError("An action parameter is required.") + + # return attrs diff --git a/src/documents/tasks.py b/src/documents/tasks.py index 6ecd30b42..309017f18 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -23,6 +23,7 @@ from documents.classifier import DocumentClassifier from documents.classifier import load_classifier from documents.consumer import Consumer from documents.consumer import ConsumerError +from documents.consumer import merge_overrides from documents.data_models import ConsumableDocument from documents.data_models import DocumentMetadataOverrides from documents.double_sided import collate @@ -153,6 +154,12 @@ def consume_file( overrides.asn = reader.asn logger.info(f"Found ASN in barcode: {overrides.asn}") + template_overrides = Consumer().get_template_overrides( + input_doc=input_doc.original_file, + ) + + overrides = merge_overrides(overridesA=overrides, overridesB=template_overrides) + # continue with consumption if no barcode was found document = Consumer().try_consume_file( input_doc.original_file, @@ -161,9 +168,14 @@ def consume_file( override_correspondent_id=overrides.correspondent_id, override_document_type_id=overrides.document_type_id, override_tag_ids=overrides.tag_ids, + override_storage_path_id=overrides.storage_path_id, override_created=overrides.created, override_asn=overrides.asn, override_owner_id=overrides.owner_id, + override_view_users=overrides.view_users, + override_view_groups=overrides.view_groups, + override_change_users=overrides.change_users, + override_change_groups=overrides.change_groups, task_id=self.request.id, ) diff --git a/src/documents/tests/test_management_exporter.py b/src/documents/tests/test_management_exporter.py index d7bc1000a..b86fb2ef0 100644 --- a/src/documents/tests/test_management_exporter.py +++ b/src/documents/tests/test_management_exporter.py @@ -153,7 +153,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): manifest = self._do_export(use_filename_format=use_filename_format) - self.assertEqual(len(manifest), 154) + self.assertEqual(len(manifest), 159) # dont include consumer or AnonymousUser users self.assertEqual( @@ -247,7 +247,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): self.assertEqual(Document.objects.get(id=self.d4.id).title, "wow_dec") self.assertEqual(GroupObjectPermission.objects.count(), 1) self.assertEqual(UserObjectPermission.objects.count(), 1) - self.assertEqual(Permission.objects.count(), 112) + self.assertEqual(Permission.objects.count(), 116) messages = check_sanity() # everything is alright after the test self.assertEqual(len(messages), 0) @@ -676,15 +676,15 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): os.path.join(self.dirs.media_dir, "documents"), ) - self.assertEqual(ContentType.objects.count(), 28) - self.assertEqual(Permission.objects.count(), 112) + self.assertEqual(ContentType.objects.count(), 29) + self.assertEqual(Permission.objects.count(), 116) manifest = self._do_export() with paperless_environment(): self.assertEqual( len(list(filter(lambda e: e["model"] == "auth.permission", manifest))), - 112, + 116, ) # add 1 more to db to show objects are not re-created by import Permission.objects.create( @@ -692,7 +692,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): codename="test_perm", content_type_id=1, ) - self.assertEqual(Permission.objects.count(), 113) + self.assertEqual(Permission.objects.count(), 117) # will cause an import error self.user.delete() @@ -701,5 +701,5 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): with self.assertRaises(IntegrityError): call_command("document_importer", "--no-progress-bar", self.target) - self.assertEqual(ContentType.objects.count(), 28) - self.assertEqual(Permission.objects.count(), 113) + self.assertEqual(ContentType.objects.count(), 29) + self.assertEqual(Permission.objects.count(), 117) diff --git a/src/documents/views.py b/src/documents/views.py index be6ce1ff7..e852738d7 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -86,6 +86,7 @@ from .matching import match_correspondents from .matching import match_document_types from .matching import match_storage_paths from .matching import match_tags +from .models import ConsumptionTemplate from .models import Correspondent from .models import Document from .models import DocumentType @@ -101,6 +102,7 @@ from .serialisers import AcknowledgeTasksViewSerializer from .serialisers import BulkDownloadSerializer from .serialisers import BulkEditObjectPermissionsSerializer from .serialisers import BulkEditSerializer +from .serialisers import ConsumptionTemplateSerializer from .serialisers import CorrespondentSerializer from .serialisers import DocumentListSerializer from .serialisers import DocumentSerializer @@ -1248,3 +1250,13 @@ class BulkEditObjectPermissionsView(GenericAPIView, PassUserMixin): return HttpResponseBadRequest( "Error performing bulk permissions edit, check logs for more detail.", ) + + +class ConsumptionTemplateViewSet(ModelViewSet, PassUserMixin): + model = ConsumptionTemplate + + queryset = ConsumptionTemplate.objects.all().order_by("order") + serializer_class = ConsumptionTemplateSerializer + pagination_class = StandardPagination + permission_classes = (IsAuthenticated, PaperlessObjectPermissions) + filter_backends = (ObjectOwnedOrGrantedPermissionsFilter,) diff --git a/src/paperless/urls.py b/src/paperless/urls.py index 05e772ee0..415efc4de 100644 --- a/src/paperless/urls.py +++ b/src/paperless/urls.py @@ -14,6 +14,7 @@ from documents.views import AcknowledgeTasksView from documents.views import BulkDownloadView from documents.views import BulkEditObjectPermissionsView from documents.views import BulkEditView +from documents.views import ConsumptionTemplateViewSet from documents.views import CorrespondentViewSet from documents.views import DocumentTypeViewSet from documents.views import IndexView @@ -53,6 +54,7 @@ api_router.register(r"groups", GroupViewSet, basename="groups") api_router.register(r"mail_accounts", MailAccountViewSet) api_router.register(r"mail_rules", MailRuleViewSet) api_router.register(r"share_links", ShareLinkViewSet) +api_router.register(r"consumption_templates", ConsumptionTemplateViewSet) urlpatterns = [