lgcosta/paperless-ngx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial implementation of consumption templates

Browse Source
This commit is contained in:
shamoon 2023-09-15 23:20:28 -07:00
parent 9d72d1fc81
commit 483fa245b0
9 changed files with 466 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
src/documents/consumer.py
View File

@ -4,6 +4,7 @@ import os
import tempfile
import uuid
from enum import Enum
from fnmatch import fnmatch
from pathlib import Path
from subprocess import CompletedProcess
from subprocess import run
@ -20,6 +21,8 @@ from django.utils import timezone
from filelock import FileLock
from rest_framework.reverse import reverse
from documents.data_models import DocumentMetadataOverrides
from documents.permissions import set_permissions_for_object
from documents.utils import copy_basic_file_stats
from documents.utils import copy_file_with_basic_stats
@ -27,10 +30,12 @@ from .classifier import load_classifier
from .file_handling import create_source_path_directory
from .file_handling import generate_unique_filename
from .loggers import LoggingMixin
from .models import ConsumptionTemplate
from .models import Correspondent
from .models import Document
from .models import DocumentType
from .models import FileInfo
from .models import StoragePath
from .models import Tag
from .parsers import DocumentParser
from .parsers import ParseError
@ -319,10 +324,15 @@ class Consumer(LoggingMixin):
override_correspondent_id=None,
override_document_type_id=None,
override_tag_ids=None,
override_storage_path_id=None,
task_id=None,
override_created=None,
override_asn=None,
override_owner_id=None,
override_view_users=None,
override_view_groups=None,
override_change_users=None,
override_change_groups=None,
) -> Document:
"""
Return the document object if it was successfully created.
@ -334,10 +344,15 @@ class Consumer(LoggingMixin):
self.override_correspondent_id = override_correspondent_id
self.override_document_type_id = override_document_type_id
self.override_tag_ids = override_tag_ids
self.override_storage_path_id = override_storage_path_id
self.task_id = task_id or str(uuid.uuid4())
self.override_created = override_created
self.override_asn = override_asn
self.override_owner_id = override_owner_id
self.override_view_users = override_view_users
self.override_view_groups = override_view_groups
self.override_change_users = override_change_users
self.override_change_groups = override_change_groups
self._send_progress(
0,
@ -578,6 +593,57 @@ class Consumer(LoggingMixin):
return document
def get_template_overrides(
self,
input_doc: Path,
) -> DocumentMetadataOverrides:
overrides = DocumentMetadataOverrides()
for template in ConsumptionTemplate.objects.all():
template_overrides = DocumentMetadataOverrides()
if fnmatch(
input_doc.name.lower(),
template.filter_filename.lower(),
) or input_doc.match(template.filter_path):
self.log.info(f"Document matched consumption template {template.name}")
if template.assign_tags is not None:
template_overrides.tag_ids = [
tag.pk for tag in template.assign_tags.all()
]
if template.assign_correspondent is not None:
template_overrides.correspondent_id = (
template.assign_correspondent.pk
)
if template.assign_document_type is not None:
template_overrides.document_type_id = (
template.assign_document_type.pk
)
if template.assign_storage_path is not None:
template_overrides.storage_path_id = template.assign_storage_path.pk
if template.assign_owner is not None:
template_overrides.owner_id = template.assign_owner
if template.assign_view_users is not None:
template_overrides.view_users = [
user.pk for user in template.assign_view_users.all()
]
if template.assign_view_groups is not None:
template_overrides.view_groups = [
group.pk for group in template.assign_view_groups.all()
]
if template.assign_change_users is not None:
template_overrides.change_users = [
user.pk for user in template.assign_change_users.all()
]
if template.assign_change_groups is not None:
template_overrides.change_groups = [
group.pk for group in template.assign_change_groups.all()
]
overrides = merge_overrides(
overridesA=overrides,
overridesB=template_overrides,
)
return overrides
def _store(
self,
text: str,
@ -643,6 +709,11 @@ class Consumer(LoggingMixin):
for tag_id in self.override_tag_ids:
document.tags.add(Tag.objects.get(pk=tag_id))
if self.override_storage_path_id:
document.storage_path = StoragePath.objects.get(
pk=self.override_storage_path_id,
)
if self.override_asn:
document.archive_serial_number = self.override_asn
@ -651,6 +722,24 @@ class Consumer(LoggingMixin):
pk=self.override_owner_id,
)
if (
self.override_view_users is not None
or self.override_view_groups is not None
or self.override_change_users is not None
or self.override_change_users is not None
):
permissions = {
"view": {
"users": self.override_view_users,
"groups": self.override_view_groups,
},
"change": {
"users": self.override_change_users,
"groups": self.override_change_groups,
},
}
set_permissions_for_object(permissions=permissions, object=document)
def _write(self, storage_type, source, target):
with open(source, "rb") as read_file, open(target, "wb") as write_file:
write_file.write(read_file.read())
@ -695,3 +784,28 @@ class Consumer(LoggingMixin):
self.log.warning("Script stderr:")
for line in stderr_str:
self.log.warning(line)
def merge_overrides(
overridesA: DocumentMetadataOverrides,
overridesB: DocumentMetadataOverrides,
) -> DocumentMetadataOverrides:
if overridesA.tag_ids is None:
overridesA.tag_ids = overridesB.tag_ids
if overridesA.correspondent_id is None:
overridesA.correspondent_id = overridesB.correspondent_id
if overridesA.document_type_id is None:
overridesA.document_type_id = overridesB.document_type_id
if overridesA.storage_path_id is None:
overridesA.storage_path_id = overridesB.storage_path_id
if overridesA.owner_id is None:
overridesA.owner_id = overridesB.owner_id
if overridesA.view_users is None:
overridesA.view_users = overridesB.view_users
if overridesA.view_groups is None:
overridesA.view_groups = overridesB.view_groups
if overridesA.change_users is None:
overridesA.change_users = overridesB.change_users
if overridesA.change_groups is None:
overridesA.change_groups = overridesB.change_groups
return overridesA

5
src/documents/data_models.py
View File

@ -20,9 +20,14 @@ class DocumentMetadataOverrides:
correspondent_id: Optional[int] = None
document_type_id: Optional[int] = None
tag_ids: Optional[list[int]] = None
storage_path_id: Optional[int] = None
created: Optional[datetime.datetime] = None
asn: Optional[int] = None
owner_id: Optional[int] = None
view_users: Optional[list[int]] = None
view_groups: Optional[list[int]] = None
change_users: Optional[list[int]] = None
change_groups: Optional[list[int]] = None
class DocumentSource(enum.IntEnum):

155
src/documents/migrations/1039_consumptiontemplate.py Normal file
View File

@ -0,0 +1,155 @@
# Generated by Django 4.1.11 on 2023-09-16 05:01
import django.db.models.deletion
from django.conf import settings
from django.db import migrations
from django.db import models
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
("auth", "0012_alter_user_first_name_max_length"),
("documents", "1038_sharelink"),
]
operations = [
migrations.CreateModel(
name="ConsumptionTemplate",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"name",
models.CharField(max_length=256, unique=True, verbose_name="name"),
),
("order", models.IntegerField(default=0, verbose_name="order")),
(
"filter_path",
models.CharField(
blank=True,
help_text="Only consume documents with a path that matches this if specified. Wildcards specified as * are allowed. Case insensitive.",
max_length=256,
null=True,
verbose_name="filter path",
),
),
(
"filter_filename",
models.CharField(
blank=True,
help_text="Only consume documents which entirely match this filename if specified. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive.",
max_length=256,
null=True,
verbose_name="filter filename",
),
),
(
"assign_change_groups",
models.ManyToManyField(
blank=True,
related_name="+",
to="auth.group",
verbose_name="grant change permissions to these groups",
),
),
(
"assign_change_users",
models.ManyToManyField(
blank=True,
related_name="+",
to=settings.AUTH_USER_MODEL,
verbose_name="grant change permissions to these users",
),
),
(
"assign_correspondent",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
to="documents.correspondent",
verbose_name="assign this correspondent",
),
),
(
"assign_document_type",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
to="documents.documenttype",
verbose_name="assign this document type",
),
),
(
"assign_owner",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="+",
to=settings.AUTH_USER_MODEL,
verbose_name="assign this owner",
),
),
(
"assign_storage_path",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
to="documents.storagepath",
verbose_name="assign this storage path",
),
),
(
"assign_tags",
models.ManyToManyField(
blank=True,
to="documents.tag",
verbose_name="assign this tag",
),
),
(
"assign_view_groups",
models.ManyToManyField(
blank=True,
related_name="+",
to="auth.group",
verbose_name="grant view permissions to these groups",
),
),
(
"assign_view_users",
models.ManyToManyField(
blank=True,
related_name="+",
to=settings.AUTH_USER_MODEL,
verbose_name="grant view permissions to these users",
),
),
(
"owner",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
to=settings.AUTH_USER_MODEL,
verbose_name="owner",
),
),
],
options={
"verbose_name": "consumption template",
"verbose_name_plural": "consumption templates",
},
),
]

105
src/documents/models.py
View File

@ -11,6 +11,7 @@ import dateutil.parser
import pathvalidate
from celery import states
from django.conf import settings
from django.contrib.auth.models import Group
from django.contrib.auth.models import User
from django.core.validators import MaxValueValidator
from django.core.validators import MinValueValidator
@ -735,3 +736,107 @@ class ShareLink(models.Model):
def __str__(self):
return f"Share Link for {self.document.title}"
class ConsumptionTemplate(ModelWithOwner):
class Meta:
verbose_name = _("consumption template")
verbose_name_plural = _("consumption templates")
name = models.CharField(_("name"), max_length=256, unique=True)
order = models.IntegerField(_("order"), default=0)
filter_path = models.CharField(
_("filter path"),
max_length=256,
null=True,
blank=True,
help_text=_(
"Only consume documents with a path that matches "
"this if specified. Wildcards specified as * are "
"allowed. Case insensitive.",
),
)
filter_filename = models.CharField(
_("filter filename"),
max_length=256,
null=True,
blank=True,
help_text=_(
"Only consume documents which entirely match this "
"filename if specified. Wildcards such as *.pdf or "
"*invoice* are allowed. Case insensitive.",
),
)
assign_tags = models.ManyToManyField(
Tag,
blank=True,
verbose_name=_("assign this tag"),
)
assign_document_type = models.ForeignKey(
DocumentType,
null=True,
blank=True,
on_delete=models.SET_NULL,
verbose_name=_("assign this document type"),
)
assign_correspondent = models.ForeignKey(
Correspondent,
null=True,
blank=True,
on_delete=models.SET_NULL,
verbose_name=_("assign this correspondent"),
)
assign_storage_path = models.ForeignKey(
StoragePath,
null=True,
blank=True,
on_delete=models.SET_NULL,
verbose_name=_("assign this storage path"),
)
assign_owner = models.ForeignKey(
User,
null=True,
blank=True,
on_delete=models.SET_NULL,
related_name="+",
verbose_name=_("assign this owner"),
)
assign_view_users = models.ManyToManyField(
User,
blank=True,
related_name="+",
verbose_name=_("grant view permissions to these users"),
)
assign_view_groups = models.ManyToManyField(
Group,
blank=True,
related_name="+",
verbose_name=_("grant view permissions to these groups"),
)
assign_change_users = models.ManyToManyField(
User,
blank=True,
related_name="+",
verbose_name=_("grant change permissions to these users"),
)
assign_change_groups = models.ManyToManyField(
Group,
blank=True,
related_name="+",
verbose_name=_("grant change permissions to these groups"),
)
def __str__(self):
return f"{self.name}"

53
src/documents/serialisers.py
View File

@ -20,6 +20,7 @@ from documents.permissions import get_groups_with_only_permission
from documents.permissions import set_permissions_for_object
from . import bulk_edit
from .models import ConsumptionTemplate
from .models import Correspondent
from .models import Document
from .models import DocumentType
@ -1035,3 +1036,55 @@ class BulkEditObjectPermissionsSerializer(serializers.Serializer, SetPermissions
self._validate_permissions(permissions)
return attrs
class ConsumptionTemplateSerializer(OwnedObjectSerializer):
assign_correspondent = CorrespondentField(allow_null=True, required=False)
assign_tags = TagsField(many=True, allow_null=True, required=False)
assign_document_type = DocumentTypeField(allow_null=True, required=False)
assign_storage_path = StoragePathField(allow_null=True, required=False)
order = serializers.IntegerField(required=False)
class Meta:
model = ConsumptionTemplate
fields = [
"id",
"name",
"filter_path",
"filter_filename",
"assign_tags",
"assign_correspondent",
"assign_document_type",
"assign_storage_path",
"assign_owner",
"assign_view_users",
"assign_view_groups",
"assign_change_users",
"assign_change_groups",
"order",
"owner",
"user_can_change",
"permissions",
"set_permissions",
]
def update(self, instance, validated_data):
super().update(instance, validated_data)
return instance
# def create(self, validated_data):
# if "assign_tags" in validated_data:
# assign_tags = validated_data.pop("assign_tags")
# mail_rule = super().create(validated_data)
# if assign_tags:
# mail_rule.assign_tags.set(assign_tags)
# return mail_rule
# def validate(self, attrs):
# if (
# attrs["action"] == ConsumptionTemplate.MailAction.TAG
# or attrs["action"] == ConsumptionTemplate.MailAction.MOVE
# ) and attrs["action_parameter"] is None:
# raise serializers.ValidationError("An action parameter is required.")
# return attrs

12
src/documents/tasks.py
View File

@ -23,6 +23,7 @@ from documents.classifier import DocumentClassifier
from documents.classifier import load_classifier
from documents.consumer import Consumer
from documents.consumer import ConsumerError
from documents.consumer import merge_overrides
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides
from documents.double_sided import collate
@ -153,6 +154,12 @@ def consume_file(
overrides.asn = reader.asn
logger.info(f"Found ASN in barcode: {overrides.asn}")
template_overrides = Consumer().get_template_overrides(
input_doc=input_doc.original_file,
)
overrides = merge_overrides(overridesA=overrides, overridesB=template_overrides)
# continue with consumption if no barcode was found
document = Consumer().try_consume_file(
input_doc.original_file,
@ -161,9 +168,14 @@ def consume_file(
override_correspondent_id=overrides.correspondent_id,
override_document_type_id=overrides.document_type_id,
override_tag_ids=overrides.tag_ids,
override_storage_path_id=overrides.storage_path_id,
override_created=overrides.created,
override_asn=overrides.asn,
override_owner_id=overrides.owner_id,
override_view_users=overrides.view_users,
override_view_groups=overrides.view_groups,
override_change_users=overrides.change_users,
override_change_groups=overrides.change_groups,
task_id=self.request.id,
)

16
src/documents/tests/test_management_exporter.py
View File

@ -153,7 +153,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
manifest = self._do_export(use_filename_format=use_filename_format)
self.assertEqual(len(manifest), 154)
self.assertEqual(len(manifest), 159)
# dont include consumer or AnonymousUser users
self.assertEqual(
@ -247,7 +247,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
self.assertEqual(Document.objects.get(id=self.d4.id).title, "wow_dec")
self.assertEqual(GroupObjectPermission.objects.count(), 1)
self.assertEqual(UserObjectPermission.objects.count(), 1)
self.assertEqual(Permission.objects.count(), 112)
self.assertEqual(Permission.objects.count(), 116)
messages = check_sanity()
# everything is alright after the test
self.assertEqual(len(messages), 0)
@ -676,15 +676,15 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
os.path.join(self.dirs.media_dir, "documents"),
)
self.assertEqual(ContentType.objects.count(), 28)
self.assertEqual(Permission.objects.count(), 112)
self.assertEqual(ContentType.objects.count(), 29)
self.assertEqual(Permission.objects.count(), 116)
manifest = self._do_export()
with paperless_environment():
self.assertEqual(
len(list(filter(lambda e: e["model"] == "auth.permission", manifest))),
112,
116,
)
# add 1 more to db to show objects are not re-created by import
Permission.objects.create(
@ -692,7 +692,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
codename="test_perm",
content_type_id=1,
)
self.assertEqual(Permission.objects.count(), 113)
self.assertEqual(Permission.objects.count(), 117)
# will cause an import error
self.user.delete()
@ -701,5 +701,5 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
with self.assertRaises(IntegrityError):
call_command("document_importer", "--no-progress-bar", self.target)
self.assertEqual(ContentType.objects.count(), 28)
self.assertEqual(Permission.objects.count(), 113)
self.assertEqual(ContentType.objects.count(), 29)
self.assertEqual(Permission.objects.count(), 117)

12
src/documents/views.py
View File

@ -86,6 +86,7 @@ from .matching import match_correspondents
from .matching import match_document_types
from .matching import match_storage_paths
from .matching import match_tags
from .models import ConsumptionTemplate
from .models import Correspondent
from .models import Document
from .models import DocumentType
@ -101,6 +102,7 @@ from .serialisers import AcknowledgeTasksViewSerializer
from .serialisers import BulkDownloadSerializer
from .serialisers import BulkEditObjectPermissionsSerializer
from .serialisers import BulkEditSerializer
from .serialisers import ConsumptionTemplateSerializer
from .serialisers import CorrespondentSerializer
from .serialisers import DocumentListSerializer
from .serialisers import DocumentSerializer
@ -1248,3 +1250,13 @@ class BulkEditObjectPermissionsView(GenericAPIView, PassUserMixin):
return HttpResponseBadRequest(
"Error performing bulk permissions edit, check logs for more detail.",
)
class ConsumptionTemplateViewSet(ModelViewSet, PassUserMixin):
model = ConsumptionTemplate
queryset = ConsumptionTemplate.objects.all().order_by("order")
serializer_class = ConsumptionTemplateSerializer
pagination_class = StandardPagination
permission_classes = (IsAuthenticated, PaperlessObjectPermissions)
filter_backends = (ObjectOwnedOrGrantedPermissionsFilter,)

2
src/paperless/urls.py
View File

@ -14,6 +14,7 @@ from documents.views import AcknowledgeTasksView
from documents.views import BulkDownloadView
from documents.views import BulkEditObjectPermissionsView
from documents.views import BulkEditView
from documents.views import ConsumptionTemplateViewSet
from documents.views import CorrespondentViewSet
from documents.views import DocumentTypeViewSet
from documents.views import IndexView
@ -53,6 +54,7 @@ api_router.register(r"groups", GroupViewSet, basename="groups")
api_router.register(r"mail_accounts", MailAccountViewSet)
api_router.register(r"mail_rules", MailRuleViewSet)
api_router.register(r"share_links", ShareLinkViewSet)
api_router.register(r"consumption_templates", ConsumptionTemplateViewSet)
urlpatterns = [