From b03dc70be1a1428010b06f88f1def491aaa96195 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Sun, 22 Oct 2023 17:14:36 -0700 Subject: [PATCH] Updates the model relationships --- src/documents/index.py | 16 +- ...omfieldboolean_customfielddate_and_more.py | 196 ++++++++++++++++ .../migrations/1040_custommetadata.py | 80 ------- src/documents/models.py | 196 ++++++++++++---- src/documents/views.py | 209 +++++++----------- 5 files changed, 441 insertions(+), 256 deletions(-) create mode 100644 src/documents/migrations/1040_customfield_customfieldboolean_customfielddate_and_more.py delete mode 100644 src/documents/migrations/1040_custommetadata.py diff --git a/src/documents/index.py b/src/documents/index.py index 034f3dfb6..b9970b6c6 100644 --- a/src/documents/index.py +++ b/src/documents/index.py @@ -30,7 +30,7 @@ from whoosh.searching import ResultsPage from whoosh.searching import Searcher from whoosh.writing import AsyncWriter -from documents.models import CustomMetadata +# from documents.models import CustomMetadata from documents.models import Document from documents.models import Note from documents.models import User @@ -61,8 +61,8 @@ def get_schema(): has_path=BOOLEAN(), notes=TEXT(), num_notes=NUMERIC(sortable=True, signed=False), - custom_metadata=TEXT(), - custom_field_count=NUMERIC(sortable=True, signed=False), + # custom_metadata=TEXT(), + # custom_field_count=NUMERIC(sortable=True, signed=False), owner=TEXT(), owner_id=NUMERIC(), has_owner=BOOLEAN(), @@ -111,9 +111,9 @@ def update_document(writer: AsyncWriter, doc: Document): tags = ",".join([t.name for t in doc.tags.all()]) tags_ids = ",".join([str(t.id) for t in doc.tags.all()]) notes = ",".join([str(c.note) for c in Note.objects.filter(document=doc)]) - custom_fields = ",".join( - [str(c) for c in CustomMetadata.objects.filter(document=doc)], - ) + # custom_fields = ",".join( + # [str(c) for c in CustomMetadata.objects.filter(document=doc)], + # ) asn = doc.archive_serial_number if asn is not None and ( asn < Document.ARCHIVE_SERIAL_NUMBER_MIN @@ -153,8 +153,8 @@ def update_document(writer: AsyncWriter, doc: Document): has_path=doc.storage_path is not None, notes=notes, num_notes=len(notes), - custom_metadata=custom_fields, - custom_field_count=len(custom_fields), + # custom_metadata=custom_fields, + # custom_field_count=len(custom_fields), owner=doc.owner.username if doc.owner else None, owner_id=doc.owner.id if doc.owner else None, has_owner=doc.owner is not None, diff --git a/src/documents/migrations/1040_customfield_customfieldboolean_customfielddate_and_more.py b/src/documents/migrations/1040_customfield_customfieldboolean_customfielddate_and_more.py new file mode 100644 index 000000000..406e74733 --- /dev/null +++ b/src/documents/migrations/1040_customfield_customfieldboolean_customfielddate_and_more.py @@ -0,0 +1,196 @@ +# Generated by Django 4.2.5 on 2023-10-22 23:47 + +import django.db.models.deletion +import django.utils.timezone +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + dependencies = [ + ("documents", "1039_consumptiontemplate"), + ] + + operations = [ + migrations.CreateModel( + name="CustomField", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + models.DateTimeField( + db_index=True, + default=django.utils.timezone.now, + verbose_name="created", + ), + ), + ("name", models.CharField(max_length=128)), + ( + "data_type", + models.CharField( + choices=[ + ("string", "String"), + ("url", "URL"), + ("date", "Date"), + ("boolean", "Boolean"), + ], + default="string", + max_length=50, + verbose_name="data type", + ), + ), + ], + options={ + "verbose_name": "custom field", + "verbose_name_plural": "custom fields", + "ordering": ("created",), + }, + ), + migrations.CreateModel( + name="CustomFieldBoolean", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("value", models.BooleanField()), + ( + "parent", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + related_name="boolean", + to="documents.customfieldinstance", + ), + ), + ], + ), + migrations.CreateModel( + name="CustomFieldDate", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("value", models.DateField()), + ( + "parent", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + related_name="date", + to="documents.customfieldinstance", + ), + ), + ], + ), + migrations.CreateModel( + name="CustomFieldInstance", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + models.DateTimeField( + db_index=True, + default=django.utils.timezone.now, + verbose_name="created", + ), + ), + ( + "document", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="custom_fields", + to="documents.document", + ), + ), + ( + "field", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="fields", + to="documents.customfield", + ), + ), + ], + options={ + "verbose_name": "custom field instance", + "verbose_name_plural": "custom field instances", + "ordering": ("created",), + }, + ), + migrations.CreateModel( + name="CustomFieldShortText", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("value", models.CharField(max_length=128)), + ( + "parent", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + related_name="short_text", + to="documents.customfieldinstance", + ), + ), + ], + ), + migrations.CreateModel( + name="CustomFieldUrl", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("value", models.URLField()), + ( + "parent", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + related_name="url", + to="documents.customfieldinstance", + ), + ), + ], + ), + ] diff --git a/src/documents/migrations/1040_custommetadata.py b/src/documents/migrations/1040_custommetadata.py deleted file mode 100644 index 2a1800ffd..000000000 --- a/src/documents/migrations/1040_custommetadata.py +++ /dev/null @@ -1,80 +0,0 @@ -# Generated by Django 4.2.5 on 2023-10-20 15:48 - -import django.db.models.deletion -import django.utils.timezone -from django.conf import settings -from django.db import migrations -from django.db import models - - -class Migration(migrations.Migration): - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ("documents", "1039_consumptiontemplate"), - ] - - operations = [ - migrations.CreateModel( - name="CustomMetadata", - fields=[ - ( - "id", - models.AutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "data_type", - models.CharField( - choices=[ - ("string", "String"), - ("url", "URL"), - ("date", "Date"), - ], - default="string", - max_length=50, - ), - ), - ("data", models.CharField(blank=True, max_length=512)), - ("name", models.CharField(blank=True, max_length=512)), - ( - "created", - models.DateTimeField( - db_index=True, - default=django.utils.timezone.now, - verbose_name="created", - ), - ), - ( - "document", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - related_name="metadata", - to="documents.document", - verbose_name="document", - ), - ), - ( - "user", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name="metadata", - to=settings.AUTH_USER_MODEL, - verbose_name="user", - ), - ), - ], - options={ - "verbose_name": "custom metadata", - "verbose_name_plural": "custom metadata", - "ordering": ("created",), - }, - ), - ] diff --git a/src/documents/models.py b/src/documents/models.py index f90cb421e..7e298d7d5 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -885,27 +885,46 @@ if settings.AUDIT_LOG_ENABLED: auditlog.register(Note) -class CustomMetadata(models.Model): - class DataType(models.TextChoices): +class CustomField(models.Model): + """ + Defines the name and type of a custom field + """ + + class FieldDataType(models.TextChoices): STRING = ("string", _("String")) URL = ("url", _("URL")) DATE = ("date", _("Date")) + BOOL = ("boolean"), _("Boolean") + + created = models.DateTimeField( + _("created"), + default=timezone.now, + db_index=True, + ) + + name = models.CharField(max_length=128) data_type = models.CharField( + _("data type"), max_length=50, - choices=DataType.choices, - default=DataType.STRING, + choices=FieldDataType.choices, + default=FieldDataType.STRING, ) - data = models.CharField( - max_length=512, - blank=True, - ) + class Meta: + ordering = ("created",) + verbose_name = _("custom field") + verbose_name_plural = _("custom fields") - name = models.CharField( - max_length=512, - blank=True, - ) + def __str__(self) -> str: + return f"{self.name} : {self.data_type}" + + +class CustomFieldInstance(models.Model): + """ + A single instance of a field, attached to a CustomField for the name and type + and attached to a single Document to be metadata for it + """ created = models.DateTimeField( _("created"), @@ -915,51 +934,146 @@ class CustomMetadata(models.Model): document = models.ForeignKey( Document, - blank=True, - null=True, - related_name="metadata", + blank=False, + null=False, on_delete=models.CASCADE, - verbose_name=_("document"), + related_name="custom_fields", ) - user = models.ForeignKey( - User, - blank=True, - null=True, - related_name="metadata", - on_delete=models.SET_NULL, - verbose_name=_("user"), + field = models.ForeignKey( + CustomField, + blank=False, + null=False, + on_delete=models.CASCADE, + related_name="fields", ) class Meta: ordering = ("created",) - verbose_name = _("custom metadata") - verbose_name_plural = _("custom metadata") + verbose_name = _("custom field instance") + verbose_name_plural = _("custom field instances") - def __str__(self): - return f"{self.data_type} : {self.name} : {self.data}" + def __str__(self) -> str: + return str(self.field) + f" : {self.value}" + + @property + def value(self): + """ + Based on the data type, access the actual value the instance stores + """ + if self.field.data_type == CustomField.FieldDataType.STRING: + return self.short_text.value + elif self.field.data_type == CustomField.FieldDataType.URL: + return self.url.value + elif self.field.data_type == CustomField.FieldDataType.DATE: + return self.date.value + elif self.field.data_type == CustomField.FieldDataType.BOOL: + return self.boolean.value + raise NotImplementedError(self.field.data_type) + + @property + def field_type(self): + """ + Based on the data type, quick access to class for storing that value + """ + if self.field.data_type == CustomField.FieldDataType.STRING: + return CustomFieldShortText + elif self.field.data_type == CustomField.FieldDataType.URL: + return CustomFieldUrl + elif self.field.data_type == CustomField.FieldDataType.DATE: + return CustomFieldDate + elif self.field.data_type == CustomField.FieldDataType.BOOL: + return CustomFieldBoolean + raise NotImplementedError(self.field.data_type) def to_json(self) -> dict[str, str]: return { "id": self.id, "created": self.created, - "type": self.data_type, - "name": self.name, - "data": self.data, - "user": { - "id": self.user.id, - "username": self.user.username, - "first_name": self.user.first_name, - "last_name": self.user.last_name, - }, + "type": self.field.data_type, + "name": self.field.name, + "data": self.value, } @staticmethod - def from_json(document: Document, user: User, data) -> "CustomMetadata": - return CustomMetadata.objects.create( + def from_json( + document: Document, + field: CustomField, + data, + ) -> "CustomFieldInstance": + instance = CustomFieldInstance.objects.create( document=document, + field=field, data_type=data["type"], - name=data["name"], - data=data["data"], - user=user, ) + instance.field_type.objects.create(value=data["value"], parent=instance) + + return field + + +class CustomFieldShortText(models.Model): + """ + Data storage for a short text custom field + """ + + value = models.CharField(max_length=128) + parent = models.OneToOneField( + CustomFieldInstance, + on_delete=models.CASCADE, + related_name="short_text", + parent_link=True, + ) + + def __str__(self) -> str: + return f"{self.value}" + + +class CustomFieldBoolean(models.Model): + """ + Data storage for a boolean custom field + """ + + value = models.BooleanField() + parent = models.OneToOneField( + CustomFieldInstance, + on_delete=models.CASCADE, + related_name="boolean", + parent_link=True, + ) + + def __str__(self) -> str: + return f"{self.value}" + + +class CustomFieldUrl(models.Model): + """ + Data storage for a URL custom field + """ + + value = models.URLField() + parent = models.OneToOneField( + CustomFieldInstance, + on_delete=models.CASCADE, + related_name="url", + parent_link=True, + ) + + def __str__(self) -> str: + return f"{self.value}" + + +class CustomFieldDate(models.Model): + """ + Data storage for a date custom field + """ + + value = models.DateField() + parent = models.OneToOneField( + CustomFieldInstance, + on_delete=models.CASCADE, + related_name="date", + parent_link=True, + ) + + def __str__(self) -> str: + return f"{self.value}" diff --git a/src/documents/views.py b/src/documents/views.py index 7949220b8..4538606eb 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -1,10 +1,8 @@ import itertools -import json import logging import os import re import tempfile -import urllib import zipfile from datetime import datetime from pathlib import Path @@ -28,7 +26,6 @@ from django.http import HttpResponse from django.http import HttpResponseBadRequest from django.http import HttpResponseForbidden from django.http import HttpResponseRedirect -from django.http import HttpResponseServerError from django.shortcuts import get_object_or_404 from django.utils import timezone from django.utils.decorators import method_decorator @@ -38,7 +35,6 @@ from django.views.decorators.cache import cache_control from django.views.generic import TemplateView from django_filters.rest_framework import DjangoFilterBackend from langdetect import detect -from packaging import version as packaging_version from rest_framework import parsers from rest_framework.decorators import action from rest_framework.exceptions import NotFound @@ -79,7 +75,8 @@ from documents.matching import match_storage_paths from documents.matching import match_tags from documents.models import ConsumptionTemplate from documents.models import Correspondent -from documents.models import CustomMetadata + +# from documents.models import CustomMetadata from documents.models import Document from documents.models import DocumentType from documents.models import Note @@ -113,7 +110,6 @@ from documents.serialisers import TagSerializerVersion1 from documents.serialisers import TasksViewSerializer from documents.serialisers import UiSettingsViewSerializer from documents.tasks import consume_file -from paperless import version from paperless.db import GnuPG from paperless.views import StandardPagination @@ -624,99 +620,99 @@ class DocumentViewSet( ] return Response(links) - @action(methods=["get", "post", "delete"], detail=True) - def custom_metadata(self, request, pk=None) -> Response: - def package_custom_metadata(doc: Document): - return [ - c.to_json() - for c in CustomMetadata.objects.filter(document=doc).order_by( - "-created", - ) - ] + # @action(methods=["get", "post", "delete"], detail=True) + # def custom_metadata(self, request, pk=None) -> Response: + # def package_custom_metadata(doc: Document): + # return [ + # c.to_json() + # for c in CustomMetadata.objects.filter(document=doc).order_by( + # "-created", + # ) + # ] - request.user = request.user - try: - doc = Document.objects.get(pk=pk) - if request.user is not None and not has_perms_owner_aware( - request.user, - "view_document", - doc, - ): - return HttpResponseForbidden( - "Insufficient permissions to view custom metadata", - ) - except Document.DoesNotExist: - raise Http404 + # request.user = request.user + # try: + # doc = Document.objects.get(pk=pk) + # if request.user is not None and not has_perms_owner_aware( + # request.user, + # "view_document", + # doc, + # ): + # return HttpResponseForbidden( + # "Insufficient permissions to view custom metadata", + # ) + # except Document.DoesNotExist: + # raise Http404 - if request.method == "GET": - try: - return Response(package_custom_metadata(doc)) - except Exception as e: - logger.warning(f"An error occurred retrieving custom metadata: {e!s}") - return HttpResponseServerError( - { - "error": ( - "Error retrieving custom metadata," - " check logs for more detail." - ), - }, - ) - elif request.method == "POST": - try: - if request.user is not None and not has_perms_owner_aware( - request.user, - "change_document", - doc, - ): - return HttpResponseForbidden( - "Insufficient permissions to create custom metadata", - ) + # if request.method == "GET": + # try: + # return Response(package_custom_metadata(doc)) + # except Exception as e: + # logger.warning(f"An error occurred retrieving custom metadata: {e!s}") + # return HttpResponseServerError( + # { + # "error": ( + # "Error retrieving custom metadata," + # " check logs for more detail." + # ), + # }, + # ) + # elif request.method == "POST": + # try: + # if request.user is not None and not has_perms_owner_aware( + # request.user, + # "change_document", + # doc, + # ): + # return HttpResponseForbidden( + # "Insufficient permissions to create custom metadata", + # ) - CustomMetadata.from_json(doc, request.user, request.data) + # CustomMetadata.from_json(doc, request.user, request.data) - doc.modified = timezone.now() - doc.save() + # doc.modified = timezone.now() + # doc.save() - from documents import index + # from documents import index - index.add_or_update_document(self.get_object()) + # index.add_or_update_document(self.get_object()) - return Response(package_custom_metadata(doc)) - except Exception as e: - logger.warning(f"An error occurred saving custom metadata: {e!s}") - return HttpResponseServerError( - { - "error": ( - "Error saving custom metadata, " - "check logs for more detail." - ), - }, - ) - elif request.method == "DELETE": - if request.user is not None and not has_perms_owner_aware( - request.user, - "change_document", - doc, - ): - return HttpResponseForbidden( - "Insufficient permissions to delete custom metadata", - ) + # return Response(package_custom_metadata(doc)) + # except Exception as e: + # logger.warning(f"An error occurred saving custom metadata: {e!s}") + # return HttpResponseServerError( + # { + # "error": ( + # "Error saving custom metadata, " + # "check logs for more detail." + # ), + # }, + # ) + # elif request.method == "DELETE": + # if request.user is not None and not has_perms_owner_aware( + # request.user, + # "change_document", + # doc, + # ): + # return HttpResponseForbidden( + # "Insufficient permissions to delete custom metadata", + # ) - metadata = CustomMetadata.objects.get(id=int(request.GET.get("id"))) - metadata.delete() + # metadata = CustomMetadata.objects.get(id=int(request.GET.get("id"))) + # metadata.delete() - doc.modified = timezone.now() - doc.save() + # doc.modified = timezone.now() + # doc.save() - from documents import index + # from documents import index - index.add_or_update_document(self.get_object()) + # index.add_or_update_document(self.get_object()) - return Response(package_custom_metadata(doc)) + # return Response(package_custom_metadata(doc)) - return Response( - {"error": "unreachable error was reached for custom metadata"}, - ) # pragma: no cover + # return Response( + # {"error": "unreachable error was reached for custom metadata"}, + # ) # pragma: no cover class SearchResultSerializer(DocumentSerializer, PassUserMixin): @@ -1171,47 +1167,6 @@ class BulkDownloadView(GenericAPIView): return response -class RemoteVersionView(GenericAPIView): - def get(self, request, format=None): - remote_version = "0.0.0" - is_greater_than_current = False - current_version = packaging_version.parse(version.__full_version_str__) - try: - req = urllib.request.Request( - "https://api.github.com/repos/paperless-ngx/" - "paperless-ngx/releases/latest", - ) - # Ensure a JSON response - req.add_header("Accept", "application/json") - - with urllib.request.urlopen(req) as response: - remote = response.read().decode("utf-8") - try: - remote_json = json.loads(remote) - remote_version = remote_json["tag_name"] - # Basically PEP 616 but that only went in 3.9 - if remote_version.startswith("ngx-"): - remote_version = remote_version[len("ngx-") :] - except ValueError: - logger.debug("An error occurred parsing remote version json") - except urllib.error.URLError: - logger.debug("An error occurred checking for available updates") - - is_greater_than_current = ( - packaging_version.parse( - remote_version, - ) - > current_version - ) - - return Response( - { - "version": remote_version, - "update_available": is_greater_than_current, - }, - ) - - class StoragePathViewSet(ModelViewSet, PassUserMixin): model = StoragePath