Updates the model relationships

src/documents/index.py

@@ -30,7 +30,7 @@ from whoosh.searching import ResultsPage
 from whoosh.searching import Searcher
 from whoosh.writing import AsyncWriter
 
-from documents.models import CustomMetadata
+# from documents.models import CustomMetadata
 from documents.models import Document
 from documents.models import Note
 from documents.models import User
@@ -61,8 +61,8 @@ def get_schema():
         has_path=BOOLEAN(),
         notes=TEXT(),
         num_notes=NUMERIC(sortable=True, signed=False),
-        custom_metadata=TEXT(),
+        # custom_metadata=TEXT(),
-        custom_field_count=NUMERIC(sortable=True, signed=False),
+        # custom_field_count=NUMERIC(sortable=True, signed=False),
         owner=TEXT(),
         owner_id=NUMERIC(),
         has_owner=BOOLEAN(),
@@ -111,9 +111,9 @@ def update_document(writer: AsyncWriter, doc: Document):
     tags = ",".join([t.name for t in doc.tags.all()])
     tags_ids = ",".join([str(t.id) for t in doc.tags.all()])
     notes = ",".join([str(c.note) for c in Note.objects.filter(document=doc)])
-    custom_fields = ",".join(
+    # custom_fields = ",".join(
-        [str(c) for c in CustomMetadata.objects.filter(document=doc)],
+    #    [str(c) for c in CustomMetadata.objects.filter(document=doc)],
-    )
+    # )
     asn = doc.archive_serial_number
     if asn is not None and (
         asn < Document.ARCHIVE_SERIAL_NUMBER_MIN
@@ -153,8 +153,8 @@ def update_document(writer: AsyncWriter, doc: Document):
         has_path=doc.storage_path is not None,
         notes=notes,
         num_notes=len(notes),
-        custom_metadata=custom_fields,
+        # custom_metadata=custom_fields,
-        custom_field_count=len(custom_fields),
+        # custom_field_count=len(custom_fields),
         owner=doc.owner.username if doc.owner else None,
         owner_id=doc.owner.id if doc.owner else None,
         has_owner=doc.owner is not None,

src/documents/migrations/1040_customfield_customfieldboolean_customfielddate_and_more.py

@@ -0,0 +1,196 @@
+# Generated by Django 4.2.5 on 2023-10-22 23:47
+
+import django.db.models.deletion
+import django.utils.timezone
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("documents", "1039_consumptiontemplate"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="CustomField",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    models.DateTimeField(
+                        db_index=True,
+                        default=django.utils.timezone.now,
+                        verbose_name="created",
+                    ),
+                ),
+                ("name", models.CharField(max_length=128)),
+                (
+                    "data_type",
+                    models.CharField(
+                        choices=[
+                            ("string", "String"),
+                            ("url", "URL"),
+                            ("date", "Date"),
+                            ("boolean", "Boolean"),
+                        ],
+                        default="string",
+                        max_length=50,
+                        verbose_name="data type",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "custom field",
+                "verbose_name_plural": "custom fields",
+                "ordering": ("created",),
+            },
+        ),
+        migrations.CreateModel(
+            name="CustomFieldBoolean",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("value", models.BooleanField()),
+                (
+                    "parent",
+                    models.OneToOneField(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        related_name="boolean",
+                        to="documents.customfieldinstance",
+                    ),
+                ),
+            ],
+        ),
+        migrations.CreateModel(
+            name="CustomFieldDate",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("value", models.DateField()),
+                (
+                    "parent",
+                    models.OneToOneField(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        related_name="date",
+                        to="documents.customfieldinstance",
+                    ),
+                ),
+            ],
+        ),
+        migrations.CreateModel(
+            name="CustomFieldInstance",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    models.DateTimeField(
+                        db_index=True,
+                        default=django.utils.timezone.now,
+                        verbose_name="created",
+                    ),
+                ),
+                (
+                    "document",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="custom_fields",
+                        to="documents.document",
+                    ),
+                ),
+                (
+                    "field",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="fields",
+                        to="documents.customfield",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "custom field instance",
+                "verbose_name_plural": "custom field instances",
+                "ordering": ("created",),
+            },
+        ),
+        migrations.CreateModel(
+            name="CustomFieldShortText",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("value", models.CharField(max_length=128)),
+                (
+                    "parent",
+                    models.OneToOneField(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        related_name="short_text",
+                        to="documents.customfieldinstance",
+                    ),
+                ),
+            ],
+        ),
+        migrations.CreateModel(
+            name="CustomFieldUrl",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("value", models.URLField()),
+                (
+                    "parent",
+                    models.OneToOneField(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        related_name="url",
+                        to="documents.customfieldinstance",
+                    ),
+                ),
+            ],
+        ),
+    ]

src/documents/migrations/1040_custommetadata.py

@@ -1,80 +0,0 @@
-# Generated by Django 4.2.5 on 2023-10-20 15:48
-
-import django.db.models.deletion
-import django.utils.timezone
-from django.conf import settings
-from django.db import migrations
-from django.db import models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-        ("documents", "1039_consumptiontemplate"),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name="CustomMetadata",
-            fields=[
-                (
-                    "id",
-                    models.AutoField(
-                        auto_created=True,
-                        primary_key=True,
-                        serialize=False,
-                        verbose_name="ID",
-                    ),
-                ),
-                (
-                    "data_type",
-                    models.CharField(
-                        choices=[
-                            ("string", "String"),
-                            ("url", "URL"),
-                            ("date", "Date"),
-                        ],
-                        default="string",
-                        max_length=50,
-                    ),
-                ),
-                ("data", models.CharField(blank=True, max_length=512)),
-                ("name", models.CharField(blank=True, max_length=512)),
-                (
-                    "created",
-                    models.DateTimeField(
-                        db_index=True,
-                        default=django.utils.timezone.now,
-                        verbose_name="created",
-                    ),
-                ),
-                (
-                    "document",
-                    models.ForeignKey(
-                        blank=True,
-                        null=True,
-                        on_delete=django.db.models.deletion.CASCADE,
-                        related_name="metadata",
-                        to="documents.document",
-                        verbose_name="document",
-                    ),
-                ),
-                (
-                    "user",
-                    models.ForeignKey(
-                        blank=True,
-                        null=True,
-                        on_delete=django.db.models.deletion.SET_NULL,
-                        related_name="metadata",
-                        to=settings.AUTH_USER_MODEL,
-                        verbose_name="user",
-                    ),
-                ),
-            ],
-            options={
-                "verbose_name": "custom metadata",
-                "verbose_name_plural": "custom metadata",
-                "ordering": ("created",),
-            },
-        ),
-    ]

src/documents/models.py

@@ -885,27 +885,46 @@ if settings.AUDIT_LOG_ENABLED:
     auditlog.register(Note)
 
 
-class CustomMetadata(models.Model):
+class CustomField(models.Model):
-    class DataType(models.TextChoices):
+    """
+    Defines the name and type of a custom field
+    """
+
+    class FieldDataType(models.TextChoices):
         STRING = ("string", _("String"))
         URL = ("url", _("URL"))
         DATE = ("date", _("Date"))
+        BOOL = ("boolean"), _("Boolean")
+
+    created = models.DateTimeField(
+        _("created"),
+        default=timezone.now,
+        db_index=True,
+    )
+
+    name = models.CharField(max_length=128)
 
     data_type = models.CharField(
+        _("data type"),
         max_length=50,
-        choices=DataType.choices,
+        choices=FieldDataType.choices,
-        default=DataType.STRING,
+        default=FieldDataType.STRING,
     )
 
-    data = models.CharField(
+    class Meta:
-        max_length=512,
+        ordering = ("created",)
-        blank=True,
+        verbose_name = _("custom field")
-    )
+        verbose_name_plural = _("custom fields")
 
-    name = models.CharField(
+    def __str__(self) -> str:
-        max_length=512,
+        return f"{self.name} : {self.data_type}"
-        blank=True,
+
-    )
+
+class CustomFieldInstance(models.Model):
+    """
+    A single instance of a field, attached to a CustomField for the name and type
+    and attached to a single Document to be metadata for it
+    """
 
     created = models.DateTimeField(
         _("created"),
@@ -915,51 +934,146 @@ class CustomMetadata(models.Model):
 
     document = models.ForeignKey(
         Document,
-        blank=True,
+        blank=False,
-        null=True,
+        null=False,
-        related_name="metadata",
         on_delete=models.CASCADE,
-        verbose_name=_("document"),
+        related_name="custom_fields",
     )
 
-    user = models.ForeignKey(
+    field = models.ForeignKey(
-        User,
+        CustomField,
-        blank=True,
+        blank=False,
-        null=True,
+        null=False,
-        related_name="metadata",
+        on_delete=models.CASCADE,
-        on_delete=models.SET_NULL,
+        related_name="fields",
-        verbose_name=_("user"),
     )
 
     class Meta:
         ordering = ("created",)
-        verbose_name = _("custom metadata")
+        verbose_name = _("custom field instance")
-        verbose_name_plural = _("custom metadata")
+        verbose_name_plural = _("custom field instances")
 
-    def __str__(self):
+    def __str__(self) -> str:
-        return f"{self.data_type} : {self.name} : {self.data}"
+        return str(self.field) + f" : {self.value}"
+
+    @property
+    def value(self):
+        """
+        Based on the data type, access the actual value the instance stores
+        """
+        if self.field.data_type == CustomField.FieldDataType.STRING:
+            return self.short_text.value
+        elif self.field.data_type == CustomField.FieldDataType.URL:
+            return self.url.value
+        elif self.field.data_type == CustomField.FieldDataType.DATE:
+            return self.date.value
+        elif self.field.data_type == CustomField.FieldDataType.BOOL:
+            return self.boolean.value
+        raise NotImplementedError(self.field.data_type)
+
+    @property
+    def field_type(self):
+        """
+        Based on the data type, quick access to class for storing that value
+        """
+        if self.field.data_type == CustomField.FieldDataType.STRING:
+            return CustomFieldShortText
+        elif self.field.data_type == CustomField.FieldDataType.URL:
+            return CustomFieldUrl
+        elif self.field.data_type == CustomField.FieldDataType.DATE:
+            return CustomFieldDate
+        elif self.field.data_type == CustomField.FieldDataType.BOOL:
+            return CustomFieldBoolean
+        raise NotImplementedError(self.field.data_type)
 
     def to_json(self) -> dict[str, str]:
         return {
             "id": self.id,
             "created": self.created,
-            "type": self.data_type,
+            "type": self.field.data_type,
-            "name": self.name,
+            "name": self.field.name,
-            "data": self.data,
+            "data": self.value,
-            "user": {
-                "id": self.user.id,
-                "username": self.user.username,
-                "first_name": self.user.first_name,
-                "last_name": self.user.last_name,
-            },
         }
 
     @staticmethod
-    def from_json(document: Document, user: User, data) -> "CustomMetadata":
+    def from_json(
-        return CustomMetadata.objects.create(
+        document: Document,
+        field: CustomField,
+        data,
+    ) -> "CustomFieldInstance":
+        instance = CustomFieldInstance.objects.create(
             document=document,
+            field=field,
             data_type=data["type"],
-            name=data["name"],
-            data=data["data"],
-            user=user,
         )
+        instance.field_type.objects.create(value=data["value"], parent=instance)
+
+        return field
+
+
+class CustomFieldShortText(models.Model):
+    """
+    Data storage for a short text custom field
+    """
+
+    value = models.CharField(max_length=128)
+    parent = models.OneToOneField(
+        CustomFieldInstance,
+        on_delete=models.CASCADE,
+        related_name="short_text",
+        parent_link=True,
+    )
+
+    def __str__(self) -> str:
+        return f"{self.value}"
+
+
+class CustomFieldBoolean(models.Model):
+    """
+    Data storage for a boolean custom field
+    """
+
+    value = models.BooleanField()
+    parent = models.OneToOneField(
+        CustomFieldInstance,
+        on_delete=models.CASCADE,
+        related_name="boolean",
+        parent_link=True,
+    )
+
+    def __str__(self) -> str:
+        return f"{self.value}"
+
+
+class CustomFieldUrl(models.Model):
+    """
+    Data storage for a URL custom field
+    """
+
+    value = models.URLField()
+    parent = models.OneToOneField(
+        CustomFieldInstance,
+        on_delete=models.CASCADE,
+        related_name="url",
+        parent_link=True,
+    )
+
+    def __str__(self) -> str:
+        return f"{self.value}"
+
+
+class CustomFieldDate(models.Model):
+    """
+    Data storage for a date custom field
+    """
+
+    value = models.DateField()
+    parent = models.OneToOneField(
+        CustomFieldInstance,
+        on_delete=models.CASCADE,
+        related_name="date",
+        parent_link=True,
+    )
+
+    def __str__(self) -> str:
+        return f"{self.value}"

src/documents/views.py

@@ -1,10 +1,8 @@
 import itertools
-import json
 import logging
 import os
 import re
 import tempfile
-import urllib
 import zipfile
 from datetime import datetime
 from pathlib import Path
@@ -28,7 +26,6 @@ from django.http import HttpResponse
 from django.http import HttpResponseBadRequest
 from django.http import HttpResponseForbidden
 from django.http import HttpResponseRedirect
-from django.http import HttpResponseServerError
 from django.shortcuts import get_object_or_404
 from django.utils import timezone
 from django.utils.decorators import method_decorator
@@ -38,7 +35,6 @@ from django.views.decorators.cache import cache_control
 from django.views.generic import TemplateView
 from django_filters.rest_framework import DjangoFilterBackend
 from langdetect import detect
-from packaging import version as packaging_version
 from rest_framework import parsers
 from rest_framework.decorators import action
 from rest_framework.exceptions import NotFound
@@ -79,7 +75,8 @@ from documents.matching import match_storage_paths
 from documents.matching import match_tags
 from documents.models import ConsumptionTemplate
 from documents.models import Correspondent
-from documents.models import CustomMetadata
+
+# from documents.models import CustomMetadata
 from documents.models import Document
 from documents.models import DocumentType
 from documents.models import Note
@@ -113,7 +110,6 @@ from documents.serialisers import TagSerializerVersion1
 from documents.serialisers import TasksViewSerializer
 from documents.serialisers import UiSettingsViewSerializer
 from documents.tasks import consume_file
-from paperless import version
 from paperless.db import GnuPG
 from paperless.views import StandardPagination
 
@@ -624,99 +620,99 @@ class DocumentViewSet(
             ]
             return Response(links)
 
-    @action(methods=["get", "post", "delete"], detail=True)
+    # @action(methods=["get", "post", "delete"], detail=True)
-    def custom_metadata(self, request, pk=None) -> Response:
+    # def custom_metadata(self, request, pk=None) -> Response:
-        def package_custom_metadata(doc: Document):
+    #     def package_custom_metadata(doc: Document):
-            return [
+    #         return [
-                c.to_json()
+    #             c.to_json()
-                for c in CustomMetadata.objects.filter(document=doc).order_by(
+    #             for c in CustomMetadata.objects.filter(document=doc).order_by(
-                    "-created",
+    #                 "-created",
-                )
+    #             )
-            ]
+    #         ]
 
-        request.user = request.user
+    #     request.user = request.user
-        try:
+    #     try:
-            doc = Document.objects.get(pk=pk)
+    #         doc = Document.objects.get(pk=pk)
-            if request.user is not None and not has_perms_owner_aware(
+    #         if request.user is not None and not has_perms_owner_aware(
-                request.user,
+    #             request.user,
-                "view_document",
+    #             "view_document",
-                doc,
+    #             doc,
-            ):
+    #         ):
-                return HttpResponseForbidden(
+    #             return HttpResponseForbidden(
-                    "Insufficient permissions to view custom metadata",
+    #                 "Insufficient permissions to view custom metadata",
-                )
+    #             )
-        except Document.DoesNotExist:
+    #     except Document.DoesNotExist:
-            raise Http404
+    #         raise Http404
 
-        if request.method == "GET":
+    #     if request.method == "GET":
-            try:
+    #         try:
-                return Response(package_custom_metadata(doc))
+    #             return Response(package_custom_metadata(doc))
-            except Exception as e:
+    #         except Exception as e:
-                logger.warning(f"An error occurred retrieving custom metadata: {e!s}")
+    #             logger.warning(f"An error occurred retrieving custom metadata: {e!s}")
-                return HttpResponseServerError(
+    #             return HttpResponseServerError(
-                    {
+    #                 {
-                        "error": (
+    #                     "error": (
-                            "Error retrieving custom metadata,"
+    #                         "Error retrieving custom metadata,"
-                            " check logs for more detail."
+    #                         " check logs for more detail."
-                        ),
+    #                     ),
-                    },
+    #                 },
-                )
+    #             )
-        elif request.method == "POST":
+    #     elif request.method == "POST":
-            try:
+    #         try:
-                if request.user is not None and not has_perms_owner_aware(
+    #             if request.user is not None and not has_perms_owner_aware(
-                    request.user,
+    #                 request.user,
-                    "change_document",
+    #                 "change_document",
-                    doc,
+    #                 doc,
-                ):
+    #             ):
-                    return HttpResponseForbidden(
+    #                 return HttpResponseForbidden(
-                        "Insufficient permissions to create custom metadata",
+    #                     "Insufficient permissions to create custom metadata",
-                    )
+    #                 )
 
-                CustomMetadata.from_json(doc, request.user, request.data)
+    #             CustomMetadata.from_json(doc, request.user, request.data)
 
-                doc.modified = timezone.now()
+    #             doc.modified = timezone.now()
-                doc.save()
+    #             doc.save()
 
-                from documents import index
+    #             from documents import index
 
-                index.add_or_update_document(self.get_object())
+    #             index.add_or_update_document(self.get_object())
 
-                return Response(package_custom_metadata(doc))
+    #             return Response(package_custom_metadata(doc))
-            except Exception as e:
+    #         except Exception as e:
-                logger.warning(f"An error occurred saving custom metadata: {e!s}")
+    #             logger.warning(f"An error occurred saving custom metadata: {e!s}")
-                return HttpResponseServerError(
+    #             return HttpResponseServerError(
-                    {
+    #                 {
-                        "error": (
+    #                     "error": (
-                            "Error saving custom metadata, "
+    #                         "Error saving custom metadata, "
-                            "check logs for more detail."
+    #                         "check logs for more detail."
-                        ),
+    #                     ),
-                    },
+    #                 },
-                )
+    #             )
-        elif request.method == "DELETE":
+    #     elif request.method == "DELETE":
-            if request.user is not None and not has_perms_owner_aware(
+    #         if request.user is not None and not has_perms_owner_aware(
-                request.user,
+    #             request.user,
-                "change_document",
+    #             "change_document",
-                doc,
+    #             doc,
-            ):
+    #         ):
-                return HttpResponseForbidden(
+    #             return HttpResponseForbidden(
-                    "Insufficient permissions to delete custom metadata",
+    #                 "Insufficient permissions to delete custom metadata",
-                )
+    #             )
 
-            metadata = CustomMetadata.objects.get(id=int(request.GET.get("id")))
+    #         metadata = CustomMetadata.objects.get(id=int(request.GET.get("id")))
-            metadata.delete()
+    #         metadata.delete()
 
-            doc.modified = timezone.now()
+    #         doc.modified = timezone.now()
-            doc.save()
+    #         doc.save()
 
-            from documents import index
+    #         from documents import index
 
-            index.add_or_update_document(self.get_object())
+    #         index.add_or_update_document(self.get_object())
 
-            return Response(package_custom_metadata(doc))
+    #         return Response(package_custom_metadata(doc))
 
-        return Response(
+    #     return Response(
-            {"error": "unreachable error was reached for custom metadata"},
+    #         {"error": "unreachable error was reached for custom metadata"},
-        )  # pragma: no cover
+    #     )  # pragma: no cover
 
 
 class SearchResultSerializer(DocumentSerializer, PassUserMixin):
@@ -1171,47 +1167,6 @@ class BulkDownloadView(GenericAPIView):
             return response
 
 
-class RemoteVersionView(GenericAPIView):
-    def get(self, request, format=None):
-        remote_version = "0.0.0"
-        is_greater_than_current = False
-        current_version = packaging_version.parse(version.__full_version_str__)
-        try:
-            req = urllib.request.Request(
-                "https://api.github.com/repos/paperless-ngx/"
-                "paperless-ngx/releases/latest",
-            )
-            # Ensure a JSON response
-            req.add_header("Accept", "application/json")
-
-            with urllib.request.urlopen(req) as response:
-                remote = response.read().decode("utf-8")
-            try:
-                remote_json = json.loads(remote)
-                remote_version = remote_json["tag_name"]
-                # Basically PEP 616 but that only went in 3.9
-                if remote_version.startswith("ngx-"):
-                    remote_version = remote_version[len("ngx-") :]
-            except ValueError:
-                logger.debug("An error occurred parsing remote version json")
-        except urllib.error.URLError:
-            logger.debug("An error occurred checking for available updates")
-
-        is_greater_than_current = (
-            packaging_version.parse(
-                remote_version,
-            )
-            > current_version
-        )
-
-        return Response(
-            {
-                "version": remote_version,
-                "update_available": is_greater_than_current,
-            },
-        )
-
-
 class StoragePathViewSet(ModelViewSet, PassUserMixin):
     model = StoragePath