lgcosta/paperless-ngx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updates the model relationships

Browse Source
This commit is contained in:
Trenton H 2023-10-22 17:14:36 -07:00 committed by shamoon
parent 021b1031dd
commit b03dc70be1
5 changed files with 441 additions and 256 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/documents/index.py
View File

@ -30,7 +30,7 @@ from whoosh.searching import ResultsPage
from whoosh.searching import Searcher
from whoosh.writing import AsyncWriter
from documents.models import CustomMetadata
# from documents.models import CustomMetadata
from documents.models import Document
from documents.models import Note
from documents.models import User
@ -61,8 +61,8 @@ def get_schema():
has_path=BOOLEAN(),
notes=TEXT(),
num_notes=NUMERIC(sortable=True, signed=False),
custom_metadata=TEXT(),
custom_field_count=NUMERIC(sortable=True, signed=False),
# custom_metadata=TEXT(),
# custom_field_count=NUMERIC(sortable=True, signed=False),
owner=TEXT(),
owner_id=NUMERIC(),
has_owner=BOOLEAN(),
@ -111,9 +111,9 @@ def update_document(writer: AsyncWriter, doc: Document):
tags = ",".join([t.name for t in doc.tags.all()])
tags_ids = ",".join([str(t.id) for t in doc.tags.all()])
notes = ",".join([str(c.note) for c in Note.objects.filter(document=doc)])
custom_fields = ",".join(
[str(c) for c in CustomMetadata.objects.filter(document=doc)],
)
# custom_fields = ",".join(
# [str(c) for c in CustomMetadata.objects.filter(document=doc)],
# )
asn = doc.archive_serial_number
if asn is not None and (
asn < Document.ARCHIVE_SERIAL_NUMBER_MIN
@ -153,8 +153,8 @@ def update_document(writer: AsyncWriter, doc: Document):
has_path=doc.storage_path is not None,
notes=notes,
num_notes=len(notes),
custom_metadata=custom_fields,
custom_field_count=len(custom_fields),
# custom_metadata=custom_fields,
# custom_field_count=len(custom_fields),
owner=doc.owner.username if doc.owner else None,
owner_id=doc.owner.id if doc.owner else None,
has_owner=doc.owner is not None,

196
src/documents/migrations/1040_customfield_customfieldboolean_customfielddate_and_more.py Normal file
View File

@ -0,0 +1,196 @@
# Generated by Django 4.2.5 on 2023-10-22 23:47
import django.db.models.deletion
import django.utils.timezone
from django.db import migrations
from django.db import models
class Migration(migrations.Migration):
dependencies = [
("documents", "1039_consumptiontemplate"),
]
operations = [
migrations.CreateModel(
name="CustomField",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"created",
models.DateTimeField(
db_index=True,
default=django.utils.timezone.now,
verbose_name="created",
),
),
("name", models.CharField(max_length=128)),
(
"data_type",
models.CharField(
choices=[
("string", "String"),
("url", "URL"),
("date", "Date"),
("boolean", "Boolean"),
],
default="string",
max_length=50,
verbose_name="data type",
),
),
],
options={
"verbose_name": "custom field",
"verbose_name_plural": "custom fields",
"ordering": ("created",),
},
),
migrations.CreateModel(
name="CustomFieldBoolean",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("value", models.BooleanField()),
(
"parent",
models.OneToOneField(
on_delete=django.db.models.deletion.CASCADE,
parent_link=True,
related_name="boolean",
to="documents.customfieldinstance",
),
),
],
),
migrations.CreateModel(
name="CustomFieldDate",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("value", models.DateField()),
(
"parent",
models.OneToOneField(
on_delete=django.db.models.deletion.CASCADE,
parent_link=True,
related_name="date",
to="documents.customfieldinstance",
),
),
],
),
migrations.CreateModel(
name="CustomFieldInstance",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"created",
models.DateTimeField(
db_index=True,
default=django.utils.timezone.now,
verbose_name="created",
),
),
(
"document",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="custom_fields",
to="documents.document",
),
),
(
"field",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="fields",
to="documents.customfield",
),
),
],
options={
"verbose_name": "custom field instance",
"verbose_name_plural": "custom field instances",
"ordering": ("created",),
},
),
migrations.CreateModel(
name="CustomFieldShortText",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("value", models.CharField(max_length=128)),
(
"parent",
models.OneToOneField(
on_delete=django.db.models.deletion.CASCADE,
parent_link=True,
related_name="short_text",
to="documents.customfieldinstance",
),
),
],
),
migrations.CreateModel(
name="CustomFieldUrl",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("value", models.URLField()),
(
"parent",
models.OneToOneField(
on_delete=django.db.models.deletion.CASCADE,
parent_link=True,
related_name="url",
to="documents.customfieldinstance",
),
),
],
),
]

80
src/documents/migrations/1040_custommetadata.py
View File

@ -1,80 +0,0 @@
# Generated by Django 4.2.5 on 2023-10-20 15:48
import django.db.models.deletion
import django.utils.timezone
from django.conf import settings
from django.db import migrations
from django.db import models
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
("documents", "1039_consumptiontemplate"),
]
operations = [
migrations.CreateModel(
name="CustomMetadata",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"data_type",
models.CharField(
choices=[
("string", "String"),
("url", "URL"),
("date", "Date"),
],
default="string",
max_length=50,
),
),
("data", models.CharField(blank=True, max_length=512)),
("name", models.CharField(blank=True, max_length=512)),
(
"created",
models.DateTimeField(
db_index=True,
default=django.utils.timezone.now,
verbose_name="created",
),
),
(
"document",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="metadata",
to="documents.document",
verbose_name="document",
),
),
(
"user",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="metadata",
to=settings.AUTH_USER_MODEL,
verbose_name="user",
),
),
],
options={
"verbose_name": "custom metadata",
"verbose_name_plural": "custom metadata",
"ordering": ("created",),
},
),
]

196
src/documents/models.py
View File

@ -885,27 +885,46 @@ if settings.AUDIT_LOG_ENABLED:
auditlog.register(Note)
class CustomMetadata(models.Model):
class DataType(models.TextChoices):
class CustomField(models.Model):
"""
Defines the name and type of a custom field
"""
class FieldDataType(models.TextChoices):
STRING = ("string", _("String"))
URL = ("url", _("URL"))
DATE = ("date", _("Date"))
BOOL = ("boolean"), _("Boolean")
created = models.DateTimeField(
_("created"),
default=timezone.now,
db_index=True,
)
name = models.CharField(max_length=128)
data_type = models.CharField(
_("data type"),
max_length=50,
choices=DataType.choices,
default=DataType.STRING,
choices=FieldDataType.choices,
default=FieldDataType.STRING,
)
data = models.CharField(
max_length=512,
blank=True,
)
class Meta:
ordering = ("created",)
verbose_name = _("custom field")
verbose_name_plural = _("custom fields")
name = models.CharField(
max_length=512,
blank=True,
)
def __str__(self) -> str:
return f"{self.name} : {self.data_type}"
class CustomFieldInstance(models.Model):
"""
A single instance of a field, attached to a CustomField for the name and type
and attached to a single Document to be metadata for it
"""
created = models.DateTimeField(
_("created"),
@ -915,51 +934,146 @@ class CustomMetadata(models.Model):
document = models.ForeignKey(
Document,
blank=True,
null=True,
related_name="metadata",
blank=False,
null=False,
on_delete=models.CASCADE,
verbose_name=_("document"),
related_name="custom_fields",
)
user = models.ForeignKey(
User,
blank=True,
null=True,
related_name="metadata",
on_delete=models.SET_NULL,
verbose_name=_("user"),
field = models.ForeignKey(
CustomField,
blank=False,
null=False,
on_delete=models.CASCADE,
related_name="fields",
)
class Meta:
ordering = ("created",)
verbose_name = _("custom metadata")
verbose_name_plural = _("custom metadata")
verbose_name = _("custom field instance")
verbose_name_plural = _("custom field instances")
def __str__(self):
return f"{self.data_type} : {self.name} : {self.data}"
def __str__(self) -> str:
return str(self.field) + f" : {self.value}"
@property
def value(self):
"""
Based on the data type, access the actual value the instance stores
"""
if self.field.data_type == CustomField.FieldDataType.STRING:
return self.short_text.value
elif self.field.data_type == CustomField.FieldDataType.URL:
return self.url.value
elif self.field.data_type == CustomField.FieldDataType.DATE:
return self.date.value
elif self.field.data_type == CustomField.FieldDataType.BOOL:
return self.boolean.value
raise NotImplementedError(self.field.data_type)
@property
def field_type(self):
"""
Based on the data type, quick access to class for storing that value
"""
if self.field.data_type == CustomField.FieldDataType.STRING:
return CustomFieldShortText
elif self.field.data_type == CustomField.FieldDataType.URL:
return CustomFieldUrl
elif self.field.data_type == CustomField.FieldDataType.DATE:
return CustomFieldDate
elif self.field.data_type == CustomField.FieldDataType.BOOL:
return CustomFieldBoolean
raise NotImplementedError(self.field.data_type)
def to_json(self) -> dict[str, str]:
return {
"id": self.id,
"created": self.created,
"type": self.data_type,
"name": self.name,
"data": self.data,
"user": {
"id": self.user.id,
"username": self.user.username,
"first_name": self.user.first_name,
"last_name": self.user.last_name,
},
"type": self.field.data_type,
"name": self.field.name,
"data": self.value,
}
@staticmethod
def from_json(document: Document, user: User, data) -> "CustomMetadata":
return CustomMetadata.objects.create(
def from_json(
document: Document,
field: CustomField,
data,
) -> "CustomFieldInstance":
instance = CustomFieldInstance.objects.create(
document=document,
field=field,
data_type=data["type"],
name=data["name"],
data=data["data"],
user=user,
)
instance.field_type.objects.create(value=data["value"], parent=instance)
return field
class CustomFieldShortText(models.Model):
"""
Data storage for a short text custom field
"""
value = models.CharField(max_length=128)
parent = models.OneToOneField(
CustomFieldInstance,
on_delete=models.CASCADE,
related_name="short_text",
parent_link=True,
)
def __str__(self) -> str:
return f"{self.value}"
class CustomFieldBoolean(models.Model):
"""
Data storage for a boolean custom field
"""
value = models.BooleanField()
parent = models.OneToOneField(
CustomFieldInstance,
on_delete=models.CASCADE,
related_name="boolean",
parent_link=True,
)
def __str__(self) -> str:
return f"{self.value}"
class CustomFieldUrl(models.Model):
"""
Data storage for a URL custom field
"""
value = models.URLField()
parent = models.OneToOneField(
CustomFieldInstance,
on_delete=models.CASCADE,
related_name="url",
parent_link=True,
)
def __str__(self) -> str:
return f"{self.value}"
class CustomFieldDate(models.Model):
"""
Data storage for a date custom field
"""
value = models.DateField()
parent = models.OneToOneField(
CustomFieldInstance,
on_delete=models.CASCADE,
related_name="date",
parent_link=True,
)
def __str__(self) -> str:
return f"{self.value}"

209
src/documents/views.py
View File

@ -1,10 +1,8 @@
import itertools
import json
import logging
import os
import re
import tempfile
import urllib
import zipfile
from datetime import datetime
from pathlib import Path
@ -28,7 +26,6 @@ from django.http import HttpResponse
from django.http import HttpResponseBadRequest
from django.http import HttpResponseForbidden
from django.http import HttpResponseRedirect
from django.http import HttpResponseServerError
from django.shortcuts import get_object_or_404
from django.utils import timezone
from django.utils.decorators import method_decorator
@ -38,7 +35,6 @@ from django.views.decorators.cache import cache_control
from django.views.generic import TemplateView
from django_filters.rest_framework import DjangoFilterBackend
from langdetect import detect
from packaging import version as packaging_version
from rest_framework import parsers
from rest_framework.decorators import action
from rest_framework.exceptions import NotFound
@ -79,7 +75,8 @@ from documents.matching import match_storage_paths
from documents.matching import match_tags
from documents.models import ConsumptionTemplate
from documents.models import Correspondent
from documents.models import CustomMetadata
# from documents.models import CustomMetadata
from documents.models import Document
from documents.models import DocumentType
from documents.models import Note
@ -113,7 +110,6 @@ from documents.serialisers import TagSerializerVersion1
from documents.serialisers import TasksViewSerializer
from documents.serialisers import UiSettingsViewSerializer
from documents.tasks import consume_file
from paperless import version
from paperless.db import GnuPG
from paperless.views import StandardPagination
@ -624,99 +620,99 @@ class DocumentViewSet(
]
return Response(links)
@action(methods=["get", "post", "delete"], detail=True)
def custom_metadata(self, request, pk=None) -> Response:
def package_custom_metadata(doc: Document):
return [
c.to_json()
for c in CustomMetadata.objects.filter(document=doc).order_by(
"-created",
)
]
# @action(methods=["get", "post", "delete"], detail=True)
# def custom_metadata(self, request, pk=None) -> Response:
# def package_custom_metadata(doc: Document):
# return [
# c.to_json()
# for c in CustomMetadata.objects.filter(document=doc).order_by(
# "-created",
# )
# ]
request.user = request.user
try:
doc = Document.objects.get(pk=pk)
if request.user is not None and not has_perms_owner_aware(
request.user,
"view_document",
doc,
):
return HttpResponseForbidden(
"Insufficient permissions to view custom metadata",
)
except Document.DoesNotExist:
raise Http404
# request.user = request.user
# try:
# doc = Document.objects.get(pk=pk)
# if request.user is not None and not has_perms_owner_aware(
# request.user,
# "view_document",
# doc,
# ):
# return HttpResponseForbidden(
# "Insufficient permissions to view custom metadata",
# )
# except Document.DoesNotExist:
# raise Http404
if request.method == "GET":
try:
return Response(package_custom_metadata(doc))
except Exception as e:
logger.warning(f"An error occurred retrieving custom metadata: {e!s}")
return HttpResponseServerError(
{
"error": (
"Error retrieving custom metadata,"
" check logs for more detail."
),
},
)
elif request.method == "POST":
try:
if request.user is not None and not has_perms_owner_aware(
request.user,
"change_document",
doc,
):
return HttpResponseForbidden(
"Insufficient permissions to create custom metadata",
)
# if request.method == "GET":
# try:
# return Response(package_custom_metadata(doc))
# except Exception as e:
# logger.warning(f"An error occurred retrieving custom metadata: {e!s}")
# return HttpResponseServerError(
# {
# "error": (
# "Error retrieving custom metadata,"
# " check logs for more detail."
# ),
# },
# )
# elif request.method == "POST":
# try:
# if request.user is not None and not has_perms_owner_aware(
# request.user,
# "change_document",
# doc,
# ):
# return HttpResponseForbidden(
# "Insufficient permissions to create custom metadata",
# )
CustomMetadata.from_json(doc, request.user, request.data)
# CustomMetadata.from_json(doc, request.user, request.data)
doc.modified = timezone.now()
doc.save()
# doc.modified = timezone.now()
# doc.save()
from documents import index
# from documents import index
index.add_or_update_document(self.get_object())
# index.add_or_update_document(self.get_object())
return Response(package_custom_metadata(doc))
except Exception as e:
logger.warning(f"An error occurred saving custom metadata: {e!s}")
return HttpResponseServerError(
{
"error": (
"Error saving custom metadata, "
"check logs for more detail."
),
},
)
elif request.method == "DELETE":
if request.user is not None and not has_perms_owner_aware(
request.user,
"change_document",
doc,
):
return HttpResponseForbidden(
"Insufficient permissions to delete custom metadata",
)
# return Response(package_custom_metadata(doc))
# except Exception as e:
# logger.warning(f"An error occurred saving custom metadata: {e!s}")
# return HttpResponseServerError(
# {
# "error": (
# "Error saving custom metadata, "
# "check logs for more detail."
# ),
# },
# )
# elif request.method == "DELETE":
# if request.user is not None and not has_perms_owner_aware(
# request.user,
# "change_document",
# doc,
# ):
# return HttpResponseForbidden(
# "Insufficient permissions to delete custom metadata",
# )
metadata = CustomMetadata.objects.get(id=int(request.GET.get("id")))
metadata.delete()
# metadata = CustomMetadata.objects.get(id=int(request.GET.get("id")))
# metadata.delete()
doc.modified = timezone.now()
doc.save()
# doc.modified = timezone.now()
# doc.save()
from documents import index
# from documents import index
index.add_or_update_document(self.get_object())
# index.add_or_update_document(self.get_object())
return Response(package_custom_metadata(doc))
# return Response(package_custom_metadata(doc))
return Response(
{"error": "unreachable error was reached for custom metadata"},
) # pragma: no cover
# return Response(
# {"error": "unreachable error was reached for custom metadata"},
# ) # pragma: no cover
class SearchResultSerializer(DocumentSerializer, PassUserMixin):
@ -1171,47 +1167,6 @@ class BulkDownloadView(GenericAPIView):
return response
class RemoteVersionView(GenericAPIView):
def get(self, request, format=None):
remote_version = "0.0.0"
is_greater_than_current = False
current_version = packaging_version.parse(version.__full_version_str__)
try:
req = urllib.request.Request(
"https://api.github.com/repos/paperless-ngx/"
"paperless-ngx/releases/latest",
)
# Ensure a JSON response
req.add_header("Accept", "application/json")
with urllib.request.urlopen(req) as response:
remote = response.read().decode("utf-8")
try:
remote_json = json.loads(remote)
remote_version = remote_json["tag_name"]
# Basically PEP 616 but that only went in 3.9
if remote_version.startswith("ngx-"):
remote_version = remote_version[len("ngx-") :]
except ValueError:
logger.debug("An error occurred parsing remote version json")
except urllib.error.URLError:
logger.debug("An error occurred checking for available updates")
is_greater_than_current = (
packaging_version.parse(
remote_version,
)
> current_version
)
return Response(
{
"version": remote_version,
"update_available": is_greater_than_current,
},
)
class StoragePathViewSet(ModelViewSet, PassUserMixin):
model = StoragePath