lgcosta/paperless-ngx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix doc cleanup signals, testing

Browse Source
This commit is contained in:
shamoon
2024-04-28 23:17:58 -07:00
parent 0b35ce3619
commit 413320674b
10 changed files with 252 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/documents/serialisers.py
View File

@@ -1868,7 +1868,7 @@ class WorkflowSerializer(serializers.ModelSerializer):
class TrashSerializer(SerializerWithPerms):
documents = serializers.ListField(
required=True,
required=False,
label="Documents",
write_only=True,
child=serializers.IntegerField(),

9
src/documents/signals/handlers.py
View File

@@ -1,7 +1,6 @@
import logging
import os
import shutil
from datetime import timedelta
from typing import Optional
from celery import states
@@ -302,12 +301,8 @@ def set_storage_path(
document.save(update_fields=("storage_path",))
@receiver(models.signals.post_delete, sender=Document)
def cleanup_document_deletion(sender, instance, force=False, **kwargs):
if not force:
now = timezone.localtime(timezone.now())
if now - instance.deleted_at < timedelta(days=settings.EMPTY_TRASH_DELAY):
return
# see empty_trash in documents/tasks.py for signal handling
def cleanup_document_deletion(sender, instance, **kwargs):
with FileLock(settings.MEDIA_LOCK):
if settings.TRASH_DIR:
# Find a non-conflicting filename in case a document with the same

16
src/documents/tasks.py
View File

@@ -11,8 +11,8 @@ import tqdm
from celery import Task
from celery import shared_task
from django.conf import settings
from django.db import models
from django.db import transaction
from django.db.models.signals import post_delete
from django.db.models.signals import post_save
from django.utils import timezone
from filelock import FileLock
@@ -44,6 +44,7 @@ from documents.plugins.base import StopConsumeTaskError
from documents.plugins.helpers import ProgressStatusOptions
from documents.sanity_checker import SanityCheckFailedException
from documents.signals import document_updated
from documents.signals.handlers import cleanup_document_deletion
if settings.AUDIT_LOG_ENABLED:
import json
@@ -307,10 +308,11 @@ def empty_trash(doc_ids=None):
if doc_ids is not None
else Document.deleted_objects.filter(deleted_at__gt=cutoff)
)
# Temporarily connect the cleanup handler (hard_delete calls delete)
models.signals.post_delete.connect(cleanup_document_deletion, sender=Document)
for doc in documents:
doc.delete()
post_delete.send(
sender=Document,
instance=doc,
force=True,
)
doc.hard_delete()
models.signals.post_delete.disconnect(cleanup_document_deletion, sender=Document)

1
src/documents/tests/test_api_uisettings.py
View File

@@ -40,6 +40,7 @@ class TestApiUiSettings(DirectoriesMixin, APITestCase):
"app_title": None,
"app_logo": None,
"auditlog_enabled": True,
"trash_delay": 30,
"update_checking": {
"backend_setting": "default",
},

30
src/documents/tests/test_document_model.py
View File

@@ -10,6 +10,7 @@ from django.utils import timezone
from documents.models import Correspondent
from documents.models import Document
from documents.tasks import empty_trash
class TestDocument(TestCase):
@@ -43,10 +44,39 @@ class TestDocument(TestCase):
with mock.patch("documents.signals.handlers.os.unlink") as mock_unlink:
document.delete()
empty_trash([document.pk])
mock_unlink.assert_any_call(file_path)
mock_unlink.assert_any_call(thumb_path)
self.assertEqual(mock_unlink.call_count, 2)
def test_document_soft_delete(self):
document = Document.objects.create(
correspondent=Correspondent.objects.create(name="Test0"),
title="Title",
content="content",
checksum="checksum",
mime_type="application/pdf",
)
file_path = document.source_path
thumb_path = document.thumbnail_path
Path(file_path).touch()
Path(thumb_path).touch()
with mock.patch("documents.signals.handlers.os.unlink") as mock_unlink:
document.delete()
self.assertEqual(mock_unlink.call_count, 0)
self.assertEqual(Document.objects.count(), 0)
document.restore(strict=False)
self.assertEqual(Document.objects.count(), 1)
document.delete()
empty_trash([document.pk])
self.assertEqual(mock_unlink.call_count, 2)
def test_file_name(self):
doc = Document(
mime_type="application/pdf",

15
src/documents/tests/test_file_handling.py
View File

@@ -19,6 +19,7 @@ from documents.models import Correspondent
from documents.models import Document
from documents.models import DocumentType
from documents.models import StoragePath
from documents.tasks import empty_trash
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import FileSystemAssertsMixin
@@ -169,6 +170,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
# Ensure that filename is properly generated
document.filename = generate_filename(document)
document.save()
self.assertEqual(document.filename, "none/none.pdf")
create_source_path_directory(document.source_path)
@@ -176,6 +178,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
# Ensure file deletion after delete
document.delete()
empty_trash([document.pk])
self.assertIsNotFile(
os.path.join(settings.ORIGINALS_DIR, "none", "none.pdf"),
)
@@ -185,7 +188,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
FILENAME_FORMAT="{correspondent}/{correspondent}",
TRASH_DIR=tempfile.mkdtemp(),
)
def test_document_delete_trash(self):
def test_document_delete_trash_dir(self):
document = Document()
document.mime_type = "application/pdf"
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
@@ -193,6 +196,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
# Ensure that filename is properly generated
document.filename = generate_filename(document)
document.save()
self.assertEqual(document.filename, "none/none.pdf")
create_source_path_directory(document.source_path)
@@ -201,6 +205,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
# Ensure file was moved to trash after delete
self.assertIsNotFile(os.path.join(settings.TRASH_DIR, "none", "none.pdf"))
document.delete()
empty_trash([document.pk])
self.assertIsNotFile(
os.path.join(settings.ORIGINALS_DIR, "none", "none.pdf"),
)
@@ -214,9 +219,11 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
document.save()
document.filename = generate_filename(document)
document.save()
create_source_path_directory(document.source_path)
Path(document.source_path).touch()
document.delete()
empty_trash([document.pk])
self.assertIsFile(os.path.join(settings.TRASH_DIR, "none_01.pdf"))
@override_settings(FILENAME_FORMAT="{correspondent}/{correspondent}")
@@ -227,6 +234,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
document.save()
document.delete()
empty_trash([document.pk])
@override_settings(FILENAME_FORMAT="{correspondent}/{correspondent}")
def test_directory_not_empty(self):
@@ -436,6 +444,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
# Ensure that filename is properly generated
document.filename = generate_filename(document)
document.save()
self.assertEqual(document.filename, "none/none/none.pdf")
create_source_path_directory(document.source_path)
Path(document.source_path).touch()
@@ -444,6 +453,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
self.assertIsDir(os.path.join(settings.ORIGINALS_DIR, "none/none"))
document.delete()
empty_trash([document.pk])
self.assertIsNotFile(
os.path.join(settings.ORIGINALS_DIR, "none/none/none.pdf"),
@@ -550,6 +560,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
self.assertEqual(document2.filename, "qwe_01.pdf")
document.delete()
empty_trash([document.pk])
self.assertIsNotFile(document.source_path)
@@ -819,6 +830,7 @@ class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, Test
self.assertIsFile(doc.archive_path)
doc.delete()
empty_trash([doc.pk])
self.assertIsNotFile(original)
self.assertIsNotFile(archive)
@@ -854,6 +866,7 @@ class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, Test
self.assertIsFile(doc2.source_path)
doc2.delete()
empty_trash([doc2.pk])
self.assertIsFile(doc1.source_path)
self.assertIsFile(doc1.archive_path)

17
src/paperless/tests/test_settings.py
View File

@@ -156,6 +156,7 @@ class TestCeleryScheduleParsing(TestCase):
CLASSIFIER_EXPIRE_TIME = 59.0 * 60.0
INDEX_EXPIRE_TIME = 23.0 * 60.0 * 60.0
SANITY_EXPIRE_TIME = ((7.0 * 24.0) - 1.0) * 60.0 * 60.0
EMPTY_TRASH_EXPIRE_TIME = 23.0 * 60.0 * 60.0
def test_schedule_configuration_default(self):
"""
@@ -190,6 +191,11 @@ class TestCeleryScheduleParsing(TestCase):
"schedule": crontab(minute=30, hour=0, day_of_week="sun"),
"options": {"expires": self.SANITY_EXPIRE_TIME},
},
"Empty trash": {
"task": "documents.tasks.empty_trash",
"schedule": crontab(minute=0, hour="1"),
"options": {"expires": self.EMPTY_TRASH_EXPIRE_TIME},
},
},
schedule,
)
@@ -232,6 +238,11 @@ class TestCeleryScheduleParsing(TestCase):
"schedule": crontab(minute=30, hour=0, day_of_week="sun"),
"options": {"expires": self.SANITY_EXPIRE_TIME},
},
"Empty trash": {
"task": "documents.tasks.empty_trash",
"schedule": crontab(minute=0, hour="1"),
"options": {"expires": self.EMPTY_TRASH_EXPIRE_TIME},
},
},
schedule,
)
@@ -266,6 +277,11 @@ class TestCeleryScheduleParsing(TestCase):
"schedule": crontab(minute=30, hour=0, day_of_week="sun"),
"options": {"expires": self.SANITY_EXPIRE_TIME},
},
"Empty trash": {
"task": "documents.tasks.empty_trash",
"schedule": crontab(minute=0, hour="1"),
"options": {"expires": self.EMPTY_TRASH_EXPIRE_TIME},
},
},
schedule,
)
@@ -286,6 +302,7 @@ class TestCeleryScheduleParsing(TestCase):
"PAPERLESS_TRAIN_TASK_CRON": "disable",
"PAPERLESS_SANITY_TASK_CRON": "disable",
"PAPERLESS_INDEX_TASK_CRON": "disable",
"PAPERLESS_EMPTY_TRASH_TASK_CRON": "disable",
},
):
schedule = _parse_beat_schedule()