From 3d307adf82e009d963d80950a2c299c347340efb Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Tue, 9 Apr 2024 10:10:06 -0700 Subject: [PATCH] Working plugin transition for consumer --- paperless-ngx.code-workspace | 33 ++++++++++ src/documents/consumer.py | 103 ++++++++++--------------------- src/documents/plugins/base.py | 3 +- src/documents/plugins/helpers.py | 2 +- src/documents/tasks.py | 32 +--------- src/paperless/settings.py | 8 +-- 6 files changed, 74 insertions(+), 107 deletions(-) create mode 100644 paperless-ngx.code-workspace diff --git a/paperless-ngx.code-workspace b/paperless-ngx.code-workspace new file mode 100644 index 000000000..561ffae79 --- /dev/null +++ b/paperless-ngx.code-workspace @@ -0,0 +1,33 @@ +{ + "folders": [ + { + "path": "." + }, + { + "path": "./src", + "name": "Backend" + }, + { + "path": "./src-ui", + "name": "Frontend" + }, + { + "path": "./.github", + "name": "CI/CD" + } + + ], + "settings": { + "files.exclude": { + "**/__pycache__": true, + "**/.mypy_cache": true, + "**/.ruff_cache": true, + "**/.pytest_cache": true, + "**/.idea": true, + "**/.venv": true, + "**/.coverage": true, + "**/coverage.json": true + }, + "python.defaultInterpreterPath": "./.venv/bin/python3.exe", + } +} diff --git a/src/documents/consumer.py b/src/documents/consumer.py index c735ed4c8..637dc72f9 100644 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -2,15 +2,12 @@ import datetime import hashlib import os import tempfile -import uuid from enum import Enum from pathlib import Path from typing import TYPE_CHECKING from typing import Optional import magic -from asgiref.sync import async_to_sync -from channels.layers import get_channel_layer from django.conf import settings from django.contrib.auth.models import User from django.db import transaction @@ -45,6 +42,7 @@ from documents.plugins.base import AlwaysRunPluginMixin from documents.plugins.base import ConsumeTaskPlugin from documents.plugins.base import NoCleanupPluginMixin from documents.plugins.base import NoSetupPluginMixin +from documents.plugins.helpers import ProgressStatusOptions from documents.signals import document_consumption_finished from documents.signals import document_consumption_started from documents.utils import copy_basic_file_stats @@ -254,9 +252,15 @@ class ConsumerFilePhase(str, Enum): FAILED = "FAILED" -class Consumer(LoggingMixin): +class ConsumerPlugin(AlwaysRunPluginMixin, ConsumeTaskPlugin, LoggingMixin): logging_name = "paperless.consumer" + def setup(self) -> None: + pass + + def cleanup(self) -> None: + pass + def _send_progress( self, current_progress: int, @@ -265,19 +269,15 @@ class Consumer(LoggingMixin): message: Optional[ConsumerStatusShortMessage] = None, document_id=None, ): # pragma: no cover - payload = { - "filename": os.path.basename(self.filename) if self.filename else None, - "task_id": self.task_id, - "current_progress": current_progress, - "max_progress": max_progress, - "status": status, - "message": message, - "document_id": document_id, - "owner_id": self.override_owner_id if self.override_owner_id else None, - } - async_to_sync(self.channel_layer.group_send)( - "status_updates", - {"type": "status_update", "data": payload}, + self.status_mgr.send_progress( + ProgressStatusOptions.WORKING, + message, + current_progress, + max_progress, + { + "document_id": document_id, + "owner_id": self.override_owner_id if self.override_owner_id else None, + }, ) def _fail( @@ -291,22 +291,6 @@ class Consumer(LoggingMixin): self.log.error(log_message or message, exc_info=exc_info) raise ConsumerError(f"{self.filename}: {log_message or message}") from exception - def __init__(self): - super().__init__() - self.path: Optional[Path] = None - self.original_path: Optional[Path] = None - self.filename = None - self.override_title = None - self.override_correspondent_id = None - self.override_tag_ids = None - self.override_document_type_id = None - self.override_asn = None - self.task_id = None - self.override_owner_id = None - self.override_custom_field_ids = None - - self.channel_layer = get_channel_layer() - def pre_check_file_exists(self): """ Confirm the input file still exists where it should @@ -486,45 +470,26 @@ class Consumer(LoggingMixin): exception=e, ) - def try_consume_file( - self, - path: Path, - override_filename=None, - override_title=None, - override_correspondent_id=None, - override_document_type_id=None, - override_tag_ids=None, - override_storage_path_id=None, - task_id=None, - override_created=None, - override_asn=None, - override_owner_id=None, - override_view_users=None, - override_view_groups=None, - override_change_users=None, - override_change_groups=None, - override_custom_field_ids=None, - ) -> Document: + def run(self) -> str: """ Return the document object if it was successfully created. """ - self.original_path = Path(path).resolve() - self.filename = override_filename or self.original_path.name - self.override_title = override_title - self.override_correspondent_id = override_correspondent_id - self.override_document_type_id = override_document_type_id - self.override_tag_ids = override_tag_ids - self.override_storage_path_id = override_storage_path_id - self.task_id = task_id or str(uuid.uuid4()) - self.override_created = override_created - self.override_asn = override_asn - self.override_owner_id = override_owner_id - self.override_view_users = override_view_users - self.override_view_groups = override_view_groups - self.override_change_users = override_change_users - self.override_change_groups = override_change_groups - self.override_custom_field_ids = override_custom_field_ids + self.original_path = self.input_doc.original_file + self.filename = self.metadata.filename or self.input_doc.original_file.name + self.override_title = self.metadata.title + self.override_correspondent_id = self.metadata.correspondent_id + self.override_document_type_id = self.metadata.document_type_id + self.override_tag_ids = self.metadata.tag_ids + self.override_storage_path_id = self.metadata.storage_path_id + self.override_created = self.metadata.created + self.override_asn = self.metadata.asn + self.override_owner_id = self.metadata.owner_id + self.override_view_users = self.metadata.view_users + self.override_view_groups = self.metadata.view_groups + self.override_change_users = self.metadata.change_users + self.override_change_groups = self.metadata.change_groups + self.override_custom_field_ids = self.metadata.custom_field_ids self._send_progress( 0, @@ -766,7 +731,7 @@ class Consumer(LoggingMixin): # Return the most up to date fields document.refresh_from_db() - return document + return f"Success. New document id {document.pk} created" def _parse_title_placeholders(self, title: str) -> str: local_added = timezone.localtime(timezone.now()) diff --git a/src/documents/plugins/base.py b/src/documents/plugins/base.py index aec4887be..14d6ea696 100644 --- a/src/documents/plugins/base.py +++ b/src/documents/plugins/base.py @@ -67,7 +67,8 @@ class ConsumeTaskPlugin(abc.ABC): self.status_mgr = status_mgr self.task_id: Final = task_id - @abc.abstractproperty + @property + @abc.abstractmethod def able_to_run(self) -> bool: """ Return True if the conditions are met for the plugin to run, False otherwise diff --git a/src/documents/plugins/helpers.py b/src/documents/plugins/helpers.py index 27d03f30f..2d3686db4 100644 --- a/src/documents/plugins/helpers.py +++ b/src/documents/plugins/helpers.py @@ -57,7 +57,7 @@ class ProgressManager: message: str, current_progress: int, max_progress: int, - extra_args: Optional[dict[str, Union[str, int]]] = None, + extra_args: Optional[dict[str, Union[str, int, None]]] = None, ) -> None: # Ensure the layer is open self.open() diff --git a/src/documents/tasks.py b/src/documents/tasks.py index 0ab55ac45..c207b5672 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -21,8 +21,7 @@ from documents.barcodes import BarcodePlugin from documents.caching import clear_document_caches from documents.classifier import DocumentClassifier from documents.classifier import load_classifier -from documents.consumer import Consumer -from documents.consumer import ConsumerError +from documents.consumer import ConsumerPlugin from documents.consumer import WorkflowTriggerPlugin from documents.data_models import ConsumableDocument from documents.data_models import DocumentMetadataOverrides @@ -115,6 +114,7 @@ def consume_file( CollatePlugin, BarcodePlugin, WorkflowTriggerPlugin, + ConsumerPlugin, ] with ProgressManager( @@ -162,34 +162,6 @@ def consume_file( finally: plugin.cleanup() - # continue with consumption if no barcode was found - document = Consumer().try_consume_file( - input_doc.original_file, - override_filename=overrides.filename, - override_title=overrides.title, - override_correspondent_id=overrides.correspondent_id, - override_document_type_id=overrides.document_type_id, - override_tag_ids=overrides.tag_ids, - override_storage_path_id=overrides.storage_path_id, - override_created=overrides.created, - override_asn=overrides.asn, - override_owner_id=overrides.owner_id, - override_view_users=overrides.view_users, - override_view_groups=overrides.view_groups, - override_change_users=overrides.change_users, - override_change_groups=overrides.change_groups, - override_custom_field_ids=overrides.custom_field_ids, - task_id=self.request.id, - ) - - if document: - return f"Success. New document id {document.pk} created" - else: - raise ConsumerError( - "Unknown error: Returned document was null, but " - "no error message was given.", - ) - @shared_task def sanity_check(): diff --git a/src/paperless/settings.py b/src/paperless/settings.py index 64af7c9b7..72d3321cd 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -7,7 +7,6 @@ import re import tempfile from os import PathLike from pathlib import Path -from platform import machine from typing import Final from typing import Optional from typing import Union @@ -112,7 +111,7 @@ def __get_list( return [] -def _parse_redis_url(env_redis: Optional[str]) -> tuple[str]: +def _parse_redis_url(env_redis: Optional[str]) -> tuple[str, str]: """ Gets the Redis information from the environment or a default and handles converting from incompatible django_channels and celery formats. @@ -371,10 +370,7 @@ ASGI_APPLICATION = "paperless.asgi.application" STATIC_URL = os.getenv("PAPERLESS_STATIC_URL", BASE_URL + "static/") WHITENOISE_STATIC_PREFIX = "/static/" -if machine().lower() == "aarch64": # pragma: no cover - _static_backend = "django.contrib.staticfiles.storage.StaticFilesStorage" -else: - _static_backend = "whitenoise.storage.CompressedStaticFilesStorage" +_static_backend = "django.contrib.staticfiles.storage.StaticFilesStorage" STORAGES = { "staticfiles": {