Working plugin transition for consumer

paperless-ngx.code-workspace

@@ -0,0 +1,33 @@
+{
+	"folders": [
+		{
+			"path": "."
+		},
+		{
+			"path": "./src",
+			"name": "Backend"
+		},
+		{
+			"path": "./src-ui",
+			"name": "Frontend"
+		},
+		{
+			"path": "./.github",
+			"name": "CI/CD"
+		}
+
+	],
+	"settings": {
+		"files.exclude": {
+			"**/__pycache__": true,
+			"**/.mypy_cache": true,
+			"**/.ruff_cache": true,
+			"**/.pytest_cache": true,
+			"**/.idea": true,
+			"**/.venv": true,
+			"**/.coverage": true,
+			"**/coverage.json": true
+	},
+	"python.defaultInterpreterPath": "./.venv/bin/python3.exe",
+	}
+}

src/documents/consumer.py

@@ -2,15 +2,12 @@ import datetime
 import hashlib
 import os
 import tempfile
-import uuid
 from enum import Enum
 from pathlib import Path
 from typing import TYPE_CHECKING
 from typing import Optional
 
 import magic
-from asgiref.sync import async_to_sync
-from channels.layers import get_channel_layer
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.db import transaction
@@ -45,6 +42,7 @@ from documents.plugins.base import AlwaysRunPluginMixin
 from documents.plugins.base import ConsumeTaskPlugin
 from documents.plugins.base import NoCleanupPluginMixin
 from documents.plugins.base import NoSetupPluginMixin
+from documents.plugins.helpers import ProgressStatusOptions
 from documents.signals import document_consumption_finished
 from documents.signals import document_consumption_started
 from documents.utils import copy_basic_file_stats
@@ -254,9 +252,15 @@ class ConsumerFilePhase(str, Enum):
     FAILED = "FAILED"
 
 
-class Consumer(LoggingMixin):
+class ConsumerPlugin(AlwaysRunPluginMixin, ConsumeTaskPlugin, LoggingMixin):
     logging_name = "paperless.consumer"
 
+    def setup(self) -> None:
+        pass
+
+    def cleanup(self) -> None:
+        pass
+
     def _send_progress(
         self,
         current_progress: int,
@@ -265,19 +269,15 @@ class Consumer(LoggingMixin):
         message: Optional[ConsumerStatusShortMessage] = None,
         document_id=None,
     ):  # pragma: no cover
-        payload = {
-            "filename": os.path.basename(self.filename) if self.filename else None,
-            "task_id": self.task_id,
-            "current_progress": current_progress,
-            "max_progress": max_progress,
-            "status": status,
-            "message": message,
+        self.status_mgr.send_progress(
+            ProgressStatusOptions.WORKING,
+            message,
+            current_progress,
+            max_progress,
+            {
                 "document_id": document_id,
                 "owner_id": self.override_owner_id if self.override_owner_id else None,
-        }
-        async_to_sync(self.channel_layer.group_send)(
-            "status_updates",
-            {"type": "status_update", "data": payload},
+            },
         )
 
     def _fail(
@@ -291,22 +291,6 @@ class Consumer(LoggingMixin):
         self.log.error(log_message or message, exc_info=exc_info)
         raise ConsumerError(f"{self.filename}: {log_message or message}") from exception
 
-    def __init__(self):
-        super().__init__()
-        self.path: Optional[Path] = None
-        self.original_path: Optional[Path] = None
-        self.filename = None
-        self.override_title = None
-        self.override_correspondent_id = None
-        self.override_tag_ids = None
-        self.override_document_type_id = None
-        self.override_asn = None
-        self.task_id = None
-        self.override_owner_id = None
-        self.override_custom_field_ids = None
-
-        self.channel_layer = get_channel_layer()
-
     def pre_check_file_exists(self):
         """
         Confirm the input file still exists where it should
@@ -486,45 +470,26 @@ class Consumer(LoggingMixin):
                 exception=e,
             )
 
-    def try_consume_file(
-        self,
-        path: Path,
-        override_filename=None,
-        override_title=None,
-        override_correspondent_id=None,
-        override_document_type_id=None,
-        override_tag_ids=None,
-        override_storage_path_id=None,
-        task_id=None,
-        override_created=None,
-        override_asn=None,
-        override_owner_id=None,
-        override_view_users=None,
-        override_view_groups=None,
-        override_change_users=None,
-        override_change_groups=None,
-        override_custom_field_ids=None,
-    ) -> Document:
+    def run(self) -> str:
         """
         Return the document object if it was successfully created.
         """
 
-        self.original_path = Path(path).resolve()
-        self.filename = override_filename or self.original_path.name
-        self.override_title = override_title
-        self.override_correspondent_id = override_correspondent_id
-        self.override_document_type_id = override_document_type_id
-        self.override_tag_ids = override_tag_ids
-        self.override_storage_path_id = override_storage_path_id
-        self.task_id = task_id or str(uuid.uuid4())
-        self.override_created = override_created
-        self.override_asn = override_asn
-        self.override_owner_id = override_owner_id
-        self.override_view_users = override_view_users
-        self.override_view_groups = override_view_groups
-        self.override_change_users = override_change_users
-        self.override_change_groups = override_change_groups
-        self.override_custom_field_ids = override_custom_field_ids
+        self.original_path = self.input_doc.original_file
+        self.filename = self.metadata.filename or self.input_doc.original_file.name
+        self.override_title = self.metadata.title
+        self.override_correspondent_id = self.metadata.correspondent_id
+        self.override_document_type_id = self.metadata.document_type_id
+        self.override_tag_ids = self.metadata.tag_ids
+        self.override_storage_path_id = self.metadata.storage_path_id
+        self.override_created = self.metadata.created
+        self.override_asn = self.metadata.asn
+        self.override_owner_id = self.metadata.owner_id
+        self.override_view_users = self.metadata.view_users
+        self.override_view_groups = self.metadata.view_groups
+        self.override_change_users = self.metadata.change_users
+        self.override_change_groups = self.metadata.change_groups
+        self.override_custom_field_ids = self.metadata.custom_field_ids
 
         self._send_progress(
             0,
@@ -766,7 +731,7 @@ class Consumer(LoggingMixin):
         # Return the most up to date fields
         document.refresh_from_db()
 
-        return document
+        return f"Success. New document id {document.pk} created"
 
     def _parse_title_placeholders(self, title: str) -> str:
         local_added = timezone.localtime(timezone.now())

src/documents/plugins/base.py

@@ -67,7 +67,8 @@ class ConsumeTaskPlugin(abc.ABC):
         self.status_mgr = status_mgr
         self.task_id: Final = task_id
 
-    @abc.abstractproperty
+    @property
+    @abc.abstractmethod
     def able_to_run(self) -> bool:
         """
         Return True if the conditions are met for the plugin to run, False otherwise

src/documents/plugins/helpers.py

@@ -57,7 +57,7 @@ class ProgressManager:
         message: str,
         current_progress: int,
         max_progress: int,
-        extra_args: Optional[dict[str, Union[str, int]]] = None,
+        extra_args: Optional[dict[str, Union[str, int, None]]] = None,
     ) -> None:
         # Ensure the layer is open
         self.open()

src/documents/tasks.py

@@ -21,8 +21,7 @@ from documents.barcodes import BarcodePlugin
 from documents.caching import clear_document_caches
 from documents.classifier import DocumentClassifier
 from documents.classifier import load_classifier
-from documents.consumer import Consumer
-from documents.consumer import ConsumerError
+from documents.consumer import ConsumerPlugin
 from documents.consumer import WorkflowTriggerPlugin
 from documents.data_models import ConsumableDocument
 from documents.data_models import DocumentMetadataOverrides
@@ -115,6 +114,7 @@ def consume_file(
         CollatePlugin,
         BarcodePlugin,
         WorkflowTriggerPlugin,
+        ConsumerPlugin,
     ]
 
     with ProgressManager(
@@ -162,34 +162,6 @@ def consume_file(
             finally:
                 plugin.cleanup()
 
-    # continue with consumption if no barcode was found
-    document = Consumer().try_consume_file(
-        input_doc.original_file,
-        override_filename=overrides.filename,
-        override_title=overrides.title,
-        override_correspondent_id=overrides.correspondent_id,
-        override_document_type_id=overrides.document_type_id,
-        override_tag_ids=overrides.tag_ids,
-        override_storage_path_id=overrides.storage_path_id,
-        override_created=overrides.created,
-        override_asn=overrides.asn,
-        override_owner_id=overrides.owner_id,
-        override_view_users=overrides.view_users,
-        override_view_groups=overrides.view_groups,
-        override_change_users=overrides.change_users,
-        override_change_groups=overrides.change_groups,
-        override_custom_field_ids=overrides.custom_field_ids,
-        task_id=self.request.id,
-    )
-
-    if document:
-        return f"Success. New document id {document.pk} created"
-    else:
-        raise ConsumerError(
-            "Unknown error: Returned document was null, but "
-            "no error message was given.",
-        )
-
 
 @shared_task
 def sanity_check():

src/paperless/settings.py

@@ -7,7 +7,6 @@ import re
 import tempfile
 from os import PathLike
 from pathlib import Path
-from platform import machine
 from typing import Final
 from typing import Optional
 from typing import Union
@@ -112,7 +111,7 @@ def __get_list(
         return []
 
 
-def _parse_redis_url(env_redis: Optional[str]) -> tuple[str]:
+def _parse_redis_url(env_redis: Optional[str]) -> tuple[str, str]:
     """
     Gets the Redis information from the environment or a default and handles
     converting from incompatible django_channels and celery formats.
@@ -371,10 +370,7 @@ ASGI_APPLICATION = "paperless.asgi.application"
 STATIC_URL = os.getenv("PAPERLESS_STATIC_URL", BASE_URL + "static/")
 WHITENOISE_STATIC_PREFIX = "/static/"
 
-if machine().lower() == "aarch64":  # pragma: no cover
 _static_backend = "django.contrib.staticfiles.storage.StaticFilesStorage"
-else:
-    _static_backend = "whitenoise.storage.CompressedStaticFilesStorage"
 
 STORAGES = {
     "staticfiles": {