lgcosta/paperless-ngx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Working plugin transition for consumer

Browse Source
This commit is contained in:
Trenton H 2024-04-09 10:10:06 -07:00
parent e837f1e85b
commit 3d307adf82
6 changed files with 74 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
paperless-ngx.code-workspace Normal file
View File

@ -0,0 +1,33 @@
{
"folders": [
{
"path": "."
},
{
"path": "./src",
"name": "Backend"
},
{
"path": "./src-ui",
"name": "Frontend"
},
{
"path": "./.github",
"name": "CI/CD"
}
],
"settings": {
"files.exclude": {
"**/__pycache__": true,
"**/.mypy_cache": true,
"**/.ruff_cache": true,
"**/.pytest_cache": true,
"**/.idea": true,
"**/.venv": true,
"**/.coverage": true,
"**/coverage.json": true
},
"python.defaultInterpreterPath": "./.venv/bin/python3.exe",
}
}

99
src/documents/consumer.py
View File

@ -2,15 +2,12 @@ import datetime
import hashlib
import os
import tempfile
import uuid
from enum import Enum
from pathlib import Path
from typing import TYPE_CHECKING
from typing import Optional
import magic
from asgiref.sync import async_to_sync
from channels.layers import get_channel_layer
from django.conf import settings
from django.contrib.auth.models import User
from django.db import transaction
@ -45,6 +42,7 @@ from documents.plugins.base import AlwaysRunPluginMixin
from documents.plugins.base import ConsumeTaskPlugin
from documents.plugins.base import NoCleanupPluginMixin
from documents.plugins.base import NoSetupPluginMixin
from documents.plugins.helpers import ProgressStatusOptions
from documents.signals import document_consumption_finished
from documents.signals import document_consumption_started
from documents.utils import copy_basic_file_stats
@ -254,9 +252,15 @@ class ConsumerFilePhase(str, Enum):
FAILED = "FAILED"
class Consumer(LoggingMixin):
class ConsumerPlugin(AlwaysRunPluginMixin, ConsumeTaskPlugin, LoggingMixin):
logging_name = "paperless.consumer"
def setup(self) -> None:
pass
def cleanup(self) -> None:
pass
def _send_progress(
self,
current_progress: int,
@ -265,19 +269,15 @@ class Consumer(LoggingMixin):
message: Optional[ConsumerStatusShortMessage] = None,
document_id=None,
): # pragma: no cover
payload = {
"filename": os.path.basename(self.filename) if self.filename else None,
"task_id": self.task_id,
"current_progress": current_progress,
"max_progress": max_progress,
"status": status,
"message": message,
self.status_mgr.send_progress(
ProgressStatusOptions.WORKING,
message,
current_progress,
max_progress,
{
"document_id": document_id,
"owner_id": self.override_owner_id if self.override_owner_id else None,
}
async_to_sync(self.channel_layer.group_send)(
"status_updates",
{"type": "status_update", "data": payload},
},
)
def _fail(
@ -291,22 +291,6 @@ class Consumer(LoggingMixin):
self.log.error(log_message or message, exc_info=exc_info)
raise ConsumerError(f"{self.filename}: {log_message or message}") from exception
def __init__(self):
super().__init__()
self.path: Optional[Path] = None
self.original_path: Optional[Path] = None
self.filename = None
self.override_title = None
self.override_correspondent_id = None
self.override_tag_ids = None
self.override_document_type_id = None
self.override_asn = None
self.task_id = None
self.override_owner_id = None
self.override_custom_field_ids = None
self.channel_layer = get_channel_layer()
def pre_check_file_exists(self):
"""
Confirm the input file still exists where it should
@ -486,45 +470,26 @@ class Consumer(LoggingMixin):
exception=e,
)
def try_consume_file(
self,
path: Path,
override_filename=None,
override_title=None,
override_correspondent_id=None,
override_document_type_id=None,
override_tag_ids=None,
override_storage_path_id=None,
task_id=None,
override_created=None,
override_asn=None,
override_owner_id=None,
override_view_users=None,
override_view_groups=None,
override_change_users=None,
override_change_groups=None,
override_custom_field_ids=None,
) -> Document:
def run(self) -> str:
"""
Return the document object if it was successfully created.
"""
self.original_path = Path(path).resolve()
self.filename = override_filename or self.original_path.name
self.override_title = override_title
self.override_correspondent_id = override_correspondent_id
self.override_document_type_id = override_document_type_id
self.override_tag_ids = override_tag_ids
self.override_storage_path_id = override_storage_path_id
self.task_id = task_id or str(uuid.uuid4())
self.override_created = override_created
self.override_asn = override_asn
self.override_owner_id = override_owner_id
self.override_view_users = override_view_users
self.override_view_groups = override_view_groups
self.override_change_users = override_change_users
self.override_change_groups = override_change_groups
self.override_custom_field_ids = override_custom_field_ids
self.original_path = self.input_doc.original_file
self.filename = self.metadata.filename or self.input_doc.original_file.name
self.override_title = self.metadata.title
self.override_correspondent_id = self.metadata.correspondent_id
self.override_document_type_id = self.metadata.document_type_id
self.override_tag_ids = self.metadata.tag_ids
self.override_storage_path_id = self.metadata.storage_path_id
self.override_created = self.metadata.created
self.override_asn = self.metadata.asn
self.override_owner_id = self.metadata.owner_id
self.override_view_users = self.metadata.view_users
self.override_view_groups = self.metadata.view_groups
self.override_change_users = self.metadata.change_users
self.override_change_groups = self.metadata.change_groups
self.override_custom_field_ids = self.metadata.custom_field_ids
self._send_progress(
0,
@ -766,7 +731,7 @@ class Consumer(LoggingMixin):
# Return the most up to date fields
document.refresh_from_db()
return document
return f"Success. New document id {document.pk} created"
def _parse_title_placeholders(self, title: str) -> str:
local_added = timezone.localtime(timezone.now())

3
src/documents/plugins/base.py
View File

@ -67,7 +67,8 @@ class ConsumeTaskPlugin(abc.ABC):
self.status_mgr = status_mgr
self.task_id: Final = task_id
@abc.abstractproperty
@property
@abc.abstractmethod
def able_to_run(self) -> bool:
"""
Return True if the conditions are met for the plugin to run, False otherwise

2
src/documents/plugins/helpers.py
View File

@ -57,7 +57,7 @@ class ProgressManager:
message: str,
current_progress: int,
max_progress: int,
extra_args: Optional[dict[str, Union[str, int]]] = None,
extra_args: Optional[dict[str, Union[str, int, None]]] = None,
) -> None:
# Ensure the layer is open
self.open()

32
src/documents/tasks.py
View File

@ -21,8 +21,7 @@ from documents.barcodes import BarcodePlugin
from documents.caching import clear_document_caches
from documents.classifier import DocumentClassifier
from documents.classifier import load_classifier
from documents.consumer import Consumer
from documents.consumer import ConsumerError
from documents.consumer import ConsumerPlugin
from documents.consumer import WorkflowTriggerPlugin
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides
@ -115,6 +114,7 @@ def consume_file(
CollatePlugin,
BarcodePlugin,
WorkflowTriggerPlugin,
ConsumerPlugin,
]
with ProgressManager(
@ -162,34 +162,6 @@ def consume_file(
finally:
plugin.cleanup()
# continue with consumption if no barcode was found
document = Consumer().try_consume_file(
input_doc.original_file,
override_filename=overrides.filename,
override_title=overrides.title,
override_correspondent_id=overrides.correspondent_id,
override_document_type_id=overrides.document_type_id,
override_tag_ids=overrides.tag_ids,
override_storage_path_id=overrides.storage_path_id,
override_created=overrides.created,
override_asn=overrides.asn,
override_owner_id=overrides.owner_id,
override_view_users=overrides.view_users,
override_view_groups=overrides.view_groups,
override_change_users=overrides.change_users,
override_change_groups=overrides.change_groups,
override_custom_field_ids=overrides.custom_field_ids,
task_id=self.request.id,
)
if document:
return f"Success. New document id {document.pk} created"
else:
raise ConsumerError(
"Unknown error: Returned document was null, but "
"no error message was given.",
)
@shared_task
def sanity_check():

8
src/paperless/settings.py
View File

@ -7,7 +7,6 @@ import re
import tempfile
from os import PathLike
from pathlib import Path
from platform import machine
from typing import Final
from typing import Optional
from typing import Union
@ -112,7 +111,7 @@ def __get_list(
return []
def _parse_redis_url(env_redis: Optional[str]) -> tuple[str]:
def _parse_redis_url(env_redis: Optional[str]) -> tuple[str, str]:
"""
Gets the Redis information from the environment or a default and handles
converting from incompatible django_channels and celery formats.
@ -371,10 +370,7 @@ ASGI_APPLICATION = "paperless.asgi.application"
STATIC_URL = os.getenv("PAPERLESS_STATIC_URL", BASE_URL + "static/")
WHITENOISE_STATIC_PREFIX = "/static/"
if machine().lower() == "aarch64": # pragma: no cover
_static_backend = "django.contrib.staticfiles.storage.StaticFilesStorage"
else:
_static_backend = "whitenoise.storage.CompressedStaticFilesStorage"
_static_backend = "django.contrib.staticfiles.storage.StaticFilesStorage"
STORAGES = {
"staticfiles": {