From 2da5e46386ea77622c576f4d5c7fd87eb77023c3 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Sat, 13 Jan 2024 08:11:14 -0800 Subject: [PATCH 1/3] Refactor file consumption task to allow beginnings of a plugin system (#5367) --- src/documents/barcodes.py | 265 ++++++++++++----------- src/documents/consumer.py | 136 ++++++------ src/documents/double_sided.py | 230 +++++++++++--------- src/documents/plugins/__init__.py | 0 src/documents/plugins/base.py | 131 +++++++++++ src/documents/plugins/helpers.py | 82 +++++++ src/documents/tasks.py | 114 +++++----- src/documents/tests/test_barcodes.py | 237 +++++++------------- src/documents/tests/test_double_sided.py | 12 +- src/documents/tests/test_workflows.py | 19 +- src/documents/tests/utils.py | 72 ++++++ 11 files changed, 767 insertions(+), 531 deletions(-) create mode 100644 src/documents/plugins/__init__.py create mode 100644 src/documents/plugins/base.py create mode 100644 src/documents/plugins/helpers.py diff --git a/src/documents/barcodes.py b/src/documents/barcodes.py index 5a2c3381a..606451f84 100644 --- a/src/documents/barcodes.py +++ b/src/documents/barcodes.py @@ -3,7 +3,6 @@ import re import tempfile from dataclasses import dataclass from pathlib import Path -from typing import Final from typing import Optional from django.conf import settings @@ -15,8 +14,9 @@ from PIL import Image from documents.converters import convert_from_tiff_to_pdf from documents.data_models import ConsumableDocument -from documents.data_models import DocumentMetadataOverrides -from documents.data_models import DocumentSource +from documents.plugins.base import ConsumeTaskPlugin +from documents.plugins.base import StopConsumeTaskError +from documents.plugins.helpers import ProgressStatusOptions from documents.utils import copy_basic_file_stats from documents.utils import copy_file_with_basic_stats @@ -26,7 +26,7 @@ logger = logging.getLogger("paperless.barcodes") @dataclass(frozen=True) class Barcode: """ - Holds the information about a single barcode and its location + Holds the information about a single barcode and its location in a document """ page: int @@ -49,77 +49,111 @@ class Barcode: return self.value.startswith(settings.CONSUMER_ASN_BARCODE_PREFIX) -class BarcodeReader: - def __init__(self, filepath: Path, mime_type: str) -> None: - self.file: Final[Path] = filepath - self.mime: Final[str] = mime_type - self.pdf_file: Path = self.file - self.barcodes: list[Barcode] = [] - self._tiff_conversion_done = False - self.temp_dir: Optional[tempfile.TemporaryDirectory] = None +class BarcodePlugin(ConsumeTaskPlugin): + NAME: str = "BarcodePlugin" + @property + def able_to_run(self) -> bool: + """ + Able to run if: + - ASN from barcode detection is enabled or + - Barcode support is enabled and the mime type is supported + """ if settings.CONSUMER_BARCODE_TIFF_SUPPORT: - self.SUPPORTED_FILE_MIMES = {"application/pdf", "image/tiff"} + supported_mimes = {"application/pdf", "image/tiff"} else: - self.SUPPORTED_FILE_MIMES = {"application/pdf"} + supported_mimes = {"application/pdf"} - def __enter__(self): - if self.supported_mime_type: - self.temp_dir = tempfile.TemporaryDirectory(prefix="paperless-barcodes") - return self + return ( + settings.CONSUMER_ENABLE_ASN_BARCODE or settings.CONSUMER_ENABLE_BARCODES + ) and self.input_doc.mime_type in supported_mimes - def __exit__(self, exc_type, exc_val, exc_tb): - if self.temp_dir is not None: - self.temp_dir.cleanup() - self.temp_dir = None + def setup(self): + self.temp_dir = tempfile.TemporaryDirectory( + dir=self.base_tmp_dir, + prefix="barcode", + ) + self.pdf_file = self.input_doc.original_file + self._tiff_conversion_done = False + self.barcodes: list[Barcode] = [] - @property - def supported_mime_type(self) -> bool: - """ - Return True if the given mime type is supported for barcodes, false otherwise - """ - return self.mime in self.SUPPORTED_FILE_MIMES + def run(self) -> Optional[str]: + # Maybe do the conversion of TIFF to PDF + self.convert_from_tiff_to_pdf() - @property - def asn(self) -> Optional[int]: - """ - Search the parsed barcodes for any ASNs. - The first barcode that starts with CONSUMER_ASN_BARCODE_PREFIX - is considered the ASN to be used. - Returns the detected ASN (or None) - """ - asn = None - - if not self.supported_mime_type: - return None - - # Ensure the barcodes have been read + # Locate any barcodes in the files self.detect() - # get the first barcode that starts with CONSUMER_ASN_BARCODE_PREFIX - asn_text = next( - (x.value for x in self.barcodes if x.is_asn), - None, + # Update/overwrite an ASN if possible + located_asn = self.asn + if located_asn is not None: + logger.info(f"Found ASN in barcode: {located_asn}") + self.metadata.asn = located_asn + + separator_pages = self.get_separation_pages() + if not separator_pages: + return "No pages to split on!" + + # We have pages to split against + + # Note this does NOT use the base_temp_dir, as that will be removed + tmp_dir = Path( + tempfile.mkdtemp( + dir=settings.SCRATCH_DIR, + prefix="paperless-barcode-split-", + ), + ).resolve() + + from documents import tasks + + # Create the split document tasks + for new_document in self.separate_pages(separator_pages): + copy_file_with_basic_stats(new_document, tmp_dir / new_document.name) + + task = tasks.consume_file.delay( + ConsumableDocument( + # Same source, for templates + source=self.input_doc.source, + mailrule_id=self.input_doc.mailrule_id, + # Can't use same folder or the consume might grab it again + original_file=(tmp_dir / new_document.name).resolve(), + ), + # All the same metadata + self.metadata, + ) + logger.info(f"Created new task {task.id} for {new_document.name}") + + # This file is now two or more files + self.input_doc.original_file.unlink() + + msg = "Barcode splitting complete!" + + # Update the progress to complete + self.status_mgr.send_progress(ProgressStatusOptions.SUCCESS, msg, 100, 100) + + # Request the consume task stops + raise StopConsumeTaskError(msg) + + def cleanup(self) -> None: + self.temp_dir.cleanup() + + def convert_from_tiff_to_pdf(self): + """ + May convert a TIFF image into a PDF, if the input is a TIFF and + the TIFF has not been made into a PDF + """ + # Nothing to do, pdf_file is already assigned correctly + if self.input_doc.mime_type != "image/tiff" or self._tiff_conversion_done: + return + + self.pdf_file = convert_from_tiff_to_pdf( + self.input_doc.original_file, + Path(self.temp_dir.name), ) - - if asn_text: - logger.debug(f"Found ASN Barcode: {asn_text}") - # remove the prefix and remove whitespace - asn_text = asn_text[len(settings.CONSUMER_ASN_BARCODE_PREFIX) :].strip() - - # remove non-numeric parts of the remaining string - asn_text = re.sub(r"\D", "", asn_text) - - # now, try parsing the ASN number - try: - asn = int(asn_text) - except ValueError as e: - logger.warning(f"Failed to parse ASN number because: {e}") - - return asn + self._tiff_conversion_done = True @staticmethod - def read_barcodes_zxing(image: Image) -> list[str]: + def read_barcodes_zxing(image: Image.Image) -> list[str]: barcodes = [] import zxingcpp @@ -135,7 +169,7 @@ class BarcodeReader: return barcodes @staticmethod - def read_barcodes_pyzbar(image: Image) -> list[str]: + def read_barcodes_pyzbar(image: Image.Image) -> list[str]: barcodes = [] from pyzbar import pyzbar @@ -154,18 +188,6 @@ class BarcodeReader: return barcodes - def convert_from_tiff_to_pdf(self): - """ - May convert a TIFF image into a PDF, if the input is a TIFF and - the TIFF has not been made into a PDF - """ - # Nothing to do, pdf_file is already assigned correctly - if self.mime != "image/tiff" or self._tiff_conversion_done: - return - - self._tiff_conversion_done = True - self.pdf_file = convert_from_tiff_to_pdf(self.file, Path(self.temp_dir.name)) - def detect(self) -> None: """ Scan all pages of the PDF as images, updating barcodes and the pages @@ -218,10 +240,45 @@ class BarcodeReader: # This file is really borked, allow the consumption to continue # but it may fail further on except Exception as e: # pragma: no cover - logger.exception( + logger.warning( f"Exception during barcode scanning: {e}", ) + @property + def asn(self) -> Optional[int]: + """ + Search the parsed barcodes for any ASNs. + The first barcode that starts with CONSUMER_ASN_BARCODE_PREFIX + is considered the ASN to be used. + Returns the detected ASN (or None) + """ + asn = None + + # Ensure the barcodes have been read + self.detect() + + # get the first barcode that starts with CONSUMER_ASN_BARCODE_PREFIX + asn_text = next( + (x.value for x in self.barcodes if x.is_asn), + None, + ) + + if asn_text: + logger.debug(f"Found ASN Barcode: {asn_text}") + # remove the prefix and remove whitespace + asn_text = asn_text[len(settings.CONSUMER_ASN_BARCODE_PREFIX) :].strip() + + # remove non-numeric parts of the remaining string + asn_text = re.sub(r"\D", "", asn_text) + + # now, try parsing the ASN number + try: + asn = int(asn_text) + except ValueError as e: + logger.warning(f"Failed to parse ASN number because: {e}") + + return asn + def get_separation_pages(self) -> dict[int, bool]: """ Search the parsed barcodes for separators and returns a dict of page @@ -251,7 +308,7 @@ class BarcodeReader: """ document_paths = [] - fname = self.file.stem + fname = self.input_doc.original_file.stem with Pdf.open(self.pdf_file) as input_pdf: # Start with an empty document current_document: list[Page] = [] @@ -292,58 +349,8 @@ class BarcodeReader: with open(savepath, "wb") as out: dst.save(out) - copy_basic_file_stats(self.file, savepath) + copy_basic_file_stats(self.input_doc.original_file, savepath) document_paths.append(savepath) return document_paths - - def separate( - self, - source: DocumentSource, - overrides: DocumentMetadataOverrides, - ) -> bool: - """ - Separates the document, based on barcodes and configuration, creating new - documents as required in the appropriate location. - - Returns True if a split happened, False otherwise - """ - # Do nothing - if not self.supported_mime_type: - logger.warning(f"Unsupported file format for barcode reader: {self.mime}") - return False - - # Does nothing unless needed - self.convert_from_tiff_to_pdf() - - # Actually read the codes, if any - self.detect() - - separator_pages = self.get_separation_pages() - - # Also do nothing - if not separator_pages: - logger.warning("No pages to split on!") - return False - - tmp_dir = Path(tempfile.mkdtemp(prefix="paperless-barcode-split-")).resolve() - - from documents import tasks - - # Create the split document tasks - for new_document in self.separate_pages(separator_pages): - copy_file_with_basic_stats(new_document, tmp_dir / new_document.name) - - tasks.consume_file.delay( - ConsumableDocument( - # Same source, for templates - source=source, - # Can't use same folder or the consume might grab it again - original_file=(tmp_dir / new_document.name).resolve(), - ), - # All the same metadata - overrides, - ) - logger.info("Barcode splitting complete!") - return True diff --git a/src/documents/consumer.py b/src/documents/consumer.py index 06e9f68fc..01b25edea 100644 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -21,7 +21,6 @@ from filelock import FileLock from rest_framework.reverse import reverse from documents.classifier import load_classifier -from documents.data_models import ConsumableDocument from documents.data_models import DocumentMetadataOverrides from documents.file_handling import create_source_path_directory from documents.file_handling import generate_unique_filename @@ -42,12 +41,83 @@ from documents.parsers import ParseError from documents.parsers import get_parser_class_for_mime_type from documents.parsers import parse_date from documents.permissions import set_permissions_for_object +from documents.plugins.base import AlwaysRunPluginMixin +from documents.plugins.base import ConsumeTaskPlugin +from documents.plugins.base import NoCleanupPluginMixin +from documents.plugins.base import NoSetupPluginMixin from documents.signals import document_consumption_finished from documents.signals import document_consumption_started from documents.utils import copy_basic_file_stats from documents.utils import copy_file_with_basic_stats +class WorkflowTriggerPlugin( + NoCleanupPluginMixin, + NoSetupPluginMixin, + AlwaysRunPluginMixin, + ConsumeTaskPlugin, +): + NAME: str = "WorkflowTriggerPlugin" + + def run(self) -> Optional[str]: + """ + Get overrides from matching workflows + """ + overrides = DocumentMetadataOverrides() + for workflow in Workflow.objects.filter(enabled=True).order_by("order"): + template_overrides = DocumentMetadataOverrides() + + if document_matches_workflow( + self.input_doc, + workflow, + WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, + ): + for action in workflow.actions.all(): + if action.assign_title is not None: + template_overrides.title = action.assign_title + if action.assign_tags is not None: + template_overrides.tag_ids = [ + tag.pk for tag in action.assign_tags.all() + ] + if action.assign_correspondent is not None: + template_overrides.correspondent_id = ( + action.assign_correspondent.pk + ) + if action.assign_document_type is not None: + template_overrides.document_type_id = ( + action.assign_document_type.pk + ) + if action.assign_storage_path is not None: + template_overrides.storage_path_id = ( + action.assign_storage_path.pk + ) + if action.assign_owner is not None: + template_overrides.owner_id = action.assign_owner.pk + if action.assign_view_users is not None: + template_overrides.view_users = [ + user.pk for user in action.assign_view_users.all() + ] + if action.assign_view_groups is not None: + template_overrides.view_groups = [ + group.pk for group in action.assign_view_groups.all() + ] + if action.assign_change_users is not None: + template_overrides.change_users = [ + user.pk for user in action.assign_change_users.all() + ] + if action.assign_change_groups is not None: + template_overrides.change_groups = [ + group.pk for group in action.assign_change_groups.all() + ] + if action.assign_custom_fields is not None: + template_overrides.custom_field_ids = [ + field.pk for field in action.assign_custom_fields.all() + ] + + overrides.update(template_overrides) + self.metadata.update(overrides) + + class ConsumerError(Exception): pass @@ -602,70 +672,6 @@ class Consumer(LoggingMixin): return document - def get_workflow_overrides( - self, - input_doc: ConsumableDocument, - ) -> DocumentMetadataOverrides: - """ - Get overrides from matching workflows - """ - overrides = DocumentMetadataOverrides() - for workflow in Workflow.objects.filter(enabled=True).order_by("order"): - template_overrides = DocumentMetadataOverrides() - - if document_matches_workflow( - input_doc, - workflow, - WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, - ): - for action in workflow.actions.all(): - self.log.info( - f"Applying overrides in {action} from {workflow}", - ) - if action.assign_title is not None: - template_overrides.title = action.assign_title - if action.assign_tags is not None: - template_overrides.tag_ids = [ - tag.pk for tag in action.assign_tags.all() - ] - if action.assign_correspondent is not None: - template_overrides.correspondent_id = ( - action.assign_correspondent.pk - ) - if action.assign_document_type is not None: - template_overrides.document_type_id = ( - action.assign_document_type.pk - ) - if action.assign_storage_path is not None: - template_overrides.storage_path_id = ( - action.assign_storage_path.pk - ) - if action.assign_owner is not None: - template_overrides.owner_id = action.assign_owner.pk - if action.assign_view_users is not None: - template_overrides.view_users = [ - user.pk for user in action.assign_view_users.all() - ] - if action.assign_view_groups is not None: - template_overrides.view_groups = [ - group.pk for group in action.assign_view_groups.all() - ] - if action.assign_change_users is not None: - template_overrides.change_users = [ - user.pk for user in action.assign_change_users.all() - ] - if action.assign_change_groups is not None: - template_overrides.change_groups = [ - group.pk for group in action.assign_change_groups.all() - ] - if action.assign_custom_fields is not None: - template_overrides.custom_field_ids = [ - field.pk for field in action.assign_custom_fields.all() - ] - - overrides.update(template_overrides) - return overrides - def _parse_title_placeholders(self, title: str) -> str: local_added = timezone.localtime(timezone.now()) diff --git a/src/documents/double_sided.py b/src/documents/double_sided.py index 5acde1597..bfe66f4fe 100644 --- a/src/documents/double_sided.py +++ b/src/documents/double_sided.py @@ -3,127 +3,145 @@ import logging import os import shutil from pathlib import Path +from typing import Final +from typing import Optional from django.conf import settings from pikepdf import Pdf from documents.consumer import ConsumerError from documents.converters import convert_from_tiff_to_pdf -from documents.data_models import ConsumableDocument +from documents.plugins.base import ConsumeTaskPlugin +from documents.plugins.base import NoCleanupPluginMixin +from documents.plugins.base import NoSetupPluginMixin +from documents.plugins.base import StopConsumeTaskError logger = logging.getLogger("paperless.double_sided") # Hardcoded for now, could be made a configurable setting if needed -TIMEOUT_MINUTES = 30 +TIMEOUT_MINUTES: Final[int] = 30 +TIMEOUT_SECONDS: Final[int] = TIMEOUT_MINUTES * 60 # Used by test cases STAGING_FILE_NAME = "double-sided-staging.pdf" -def collate(input_doc: ConsumableDocument) -> str: - """ - Tries to collate pages from 2 single sided scans of a double sided - document. +class CollatePlugin(NoCleanupPluginMixin, NoSetupPluginMixin, ConsumeTaskPlugin): + NAME: str = "CollatePlugin" - When called with a file, it checks whether or not a staging file - exists, if not, the current file is turned into that staging file - containing the odd numbered pages. - - If a staging file exists, and it is not too old, the current file is - considered to be the second part (the even numbered pages) and it will - collate the pages of both, the pages of the second file will be added - in reverse order, since the ADF will have scanned the pages from bottom - to top. - - Returns a status message on success, or raises a ConsumerError - in case of failure. - """ - - # Make sure scratch dir exists, Consumer might not have run yet - settings.SCRATCH_DIR.mkdir(exist_ok=True) - - if input_doc.mime_type == "application/pdf": - pdf_file = input_doc.original_file - elif ( - input_doc.mime_type == "image/tiff" - and settings.CONSUMER_COLLATE_DOUBLE_SIDED_TIFF_SUPPORT - ): - pdf_file = convert_from_tiff_to_pdf( - input_doc.original_file, - settings.SCRATCH_DIR, - ) - input_doc.original_file.unlink() - else: - raise ConsumerError("Unsupported file type for collation of double-sided scans") - - staging = settings.SCRATCH_DIR / STAGING_FILE_NAME - - valid_staging_exists = False - if staging.exists(): - stats = os.stat(str(staging)) - # if the file is older than the timeout, we don't consider - # it valid - if dt.datetime.now().timestamp() - stats.st_mtime > TIMEOUT_MINUTES * 60: - logger.warning("Outdated double sided staging file exists, deleting it") - os.unlink(str(staging)) - else: - valid_staging_exists = True - - if valid_staging_exists: - try: - # Collate pages from second PDF in reverse order - with Pdf.open(staging) as pdf1, Pdf.open(pdf_file) as pdf2: - pdf2.pages.reverse() - try: - for i, page in enumerate(pdf2.pages): - pdf1.pages.insert(2 * i + 1, page) - except IndexError: - raise ConsumerError( - "This second file (even numbered pages) contains more " - "pages than the first/odd numbered one. This means the " - "two uploaded files don't belong to the same double-" - "sided scan. Please retry, starting with the odd " - "numbered pages again.", - ) - # Merged file has the same path, but without the - # double-sided subdir. Therefore, it is also in the - # consumption dir and will be picked up for processing - old_file = input_doc.original_file - new_file = Path( - *( - part - for part in old_file.with_name( - f"{old_file.stem}-collated.pdf", - ).parts - if part != settings.CONSUMER_COLLATE_DOUBLE_SIDED_SUBDIR_NAME - ), - ) - # If the user didn't create the subdirs yet, do it for them - new_file.parent.mkdir(parents=True, exist_ok=True) - pdf1.save(new_file) - logger.info("Collated documents into new file %s", new_file) - return ( - "Success. Even numbered pages of double sided scan collated " - "with odd pages" - ) - finally: - # Delete staging and recently uploaded file no matter what. - # If any error occurs, the user needs to be able to restart - # the process from scratch; after all, the staging file - # with the odd numbered pages might be the culprit - pdf_file.unlink() - staging.unlink() - - else: - shutil.move(pdf_file, staging) - # update access to modification time so we know if the file - # is outdated when another file gets uploaded - os.utime(staging, (dt.datetime.now().timestamp(),) * 2) - logger.info( - "Got scan with odd numbered pages of double-sided scan, moved it to %s", - staging, - ) + @property + def able_to_run(self) -> bool: return ( - "Received odd numbered pages of double sided scan, waiting up to " - f"{TIMEOUT_MINUTES} minutes for even numbered pages" + settings.CONSUMER_ENABLE_COLLATE_DOUBLE_SIDED + and settings.CONSUMER_COLLATE_DOUBLE_SIDED_SUBDIR_NAME + in self.input_doc.original_file.parts ) + + def run(self) -> Optional[str]: + """ + Tries to collate pages from 2 single sided scans of a double sided + document. + + When called with a file, it checks whether or not a staging file + exists, if not, the current file is turned into that staging file + containing the odd numbered pages. + + If a staging file exists, and it is not too old, the current file is + considered to be the second part (the even numbered pages) and it will + collate the pages of both, the pages of the second file will be added + in reverse order, since the ADF will have scanned the pages from bottom + to top. + + Returns a status message on success, or raises a ConsumerError + in case of failure. + """ + + if self.input_doc.mime_type == "application/pdf": + pdf_file = self.input_doc.original_file + elif ( + self.input_doc.mime_type == "image/tiff" + and settings.CONSUMER_COLLATE_DOUBLE_SIDED_TIFF_SUPPORT + ): + pdf_file = convert_from_tiff_to_pdf( + self.input_doc.original_file, + self.base_tmp_dir, + ) + self.input_doc.original_file.unlink() + else: + raise ConsumerError( + "Unsupported file type for collation of double-sided scans", + ) + + staging: Path = settings.SCRATCH_DIR / STAGING_FILE_NAME + + valid_staging_exists = False + if staging.exists(): + stats = staging.stat() + # if the file is older than the timeout, we don't consider + # it valid + if (dt.datetime.now().timestamp() - stats.st_mtime) > TIMEOUT_SECONDS: + logger.warning("Outdated double sided staging file exists, deleting it") + staging.unlink() + else: + valid_staging_exists = True + + if valid_staging_exists: + try: + # Collate pages from second PDF in reverse order + with Pdf.open(staging) as pdf1, Pdf.open(pdf_file) as pdf2: + pdf2.pages.reverse() + try: + for i, page in enumerate(pdf2.pages): + pdf1.pages.insert(2 * i + 1, page) + except IndexError: + raise ConsumerError( + "This second file (even numbered pages) contains more " + "pages than the first/odd numbered one. This means the " + "two uploaded files don't belong to the same double-" + "sided scan. Please retry, starting with the odd " + "numbered pages again.", + ) + # Merged file has the same path, but without the + # double-sided subdir. Therefore, it is also in the + # consumption dir and will be picked up for processing + old_file = self.input_doc.original_file + new_file = Path( + *( + part + for part in old_file.with_name( + f"{old_file.stem}-collated.pdf", + ).parts + if part + != settings.CONSUMER_COLLATE_DOUBLE_SIDED_SUBDIR_NAME + ), + ) + # If the user didn't create the subdirs yet, do it for them + new_file.parent.mkdir(parents=True, exist_ok=True) + pdf1.save(new_file) + logger.info("Collated documents into new file %s", new_file) + raise StopConsumeTaskError( + "Success. Even numbered pages of double sided scan collated " + "with odd pages", + ) + finally: + # Delete staging and recently uploaded file no matter what. + # If any error occurs, the user needs to be able to restart + # the process from scratch; after all, the staging file + # with the odd numbered pages might be the culprit + pdf_file.unlink() + staging.unlink() + + else: + shutil.move(pdf_file, staging) + # update access to modification time so we know if the file + # is outdated when another file gets uploaded + timestamp = dt.datetime.now().timestamp() + os.utime(staging, (timestamp, timestamp)) + logger.info( + "Got scan with odd numbered pages of double-sided scan, moved it to %s", + staging, + ) + raise StopConsumeTaskError( + "Received odd numbered pages of double sided scan, waiting up to " + f"{TIMEOUT_MINUTES} minutes for even numbered pages", + ) diff --git a/src/documents/plugins/__init__.py b/src/documents/plugins/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/documents/plugins/base.py b/src/documents/plugins/base.py new file mode 100644 index 000000000..aec4887be --- /dev/null +++ b/src/documents/plugins/base.py @@ -0,0 +1,131 @@ +import abc +from pathlib import Path +from typing import Final +from typing import Optional + +from documents.data_models import ConsumableDocument +from documents.data_models import DocumentMetadataOverrides +from documents.plugins.helpers import ProgressManager + + +class StopConsumeTaskError(Exception): + """ + A plugin setup or run may raise this to exit the asynchronous consume task. + + Most likely, this means it has created one or more new tasks to execute instead, + such as when a barcode has been used to create new documents + """ + + def __init__(self, message: str) -> None: + self.message = message + super().__init__(message) + + +class ConsumeTaskPlugin(abc.ABC): + """ + Defines the interface for a plugin for the document consume task + Meanings as per RFC2119 (https://datatracker.ietf.org/doc/html/rfc2119) + + Plugin Implementation + + The plugin SHALL implement property able_to_run and methods setup, run and cleanup. + The plugin property able_to_run SHALL return True if the plugin is able to run, given the conditions, settings and document information. + The plugin property able_to_run MAY be hardcoded to return True. + The plugin setup SHOULD perform any resource creation or additional initialization needed to run the document. + The plugin setup MAY be a non-operation. + The plugin cleanup SHOULD perform resource cleanup, including in the event of an error. + The plugin cleanup MAY be a non-operation. + The plugin run SHALL perform any operations against the document or system state required for the plugin. + The plugin run MAY update the document metadata. + The plugin run MAY return an informational message. + The plugin run MAY raise StopConsumeTaskError to cease any further operations against the document. + + Plugin Manager Implementation + + The plugin manager SHALL provide the plugin with the input document, document metadata, progress manager and a created temporary directory. + The plugin manager SHALL execute the plugin setup, run and cleanup, in that order IF the plugin property able_to_run is True. + The plugin manager SHOULD log the return message of executing a plugin's run. + The plugin manager SHALL always execute the plugin cleanup, IF the plugin property able_to_run is True. + The plugin manager SHALL cease calling plugins and exit the task IF a plugin raises StopConsumeTaskError. + The plugin manager SHOULD return the StopConsumeTaskError message IF a plugin raises StopConsumeTaskError. + """ + + NAME: str = "ConsumeTaskPlugin" + + def __init__( + self, + input_doc: ConsumableDocument, + metadata: DocumentMetadataOverrides, + status_mgr: ProgressManager, + base_tmp_dir: Path, + task_id: str, + ) -> None: + super().__init__() + self.input_doc = input_doc + self.metadata = metadata + self.base_tmp_dir: Final = base_tmp_dir + self.status_mgr = status_mgr + self.task_id: Final = task_id + + @abc.abstractproperty + def able_to_run(self) -> bool: + """ + Return True if the conditions are met for the plugin to run, False otherwise + + If False, setup(), run() and cleanup() will not be called + """ + + @abc.abstractmethod + def setup(self) -> None: + """ + Allows the plugin to perform any additional setup it may need, such as creating + a temporary directory, copying a file somewhere, etc. + + Executed before run() + + In general, this should be the "light" work, not the bulk of processing + """ + + @abc.abstractmethod + def run(self) -> Optional[str]: + """ + The bulk of plugin processing, this does whatever action the plugin is for. + + Executed after setup() and before cleanup() + """ + + @abc.abstractmethod + def cleanup(self) -> None: + """ + Allows the plugin to execute any cleanup it may require + + Executed after run(), even in the case of error + """ + + +class AlwaysRunPluginMixin(ConsumeTaskPlugin): + """ + A plugin which is always able to run + """ + + @property + def able_to_run(self) -> bool: + return True + + +class NoSetupPluginMixin(ConsumeTaskPlugin): + """ + A plugin which requires no setup + """ + + def setup(self) -> None: + pass + + +class NoCleanupPluginMixin(ConsumeTaskPlugin): + """ + A plugin which needs to clean up no files + """ + + def cleanup(self) -> None: + pass diff --git a/src/documents/plugins/helpers.py b/src/documents/plugins/helpers.py new file mode 100644 index 000000000..92fe1255b --- /dev/null +++ b/src/documents/plugins/helpers.py @@ -0,0 +1,82 @@ +import enum +from typing import TYPE_CHECKING +from typing import Optional +from typing import Union + +from asgiref.sync import async_to_sync +from channels.layers import get_channel_layer +from channels_redis.pubsub import RedisPubSubChannelLayer + + +class ProgressStatusOptions(str, enum.Enum): + STARTED = "STARTED" + WORKING = "WORKING" + SUCCESS = "SUCCESS" + FAILED = "FAILED" + + +class ProgressManager: + """ + Handles sending of progress information via the channel layer, with proper management + of the open/close of the layer to ensure messages go out and everything is cleaned up + """ + + def __init__(self, filename: str, task_id: Optional[str] = None) -> None: + self.filename = filename + self._channel: Optional[RedisPubSubChannelLayer] = None + self.task_id = task_id + + def __enter__(self): + self.open() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def open(self) -> None: + """ + If not already opened, gets the default channel layer + opened and ready to send messages + """ + if self._channel is None: + self._channel = get_channel_layer() + + def close(self) -> None: + """ + If it was opened, flushes the channel layer + """ + if self._channel is not None: + async_to_sync(self._channel.flush) + self._channel = None + + def send_progress( + self, + status: ProgressStatusOptions, + message: str, + current_progress: int, + max_progress: int, + extra_args: Optional[dict[str, Union[str, int]]] = None, + ) -> None: + # Ensure the layer is open + self.open() + + # Just for IDEs + if TYPE_CHECKING: + assert self._channel is not None + + payload = { + "type": "status_update", + "data": { + "filename": self.filename, + "task_id": self.task_id, + "current_progress": current_progress, + "max_progress": max_progress, + "status": status, + "message": message, + }, + } + if extra_args is not None: + payload["data"].update(extra_args) + + # Construct and send the update + async_to_sync(self._channel.group_send)("status_updates", payload) diff --git a/src/documents/tasks.py b/src/documents/tasks.py index abb9cd39d..a83c2e6cd 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -2,30 +2,30 @@ import hashlib import logging import shutil import uuid +from pathlib import Path +from tempfile import TemporaryDirectory from typing import Optional import tqdm -from asgiref.sync import async_to_sync from celery import Task from celery import shared_task -from channels.layers import get_channel_layer from django.conf import settings from django.db import transaction from django.db.models.signals import post_save from filelock import FileLock -from redis.exceptions import ConnectionError from whoosh.writing import AsyncWriter from documents import index from documents import sanity_checker -from documents.barcodes import BarcodeReader +from documents.barcodes import BarcodePlugin from documents.classifier import DocumentClassifier from documents.classifier import load_classifier from documents.consumer import Consumer from documents.consumer import ConsumerError +from documents.consumer import WorkflowTriggerPlugin from documents.data_models import ConsumableDocument from documents.data_models import DocumentMetadataOverrides -from documents.double_sided import collate +from documents.double_sided import CollatePlugin from documents.file_handling import create_source_path_directory from documents.file_handling import generate_unique_filename from documents.models import Correspondent @@ -35,6 +35,10 @@ from documents.models import StoragePath from documents.models import Tag from documents.parsers import DocumentParser from documents.parsers import get_parser_class_for_mime_type +from documents.plugins.base import ConsumeTaskPlugin +from documents.plugins.base import ProgressManager +from documents.plugins.base import StopConsumeTaskError +from documents.plugins.helpers import ProgressStatusOptions from documents.sanity_checker import SanityCheckFailedException from documents.signals import document_updated @@ -102,70 +106,60 @@ def consume_file( input_doc: ConsumableDocument, overrides: Optional[DocumentMetadataOverrides] = None, ): - def send_progress(status="SUCCESS", message="finished"): - payload = { - "filename": overrides.filename or input_doc.original_file.name, - "task_id": None, - "current_progress": 100, - "max_progress": 100, - "status": status, - "message": message, - } - try: - async_to_sync(get_channel_layer().group_send)( - "status_updates", - {"type": "status_update", "data": payload}, - ) - except ConnectionError as e: - logger.warning(f"ConnectionError on status send: {e!s}") - # Default no overrides if overrides is None: overrides = DocumentMetadataOverrides() - # Handle collation of double-sided documents scanned in two parts - if settings.CONSUMER_ENABLE_COLLATE_DOUBLE_SIDED and ( - settings.CONSUMER_COLLATE_DOUBLE_SIDED_SUBDIR_NAME - in input_doc.original_file.parts - ): - try: - msg = collate(input_doc) - send_progress(message=msg) - return msg - except ConsumerError as e: - send_progress(status="FAILURE", message=e.args[0]) - raise e + plugins: list[type[ConsumeTaskPlugin]] = [ + CollatePlugin, + BarcodePlugin, + WorkflowTriggerPlugin, + ] - # read all barcodes in the current document - if settings.CONSUMER_ENABLE_BARCODES or settings.CONSUMER_ENABLE_ASN_BARCODE: - with BarcodeReader(input_doc.original_file, input_doc.mime_type) as reader: - if settings.CONSUMER_ENABLE_BARCODES and reader.separate( - input_doc.source, + with ProgressManager( + overrides.filename or input_doc.original_file.name, + self.request.id, + ) as status_mgr, TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir: + tmp_dir = Path(tmp_dir) + for plugin_class in plugins: + plugin_name = plugin_class.NAME + + plugin = plugin_class( + input_doc, overrides, - ): - # notify the sender, otherwise the progress bar - # in the UI stays stuck - send_progress() - # consuming stops here, since the original document with - # the barcodes has been split and will be consumed separately - input_doc.original_file.unlink() - return "File successfully split" + status_mgr, + tmp_dir, + self.request.id, + ) - # try reading the ASN from barcode - if ( - settings.CONSUMER_ENABLE_ASN_BARCODE - and (located_asn := reader.asn) is not None - ): - # Note this will take precedence over an API provided ASN - # But it's from a physical barcode, so that's good - overrides.asn = located_asn - logger.info(f"Found ASN in barcode: {overrides.asn}") + if not plugin.able_to_run: + logger.debug(f"Skipping plugin {plugin_name}") + continue - template_overrides = Consumer().get_workflow_overrides( - input_doc=input_doc, - ) + try: + logger.debug(f"Executing plugin {plugin_name}") + plugin.setup() - overrides.update(template_overrides) + msg = plugin.run() + + if msg is not None: + logger.info(f"{plugin_name} completed with: {msg}") + else: + logger.info(f"{plugin_name} completed with no message") + + overrides = plugin.metadata + + except StopConsumeTaskError as e: + logger.info(f"{plugin_name} requested task exit: {e.message}") + return e.message + + except Exception as e: + logger.exception(f"{plugin_name} failed: {e}") + status_mgr.send_progress(ProgressStatusOptions.FAILED, f"{e}", 100, 100) + raise + + finally: + plugin.cleanup() # continue with consumption if no barcode was found document = Consumer().try_consume_file( diff --git a/src/documents/tests/test_barcodes.py b/src/documents/tests/test_barcodes.py index e4d8ccc57..4552a2b77 100644 --- a/src/documents/tests/test_barcodes.py +++ b/src/documents/tests/test_barcodes.py @@ -1,4 +1,7 @@ import shutil +from collections.abc import Generator +from contextlib import contextmanager +from pathlib import Path from unittest import mock import pytest @@ -7,14 +10,13 @@ from django.test import TestCase from django.test import override_settings from documents import tasks -from documents.barcodes import BarcodeReader -from documents.consumer import ConsumerError +from documents.barcodes import BarcodePlugin from documents.data_models import ConsumableDocument from documents.data_models import DocumentMetadataOverrides from documents.data_models import DocumentSource -from documents.models import Document from documents.tests.utils import DirectoriesMixin from documents.tests.utils import DocumentConsumeDelayMixin +from documents.tests.utils import DummyProgressManager from documents.tests.utils import FileSystemAssertsMixin from documents.tests.utils import SampleDirMixin @@ -26,8 +28,29 @@ except ImportError: HAS_ZXING_LIB = False +class GetReaderPluginMixin: + @contextmanager + def get_reader(self, filepath: Path) -> Generator[BarcodePlugin, None, None]: + reader = BarcodePlugin( + ConsumableDocument(DocumentSource.ConsumeFolder, original_file=filepath), + DocumentMetadataOverrides(), + DummyProgressManager(filepath.name, None), + self.dirs.scratch_dir, + "task-id", + ) + reader.setup() + yield reader + reader.cleanup() + + @override_settings(CONSUMER_BARCODE_SCANNER="PYZBAR") -class TestBarcode(DirectoriesMixin, FileSystemAssertsMixin, SampleDirMixin, TestCase): +class TestBarcode( + DirectoriesMixin, + FileSystemAssertsMixin, + SampleDirMixin, + GetReaderPluginMixin, + TestCase, +): def test_scan_file_for_separating_barcodes(self): """ GIVEN: @@ -39,7 +62,7 @@ class TestBarcode(DirectoriesMixin, FileSystemAssertsMixin, SampleDirMixin, Test """ test_file = self.BARCODE_SAMPLE_DIR / "patch-code-t.pdf" - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: reader.detect() separator_page_numbers = reader.get_separation_pages() @@ -60,7 +83,7 @@ class TestBarcode(DirectoriesMixin, FileSystemAssertsMixin, SampleDirMixin, Test """ test_file = self.BARCODE_SAMPLE_DIR / "patch-code-t-middle.tiff" - with BarcodeReader(test_file, "image/tiff") as reader: + with self.get_reader(test_file) as reader: reader.detect() separator_page_numbers = reader.get_separation_pages() @@ -80,7 +103,7 @@ class TestBarcode(DirectoriesMixin, FileSystemAssertsMixin, SampleDirMixin, Test """ test_file = self.BARCODE_SAMPLE_DIR / "patch-code-t-middle-alpha.tiff" - with BarcodeReader(test_file, "image/tiff") as reader: + with self.get_reader(test_file) as reader: reader.detect() separator_page_numbers = reader.get_separation_pages() @@ -97,7 +120,7 @@ class TestBarcode(DirectoriesMixin, FileSystemAssertsMixin, SampleDirMixin, Test - No pages to split on """ test_file = self.SAMPLE_DIR / "simple.pdf" - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: reader.detect() separator_page_numbers = reader.get_separation_pages() @@ -115,7 +138,7 @@ class TestBarcode(DirectoriesMixin, FileSystemAssertsMixin, SampleDirMixin, Test """ test_file = self.BARCODE_SAMPLE_DIR / "patch-code-t-middle.pdf" - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: reader.detect() separator_page_numbers = reader.get_separation_pages() @@ -133,7 +156,7 @@ class TestBarcode(DirectoriesMixin, FileSystemAssertsMixin, SampleDirMixin, Test """ test_file = self.BARCODE_SAMPLE_DIR / "several-patcht-codes.pdf" - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: reader.detect() separator_page_numbers = reader.get_separation_pages() @@ -158,7 +181,7 @@ class TestBarcode(DirectoriesMixin, FileSystemAssertsMixin, SampleDirMixin, Test ]: test_file = self.BARCODE_SAMPLE_DIR / test_file - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: reader.detect() separator_page_numbers = reader.get_separation_pages() @@ -177,7 +200,7 @@ class TestBarcode(DirectoriesMixin, FileSystemAssertsMixin, SampleDirMixin, Test """ test_file = self.BARCODE_SAMPLE_DIR / "patch-code-t-middle-unreadable.pdf" - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: reader.detect() separator_page_numbers = reader.get_separation_pages() @@ -195,7 +218,7 @@ class TestBarcode(DirectoriesMixin, FileSystemAssertsMixin, SampleDirMixin, Test """ test_file = self.BARCODE_SAMPLE_DIR / "barcode-fax-image.pdf" - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: reader.detect() separator_page_numbers = reader.get_separation_pages() @@ -214,7 +237,7 @@ class TestBarcode(DirectoriesMixin, FileSystemAssertsMixin, SampleDirMixin, Test """ test_file = self.BARCODE_SAMPLE_DIR / "patch-code-t-qr.pdf" - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: reader.detect() separator_page_numbers = reader.get_separation_pages() @@ -234,7 +257,7 @@ class TestBarcode(DirectoriesMixin, FileSystemAssertsMixin, SampleDirMixin, Test """ test_file = self.BARCODE_SAMPLE_DIR / "barcode-39-custom.pdf" - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: reader.detect() separator_page_numbers = reader.get_separation_pages() @@ -255,7 +278,7 @@ class TestBarcode(DirectoriesMixin, FileSystemAssertsMixin, SampleDirMixin, Test """ test_file = self.BARCODE_SAMPLE_DIR / "barcode-qr-custom.pdf" - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: reader.detect() separator_page_numbers = reader.get_separation_pages() @@ -276,7 +299,7 @@ class TestBarcode(DirectoriesMixin, FileSystemAssertsMixin, SampleDirMixin, Test """ test_file = self.BARCODE_SAMPLE_DIR / "barcode-128-custom.pdf" - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: reader.detect() separator_page_numbers = reader.get_separation_pages() @@ -296,7 +319,7 @@ class TestBarcode(DirectoriesMixin, FileSystemAssertsMixin, SampleDirMixin, Test """ test_file = self.BARCODE_SAMPLE_DIR / "barcode-39-custom.pdf" - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: reader.detect() separator_page_numbers = reader.get_separation_pages() @@ -315,7 +338,7 @@ class TestBarcode(DirectoriesMixin, FileSystemAssertsMixin, SampleDirMixin, Test """ test_file = self.BARCODE_SAMPLE_DIR / "many-qr-codes.pdf" - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: reader.detect() separator_page_numbers = reader.get_separation_pages() @@ -334,7 +357,7 @@ class TestBarcode(DirectoriesMixin, FileSystemAssertsMixin, SampleDirMixin, Test """ test_file = self.SAMPLE_DIR / "password-is-test.pdf" with self.assertLogs("paperless.barcodes", level="WARNING") as cm: - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: reader.detect() warning = cm.output[0] expected_str = "WARNING:paperless.barcodes:File is likely password protected, not checking for barcodes" @@ -356,7 +379,7 @@ class TestBarcode(DirectoriesMixin, FileSystemAssertsMixin, SampleDirMixin, Test """ test_file = self.BARCODE_SAMPLE_DIR / "patch-code-t-middle.pdf" - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: documents = reader.separate_pages({1: False}) self.assertEqual(reader.pdf_file, test_file) @@ -373,7 +396,7 @@ class TestBarcode(DirectoriesMixin, FileSystemAssertsMixin, SampleDirMixin, Test """ test_file = self.BARCODE_SAMPLE_DIR / "patch-code-t-double.pdf" - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: documents = reader.separate_pages({1: False, 2: False}) self.assertEqual(len(documents), 2) @@ -385,32 +408,18 @@ class TestBarcode(DirectoriesMixin, FileSystemAssertsMixin, SampleDirMixin, Test WHEN: - No separation pages are provided THEN: - - No new documents are produced - - A warning is logged + - Nothing happens """ test_file = self.SAMPLE_DIR / "simple.pdf" - with self.assertLogs("paperless.barcodes", level="WARNING") as cm: - with BarcodeReader(test_file, "application/pdf") as reader: - self.assertFalse( - reader.separate( - DocumentSource.ApiUpload, - DocumentMetadataOverrides(), - ), - ) - self.assertEqual( - cm.output, - [ - "WARNING:paperless.barcodes:No pages to split on!", - ], - ) + with self.get_reader(test_file) as reader: + self.assertEqual("No pages to split on!", reader.run()) @override_settings( CONSUMER_ENABLE_BARCODES=True, CONSUMER_BARCODE_TIFF_SUPPORT=True, ) - @mock.patch("documents.consumer.Consumer.try_consume_file") - def test_consume_barcode_unsupported_jpg_file(self, m): + def test_consume_barcode_unsupported_jpg_file(self): """ GIVEN: - JPEG image as input @@ -422,35 +431,8 @@ class TestBarcode(DirectoriesMixin, FileSystemAssertsMixin, SampleDirMixin, Test """ test_file = self.SAMPLE_DIR / "simple.jpg" - dst = settings.SCRATCH_DIR / "simple.jpg" - shutil.copy(test_file, dst) - - with self.assertLogs("paperless.barcodes", level="WARNING") as cm: - self.assertIn( - "Success", - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=dst, - ), - None, - ), - ) - - self.assertListEqual( - cm.output, - [ - "WARNING:paperless.barcodes:Unsupported file format for barcode reader: image/jpeg", - ], - ) - m.assert_called_once() - - args, kwargs = m.call_args - self.assertIsNone(kwargs["override_filename"]) - self.assertIsNone(kwargs["override_title"]) - self.assertIsNone(kwargs["override_correspondent_id"]) - self.assertIsNone(kwargs["override_document_type_id"]) - self.assertIsNone(kwargs["override_tag_ids"]) + with self.get_reader(test_file) as reader: + self.assertFalse(reader.able_to_run) @override_settings( CONSUMER_ENABLE_BARCODES=True, @@ -467,7 +449,7 @@ class TestBarcode(DirectoriesMixin, FileSystemAssertsMixin, SampleDirMixin, Test """ test_file = self.BARCODE_SAMPLE_DIR / "split-by-asn-2.pdf" - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: reader.detect() separator_page_numbers = reader.get_separation_pages() @@ -504,7 +486,7 @@ class TestBarcode(DirectoriesMixin, FileSystemAssertsMixin, SampleDirMixin, Test """ test_file = self.BARCODE_SAMPLE_DIR / "split-by-asn-1.pdf" - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: reader.detect() separator_page_numbers = reader.get_separation_pages() @@ -550,7 +532,7 @@ class TestBarcodeNewConsume( overrides = DocumentMetadataOverrides(tag_ids=[1, 2, 9]) - with mock.patch("documents.tasks.async_to_sync") as progress_mocker: + with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): self.assertEqual( tasks.consume_file( ConsumableDocument( @@ -559,10 +541,8 @@ class TestBarcodeNewConsume( ), overrides, ), - "File successfully split", + "Barcode splitting complete!", ) - # We let the consumer know progress is done - progress_mocker.assert_called_once() # 2 new document consume tasks created self.assertEqual(self.consume_file_mock.call_count, 2) @@ -580,7 +560,20 @@ class TestBarcodeNewConsume( self.assertEqual(overrides, new_doc_overrides) -class TestAsnBarcode(DirectoriesMixin, SampleDirMixin, TestCase): +class TestAsnBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, TestCase): + @contextmanager + def get_reader(self, filepath: Path) -> BarcodePlugin: + reader = BarcodePlugin( + ConsumableDocument(DocumentSource.ConsumeFolder, original_file=filepath), + DocumentMetadataOverrides(), + DummyProgressManager(filepath.name, None), + self.dirs.scratch_dir, + "task-id", + ) + reader.setup() + yield reader + reader.cleanup() + @override_settings(CONSUMER_ASN_BARCODE_PREFIX="CUSTOM-PREFIX-") def test_scan_file_for_asn_custom_prefix(self): """ @@ -594,7 +587,7 @@ class TestAsnBarcode(DirectoriesMixin, SampleDirMixin, TestCase): - The ASN integer value is correct """ test_file = self.BARCODE_SAMPLE_DIR / "barcode-39-asn-custom-prefix.pdf" - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: asn = reader.asn self.assertEqual(reader.pdf_file, test_file) @@ -613,7 +606,7 @@ class TestAsnBarcode(DirectoriesMixin, SampleDirMixin, TestCase): """ test_file = self.BARCODE_SAMPLE_DIR / "barcode-39-asn-123.pdf" - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: asn = reader.asn self.assertEqual(reader.pdf_file, test_file) @@ -630,55 +623,12 @@ class TestAsnBarcode(DirectoriesMixin, SampleDirMixin, TestCase): """ test_file = self.BARCODE_SAMPLE_DIR / "patch-code-t.pdf" - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: asn = reader.asn self.assertEqual(reader.pdf_file, test_file) self.assertEqual(asn, None) - @override_settings(CONSUMER_ENABLE_ASN_BARCODE=True) - def test_scan_file_for_asn_already_exists(self): - """ - GIVEN: - - PDF with an ASN barcode - - ASN value already exists - WHEN: - - File is scanned for barcodes - THEN: - - ASN is retrieved from the document - - Consumption fails - """ - - Document.objects.create( - title="WOW", - content="the content", - archive_serial_number=123, - checksum="456", - mime_type="application/pdf", - ) - - test_file = self.BARCODE_SAMPLE_DIR / "barcode-39-asn-123.pdf" - - dst = settings.SCRATCH_DIR / "barcode-39-asn-123.pdf" - shutil.copy(test_file, dst) - - with mock.patch("documents.consumer.Consumer._send_progress"): - with self.assertRaises(ConsumerError) as cm, self.assertLogs( - "paperless.consumer", - level="ERROR", - ) as logs_cm: - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=dst, - ), - None, - ) - self.assertIn("Not consuming barcode-39-asn-123.pdf", str(cm.exception)) - error_str = logs_cm.output[0] - expected_str = "ERROR:paperless.consumer:Not consuming barcode-39-asn-123.pdf: Given ASN already exists!" - self.assertEqual(expected_str, error_str) - def test_scan_file_for_asn_barcode_invalid(self): """ GIVEN: @@ -692,7 +642,7 @@ class TestAsnBarcode(DirectoriesMixin, SampleDirMixin, TestCase): """ test_file = self.BARCODE_SAMPLE_DIR / "barcode-39-asn-invalid.pdf" - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: asn = reader.asn self.assertEqual(reader.pdf_file, test_file) @@ -718,7 +668,9 @@ class TestAsnBarcode(DirectoriesMixin, SampleDirMixin, TestCase): dst = settings.SCRATCH_DIR / "barcode-39-asn-123.pdf" shutil.copy(test_file, dst) - with mock.patch("documents.consumer.Consumer.try_consume_file") as mocked_call: + with mock.patch( + "documents.consumer.Consumer.try_consume_file", + ) as mocked_consumer: tasks.consume_file( ConsumableDocument( source=DocumentSource.ConsumeFolder, @@ -726,40 +678,11 @@ class TestAsnBarcode(DirectoriesMixin, SampleDirMixin, TestCase): ), None, ) - - args, kwargs = mocked_call.call_args + mocked_consumer.assert_called_once() + args, kwargs = mocked_consumer.call_args self.assertEqual(kwargs["override_asn"], 123) - @override_settings(CONSUMER_ENABLE_ASN_BARCODE=True) - def test_asn_too_large(self): - """ - GIVEN: - - ASN from barcode enabled - - Barcode contains too large an ASN value - WHEN: - - ASN from barcode checked for correctness - THEN: - - Exception is raised regarding size limits - """ - src = self.BARCODE_SAMPLE_DIR / "barcode-128-asn-too-large.pdf" - - dst = self.dirs.scratch_dir / "barcode-128-asn-too-large.pdf" - shutil.copy(src, dst) - - input_doc = ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=dst, - ) - - with mock.patch("documents.consumer.Consumer._send_progress"): - self.assertRaisesMessage( - ConsumerError, - "Given ASN 4294967296 is out of range [0, 4,294,967,295]", - tasks.consume_file, - input_doc, - ) - @override_settings(CONSUMER_BARCODE_SCANNER="PYZBAR") def test_scan_file_for_qrcode_without_upscale(self): """ @@ -774,7 +697,7 @@ class TestAsnBarcode(DirectoriesMixin, SampleDirMixin, TestCase): test_file = self.BARCODE_SAMPLE_DIR / "barcode-qr-asn-000123-upscale-dpi.pdf" - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: reader.detect() self.assertEqual(len(reader.barcodes), 0) @@ -796,7 +719,7 @@ class TestAsnBarcode(DirectoriesMixin, SampleDirMixin, TestCase): test_file = self.BARCODE_SAMPLE_DIR / "barcode-qr-asn-000123-upscale-dpi.pdf" - with BarcodeReader(test_file, "application/pdf") as reader: + with self.get_reader(test_file) as reader: reader.detect() self.assertEqual(len(reader.barcodes), 1) self.assertEqual(reader.asn, 123) diff --git a/src/documents/tests/test_double_sided.py b/src/documents/tests/test_double_sided.py index 88cbe7d87..c66594491 100644 --- a/src/documents/tests/test_double_sided.py +++ b/src/documents/tests/test_double_sided.py @@ -17,6 +17,7 @@ from documents.data_models import DocumentSource from documents.double_sided import STAGING_FILE_NAME from documents.double_sided import TIMEOUT_MINUTES from documents.tests.utils import DirectoriesMixin +from documents.tests.utils import DummyProgressManager from documents.tests.utils import FileSystemAssertsMixin @@ -42,9 +43,10 @@ class TestDoubleSided(DirectoriesMixin, FileSystemAssertsMixin, TestCase): dst = self.dirs.double_sided_dir / dstname dst.parent.mkdir(parents=True, exist_ok=True) shutil.copy(src, dst) - with mock.patch("documents.tasks.async_to_sync"), mock.patch( - "documents.consumer.async_to_sync", - ): + with mock.patch( + "documents.tasks.ProgressManager", + DummyProgressManager, + ), mock.patch("documents.consumer.async_to_sync"): msg = tasks.consume_file( ConsumableDocument( source=DocumentSource.ConsumeFolder, @@ -211,7 +213,7 @@ class TestDoubleSided(DirectoriesMixin, FileSystemAssertsMixin, TestCase): """ msg = self.consume_file("simple.pdf", Path("..") / "simple.pdf") self.assertIsNotFile(self.staging_file) - self.assertRegex(msg, "Success. New document .* created") + self.assertRegex(msg, r"Success. New document id \d+ created") def test_subdirectory_upload(self): """ @@ -250,4 +252,4 @@ class TestDoubleSided(DirectoriesMixin, FileSystemAssertsMixin, TestCase): """ msg = self.consume_file("simple.pdf") self.assertIsNotFile(self.staging_file) - self.assertRegex(msg, "Success. New document .* created") + self.assertRegex(msg, r"Success. New document id \d+ created") diff --git a/src/documents/tests/test_workflows.py b/src/documents/tests/test_workflows.py index b688eecc9..e92a00682 100644 --- a/src/documents/tests/test_workflows.py +++ b/src/documents/tests/test_workflows.py @@ -24,6 +24,7 @@ from documents.models import WorkflowAction from documents.models import WorkflowTrigger from documents.signals import document_consumption_finished from documents.tests.utils import DirectoriesMixin +from documents.tests.utils import DummyProgressManager from documents.tests.utils import FileSystemAssertsMixin from paperless_mail.models import MailAccount from paperless_mail.models import MailRule @@ -126,7 +127,7 @@ class TestWorkflows(DirectoriesMixin, FileSystemAssertsMixin, APITestCase): test_file = self.SAMPLE_DIR / "simple.pdf" - with mock.patch("documents.tasks.async_to_sync"): + with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): with self.assertLogs("paperless.matching", level="INFO") as cm: tasks.consume_file( ConsumableDocument( @@ -203,7 +204,7 @@ class TestWorkflows(DirectoriesMixin, FileSystemAssertsMixin, APITestCase): w.save() test_file = self.SAMPLE_DIR / "simple.pdf" - with mock.patch("documents.tasks.async_to_sync"): + with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): with self.assertLogs("paperless.matching", level="INFO") as cm: tasks.consume_file( ConsumableDocument( @@ -294,7 +295,7 @@ class TestWorkflows(DirectoriesMixin, FileSystemAssertsMixin, APITestCase): test_file = self.SAMPLE_DIR / "simple.pdf" - with mock.patch("documents.tasks.async_to_sync"): + with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): with self.assertLogs("paperless.matching", level="INFO") as cm: tasks.consume_file( ConsumableDocument( @@ -356,7 +357,7 @@ class TestWorkflows(DirectoriesMixin, FileSystemAssertsMixin, APITestCase): test_file = self.SAMPLE_DIR / "simple.pdf" - with mock.patch("documents.tasks.async_to_sync"): + with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): with self.assertLogs("paperless.matching", level="DEBUG") as cm: tasks.consume_file( ConsumableDocument( @@ -407,7 +408,7 @@ class TestWorkflows(DirectoriesMixin, FileSystemAssertsMixin, APITestCase): test_file = self.SAMPLE_DIR / "simple.pdf" - with mock.patch("documents.tasks.async_to_sync"): + with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): with self.assertLogs("paperless.matching", level="DEBUG") as cm: tasks.consume_file( ConsumableDocument( @@ -468,7 +469,7 @@ class TestWorkflows(DirectoriesMixin, FileSystemAssertsMixin, APITestCase): test_file = self.SAMPLE_DIR / "simple.pdf" - with mock.patch("documents.tasks.async_to_sync"): + with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): with self.assertLogs("paperless.matching", level="DEBUG") as cm: tasks.consume_file( ConsumableDocument( @@ -529,7 +530,7 @@ class TestWorkflows(DirectoriesMixin, FileSystemAssertsMixin, APITestCase): test_file = self.SAMPLE_DIR / "simple.pdf" - with mock.patch("documents.tasks.async_to_sync"): + with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): with self.assertLogs("paperless.matching", level="DEBUG") as cm: tasks.consume_file( ConsumableDocument( @@ -591,7 +592,7 @@ class TestWorkflows(DirectoriesMixin, FileSystemAssertsMixin, APITestCase): test_file = self.SAMPLE_DIR / "simple.pdf" - with mock.patch("documents.tasks.async_to_sync"): + with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): with self.assertLogs("paperless.matching", level="DEBUG") as cm: tasks.consume_file( ConsumableDocument( @@ -686,7 +687,7 @@ class TestWorkflows(DirectoriesMixin, FileSystemAssertsMixin, APITestCase): test_file = self.SAMPLE_DIR / "simple.pdf" - with mock.patch("documents.tasks.async_to_sync"): + with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): with self.assertLogs("paperless.matching", level="INFO") as cm: tasks.consume_file( ConsumableDocument( diff --git a/src/documents/tests/utils.py b/src/documents/tests/utils.py index 0b6d8fcad..4c3305d13 100644 --- a/src/documents/tests/utils.py +++ b/src/documents/tests/utils.py @@ -9,6 +9,7 @@ from os import PathLike from pathlib import Path from typing import Any from typing import Callable +from typing import Optional from typing import Union from unittest import mock @@ -23,6 +24,7 @@ from django.test import override_settings from documents.data_models import ConsumableDocument from documents.data_models import DocumentMetadataOverrides from documents.parsers import ParseError +from documents.plugins.helpers import ProgressStatusOptions def setup_directories(): @@ -146,6 +148,11 @@ def util_call_with_backoff( class DirectoriesMixin: + """ + Creates and overrides settings for all folders and paths, then ensures + they are cleaned up on exit + """ + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.dirs = None @@ -160,6 +167,10 @@ class DirectoriesMixin: class FileSystemAssertsMixin: + """ + Utilities for checks various state information of the file system + """ + def assertIsFile(self, path: Union[PathLike, str]): self.assertTrue(Path(path).resolve().is_file(), f"File does not exist: {path}") @@ -188,6 +199,11 @@ class FileSystemAssertsMixin: class ConsumerProgressMixin: + """ + Mocks the Consumer _send_progress, preventing attempts to connect to Redis + and allowing access to its calls for verification + """ + def setUp(self) -> None: self.send_progress_patcher = mock.patch( "documents.consumer.Consumer._send_progress", @@ -310,3 +326,59 @@ class SampleDirMixin: SAMPLE_DIR = Path(__file__).parent / "samples" BARCODE_SAMPLE_DIR = SAMPLE_DIR / "barcodes" + + +class DummyProgressManager: + """ + A dummy handler for progress management that doesn't actually try to + connect to Redis. Payloads are stored for test assertions if needed. + + Use it with + mock.patch("documents.tasks.ProgressManager", DummyProgressManager) + """ + + def __init__(self, filename: str, task_id: Optional[str] = None) -> None: + self.filename = filename + self.task_id = task_id + print("hello world") + self.payloads = [] + + def __enter__(self): + self.open() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def open(self) -> None: + pass + + def close(self) -> None: + pass + + def send_progress( + self, + status: ProgressStatusOptions, + message: str, + current_progress: int, + max_progress: int, + extra_args: Optional[dict[str, Union[str, int]]] = None, + ) -> None: + # Ensure the layer is open + self.open() + + payload = { + "type": "status_update", + "data": { + "filename": self.filename, + "task_id": self.task_id, + "current_progress": current_progress, + "max_progress": max_progress, + "status": status, + "message": message, + }, + } + if extra_args is not None: + payload["data"].update(extra_args) + + self.payloads.append(payload) From 2a6e79acc8c1c105b3c7b36aa0c07413adcac0d8 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Sat, 13 Jan 2024 11:57:25 -0800 Subject: [PATCH 2/3] Feature: app branding (#5357) --- docs/configuration.md | 16 +- src-ui/messages.xlf | 217 +++++++++++------- src-ui/src/app/app.module.ts | 2 + .../admin/config/config.component.html | 1 + .../admin/config/config.component.spec.ts | 40 ++++ .../admin/config/config.component.ts | 28 ++- .../app-frame/app-frame.component.html | 16 +- .../app-frame/app-frame.component.scss | 11 +- .../app-frame/app-frame.component.ts | 4 + .../common/input/file/file.component.html | 37 +++ .../common/input/file/file.component.scss | 0 .../common/input/file/file.component.spec.ts | 41 ++++ .../common/input/file/file.component.ts | 53 +++++ .../common/logo/logo.component.html | 40 ++-- .../common/logo/logo.component.spec.ts | 11 + .../components/common/logo/logo.component.ts | 14 ++ .../dashboard/dashboard.component.ts | 6 +- .../not-found/not-found.component.spec.ts | 2 + src-ui/src/app/data/paperless-config.ts | 18 ++ src-ui/src/app/data/ui-settings.ts | 12 + .../src/app/services/config.service.spec.ts | 22 ++ src-ui/src/app/services/config.service.ts | 14 ++ .../src/app/services/settings.service.spec.ts | 12 + src-ui/src/app/services/settings.service.ts | 3 + src-ui/src/styles.scss | 3 + src/documents/tests/test_api_app_config.py | 59 +++++ src/documents/tests/test_api_uisettings.py | 2 + src/documents/views.py | 11 + src/paperless/config.py | 29 ++- ...licationconfiguration_app_logo_and_more.py | 33 +++ src/paperless/models.py | 18 ++ src/paperless/serialisers.py | 5 + src/paperless/settings.py | 4 + src/paperless/urls.py | 9 + 34 files changed, 675 insertions(+), 118 deletions(-) create mode 100644 src-ui/src/app/components/common/input/file/file.component.html create mode 100644 src-ui/src/app/components/common/input/file/file.component.scss create mode 100644 src-ui/src/app/components/common/input/file/file.component.spec.ts create mode 100644 src-ui/src/app/components/common/input/file/file.component.ts create mode 100644 src/paperless/migrations/0002_applicationconfiguration_app_logo_and_more.py diff --git a/docs/configuration.md b/docs/configuration.md index 5ca6bf701..b68198619 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -4,9 +4,9 @@ Paperless provides a wide range of customizations. Depending on how you run paperless, these settings have to be defined in different places. Certain configuration options may be set via the UI. This currently includes -common [OCR](#ocr) related settings. If set, these will take preference over the -settings via environment variables. If not set, the environment setting or applicable -default will be utilized instead. +common [OCR](#ocr) related settings and some frontend settings. If set, these will take +preference over the settings via environment variables. If not set, the environment setting +or applicable default will be utilized instead. - If you run paperless on docker, `paperless.conf` is not used. Rather, configure paperless by copying necessary options to @@ -1329,7 +1329,15 @@ started by the container. You can read more about this in the [advanced documentation](advanced_usage.md#celery-monitoring). -## Update Checking {#update-checking} +## Frontend Settings + +#### [`PAPERLESS_APP_TITLE=`](#PAPERLESS_APP_TITLE) {#PAPERLESS_APP_TITLE} + +: If set, overrides the default name "Paperless-ngx" + +#### [`PAPERLESS_APP_LOGO=`](#PAPERLESS_APP_LOGO) {#PAPERLESS_APP_LOGO} + +: Path to an image file in the /media/logo directory, must include 'logo', e.g. `/logo/Atari_logo.svg` #### [`PAPERLESS_ENABLE_UPDATE_CHECK=`](#PAPERLESS_ENABLE_UPDATE_CHECK) {#PAPERLESS_ENABLE_UPDATE_CHECK} diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf index 9581095ad..d2fd2a0b1 100644 --- a/src-ui/messages.xlf +++ b/src-ui/messages.xlf @@ -439,7 +439,7 @@ Discard src/app/components/admin/config/config.component.html - 48 + 49 src/app/components/document-detail/document-detail.component.html @@ -450,7 +450,7 @@ Save src/app/components/admin/config/config.component.html - 51 + 52 src/app/components/admin/settings/settings.component.html @@ -513,28 +513,42 @@ Error retrieving config src/app/components/admin/config/config.component.ts - 79 + 81 Invalid JSON src/app/components/admin/config/config.component.ts - 105 + 107 Configuration updated src/app/components/admin/config/config.component.ts - 148 + 151 An error occurred updating configuration src/app/components/admin/config/config.component.ts - 153 + 156 + + + + File successfully updated + + src/app/components/admin/config/config.component.ts + 178 + + + + An error occurred uploading file + + src/app/components/admin/config/config.component.ts + 183 @@ -545,11 +559,11 @@ src/app/components/app-frame/app-frame.component.html - 309 + 319 src/app/components/app-frame/app-frame.component.html - 314 + 324 @@ -646,15 +660,15 @@ src/app/components/app-frame/app-frame.component.html - 59 + 69 src/app/components/app-frame/app-frame.component.html - 267 + 277 src/app/components/app-frame/app-frame.component.html - 271 + 281 @@ -1123,7 +1137,7 @@ src/app/components/app-frame/app-frame.component.html - 116 + 126 @@ -1517,7 +1531,7 @@ src/app/components/app-frame/app-frame.component.ts - 117 + 121 @@ -1535,7 +1549,7 @@ src/app/components/app-frame/app-frame.component.html - 296 + 306 @@ -1722,11 +1736,11 @@ src/app/components/app-frame/app-frame.component.html - 285 + 295 src/app/components/app-frame/app-frame.component.html - 289 + 299 @@ -2035,66 +2049,65 @@ 180 - - Paperless-ngx + + by Paperless-ngx src/app/components/app-frame/app-frame.component.html - 15 + 20 - app title Search documents src/app/components/app-frame/app-frame.component.html - 23 + 33 Logged in as src/app/components/app-frame/app-frame.component.html - 47 + 57 My Profile src/app/components/app-frame/app-frame.component.html - 53 + 63 Logout src/app/components/app-frame/app-frame.component.html - 64 + 74 Documentation src/app/components/app-frame/app-frame.component.html - 71 + 81 src/app/components/app-frame/app-frame.component.html - 319 + 329 src/app/components/app-frame/app-frame.component.html - 324 + 334 Dashboard src/app/components/app-frame/app-frame.component.html - 96 + 106 src/app/components/app-frame/app-frame.component.html - 100 + 110 src/app/components/dashboard/dashboard.component.html @@ -2105,11 +2118,11 @@ Documents src/app/components/app-frame/app-frame.component.html - 105 + 115 src/app/components/app-frame/app-frame.component.html - 109 + 119 src/app/components/document-list/document-list.component.ts @@ -2136,36 +2149,36 @@ Open documents src/app/components/app-frame/app-frame.component.html - 150 + 160 Close all src/app/components/app-frame/app-frame.component.html - 174 + 184 src/app/components/app-frame/app-frame.component.html - 178 + 188 Manage src/app/components/app-frame/app-frame.component.html - 186 + 196 Correspondents src/app/components/app-frame/app-frame.component.html - 192 + 202 src/app/components/app-frame/app-frame.component.html - 196 + 206 src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html @@ -2176,11 +2189,11 @@ Tags src/app/components/app-frame/app-frame.component.html - 201 + 211 src/app/components/app-frame/app-frame.component.html - 206 + 216 src/app/components/common/input/tags/tags.component.ts @@ -2207,11 +2220,11 @@ Document Types src/app/components/app-frame/app-frame.component.html - 212 + 222 src/app/components/app-frame/app-frame.component.html - 216 + 226 src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html @@ -2222,11 +2235,11 @@ Storage Paths src/app/components/app-frame/app-frame.component.html - 221 + 231 src/app/components/app-frame/app-frame.component.html - 225 + 235 src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html @@ -2237,11 +2250,11 @@ Custom Fields src/app/components/app-frame/app-frame.component.html - 230 + 240 src/app/components/app-frame/app-frame.component.html - 234 + 244 src/app/components/common/custom-fields-dropdown/custom-fields-dropdown.component.html @@ -2256,11 +2269,11 @@ Workflows src/app/components/app-frame/app-frame.component.html - 241 + 251 src/app/components/app-frame/app-frame.component.html - 245 + 255 src/app/components/manage/workflows/workflows.component.html @@ -2271,99 +2284,99 @@ Mail src/app/components/app-frame/app-frame.component.html - 250 + 260 src/app/components/app-frame/app-frame.component.html - 255 + 265 Administration src/app/components/app-frame/app-frame.component.html - 261 + 271 Configuration src/app/components/app-frame/app-frame.component.html - 276 + 286 src/app/components/app-frame/app-frame.component.html - 280 + 290 File Tasks src/app/components/app-frame/app-frame.component.html - 303,305 + 313,315 GitHub src/app/components/app-frame/app-frame.component.html - 331 + 341 is available. src/app/components/app-frame/app-frame.component.html - 340,341 + 350,351 Click to view. src/app/components/app-frame/app-frame.component.html - 341 + 351 Paperless-ngx can automatically check for updates src/app/components/app-frame/app-frame.component.html - 345 + 355 How does this work? src/app/components/app-frame/app-frame.component.html - 352,354 + 362,364 Update available src/app/components/app-frame/app-frame.component.html - 368 + 378 Sidebar views updated src/app/components/app-frame/app-frame.component.ts - 259 + 263 Error updating sidebar views src/app/components/app-frame/app-frame.component.ts - 262 + 266 An error occurred while saving update checking settings. src/app/components/app-frame/app-frame.component.ts - 283 + 287 @@ -3696,6 +3709,14 @@ src/app/components/common/input/document-link/document-link.component.html 11 + + src/app/components/common/input/file/file.component.html + 11 + + + src/app/components/common/input/file/file.component.html + 25 + src/app/components/common/input/number/number.component.html 11 @@ -3757,6 +3778,13 @@ 44 + + Upload + + src/app/components/common/input/file/file.component.html + 17 + + Show password @@ -4188,32 +4216,32 @@ 44 - - Hello , welcome to Paperless-ngx + + Hello , welcome to src/app/components/dashboard/dashboard.component.ts - 38 + 41 - - Welcome to Paperless-ngx + + Welcome to src/app/components/dashboard/dashboard.component.ts - 40 + 43 Dashboard updated src/app/components/dashboard/dashboard.component.ts - 71 + 74 Error updating dashboard src/app/components/dashboard/dashboard.component.ts - 74 + 77 @@ -6443,102 +6471,123 @@ 46 + + General Settings + + src/app/data/paperless-config.ts + 50 + + OCR Settings src/app/data/paperless-config.ts - 49 + 51 Output Type src/app/data/paperless-config.ts - 73 + 75 Language src/app/data/paperless-config.ts - 81 + 83 Pages src/app/data/paperless-config.ts - 88 + 90 Mode src/app/data/paperless-config.ts - 95 + 97 Skip Archive File src/app/data/paperless-config.ts - 103 + 105 Image DPI src/app/data/paperless-config.ts - 111 + 113 Clean src/app/data/paperless-config.ts - 118 + 120 Deskew src/app/data/paperless-config.ts - 126 + 128 Rotate Pages src/app/data/paperless-config.ts - 133 + 135 Rotate Pages Threshold src/app/data/paperless-config.ts - 140 + 142 Max Image Pixels src/app/data/paperless-config.ts - 147 + 149 Color Conversion Strategy src/app/data/paperless-config.ts - 154 + 156 OCR Arguments src/app/data/paperless-config.ts - 162 + 164 + + + + Application Logo + + src/app/data/paperless-config.ts + 171 + + + + Application Title + + src/app/data/paperless-config.ts + 178 diff --git a/src-ui/src/app/app.module.ts b/src-ui/src/app/app.module.ts index ad76bdb74..a97897c36 100644 --- a/src-ui/src/app/app.module.ts +++ b/src-ui/src/app/app.module.ts @@ -110,6 +110,7 @@ import { DocumentLinkComponent } from './components/common/input/document-link/d import { PreviewPopupComponent } from './components/common/preview-popup/preview-popup.component' import { SwitchComponent } from './components/common/input/switch/switch.component' import { ConfigComponent } from './components/admin/config/config.component' +import { FileComponent } from './components/common/input/file/file.component' import localeAf from '@angular/common/locales/af' import localeAr from '@angular/common/locales/ar' @@ -267,6 +268,7 @@ function initializeApp(settings: SettingsService) { PreviewPopupComponent, SwitchComponent, ConfigComponent, + FileComponent, ], imports: [ BrowserModule, diff --git a/src-ui/src/app/components/admin/config/config.component.html b/src-ui/src/app/components/admin/config/config.component.html index 6f5fc4bac..53beb60e0 100644 --- a/src-ui/src/app/components/admin/config/config.component.html +++ b/src-ui/src/app/components/admin/config/config.component.html @@ -30,6 +30,7 @@ @case (ConfigOptionType.Boolean) { } @case (ConfigOptionType.String) { } @case (ConfigOptionType.JSON) { } + @case (ConfigOptionType.File) { } } diff --git a/src-ui/src/app/components/admin/config/config.component.spec.ts b/src-ui/src/app/components/admin/config/config.component.spec.ts index 5d70881b6..6c5472159 100644 --- a/src-ui/src/app/components/admin/config/config.component.spec.ts +++ b/src-ui/src/app/components/admin/config/config.component.spec.ts @@ -15,12 +15,15 @@ import { SwitchComponent } from '../../common/input/switch/switch.component' import { FormsModule, ReactiveFormsModule } from '@angular/forms' import { PageHeaderComponent } from '../../common/page-header/page-header.component' import { SelectComponent } from '../../common/input/select/select.component' +import { FileComponent } from '../../common/input/file/file.component' +import { SettingsService } from 'src/app/services/settings.service' describe('ConfigComponent', () => { let component: ConfigComponent let fixture: ComponentFixture let configService: ConfigService let toastService: ToastService + let settingService: SettingsService beforeEach(async () => { await TestBed.configureTestingModule({ @@ -30,6 +33,7 @@ describe('ConfigComponent', () => { SelectComponent, NumberComponent, SwitchComponent, + FileComponent, PageHeaderComponent, ], imports: [ @@ -44,6 +48,7 @@ describe('ConfigComponent', () => { configService = TestBed.inject(ConfigService) toastService = TestBed.inject(ToastService) + settingService = TestBed.inject(SettingsService) fixture = TestBed.createComponent(ConfigComponent) component = fixture.componentInstance fixture.detectChanges() @@ -100,4 +105,39 @@ describe('ConfigComponent', () => { component.configForm.patchValue({ user_args: '{ "foo": "bar" }' }) expect(component.errors).toEqual({ user_args: null }) }) + + it('should upload file, show error if necessary', () => { + const uploadSpy = jest.spyOn(configService, 'uploadFile') + const errorSpy = jest.spyOn(toastService, 'showError') + uploadSpy.mockReturnValueOnce( + throwError(() => new Error('Error uploading file')) + ) + component.uploadFile(new File([], 'test.png'), 'app_logo') + expect(uploadSpy).toHaveBeenCalled() + expect(errorSpy).toHaveBeenCalled() + uploadSpy.mockReturnValueOnce( + of({ app_logo: 'https://example.com/logo/test.png' } as any) + ) + component.uploadFile(new File([], 'test.png'), 'app_logo') + expect(component.initialConfig).toEqual({ + app_logo: 'https://example.com/logo/test.png', + }) + }) + + it('should refresh ui settings after save or upload', () => { + const saveSpy = jest.spyOn(configService, 'saveConfig') + const initSpy = jest.spyOn(settingService, 'initializeSettings') + saveSpy.mockReturnValueOnce( + of({ output_type: OutputTypeConfig.PDF_A } as any) + ) + component.saveConfig() + expect(initSpy).toHaveBeenCalled() + + const uploadSpy = jest.spyOn(configService, 'uploadFile') + uploadSpy.mockReturnValueOnce( + of({ app_logo: 'https://example.com/logo/test.png' } as any) + ) + component.uploadFile(new File([], 'test.png'), 'app_logo') + expect(initSpy).toHaveBeenCalled() + }) }) diff --git a/src-ui/src/app/components/admin/config/config.component.ts b/src-ui/src/app/components/admin/config/config.component.ts index 66d7b537f..63e66d456 100644 --- a/src-ui/src/app/components/admin/config/config.component.ts +++ b/src-ui/src/app/components/admin/config/config.component.ts @@ -19,6 +19,7 @@ import { ConfigService } from 'src/app/services/config.service' import { ToastService } from 'src/app/services/toast.service' import { ComponentWithPermissions } from '../../with-permissions/with-permissions.component' import { DirtyComponent, dirtyCheck } from '@ngneat/dirty-check-forms' +import { SettingsService } from 'src/app/services/settings.service' @Component({ selector: 'pngx-config', @@ -55,7 +56,8 @@ export class ConfigComponent constructor( private configService: ConfigService, - private toastService: ToastService + private toastService: ToastService, + private settingsService: SettingsService ) { super() this.configForm.addControl('id', new FormControl()) @@ -145,6 +147,7 @@ export class ConfigComponent this.loading = false this.initialize(config) this.store.next(config) + this.settingsService.initializeSettings().subscribe() this.toastService.showInfo($localize`Configuration updated`) }, error: (e) => { @@ -160,4 +163,27 @@ export class ConfigComponent public discardChanges() { this.configForm.reset(this.initialConfig) } + + public uploadFile(file: File, key: string) { + this.loading = true + this.configService + .uploadFile(file, this.configForm.value['id'], key) + .pipe(takeUntil(this.unsubscribeNotifier), first()) + .subscribe({ + next: (config) => { + this.loading = false + this.initialize(config) + this.store.next(config) + this.settingsService.initializeSettings().subscribe() + this.toastService.showInfo($localize`File successfully updated`) + }, + error: (e) => { + this.loading = false + this.toastService.showError( + $localize`An error occurred uploading file`, + e + ) + }, + }) + } } diff --git a/src-ui/src/app/components/app-frame/app-frame.component.html b/src-ui/src/app/components/app-frame/app-frame.component.html index 32241333e..ba21c8bac 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.html +++ b/src-ui/src/app/components/app-frame/app-frame.component.html @@ -4,15 +4,25 @@ (click)="isMenuCollapsed = !isMenuCollapsed"> - - Paperless-ngx +
+ @if (customAppTitle?.length) { +
+ {{customAppTitle}} + by Paperless-ngx +
+ } @else { + Paperless-ngx + } +
diff --git a/src-ui/src/app/components/app-frame/app-frame.component.scss b/src-ui/src/app/components/app-frame/app-frame.component.scss index f8a32ecfb..5dcac760f 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.scss +++ b/src-ui/src/app/components/app-frame/app-frame.component.scss @@ -217,9 +217,16 @@ main { */ .navbar-brand { - padding-top: 0.75rem; - padding-bottom: 0.75rem; font-size: 1rem; + + .flex-column { + padding: 0.15rem 0; + } + + .byline { + font-size: 0.5rem; + letter-spacing: 0.1rem; + } } @media screen and (min-width: 768px) { diff --git a/src-ui/src/app/components/app-frame/app-frame.component.ts b/src-ui/src/app/components/app-frame/app-frame.component.ts index 0e877b7ce..cfc9740a4 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.ts +++ b/src-ui/src/app/components/app-frame/app-frame.component.ts @@ -102,6 +102,10 @@ export class AppFrameComponent }, 200) // slightly longer than css animation for slim sidebar } + get customAppTitle(): string { + return this.settingsService.get(SETTINGS_KEYS.APP_TITLE) + } + get slimSidebarEnabled(): boolean { return this.settingsService.get(SETTINGS_KEYS.SLIM_SIDEBAR) } diff --git a/src-ui/src/app/components/common/input/file/file.component.html b/src-ui/src/app/components/common/input/file/file.component.html new file mode 100644 index 000000000..9ad82a99c --- /dev/null +++ b/src-ui/src/app/components/common/input/file/file.component.html @@ -0,0 +1,37 @@ +
+
+
+ @if (title) { + + } + @if (removable) { + + } +
+
+ + +
+ @if (filename) { +
+ {{filename}} + +
+ } + + @if (hint) { + + } +
+ {{error}} +
+
+
diff --git a/src-ui/src/app/components/common/input/file/file.component.scss b/src-ui/src/app/components/common/input/file/file.component.scss new file mode 100644 index 000000000..e69de29bb diff --git a/src-ui/src/app/components/common/input/file/file.component.spec.ts b/src-ui/src/app/components/common/input/file/file.component.spec.ts new file mode 100644 index 000000000..86d135188 --- /dev/null +++ b/src-ui/src/app/components/common/input/file/file.component.spec.ts @@ -0,0 +1,41 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing' + +import { FileComponent } from './file.component' +import { HttpClientTestingModule } from '@angular/common/http/testing' +import { FormsModule, ReactiveFormsModule } from '@angular/forms' + +describe('FileComponent', () => { + let component: FileComponent + let fixture: ComponentFixture + + beforeEach(async () => { + await TestBed.configureTestingModule({ + declarations: [FileComponent], + imports: [FormsModule, ReactiveFormsModule, HttpClientTestingModule], + }).compileComponents() + + fixture = TestBed.createComponent(FileComponent) + component = fixture.componentInstance + fixture.detectChanges() + }) + + it('should update file on change', () => { + const event = { target: { files: [new File([], 'test.png')] } } + component.onFile(event as any) + expect(component.file.name).toEqual('test.png') + }) + + it('should get filename', () => { + component.value = 'https://example.com:8000/logo/filename.svg' + expect(component.filename).toEqual('filename.svg') + }) + + it('should fire upload event', () => { + let firedFile + component.file = new File([], 'test.png') + component.upload.subscribe((file) => (firedFile = file)) + component.uploadClicked() + expect(firedFile.name).toEqual('test.png') + expect(component.file).toBeUndefined() + }) +}) diff --git a/src-ui/src/app/components/common/input/file/file.component.ts b/src-ui/src/app/components/common/input/file/file.component.ts new file mode 100644 index 000000000..0506dcc5b --- /dev/null +++ b/src-ui/src/app/components/common/input/file/file.component.ts @@ -0,0 +1,53 @@ +import { + Component, + ElementRef, + EventEmitter, + Output, + ViewChild, + forwardRef, +} from '@angular/core' +import { NG_VALUE_ACCESSOR } from '@angular/forms' +import { AbstractInputComponent } from '../abstract-input' + +@Component({ + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => FileComponent), + multi: true, + }, + ], + selector: 'pngx-input-file', + templateUrl: './file.component.html', + styleUrl: './file.component.scss', +}) +export class FileComponent extends AbstractInputComponent { + @Output() + upload = new EventEmitter() + + public file: File + + @ViewChild('fileInput') fileInput: ElementRef + + get filename(): string { + return this.value + ? this.value.substring(this.value.lastIndexOf('/') + 1) + : null + } + + onFile(event: Event) { + this.file = (event.target as HTMLInputElement).files[0] + } + + uploadClicked() { + this.upload.emit(this.file) + this.clear() + } + + clear() { + this.file = undefined + this.fileInput.nativeElement.value = null + this.writeValue(null) + this.onChange(null) + } +} diff --git a/src-ui/src/app/components/common/logo/logo.component.html b/src-ui/src/app/components/common/logo/logo.component.html index af08e41fd..6fb003396 100644 --- a/src-ui/src/app/components/common/logo/logo.component.html +++ b/src-ui/src/app/components/common/logo/logo.component.html @@ -1,18 +1,22 @@ - - - - - - - - - - - - - - - - - - +@if (customLogo) { + +} @else { + + + + + + + + + + + + + + + + + + +} diff --git a/src-ui/src/app/components/common/logo/logo.component.spec.ts b/src-ui/src/app/components/common/logo/logo.component.spec.ts index 921ea3765..5b64177a4 100644 --- a/src-ui/src/app/components/common/logo/logo.component.spec.ts +++ b/src-ui/src/app/components/common/logo/logo.component.spec.ts @@ -2,15 +2,21 @@ import { ComponentFixture, TestBed } from '@angular/core/testing' import { LogoComponent } from './logo.component' import { By } from '@angular/platform-browser' +import { HttpClientTestingModule } from '@angular/common/http/testing' +import { SettingsService } from 'src/app/services/settings.service' +import { SETTINGS_KEYS } from 'src/app/data/ui-settings' describe('LogoComponent', () => { let component: LogoComponent let fixture: ComponentFixture + let settingsService: SettingsService beforeEach(() => { TestBed.configureTestingModule({ declarations: [LogoComponent], + imports: [HttpClientTestingModule], }) + settingsService = TestBed.inject(SettingsService) fixture = TestBed.createComponent(LogoComponent) component = fixture.componentInstance fixture.detectChanges() @@ -33,4 +39,9 @@ describe('LogoComponent', () => { 'height:10em' ) }) + + it('should support getting custom logo', () => { + settingsService.set(SETTINGS_KEYS.APP_LOGO, '/logo/test.png') + expect(component.customLogo).toEqual('http://localhost:8000/logo/test.png') + }) }) diff --git a/src-ui/src/app/components/common/logo/logo.component.ts b/src-ui/src/app/components/common/logo/logo.component.ts index 3320a621a..7404ea865 100644 --- a/src-ui/src/app/components/common/logo/logo.component.ts +++ b/src-ui/src/app/components/common/logo/logo.component.ts @@ -1,4 +1,7 @@ import { Component, Input } from '@angular/core' +import { SETTINGS_KEYS } from 'src/app/data/ui-settings' +import { SettingsService } from 'src/app/services/settings.service' +import { environment } from 'src/environments/environment' @Component({ selector: 'pngx-logo', @@ -12,6 +15,17 @@ export class LogoComponent { @Input() height = '6em' + get customLogo(): string { + return this.settingsService.get(SETTINGS_KEYS.APP_LOGO)?.length + ? environment.apiBaseUrl.replace( + /\/api\/$/, + this.settingsService.get(SETTINGS_KEYS.APP_LOGO) + ) + : null + } + + constructor(private settingsService: SettingsService) {} + getClasses() { return ['logo'].concat(this.extra_classes).join(' ') } diff --git a/src-ui/src/app/components/dashboard/dashboard.component.ts b/src-ui/src/app/components/dashboard/dashboard.component.ts index a35e7459b..906cc775a 100644 --- a/src-ui/src/app/components/dashboard/dashboard.component.ts +++ b/src-ui/src/app/components/dashboard/dashboard.component.ts @@ -5,13 +5,13 @@ import { ComponentWithPermissions } from '../with-permissions/with-permissions.c import { TourService } from 'ngx-ui-tour-ng-bootstrap' import { SavedView } from 'src/app/data/saved-view' import { ToastService } from 'src/app/services/toast.service' -import { SETTINGS_KEYS } from 'src/app/data/ui-settings' import { CdkDragDrop, CdkDragEnd, CdkDragStart, moveItemInArray, } from '@angular/cdk/drag-drop' +import { environment } from 'src/environments/environment' @Component({ selector: 'pngx-dashboard', @@ -35,9 +35,9 @@ export class DashboardComponent extends ComponentWithPermissions { get subtitle() { if (this.settingsService.displayName) { - return $localize`Hello ${this.settingsService.displayName}, welcome to Paperless-ngx` + return $localize`Hello ${this.settingsService.displayName}, welcome to ${environment.appTitle}` } else { - return $localize`Welcome to Paperless-ngx` + return $localize`Welcome to ${environment.appTitle}` } } diff --git a/src-ui/src/app/components/not-found/not-found.component.spec.ts b/src-ui/src/app/components/not-found/not-found.component.spec.ts index 2a0ab9d7c..bd3975670 100644 --- a/src-ui/src/app/components/not-found/not-found.component.spec.ts +++ b/src-ui/src/app/components/not-found/not-found.component.spec.ts @@ -2,6 +2,7 @@ import { ComponentFixture, TestBed } from '@angular/core/testing' import { NotFoundComponent } from './not-found.component' import { By } from '@angular/platform-browser' import { LogoComponent } from '../common/logo/logo.component' +import { HttpClientTestingModule } from '@angular/common/http/testing' describe('NotFoundComponent', () => { let component: NotFoundComponent @@ -10,6 +11,7 @@ describe('NotFoundComponent', () => { beforeEach(async () => { TestBed.configureTestingModule({ declarations: [NotFoundComponent, LogoComponent], + imports: [HttpClientTestingModule], }).compileComponents() fixture = TestBed.createComponent(NotFoundComponent) diff --git a/src-ui/src/app/data/paperless-config.ts b/src-ui/src/app/data/paperless-config.ts index 69f9b46e0..3ae485ff2 100644 --- a/src-ui/src/app/data/paperless-config.ts +++ b/src-ui/src/app/data/paperless-config.ts @@ -43,9 +43,11 @@ export enum ConfigOptionType { Select = 'select', Boolean = 'boolean', JSON = 'json', + File = 'file', } export const ConfigCategory = { + General: $localize`General Settings`, OCR: $localize`OCR Settings`, } @@ -164,6 +166,20 @@ export const PaperlessConfigOptions: ConfigOption[] = [ config_key: 'PAPERLESS_OCR_USER_ARGS', category: ConfigCategory.OCR, }, + { + key: 'app_logo', + title: $localize`Application Logo`, + type: ConfigOptionType.File, + config_key: 'PAPERLESS_APP_LOGO', + category: ConfigCategory.General, + }, + { + key: 'app_title', + title: $localize`Application Title`, + type: ConfigOptionType.String, + config_key: 'PAPERLESS_APP_TITLE', + category: ConfigCategory.General, + }, ] export interface PaperlessConfig extends ObjectWithId { @@ -180,4 +196,6 @@ export interface PaperlessConfig extends ObjectWithId { max_image_pixels: number color_conversion_strategy: ColorConvertConfig user_args: object + app_logo: string + app_title: string } diff --git a/src-ui/src/app/data/ui-settings.ts b/src-ui/src/app/data/ui-settings.ts index 329fc6aa0..e23e490e9 100644 --- a/src-ui/src/app/data/ui-settings.ts +++ b/src-ui/src/app/data/ui-settings.ts @@ -14,6 +14,8 @@ export interface UiSetting { export const SETTINGS_KEYS = { LANGUAGE: 'language', + APP_LOGO: 'app_logo', + APP_TITLE: 'app_title', // maintain old general-settings: for backwards compatibility BULK_EDIT_CONFIRMATION_DIALOGS: 'general-settings:bulk-edit:confirmation-dialogs', @@ -194,4 +196,14 @@ export const SETTINGS: UiSetting[] = [ type: 'array', default: [], }, + { + key: SETTINGS_KEYS.APP_LOGO, + type: 'string', + default: '', + }, + { + key: SETTINGS_KEYS.APP_TITLE, + type: 'string', + default: '', + }, ] diff --git a/src-ui/src/app/services/config.service.spec.ts b/src-ui/src/app/services/config.service.spec.ts index 3cfadb051..4fb24727f 100644 --- a/src-ui/src/app/services/config.service.spec.ts +++ b/src-ui/src/app/services/config.service.spec.ts @@ -39,4 +39,26 @@ describe('ConfigService', () => { ) expect(req.request.method).toEqual('PATCH') }) + + it('should support upload file with form data', () => { + service.uploadFile(new File([], 'test.png'), 1, 'app_logo').subscribe() + const req = httpTestingController.expectOne( + `${environment.apiBaseUrl}config/1/` + ) + expect(req.request.method).toEqual('PATCH') + expect(req.request.body).not.toBeNull() + }) + + it('should not pass string app_logo', () => { + service + .saveConfig({ + id: 1, + app_logo: '/logo/foobar.png', + } as PaperlessConfig) + .subscribe() + const req = httpTestingController.expectOne( + `${environment.apiBaseUrl}config/1/` + ) + expect(req.request.body).toEqual({ id: 1 }) + }) }) diff --git a/src-ui/src/app/services/config.service.ts b/src-ui/src/app/services/config.service.ts index 19158b3ce..538aafbdd 100644 --- a/src-ui/src/app/services/config.service.ts +++ b/src-ui/src/app/services/config.service.ts @@ -20,8 +20,22 @@ export class ConfigService { } saveConfig(config: PaperlessConfig): Observable { + // dont pass string + if (typeof config.app_logo === 'string') delete config.app_logo return this.http .patch(`${this.baseUrl}${config.id}/`, config) .pipe(first()) } + + uploadFile( + file: File, + configID: number, + configKey: string + ): Observable { + let formData = new FormData() + formData.append(configKey, file, file.name) + return this.http + .patch(`${this.baseUrl}${configID}/`, formData) + .pipe(first()) + } } diff --git a/src-ui/src/app/services/settings.service.spec.ts b/src-ui/src/app/services/settings.service.spec.ts index 0c148cec2..ff0a9837b 100644 --- a/src-ui/src/app/services/settings.service.spec.ts +++ b/src-ui/src/app/services/settings.service.spec.ts @@ -301,4 +301,16 @@ describe('SettingsService', () => { .expectOne(`${environment.apiBaseUrl}ui_settings/`) .flush(ui_settings) }) + + it('should update environment app title if set', () => { + const settings = Object.assign({}, ui_settings) + settings.settings['app_title'] = 'FooBar' + const req = httpTestingController.expectOne( + `${environment.apiBaseUrl}ui_settings/` + ) + req.flush(settings) + expect(environment.appTitle).toEqual('FooBar') + // post for migrate + httpTestingController.expectOne(`${environment.apiBaseUrl}ui_settings/`) + }) }) diff --git a/src-ui/src/app/services/settings.service.ts b/src-ui/src/app/services/settings.service.ts index 9f8560322..4bbeb1dde 100644 --- a/src-ui/src/app/services/settings.service.ts +++ b/src-ui/src/app/services/settings.service.ts @@ -270,6 +270,9 @@ export class SettingsService { first(), tap((uisettings) => { Object.assign(this.settings, uisettings.settings) + if (this.get(SETTINGS_KEYS.APP_TITLE)?.length) { + environment.appTitle = this.get(SETTINGS_KEYS.APP_TITLE) + } this.maybeMigrateSettings() // to update lang cookie if (this.settings['language']?.length) diff --git a/src-ui/src/styles.scss b/src-ui/src/styles.scss index c8e8e8d5c..0dc58403a 100644 --- a/src-ui/src/styles.scss +++ b/src-ui/src/styles.scss @@ -11,6 +11,9 @@ $grid-breakpoints: ( xxxl: 2400px ); +$form-file-button-bg: var(--bs-body-bg); +$form-file-button-hover-bg: var(--pngx-bg-alt); + @import "node_modules/bootstrap/scss/bootstrap"; @import "theme"; @import "~@ng-select/ng-select/themes/default.theme.css"; diff --git a/src/documents/tests/test_api_app_config.py b/src/documents/tests/test_api_app_config.py index a12d2a695..ba14e664a 100644 --- a/src/documents/tests/test_api_app_config.py +++ b/src/documents/tests/test_api_app_config.py @@ -1,4 +1,5 @@ import json +import os from django.contrib.auth.models import User from rest_framework import status @@ -49,10 +50,34 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase): "rotate_pages_threshold": None, "max_image_pixels": None, "color_conversion_strategy": None, + "app_title": None, + "app_logo": None, }, ), ) + def test_api_get_ui_settings_with_config(self): + """ + GIVEN: + - Existing config with app_title, app_logo specified + WHEN: + - API to retrieve uisettings is called + THEN: + - app_title and app_logo are included + """ + config = ApplicationConfiguration.objects.first() + config.app_title = "Fancy New Title" + config.app_logo = "/logo/example.jpg" + config.save() + response = self.client.get("/api/ui_settings/", format="json") + self.assertDictContainsSubset( + { + "app_title": config.app_title, + "app_logo": config.app_logo, + }, + response.data["settings"], + ) + def test_api_update_config(self): """ GIVEN: @@ -100,3 +125,37 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase): config = ApplicationConfiguration.objects.first() self.assertEqual(config.user_args, None) self.assertEqual(config.language, None) + + def test_api_replace_app_logo(self): + """ + GIVEN: + - Existing config with app_logo specified + WHEN: + - API to replace app_logo is called + THEN: + - old app_logo file is deleted + """ + with open( + os.path.join(os.path.dirname(__file__), "samples", "simple.jpg"), + "rb", + ) as f: + self.client.patch( + f"{self.ENDPOINT}1/", + { + "app_logo": f, + }, + ) + config = ApplicationConfiguration.objects.first() + old_logo = config.app_logo + self.assertTrue(os.path.exists(old_logo.path)) + with open( + os.path.join(os.path.dirname(__file__), "samples", "simple.png"), + "rb", + ) as f: + self.client.patch( + f"{self.ENDPOINT}1/", + { + "app_logo": f, + }, + ) + self.assertFalse(os.path.exists(old_logo.path)) diff --git a/src/documents/tests/test_api_uisettings.py b/src/documents/tests/test_api_uisettings.py index da9f2914d..bde4808d4 100644 --- a/src/documents/tests/test_api_uisettings.py +++ b/src/documents/tests/test_api_uisettings.py @@ -35,6 +35,8 @@ class TestApiUiSettings(DirectoriesMixin, APITestCase): self.assertDictEqual( response.data["settings"], { + "app_title": None, + "app_logo": None, "update_checking": { "backend_setting": "default", }, diff --git a/src/documents/views.py b/src/documents/views.py index d6b90cbfd..b545a1466 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -120,6 +120,7 @@ from documents.serialisers import WorkflowTriggerSerializer from documents.signals import document_updated from documents.tasks import consume_file from paperless import version +from paperless.config import GeneralConfig from paperless.db import GnuPG from paperless.views import StandardPagination @@ -1164,6 +1165,16 @@ class UiSettingsView(GenericAPIView): ui_settings["update_checking"] = { "backend_setting": settings.ENABLE_UPDATE_CHECK, } + + general_config = GeneralConfig() + + ui_settings["app_title"] = settings.APP_TITLE + if general_config.app_title is not None and len(general_config.app_title) > 0: + ui_settings["app_title"] = general_config.app_title + ui_settings["app_logo"] = settings.APP_LOGO + if general_config.app_logo is not None and len(general_config.app_logo) > 0: + ui_settings["app_logo"] = general_config.app_logo + user_resp = { "id": user.id, "username": user.username, diff --git a/src/paperless/config.py b/src/paperless/config.py index 55d6dc3d3..4195a16db 100644 --- a/src/paperless/config.py +++ b/src/paperless/config.py @@ -8,13 +8,11 @@ from paperless.models import ApplicationConfiguration @dataclasses.dataclass -class OutputTypeConfig: +class BaseConfig: """ Almost all parsers care about the chosen PDF output format """ - output_type: str = dataclasses.field(init=False) - @staticmethod def _get_config_instance() -> ApplicationConfiguration: app_config = ApplicationConfiguration.objects.all().first() @@ -24,6 +22,15 @@ class OutputTypeConfig: app_config = ApplicationConfiguration.objects.all().first() return app_config + +@dataclasses.dataclass +class OutputTypeConfig(BaseConfig): + """ + Almost all parsers care about the chosen PDF output format + """ + + output_type: str = dataclasses.field(init=False) + def __post_init__(self) -> None: app_config = self._get_config_instance() @@ -86,3 +93,19 @@ class OcrConfig(OutputTypeConfig): user_args = {} self.user_args = user_args + + +@dataclasses.dataclass +class GeneralConfig(BaseConfig): + """ + General application settings that require global scope + """ + + app_title: str = dataclasses.field(init=False) + app_logo: str = dataclasses.field(init=False) + + def __post_init__(self) -> None: + app_config = self._get_config_instance() + + self.app_title = app_config.app_title or None + self.app_logo = app_config.app_logo.url if app_config.app_logo else None diff --git a/src/paperless/migrations/0002_applicationconfiguration_app_logo_and_more.py b/src/paperless/migrations/0002_applicationconfiguration_app_logo_and_more.py new file mode 100644 index 000000000..e6960d1a6 --- /dev/null +++ b/src/paperless/migrations/0002_applicationconfiguration_app_logo_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.9 on 2024-01-12 05:33 + +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + dependencies = [ + ("paperless", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="applicationconfiguration", + name="app_logo", + field=models.FileField( + blank=True, + null=True, + upload_to="", + verbose_name="Application logo", + ), + ), + migrations.AddField( + model_name="applicationconfiguration", + name="app_title", + field=models.CharField( + blank=True, + max_length=48, + null=True, + verbose_name="Application title", + ), + ), + ] diff --git a/src/paperless/models.py b/src/paperless/models.py index 133668dd6..72805dc56 100644 --- a/src/paperless/models.py +++ b/src/paperless/models.py @@ -1,3 +1,4 @@ +from django.core.validators import FileExtensionValidator from django.core.validators import MinValueValidator from django.db import models from django.utils.translation import gettext_lazy as _ @@ -166,6 +167,23 @@ class ApplicationConfiguration(AbstractSingletonModel): null=True, ) + app_title = models.CharField( + verbose_name=_("Application title"), + null=True, + blank=True, + max_length=48, + ) + + app_logo = models.FileField( + verbose_name=_("Application logo"), + null=True, + blank=True, + validators=[ + FileExtensionValidator(allowed_extensions=["jpg", "png", "gif", "svg"]), + ], + upload_to="logo/", + ) + class Meta: verbose_name = _("paperless application settings") diff --git a/src/paperless/serialisers.py b/src/paperless/serialisers.py index fb366f808..b724dd451 100644 --- a/src/paperless/serialisers.py +++ b/src/paperless/serialisers.py @@ -132,6 +132,11 @@ class ApplicationConfigurationSerializer(serializers.ModelSerializer): data["language"] = None return super().run_validation(data) + def update(self, instance, validated_data): + if instance.app_logo and "app_logo" in validated_data: + instance.app_logo.delete() + return super().update(instance, validated_data) + class Meta: model = ApplicationConfiguration fields = "__all__" diff --git a/src/paperless/settings.py b/src/paperless/settings.py index e13518ce3..bc815d4d5 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -367,6 +367,7 @@ STORAGES = { "staticfiles": { "BACKEND": _static_backend, }, + "default": {"BACKEND": "django.core.files.storage.FileSystemStorage"}, } _CELERY_REDIS_URL, _CHANNELS_REDIS_URL = _parse_redis_url( @@ -999,6 +1000,9 @@ ENABLE_UPDATE_CHECK = os.getenv("PAPERLESS_ENABLE_UPDATE_CHECK", "default") if ENABLE_UPDATE_CHECK != "default": ENABLE_UPDATE_CHECK = __get_boolean("PAPERLESS_ENABLE_UPDATE_CHECK") +APP_TITLE = os.getenv("PAPERLESS_APP_TITLE", None) +APP_LOGO = os.getenv("PAPERLESS_APP_LOGO", None) + ############################################################################### # Machine Learning # ############################################################################### diff --git a/src/paperless/urls.py b/src/paperless/urls.py index 25190e0d8..d45a7bf22 100644 --- a/src/paperless/urls.py +++ b/src/paperless/urls.py @@ -1,3 +1,5 @@ +import os + from django.conf import settings from django.conf.urls import include from django.contrib import admin @@ -8,6 +10,7 @@ from django.utils.translation import gettext_lazy as _ from django.views.decorators.csrf import csrf_exempt from django.views.decorators.csrf import ensure_csrf_cookie from django.views.generic import RedirectView +from django.views.static import serve from rest_framework.authtoken import views from rest_framework.routers import DefaultRouter @@ -181,6 +184,12 @@ urlpatterns = [ url=settings.STATIC_URL + "frontend/en-US/assets/%(path)s", ), ), + # App logo + re_path( + r"^logo(?P.*)$", + serve, + kwargs={"document_root": os.path.join(settings.MEDIA_ROOT, "logo")}, + ), # TODO: with localization, this is even worse! :/ # login, logout path("accounts/", include("django.contrib.auth.urls")), From 53e04e66cf52a4ebf34d7d653d33f8f988d4f5b6 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Sat, 13 Jan 2024 12:28:10 -0800 Subject: [PATCH 3/3] Enhancement: warn when outdated doc detected (#5372) * Update modified property for target docs w bidirectional links * Warn on doc change detected --- src-ui/messages.xlf | 201 ++++++++++-------- .../confirm-dialog.component.html | 4 +- .../confirm-dialog.component.ts | 6 + .../document-detail.component.spec.ts | 25 ++- .../document-detail.component.ts | 16 ++ src/documents/serialisers.py | 3 + 6 files changed, 163 insertions(+), 92 deletions(-) diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf index d2fd2a0b1..a3e9b3330 100644 --- a/src-ui/messages.xlf +++ b/src-ui/messages.xlf @@ -1931,7 +1931,7 @@ src/app/components/document-detail/document-detail.component.ts - 688 + 701 src/app/components/document-list/bulk-editor/bulk-editor.component.ts @@ -1970,7 +1970,7 @@ src/app/components/document-detail/document-detail.component.ts - 690 + 703 src/app/components/document-list/bulk-editor/bulk-editor.component.ts @@ -2394,13 +2394,6 @@ 63 - - Cancel - - src/app/components/common/confirm-dialog/confirm-dialog.component.html - 16 - - Confirmation @@ -2435,6 +2428,73 @@ 439 + + Cancel + + src/app/components/common/confirm-dialog/confirm-dialog.component.ts + 44 + + + src/app/components/common/edit-dialog/correspondent-edit-dialog/correspondent-edit-dialog.component.html + 24 + + + src/app/components/common/edit-dialog/custom-field-edit-dialog/custom-field-edit-dialog.component.html + 15 + + + src/app/components/common/edit-dialog/document-type-edit-dialog/document-type-edit-dialog.component.html + 26 + + + src/app/components/common/edit-dialog/group-edit-dialog/group-edit-dialog.component.html + 16 + + + src/app/components/common/edit-dialog/mail-account-edit-dialog/mail-account-edit-dialog.component.html + 36 + + + src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html + 48 + + + src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.html + 25 + + + src/app/components/common/edit-dialog/tag-edit-dialog/tag-edit-dialog.component.html + 27 + + + src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.html + 35 + + + src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html + 170 + + + src/app/components/common/permissions-dialog/permissions-dialog.component.html + 22 + + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html + 57 + + + src/app/components/common/select-dialog/select-dialog.component.html + 12 + + + src/app/components/document-list/bulk-editor/bulk-editor.component.html + 6 + + + src/app/components/document-list/save-view-config-dialog/save-view-config-dialog.component.html + 20 + + Create New Field @@ -2599,69 +2659,6 @@ 194 - - Cancel - - src/app/components/common/edit-dialog/correspondent-edit-dialog/correspondent-edit-dialog.component.html - 24 - - - src/app/components/common/edit-dialog/custom-field-edit-dialog/custom-field-edit-dialog.component.html - 15 - - - src/app/components/common/edit-dialog/document-type-edit-dialog/document-type-edit-dialog.component.html - 26 - - - src/app/components/common/edit-dialog/group-edit-dialog/group-edit-dialog.component.html - 16 - - - src/app/components/common/edit-dialog/mail-account-edit-dialog/mail-account-edit-dialog.component.html - 36 - - - src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html - 48 - - - src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.html - 25 - - - src/app/components/common/edit-dialog/tag-edit-dialog/tag-edit-dialog.component.html - 27 - - - src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.html - 35 - - - src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 170 - - - src/app/components/common/permissions-dialog/permissions-dialog.component.html - 22 - - - src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html - 57 - - - src/app/components/common/select-dialog/select-dialog.component.html - 12 - - - src/app/components/document-list/bulk-editor/bulk-editor.component.html - 6 - - - src/app/components/document-list/save-view-config-dialog/save-view-config-dialog.component.html - 20 - - Create new correspondent @@ -4793,47 +4790,75 @@ 276,278 + + Document changes detected + + src/app/components/document-detail/document-detail.component.ts + 298 + + + + The version of this document in your browser session appears older than the existing version. + + src/app/components/document-detail/document-detail.component.ts + 299 + + + + Saving the document here may overwrite other changes that were made. To restore the existing version, discard your changes or close the document. + + src/app/components/document-detail/document-detail.component.ts + 300 + + + + Ok + + src/app/components/document-detail/document-detail.component.ts + 301 + + Error retrieving metadata src/app/components/document-detail/document-detail.component.ts - 424 + 437 Error retrieving suggestions. src/app/components/document-detail/document-detail.component.ts - 445 + 458 Document saved successfully. src/app/components/document-detail/document-detail.component.ts - 563 + 576 src/app/components/document-detail/document-detail.component.ts - 572 + 585 Error saving document src/app/components/document-detail/document-detail.component.ts - 576 + 589 src/app/components/document-detail/document-detail.component.ts - 617 + 630 Confirm delete src/app/components/document-detail/document-detail.component.ts - 643 + 656 src/app/components/manage/management-list/management-list.component.ts @@ -4844,35 +4869,35 @@ Do you really want to delete document ""? src/app/components/document-detail/document-detail.component.ts - 644 + 657 The files for this document will be deleted permanently. This operation cannot be undone. src/app/components/document-detail/document-detail.component.ts - 645 + 658 Delete document src/app/components/document-detail/document-detail.component.ts - 647 + 660 Error deleting document src/app/components/document-detail/document-detail.component.ts - 666 + 679 Redo OCR confirm src/app/components/document-detail/document-detail.component.ts - 686 + 699 src/app/components/document-list/bulk-editor/bulk-editor.component.ts @@ -4883,28 +4908,28 @@ This operation will permanently redo OCR for this document. src/app/components/document-detail/document-detail.component.ts - 687 + 700 Redo OCR operation will begin in the background. Close and re-open or reload this document after the operation has completed to see new content. src/app/components/document-detail/document-detail.component.ts - 698 + 711 Error executing operation src/app/components/document-detail/document-detail.component.ts - 709 + 722 Page Fit src/app/components/document-detail/document-detail.component.ts - 778 + 791 diff --git a/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html b/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html index bb5577061..e25ace612 100644 --- a/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html +++ b/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html @@ -12,8 +12,8 @@ }
-