Fixing up final reports

.ruff.toml

@@ -1,8 +1,8 @@
 # https://docs.astral.sh/ruff/settings/
 # https://docs.astral.sh/ruff/rules/
 extend-select = [
-  "I",     # https://docs.astral.sh/ruff/rules/#isort-i
   "W",     # https://docs.astral.sh/ruff/rules/#pycodestyle-e-w
+  "I",     # https://docs.astral.sh/ruff/rules/#isort-i
   "UP",    # https://docs.astral.sh/ruff/rules/#pyupgrade-up
   "COM",   # https://docs.astral.sh/ruff/rules/#flake8-commas-com
   "DJ",    # https://docs.astral.sh/ruff/rules/#flake8-django-dj
@@ -12,17 +12,20 @@ extend-select = [
   "G201",  # https://docs.astral.sh/ruff/rules/#flake8-logging-format-g
   "INP",   # https://docs.astral.sh/ruff/rules/#flake8-no-pep420-inp
   "PIE",   # https://docs.astral.sh/ruff/rules/#flake8-pie-pie
+  "Q",     # https://docs.astral.sh/ruff/rules/#flake8-quotes-q
   "RSE",   # https://docs.astral.sh/ruff/rules/#flake8-raise-rse
+  "T20",   # https://docs.astral.sh/ruff/rules/#flake8-print-t20
   "SIM",   # https://docs.astral.sh/ruff/rules/#flake8-simplify-sim
   "TID",   # https://docs.astral.sh/ruff/rules/#flake8-tidy-imports-tid
-  "T20",   # https://docs.astral.sh/ruff/rules/#flake8-print-t20
+  "TCH",   # https://docs.astral.sh/ruff/rules/#flake8-type-checking-tch
   "PLC",   # https://docs.astral.sh/ruff/rules/#pylint-pl
   "PLE",   # https://docs.astral.sh/ruff/rules/#pylint-pl
   "RUF",   # https://docs.astral.sh/ruff/rules/#ruff-specific-rules-ruf
+  "FLY",   # https://docs.astral.sh/ruff/rules/#flynt-fly
 ]
 # TODO PTH https://docs.astral.sh/ruff/rules/#flake8-use-pathlib-pth
 ignore = ["DJ001", "SIM105", "RUF012"]
-fix = true
+#fix = true
 line-length = 88
 respect-gitignore = true
 src = ["src"]
@@ -32,7 +35,7 @@ show-fixes = true
 
 [per-file-ignores]
 ".github/scripts/*.py" = ["E501", "INP001", "SIM117"]
-"docker/wait-for-redis.py" = ["INP001"]
+"docker/wait-for-redis.py" = ["INP001", "T201"]
 "*/tests/*.py" = ["E501", "SIM117"]
 "*/migrations/*.py" = ["E501", "SIM", "T201"]
 "src/paperless_tesseract/tests/test_parser.py" = ["RUF001"]

src/documents/classifier.py

@@ -4,11 +4,14 @@ import pickle
 import re
 import warnings
 from collections.abc import Iterator
-from datetime import datetime
 from hashlib import sha256
-from pathlib import Path
+from typing import TYPE_CHECKING
 from typing import Optional
 
+if TYPE_CHECKING:
+    from datetime import datetime
+    from pathlib import Path
+
 from django.conf import settings
 from django.core.cache import cache
 from sklearn.exceptions import InconsistentVersionWarning

src/documents/management/commands/document_retagger.py

@@ -69,8 +69,6 @@ class Command(ProgressBarMixin, BaseCommand):
 
     def handle(self, *args, **options):
         self.handle_progress_bar_mixin(**options)
-        # Detect if we support color
-        color = self.style.ERROR("test") != "test"
 
         if options["inbox_only"]:
             queryset = Document.objects.filter(tags__is_inbox_tag=True)
@@ -96,7 +94,8 @@ class Command(ProgressBarMixin, BaseCommand):
                     use_first=options["use_first"],
                     suggest=options["suggest"],
                     base_url=options["base_url"],
-                    color=color,
+                    stdout=self.stdout,
+                    style_func=self.style,
                 )
 
             if options["document_type"]:
@@ -108,7 +107,8 @@ class Command(ProgressBarMixin, BaseCommand):
                     use_first=options["use_first"],
                     suggest=options["suggest"],
                     base_url=options["base_url"],
-                    color=color,
+                    stdout=self.stdout,
+                    style_func=self.style,
                 )
 
             if options["tags"]:
@@ -119,7 +119,8 @@ class Command(ProgressBarMixin, BaseCommand):
                     replace=options["overwrite"],
                     suggest=options["suggest"],
                     base_url=options["base_url"],
-                    color=color,
+                    stdout=self.stdout,
+                    style_func=self.style,
                 )
             if options["storage_path"]:
                 set_storage_path(
@@ -130,5 +131,6 @@ class Command(ProgressBarMixin, BaseCommand):
                     use_first=options["use_first"],
                     suggest=options["suggest"],
                     base_url=options["base_url"],
-                    color=color,
+                    stdout=self.stdout,
+                    style_func=self.style,
                 )

src/documents/management/commands/document_thumbnails.py

@@ -19,7 +19,7 @@ def _process_document(doc_id):
     if parser_class:
         parser = parser_class(logging_group=None)
     else:
-        print(f"{document} No parser for mime type {document.mime_type}")
+        print(f"{document} No parser for mime type {document.mime_type}")  # noqa: T201
         return
 
     try:
@@ -72,7 +72,7 @@ class Command(MultiProcessMixin, ProgressBarMixin, BaseCommand):
 
         if self.process_count == 1:
             for doc_id in ids:
-                _process_document(doc_id)
+                self._process_document(doc_id)
         else:  # pragma: no cover
             with multiprocessing.Pool(processes=self.process_count) as pool:
                 list(

src/documents/plugins/helpers.py

@@ -5,7 +5,9 @@ from typing import Union
 
 from asgiref.sync import async_to_sync
 from channels.layers import get_channel_layer
-from channels_redis.pubsub import RedisPubSubChannelLayer
+
+if TYPE_CHECKING:
+    from channels_redis.pubsub import RedisPubSubChannelLayer
 
 
 class ProgressStatusOptions(str, enum.Enum):

src/documents/signals/handlers.py

@@ -18,7 +18,6 @@ from django.db import close_old_connections
 from django.db import models
 from django.db.models import Q
 from django.dispatch import receiver
-from django.utils import termcolors
 from django.utils import timezone
 from filelock import FileLock
 
@@ -54,6 +53,26 @@ def add_inbox_tags(sender, document: Document, logging_group=None, **kwargs):
     document.tags.add(*inbox_tags)
 
 
+def _suggestion_printer(
+    stdout,
+    style_func,
+    suggestion_type: str,
+    document: Document,
+    selected: MatchingModel,
+    base_url: Optional[str] = None,
+):
+    """
+    Smaller helper to reduce duplication when just outputting suggestions to the console
+    """
+    doc_str = str(document)
+    if base_url is not None:
+        stdout.write(style_func.SUCCESS(doc_str))
+        stdout.write(style_func.SUCCESS(f"{base_url}/documents/{document.pk}"))
+    else:
+        stdout.write(style_func.SUCCESS(f"{doc_str} [{document.pk}]"))
+    stdout.write(f"Suggest {suggestion_type}: {selected}")
+
+
 def set_correspondent(
     sender,
     document: Document,
@@ -63,7 +82,8 @@ def set_correspondent(
     use_first=True,
     suggest=False,
     base_url=None,
-    color=False,
+    stdout=None,
+    style_func=None,
     **kwargs,
 ):
     if document.correspondent and not replace:
@@ -90,23 +110,14 @@ def set_correspondent(
 
     if selected or replace:
         if suggest:
-            if base_url:
+            _suggestion_printer(
-                print(
+                stdout,
-                    termcolors.colorize(str(document), fg="green")
+                style_func,
-                    if color
+                "correspondent",
-                    else str(document),
+                document,
-                )
+                selected,
-                print(f"{base_url}/documents/{document.pk}")
+                base_url,
-            else:
+            )
-                print(
-                    (
-                        termcolors.colorize(str(document), fg="green")
-                        if color
-                        else str(document)
-                    )
-                    + f" [{document.pk}]",
-                )
-            print(f"Suggest correspondent {selected}")
         else:
             logger.info(
                 f"Assigning correspondent {selected} to {document}",
@@ -126,7 +137,8 @@ def set_document_type(
     use_first=True,
     suggest=False,
     base_url=None,
-    color=False,
+    stdout=None,
+    style_func=None,
     **kwargs,
 ):
     if document.document_type and not replace:
@@ -154,23 +166,14 @@ def set_document_type(
 
     if selected or replace:
         if suggest:
-            if base_url:
+            _suggestion_printer(
-                print(
+                stdout,
-                    termcolors.colorize(str(document), fg="green")
+                style_func,
-                    if color
+                "document type",
-                    else str(document),
+                document,
-                )
+                selected,
-                print(f"{base_url}/documents/{document.pk}")
+                base_url,
-            else:
+            )
-                print(
-                    (
-                        termcolors.colorize(str(document), fg="green")
-                        if color
-                        else str(document)
-                    )
-                    + f" [{document.pk}]",
-                )
-            print(f"Suggest document type {selected}")
         else:
             logger.info(
                 f"Assigning document type {selected} to {document}",
@@ -189,7 +192,8 @@ def set_tags(
     replace=False,
     suggest=False,
     base_url=None,
-    color=False,
+    stdout=None,
+    style_func=None,
     **kwargs,
 ):
     if replace:
@@ -212,26 +216,16 @@ def set_tags(
         ]
         if not relevant_tags and not extra_tags:
             return
+        doc_str = style_func.SUCCESS(str(document))
         if base_url:
-            print(
+            stdout.write(doc_str)
-                termcolors.colorize(str(document), fg="green")
+            stdout.write(f"{base_url}/documents/{document.pk}")
-                if color
-                else str(document),
-            )
-            print(f"{base_url}/documents/{document.pk}")
         else:
-            print(
+            stdout.write(doc_str + style_func.SUCCESS(f" [{document.pk}]"))
-                (
-                    termcolors.colorize(str(document), fg="green")
-                    if color
-                    else str(document)
-                )
-                + f" [{document.pk}]",
-            )
         if relevant_tags:
-            print("Suggest tags: " + ", ".join([t.name for t in relevant_tags]))
+            stdout.write("Suggest tags: " + ", ".join([t.name for t in relevant_tags]))
         if extra_tags:
-            print("Extra tags: " + ", ".join([t.name for t in extra_tags]))
+            stdout.write("Extra tags: " + ", ".join([t.name for t in extra_tags]))
     else:
         if not relevant_tags:
             return
@@ -254,7 +248,8 @@ def set_storage_path(
     use_first=True,
     suggest=False,
     base_url=None,
-    color=False,
+    stdout=None,
+    style_func=None,
     **kwargs,
 ):
     if document.storage_path and not replace:
@@ -285,23 +280,14 @@ def set_storage_path(
 
     if selected or replace:
         if suggest:
-            if base_url:
+            _suggestion_printer(
-                print(
+                stdout,
-                    termcolors.colorize(str(document), fg="green")
+                style_func,
-                    if color
+                "storage directory",
-                    else str(document),
+                document,
-                )
+                selected,
-                print(f"{base_url}/documents/{document.pk}")
+                base_url,
-            else:
+            )
-                print(
-                    (
-                        termcolors.colorize(str(document), fg="green")
-                        if color
-                        else str(document)
-                    )
-                    + f" [{document.pk}]",
-                )
-            print(f"Suggest storage directory {selected}")
         else:
             logger.info(
                 f"Assigning storage path {selected} to {document}",

src/documents/tests/test_workflows.py

@@ -1,16 +1,19 @@
 from datetime import timedelta
 from pathlib import Path
+from typing import TYPE_CHECKING
 from unittest import mock
 
 from django.contrib.auth.models import Group
 from django.contrib.auth.models import User
-from django.db.models import QuerySet
 from django.utils import timezone
 from guardian.shortcuts import assign_perm
 from guardian.shortcuts import get_groups_with_perms
 from guardian.shortcuts import get_users_with_perms
 from rest_framework.test import APITestCase
 
+if TYPE_CHECKING:
+    from django.db.models import QuerySet
+
 from documents import tasks
 from documents.data_models import ConsumableDocument
 from documents.data_models import DocumentSource