From fb82aa0ee1b84b8b5fab04116061d2ec4a5fa90b Mon Sep 17 00:00:00 2001 From: pkrahmer <5699756+pkrahmer@users.noreply.github.com> Date: Mon, 5 Feb 2024 18:38:19 +0100 Subject: [PATCH 1/8] Feature: Allow tagging by putting barcode stickers on documents (#5580) --- docs/configuration.md | 49 +++++++++++ paperless.conf.example | 2 + src/documents/barcodes.py | 62 +++++++++++++- src/documents/tests/test_barcodes.py | 123 +++++++++++++++++++++++++++ src/paperless/settings.py | 13 +++ 5 files changed, 248 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index f473921cb..e99e0a085 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1173,6 +1173,55 @@ combination with PAPERLESS_CONSUMER_BARCODE_UPSCALE bigger than 1.0. Defaults to "300" +#### [`PAPERLESS_CONSUMER_ENABLE_TAG_BARCODE=`](#PAPERLESS_CONSUMER_ENABLE_TAG_BARCODE) {#PAPERLESS_CONSUMER_ENABLE_TAG_BARCODE} + +: Enables the detection of barcodes in the scanned document and +assigns or creates tags if a properly formatted barcode is detected. + + The barcode must match one of the (configurable) regular expressions. + If the barcode text contains ',' (comma), it is split into multiple + barcodes which are individually processed for tagging. + + Matching is case insensitive. + + Defaults to false. + +#### [`PAPERLESS_CONSUMER_TAG_BARCODE_MAPPING=`](#PAPERLESS_CONSUMER_TAG_BARCODE_MAPPING) {#PAPERLESS_CONSUMER_TAG_BARCODE_MAPPING} + +: Defines a dictionary of filter regex and substitute expressions. + + Syntax: {"": "" [,...]]} + + A barcode is considered for tagging if the barcode text matches + at least one of the provided pattern. + + If a match is found, the rule is applied. This allows very + versatile reformatting and mapping of barcode pattern to tag values. + + If a tag is not found it will be created. + + Defaults to: + + {"TAG:(.*)": "\\g<1>"} which defines + - a regex TAG:(.*) which includes barcodes beginning with TAG: + followed by any text that gets stored into match group #1 and + - a substitute \\g<1> that replaces the original barcode text + by the content in match group #1. + Consequently, the tag is the barcode text without its TAG: prefix. + + More examples: + + {"ASN12.*": "JOHN", "ASN13.*": "SMITH"} for example maps + - ASN12nnnn barcodes to the tag JOHN and + - ASN13nnnn barcodes to the tag SMITH. + + {"T-J": "JOHN", "T-S": "SMITH", "T-D": "DOE"} directly maps + - T-J barcodes to the tag JOHN, + - T-S barcodes to the tag SMITH and + - T-D barcodes to the tag DOE. + + Please refer to the Python regex documentation for more information. + ## Audit Trail #### [`PAPERLESS_AUDIT_LOG_ENABLED=`](#PAPERLESS_AUDIT_LOG_ENABLED) {#PAPERLESS_AUDIT_LOG_ENABLED} diff --git a/paperless.conf.example b/paperless.conf.example index 1610dcda9..db557a7b6 100644 --- a/paperless.conf.example +++ b/paperless.conf.example @@ -68,6 +68,8 @@ #PAPERLESS_CONSUMER_BARCODE_STRING=PATCHT #PAPERLESS_CONSUMER_BARCODE_UPSCALE=0.0 #PAPERLESS_CONSUMER_BARCODE_DPI=300 +#PAPERLESS_CONSUMER_ENABLE_TAG_BARCODE=false +#PAPERLESS_CONSUMER_TAG_BARCODE_MAPPING={"TAG:(.*)": "\\g<1>"} #PAPERLESS_CONSUMER_ENABLE_COLLATE_DOUBLE_SIDED=false #PAPERLESS_CONSUMER_COLLATE_DOUBLE_SIDED_SUBDIR_NAME=double-sided #PAPERLESS_CONSUMER_COLLATE_DOUBLE_SIDED_TIFF_SUPPORT=false diff --git a/src/documents/barcodes.py b/src/documents/barcodes.py index 606451f84..4bfb9b791 100644 --- a/src/documents/barcodes.py +++ b/src/documents/barcodes.py @@ -14,6 +14,7 @@ from PIL import Image from documents.converters import convert_from_tiff_to_pdf from documents.data_models import ConsumableDocument +from documents.models import Tag from documents.plugins.base import ConsumeTaskPlugin from documents.plugins.base import StopConsumeTaskError from documents.plugins.helpers import ProgressStatusOptions @@ -65,7 +66,9 @@ class BarcodePlugin(ConsumeTaskPlugin): supported_mimes = {"application/pdf"} return ( - settings.CONSUMER_ENABLE_ASN_BARCODE or settings.CONSUMER_ENABLE_BARCODES + settings.CONSUMER_ENABLE_ASN_BARCODE + or settings.CONSUMER_ENABLE_BARCODES + or settings.CONSUMER_ENABLE_TAG_BARCODE ) and self.input_doc.mime_type in supported_mimes def setup(self): @@ -90,6 +93,16 @@ class BarcodePlugin(ConsumeTaskPlugin): logger.info(f"Found ASN in barcode: {located_asn}") self.metadata.asn = located_asn + # try reading tags from barcodes + if settings.CONSUMER_ENABLE_TAG_BARCODE: + tags = self.tags + if tags is not None and len(tags) > 0: + if self.metadata.tag_ids: + self.metadata.tag_ids += tags + else: + self.metadata.tag_ids = tags + logger.info(f"Found tags in barcode: {tags}") + separator_pages = self.get_separation_pages() if not separator_pages: return "No pages to split on!" @@ -279,6 +292,53 @@ class BarcodePlugin(ConsumeTaskPlugin): return asn + @property + def tags(self) -> Optional[list[int]]: + """ + Search the parsed barcodes for any tags. + Returns the detected tag ids (or empty list) + """ + tags = [] + + # Ensure the barcodes have been read + self.detect() + + for x in self.barcodes: + tag_texts = x.value + + for raw in tag_texts.split(","): + try: + tag = None + for regex in settings.CONSUMER_TAG_BARCODE_MAPPING: + if re.match(regex, raw, flags=re.IGNORECASE): + sub = settings.CONSUMER_TAG_BARCODE_MAPPING[regex] + tag = ( + re.sub(regex, sub, raw, flags=re.IGNORECASE) + if sub + else raw + ) + break + + if tag: + tag = Tag.objects.get_or_create( + name__iexact=tag, + defaults={"name": tag}, + )[0] + + logger.debug( + f"Found Tag Barcode '{raw}', substituted " + f"to '{tag}' and mapped to " + f"tag #{tag.pk}.", + ) + tags.append(tag.pk) + + except Exception as e: + logger.error( + f"Failed to find or create TAG '{raw}' because: {e}", + ) + + return tags + def get_separation_pages(self) -> dict[int, bool]: """ Search the parsed barcodes for separators and returns a dict of page diff --git a/src/documents/tests/test_barcodes.py b/src/documents/tests/test_barcodes.py index 4552a2b77..3dd6d62ff 100644 --- a/src/documents/tests/test_barcodes.py +++ b/src/documents/tests/test_barcodes.py @@ -14,6 +14,7 @@ from documents.barcodes import BarcodePlugin from documents.data_models import ConsumableDocument from documents.data_models import DocumentMetadataOverrides from documents.data_models import DocumentSource +from documents.models import Tag from documents.tests.utils import DirectoriesMixin from documents.tests.utils import DocumentConsumeDelayMixin from documents.tests.utils import DummyProgressManager @@ -741,3 +742,125 @@ class TestBarcodeZxing(TestBarcode): @override_settings(CONSUMER_BARCODE_SCANNER="ZXING") class TestAsnBarcodesZxing(TestAsnBarcode): pass + + +class TestTagBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, TestCase): + @contextmanager + def get_reader(self, filepath: Path) -> BarcodePlugin: + reader = BarcodePlugin( + ConsumableDocument(DocumentSource.ConsumeFolder, original_file=filepath), + DocumentMetadataOverrides(), + DummyProgressManager(filepath.name, None), + self.dirs.scratch_dir, + "task-id", + ) + reader.setup() + yield reader + reader.cleanup() + + @override_settings(CONSUMER_ENABLE_TAG_BARCODE=True) + def test_scan_file_without_matching_barcodes(self): + """ + GIVEN: + - PDF containing tag barcodes but none with matching prefix (default "TAG:") + WHEN: + - File is scanned for barcodes + THEN: + - No TAG has been created + """ + test_file = self.BARCODE_SAMPLE_DIR / "barcode-39-asn-custom-prefix.pdf" + with self.get_reader(test_file) as reader: + reader.run() + tags = reader.metadata.tag_ids + self.assertEqual(tags, None) + + @override_settings( + CONSUMER_ENABLE_TAG_BARCODE=False, + CONSUMER_TAG_BARCODE_MAPPING={"CUSTOM-PREFIX-(.*)": "\\g<1>"}, + ) + def test_scan_file_with_matching_barcode_but_function_disabled(self): + """ + GIVEN: + - PDF containing a tag barcode with matching custom prefix + - The tag barcode functionality is disabled + WHEN: + - File is scanned for barcodes + THEN: + - No TAG has been created + """ + test_file = self.BARCODE_SAMPLE_DIR / "barcode-39-asn-custom-prefix.pdf" + with self.get_reader(test_file) as reader: + reader.run() + tags = reader.metadata.tag_ids + self.assertEqual(tags, None) + + @override_settings( + CONSUMER_ENABLE_TAG_BARCODE=True, + CONSUMER_TAG_BARCODE_MAPPING={"CUSTOM-PREFIX-(.*)": "\\g<1>"}, + ) + def test_scan_file_for_tag_custom_prefix(self): + """ + GIVEN: + - PDF containing a tag barcode with custom prefix + - The barcode mapping accepts this prefix and removes it from the mapped tag value + - The created tag is the non-prefixed values + WHEN: + - File is scanned for barcodes + THEN: + - The TAG is located + - One TAG has been created + """ + test_file = self.BARCODE_SAMPLE_DIR / "barcode-39-asn-custom-prefix.pdf" + with self.get_reader(test_file) as reader: + reader.metadata.tag_ids = [99] + reader.run() + self.assertEqual(reader.pdf_file, test_file) + tags = reader.metadata.tag_ids + self.assertEqual(len(tags), 2) + self.assertEqual(tags[0], 99) + self.assertEqual(Tag.objects.get(name__iexact="00123").pk, tags[1]) + + @override_settings( + CONSUMER_ENABLE_TAG_BARCODE=True, + CONSUMER_TAG_BARCODE_MAPPING={"ASN(.*)": "\\g<1>"}, + ) + def test_scan_file_for_many_custom_tags(self): + """ + GIVEN: + - PDF containing multiple tag barcode with custom prefix + - The barcode mapping accepts this prefix and removes it from the mapped tag value + - The created tags are the non-prefixed values + WHEN: + - File is scanned for barcodes + THEN: + - The TAG is located + - File Tags have been created + """ + test_file = self.BARCODE_SAMPLE_DIR / "split-by-asn-1.pdf" + with self.get_reader(test_file) as reader: + reader.run() + tags = reader.metadata.tag_ids + self.assertEqual(len(tags), 5) + self.assertEqual(Tag.objects.get(name__iexact="00123").pk, tags[0]) + self.assertEqual(Tag.objects.get(name__iexact="00124").pk, tags[1]) + self.assertEqual(Tag.objects.get(name__iexact="00125").pk, tags[2]) + self.assertEqual(Tag.objects.get(name__iexact="00126").pk, tags[3]) + self.assertEqual(Tag.objects.get(name__iexact="00127").pk, tags[4]) + + @override_settings( + CONSUMER_ENABLE_TAG_BARCODE=True, + CONSUMER_TAG_BARCODE_MAPPING={"CUSTOM-PREFIX-(.*)": "\\g<3>"}, + ) + def test_scan_file_for_tag_raises_value_error(self): + """ + GIVEN: + - Any error occurs during tag barcode processing + THEN: + - The processing should be skipped and not break the import + """ + test_file = self.BARCODE_SAMPLE_DIR / "barcode-39-asn-custom-prefix.pdf" + with self.get_reader(test_file) as reader: + reader.run() + # expect error to be caught and logged only + tags = reader.metadata.tag_ids + self.assertEqual(tags, None) diff --git a/src/paperless/settings.py b/src/paperless/settings.py index 7179f0358..4f7894acc 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -853,6 +853,19 @@ CONSUMER_BARCODE_UPSCALE: Final[float] = __get_float( CONSUMER_BARCODE_DPI: Final[int] = __get_int("PAPERLESS_CONSUMER_BARCODE_DPI", 300) +CONSUMER_ENABLE_TAG_BARCODE: Final[bool] = __get_boolean( + "PAPERLESS_CONSUMER_ENABLE_TAG_BARCODE", +) + +CONSUMER_TAG_BARCODE_MAPPING = dict( + json.loads( + os.getenv( + "PAPERLESS_CONSUMER_TAG_BARCODE_MAPPING", + '{"TAG:(.*)": "\\\\g<1>"}', + ), + ), +) + CONSUMER_ENABLE_COLLATE_DOUBLE_SIDED: Final[bool] = __get_boolean( "PAPERLESS_CONSUMER_ENABLE_COLLATE_DOUBLE_SIDED", ) From 4813a7bc70d800a96538ebc5313bff6b57470c5d Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Mon, 5 Feb 2024 13:46:59 -0800 Subject: [PATCH 2/8] Chore: Adds additional rules for Ruff linter (#5660) --- .ruff.toml | 34 ++++- src/documents/caching.py | 1 - src/documents/classifier.py | 7 +- .../management/commands/document_retagger.py | 14 +- .../commands/document_thumbnails.py | 2 +- src/documents/plugins/helpers.py | 4 +- src/documents/signals/handlers.py | 130 ++++++++---------- src/documents/tests/test_api_bulk_download.py | 2 - .../tests/test_management_consumer.py | 8 +- src/documents/tests/test_management_fuzzy.py | 2 +- src/documents/tests/test_workflows.py | 5 +- src/documents/tests/utils.py | 1 - src/paperless/auth.py | 6 +- 13 files changed, 117 insertions(+), 99 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index c69de0ee1..497d3f1eb 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,7 +1,29 @@ -# https://beta.ruff.rs/docs/settings/ -# https://beta.ruff.rs/docs/rules/ -extend-select = ["I", "W", "UP", "COM", "DJ", "EXE", "ISC", "ICN", "G201", "INP", "PIE", "RSE", "SIM", "TID", "PLC", "PLE", "RUF"] -# TODO PTH +# https://docs.astral.sh/ruff/settings/ +# https://docs.astral.sh/ruff/rules/ +extend-select = [ + "W", # https://docs.astral.sh/ruff/rules/#pycodestyle-e-w + "I", # https://docs.astral.sh/ruff/rules/#isort-i + "UP", # https://docs.astral.sh/ruff/rules/#pyupgrade-up + "COM", # https://docs.astral.sh/ruff/rules/#flake8-commas-com + "DJ", # https://docs.astral.sh/ruff/rules/#flake8-django-dj + "EXE", # https://docs.astral.sh/ruff/rules/#flake8-executable-exe + "ISC", # https://docs.astral.sh/ruff/rules/#flake8-implicit-str-concat-isc + "ICN", # https://docs.astral.sh/ruff/rules/#flake8-import-conventions-icn + "G201", # https://docs.astral.sh/ruff/rules/#flake8-logging-format-g + "INP", # https://docs.astral.sh/ruff/rules/#flake8-no-pep420-inp + "PIE", # https://docs.astral.sh/ruff/rules/#flake8-pie-pie + "Q", # https://docs.astral.sh/ruff/rules/#flake8-quotes-q + "RSE", # https://docs.astral.sh/ruff/rules/#flake8-raise-rse + "T20", # https://docs.astral.sh/ruff/rules/#flake8-print-t20 + "SIM", # https://docs.astral.sh/ruff/rules/#flake8-simplify-sim + "TID", # https://docs.astral.sh/ruff/rules/#flake8-tidy-imports-tid + "TCH", # https://docs.astral.sh/ruff/rules/#flake8-type-checking-tch + "PLC", # https://docs.astral.sh/ruff/rules/#pylint-pl + "PLE", # https://docs.astral.sh/ruff/rules/#pylint-pl + "RUF", # https://docs.astral.sh/ruff/rules/#ruff-specific-rules-ruf + "FLY", # https://docs.astral.sh/ruff/rules/#flynt-fly +] +# TODO PTH https://docs.astral.sh/ruff/rules/#flake8-use-pathlib-pth ignore = ["DJ001", "SIM105", "RUF012"] fix = true line-length = 88 @@ -13,9 +35,9 @@ show-fixes = true [per-file-ignores] ".github/scripts/*.py" = ["E501", "INP001", "SIM117"] -"docker/wait-for-redis.py" = ["INP001"] +"docker/wait-for-redis.py" = ["INP001", "T201"] "*/tests/*.py" = ["E501", "SIM117"] -"*/migrations/*.py" = ["E501", "SIM"] +"*/migrations/*.py" = ["E501", "SIM", "T201"] "src/paperless_tesseract/tests/test_parser.py" = ["RUF001"] "src/documents/models.py" = ["SIM115"] diff --git a/src/documents/caching.py b/src/documents/caching.py index 9b8607dd8..d80f319f7 100644 --- a/src/documents/caching.py +++ b/src/documents/caching.py @@ -90,7 +90,6 @@ def set_suggestions_cache( """ if classifier is not None: doc_key = get_suggestion_cache_key(document_id) - print(classifier.last_auto_type_hash) cache.set( doc_key, SuggestionCacheData( diff --git a/src/documents/classifier.py b/src/documents/classifier.py index 6180a8671..aa0eb70b6 100644 --- a/src/documents/classifier.py +++ b/src/documents/classifier.py @@ -4,11 +4,14 @@ import pickle import re import warnings from collections.abc import Iterator -from datetime import datetime from hashlib import sha256 -from pathlib import Path +from typing import TYPE_CHECKING from typing import Optional +if TYPE_CHECKING: + from datetime import datetime + from pathlib import Path + from django.conf import settings from django.core.cache import cache from sklearn.exceptions import InconsistentVersionWarning diff --git a/src/documents/management/commands/document_retagger.py b/src/documents/management/commands/document_retagger.py index dda3ecebc..10bb54b71 100644 --- a/src/documents/management/commands/document_retagger.py +++ b/src/documents/management/commands/document_retagger.py @@ -69,8 +69,6 @@ class Command(ProgressBarMixin, BaseCommand): def handle(self, *args, **options): self.handle_progress_bar_mixin(**options) - # Detect if we support color - color = self.style.ERROR("test") != "test" if options["inbox_only"]: queryset = Document.objects.filter(tags__is_inbox_tag=True) @@ -96,7 +94,8 @@ class Command(ProgressBarMixin, BaseCommand): use_first=options["use_first"], suggest=options["suggest"], base_url=options["base_url"], - color=color, + stdout=self.stdout, + style_func=self.style, ) if options["document_type"]: @@ -108,7 +107,8 @@ class Command(ProgressBarMixin, BaseCommand): use_first=options["use_first"], suggest=options["suggest"], base_url=options["base_url"], - color=color, + stdout=self.stdout, + style_func=self.style, ) if options["tags"]: @@ -119,7 +119,8 @@ class Command(ProgressBarMixin, BaseCommand): replace=options["overwrite"], suggest=options["suggest"], base_url=options["base_url"], - color=color, + stdout=self.stdout, + style_func=self.style, ) if options["storage_path"]: set_storage_path( @@ -130,5 +131,6 @@ class Command(ProgressBarMixin, BaseCommand): use_first=options["use_first"], suggest=options["suggest"], base_url=options["base_url"], - color=color, + stdout=self.stdout, + style_func=self.style, ) diff --git a/src/documents/management/commands/document_thumbnails.py b/src/documents/management/commands/document_thumbnails.py index ecd265102..d4653f0b3 100644 --- a/src/documents/management/commands/document_thumbnails.py +++ b/src/documents/management/commands/document_thumbnails.py @@ -19,7 +19,7 @@ def _process_document(doc_id): if parser_class: parser = parser_class(logging_group=None) else: - print(f"{document} No parser for mime type {document.mime_type}") + print(f"{document} No parser for mime type {document.mime_type}") # noqa: T201 return try: diff --git a/src/documents/plugins/helpers.py b/src/documents/plugins/helpers.py index 92fe1255b..27d03f30f 100644 --- a/src/documents/plugins/helpers.py +++ b/src/documents/plugins/helpers.py @@ -5,7 +5,9 @@ from typing import Union from asgiref.sync import async_to_sync from channels.layers import get_channel_layer -from channels_redis.pubsub import RedisPubSubChannelLayer + +if TYPE_CHECKING: + from channels_redis.pubsub import RedisPubSubChannelLayer class ProgressStatusOptions(str, enum.Enum): diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py index 2b717e042..c8657ce1d 100644 --- a/src/documents/signals/handlers.py +++ b/src/documents/signals/handlers.py @@ -18,7 +18,6 @@ from django.db import close_old_connections from django.db import models from django.db.models import Q from django.dispatch import receiver -from django.utils import termcolors from django.utils import timezone from filelock import FileLock @@ -54,6 +53,26 @@ def add_inbox_tags(sender, document: Document, logging_group=None, **kwargs): document.tags.add(*inbox_tags) +def _suggestion_printer( + stdout, + style_func, + suggestion_type: str, + document: Document, + selected: MatchingModel, + base_url: Optional[str] = None, +): + """ + Smaller helper to reduce duplication when just outputting suggestions to the console + """ + doc_str = str(document) + if base_url is not None: + stdout.write(style_func.SUCCESS(doc_str)) + stdout.write(style_func.SUCCESS(f"{base_url}/documents/{document.pk}")) + else: + stdout.write(style_func.SUCCESS(f"{doc_str} [{document.pk}]")) + stdout.write(f"Suggest {suggestion_type}: {selected}") + + def set_correspondent( sender, document: Document, @@ -63,7 +82,8 @@ def set_correspondent( use_first=True, suggest=False, base_url=None, - color=False, + stdout=None, + style_func=None, **kwargs, ): if document.correspondent and not replace: @@ -90,23 +110,14 @@ def set_correspondent( if selected or replace: if suggest: - if base_url: - print( - termcolors.colorize(str(document), fg="green") - if color - else str(document), - ) - print(f"{base_url}/documents/{document.pk}") - else: - print( - ( - termcolors.colorize(str(document), fg="green") - if color - else str(document) - ) - + f" [{document.pk}]", - ) - print(f"Suggest correspondent {selected}") + _suggestion_printer( + stdout, + style_func, + "correspondent", + document, + selected, + base_url, + ) else: logger.info( f"Assigning correspondent {selected} to {document}", @@ -126,7 +137,8 @@ def set_document_type( use_first=True, suggest=False, base_url=None, - color=False, + stdout=None, + style_func=None, **kwargs, ): if document.document_type and not replace: @@ -154,23 +166,14 @@ def set_document_type( if selected or replace: if suggest: - if base_url: - print( - termcolors.colorize(str(document), fg="green") - if color - else str(document), - ) - print(f"{base_url}/documents/{document.pk}") - else: - print( - ( - termcolors.colorize(str(document), fg="green") - if color - else str(document) - ) - + f" [{document.pk}]", - ) - print(f"Suggest document type {selected}") + _suggestion_printer( + stdout, + style_func, + "document type", + document, + selected, + base_url, + ) else: logger.info( f"Assigning document type {selected} to {document}", @@ -189,7 +192,8 @@ def set_tags( replace=False, suggest=False, base_url=None, - color=False, + stdout=None, + style_func=None, **kwargs, ): if replace: @@ -212,26 +216,16 @@ def set_tags( ] if not relevant_tags and not extra_tags: return + doc_str = style_func.SUCCESS(str(document)) if base_url: - print( - termcolors.colorize(str(document), fg="green") - if color - else str(document), - ) - print(f"{base_url}/documents/{document.pk}") + stdout.write(doc_str) + stdout.write(f"{base_url}/documents/{document.pk}") else: - print( - ( - termcolors.colorize(str(document), fg="green") - if color - else str(document) - ) - + f" [{document.pk}]", - ) + stdout.write(doc_str + style_func.SUCCESS(f" [{document.pk}]")) if relevant_tags: - print("Suggest tags: " + ", ".join([t.name for t in relevant_tags])) + stdout.write("Suggest tags: " + ", ".join([t.name for t in relevant_tags])) if extra_tags: - print("Extra tags: " + ", ".join([t.name for t in extra_tags])) + stdout.write("Extra tags: " + ", ".join([t.name for t in extra_tags])) else: if not relevant_tags: return @@ -254,7 +248,8 @@ def set_storage_path( use_first=True, suggest=False, base_url=None, - color=False, + stdout=None, + style_func=None, **kwargs, ): if document.storage_path and not replace: @@ -285,23 +280,14 @@ def set_storage_path( if selected or replace: if suggest: - if base_url: - print( - termcolors.colorize(str(document), fg="green") - if color - else str(document), - ) - print(f"{base_url}/documents/{document.pk}") - else: - print( - ( - termcolors.colorize(str(document), fg="green") - if color - else str(document) - ) - + f" [{document.pk}]", - ) - print(f"Suggest storage directory {selected}") + _suggestion_printer( + stdout, + style_func, + "storage directory", + document, + selected, + base_url, + ) else: logger.info( f"Assigning storage path {selected} to {document}", diff --git a/src/documents/tests/test_api_bulk_download.py b/src/documents/tests/test_api_bulk_download.py index 57912c65c..43299b77d 100644 --- a/src/documents/tests/test_api_bulk_download.py +++ b/src/documents/tests/test_api_bulk_download.py @@ -246,8 +246,6 @@ class TestBulkDownload(DirectoriesMixin, APITestCase): self.doc3.title = "Title 2 - Doc 3" self.doc3.save() - print(self.doc3.archive_path) - print(self.doc3.archive_filename) response = self.client.post( self.ENDPOINT, diff --git a/src/documents/tests/test_management_consumer.py b/src/documents/tests/test_management_consumer.py index 99d5d410e..7e2707403 100644 --- a/src/documents/tests/test_management_consumer.py +++ b/src/documents/tests/test_management_consumer.py @@ -88,10 +88,10 @@ class ConsumerThreadMixin(DocumentConsumeDelayMixin): ): eq = filecmp.cmp(input_doc.original_file, self.sample_file, shallow=False) if not eq: - print("Consumed an INVALID file.") + print("Consumed an INVALID file.") # noqa: T201 raise ConsumerError("Incomplete File READ FAILED") else: - print("Consumed a perfectly valid file.") + print("Consumed a perfectly valid file.") # noqa: T201 def slow_write_file(self, target, incomplete=False): with open(self.sample_file, "rb") as f: @@ -102,11 +102,11 @@ class ConsumerThreadMixin(DocumentConsumeDelayMixin): with open(target, "wb") as f: # this will take 2 seconds, since the file is about 20k. - print("Start writing file.") + print("Start writing file.") # noqa: T201 for b in chunked(1000, pdf_bytes): f.write(b) sleep(0.1) - print("file completed.") + print("file completed.") # noqa: T201 @override_settings( diff --git a/src/documents/tests/test_management_fuzzy.py b/src/documents/tests/test_management_fuzzy.py index c215c43ca..7cc1f265e 100644 --- a/src/documents/tests/test_management_fuzzy.py +++ b/src/documents/tests/test_management_fuzzy.py @@ -196,7 +196,7 @@ class TestFuzzyMatchCommand(TestCase): self.assertEqual(Document.objects.count(), 3) stdout, _ = self.call_command("--delete") - print(stdout) + lines = [x.strip() for x in stdout.split("\n") if len(x.strip())] self.assertEqual(len(lines), 3) self.assertEqual( diff --git a/src/documents/tests/test_workflows.py b/src/documents/tests/test_workflows.py index ba5c53a78..95f903239 100644 --- a/src/documents/tests/test_workflows.py +++ b/src/documents/tests/test_workflows.py @@ -1,16 +1,19 @@ from datetime import timedelta from pathlib import Path +from typing import TYPE_CHECKING from unittest import mock from django.contrib.auth.models import Group from django.contrib.auth.models import User -from django.db.models import QuerySet from django.utils import timezone from guardian.shortcuts import assign_perm from guardian.shortcuts import get_groups_with_perms from guardian.shortcuts import get_users_with_perms from rest_framework.test import APITestCase +if TYPE_CHECKING: + from django.db.models import QuerySet + from documents import tasks from documents.data_models import ConsumableDocument from documents.data_models import DocumentSource diff --git a/src/documents/tests/utils.py b/src/documents/tests/utils.py index 4c3305d13..ba435d5c3 100644 --- a/src/documents/tests/utils.py +++ b/src/documents/tests/utils.py @@ -340,7 +340,6 @@ class DummyProgressManager: def __init__(self, filename: str, task_id: Optional[str] = None) -> None: self.filename = filename self.task_id = task_id - print("hello world") self.payloads = [] def __enter__(self): diff --git a/src/paperless/auth.py b/src/paperless/auth.py index 98e2a8b30..ba9320b5d 100644 --- a/src/paperless/auth.py +++ b/src/paperless/auth.py @@ -1,3 +1,5 @@ +import logging + from django.conf import settings from django.contrib import auth from django.contrib.auth.middleware import PersistentRemoteUserMiddleware @@ -6,6 +8,8 @@ from django.http import HttpRequest from django.utils.deprecation import MiddlewareMixin from rest_framework import authentication +logger = logging.getLogger("paperless.auth") + class AutoLoginMiddleware(MiddlewareMixin): def process_request(self, request: HttpRequest): @@ -35,7 +39,7 @@ class AngularApiAuthenticationOverride(authentication.BaseAuthentication): and request.headers["Referer"].startswith("http://localhost:4200/") ): user = User.objects.filter(is_staff=True).first() - print(f"Auto-Login with user {user}") + logger.debug(f"Auto-Login with user {user}") return (user, None) else: return None From 4606caeaa8785c16077094e526232a7a56b69e71 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue, 6 Feb 2024 07:16:15 -0800 Subject: [PATCH 3/8] Chore: Use memory cache backend in debug mode (#5666) --- src/paperless/settings.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/paperless/settings.py b/src/paperless/settings.py index 4f7894acc..d485415ca 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -771,6 +771,11 @@ CACHES = { }, } +if DEBUG and os.getenv("PAPERLESS_CACHE_BACKEND") is None: + CACHES["default"][ + "BACKEND" + ] = "django.core.cache.backends.locmem.LocMemCache" # pragma: no cover + def default_threads_per_worker(task_workers) -> int: # always leave one core open From aaa130e20de4f8fd178055c34b360b7fb11f0aa0 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue, 6 Feb 2024 07:31:07 -0800 Subject: [PATCH 4/8] Feature: allow create objects from bulk edit (#5667) --- .../filterable-dropdown.component.html | 16 +- .../filterable-dropdown.component.spec.ts | 42 ++++ .../filterable-dropdown.component.ts | 23 ++- .../bulk-editor/bulk-editor.component.html | 4 + .../bulk-editor/bulk-editor.component.spec.ts | 194 ++++++++++++++++++ .../bulk-editor/bulk-editor.component.ts | 93 ++++++++- 6 files changed, 364 insertions(+), 8 deletions(-) diff --git a/src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.html b/src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.html index 599faa988..cac217716 100644 --- a/src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.html +++ b/src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.html @@ -45,10 +45,18 @@ } @if (editing) { - + @if ((selectionModel.itemsSorted | filter: filterText).length === 0 && createRef !== undefined) { + + } + @if ((selectionModel.itemsSorted | filter: filterText).length > 0) { + + } } @if (!editing && manyToOne) {
diff --git a/src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.spec.ts b/src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.spec.ts index f88667f34..58aa029ee 100644 --- a/src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.spec.ts +++ b/src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.spec.ts @@ -500,4 +500,46 @@ describe('FilterableDropdownComponent & FilterableDropdownSelectionModel', () => selectionModel.apply() expect(selectionModel.itemsSorted).toEqual([nullItem, items[1], items[0]]) }) + + it('should set support create, keep open model and call createRef method', fakeAsync(() => { + component.items = items + component.icon = 'tag-fill' + component.selectionModel = selectionModel + fixture.nativeElement + .querySelector('button') + .dispatchEvent(new MouseEvent('click')) // open + fixture.detectChanges() + tick(100) + + component.filterText = 'Test Filter Text' + component.createRef = jest.fn() + component.createClicked() + expect(component.creating).toBeTruthy() + expect(component.createRef).toHaveBeenCalledWith('Test Filter Text') + const openSpy = jest.spyOn(component.dropdown, 'open') + component.dropdownOpenChange(false) + expect(openSpy).toHaveBeenCalled() // should keep open + })) + + it('should call create on enter inside filter field if 0 items remain while editing', fakeAsync(() => { + component.items = items + component.icon = 'tag-fill' + component.editing = true + component.createRef = jest.fn() + const createSpy = jest.spyOn(component, 'createClicked') + expect(component.selectionModel.getSelectedItems()).toEqual([]) + fixture.nativeElement + .querySelector('button') + .dispatchEvent(new MouseEvent('click')) // open + fixture.detectChanges() + tick(100) + component.filterText = 'FooBar' + fixture.detectChanges() + component.listFilterTextInput.nativeElement.dispatchEvent( + new KeyboardEvent('keyup', { key: 'Enter' }) + ) + expect(component.selectionModel.getSelectedItems()).toEqual([]) + tick(300) + expect(createSpy).toHaveBeenCalled() + })) }) diff --git a/src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.ts b/src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.ts index 26b036db9..bb1a9da27 100644 --- a/src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.ts +++ b/src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.ts @@ -398,6 +398,11 @@ export class FilterableDropdownComponent { @Input() disabled = false + @Input() + createRef: (name) => void + + creating: boolean = false + @Output() apply = new EventEmitter() @@ -437,6 +442,11 @@ export class FilterableDropdownComponent { } } + createClicked() { + this.creating = true + this.createRef(this.filterText) + } + dropdownOpenChange(open: boolean): void { if (open) { setTimeout(() => { @@ -448,9 +458,14 @@ export class FilterableDropdownComponent { } this.opened.next(this) } else { - this.filterText = '' - if (this.applyOnClose && this.selectionModel.isDirty()) { - this.apply.emit(this.selectionModel.diff()) + if (this.creating) { + this.dropdown.open() + this.creating = false + } else { + this.filterText = '' + if (this.applyOnClose && this.selectionModel.isDirty()) { + this.apply.emit(this.selectionModel.diff()) + } } } } @@ -466,6 +481,8 @@ export class FilterableDropdownComponent { this.dropdown.close() } }, 200) + } else if (filtered.length == 0 && this.createRef) { + this.createClicked() } } diff --git a/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.html b/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.html index 0c261df67..686c07bb3 100644 --- a/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.html +++ b/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.html @@ -25,6 +25,7 @@ [editing]="true" [manyToOne]="true" [applyOnClose]="applyOnClose" + [createRef]="createTag.bind(this)" (opened)="openTagsDropdown()" [(selectionModel)]="tagSelectionModel" [documentCounts]="tagDocumentCounts" @@ -38,6 +39,7 @@ [disabled]="!userCanEditAll" [editing]="true" [applyOnClose]="applyOnClose" + [createRef]="createCorrespondent.bind(this)" (opened)="openCorrespondentDropdown()" [(selectionModel)]="correspondentSelectionModel" [documentCounts]="correspondentDocumentCounts" @@ -51,6 +53,7 @@ [disabled]="!userCanEditAll" [editing]="true" [applyOnClose]="applyOnClose" + [createRef]="createDocumentType.bind(this)" (opened)="openDocumentTypeDropdown()" [(selectionModel)]="documentTypeSelectionModel" [documentCounts]="documentTypeDocumentCounts" @@ -64,6 +67,7 @@ [disabled]="!userCanEditAll" [editing]="true" [applyOnClose]="applyOnClose" + [createRef]="createStoragePath.bind(this)" (opened)="openStoragePathDropdown()" [(selectionModel)]="storagePathsSelectionModel" [documentCounts]="storagePathDocumentCounts" diff --git a/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.spec.ts b/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.spec.ts index 42f8b6d1d..4da9f36df 100644 --- a/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.spec.ts +++ b/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.spec.ts @@ -42,6 +42,16 @@ import { NgSelectModule } from '@ng-select/ng-select' import { GroupService } from 'src/app/services/rest/group.service' import { NgxBootstrapIconsModule, allIcons } from 'ngx-bootstrap-icons' import { SwitchComponent } from '../../common/input/switch/switch.component' +import { EditDialogMode } from '../../common/edit-dialog/edit-dialog.component' +import { TagEditDialogComponent } from '../../common/edit-dialog/tag-edit-dialog/tag-edit-dialog.component' +import { Results } from 'src/app/data/results' +import { Tag } from 'src/app/data/tag' +import { Correspondent } from 'src/app/data/correspondent' +import { DocumentType } from 'src/app/data/document-type' +import { StoragePath } from 'src/app/data/storage-path' +import { CorrespondentEditDialogComponent } from '../../common/edit-dialog/correspondent-edit-dialog/correspondent-edit-dialog.component' +import { DocumentTypeEditDialogComponent } from '../../common/edit-dialog/document-type-edit-dialog/document-type-edit-dialog.component' +import { StoragePathEditDialogComponent } from '../../common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component' const selectionData: SelectionData = { selected_tags: [ @@ -65,6 +75,10 @@ describe('BulkEditorComponent', () => { let documentService: DocumentService let toastService: ToastService let modalService: NgbModal + let tagService: TagService + let correspondentsService: CorrespondentService + let documentTypeService: DocumentTypeService + let storagePathService: StoragePathService let httpTestingController: HttpTestingController beforeEach(async () => { @@ -165,6 +179,10 @@ describe('BulkEditorComponent', () => { documentService = TestBed.inject(DocumentService) toastService = TestBed.inject(ToastService) modalService = TestBed.inject(NgbModal) + tagService = TestBed.inject(TagService) + correspondentsService = TestBed.inject(CorrespondentService) + documentTypeService = TestBed.inject(DocumentTypeService) + storagePathService = TestBed.inject(StoragePathService) httpTestingController = TestBed.inject(HttpTestingController) fixture = TestBed.createComponent(BulkEditorComponent) @@ -902,4 +920,180 @@ describe('BulkEditorComponent', () => { `${environment.apiBaseUrl}documents/storage_paths/` ) }) + + it('should support create new tag', () => { + const name = 'New Tag' + const newTag = { id: 101, name: 'New Tag' } + const tags: Results = { + results: [ + { id: 1, name: 'Tag 1' }, + { id: 2, name: 'Tag 2' }, + ], + count: 2, + all: [1, 2], + } + + const modalInstance = { + componentInstance: { + dialogMode: EditDialogMode.CREATE, + object: { name }, + succeeded: of(newTag), + }, + } + const tagListAllSpy = jest.spyOn(tagService, 'listAll') + tagListAllSpy.mockReturnValue(of(tags)) + + const tagSelectionModelToggleSpy = jest.spyOn( + component.tagSelectionModel, + 'toggle' + ) + + const modalServiceOpenSpy = jest.spyOn(modalService, 'open') + modalServiceOpenSpy.mockReturnValue(modalInstance as any) + + component.createTag(name) + + expect(modalServiceOpenSpy).toHaveBeenCalledWith(TagEditDialogComponent, { + backdrop: 'static', + }) + expect(tagListAllSpy).toHaveBeenCalled() + + expect(tagSelectionModelToggleSpy).toHaveBeenCalledWith(newTag.id) + expect(component.tags).toEqual(tags.results) + }) + + it('should support create new correspondent', () => { + const name = 'New Correspondent' + const newCorrespondent = { id: 101, name: 'New Correspondent' } + const correspondents: Results = { + results: [ + { id: 1, name: 'Correspondent 1' }, + { id: 2, name: 'Correspondent 2' }, + ], + count: 2, + all: [1, 2], + } + + const modalInstance = { + componentInstance: { + dialogMode: EditDialogMode.CREATE, + object: { name }, + succeeded: of(newCorrespondent), + }, + } + const correspondentsListAllSpy = jest.spyOn( + correspondentsService, + 'listAll' + ) + correspondentsListAllSpy.mockReturnValue(of(correspondents)) + + const correspondentSelectionModelToggleSpy = jest.spyOn( + component.correspondentSelectionModel, + 'toggle' + ) + + const modalServiceOpenSpy = jest.spyOn(modalService, 'open') + modalServiceOpenSpy.mockReturnValue(modalInstance as any) + + component.createCorrespondent(name) + + expect(modalServiceOpenSpy).toHaveBeenCalledWith( + CorrespondentEditDialogComponent, + { backdrop: 'static' } + ) + expect(correspondentsListAllSpy).toHaveBeenCalled() + + expect(correspondentSelectionModelToggleSpy).toHaveBeenCalledWith( + newCorrespondent.id + ) + expect(component.correspondents).toEqual(correspondents.results) + }) + + it('should support create new document type', () => { + const name = 'New Document Type' + const newDocumentType = { id: 101, name: 'New Document Type' } + const documentTypes: Results = { + results: [ + { id: 1, name: 'Document Type 1' }, + { id: 2, name: 'Document Type 2' }, + ], + count: 2, + all: [1, 2], + } + + const modalInstance = { + componentInstance: { + dialogMode: EditDialogMode.CREATE, + object: { name }, + succeeded: of(newDocumentType), + }, + } + const documentTypesListAllSpy = jest.spyOn(documentTypeService, 'listAll') + documentTypesListAllSpy.mockReturnValue(of(documentTypes)) + + const documentTypeSelectionModelToggleSpy = jest.spyOn( + component.documentTypeSelectionModel, + 'toggle' + ) + + const modalServiceOpenSpy = jest.spyOn(modalService, 'open') + modalServiceOpenSpy.mockReturnValue(modalInstance as any) + + component.createDocumentType(name) + + expect(modalServiceOpenSpy).toHaveBeenCalledWith( + DocumentTypeEditDialogComponent, + { backdrop: 'static' } + ) + expect(documentTypesListAllSpy).toHaveBeenCalled() + + expect(documentTypeSelectionModelToggleSpy).toHaveBeenCalledWith( + newDocumentType.id + ) + expect(component.documentTypes).toEqual(documentTypes.results) + }) + + it('should support create new storage path', () => { + const name = 'New Storage Path' + const newStoragePath = { id: 101, name: 'New Storage Path' } + const storagePaths: Results = { + results: [ + { id: 1, name: 'Storage Path 1' }, + { id: 2, name: 'Storage Path 2' }, + ], + count: 2, + all: [1, 2], + } + + const modalInstance = { + componentInstance: { + dialogMode: EditDialogMode.CREATE, + object: { name }, + succeeded: of(newStoragePath), + }, + } + const storagePathsListAllSpy = jest.spyOn(storagePathService, 'listAll') + storagePathsListAllSpy.mockReturnValue(of(storagePaths)) + + const storagePathsSelectionModelToggleSpy = jest.spyOn( + component.storagePathsSelectionModel, + 'toggle' + ) + + const modalServiceOpenSpy = jest.spyOn(modalService, 'open') + modalServiceOpenSpy.mockReturnValue(modalInstance as any) + + component.createStoragePath(name) + + expect(modalServiceOpenSpy).toHaveBeenCalledWith( + StoragePathEditDialogComponent, + { backdrop: 'static' } + ) + expect(storagePathsListAllSpy).toHaveBeenCalled() + + expect(storagePathsSelectionModelToggleSpy).toHaveBeenCalledWith( + newStoragePath.id + ) + expect(component.storagePaths).toEqual(storagePaths.results) + }) }) diff --git a/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.ts b/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.ts index 49d4c070f..0bfb287cb 100644 --- a/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.ts +++ b/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.ts @@ -33,7 +33,12 @@ import { PermissionType, } from 'src/app/services/permissions.service' import { FormControl, FormGroup } from '@angular/forms' -import { first, Subject, takeUntil } from 'rxjs' +import { first, map, Subject, switchMap, takeUntil } from 'rxjs' +import { CorrespondentEditDialogComponent } from '../../common/edit-dialog/correspondent-edit-dialog/correspondent-edit-dialog.component' +import { EditDialogMode } from '../../common/edit-dialog/edit-dialog.component' +import { TagEditDialogComponent } from '../../common/edit-dialog/tag-edit-dialog/tag-edit-dialog.component' +import { DocumentTypeEditDialogComponent } from '../../common/edit-dialog/document-type-edit-dialog/document-type-edit-dialog.component' +import { StoragePathEditDialogComponent } from '../../common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component' @Component({ selector: 'pngx-bulk-editor', @@ -479,6 +484,92 @@ export class BulkEditorComponent } } + createTag(name: string) { + let modal = this.modalService.open(TagEditDialogComponent, { + backdrop: 'static', + }) + modal.componentInstance.dialogMode = EditDialogMode.CREATE + modal.componentInstance.object = { name } + modal.componentInstance.succeeded + .pipe( + switchMap((newTag) => { + return this.tagService + .listAll() + .pipe(map((tags) => ({ newTag, tags }))) + }) + ) + .pipe(takeUntil(this.unsubscribeNotifier)) + .subscribe(({ newTag, tags }) => { + this.tags = tags.results + this.tagSelectionModel.toggle(newTag.id) + }) + } + + createCorrespondent(name: string) { + let modal = this.modalService.open(CorrespondentEditDialogComponent, { + backdrop: 'static', + }) + modal.componentInstance.dialogMode = EditDialogMode.CREATE + modal.componentInstance.object = { name } + modal.componentInstance.succeeded + .pipe( + switchMap((newCorrespondent) => { + return this.correspondentService + .listAll() + .pipe( + map((correspondents) => ({ newCorrespondent, correspondents })) + ) + }) + ) + .pipe(takeUntil(this.unsubscribeNotifier)) + .subscribe(({ newCorrespondent, correspondents }) => { + this.correspondents = correspondents.results + this.correspondentSelectionModel.toggle(newCorrespondent.id) + }) + } + + createDocumentType(name: string) { + let modal = this.modalService.open(DocumentTypeEditDialogComponent, { + backdrop: 'static', + }) + modal.componentInstance.dialogMode = EditDialogMode.CREATE + modal.componentInstance.object = { name } + modal.componentInstance.succeeded + .pipe( + switchMap((newDocumentType) => { + return this.documentTypeService + .listAll() + .pipe(map((documentTypes) => ({ newDocumentType, documentTypes }))) + }) + ) + .pipe(takeUntil(this.unsubscribeNotifier)) + .subscribe(({ newDocumentType, documentTypes }) => { + this.documentTypes = documentTypes.results + this.documentTypeSelectionModel.toggle(newDocumentType.id) + }) + } + + createStoragePath(name: string) { + let modal = this.modalService.open(StoragePathEditDialogComponent, { + backdrop: 'static', + }) + modal.componentInstance.dialogMode = EditDialogMode.CREATE + modal.componentInstance.object = { name } + modal.componentInstance.succeeded + .pipe( + switchMap((newStoragePath) => { + return this.storagePathService + .listAll() + .pipe(map((storagePaths) => ({ newStoragePath, storagePaths }))) + }) + ) + .pipe(takeUntil(this.unsubscribeNotifier)) + .subscribe(({ newStoragePath, storagePaths }) => { + this.storagePaths = storagePaths.results + this.storagePathsSelectionModel.toggle(newStoragePath.id) + }) + } + applyDelete() { let modal = this.modalService.open(ConfirmDialogComponent, { backdrop: 'static', From 718171a1258628558ef6110fada9cac6e79e8d92 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue, 6 Feb 2024 14:50:08 -0800 Subject: [PATCH 5/8] Resolve svg rem width/height attribute warning --- .../document-card-small.component.html | 14 +++++++------- .../document-list/document-list.component.html | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html index cdbd88825..ea9ba9914 100644 --- a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html +++ b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html @@ -25,7 +25,7 @@ @if (notesEnabled && document.notes.length) { - + {{document.notes.length}} } @@ -43,14 +43,14 @@ @if (document.document_type) { } @if (document.storage_path) { } @@ -63,25 +63,25 @@
- + {{document.created_date | customDate:'mediumDate'}}
@if (document.archive_serial_number | isNumber) {
- + #{{document.archive_serial_number}}
} @if (document.owner && document.owner !== settingsService.currentUser.id) {
- + {{document.owner | username}}
} @if (document.is_shared_by_requester) {
- + Shared
} diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html index 5de7ff6e7..5409306c6 100644 --- a/src-ui/src/app/components/document-list/document-list.component.html +++ b/src-ui/src/app/components/document-list/document-list.component.html @@ -232,7 +232,7 @@ @if (d.notes.length) { - + {{d.notes.length}} } From e98da2e72cc32a31a4db25a503504d2441d49bc9 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Wed, 7 Feb 2024 07:12:24 -0800 Subject: [PATCH 6/8] Fix whitespace in some frontend html files --- .../users-groups/users-groups.component.html | 112 +++++++-------- .../custom-fields.component.html | 24 ++-- .../manage/mail/mail.component.html | 130 +++++++++--------- .../manage/workflows/workflows.component.html | 24 ++-- 4 files changed, 145 insertions(+), 145 deletions(-) diff --git a/src-ui/src/app/components/admin/users-groups/users-groups.component.html b/src-ui/src/app/components/admin/users-groups/users-groups.component.html index 3f91842d4..a485a97ef 100644 --- a/src-ui/src/app/components/admin/users-groups/users-groups.component.html +++ b/src-ui/src/app/components/admin/users-groups/users-groups.component.html @@ -33,64 +33,64 @@
+ +
+ + + + } + +} + +@if (groups) { +

+ Groups + +

+ @if (groups.length > 0) { +
    +
  • +
    +
    Name
    +
    +
    +
    Actions
    +
    +
    • + @for (group of groups; track group) { +
  • +
    +
    +
    +
    +
    +
    + - -
    +
    -
    • - } -
- } - - @if (groups) { -

- Groups - -

- @if (groups.length > 0) { -
    -
  • -
    -
    Name
    -
    -
    -
    Actions
    -
    -
    • - @for (group of groups; track group) { -
  • -
    -
    -
    -
    -
    -
    - - -
    -
    -
    -
    • - } - @if (groups.length === 0) { -
  • No groups defined
    • - } -
- } - } - - @if (!users || !groups) { -
- -
- } + + } + @if (groups.length === 0) { +
  • No groups defined
    • + } + + } +} + +@if (!users || !groups) { +
    + + +
    +} diff --git a/src-ui/src/app/components/manage/custom-fields/custom-fields.component.html b/src-ui/src/app/components/manage/custom-fields/custom-fields.component.html index a803aae9c..2d4c77f93 100644 --- a/src-ui/src/app/components/manage/custom-fields/custom-fields.component.html +++ b/src-ui/src/app/components/manage/custom-fields/custom-fields.component.html @@ -29,16 +29,16 @@
    - -
    - + + - - } - @if (fields.length === 0) { -
  • No fields defined.
    • - } - + + + + } + @if (fields.length === 0) { +
  • No fields defined.
    • + } + diff --git a/src-ui/src/app/components/manage/mail/mail.component.html b/src-ui/src/app/components/manage/mail/mail.component.html index 68187c6bd..ef294fe43 100644 --- a/src-ui/src/app/components/manage/mail/mail.component.html +++ b/src-ui/src/app/components/manage/mail/mail.component.html @@ -32,72 +32,72 @@
    - - -
    - - - - } - @if (mailAccounts.length === 0) { -
  • No mail accounts defined.
    • - } - - - - - -

    - Mail rules - -

    -
      -
    • -
      -
      Name
      -
      Account
      -
      Actions
      + + +
      -
      • + + + + } + @if (mailAccounts.length === 0) { +
    • No mail accounts defined.
      • + } +
    - @for (rule of mailRules; track rule) { -
  • -
    -
    -
    {{(mailAccountService.getCached(rule.account) | async)?.name}}
    -
    -
    - - - -
    -
    -
    -
    • - } - @if (mailRules.length === 0) { -
  • No mail rules defined.
    • - } - +     - + +

    + Mail rules + +

    +
      +
    • +
      +
      Name
      +
      Account
      +
      Actions
      +
      +
      • - @if (!mailAccounts || !mailRules) { -
      - - -
      - } + @for (rule of mailRules; track rule) { +
    • +
      +
      +
      {{(mailAccountService.getCached(rule.account) | async)?.name}}
      +
      +
      + + + +
      +
      +
      +
      • + } + @if (mailRules.length === 0) { +
    • No mail rules defined.
      • + } +
    + +     + +@if (!mailAccounts || !mailRules) { +
    + + +
    +} diff --git a/src-ui/src/app/components/manage/workflows/workflows.component.html b/src-ui/src/app/components/manage/workflows/workflows.component.html index 9dc214af4..9bc68fb49 100644 --- a/src-ui/src/app/components/manage/workflows/workflows.component.html +++ b/src-ui/src/app/components/manage/workflows/workflows.component.html @@ -33,16 +33,16 @@
    - -
    - + + - - } - @if (workflows.length === 0) { -
  • No workflows defined.
    • - } - + + + + } + @if (workflows.length === 0) { +
  • No workflows defined.
    • + } + From b47f30183114a73ef744aef1012d70c2cd56971e Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Wed, 7 Feb 2024 18:45:59 -0800 Subject: [PATCH 7/8] Adds the Python environment flags for no byte code saving, reducing the image size slightly (#5677) --- Dockerfile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 73bd0cf57..ea5bd6026 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,8 +39,6 @@ RUN set -eux \ # - Don't leave anything extra in here FROM docker.io/python:3.11-slim-bookworm as main-app -ENV PYTHONWARNINGS="ignore:::django.http.response:517" - LABEL org.opencontainers.image.authors="paperless-ngx team " LABEL org.opencontainers.image.documentation="https://docs.paperless-ngx.com/" LABEL org.opencontainers.image.source="https://github.com/paperless-ngx/paperless-ngx" @@ -57,6 +55,12 @@ ARG JBIG2ENC_VERSION=0.29 ARG QPDF_VERSION=11.6.4 ARG GS_VERSION=10.02.1 +# Set Python environment variables +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + # Ignore warning from Whitenoise + PYTHONWARNINGS="ignore:::django.http.response:517" + # # Begin installation and configuration # Order the steps below from least often changed to most From c508be6ecd417863b87662fa664952e97487e41c Mon Sep 17 00:00:00 2001 From: Moritz Pflanzer Date: Thu, 8 Feb 2024 17:15:38 +0100 Subject: [PATCH 8/8] Feature: OIDC & social authentication (#5190) --------- Co-authored-by: Moritz Pflanzer Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com> --- Pipfile | 1 + Pipfile.lock | 8 + docs/advanced_usage.md | 39 +++ docs/configuration.md | 44 +++ src-ui/messages.xlf | 73 +++- .../app-frame/app-frame.component.spec.ts | 22 ++ .../app-frame/app-frame.component.ts | 21 +- .../profile-edit-dialog.component.html | 37 ++ .../profile-edit-dialog.component.spec.ts | 73 ++++ .../profile-edit-dialog.component.ts | 31 ++ src-ui/src/app/data/user-profile.ts | 13 + .../services/django-messages.service.spec.ts | 30 ++ .../app/services/django-messages.service.ts | 27 ++ .../src/app/services/profile.service.spec.ts | 16 + src-ui/src/app/services/profile.service.ts | 18 +- .../{registration => account}/login.html | 36 +- .../password_reset.html} | 6 +- .../password_reset_done.html | 0 .../password_reset_from_key.html} | 14 +- .../password_reset_from_key_done.html} | 2 +- src/documents/templates/index.html | 7 + .../authentication_error.html} | 14 +- .../templates/socialaccount/login.html | 52 +++ .../templates/socialaccount/signup.html | 77 +++++ src/documents/tests/test_api_profile.py | 198 ++++++++++- .../tests/test_management_exporter.py | 18 +- src/locale/en_US/LC_MESSAGES/django.po | 322 ++++++++++-------- src/paperless/adapter.py | 30 ++ src/paperless/serialisers.py | 23 ++ src/paperless/settings.py | 25 ++ src/paperless/tests/test_adapter.py | 43 +++ src/paperless/urls.py | 12 +- src/paperless/views.py | 55 +++ 33 files changed, 1197 insertions(+), 190 deletions(-) create mode 100644 src-ui/src/app/services/django-messages.service.spec.ts create mode 100644 src-ui/src/app/services/django-messages.service.ts rename src/documents/templates/{registration => account}/login.html (83%) rename src/documents/templates/{registration/password_reset_form.html => account/password_reset.html} (96%) rename src/documents/templates/{registration => account}/password_reset_done.html (100%) rename src/documents/templates/{registration/password_reset_confirm.html => account/password_reset_from_key.html} (92%) rename src/documents/templates/{registration/password_reset_complete.html => account/password_reset_from_key_done.html} (99%) rename src/documents/templates/{registration/logged_out.html => socialaccount/authentication_error.html} (93%) create mode 100644 src/documents/templates/socialaccount/login.html create mode 100644 src/documents/templates/socialaccount/signup.html create mode 100644 src/paperless/adapter.py create mode 100644 src/paperless/tests/test_adapter.py diff --git a/Pipfile b/Pipfile index afc294412..6d5c7f861 100644 --- a/Pipfile +++ b/Pipfile @@ -8,6 +8,7 @@ dateparser = "~=1.2" # WARNING: django does not use semver. # Only patch versions are guaranteed to not introduce breaking changes. django = "~=4.2.9" +django-allauth = "*" django-auditlog = "*" django-celery-results = "*" django-compression-middleware = "*" diff --git a/Pipfile.lock b/Pipfile.lock index ceef1c8a9..d177517c2 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -452,6 +452,14 @@ "markers": "python_version >= '3.8'", "version": "==4.2.9" }, + "django-allauth": { + "hashes": [ + "sha256:ec19efb80b34d2f18bd831eab9b10b6301f58d1cce9f39af35f497b7e5b0a141" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==0.59.0" + }, "django-auditlog": { "hashes": [ "sha256:7bc2c87e4aff62dec9785d1b2359a2b27148f8c286f8a52b9114fc7876c5a9f7", diff --git a/docs/advanced_usage.md b/docs/advanced_usage.md index 04626fe41..46d9c2b4b 100644 --- a/docs/advanced_usage.md +++ b/docs/advanced_usage.md @@ -640,3 +640,42 @@ single-sided split marker page, the split document(s) will have an empty page at whatever else was on the backside of the split marker page.) You can work around that by having a split marker page that has the split barcode on _both_ sides. This way, the extra page will get automatically removed. + +## SSO and third party authentication with Paperless-ngx + +Paperless-ngx has a built-in authentication system from Django but you can easily integrate an +external authentication solution using one of the following methods: + +### Remote User authentication + +This is a simple option that uses remote user authentication made available by certain SSO +applications. See the relevant configuration options for more information: +[PAPERLESS_ENABLE_HTTP_REMOTE_USER](configuration.md#PAPERLESS_ENABLE_HTTP_REMOTE_USER) and +[PAPERLESS_HTTP_REMOTE_USER_HEADER_NAME](configuration.md#PAPERLESS_HTTP_REMOTE_USER_HEADER_NAME) + +### OpenID Connect and social authentication + +Version 2.5.0 of Paperless-ngx added support for integrating other authentication systems via +the [django-allauth](https://github.com/pennersr/django-allauth) package. Once set up, users +can either log in or (optionally) sign up using any third party systems you integrate. See the +relevant [configuration settings](configuration.md#PAPERLESS_SOCIALACCOUNT_PROVIDERS) and +[django-allauth docs](https://docs.allauth.org/en/latest/socialaccount/configuration.html) +for more information. + +As an example, to set up login via Github, the following environment variables would need to be +set: + +```conf +PAPERLESS_APPS="allauth.socialaccount.providers.github" +PAPERLESS_SOCIALACCOUNT_PROVIDERS='{"github": {"APPS": [{"provider_id": "github","name": "Github","client_id": "","secret": ""}]}}' +``` + +Or, to use OpenID Connect ("OIDC"), via Keycloak in this example: + +```conf +PAPERLESS_APPS="allauth.socialaccount.providers.openid_connect" +PAPERLESS_SOCIALACCOUNT_PROVIDERS=' +{"openid_connect": {"APPS": [{"provider_id": "keycloak","name": "Keycloak","client_id": "paperless","secret": "","settings": { "server_url": "https:///realms//.well-known/openid-configuration"}}]}}' +``` + +More details about configuration option for various providers can be found in the allauth documentation: https://docs.allauth.org/en/latest/socialaccount/providers/index.html#provider-specifics diff --git a/docs/configuration.md b/docs/configuration.md index e99e0a085..3d1b1d1d1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -535,6 +535,42 @@ This is for use with self-signed certificates against local IMAP servers. Settings this value has security implications for the security of your email. Understand what it does and be sure you need to before setting. +#### [`PAPERLESS_SOCIALACCOUNT_PROVIDERS=`](#PAPERLESS_SOCIALACCOUNT_PROVIDERS) {#PAPERLESS_SOCIALACCOUNT_PROVIDERS} + +: This variable is used to setup login and signup via social account providers which are compatible with django-allauth. +See the corresponding [django-allauth documentation](https://docs.allauth.org/en/0.60.0/socialaccount/providers/index.html) +for a list of provider configurations. You will also likely need to include the relevant Django 'application' inside the +[PAPERLESS_APPS](#PAPERLESS_APPS) setting. + + Defaults to None, which does not enable any third party authentication systems. + +#### [`PAPERLESS_SOCIAL_AUTO_SIGNUP=`](#PAPERLESS_SOCIAL_AUTO_SIGNUP) {#PAPERLESS_SOCIAL_AUTO_SIGNUP} + +: Attempt to signup the user using retrieved email, username etc from the third party authentication +system. See the corresponding +[django-allauth documentation](https://docs.allauth.org/en/0.60.0/socialaccount/configuration.html) + + Defaults to False + +#### [`PAPERLESS_SOCIALACCOUNT_ALLOW_SIGNUPS=`](#PAPERLESS_SOCIALACCOUNT_ALLOW_SIGNUPS) {#PAPERLESS_SOCIALACCOUNT_ALLOW_SIGNUPS} + +: Allow users to signup for a new Paperless-ngx account using any setup third party authentication systems. + + Defaults to True + +#### [`PAPERLESS_ACCOUNT_ALLOW_SIGNUPS=`](#PAPERLESS_ACCOUNT_ALLOW_SIGNUPS) {#PAPERLESS_ACCOUNT_ALLOW_SIGNUPS} + +: Allow users to signup for a new Paperless-ngx account. + + Defaults to False + +#### [`PAPERLESS_ACCOUNT_DEFAULT_HTTP_PROTOCOL=`](#PAPERLESS_ACCOUNT_DEFAULT_HTTP_PROTOCOL) {#PAPERLESS_ACCOUNT_DEFAULT_HTTP_PROTOCOL} + +: The protocol used when generating URLs, e.g. login callback URLs. See the corresponding +[django-allauth documentation](https://docs.allauth.org/en/latest/account/configuration.html) + + Defaults to 'https' + ## OCR settings {#ocr} Paperless uses [OCRmyPDF](https://ocrmypdf.readthedocs.io/en/latest/) @@ -905,6 +941,14 @@ documents. Default is none, which disables the temporary directory. +#### [`PAPERLESS_APPS=`](#PAPERLESS_APPS) {#PAPERLESS_APPS} + +: A comma-separated list of Django apps to be included in Django's +[`INSTALLED_APPS`](https://docs.djangoproject.com/en/5.0/ref/applications/). This setting should +be used with caution! + + Defaults to None, which does not add any additional apps. + ## Document Consumption {#consume_config} #### [`PAPERLESS_CONSUMER_DELETE_DUPLICATES=`](#PAPERLESS_CONSUMER_DELETE_DUPLICATES) {#PAPERLESS_CONSUMER_DELETE_DUPLICATES} diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf index 9f163e3b8..f2b356d9a 100644 --- a/src-ui/messages.xlf +++ b/src-ui/messages.xlf @@ -502,7 +502,7 @@ src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html - 55 + 92 src/app/components/document-detail/document-detail.component.html @@ -1563,7 +1563,7 @@ src/app/components/app-frame/app-frame.component.ts - 121 + 140 @@ -1938,7 +1938,7 @@ src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts - 145 + 159 @@ -2405,21 +2405,21 @@ Sidebar views updated src/app/components/app-frame/app-frame.component.ts - 263 + 282 Error updating sidebar views src/app/components/app-frame/app-frame.component.ts - 266 + 285 An error occurred while saving update checking settings. src/app/components/app-frame/app-frame.component.ts - 287 + 306 @@ -2523,7 +2523,7 @@ src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html - 54 + 91 src/app/components/common/select-dialog/select-dialog.component.html @@ -4103,39 +4103,88 @@ 50 + + Connected social accounts + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html + 54 + + + + Set a password before disconnecting social account. + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html + 58 + + + + Disconnect social account + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html + 68 + + + + Disconnect + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html + 69 + + + + Warning: disconnecting social accounts cannot be undone + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html + 74 + + + + Connect new social account + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html + 79 + + Emails must match src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts - 94 + 108 Passwords must match src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts - 122 + 136 Profile updated successfully src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts - 142 + 156 Error saving profile src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts - 154 + 168 Error generating auth token src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts - 171 + 185 + + + + Error disconnecting social account + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts + 210 diff --git a/src-ui/src/app/components/app-frame/app-frame.component.spec.ts b/src-ui/src/app/components/app-frame/app-frame.component.spec.ts index 64877bb09..e1a553047 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.spec.ts +++ b/src-ui/src/app/components/app-frame/app-frame.component.spec.ts @@ -21,6 +21,10 @@ import { IfPermissionsDirective } from 'src/app/directives/if-permissions.direct import { FormsModule, ReactiveFormsModule } from '@angular/forms' import { of, throwError } from 'rxjs' import { ToastService } from 'src/app/services/toast.service' +import { + DjangoMessageLevel, + DjangoMessagesService, +} from 'src/app/services/django-messages.service' import { environment } from 'src/environments/environment' import { OpenDocumentsService } from 'src/app/services/open-documents.service' import { ActivatedRoute, Router } from '@angular/router' @@ -83,6 +87,7 @@ describe('AppFrameComponent', () => { let permissionsService: PermissionsService let remoteVersionService: RemoteVersionService let toastService: ToastService + let messagesService: DjangoMessagesService let openDocumentsService: OpenDocumentsService let searchService: SearchService let documentListViewService: DocumentListViewService @@ -123,6 +128,7 @@ describe('AppFrameComponent', () => { RemoteVersionService, IfPermissionsDirective, ToastService, + DjangoMessagesService, OpenDocumentsService, SearchService, NgbModal, @@ -151,6 +157,7 @@ describe('AppFrameComponent', () => { permissionsService = TestBed.inject(PermissionsService) remoteVersionService = TestBed.inject(RemoteVersionService) toastService = TestBed.inject(ToastService) + messagesService = TestBed.inject(DjangoMessagesService) openDocumentsService = TestBed.inject(OpenDocumentsService) searchService = TestBed.inject(SearchService) documentListViewService = TestBed.inject(DocumentListViewService) @@ -393,4 +400,19 @@ describe('AppFrameComponent', () => { backdrop: 'static', }) }) + + it('should show toasts for django messages', () => { + const toastErrorSpy = jest.spyOn(toastService, 'showError') + const toastInfoSpy = jest.spyOn(toastService, 'showInfo') + jest.spyOn(messagesService, 'get').mockReturnValue([ + { level: DjangoMessageLevel.WARNING, message: 'Test warning' }, + { level: DjangoMessageLevel.ERROR, message: 'Test error' }, + { level: DjangoMessageLevel.SUCCESS, message: 'Test success' }, + { level: DjangoMessageLevel.INFO, message: 'Test info' }, + { level: DjangoMessageLevel.DEBUG, message: 'Test debug' }, + ]) + component.ngOnInit() + expect(toastErrorSpy).toHaveBeenCalledTimes(2) + expect(toastInfoSpy).toHaveBeenCalledTimes(3) + }) }) diff --git a/src-ui/src/app/components/app-frame/app-frame.component.ts b/src-ui/src/app/components/app-frame/app-frame.component.ts index cfc9740a4..ab9322380 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.ts +++ b/src-ui/src/app/components/app-frame/app-frame.component.ts @@ -12,6 +12,10 @@ import { } from 'rxjs/operators' import { Document } from 'src/app/data/document' import { OpenDocumentsService } from 'src/app/services/open-documents.service' +import { + DjangoMessageLevel, + DjangoMessagesService, +} from 'src/app/services/django-messages.service' import { SavedViewService } from 'src/app/services/rest/saved-view.service' import { SearchService } from 'src/app/services/rest/search.service' import { environment } from 'src/environments/environment' @@ -73,7 +77,8 @@ export class AppFrameComponent public tasksService: TasksService, private readonly toastService: ToastService, private modalService: NgbModal, - permissionsService: PermissionsService + public permissionsService: PermissionsService, + private djangoMessagesService: DjangoMessagesService ) { super() @@ -92,6 +97,20 @@ export class AppFrameComponent this.checkForUpdates() } this.tasksService.reload() + + this.djangoMessagesService.get().forEach((message) => { + switch (message.level) { + case DjangoMessageLevel.ERROR: + case DjangoMessageLevel.WARNING: + this.toastService.showError(message.message) + break + case DjangoMessageLevel.SUCCESS: + case DjangoMessageLevel.INFO: + case DjangoMessageLevel.DEBUG: + this.toastService.showInfo(message.message) + break + } + }) } toggleSlimSidebar(): void { diff --git a/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html b/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html index 394ba4449..6b06dfa8e 100644 --- a/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html +++ b/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html @@ -49,6 +49,43 @@
    Warning: changing the token cannot be undone
    + @if (socialAccounts?.length > 0) { +
    +

    Connected social accounts

    +
      + @for (account of socialAccounts; track account.id) { +
    • + {{account.name}} ({{account.provider}}) + +
      • + } +
    +
    Warning: disconnecting social accounts cannot be undone
    +
    + } + @if (socialAccountProviders?.length > 0) { +
    +

    Connect new social account

    +
    + @for (provider of socialAccountProviders; track provider.name) { + + {{provider.name}}  + + } +
    +
    + }
    diff --git a/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.spec.ts b/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.spec.ts index 5deb26c8d..36888d4bd 100644 --- a/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.spec.ts +++ b/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.spec.ts @@ -12,6 +12,7 @@ import { NgbAccordionModule, NgbActiveModal, NgbModalModule, + NgbPopoverModule, } from '@ng-bootstrap/ng-bootstrap' import { HttpClientModule } from '@angular/common/http' import { TextComponent } from '../input/text/text.component' @@ -21,13 +22,22 @@ import { ToastService } from 'src/app/services/toast.service' import { Clipboard } from '@angular/cdk/clipboard' import { NgxBootstrapIconsModule, allIcons } from 'ngx-bootstrap-icons' +const socialAccount = { + id: 1, + provider: 'test_provider', + name: 'Test Provider', +} const profile = { email: 'foo@bar.com', password: '*********', first_name: 'foo', last_name: 'bar', auth_token: '123456789abcdef', + social_accounts: [socialAccount], } +const socialAccountProviders = [ + { name: 'Test Provider', login_url: 'https://example.com' }, +] describe('ProfileEditDialogComponent', () => { let component: ProfileEditDialogComponent @@ -51,6 +61,7 @@ describe('ProfileEditDialogComponent', () => { NgbModalModule, NgbAccordionModule, NgxBootstrapIconsModule.pick(allIcons), + NgbPopoverModule, ], }) profileService = TestBed.inject(ProfileService) @@ -64,6 +75,11 @@ describe('ProfileEditDialogComponent', () => { it('should get profile on init, display in form', () => { const getSpy = jest.spyOn(profileService, 'get') getSpy.mockReturnValue(of(profile)) + const getProvidersSpy = jest.spyOn( + profileService, + 'getSocialAccountProviders' + ) + getProvidersSpy.mockReturnValue(of(socialAccountProviders)) component.ngOnInit() expect(getSpy).toHaveBeenCalled() fixture.detectChanges() @@ -103,6 +119,11 @@ describe('ProfileEditDialogComponent', () => { expect(component.form.get('email_confirm').enabled).toBeFalsy() const getSpy = jest.spyOn(profileService, 'get') getSpy.mockReturnValue(of(profile)) + const getProvidersSpy = jest.spyOn( + profileService, + 'getSocialAccountProviders' + ) + getProvidersSpy.mockReturnValue(of(socialAccountProviders)) component.ngOnInit() component.form.get('email').patchValue('foo@bar2.com') component.onEmailKeyUp({ target: { value: 'foo@bar2.com' } } as any) @@ -134,6 +155,12 @@ describe('ProfileEditDialogComponent', () => { expect(component.form.get('password_confirm').enabled).toBeFalsy() const getSpy = jest.spyOn(profileService, 'get') getSpy.mockReturnValue(of(profile)) + const getProvidersSpy = jest.spyOn( + profileService, + 'getSocialAccountProviders' + ) + getProvidersSpy.mockReturnValue(of(socialAccountProviders)) + component.hasUsablePassword = true component.ngOnInit() component.form.get('password').patchValue('new*pass') component.onPasswordKeyUp({ @@ -167,6 +194,11 @@ describe('ProfileEditDialogComponent', () => { it('should logout on save if password changed', fakeAsync(() => { const getSpy = jest.spyOn(profileService, 'get') getSpy.mockReturnValue(of(profile)) + const getProvidersSpy = jest.spyOn( + profileService, + 'getSocialAccountProviders' + ) + getProvidersSpy.mockReturnValue(of(socialAccountProviders)) component.ngOnInit() component['newPassword'] = 'new*pass' component.form.get('password').patchValue('new*pass') @@ -189,6 +221,11 @@ describe('ProfileEditDialogComponent', () => { it('should support auth token copy', fakeAsync(() => { const getSpy = jest.spyOn(profileService, 'get') getSpy.mockReturnValue(of(profile)) + const getProvidersSpy = jest.spyOn( + profileService, + 'getSocialAccountProviders' + ) + getProvidersSpy.mockReturnValue(of(socialAccountProviders)) component.ngOnInit() const copySpy = jest.spyOn(clipboard, 'copy') component.copyAuthToken() @@ -220,4 +257,40 @@ describe('ProfileEditDialogComponent', () => { ) expect(component.form.get('auth_token').value).toEqual(newToken) }) + + it('should get social account providers on init', () => { + const getSpy = jest.spyOn(profileService, 'get') + getSpy.mockReturnValue(of(profile)) + const getProvidersSpy = jest.spyOn( + profileService, + 'getSocialAccountProviders' + ) + getProvidersSpy.mockReturnValue(of(socialAccountProviders)) + component.ngOnInit() + expect(getProvidersSpy).toHaveBeenCalled() + }) + + it('should remove disconnected social account from component, show error if needed', () => { + const disconnectSpy = jest.spyOn(profileService, 'disconnectSocialAccount') + const getSpy = jest.spyOn(profileService, 'get') + getSpy.mockImplementation(() => of(profile)) + component.ngOnInit() + + const errorSpy = jest.spyOn(toastService, 'showError') + + expect(component.socialAccounts).toContainEqual(socialAccount) + + // fail first + disconnectSpy.mockReturnValueOnce( + throwError(() => new Error('unable to disconnect')) + ) + component.disconnectSocialAccount(socialAccount.id) + expect(errorSpy).toHaveBeenCalled() + + // succeed + disconnectSpy.mockReturnValue(of(socialAccount.id)) + component.disconnectSocialAccount(socialAccount.id) + expect(disconnectSpy).toHaveBeenCalled() + expect(component.socialAccounts).not.toContainEqual(socialAccount) + }) }) diff --git a/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts b/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts index d89d49829..77cba4505 100644 --- a/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts +++ b/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts @@ -2,6 +2,7 @@ import { Component, OnDestroy, OnInit } from '@angular/core' import { FormControl, FormGroup } from '@angular/forms' import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap' import { ProfileService } from 'src/app/services/profile.service' +import { SocialAccount, SocialAccountProvider } from 'src/app/data/user-profile' import { ToastService } from 'src/app/services/toast.service' import { Subject, takeUntil } from 'rxjs' import { Clipboard } from '@angular/cdk/clipboard' @@ -30,6 +31,7 @@ export class ProfileEditDialogComponent implements OnInit, OnDestroy { private newPassword: string private passwordConfirm: string public showPasswordConfirm: boolean = false + public hasUsablePassword: boolean = false private currentEmail: string private newEmail: string @@ -38,6 +40,9 @@ export class ProfileEditDialogComponent implements OnInit, OnDestroy { public copied: boolean = false + public socialAccounts: SocialAccount[] = [] + public socialAccountProviders: SocialAccountProvider[] = [] + constructor( private profileService: ProfileService, public activeModal: NgbActiveModal, @@ -59,10 +64,19 @@ export class ProfileEditDialogComponent implements OnInit, OnDestroy { this.onEmailChange() }) this.currentPassword = profile.password + this.hasUsablePassword = profile.has_usable_password this.form.get('password').valueChanges.subscribe((newPassword) => { this.newPassword = newPassword this.onPasswordChange() }) + this.socialAccounts = profile.social_accounts + }) + + this.profileService + .getSocialAccountProviders() + .pipe(takeUntil(this.unsubscribeNotifier)) + .subscribe((providers) => { + this.socialAccountProviders = providers }) } @@ -182,4 +196,21 @@ export class ProfileEditDialogComponent implements OnInit, OnDestroy { this.copied = false }, 3000) } + + disconnectSocialAccount(id: number): void { + this.profileService + .disconnectSocialAccount(id) + .pipe(takeUntil(this.unsubscribeNotifier)) + .subscribe({ + next: (id: number) => { + this.socialAccounts = this.socialAccounts.filter((a) => a.id != id) + }, + error: (error) => { + this.toastService.showError( + $localize`Error disconnecting social account`, + error + ) + }, + }) + } } diff --git a/src-ui/src/app/data/user-profile.ts b/src-ui/src/app/data/user-profile.ts index 94012925a..554f6f0e1 100644 --- a/src-ui/src/app/data/user-profile.ts +++ b/src-ui/src/app/data/user-profile.ts @@ -1,7 +1,20 @@ +export interface SocialAccount { + id: number + provider: string + name: string +} + +export interface SocialAccountProvider { + name: string + login_url: string +} + export interface PaperlessUserProfile { email?: string password?: string first_name?: string last_name?: string auth_token?: string + social_accounts?: SocialAccount[] + has_usable_password?: boolean } diff --git a/src-ui/src/app/services/django-messages.service.spec.ts b/src-ui/src/app/services/django-messages.service.spec.ts new file mode 100644 index 000000000..5e95f32f4 --- /dev/null +++ b/src-ui/src/app/services/django-messages.service.spec.ts @@ -0,0 +1,30 @@ +import { TestBed } from '@angular/core/testing' + +import { + DjangoMessageLevel, + DjangoMessagesService, +} from './django-messages.service' + +const messages = [ + { level: DjangoMessageLevel.ERROR, message: 'Error Message' }, + { level: DjangoMessageLevel.INFO, message: 'Info Message' }, +] + +describe('DjangoMessagesService', () => { + let service: DjangoMessagesService + + beforeEach(() => { + window['DJANGO_MESSAGES'] = messages + TestBed.configureTestingModule({ + providers: [DjangoMessagesService], + }) + service = TestBed.inject(DjangoMessagesService) + }) + + it('should retrieve global django messages if present', () => { + expect(service.get()).toEqual(messages) + + window['DJANGO_MESSAGES'] = undefined + expect(service.get()).toEqual([]) + }) +}) diff --git a/src-ui/src/app/services/django-messages.service.ts b/src-ui/src/app/services/django-messages.service.ts new file mode 100644 index 000000000..fddbe881a --- /dev/null +++ b/src-ui/src/app/services/django-messages.service.ts @@ -0,0 +1,27 @@ +import { Injectable } from '@angular/core' + +// see https://docs.djangoproject.com/en/5.0/ref/contrib/messages/#message-tags +export enum DjangoMessageLevel { + DEBUG = 'debug', + INFO = 'info', + SUCCESS = 'success', + WARNING = 'warning', + ERROR = 'error', +} + +export interface DjangoMessage { + level: DjangoMessageLevel + message: string +} + +@Injectable({ + providedIn: 'root', +}) +export class DjangoMessagesService { + constructor() {} + + get(): DjangoMessage[] { + // These are embedded in the HTML as raw JS, the service is for convenience + return window['DJANGO_MESSAGES'] ?? [] + } +} diff --git a/src-ui/src/app/services/profile.service.spec.ts b/src-ui/src/app/services/profile.service.spec.ts index f19a1312e..538911ac3 100644 --- a/src-ui/src/app/services/profile.service.spec.ts +++ b/src-ui/src/app/services/profile.service.spec.ts @@ -51,4 +51,20 @@ describe('ProfileService', () => { ) expect(req.request.method).toEqual('POST') }) + + it('supports disconnecting a social account', () => { + service.disconnectSocialAccount(1).subscribe() + const req = httpTestingController.expectOne( + `${environment.apiBaseUrl}profile/disconnect_social_account/` + ) + expect(req.request.method).toEqual('POST') + }) + + it('calls get social account provider endpoint', () => { + service.getSocialAccountProviders().subscribe() + const req = httpTestingController.expectOne( + `${environment.apiBaseUrl}profile/social_account_providers/` + ) + expect(req.request.method).toEqual('GET') + }) }) diff --git a/src-ui/src/app/services/profile.service.ts b/src-ui/src/app/services/profile.service.ts index de5aeb7a4..32e06cce0 100644 --- a/src-ui/src/app/services/profile.service.ts +++ b/src-ui/src/app/services/profile.service.ts @@ -1,7 +1,10 @@ import { HttpClient } from '@angular/common/http' import { Injectable } from '@angular/core' import { Observable } from 'rxjs' -import { PaperlessUserProfile } from '../data/user-profile' +import { + PaperlessUserProfile, + SocialAccountProvider, +} from '../data/user-profile' import { environment } from 'src/environments/environment' @Injectable({ @@ -31,4 +34,17 @@ export class ProfileService { {} ) } + + disconnectSocialAccount(id: number): Observable { + return this.http.post( + `${environment.apiBaseUrl}${this.endpoint}/disconnect_social_account/`, + { id: id } + ) + } + + getSocialAccountProviders(): Observable { + return this.http.get( + `${environment.apiBaseUrl}${this.endpoint}/social_account_providers/` + ) + } } diff --git a/src/documents/templates/registration/login.html b/src/documents/templates/account/login.html similarity index 83% rename from src/documents/templates/registration/login.html rename to src/documents/templates/account/login.html index ba57013ac..777f65409 100644 --- a/src/documents/templates/registration/login.html +++ b/src/documents/templates/account/login.html @@ -18,7 +18,8 @@ - + diff --git a/src/documents/templates/registration/logged_out.html b/src/documents/templates/socialaccount/authentication_error.html similarity index 93% rename from src/documents/templates/registration/logged_out.html rename to src/documents/templates/socialaccount/authentication_error.html index 7d7491e3f..1bd3d72e9 100644 --- a/src/documents/templates/registration/logged_out.html +++ b/src/documents/templates/socialaccount/authentication_error.html @@ -2,23 +2,25 @@ {% load static %} {% load i18n %} +{% load allauth %} - + - {% translate "Paperless-ngx signed out" %} + {% translate "Paperless-ngx social account sign in" %} - +
    + {% csrf_token %} -

    {% translate "You have been successfully logged out. Bye!" %}

    - {% translate "Sign in again" %} -
    + {% url 'account_login' as login_url %} +

    {% blocktranslate %}An error occurred while attempting to login via your social network account. Back to the login page{% endblocktranslate %}

    +
    diff --git a/src/documents/templates/socialaccount/login.html b/src/documents/templates/socialaccount/login.html new file mode 100644 index 000000000..f135a77fe --- /dev/null +++ b/src/documents/templates/socialaccount/login.html @@ -0,0 +1,52 @@ + + +{% load static %} +{% load i18n %} +{% load allauth %} + + + + + + + + + + {% translate "Paperless-ngx social account sign in" %} + + + + + + +
    + +
    + + diff --git a/src/documents/templates/socialaccount/signup.html b/src/documents/templates/socialaccount/signup.html new file mode 100644 index 000000000..ef208d8ad --- /dev/null +++ b/src/documents/templates/socialaccount/signup.html @@ -0,0 +1,77 @@ + + +{% load static %} +{% load i18n %} + + + + + + + + + + {% translate "Paperless-ngx social account sign up" %} + + + + + + +
    + +
    + + diff --git a/src/documents/tests/test_api_profile.py b/src/documents/tests/test_api_profile.py index 9e12b1ed3..eede0d2b0 100644 --- a/src/documents/tests/test_api_profile.py +++ b/src/documents/tests/test_api_profile.py @@ -1,3 +1,7 @@ +from unittest import mock + +from allauth.socialaccount.models import SocialAccount +from allauth.socialaccount.models import SocialApp from django.contrib.auth.models import User from rest_framework import status from rest_framework.authtoken.models import Token @@ -6,6 +10,44 @@ from rest_framework.test import APITestCase from documents.tests.utils import DirectoriesMixin +# see allauth.socialaccount.providers.openid.provider.OpenIDProvider +class MockOpenIDProvider: + id = "openid" + name = "OpenID" + + def get_brands(self): + default_servers = [ + dict(id="yahoo", name="Yahoo", openid_url="http://me.yahoo.com"), + dict(id="hyves", name="Hyves", openid_url="http://hyves.nl"), + ] + return default_servers + + def get_login_url(self, request, **kwargs): + return "openid/login/" + + +# see allauth.socialaccount.providers.openid_connect.provider.OpenIDConnectProviderAccount +class MockOpenIDConnectProviderAccount: + def __init__(self, mock_social_account_dict): + self.account = mock_social_account_dict + + def to_str(self): + return self.account["name"] + + +# see allauth.socialaccount.providers.openid_connect.provider.OpenIDConnectProvider +class MockOpenIDConnectProvider: + id = "openid_connect" + name = "OpenID Connect" + + def __init__(self, app=None): + self.app = app + self.name = app.name + + def get_login_url(self, request, **kwargs): + return f"{self.app.provider_id}/login/?process=connect" + + class TestApiProfile(DirectoriesMixin, APITestCase): ENDPOINT = "/api/profile/" @@ -19,6 +61,17 @@ class TestApiProfile(DirectoriesMixin, APITestCase): ) self.client.force_authenticate(user=self.user) + def setupSocialAccount(self): + SocialApp.objects.create( + name="Keycloak", + provider="openid_connect", + provider_id="keycloak-test", + ) + self.user.socialaccount_set.add( + SocialAccount(uid="123456789", provider="keycloak-test"), + bulk=False, + ) + def test_get_profile(self): """ GIVEN: @@ -28,7 +81,6 @@ class TestApiProfile(DirectoriesMixin, APITestCase): THEN: - Profile is returned """ - response = self.client.get(self.ENDPOINT) self.assertEqual(response.status_code, status.HTTP_200_OK) @@ -37,6 +89,52 @@ class TestApiProfile(DirectoriesMixin, APITestCase): self.assertEqual(response.data["first_name"], self.user.first_name) self.assertEqual(response.data["last_name"], self.user.last_name) + @mock.patch( + "allauth.socialaccount.models.SocialAccount.get_provider_account", + ) + @mock.patch( + "allauth.socialaccount.adapter.DefaultSocialAccountAdapter.list_providers", + ) + def test_get_profile_w_social(self, mock_list_providers, mock_get_provider_account): + """ + GIVEN: + - Configured user and setup social account + WHEN: + - API call is made to get profile + THEN: + - Profile is returned with social accounts + """ + self.setupSocialAccount() + + openid_provider = ( + MockOpenIDConnectProvider( + app=SocialApp.objects.get(provider_id="keycloak-test"), + ), + ) + mock_list_providers.return_value = [ + openid_provider, + ] + mock_get_provider_account.return_value = MockOpenIDConnectProviderAccount( + mock_social_account_dict={ + "name": openid_provider[0].name, + }, + ) + + response = self.client.get(self.ENDPOINT) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + self.assertEqual( + response.data["social_accounts"], + [ + { + "id": 1, + "provider": "keycloak-test", + "name": "Keycloak", + }, + ], + ) + def test_update_profile(self): """ GIVEN: @@ -103,3 +201,101 @@ class TestApiProfile(DirectoriesMixin, APITestCase): response = self.client.post(f"{self.ENDPOINT}generate_auth_token/") self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) + + @mock.patch( + "allauth.socialaccount.adapter.DefaultSocialAccountAdapter.list_providers", + ) + def test_get_social_account_providers( + self, + mock_list_providers, + ): + """ + GIVEN: + - Configured user + WHEN: + - API call is made to get social account providers + THEN: + - Social account providers are returned + """ + self.setupSocialAccount() + + mock_list_providers.return_value = [ + MockOpenIDConnectProvider( + app=SocialApp.objects.get(provider_id="keycloak-test"), + ), + ] + + response = self.client.get(f"{self.ENDPOINT}social_account_providers/") + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual( + response.data[0]["name"], + "Keycloak", + ) + self.assertIn( + "keycloak-test/login/?process=connect", + response.data[0]["login_url"], + ) + + @mock.patch( + "allauth.socialaccount.adapter.DefaultSocialAccountAdapter.list_providers", + ) + def test_get_social_account_providers_openid( + self, + mock_list_providers, + ): + """ + GIVEN: + - Configured user and openid social account provider + WHEN: + - API call is made to get social account providers + THEN: + - Brands for openid provider are returned + """ + + mock_list_providers.return_value = [ + MockOpenIDProvider(), + ] + + response = self.client.get(f"{self.ENDPOINT}social_account_providers/") + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual( + len(response.data), + 2, + ) + + def test_disconnect_social_account(self): + """ + GIVEN: + - Configured user + WHEN: + - API call is made to disconnect a social account + THEN: + - Social account is deleted from the user or request fails + """ + self.setupSocialAccount() + + # Test with invalid id + response = self.client.post( + f"{self.ENDPOINT}disconnect_social_account/", + {"id": -1}, + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + # Test with valid id + social_account_id = self.user.socialaccount_set.all()[0].pk + + response = self.client.post( + f"{self.ENDPOINT}disconnect_social_account/", + {"id": social_account_id}, + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data, social_account_id) + + self.assertEqual( + len(self.user.socialaccount_set.filter(pk=social_account_id)), + 0, + ) diff --git a/src/documents/tests/test_management_exporter.py b/src/documents/tests/test_management_exporter.py index 888572b58..226b89694 100644 --- a/src/documents/tests/test_management_exporter.py +++ b/src/documents/tests/test_management_exporter.py @@ -177,9 +177,9 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): os.path.join(self.dirs.media_dir, "documents"), ) - manifest = self._do_export(use_filename_format=use_filename_format) + num_permission_objects = Permission.objects.count() - self.assertEqual(len(manifest), 190) + manifest = self._do_export(use_filename_format=use_filename_format) # dont include consumer or AnonymousUser users self.assertEqual( @@ -273,7 +273,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): self.assertEqual(Document.objects.get(id=self.d4.id).title, "wow_dec") self.assertEqual(GroupObjectPermission.objects.count(), 1) self.assertEqual(UserObjectPermission.objects.count(), 1) - self.assertEqual(Permission.objects.count(), 136) + self.assertEqual(Permission.objects.count(), num_permission_objects) messages = check_sanity() # everything is alright after the test self.assertEqual(len(messages), 0) @@ -753,15 +753,15 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): os.path.join(self.dirs.media_dir, "documents"), ) - self.assertEqual(ContentType.objects.count(), 34) - self.assertEqual(Permission.objects.count(), 136) + num_content_type_objects = ContentType.objects.count() + num_permission_objects = Permission.objects.count() manifest = self._do_export() with paperless_environment(): self.assertEqual( len(list(filter(lambda e: e["model"] == "auth.permission", manifest))), - 136, + num_permission_objects, ) # add 1 more to db to show objects are not re-created by import Permission.objects.create( @@ -769,7 +769,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): codename="test_perm", content_type_id=1, ) - self.assertEqual(Permission.objects.count(), 137) + self.assertEqual(Permission.objects.count(), num_permission_objects + 1) # will cause an import error self.user.delete() @@ -778,5 +778,5 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): with self.assertRaises(IntegrityError): call_command("document_importer", "--no-progress-bar", self.target) - self.assertEqual(ContentType.objects.count(), 34) - self.assertEqual(Permission.objects.count(), 137) + self.assertEqual(ContentType.objects.count(), num_content_type_objects) + self.assertEqual(Permission.objects.count(), num_permission_objects + 1) diff --git a/src/locale/en_US/LC_MESSAGES/django.po b/src/locale/en_US/LC_MESSAGES/django.po index 0c7242462..7ee0c0d8b 100644 --- a/src/locale/en_US/LC_MESSAGES/django.po +++ b/src/locale/en_US/LC_MESSAGES/django.po @@ -2,7 +2,7 @@ msgid "" msgstr "" "Project-Id-Version: paperless-ngx\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2024-02-02 20:17-0800\n" +"POT-Creation-Date: 2024-02-07 06:20+0000\n" "PO-Revision-Date: 2022-02-17 04:17\n" "Last-Translator: \n" "Language-Team: English\n" @@ -777,15 +777,136 @@ msgstr "" msgid "Invalid color." msgstr "" -#: documents/serialisers.py:1049 +#: documents/serialisers.py:1060 #, python-format msgid "File type %(type)s not supported" msgstr "" -#: documents/serialisers.py:1152 +#: documents/serialisers.py:1163 msgid "Invalid variable detected." msgstr "" +#: documents/templates/account/login.html:14 +msgid "Paperless-ngx sign in" +msgstr "" + +#: documents/templates/account/login.html:47 +msgid "Please sign in." +msgstr "" + +#: documents/templates/account/login.html:50 +msgid "Your username and password didn't match. Please try again." +msgstr "" + +#: documents/templates/account/login.html:54 +msgid "Share link was not found." +msgstr "" + +#: documents/templates/account/login.html:58 +msgid "Share link has expired." +msgstr "" + +#: documents/templates/account/login.html:61 +#: documents/templates/socialaccount/signup.html:56 +msgid "Username" +msgstr "" + +#: documents/templates/account/login.html:62 +msgid "Password" +msgstr "" + +#: documents/templates/account/login.html:72 +msgid "Sign in" +msgstr "" + +#: documents/templates/account/login.html:76 +msgid "Forgot your password?" +msgstr "" + +#: documents/templates/account/login.html:83 +msgid "or sign in via" +msgstr "" + +#: documents/templates/account/password_reset.html:15 +msgid "Paperless-ngx reset password request" +msgstr "" + +#: documents/templates/account/password_reset.html:43 +msgid "" +"Enter your email address below, and we'll email instructions for setting a " +"new one." +msgstr "" + +#: documents/templates/account/password_reset.html:46 +msgid "An error occurred. Please try again." +msgstr "" + +#: documents/templates/account/password_reset.html:49 +#: documents/templates/socialaccount/signup.html:57 +msgid "Email" +msgstr "" + +#: documents/templates/account/password_reset.html:56 +msgid "Send me instructions!" +msgstr "" + +#: documents/templates/account/password_reset_done.html:14 +msgid "Paperless-ngx reset password sent" +msgstr "" + +#: documents/templates/account/password_reset_done.html:40 +msgid "Check your inbox." +msgstr "" + +#: documents/templates/account/password_reset_done.html:41 +msgid "" +"We've emailed you instructions for setting your password. You should receive " +"the email shortly!" +msgstr "" + +#: documents/templates/account/password_reset_from_key.html:15 +msgid "Paperless-ngx reset password confirmation" +msgstr "" + +#: documents/templates/account/password_reset_from_key.html:44 +msgid "request a new password reset" +msgstr "" + +#: documents/templates/account/password_reset_from_key.html:46 +msgid "Set a new password." +msgstr "" + +#: documents/templates/account/password_reset_from_key.html:50 +msgid "Passwords did not match or too weak. Try again." +msgstr "" + +#: documents/templates/account/password_reset_from_key.html:53 +msgid "New Password" +msgstr "" + +#: documents/templates/account/password_reset_from_key.html:54 +msgid "Confirm Password" +msgstr "" + +#: documents/templates/account/password_reset_from_key.html:65 +msgid "Change my password" +msgstr "" + +#: documents/templates/account/password_reset_from_key_done.html:14 +msgid "Paperless-ngx reset password complete" +msgstr "" + +#: documents/templates/account/password_reset_from_key_done.html:40 +msgid "Password reset complete." +msgstr "" + +#: documents/templates/account/password_reset_from_key_done.html:42 +#, python-format +msgid "" +"Your new password has been set. You can now log " +"in" +msgstr "" + #: documents/templates/index.html:79 msgid "Paperless-ngx is loading..." msgstr "" @@ -798,131 +919,40 @@ msgstr "" msgid "Here's a link to the docs." msgstr "" -#: documents/templates/registration/logged_out.html:14 -msgid "Paperless-ngx signed out" +#: documents/templates/socialaccount/authentication_error.html:15 +#: documents/templates/socialaccount/login.html:15 +msgid "Paperless-ngx social account sign in" msgstr "" -#: documents/templates/registration/logged_out.html:40 -msgid "You have been successfully logged out. Bye!" -msgstr "" - -#: documents/templates/registration/logged_out.html:41 -msgid "Sign in again" -msgstr "" - -#: documents/templates/registration/login.html:14 -msgid "Paperless-ngx sign in" -msgstr "" - -#: documents/templates/registration/login.html:41 -msgid "Please sign in." -msgstr "" - -#: documents/templates/registration/login.html:44 -msgid "Your username and password didn't match. Please try again." -msgstr "" - -#: documents/templates/registration/login.html:48 -msgid "Share link was not found." -msgstr "" - -#: documents/templates/registration/login.html:52 -msgid "Share link has expired." -msgstr "" - -#: documents/templates/registration/login.html:55 -msgid "Username" -msgstr "" - -#: documents/templates/registration/login.html:56 -msgid "Password" -msgstr "" - -#: documents/templates/registration/login.html:66 -msgid "Sign in" -msgstr "" - -#: documents/templates/registration/login.html:70 -msgid "Forgot your password?" -msgstr "" - -#: documents/templates/registration/password_reset_complete.html:14 -msgid "Paperless-ngx reset password complete" -msgstr "" - -#: documents/templates/registration/password_reset_complete.html:40 -msgid "Password reset complete." -msgstr "" - -#: documents/templates/registration/password_reset_complete.html:42 +#: documents/templates/socialaccount/authentication_error.html:43 #, python-format msgid "" -"Your new password has been set. You can now log " -"in" +"An error occurred while attempting to login via your social network account. " +"Back to the login page" msgstr "" -#: documents/templates/registration/password_reset_confirm.html:14 -msgid "Paperless-ngx reset password confirmation" +#: documents/templates/socialaccount/login.html:44 +#, python-format +msgid "You are about to connect a new third-party account from %(provider)s." msgstr "" -#: documents/templates/registration/password_reset_confirm.html:42 -msgid "Set a new password." +#: documents/templates/socialaccount/login.html:47 +msgid "Continue" msgstr "" -#: documents/templates/registration/password_reset_confirm.html:46 -msgid "Passwords did not match or too weak. Try again." +#: documents/templates/socialaccount/signup.html:14 +msgid "Paperless-ngx social account sign up" msgstr "" -#: documents/templates/registration/password_reset_confirm.html:49 -msgid "New Password" -msgstr "" - -#: documents/templates/registration/password_reset_confirm.html:50 -msgid "Confirm Password" -msgstr "" - -#: documents/templates/registration/password_reset_confirm.html:61 -msgid "Change my password" -msgstr "" - -#: documents/templates/registration/password_reset_confirm.html:65 -msgid "request a new password reset" -msgstr "" - -#: documents/templates/registration/password_reset_done.html:14 -msgid "Paperless-ngx reset password sent" -msgstr "" - -#: documents/templates/registration/password_reset_done.html:40 -msgid "Check your inbox." -msgstr "" - -#: documents/templates/registration/password_reset_done.html:41 +#: documents/templates/socialaccount/signup.html:53 +#, python-format msgid "" -"We've emailed you instructions for setting your password. You should receive " -"the email shortly!" +"You are about to use your %(provider_name)s account to login to\n" +"%(site_name)s. As a final step, please complete the following form:" msgstr "" -#: documents/templates/registration/password_reset_form.html:14 -msgid "Paperless-ngx reset password request" -msgstr "" - -#: documents/templates/registration/password_reset_form.html:41 -msgid "" -"Enter your email address below, and we'll email instructions for setting a " -"new one." -msgstr "" - -#: documents/templates/registration/password_reset_form.html:44 -msgid "An error occurred. Please try again." -msgstr "" - -#: documents/templates/registration/password_reset_form.html:47 -msgid "Email" -msgstr "" - -#: documents/templates/registration/password_reset_form.html:54 -msgid "Send me instructions!" +#: documents/templates/socialaccount/signup.html:72 +msgid "Sign up" msgstr "" #: documents/validators.py:17 @@ -1088,135 +1118,135 @@ msgstr "" msgid "paperless application settings" msgstr "" -#: paperless/settings.py:617 +#: paperless/settings.py:658 msgid "English (US)" msgstr "" -#: paperless/settings.py:618 +#: paperless/settings.py:659 msgid "Arabic" msgstr "" -#: paperless/settings.py:619 +#: paperless/settings.py:660 msgid "Afrikaans" msgstr "" -#: paperless/settings.py:620 +#: paperless/settings.py:661 msgid "Belarusian" msgstr "" -#: paperless/settings.py:621 +#: paperless/settings.py:662 msgid "Bulgarian" msgstr "" -#: paperless/settings.py:622 +#: paperless/settings.py:663 msgid "Catalan" msgstr "" -#: paperless/settings.py:623 +#: paperless/settings.py:664 msgid "Czech" msgstr "" -#: paperless/settings.py:624 +#: paperless/settings.py:665 msgid "Danish" msgstr "" -#: paperless/settings.py:625 +#: paperless/settings.py:666 msgid "German" msgstr "" -#: paperless/settings.py:626 +#: paperless/settings.py:667 msgid "Greek" msgstr "" -#: paperless/settings.py:627 +#: paperless/settings.py:668 msgid "English (GB)" msgstr "" -#: paperless/settings.py:628 +#: paperless/settings.py:669 msgid "Spanish" msgstr "" -#: paperless/settings.py:629 +#: paperless/settings.py:670 msgid "Finnish" msgstr "" -#: paperless/settings.py:630 +#: paperless/settings.py:671 msgid "French" msgstr "" -#: paperless/settings.py:631 +#: paperless/settings.py:672 msgid "Hungarian" msgstr "" -#: paperless/settings.py:632 +#: paperless/settings.py:673 msgid "Italian" msgstr "" -#: paperless/settings.py:633 +#: paperless/settings.py:674 msgid "Japanese" msgstr "" -#: paperless/settings.py:634 +#: paperless/settings.py:675 msgid "Luxembourgish" msgstr "" -#: paperless/settings.py:635 +#: paperless/settings.py:676 msgid "Norwegian" msgstr "" -#: paperless/settings.py:636 +#: paperless/settings.py:677 msgid "Dutch" msgstr "" -#: paperless/settings.py:637 +#: paperless/settings.py:678 msgid "Polish" msgstr "" -#: paperless/settings.py:638 +#: paperless/settings.py:679 msgid "Portuguese (Brazil)" msgstr "" -#: paperless/settings.py:639 +#: paperless/settings.py:680 msgid "Portuguese" msgstr "" -#: paperless/settings.py:640 +#: paperless/settings.py:681 msgid "Romanian" msgstr "" -#: paperless/settings.py:641 +#: paperless/settings.py:682 msgid "Russian" msgstr "" -#: paperless/settings.py:642 +#: paperless/settings.py:683 msgid "Slovak" msgstr "" -#: paperless/settings.py:643 +#: paperless/settings.py:684 msgid "Slovenian" msgstr "" -#: paperless/settings.py:644 +#: paperless/settings.py:685 msgid "Serbian" msgstr "" -#: paperless/settings.py:645 +#: paperless/settings.py:686 msgid "Swedish" msgstr "" -#: paperless/settings.py:646 +#: paperless/settings.py:687 msgid "Turkish" msgstr "" -#: paperless/settings.py:647 +#: paperless/settings.py:688 msgid "Ukrainian" msgstr "" -#: paperless/settings.py:648 +#: paperless/settings.py:689 msgid "Chinese Simplified" msgstr "" -#: paperless/urls.py:214 +#: paperless/urls.py:224 msgid "Paperless-ngx administration" msgstr "" diff --git a/src/paperless/adapter.py b/src/paperless/adapter.py new file mode 100644 index 000000000..98b0f11ba --- /dev/null +++ b/src/paperless/adapter.py @@ -0,0 +1,30 @@ +from allauth.account.adapter import DefaultAccountAdapter +from allauth.socialaccount.adapter import DefaultSocialAccountAdapter +from django.conf import settings +from django.urls import reverse + + +class CustomAccountAdapter(DefaultAccountAdapter): + def is_open_for_signup(self, request): + allow_signups = super().is_open_for_signup(request) + # Override with setting, otherwise default to super. + return getattr(settings, "ACCOUNT_ALLOW_SIGNUPS", allow_signups) + + +class CustomSocialAccountAdapter(DefaultSocialAccountAdapter): + def is_open_for_signup(self, request, sociallogin): + allow_signups = super().is_open_for_signup(request, sociallogin) + # Override with setting, otherwise default to super. + return getattr(settings, "SOCIALACCOUNT_ALLOW_SIGNUPS", allow_signups) + + def get_connect_redirect_url(self, request, socialaccount): + """ + Returns the default URL to redirect to after successfully + connecting a social account. + """ + url = reverse("base") + return url + + def populate_user(self, request, sociallogin, data): + # TODO: If default global permissions are implemented, should also be here + return super().populate_user(request, sociallogin, data) # pragma: no cover diff --git a/src/paperless/serialisers.py b/src/paperless/serialisers.py index b724dd451..4c9ed5641 100644 --- a/src/paperless/serialisers.py +++ b/src/paperless/serialisers.py @@ -1,5 +1,6 @@ import logging +from allauth.socialaccount.models import SocialAccount from django.contrib.auth.models import Group from django.contrib.auth.models import Permission from django.contrib.auth.models import User @@ -105,10 +106,30 @@ class GroupSerializer(serializers.ModelSerializer): ) +class SocialAccountSerializer(serializers.ModelSerializer): + name = serializers.SerializerMethodField() + + class Meta: + model = SocialAccount + fields = ( + "id", + "provider", + "name", + ) + + def get_name(self, obj): + return obj.get_provider_account().to_str() + + class ProfileSerializer(serializers.ModelSerializer): email = serializers.EmailField(allow_null=False) password = ObfuscatedUserPasswordField(required=False, allow_null=False) auth_token = serializers.SlugRelatedField(read_only=True, slug_field="key") + social_accounts = SocialAccountSerializer( + many=True, + read_only=True, + source="socialaccount_set", + ) class Meta: model = User @@ -118,6 +139,8 @@ class ProfileSerializer(serializers.ModelSerializer): "first_name", "last_name", "auth_token", + "social_accounts", + "has_usable_password", ) diff --git a/src/paperless/settings.py b/src/paperless/settings.py index d485415ca..d51ba9020 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -303,6 +303,9 @@ INSTALLED_APPS = [ "django_filters", "django_celery_results", "guardian", + "allauth", + "allauth.account", + "allauth.socialaccount", *env_apps, ] @@ -339,6 +342,7 @@ MIDDLEWARE = [ "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", + "allauth.account.middleware.AccountMiddleware", ] # Optional to enable compression @@ -350,6 +354,7 @@ ROOT_URLCONF = "paperless.urls" FORCE_SCRIPT_NAME = os.getenv("PAPERLESS_FORCE_SCRIPT_NAME") BASE_URL = (FORCE_SCRIPT_NAME or "") + "/" LOGIN_URL = BASE_URL + "accounts/login/" +LOGIN_REDIRECT_URL = "/dashboard" LOGOUT_REDIRECT_URL = os.getenv("PAPERLESS_LOGOUT_REDIRECT_URL") WSGI_APPLICATION = "paperless.wsgi.application" @@ -410,8 +415,28 @@ CHANNEL_LAYERS = { AUTHENTICATION_BACKENDS = [ "guardian.backends.ObjectPermissionBackend", "django.contrib.auth.backends.ModelBackend", + "allauth.account.auth_backends.AuthenticationBackend", ] +ACCOUNT_LOGOUT_ON_GET = True +ACCOUNT_DEFAULT_HTTP_PROTOCOL = os.getenv( + "PAPERLESS_ACCOUNT_DEFAULT_HTTP_PROTOCOL", + "https", +) + +ACCOUNT_ADAPTER = "paperless.adapter.CustomAccountAdapter" +ACCOUNT_ALLOW_SIGNUPS = __get_boolean("PAPERLESS_ACCOUNT_ALLOW_SIGNUPS") + +SOCIALACCOUNT_ADAPTER = "paperless.adapter.CustomSocialAccountAdapter" +SOCIALACCOUNT_ALLOW_SIGNUPS = __get_boolean( + "PAPERLESS_SOCIALACCOUNT_ALLOW_SIGNUPS", + "yes", +) +SOCIALACCOUNT_AUTO_SIGNUP = __get_boolean("PAPERLESS_SOCIAL_AUTO_SIGNUP") +SOCIALACCOUNT_PROVIDERS = json.loads( + os.getenv("PAPERLESS_SOCIALACCOUNT_PROVIDERS", "{}"), +) + AUTO_LOGIN_USERNAME = os.getenv("PAPERLESS_AUTO_LOGIN_USERNAME") if AUTO_LOGIN_USERNAME: diff --git a/src/paperless/tests/test_adapter.py b/src/paperless/tests/test_adapter.py new file mode 100644 index 000000000..ca79cbce0 --- /dev/null +++ b/src/paperless/tests/test_adapter.py @@ -0,0 +1,43 @@ +from allauth.account.adapter import get_adapter +from allauth.socialaccount.adapter import get_adapter as get_social_adapter +from django.conf import settings +from django.test import TestCase +from django.urls import reverse + + +class TestCustomAccountAdapter(TestCase): + def test_is_open_for_signup(self): + adapter = get_adapter() + + # Test when ACCOUNT_ALLOW_SIGNUPS is True + settings.ACCOUNT_ALLOW_SIGNUPS = True + self.assertTrue(adapter.is_open_for_signup(None)) + + # Test when ACCOUNT_ALLOW_SIGNUPS is False + settings.ACCOUNT_ALLOW_SIGNUPS = False + self.assertFalse(adapter.is_open_for_signup(None)) + + +class TestCustomSocialAccountAdapter(TestCase): + def test_is_open_for_signup(self): + adapter = get_social_adapter() + + # Test when SOCIALACCOUNT_ALLOW_SIGNUPS is True + settings.SOCIALACCOUNT_ALLOW_SIGNUPS = True + self.assertTrue(adapter.is_open_for_signup(None, None)) + + # Test when SOCIALACCOUNT_ALLOW_SIGNUPS is False + settings.SOCIALACCOUNT_ALLOW_SIGNUPS = False + self.assertFalse(adapter.is_open_for_signup(None, None)) + + def test_get_connect_redirect_url(self): + adapter = get_social_adapter() + request = None + socialaccount = None + + # Test the default URL + expected_url = reverse("base") + self.assertEqual( + adapter.get_connect_redirect_url(request, socialaccount), + expected_url, + ) diff --git a/src/paperless/urls.py b/src/paperless/urls.py index d45a7bf22..74f6fc108 100644 --- a/src/paperless/urls.py +++ b/src/paperless/urls.py @@ -41,10 +41,12 @@ from documents.views import WorkflowTriggerViewSet from documents.views import WorkflowViewSet from paperless.consumers import StatusConsumer from paperless.views import ApplicationConfigurationViewSet +from paperless.views import DisconnectSocialAccountView from paperless.views import FaviconView from paperless.views import GenerateAuthTokenView from paperless.views import GroupViewSet from paperless.views import ProfileView +from paperless.views import SocialAccountProvidersView from paperless.views import UserViewSet from paperless_mail.views import MailAccountTestView from paperless_mail.views import MailAccountViewSet @@ -132,6 +134,14 @@ urlpatterns = [ name="bulk_edit_object_permissions", ), path("profile/generate_auth_token/", GenerateAuthTokenView.as_view()), + path( + "profile/disconnect_social_account/", + DisconnectSocialAccountView.as_view(), + ), + path( + "profile/social_account_providers/", + SocialAccountProvidersView.as_view(), + ), re_path( "^profile/", ProfileView.as_view(), @@ -192,7 +202,7 @@ urlpatterns = [ ), # TODO: with localization, this is even worse! :/ # login, logout - path("accounts/", include("django.contrib.auth.urls")), + path("accounts/", include("allauth.urls")), # Root of the Frontend re_path( r".*", diff --git a/src/paperless/views.py b/src/paperless/views.py index 0f417b9ab..1151ceed5 100644 --- a/src/paperless/views.py +++ b/src/paperless/views.py @@ -1,10 +1,13 @@ import os from collections import OrderedDict +from allauth.socialaccount.adapter import get_adapter +from allauth.socialaccount.models import SocialAccount from django.contrib.auth.models import Group from django.contrib.auth.models import User from django.db.models.functions import Lower from django.http import HttpResponse +from django.http import HttpResponseBadRequest from django.views.generic import View from django_filters.rest_framework import DjangoFilterBackend from rest_framework.authtoken.models import Token @@ -14,6 +17,7 @@ from rest_framework.pagination import PageNumberPagination from rest_framework.permissions import DjangoObjectPermissions from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response +from rest_framework.views import APIView from rest_framework.viewsets import ModelViewSet from documents.permissions import PaperlessObjectPermissions @@ -168,3 +172,54 @@ class ApplicationConfigurationViewSet(ModelViewSet): serializer_class = ApplicationConfigurationSerializer permission_classes = (IsAuthenticated, DjangoObjectPermissions) + + +class DisconnectSocialAccountView(GenericAPIView): + """ + Disconnects a social account provider from the user account + """ + + permission_classes = [IsAuthenticated] + + def post(self, request, *args, **kwargs): + user = self.request.user + + try: + account = user.socialaccount_set.get(pk=request.data["id"]) + account_id = account.id + account.delete() + return Response(account_id) + except SocialAccount.DoesNotExist: + return HttpResponseBadRequest("Social account not found") + + +class SocialAccountProvidersView(APIView): + """ + List of social account providers + """ + + permission_classes = [IsAuthenticated] + + def get(self, request, *args, **kwargs): + adapter = get_adapter() + providers = adapter.list_providers(request) + resp = [ + {"name": p.name, "login_url": p.get_login_url(request, process="connect")} + for p in providers + if p.id != "openid" + ] + + for openid_provider in filter(lambda p: p.id == "openid", providers): + resp += [ + { + "name": b["name"], + "login_url": openid_provider.get_login_url( + request, + process="connect", + openid=b["openid_url"], + ), + } + for b in openid_provider.get_brands() + ] + + return Response(sorted(resp, key=lambda p: p["name"]))