From 607adf44f3fc609d2d49495468a61306d2fb0bd5 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Sun, 4 Feb 2024 14:20:48 -0800 Subject: [PATCH 1/6] Documentation: Make remote-user warning clearer, maybe --- docs/configuration.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index b68198619..f5ffbf9b0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -452,11 +452,12 @@ applications. This will allow authentication by simply adding a `Remote-User: ` header to a request. Use with care! You - especially *must: ensure that any such header is not passed from - your proxy server to paperless. + especially *must* ensure that any such header is not passed from + external requests to your reverse-proxy to paperless (that would + effectively bypass all authentication). - If you're exposing paperless to the internet directly, do not use - this. + If you're exposing paperless to the internet directly (i.e. + without a reverse proxy), do not use this. Also see the warning [in the official documentation](https://docs.djangoproject.com/en/4.1/howto/auth-remote-user/#configuration). From a79b9de1a286eceadd93b59c051b63398ff3c6a6 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Wed, 7 Feb 2024 09:46:20 -0800 Subject: [PATCH 2/6] Documentation: Add storage path to workflow action assignments --- docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index 28d6265ef..11b1f710a 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -331,7 +331,7 @@ Workflows allow you to filter by: There is currently one type of workflow action, "Assignment", which can assign: - Title, see [title placeholders](usage.md#title-placeholders) below -- Tags, correspondent, document types +- Tags, correspondent, document type and storage path - Document owner - View and / or edit permissions to users or groups - Custom fields. Note that no value for the field will be set From 3b2d4fe876c6d7f86f0acd2e2a668cc0e902b5fc Mon Sep 17 00:00:00 2001 From: silmaril42 <31900904+silmaril42@users.noreply.github.com> Date: Thu, 8 Feb 2024 03:46:25 +0100 Subject: [PATCH 3/6] Documentation: add detail about consumer polling behavior (#5674) --- docs/configuration.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index f5ffbf9b0..6b75f33e6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1050,8 +1050,10 @@ system changes with `inotify`. #### [`PAPERLESS_CONSUMER_POLLING_RETRY_COUNT=`](#PAPERLESS_CONSUMER_POLLING_RETRY_COUNT) {#PAPERLESS_CONSUMER_POLLING_RETRY_COUNT} -: If consumer polling is enabled, sets the number of times paperless -will check for a file to remain unmodified. +: If consumer polling is enabled, sets the maximum number of times +paperless will check for a file to remain unmodified. If a file's +modification time and size are identical for two consecutive checks, it +will be consumed. Defaults to 5. From 13201dbfffc939a8a415a929325727b86ae842aa Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Sat, 10 Feb 2024 11:02:40 -0800 Subject: [PATCH 4/6] Ensure all creations of directories create the parents too (#5711) --- src/documents/consumer.py | 8 ++++---- src/documents/index.py | 2 +- .../management/commands/document_archiver.py | 3 +-- .../management/commands/document_exporter.py | 2 +- .../management/commands/document_importer.py | 6 +++--- src/documents/parsers.py | 2 +- src/documents/tests/test_date_parsing.py | 15 --------------- src/documents/tests/test_file_handling.py | 12 ++++++------ src/documents/views.py | 4 ++-- src/paperless/settings.py | 2 +- src/paperless_mail/mail.py | 4 ++-- 11 files changed, 22 insertions(+), 38 deletions(-) diff --git a/src/documents/consumer.py b/src/documents/consumer.py index 01b25edea..92461484c 100644 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -233,10 +233,10 @@ class Consumer(LoggingMixin): """ Ensure all required directories exist before attempting to use them """ - os.makedirs(settings.SCRATCH_DIR, exist_ok=True) - os.makedirs(settings.THUMBNAIL_DIR, exist_ok=True) - os.makedirs(settings.ORIGINALS_DIR, exist_ok=True) - os.makedirs(settings.ARCHIVE_DIR, exist_ok=True) + settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) + settings.THUMBNAIL_DIR.mkdir(parents=True, exist_ok=True) + settings.ORIGINALS_DIR.mkdir(parents=True, exist_ok=True) + settings.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True) def pre_check_asn_value(self): """ diff --git a/src/documents/index.py b/src/documents/index.py index 90bc15e7b..de651c13d 100644 --- a/src/documents/index.py +++ b/src/documents/index.py @@ -88,7 +88,7 @@ def open_index(recreate=False) -> FileIndex: logger.exception("Error while opening the index, recreating.") if not os.path.isdir(settings.INDEX_DIR): - os.makedirs(settings.INDEX_DIR, exist_ok=True) + settings.INDEX_DIR.mkdir(parents=True, exist_ok=True) return create_in(settings.INDEX_DIR, get_schema()) diff --git a/src/documents/management/commands/document_archiver.py b/src/documents/management/commands/document_archiver.py index 96b3b50ab..12677dd79 100644 --- a/src/documents/management/commands/document_archiver.py +++ b/src/documents/management/commands/document_archiver.py @@ -1,6 +1,5 @@ import logging import multiprocessing -import os import tqdm from django import db @@ -52,7 +51,7 @@ class Command(MultiProcessMixin, ProgressBarMixin, BaseCommand): self.handle_processes_mixin(**options) self.handle_progress_bar_mixin(**options) - os.makedirs(settings.SCRATCH_DIR, exist_ok=True) + settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) overwrite = options["overwrite"] diff --git a/src/documents/management/commands/document_exporter.py b/src/documents/management/commands/document_exporter.py index 9bdbd1c51..faa42ba31 100644 --- a/src/documents/management/commands/document_exporter.py +++ b/src/documents/management/commands/document_exporter.py @@ -182,7 +182,7 @@ class Command(BaseCommand): if self.zip_export: self.original_target = self.target - os.makedirs(settings.SCRATCH_DIR, exist_ok=True) + settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) temp_dir = tempfile.TemporaryDirectory( dir=settings.SCRATCH_DIR, prefix="paperless-export", diff --git a/src/documents/management/commands/document_importer.py b/src/documents/management/commands/document_importer.py index c166ec0cb..88ac2b91a 100644 --- a/src/documents/management/commands/document_importer.py +++ b/src/documents/management/commands/document_importer.py @@ -243,9 +243,9 @@ class Command(BaseCommand): ) from e def _import_files_from_manifest(self, progress_bar_disable): - os.makedirs(settings.ORIGINALS_DIR, exist_ok=True) - os.makedirs(settings.THUMBNAIL_DIR, exist_ok=True) - os.makedirs(settings.ARCHIVE_DIR, exist_ok=True) + settings.ORIGINALS_DIR.mkdir(parents=True, exist_ok=True) + settings.THUMBNAIL_DIR.mkdir(parents=True, exist_ok=True) + settings.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True) self.stdout.write("Copy files into paperless...") diff --git a/src/documents/parsers.py b/src/documents/parsers.py index b791d8322..a2b4677e6 100644 --- a/src/documents/parsers.py +++ b/src/documents/parsers.py @@ -322,7 +322,7 @@ class DocumentParser(LoggingMixin): super().__init__() self.logging_group = logging_group self.settings = self.get_settings() - os.makedirs(settings.SCRATCH_DIR, exist_ok=True) + settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) self.tempdir = Path( tempfile.mkdtemp(prefix="paperless-", dir=settings.SCRATCH_DIR), ) diff --git a/src/documents/tests/test_date_parsing.py b/src/documents/tests/test_date_parsing.py index d4ea71be5..4c75e5f6b 100644 --- a/src/documents/tests/test_date_parsing.py +++ b/src/documents/tests/test_date_parsing.py @@ -1,7 +1,4 @@ import datetime -import os -import shutil -from uuid import uuid4 from dateutil import tz from django.conf import settings @@ -13,17 +10,6 @@ from documents.parsers import parse_date_generator class TestDate(TestCase): - SAMPLE_FILES = os.path.join( - os.path.dirname(__file__), - "../../paperless_tesseract/tests/samples", - ) - SCRATCH = f"/tmp/paperless-tests-{str(uuid4())[:8]}" - - def setUp(self): - os.makedirs(self.SCRATCH, exist_ok=True) - - def tearDown(self): - shutil.rmtree(self.SCRATCH) def test_date_format_1(self): text = "lorem ipsum 130218 lorem ipsum" @@ -93,7 +79,6 @@ class TestDate(TestCase): datetime.datetime(2020, 3, 1, 0, 0, tzinfo=tz.gettz(settings.TIME_ZONE)), ) - @override_settings(SCRATCH_DIR=SCRATCH) def test_date_format_9(self): text = "lorem ipsum\n27. Nullmonth 2020\nMärz 2020\nlorem ipsum" self.assertEqual( diff --git a/src/documents/tests/test_file_handling.py b/src/documents/tests/test_file_handling.py index 225008b78..a924e377b 100644 --- a/src/documents/tests/test_file_handling.py +++ b/src/documents/tests/test_file_handling.py @@ -470,12 +470,12 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase): def test_try_delete_empty_directories(self): # Create our working directory - tmp = os.path.join(settings.ORIGINALS_DIR, "test_delete_empty") - os.makedirs(tmp) + tmp: Path = settings.ORIGINALS_DIR / "test_delete_empty" + tmp.mkdir(exist_ok=True, parents=True) - os.makedirs(os.path.join(tmp, "notempty")) - Path(os.path.join(tmp, "notempty", "file")).touch() - os.makedirs(os.path.join(tmp, "notempty", "empty")) + (tmp / "notempty").mkdir(exist_ok=True, parents=True) + (tmp / "notempty" / "file").touch() + (tmp / "notempty" / "empty").mkdir(exist_ok=True, parents=True) delete_empty_directories( os.path.join(tmp, "notempty", "empty"), @@ -647,7 +647,7 @@ class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, Test existing_archive_file = os.path.join(settings.ARCHIVE_DIR, "none", "my_doc.pdf") Path(original).touch() Path(archive).touch() - os.makedirs(os.path.join(settings.ARCHIVE_DIR, "none")) + (settings.ARCHIVE_DIR / "none").mkdir(parents=True, exist_ok=True) Path(existing_archive_file).touch() doc = Document.objects.create( mime_type="application/pdf", diff --git a/src/documents/views.py b/src/documents/views.py index 3be7f4ec6..c73a8050b 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -904,7 +904,7 @@ class PostDocumentView(GenericAPIView): t = int(mktime(datetime.now().timetuple())) - os.makedirs(settings.SCRATCH_DIR, exist_ok=True) + settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) temp_file_path = Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR)) / Path( pathvalidate.sanitize_filename(doc_name), @@ -1136,7 +1136,7 @@ class BulkDownloadView(GenericAPIView): content = serializer.validated_data.get("content") follow_filename_format = serializer.validated_data.get("follow_formatting") - os.makedirs(settings.SCRATCH_DIR, exist_ok=True) + settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) temp = tempfile.NamedTemporaryFile( dir=settings.SCRATCH_DIR, suffix="-compressed-archive", diff --git a/src/paperless/settings.py b/src/paperless/settings.py index a9792db9f..163492b1e 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -689,7 +689,7 @@ USE_TZ = True setup_logging_queues() -os.makedirs(LOGGING_DIR, exist_ok=True) +LOGGING_DIR.mkdir(parents=True, exist_ok=True) LOGROTATE_MAX_SIZE = os.getenv("PAPERLESS_LOGROTATE_MAX_SIZE", 1024 * 1024) LOGROTATE_MAX_BACKUPS = os.getenv("PAPERLESS_LOGROTATE_MAX_BACKUPS", 20) diff --git a/src/paperless_mail/mail.py b/src/paperless_mail/mail.py index 6514014ca..a4ff6478c 100644 --- a/src/paperless_mail/mail.py +++ b/src/paperless_mail/mail.py @@ -710,7 +710,7 @@ class MailAccountHandler(LoggingMixin): f"{message.subject} from {message.from_}", ) - os.makedirs(settings.SCRATCH_DIR, exist_ok=True) + settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) temp_dir = Path( tempfile.mkdtemp( @@ -793,7 +793,7 @@ class MailAccountHandler(LoggingMixin): tag_ids, doc_type, ): - os.makedirs(settings.SCRATCH_DIR, exist_ok=True) + settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) _, temp_filename = tempfile.mkstemp( prefix="paperless-mail-", dir=settings.SCRATCH_DIR, From 4aeb2e1a743da76c98c083832701b9d8797093b2 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Sat, 10 Feb 2024 11:14:16 -0800 Subject: [PATCH 5/6] Bumps version to 2.5.0 --- src-ui/src/environments/environment.prod.ts | 2 +- src/paperless/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src-ui/src/environments/environment.prod.ts b/src-ui/src/environments/environment.prod.ts index 7de16a988..5bf704518 100644 --- a/src-ui/src/environments/environment.prod.ts +++ b/src-ui/src/environments/environment.prod.ts @@ -5,7 +5,7 @@ export const environment = { apiBaseUrl: document.baseURI + 'api/', apiVersion: '5', appTitle: 'Paperless-ngx', - version: '2.4.3-dev', + version: '2.5.0', webSocketHost: window.location.host, webSocketProtocol: window.location.protocol == 'https:' ? 'wss:' : 'ws:', webSocketBaseUrl: base_url.pathname + 'ws/', diff --git a/src/paperless/version.py b/src/paperless/version.py index b1926ccfb..367d5a0ef 100644 --- a/src/paperless/version.py +++ b/src/paperless/version.py @@ -1,6 +1,6 @@ from typing import Final -__version__: Final[tuple[int, int, int]] = (2, 4, 3) +__version__: Final[tuple[int, int, int]] = (2, 5, 0) # Version string like X.Y.Z __full_version_str__: Final[str] = ".".join(map(str, __version__)) # Version string like X.Y From 58cbcbd6ef116cefde48ba93236f824f6ad8ee1b Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Sat, 10 Feb 2024 11:15:12 -0800 Subject: [PATCH 6/6] Resets -dev version string --- src-ui/src/environments/environment.prod.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-ui/src/environments/environment.prod.ts b/src-ui/src/environments/environment.prod.ts index 5bf704518..7b0a1362d 100644 --- a/src-ui/src/environments/environment.prod.ts +++ b/src-ui/src/environments/environment.prod.ts @@ -5,7 +5,7 @@ export const environment = { apiBaseUrl: document.baseURI + 'api/', apiVersion: '5', appTitle: 'Paperless-ngx', - version: '2.5.0', + version: '2.5.0-dev', webSocketHost: window.location.host, webSocketProtocol: window.location.protocol == 'https:' ? 'wss:' : 'ws:', webSocketBaseUrl: base_url.pathname + 'ws/',