From bccd1d11d3f544be32423445772c6dcf001eacad Mon Sep 17 00:00:00 2001 From: Trenton Holmes <797416+stumpylog@users.noreply.github.com> Date: Thu, 6 Jun 2024 07:04:07 -0700 Subject: [PATCH] Starting testing --- .../management/commands/document_exporter.py | 16 ++----- .../management/commands/document_importer.py | 22 ++------- src/documents/management/commands/mixins.py | 34 +++++++++++++- .../tests/test_management_exporter.py | 45 ++++++++++++++++++- src/setup.cfg | 1 + 5 files changed, 84 insertions(+), 34 deletions(-) diff --git a/src/documents/management/commands/document_exporter.py b/src/documents/management/commands/document_exporter.py index f3bf65792..6097e21b3 100644 --- a/src/documents/management/commands/document_exporter.py +++ b/src/documents/management/commands/document_exporter.py @@ -31,7 +31,7 @@ if settings.AUDIT_LOG_ENABLED: from documents.file_handling import delete_empty_directories from documents.file_handling import generate_filename -from documents.management.commands.mixins import SecurityMixin +from documents.management.commands.mixins import CryptMixin from documents.models import Correspondent from documents.models import CustomField from documents.models import CustomFieldInstance @@ -47,11 +47,6 @@ from documents.models import Workflow from documents.models import WorkflowAction from documents.models import WorkflowTrigger from documents.settings import EXPORTER_ARCHIVE_NAME -from documents.settings import EXPORTER_CRYPTO_ALGO_NAME -from documents.settings import EXPORTER_CRYPTO_KEY_ITERATIONS_NAME -from documents.settings import EXPORTER_CRYPTO_KEY_SIZE_NAME -from documents.settings import EXPORTER_CRYPTO_SALT_NAME -from documents.settings import EXPORTER_CRYPTO_SETTINGS_NAME from documents.settings import EXPORTER_FILE_NAME from documents.settings import EXPORTER_THUMBNAIL_NAME from documents.utils import copy_file_with_basic_stats @@ -62,7 +57,7 @@ from paperless_mail.models import MailAccount from paperless_mail.models import MailRule -class Command(SecurityMixin, BaseCommand): +class Command(CryptMixin, BaseCommand): help = ( "Decrypt and rename all files in our collection into a given target " "directory. And include a manifest file containing document data for " @@ -375,12 +370,7 @@ class Command(SecurityMixin, BaseCommand): # 4.2.1 If needed, write the crypto values into the metadata # Django stores most of these in the field itself, we store them once here if self.passphrase: - metadata[EXPORTER_CRYPTO_SETTINGS_NAME] = { - EXPORTER_CRYPTO_ALGO_NAME: self.kdf_algorithm, - EXPORTER_CRYPTO_KEY_ITERATIONS_NAME: self.key_iterations, - EXPORTER_CRYPTO_KEY_SIZE_NAME: self.key_size, - EXPORTER_CRYPTO_SALT_NAME: self.salt, - } + metadata.update(self.get_crypt_params()) extra_metadata_path.write_text( json.dumps( metadata, diff --git a/src/documents/management/commands/document_importer.py b/src/documents/management/commands/document_importer.py index 46fb7e898..1ff86be4a 100644 --- a/src/documents/management/commands/document_importer.py +++ b/src/documents/management/commands/document_importer.py @@ -22,7 +22,7 @@ from django.db.models.signals import post_save from filelock import FileLock from documents.file_handling import create_source_path_directory -from documents.management.commands.mixins import SecurityMixin +from documents.management.commands.mixins import CryptMixin from documents.models import Correspondent from documents.models import CustomField from documents.models import CustomFieldInstance @@ -32,10 +32,6 @@ from documents.models import Note from documents.models import Tag from documents.parsers import run_convert from documents.settings import EXPORTER_ARCHIVE_NAME -from documents.settings import EXPORTER_CRYPTO_ALGO_NAME -from documents.settings import EXPORTER_CRYPTO_KEY_ITERATIONS_NAME -from documents.settings import EXPORTER_CRYPTO_KEY_SIZE_NAME -from documents.settings import EXPORTER_CRYPTO_SALT_NAME from documents.settings import EXPORTER_CRYPTO_SETTINGS_NAME from documents.settings import EXPORTER_FILE_NAME from documents.settings import EXPORTER_THUMBNAIL_NAME @@ -56,7 +52,7 @@ def disable_signal(sig, receiver, sender): sig.connect(receiver=receiver, sender=sender) -class Command(SecurityMixin, BaseCommand): +class Command(CryptMixin, BaseCommand): help = ( "Using a manifest.json file, load the data from there, and import the " "documents it refers to." @@ -182,19 +178,7 @@ class Command(SecurityMixin, BaseCommand): "No passphrase was given, but this export contains encrypted fields", ) elif EXPORTER_CRYPTO_SETTINGS_NAME in data: - # Load up the values for setting up decryption - self.kdf_algorithm: str = data[EXPORTER_CRYPTO_SETTINGS_NAME][ - EXPORTER_CRYPTO_ALGO_NAME - ] - self.key_iterations: int = data[EXPORTER_CRYPTO_SETTINGS_NAME][ - EXPORTER_CRYPTO_KEY_ITERATIONS_NAME - ] - self.key_size: int = data[EXPORTER_CRYPTO_SETTINGS_NAME][ - EXPORTER_CRYPTO_KEY_SIZE_NAME - ] - self.salt: str = data[EXPORTER_CRYPTO_SETTINGS_NAME][ - EXPORTER_CRYPTO_SALT_NAME - ] + self.load_crypt_params(data) if self.version and self.version != version.__full_version_str__: self.stdout.write( diff --git a/src/documents/management/commands/mixins.py b/src/documents/management/commands/mixins.py index 87a246695..847f027e1 100644 --- a/src/documents/management/commands/mixins.py +++ b/src/documents/management/commands/mixins.py @@ -2,12 +2,19 @@ import base64 import os from argparse import ArgumentParser from typing import Optional +from typing import Union from cryptography.fernet import Fernet from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC from django.core.management import CommandError +from documents.settings import EXPORTER_CRYPTO_ALGO_NAME +from documents.settings import EXPORTER_CRYPTO_KEY_ITERATIONS_NAME +from documents.settings import EXPORTER_CRYPTO_KEY_SIZE_NAME +from documents.settings import EXPORTER_CRYPTO_SALT_NAME +from documents.settings import EXPORTER_CRYPTO_SETTINGS_NAME + class MultiProcessMixin: """ @@ -48,7 +55,7 @@ class ProgressBarMixin: self.use_progress_bar = not self.no_progress_bar -class SecurityMixin: +class CryptMixin: """ Fully based on: https://cryptography.io/en/latest/fernet/#using-passwords-with-fernet @@ -79,6 +86,31 @@ class SecurityMixin: key_size = 32 kdf_algorithm = "pbkdf2_sha256" + def get_crypt_params(self) -> dict[str, dict[str, Union[str, int]]]: + return { + EXPORTER_CRYPTO_SETTINGS_NAME: { + EXPORTER_CRYPTO_ALGO_NAME: self.kdf_algorithm, + EXPORTER_CRYPTO_KEY_ITERATIONS_NAME: self.key_iterations, + EXPORTER_CRYPTO_KEY_SIZE_NAME: self.key_size, + EXPORTER_CRYPTO_SALT_NAME: self.salt, + }, + } + + def load_crypt_params(self, metadata: dict): + # Load up the values for setting up decryption + self.kdf_algorithm: str = metadata[EXPORTER_CRYPTO_SETTINGS_NAME][ + EXPORTER_CRYPTO_ALGO_NAME + ] + self.key_iterations: int = metadata[EXPORTER_CRYPTO_SETTINGS_NAME][ + EXPORTER_CRYPTO_KEY_ITERATIONS_NAME + ] + self.key_size: int = metadata[EXPORTER_CRYPTO_SETTINGS_NAME][ + EXPORTER_CRYPTO_KEY_SIZE_NAME + ] + self.salt: str = metadata[EXPORTER_CRYPTO_SETTINGS_NAME][ + EXPORTER_CRYPTO_SALT_NAME + ] + def setup_crypto(self, *, passphrase: str, salt: Optional[str] = None): """ Constructs a class for encryption or decryption using the specified passphrase and salt diff --git a/src/documents/tests/test_management_exporter.py b/src/documents/tests/test_management_exporter.py index dca3bf90b..a68d52aa8 100644 --- a/src/documents/tests/test_management_exporter.py +++ b/src/documents/tests/test_management_exporter.py @@ -39,6 +39,7 @@ from documents.tests.utils import DirectoriesMixin from documents.tests.utils import FileSystemAssertsMixin from documents.tests.utils import SampleDirMixin from documents.tests.utils import paperless_environment +from paperless_mail.models import MailAccount class TestExportImport( @@ -841,5 +842,47 @@ class TestExportImport( self.assertEqual(Document.objects.all().count(), 4) + +class TestCryptExportImport( + DirectoriesMixin, + FileSystemAssertsMixin, + TestCase, +): + def setUp(self) -> None: + self.target = Path(tempfile.mkdtemp()) + return super().setUp() + + def tearDown(self) -> None: + shutil.rmtree(self.target, ignore_errors=True) + return super().tearDown() + def test_export_passphrase(self): - pass + MailAccount.objects.create( + name="Test Account", + imap_server="test.imap.com", + username="myusername", + password="mypassword", + ) + + call_command( + "document_exporter", + "--no-progress-bar", + "--passphrase", + "securepassword", + self.target, + ) + + self.assertIsFile(self.target / "metadata.json") + self.assertIsFile(self.target / "manifest.json") + + data = json.loads((self.target / "manifest.json").read_text()) + + mail_accounts = list( + filter(lambda r: r["model"] == "paperless_mail.mailaccount", data), + ) + + self.assertEqual(len(mail_accounts), 1) + + mail_account_data = mail_accounts[0] + + self.assertNotEqual(mail_account_data["fields"]["password"], "mypassword") diff --git a/src/setup.cfg b/src/setup.cfg index 1877cb16e..4350c0451 100644 --- a/src/setup.cfg +++ b/src/setup.cfg @@ -4,6 +4,7 @@ addopts = --pythonwarnings=all --cov --cov-report=html --cov-report=xml --numpro env = PAPERLESS_DISABLE_DBHANDLER=true PAPERLESS_CACHE_BACKEND=django.core.cache.backends.locmem.LocMemCache +norecursedirs = locale/* [coverage:run] source =