From 9251a3054fdad54ded267de975783cdd06635519 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Thu, 6 Jun 2024 08:23:44 -0700 Subject: [PATCH] Adding documentation --- docs/administration.md | 16 +++++--- .../management/commands/document_importer.py | 18 +++++++-- .../tests/test_management_exporter.py | 35 ++++++++++++++++ .../tests/test_management_importer.py | 40 +++++++++++++++++++ 4 files changed, 100 insertions(+), 9 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index a65647836..6bdb43b64 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -248,6 +248,7 @@ optional arguments: -z, --zip -zn, --zip-name --data-only +--passphrase ``` `target` is a folder to which the data gets written. This includes @@ -309,6 +310,9 @@ value set in `-zn` or `--zip-name`. If `--data-only` is provided, only the database will be exported. This option is intended to facilitate database upgrades without needing to clean documents and thumbnails from the media directory. +If `--passphrase` is provided, it will be used to encrypt certain fields in the export. This value +must be provided to import. If this value is lost, the export cannot be imported. + !!! warning If exporting with the file name format, there may be errors due to @@ -327,16 +331,18 @@ and the script does the rest of the work: document_importer source ``` -| Option | Required | Default | Description | -| ----------- | -------- | ------- | ------------------------------------------------------------------------- | -| source | Yes | N/A | The directory containing an export | -| --data-only | No | False | If provided, only import data, do not import document files or thumbnails | +| Option | Required | Default | Description | +| ------------ | -------- | ------- | ------------------------------------------------------------------------- | +| source | Yes | N/A | The directory containing an export | +| --data-only | No | False | If provided, only import data, do not import document files or thumbnails | +| --passphrase | No | N/A | If your export was encrypted with a passphrase, must be provided | When you use the provided docker compose script, put the export inside the `export` folder in your paperless source directory. Specify `../export` as the `source`. -Note that .zip files (as can be generated from the exporter) are not supported. +Note that .zip files (as can be generated from the exporter) are not supported. You must unzip them into +the target directory first. !!! note diff --git a/src/documents/management/commands/document_importer.py b/src/documents/management/commands/document_importer.py index 1ff86be4a..351f6fd44 100644 --- a/src/documents/management/commands/document_importer.py +++ b/src/documents/management/commands/document_importer.py @@ -206,7 +206,7 @@ class Command(CryptMixin, BaseCommand): if ( self.version is not None and self.version != version.__full_version_str__ - ): + ): # pragma: no cover self.stdout.write( self.style.ERROR( "Version mismatch: " @@ -235,10 +235,10 @@ class Command(CryptMixin, BaseCommand): self.pre_check() - self.load_manifest_files() - self.load_metadata() + self.load_manifest_files() + self.check_manifest_validity() self.decrypt_secret_fields() @@ -404,8 +404,18 @@ class Command(CryptMixin, BaseCommand): # Salt has been loaded from metadata.json at this point, so it cannot be None self.setup_crypto(passphrase=self.passphrase, salt=self.salt) - for record in self.manifest: + had_an_account = False + + for index, record in enumerate(self.manifest): if record["model"] == "paperless_mail.mailaccount": record["fields"]["password"] = self.decrypt_string( value=record["fields"]["password"], ) + self.manifest[index] = record + had_an_account = True + if had_an_account: + # It's annoying, but the DB is loaded from the JSON directly + # Maybe could change that in the future? + (self.source / "manifest.json").write_text( + json.dumps(self.manifest, indent=2, ensure_ascii=False), + ) diff --git a/src/documents/tests/test_management_exporter.py b/src/documents/tests/test_management_exporter.py index a68d52aa8..abb2ef485 100644 --- a/src/documents/tests/test_management_exporter.py +++ b/src/documents/tests/test_management_exporter.py @@ -886,3 +886,38 @@ class TestCryptExportImport( mail_account_data = mail_accounts[0] self.assertNotEqual(mail_account_data["fields"]["password"], "mypassword") + + MailAccount.objects.all().delete() + + call_command( + "document_importer", + "--no-progress-bar", + "--passphrase", + "securepassword", + self.target, + ) + + account = MailAccount.objects.first() + + self.assertIsNotNone(account) + self.assertEqual(account.password, "mypassword") + + def test_import_crypt_no_passphrase(self): + call_command( + "document_exporter", + "--no-progress-bar", + "--passphrase", + "securepassword", + self.target, + ) + + with self.assertRaises(CommandError) as err: + call_command( + "document_importer", + "--no-progress-bar", + self.target, + ) + self.assertEqual( + err.msg, + "No passphrase was given, but this export contains encrypted fields", + ) diff --git a/src/documents/tests/test_management_importer.py b/src/documents/tests/test_management_importer.py index 7522c4738..9c98086b7 100644 --- a/src/documents/tests/test_management_importer.py +++ b/src/documents/tests/test_management_importer.py @@ -279,3 +279,43 @@ class TestCommandImport( "Found existing documents(s), this might indicate a non-empty installation", str(stdout.read()), ) + + def test_import_no_metadata_or_version_file(self): + stdout = StringIO() + + (self.dirs.scratch_dir / "manifest.json").touch() + + # We're not building a manifest, so it fails, but this test doesn't care + with self.assertRaises(json.decoder.JSONDecodeError): + call_command( + "document_importer", + "--no-progress-bar", + str(self.dirs.scratch_dir), + stdout=stdout, + ) + stdout.seek(0) + stdout_str = str(stdout.read()) + + self.assertIn("No version.json or metadata.json file located", stdout_str) + + def test_import_version_file(self): + stdout = StringIO() + + (self.dirs.scratch_dir / "manifest.json").touch() + (self.dirs.scratch_dir / "version.json").write_text( + json.dumps({"version": "2.8.1"}), + ) + + # We're not building a manifest, so it fails, but this test doesn't care + with self.assertRaises(json.decoder.JSONDecodeError): + call_command( + "document_importer", + "--no-progress-bar", + str(self.dirs.scratch_dir), + stdout=stdout, + ) + stdout.seek(0) + stdout_str = str(stdout.read()) + + self.assertIn("Version mismatch:", stdout_str) + self.assertIn("importing 2.8.1", stdout_str)