Merge branch 'dev' into feature-better-bs-icons

Pipfile.lock

@@ -239,12 +239,12 @@
         },
         "channels-redis": {
             "hashes": [
-                "sha256:3696f5b9fe367ea495d402ba83d7c3c99e8ca0e1354ff8d913535976ed0abf73",
+                "sha256:01c26c4d5d3a203f104bba9e5585c0305a70df390d21792386586068162027fd",
-                "sha256:6bd4f75f4ab4a7db17cee495593ace886d7e914c66f8214a1f247ff6659c073a"
+                "sha256:2c5b944a39bd984b72aa8005a3ae11637bf29b5092adeb91c9aad4ab819a8ac4"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.7'",
+            "markers": "python_version >= '3.8'",
-            "version": "==4.1.0"
+            "version": "==4.2.0"
         },
         "chardet": {
             "hashes": [
@@ -574,12 +574,12 @@
         },
         "gotenberg-client": {
             "hashes": [
-                "sha256:69e9dd5264b75ed0ba1f9eebebdc750b13d190710fd82ca0670d161c249155c9",
+                "sha256:097151c959d9ad9c6292694dac454a07a511489a353086df924f489190084425",
-                "sha256:dd0f49d3d4e01399949f39ac5024a5512566c8ded6ee457a336a5f77ce4c1a25"
+                "sha256:3d6c0449fd1afb82206bfdc2edacfe1d7d98e9de7207332b696b97a3d4dfba6b"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==0.4.1"
+            "version": "==0.5.0"
         },
         "gunicorn": {
             "hashes": [
@@ -2868,17 +2868,16 @@
                 "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa",
                 "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"
             ],
-            "index": "pypi",
             "markers": "python_version >= '3.7'",
             "version": "==3.1.3"
         },
         "markdown": {
             "hashes": [
-                "sha256:5874b47d4ee3f0b14d764324d2c94c03ea66bee56f2d929da9f2508d65e722dc",
+                "sha256:d43323865d89fc0cb9b20c75fc8ad313af307cc087e84b657d9eec768eddeadd",
-                "sha256:b65d7beb248dc22f2e8a31fb706d93798093c308dc1aba295aedeb9d41a813bd"
+                "sha256:e1ac7b3dc550ee80e602e71c1d168002f062e49f1b11e26a36264dafd4df2ef8"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.5.1"
+            "version": "==3.5.2"
         },
         "markupsafe": {
             "hashes": [
@@ -2972,12 +2971,12 @@
         },
         "mkdocs-material": {
             "hashes": [
-                "sha256:5899219f422f0a6de784232d9d40374416302ffae3c160cacc72969fcc1ee372",
+                "sha256:3d196ee67fad16b2df1a458d650a8ac1890294eaae368d26cee71bc24ad41c40",
-                "sha256:76c93a8525cceb0b395b9cedab3428bf518cf6439adef2b940f1c1574b775d89"
+                "sha256:efd7cc8ae03296d728da9bd38f4db8b07ab61f9738a0cbd0dfaf2a15a50e7343"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==9.5.3"
+            "version": "==9.5.4"
         },
         "mkdocs-material-extensions": {
             "hashes": [
@@ -3288,7 +3287,6 @@
                 "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
                 "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
             ],
-            "index": "pypi",
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.8.2"
         },
@@ -3376,6 +3374,7 @@
                 "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d",
                 "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"
             ],
+            "markers": "python_version >= '3.6'",
             "version": "==6.0.1"
         },
         "pyyaml-env-tag": {
@@ -3495,27 +3494,27 @@
         },
         "ruff": {
             "hashes": [
-                "sha256:09c415716884950080921dd6237767e52e227e397e2008e2bed410117679975b",
+                "sha256:226b517f42d59a543d6383cfe03cccf0091e3e0ed1b856c6824be03d2a75d3b6",
-                "sha256:0f58948c6d212a6b8d41cd59e349751018797ce1727f961c2fa755ad6208ba45",
+                "sha256:2f59bcf5217c661254bd6bc42d65a6fd1a8b80c48763cb5c2293295babd945dd",
-                "sha256:190a566c8f766c37074d99640cd9ca3da11d8deae2deae7c9505e68a4a30f740",
+                "sha256:5f0312ba1061e9b8c724e9a702d3c8621e3c6e6c2c9bd862550ab2951ac75c16",
-                "sha256:231d8fb11b2cc7c0366a326a66dafc6ad449d7fcdbc268497ee47e1334f66f77",
+                "sha256:6bbbc3042075871ec17f28864808540a26f0f79a4478c357d3e3d2284e832998",
-                "sha256:4b077ce83f47dd6bea1991af08b140e8b8339f0ba8cb9b7a484c30ebab18a23f",
+                "sha256:7a36fa90eb12208272a858475ec43ac811ac37e91ef868759770b71bdabe27b6",
-                "sha256:5b25093dad3b055667730a9b491129c42d45e11cdb7043b702e97125bcec48a1",
+                "sha256:9a1600942485c6e66119da294c6294856b5c86fd6df591ce293e4a4cc8e72989",
-                "sha256:6464289bd67b2344d2a5d9158d5eb81025258f169e69a46b741b396ffb0cda95",
+                "sha256:9ebb40442f7b531e136d334ef0851412410061e65d61ca8ce90d894a094feb22",
-                "sha256:934832f6ed9b34a7d5feea58972635c2039c7a3b434fe5ba2ce015064cb6e955",
+                "sha256:9fb6b3b86450d4ec6a6732f9f60c4406061b6851c4b29f944f8c9d91c3611c7a",
-                "sha256:97ce4d752f964ba559c7023a86e5f8e97f026d511e48013987623915431c7ea9",
+                "sha256:a623349a505ff768dad6bd57087e2461be8db58305ebd5577bd0e98631f9ae69",
-                "sha256:9b8f397902f92bc2e70fb6bebfa2139008dc72ae5177e66c383fa5426cb0bf2c",
+                "sha256:b13ba5d7156daaf3fd08b6b993360a96060500aca7e307d95ecbc5bb47a69296",
-                "sha256:9bd4025b9c5b429a48280785a2b71d479798a69f5c2919e7d274c5f4b32c3607",
+                "sha256:dcaab50e278ff497ee4d1fe69b29ca0a9a47cd954bb17963628fa417933c6eb1",
-                "sha256:a7f772696b4cdc0a3b2e527fc3c7ccc41cdcb98f5c80fdd4f2b8c50eb1458196",
+                "sha256:e261f1baed6291f434ffb1d5c6bd8051d1c2a26958072d38dfbec39b3dda7352",
-                "sha256:c4a88efecec23c37b11076fe676e15c6cdb1271a38f2b415e381e87fe4517f18",
+                "sha256:e3fd36e0d48aeac672aa850045e784673449ce619afc12823ea7868fcc41d8ba",
-                "sha256:e1ad00662305dcb1e987f5ec214d31f7d6a062cae3e74c1cbccef15afd96611d",
+                "sha256:e6894b00495e00c27b6ba61af1fc666f17de6140345e5ef27dd6e08fb987259d",
-                "sha256:ea0d3e950e394c4b332bcdd112aa566010a9f9c95814844a7468325290aabfd9",
+                "sha256:ee3febce7863e231a467f90e681d3d89210b900d49ce88723ce052c8761be8c7",
-                "sha256:eb85ee287b11f901037a6683b2374bb0ec82928c5cbc984f575d0437979c521a",
+                "sha256:f57de973de4edef3ad3044d6a50c02ad9fc2dff0d88587f25f1a48e3f72edf5e",
-                "sha256:f9d4d88cb6eeb4dfe20f9f0519bd2eaba8119bde87c3d5065c541dbae2b5a2cb"
+                "sha256:f988746e3c3982bea7f824c8fa318ce7f538c4dfefec99cd09c8770bd33e6539"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.7'",
-            "version": "==0.1.11"
+            "version": "==0.1.13"
         },
         "scipy": {
             "hashes": [
@@ -3668,7 +3667,6 @@
                 "sha256:d00e6be486affb5781468457b21a6cbe848c33ef43f9ea4a73b4882e5f188a44",
                 "sha256:d429c2430c93b7903914e4db9a966c7f2b068dd2ebdd2fa9b9ce094c7d459f33"
             ],
-            "index": "pypi",
             "markers": "python_version >= '3.7'",
             "version": "==3.0.0"
         },

src-ui/src/app/components/document-detail/document-detail.component.spec.ts

@@ -968,6 +968,8 @@ describe('DocumentDetailComponent', () => {
   })
 
   it('should warn when open document does not match doc retrieved from backend on init', () => {
+    let openModal: NgbModalRef
+    modalService.activeInstances.subscribe((modals) => (openModal = modals[0]))
     const modalSpy = jest.spyOn(modalService, 'open')
     const openDoc = Object.assign({}, doc)
     // simulate a document being modified elsewhere and db updated
@@ -986,6 +988,10 @@ describe('DocumentDetailComponent', () => {
     )
     fixture.detectChanges() // calls ngOnInit
     expect(modalSpy).toHaveBeenCalledWith(ConfirmDialogComponent)
+    const closeSpy = jest.spyOn(openModal, 'close')
+    const confirmDialog = openModal.componentInstance as ConfirmDialogComponent
+    confirmDialog.confirmClicked.next(confirmDialog)
+    expect(closeSpy).toHaveBeenCalled()
   })
 
   function initNormally() {

src-ui/src/app/components/document-detail/document-detail.component.ts

@@ -309,6 +309,9 @@ export class DocumentDetailComponent
               modal.componentInstance.message = $localize`Saving the document here may overwrite other changes that were made. To restore the existing version, discard your changes or close the document.`
               modal.componentInstance.cancelBtnClass = 'visually-hidden'
               modal.componentInstance.btnCaption = $localize`Ok`
+              modal.componentInstance.confirmClicked.subscribe(() =>
+                modal.close()
+              )
             }
 
             if (this.documentForm.dirty) {

src/documents/classifier.py

@@ -207,6 +207,7 @@ class DocumentClassifier:
             self.last_doc_change_time is not None
             and self.last_doc_change_time >= latest_doc_change
         ) and self.last_auto_type_hash == hasher.digest():
+            logger.info("No updates since last training")
             return False
 
         # subtract 1 since -1 (null) is also part of the classes.

src/documents/conditionals.py

@@ -0,0 +1,87 @@
+import pickle
+from datetime import datetime
+from typing import Optional
+
+from django.conf import settings
+
+from documents.classifier import DocumentClassifier
+from documents.models import Document
+
+
+def suggestions_etag(request, pk: int) -> Optional[str]:
+    """
+    Returns an optional string for the ETag, allowing browser caching of
+    suggestions if the classifier has not been changed and the suggested dates
+    setting is also unchanged
+
+    TODO: It would be nice to not duplicate the partial loading and the loading
+    between here and the actual classifier
+    """
+    if not settings.MODEL_FILE.exists():
+        return None
+    with open(settings.MODEL_FILE, "rb") as f:
+        schema_version = pickle.load(f)
+        if schema_version != DocumentClassifier.FORMAT_VERSION:
+            return None
+        _ = pickle.load(f)
+        last_auto_type_hash: bytes = pickle.load(f)
+        return f"{last_auto_type_hash}:{settings.NUMBER_OF_SUGGESTED_DATES}"
+
+
+def suggestions_last_modified(request, pk: int) -> Optional[datetime]:
+    """
+    Returns the datetime of classifier last modification.  This is slightly off,
+    as there is not way to track the suggested date setting modification, but it seems
+    unlikely that changes too often
+    """
+    if not settings.MODEL_FILE.exists():
+        return None
+    with open(settings.MODEL_FILE, "rb") as f:
+        schema_version = pickle.load(f)
+        if schema_version != DocumentClassifier.FORMAT_VERSION:
+            return None
+        last_doc_change_time = pickle.load(f)
+        return last_doc_change_time
+
+
+def metadata_etag(request, pk: int) -> Optional[str]:
+    """
+    Metadata is extracted from the original file, so use its checksum as the
+    ETag
+    """
+    try:
+        doc = Document.objects.get(pk=pk)
+        return doc.checksum
+    except Document.DoesNotExist:
+        return None
+    return None
+
+
+def metadata_last_modified(request, pk: int) -> Optional[datetime]:
+    """
+    Metadata is extracted from the original file, so use its modified.  Strictly speaking, this is
+    not the modification of the original file, but of the database object, but might as well
+    error on the side of more cautious
+    """
+    try:
+        doc = Document.objects.get(pk=pk)
+        return doc.modified
+    except Document.DoesNotExist:
+        return None
+    return None
+
+
+def preview_etag(request, pk: int) -> Optional[str]:
+    """
+    ETag for the document preview, using the original or archive checksum, depending on the request
+    """
+    try:
+        doc = Document.objects.get(pk=pk)
+        use_original = (
+            "original" in request.query_params
+            and request.query_params["original"] == "true"
+        )
+        return doc.checksum if use_original else doc.archive_checksum
+    except Document.DoesNotExist:
+        return None
+    return None

src/documents/static/signin.css

@@ -42,6 +42,10 @@ body {
   z-index: 100;
 }
 
+#inputUsername:focus~label {
+  z-index: 101;
+}
+
 #inputPassword,
 #inputPassword2 {
   border-top-left-radius: 0;

src/documents/tests/test_api_documents.py

@@ -1266,6 +1266,86 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
             },
         )
 
+    @mock.patch("documents.conditionals.pickle.load")
+    @mock.patch("documents.views.match_storage_paths")
+    @mock.patch("documents.views.match_document_types")
+    @mock.patch("documents.views.match_tags")
+    @mock.patch("documents.views.match_correspondents")
+    @override_settings(NUMBER_OF_SUGGESTED_DATES=10)
+    def test_get_suggestions_cached(
+        self,
+        match_correspondents,
+        match_tags,
+        match_document_types,
+        match_storage_paths,
+        mocked_pickle_load,
+    ):
+        """
+        GIVEN:
+           - Request for suggestions for a document
+        WHEN:
+          - Classifier has not been modified
+        THEN:
+          - Subsequent requests are returned alright
+          - ETag and last modified are called
+        """
+        settings.MODEL_FILE.touch()
+
+        from documents.classifier import DocumentClassifier
+
+        last_modified = timezone.now()
+
+        # ETag first, then modified
+        mock_effect = [
+            DocumentClassifier.FORMAT_VERSION,
+            "dont care",
+            b"thisisachecksum",
+            DocumentClassifier.FORMAT_VERSION,
+            last_modified,
+        ]
+        mocked_pickle_load.side_effect = mock_effect
+
+        doc = Document.objects.create(
+            title="test",
+            mime_type="application/pdf",
+            content="this is an invoice from 12.04.2022!",
+        )
+
+        match_correspondents.return_value = [Correspondent(id=88), Correspondent(id=2)]
+        match_tags.return_value = [Tag(id=56), Tag(id=123)]
+        match_document_types.return_value = [DocumentType(id=23)]
+        match_storage_paths.return_value = [StoragePath(id=99), StoragePath(id=77)]
+
+        response = self.client.get(f"/api/documents/{doc.pk}/suggestions/")
+        self.assertEqual(
+            response.data,
+            {
+                "correspondents": [88, 2],
+                "tags": [56, 123],
+                "document_types": [23],
+                "storage_paths": [99, 77],
+                "dates": ["2022-04-12"],
+            },
+        )
+        mocked_pickle_load.assert_called()
+        self.assertIn("Last-Modified", response.headers)
+        self.assertEqual(
+            response.headers["Last-Modified"],
+            last_modified.strftime("%a, %d %b %Y %H:%M:%S %Z").replace("UTC", "GMT"),
+        )
+        self.assertIn("ETag", response.headers)
+        self.assertEqual(
+            response.headers["ETag"],
+            f"\"b'thisisachecksum':{settings.NUMBER_OF_SUGGESTED_DATES}\"",
+        )
+
+        mocked_pickle_load.rest_mock()
+        mocked_pickle_load.side_effect = mock_effect
+
+        response = self.client.get(f"/api/documents/{doc.pk}/suggestions/")
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        mocked_pickle_load.assert_called()
+
     @mock.patch("documents.parsers.parse_date_generator")
     @override_settings(NUMBER_OF_SUGGESTED_DATES=0)
     def test_get_suggestions_dates_disabled(
@@ -1971,6 +2051,35 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
         self.assertEqual(resp.status_code, status.HTTP_200_OK)
         self.assertEqual(resp.content, b"1000")
 
+    def test_next_asn_no_documents_with_asn(self):
+        """
+        GIVEN:
+            - Existing document, but with no ASN assugned
+        WHEN:
+            - API request to get next ASN
+        THEN:
+            - ASN 1 is returned
+        """
+        user1 = User.objects.create_user(username="test1")
+        user1.user_permissions.add(*Permission.objects.all())
+        user1.save()
+
+        doc1 = Document.objects.create(
+            title="test",
+            mime_type="application/pdf",
+            content="this is a document 1",
+            checksum="1",
+        )
+        doc1.save()
+
+        self.client.force_authenticate(user1)
+
+        resp = self.client.get(
+            "/api/documents/next_asn/",
+        )
+        self.assertEqual(resp.status_code, status.HTTP_200_OK)
+        self.assertEqual(resp.content, b"1")
+
 
 class TestDocumentApiV2(DirectoriesMixin, APITestCase):
     def setUp(self):

src/documents/views.py

@@ -34,6 +34,7 @@ from django.utils.decorators import method_decorator
 from django.utils.translation import get_language
 from django.views import View
 from django.views.decorators.cache import cache_control
+from django.views.decorators.http import condition
 from django.views.generic import TemplateView
 from django_filters.rest_framework import DjangoFilterBackend
 from langdetect import detect
@@ -62,6 +63,11 @@ from documents.bulk_download import ArchiveOnlyStrategy
 from documents.bulk_download import OriginalAndArchiveStrategy
 from documents.bulk_download import OriginalsOnlyStrategy
 from documents.classifier import load_classifier
+from documents.conditionals import metadata_etag
+from documents.conditionals import metadata_last_modified
+from documents.conditionals import preview_etag
+from documents.conditionals import suggestions_etag
+from documents.conditionals import suggestions_last_modified
 from documents.data_models import ConsumableDocument
 from documents.data_models import DocumentMetadataOverrides
 from documents.data_models import DocumentSource
@@ -386,6 +392,9 @@ class DocumentViewSet(
             return None
 
     @action(methods=["get"], detail=True)
+    @method_decorator(
+        condition(etag_func=metadata_etag, last_modified_func=metadata_last_modified),
+    )
     def metadata(self, request, pk=None):
         try:
             doc = Document.objects.get(pk=pk)
@@ -430,6 +439,12 @@ class DocumentViewSet(
         return Response(meta)
 
     @action(methods=["get"], detail=True)
+    @method_decorator(
+        condition(
+            etag_func=suggestions_etag,
+            last_modified_func=suggestions_last_modified,
+        ),
+    )
     def suggestions(self, request, pk=None):
         doc = get_object_or_404(Document, pk=pk)
         if request.user is not None and not has_perms_owner_aware(
@@ -467,6 +482,8 @@ class DocumentViewSet(
         )
 
     @action(methods=["get"], detail=True)
+    @method_decorator(cache_control(public=False, max_age=5 * 60))
+    @method_decorator(condition(etag_func=preview_etag))
     def preview(self, request, pk=None):
         try:
             response = self.file_response(pk, request, "inline")
@@ -740,16 +757,12 @@ class UnifiedSearchViewSet(DocumentViewSet):
 
     @action(detail=False, methods=["GET"], name="Get Next ASN")
     def next_asn(self, request, *args, **kwargs):
-        return Response(
+        max_asn = Document.objects.aggregate(
-            (
+            Max("archive_serial_number", default=0),
-                Document.objects.filter(archive_serial_number__gte=0)
+        ).get(
-                .order_by("archive_serial_number")
+            "archive_serial_number__max",
-                .last()
-                .archive_serial_number
-                or 0
-            )
-            + 1,
         )
+        return Response(max_asn + 1)
 
 
 class LogViewSet(ViewSet):