Merge pull request #902 from paperless-ngx/v1.7.1

v1.7.1

.build-config.json

@@ -0,0 +1,9 @@
+{
+  "qpdf": {
+      "version": "10.6.3"
+    },
+  "jbig2enc": {
+      "version": "0.29",
+      "git_tag": "0.29"
+    }
+}

.editorconfig

@@ -33,5 +33,5 @@ indent_style = space
 [**/test_*.py]
 max_line_length = off
 
-[Dockerfile]
+[Dockerfile*]
 indent_style = space

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -0,0 +1,86 @@
+name: Bug report
+description: Something is not working
+title: "[BUG] Concise description of the issue"
+labels: ["bug", "unconfirmed"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Have a question? 👉 [Start a new discussion](https://github.com/paperless-ngx/paperless-ngx/discussions/new) or [ask in chat](https://matrix.to/#/#paperless:adnidor.de).
+
+        Before opening an issue, please double check:
+
+        - [The troubleshooting documentation](https://paperless-ngx.readthedocs.io/en/latest/troubleshooting.html).
+        - [The installation instructions](https://paperless-ngx.readthedocs.io/en/latest/setup.html#installation).
+        - [Existing issues and discussions](https://github.com/paperless-ngx/paperless-ngx/search?q=&type=issues).
+
+        If you encounter issues while installing or configuring Paperless-ngx, please post in the ["Support" section of the discussions](https://github.com/paperless-ngx/paperless-ngx/discussions/new?category=support).
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: A clear and concise description of what the bug is. If applicable, add screenshots to help explain your problem.
+      placeholder: |
+        Currently Paperless does not work when...
+
+        [Screenshot if applicable]
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Steps to reproduce
+      description: Steps to reproduce the behavior.
+      placeholder: |
+        1. Go to '...'
+        2. Click on '....'
+        3. See error
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Webserver logs
+      description: If available, post any logs from the web server related to your issue.
+      render: bash
+  - type: input
+    id: version
+    attributes:
+      label: Paperless-ngx version
+      placeholder: e.g. 1.6.0
+    validations:
+      required: true
+  - type: input
+    id: host-os
+    attributes:
+      label: Host OS
+      description: Host OS of the machine running paperless-ngx. Please add the architecture (uname -m) if applicable.
+      placeholder: e.g. Archlinux / Ubuntu 20.04 / Raspberry Pi `arm64`
+    validations:
+      required: true
+  - type: dropdown
+    id: install-method
+    attributes:
+      label: Installation method
+      options:
+        - Docker
+        - Bare metal
+        - Other (please describe above)
+    validations:
+      required: true
+  - type: input
+    id: browser
+    attributes:
+      label: Browser
+      description: Which browser you are using, if relevant.
+      placeholder: e.g. Chrome, Safari
+  - type: input
+    id: config-changes
+    attributes:
+      label: Configuration changes
+      description: Any configuration changes you made in `docker-compose.yml`, `docker-compose.env` or `paperless.conf`.
+  - type: input
+    id: other
+    attributes:
+      label: Other
+      description: Any other relevant details.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,50 +0,0 @@
----
-name: Bug report
-about: Something is not working
-title: '[BUG] Concise description of the issue'
-labels: ''
-assignees: ''
----
-
-<!---
-=> Before opening an issue, please check the documentation and see if it helps you resolve your issue: https://paperless-ngx.readthedocs.io/en/latest/troubleshooting.html
-=> Please also make sure that you followed the installation instructions.
-=> Please search the issues and look for similar issues before opening a bug report.
-
-=> If you would like to submit a feature request please submit one under https://github.com/paperless-ngx/paperless-ngx/discussions/categories/feature-requests
-
-=> If you encounter issues while installing of configuring Paperless-ngx, please post that in the "Support" section of the discussions. Remember that Paperless successfully runs on a variety of different systems. If paperless does not start, it's probably an issue with your system, and not an issue of paperless.
-
-=> Don't remove the [BUG] prefix from the title.
--->
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**To Reproduce**
-Steps to reproduce the behavior:
-
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-**Webserver logs**
-
-```
-If available, post any logs from the web server related to your issue.
-```
-
-**Relevant information**
-
-- Host OS of the machine running paperless: [e.g. Archlinux / Ubuntu 20.04]
-- Browser [e.g. chrome, safari]
-- Version [e.g. 1.0.0]
-- Installation method: [docker / bare metal]
-- Any configuration changes you made in `docker-compose.yml`, `docker-compose.env` or `paperless.conf`.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,11 @@
+blank_issues_enabled: false
+contact_links:
+  - name: 🤔 Questions and Help
+    url: https://github.com/paperless-ngx/paperless-ngx/discussions
+    about: This issue tracker is not for support questions. Please refer to our Discussions.
+  - name: 💬 Chat
+    url: https://matrix.to/#/#paperless:adnidor.de
+    about: Want to discuss Paperless-ngx with others? Check out our chat.
+  - name: 🚀 Feature Request
+    url: https://github.com/paperless-ngx/paperless-ngx/discussions/new?category=feature-requests
+    about: Remember to search for existing feature requests and "up-vote" any you like

.github/ISSUE_TEMPLATE/other.md

@@ -1,19 +0,0 @@
----
-name: Other
-about: Anything that is not a feature request or bug.
-title: '[Other] Title of your issue'
-labels: ''
-assignees: ''
----
-
-<!--
-
-=> Discussions, Feedback and other suggestions belong in the "Discussion" section and not on the issue tracker.
-
-=> If you would like to submit a feature request please submit one under https://github.com/paperless-ngx/paperless-ngx/discussions/categories/feature-requests
-
-=> If you encounter issues while installing of configuring Paperless-ngx, please post that in the "Support" section of the discussions. Remember that Paperless successfully runs on a variety of different systems. If paperless does not start, it's probably is an issue with your system, and not an issue of paperless.
-
-=> Don't remove the [Other] prefix from the title.
-
--->

.github/dependabot.yml

@@ -6,11 +6,14 @@ updates:
   # Enable version updates for npm
   - package-ecosystem: "npm"
     target-branch: "dev"
-    # Look for `package.json` and `lock` files in the `root` directory
+    # Look for `package.json` and `lock` files in the `/src-ui` directory
     directory: "/src-ui"
     # Check the npm registry for updates every month
     schedule:
       interval: "monthly"
+    labels:
+      - "frontend"
+      - "dependencies"
     # Add reviewers
     reviewers:
       - "paperless-ngx/frontend"
@@ -26,9 +29,13 @@ updates:
     labels:
       - "backend"
       - "dependencies"
+    # Add reviewers
+    reviewers:
+      - "paperless-ngx/backend"
 
   # Enable updates for Github Actions
   - package-ecosystem: "github-actions"
+    target-branch: "dev"
     directory: "/"
     schedule:
       # Check for updates to GitHub Actions every month
@@ -38,4 +45,4 @@ updates:
       - "dependencies"
     # Add reviewers
     reviewers:
-      - "paperless-ngx/backend"
+      - "paperless-ngx/ci-cd"

.github/release-drafter.yml

@@ -1,4 +1,7 @@
 categories:
+  - title: 'Breaking Changes'
+    labels:
+      - 'breaking-change'
   - title: 'Features'
     labels:
       - 'enhancement'
@@ -27,8 +30,7 @@ replacers: # Changes "Feature: Update checker" to "Update checker"
     replace: ''
 change-template: '- $TITLE @$AUTHOR (#$NUMBER)'
 change-title-escapes: '\<*_&#@'
-tag-prefix: "ngx-"
 template: |
-  ## Changelog
+  # Changelog
 
   $CHANGES

.github/scripts/common.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+
+
+def get_image_tag(
+    repo_name: str,
+    pkg_name: str,
+    pkg_version: str,
+) -> str:
+    """
+    Returns a string representing the normal image for a given package
+    """
+    return f"ghcr.io/{repo_name}/builder/{pkg_name}:{pkg_version}"
+
+
+def get_cache_image_tag(
+    repo_name: str,
+    pkg_name: str,
+    pkg_version: str,
+    branch_name: str,
+) -> str:
+    """
+    Returns a string representing the expected image cache tag for a given package
+
+    Registry type caching is utilized for the builder images, to allow fast
+    rebuilds, generally almost instant for the same version
+    """
+    return f"ghcr.io/{repo_name}/builder/cache/{pkg_name}:{pkg_version}"

.github/scripts/get-build-json.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+"""
+This is a helper script for the mutli-stage Docker image builder.
+It provides a single point of configuration for package version control.
+The output JSON object is used by the CI workflow to determine what versions
+to build and pull into the final Docker image.
+
+Python package information is obtained from the Pipfile.lock.  As this is
+kept updated by dependabot, it usually will need no further configuration.
+The sole exception currently is pikepdf, which has a dependency on qpdf,
+and is configured here to use the latest version of qpdf built by the workflow.
+
+Other package version information is configured directly below, generally by
+setting the version and Git information, if any.
+
+"""
+import argparse
+import json
+import os
+from pathlib import Path
+from typing import Final
+
+from common import get_cache_image_tag
+from common import get_image_tag
+
+
+def _main():
+    parser = argparse.ArgumentParser(
+        description="Generate a JSON object of information required to build the given package, based on the Pipfile.lock",
+    )
+    parser.add_argument(
+        "package",
+        help="The name of the package to generate JSON for",
+    )
+
+    PIPFILE_LOCK_PATH: Final[Path] = Path("Pipfile.lock")
+    BUILD_CONFIG_PATH: Final[Path] = Path(".build-config.json")
+
+    # Read the main config file
+    build_json: Final = json.loads(BUILD_CONFIG_PATH.read_text())
+
+    # Read Pipfile.lock file
+    pipfile_data: Final = json.loads(PIPFILE_LOCK_PATH.read_text())
+
+    args: Final = parser.parse_args()
+
+    # Read from environment variables set by GitHub Actions
+    repo_name: Final[str] = os.environ["GITHUB_REPOSITORY"]
+    branch_name: Final[str] = os.environ["GITHUB_REF_NAME"]
+
+    # Default output values
+    version = None
+    git_tag = None
+    extra_config = {}
+
+    if args.package in pipfile_data["default"]:
+        # Read the version from Pipfile.lock
+        pkg_data = pipfile_data["default"][args.package]
+        pkg_version = pkg_data["version"].split("==")[-1]
+        version = pkg_version
+
+        # Based on the package, generate the expected Git tag name
+        if args.package == "pikepdf":
+            git_tag = f"v{pkg_version}"
+        elif args.package == "psycopg2":
+            git_tag = pkg_version.replace(".", "_")
+
+        # Any extra/special values needed
+        if args.package == "pikepdf":
+            extra_config["qpdf_version"] = build_json["qpdf"]["version"]
+
+    elif args.package in build_json:
+        version = build_json[args.package]["version"]
+
+        if "git_tag" in build_json[args.package]:
+            git_tag = build_json[args.package]["git_tag"]
+    else:
+        raise NotImplementedError(args.package)
+
+    # The JSON object we'll output
+    output = {
+        "name": args.package,
+        "version": version,
+        "git_tag": git_tag,
+        "image_tag": get_image_tag(repo_name, args.package, version),
+        "cache_tag": get_cache_image_tag(
+            repo_name,
+            args.package,
+            version,
+            branch_name,
+        ),
+    }
+
+    # Add anything special a package may need
+    output.update(extra_config)
+
+    # Output the JSON info to stdout
+    print(json.dumps(output))
+
+
+if __name__ == "__main__":
+    _main()

.github/stale.yml

@@ -2,11 +2,8 @@
 daysUntilStale: 30
 # Number of days of inactivity before a stale issue is closed
 daysUntilClose: 7
-# Issues with these labels will never be considered stale
-exemptLabels:
-  - pinned
-  - security
-  - fixpending
+onlyLabels:
+  - unconfirmed
 # Label to use when marking an issue as stale
 staleLabel: stale
 # Comment to post when marking an issue as stale. Set to `false` to disable

.github/workflows/ci.yml

@@ -3,8 +3,10 @@ name: ci
 on:
   push:
     tags:
-      - ngx-*
-      - beta-*
+      # https://semver.org/#spec-item-2
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+      # https://semver.org/#spec-item-9
+      - 'v[0-9]+.[0-9]+.[0-9]+-beta.rc[0-9]+'
     branches-ignore:
       - 'translations**'
   pull_request:
@@ -45,159 +47,164 @@ jobs:
           name: documentation
           path: docs/_build/html/
 
-  code-checks-backend:
-    name: "Backend Code Checks"
-    runs-on: ubuntu-20.04
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      -
-        name: Install checkers
-        run: |
-          pipx install reorder-python-imports
-          pipx install yesqa
-          pipx install add-trailing-comma
-          pipx install flake8
-      -
-        name: Run reorder-python-imports
-        run: |
-          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs reorder-python-imports
-      -
-        name: Run yesqa
-        run: |
-          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs yesqa
-      -
-        name: Run add-trailing-comma
-        run: |
-          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs add-trailing-comma
-      # black is placed after add-trailing-comma because it may format differently
-      # if a trailing comma is added
-      -
-        name: Run black
-        uses: psf/black@stable
-        with:
-          options: "--check --diff"
-          version: "22.3.0"
-      -
-        name: Run flake8 checks
-        run: |
-          cd src/
-          flake8 --max-line-length=88 --ignore=E203,W503
+  ci-backend:
+    uses: ./.github/workflows/reusable-ci-backend.yml
 
-  code-checks-frontend:
-    name: "Frontend Code Checks"
-    runs-on: ubuntu-20.04
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-        with:
-          node-version: '16'
-      -
-        name: Install prettier
-        run: |
-          npm install prettier
-      -
-        name: Run prettier
-        run:
-          npx prettier --check --ignore-path Pipfile.lock **/*.js **/*.ts *.md **/*.md
+  ci-frontend:
+    uses: ./.github/workflows/reusable-ci-frontend.yml
 
-  tests-backend:
-    needs: [code-checks-backend]
-    name: "Backend Tests (${{ matrix.python-version }})"
+  prepare-docker-build:
+    name: Prepare Docker Pipeline Data
+    if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v'))
     runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version: ['3.8', '3.9']
-      fail-fast: false
+    needs:
+      - documentation
+      - ci-backend
+      - ci-frontend
     steps:
       -
         name: Checkout
         uses: actions/checkout@v3
-        with:
-          fetch-depth: 2
-      -
-        name: Install pipenv
-        run: pipx install pipenv
       -
         name: Set up Python
         uses: actions/setup-python@v3
         with:
-          python-version: "${{ matrix.python-version }}"
-          cache: "pipenv"
-          cache-dependency-path: 'Pipfile.lock'
+          python-version: "3.9"
       -
-        name: Install system dependencies
+        name: Setup qpdf image
+        id: qpdf-setup
         run: |
-          sudo apt-get update -qq
-          sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript optipng libzbar0 poppler-utils
-      -
-        name: Install Python dependencies
-        run: |
-          pipenv sync --dev
-      -
-        name: Tests
-        run: |
-          cd src/
-          pipenv run pytest
-      -
-        name: Get changed files
-        id: changed-files-specific
-        uses: tj-actions/changed-files@v18.1
-        with:
-          files: |
-            src/**
-      -
-        name: List all changed files
-        run: |
-          for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
-            echo "${file} was changed"
-          done
-      -
-        name: Publish coverage results
-        if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        # https://github.com/coveralls-clients/coveralls-python/issues/251
-        run: |
-          cd src/
-          pipenv run coveralls --service=github
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py qpdf)
 
-  tests-frontend:
-    needs: [code-checks-frontend]
-    name: "Frontend Tests"
-    runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        node-version: [16.x]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
-        with:
-          node-version: ${{ matrix.node-version }}
-      - run: cd src-ui && npm ci
-      - run: cd src-ui && npm run test
-      - run: cd src-ui && npm run e2e:ci
+          echo ${build_json}
+
+          echo ::set-output name=qpdf-json::${build_json}
+      -
+        name: Setup psycopg2 image
+        id: psycopg2-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py psycopg2)
+
+          echo ${build_json}
+
+          echo ::set-output name=psycopg2-json::${build_json}
+      -
+        name: Setup pikepdf image
+        id: pikepdf-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py pikepdf)
+
+          echo ${build_json}
+
+          echo ::set-output name=pikepdf-json::${build_json}
+      -
+        name: Setup jbig2enc image
+        id: jbig2enc-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py jbig2enc)
+
+          echo ${build_json}
+
+          echo ::set-output name=jbig2enc-json::${build_json}
+
+    outputs:
+
+      qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }}
+
+      pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }}
+
+      psycopg2-json: ${{ steps.psycopg2-setup.outputs.psycopg2-json }}
+
+      jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}}
+
+  build-qpdf-debs:
+    name: qpdf
+    needs:
+      - prepare-docker-build
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.qpdf
+      build-json: ${{ needs.prepare-docker-build.outputs.qpdf-json }}
+      build-args: |
+        QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
+
+  build-jbig2enc:
+    name: jbig2enc
+    needs:
+      - prepare-docker-build
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.jbig2enc
+      build-json: ${{ needs.prepare-docker-build.outputs.jbig2enc-json }}
+      build-args: |
+        JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
+
+  build-psycopg2-wheel:
+    name: psycopg2
+    needs:
+      - prepare-docker-build
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.psycopg2
+      build-json: ${{ needs.prepare-docker-build.outputs.psycopg2-json }}
+      build-args: |
+        PSYCOPG2_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).git_tag }}
+        PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
+
+  build-pikepdf-wheel:
+    name: pikepdf
+    needs:
+      - prepare-docker-build
+      - build-qpdf-debs
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.pikepdf
+      build-json: ${{ needs.prepare-docker-build.outputs.pikepdf-json }}
+      build-args: |
+        REPO=${{ github.repository }}
+        QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
+        PIKEPDF_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).git_tag }}
+        PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
 
   # build and push image to docker hub.
   build-docker-image:
-    if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || startsWith(github.ref, 'refs/tags/ngx-') || startsWith(github.ref, 'refs/tags/beta-'))
     runs-on: ubuntu-20.04
-    needs: [tests-backend, tests-frontend]
+    concurrency:
+      group: ${{ github.workflow }}-build-docker-image-${{ github.ref_name }}
+      cancel-in-progress: true
+    needs:
+      - prepare-docker-build
+      - build-psycopg2-wheel
+      - build-jbig2enc
+      - build-qpdf-debs
+      - build-pikepdf-wheel
     steps:
+      -
+        name: Check pushing to Docker Hub
+        id: docker-hub
+        # Only push to Dockerhub from the main repo
+        # Otherwise forks would require a Docker Hub account and secrets setup
+        run: |
+          if [[ ${{ github.repository }} == "paperless-ngx/paperless-ngx" ]] ; then
+            echo ::set-output name=enable::"true"
+          else
+            echo ::set-output name=enable::"false"
+          fi
       -
         name: Gather Docker metadata
         id: docker-meta
         uses: docker/metadata-action@v3
         with:
-          images: ghcr.io/${{ github.repository }}
+          images: |
+            ghcr.io/${{ github.repository }}
+            name=paperlessngx/paperless-ngx,enable=${{ steps.docker-hub.outputs.enable }}
           tags: |
-            type=match,pattern=ngx-(\d.\d.\d),group=1
+            # Tag branches with branch name
             type=ref,event=branch
-            type=ref,event=tag
+            # Process semver tags
+            # For a tag x.y.z or vX.Y.Z, output an x.y.z and x.y image tag
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
       -
         name: Checkout
         uses: actions/checkout@v3
@@ -214,6 +221,14 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Login to Docker Hub
+        uses: docker/login-action@v1
+        # Don't attempt to login is not pushing to Docker Hub
+        if: steps.docker-hub.outputs.enable == 'true'
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
       -
         name: Build and push
         uses: docker/build-push-action@v2
@@ -224,8 +239,19 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.docker-meta.outputs.tags }}
           labels: ${{ steps.docker-meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          build-args: |
+            JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
+            QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
+            PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
+            PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
+          # Get cache layers from this branch, then dev, then main
+          # This allows new branches to get at least some cache benefits, generally from dev
+          cache-from: |
+            type=registry,ref=ghcr.io/${{ github.repository }}/builder/cache/app:${{ github.ref_name }}
+            type=registry,ref=ghcr.io/${{ github.repository }}/builder/cache/app:dev
+            type=registry,ref=ghcr.io/${{ github.repository }}/builder/cache/app:main
+          cache-to: |
+            type=registry,mode=max,ref=ghcr.io/${{ github.repository }}/builder/cache/app:${{ github.ref_name }}
       -
         name: Inspect image
         run: |
@@ -243,7 +269,8 @@ jobs:
           path: src/documents/static/frontend/
 
   build-release:
-    needs: [build-docker-image, documentation]
+    needs:
+      - build-docker-image
     runs-on: ubuntu-20.04
     steps:
       -
@@ -311,8 +338,9 @@ jobs:
 
   publish-release:
     runs-on: ubuntu-20.04
-    needs: build-release
-    if: contains(github.ref, 'refs/tags/ngx-') || contains(github.ref, 'refs/tags/beta-')
+    needs:
+      - build-release
+    if: github.ref_type == 'tag' && (startsWith(github.ref_name, 'v') || contains(github.ref_name, '-beta.rc'))
     steps:
       -
         name: Download release artifact
@@ -324,12 +352,11 @@ jobs:
         name: Get version
         id: get_version
         run: |
-          if [[ $GITHUB_REF == refs/tags/ngx-* ]]; then
-            echo ::set-output name=version::${GITHUB_REF#refs/tags/ngx-}
-            echo ::set-output name=prerelease::false
-          elif [[ $GITHUB_REF == refs/tags/beta-* ]]; then
-            echo ::set-output name=version::${GITHUB_REF#refs/tags/beta-}
+          echo ::set-output name=version::${{ github.ref_name }}
+          if [[ ${{ contains(github.ref_name, '-beta.rc') }} == 'true' ]]; then
             echo ::set-output name=prerelease::true
+          else
+            echo ::set-output name=prerelease::false
           fi
       -
         name: Create Release and Changelog
@@ -337,7 +364,7 @@ jobs:
         uses: release-drafter/release-drafter@v5
         with:
           name: Paperless-ngx ${{ steps.get_version.outputs.version }}
-          tag: ngx-${{ steps.get_version.outputs.version }}
+          tag: ${{ steps.get_version.outputs.version }}
           version: ${{ steps.get_version.outputs.version }}
           prerelease: ${{ steps.get_version.outputs.prerelease }}
           publish: true # ensures release is not marked as draft

.github/workflows/codeql-analysis.yml

@@ -42,7 +42,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -51,4 +51,4 @@ jobs:
         # queries: ./path/to/local/query, your-org/your-repo/queries@main
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

.github/workflows/reusable-ci-backend.yml

@@ -0,0 +1,108 @@
+name: Backend CI Jobs
+
+on:
+  workflow_call:
+
+jobs:
+
+  code-checks-backend:
+    name: "Code Style Checks"
+    runs-on: ubuntu-20.04
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Install checkers
+        run: |
+          pipx install reorder-python-imports
+          pipx install yesqa
+          pipx install add-trailing-comma
+          pipx install flake8
+      -
+        name: Run reorder-python-imports
+        run: |
+          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs reorder-python-imports
+      -
+        name: Run yesqa
+        run: |
+          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs yesqa
+      -
+        name: Run add-trailing-comma
+        run: |
+          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs add-trailing-comma
+      # black is placed after add-trailing-comma because it may format differently
+      # if a trailing comma is added
+      -
+        name: Run black
+        uses: psf/black@stable
+        with:
+          options: "--check --diff"
+          version: "22.3.0"
+      -
+        name: Run flake8 checks
+        run: |
+          cd src/
+          flake8 --max-line-length=88 --ignore=E203,W503
+
+  tests-backend:
+    name: "Tests (${{ matrix.python-version }})"
+    runs-on: ubuntu-20.04
+    needs:
+      - code-checks-backend
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.9', '3.10']
+      fail-fast: false
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      -
+        name: Install pipenv
+        run: pipx install pipenv
+      -
+        name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: "${{ matrix.python-version }}"
+          cache: "pipenv"
+          cache-dependency-path: 'Pipfile.lock'
+      -
+        name: Install system dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript optipng libzbar0 poppler-utils
+      -
+        name: Install Python dependencies
+        run: |
+          pipenv sync --dev
+      -
+        name: Tests
+        run: |
+          cd src/
+          pipenv run pytest
+      -
+        name: Get changed files
+        id: changed-files-specific
+        uses: tj-actions/changed-files@v19
+        with:
+          files: |
+            src/**
+      -
+        name: List all changed files
+        run: |
+          for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
+            echo "${file} was changed"
+          done
+      -
+        name: Publish coverage results
+        if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        # https://github.com/coveralls-clients/coveralls-python/issues/251
+        run: |
+          cd src/
+          pipenv run coveralls --service=github

.github/workflows/reusable-ci-frontend.yml

@@ -0,0 +1,42 @@
+name: Frontend CI Jobs
+
+on:
+  workflow_call:
+
+jobs:
+
+  code-checks-frontend:
+    name: "Code Style Checks"
+    runs-on: ubuntu-20.04
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '16'
+      -
+        name: Install prettier
+        run: |
+          npm install prettier
+      -
+        name: Run prettier
+        run:
+          npx prettier --check --ignore-path Pipfile.lock **/*.js **/*.ts *.md **/*.md
+  tests-frontend:
+    name: "Tests"
+    runs-on: ubuntu-20.04
+    needs:
+      - code-checks-frontend
+    strategy:
+      matrix:
+        node-version: [16.x]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+      - run: cd src-ui && npm ci
+      - run: cd src-ui && npm run test
+      - run: cd src-ui && npm run e2e:ci

.github/workflows/reusable-workflow-builder.yml

@@ -0,0 +1,53 @@
+name: Reusable Image Builder
+
+on:
+  workflow_call:
+    inputs:
+      dockerfile:
+        required: true
+        type: string
+      build-json:
+        required: true
+        type: string
+      build-args:
+        required: false
+        default: ""
+        type: string
+
+concurrency:
+  group: ${{ github.workflow }}-${{ fromJSON(inputs.build-json).name }}-${{ fromJSON(inputs.build-json).version }}
+  cancel-in-progress: false
+
+jobs:
+  build-image:
+    name: Build ${{ fromJSON(inputs.build-json).name }} @ ${{ fromJSON(inputs.build-json).version }}
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Login to Github Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Build ${{ fromJSON(inputs.build-json).name }}
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ${{ inputs.dockerfile }}
+          tags: ${{ fromJSON(inputs.build-json).image_tag }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          build-args: ${{ inputs.build-args }}
+          push: true
+          cache-from: type=registry,ref=${{ fromJSON(inputs.build-json).cache_tag }}
+          cache-to: type=registry,mode=max,ref=${{ fromJSON(inputs.build-json).cache_tag }}

.pre-commit-config.yaml

@@ -37,7 +37,7 @@ repos:
         exclude: "(^Pipfile\\.lock$)"
   # Python hooks
   - repo: https://github.com/asottile/reorder_python_imports
-    rev: v3.0.1
+    rev: v3.1.0
     hooks:
       - id: reorder-python-imports
         exclude: "(migrations)"
@@ -47,7 +47,7 @@ repos:
       - id: yesqa
         exclude: "(migrations)"
   - repo: https://github.com/asottile/add-trailing-comma
-    rev: "v2.2.2"
+    rev: "v2.2.3"
     hooks:
       - id: add-trailing-comma
         exclude: "(migrations)"
@@ -62,11 +62,25 @@ repos:
     rev: 22.3.0
     hooks:
       - id: black
-  # Dockerfile hooks
-  - repo: https://github.com/pryorda/dockerfilelint-precommit-hooks
-    rev: "v0.1.0"
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v2.32.1
     hooks:
-      - id: dockerfilelint
+      - id: pyupgrade
+        exclude: "(migrations)"
+        args:
+          - "--py38-plus"
+  # Dockerfile hooks
+  - repo: https://github.com/AleksaC/hadolint-py
+    rev: v2.10.0
+    hooks:
+      - id: hadolint
+        args:
+          - --ignore
+          - DL3008      # https://github.com/hadolint/hadolint/wiki/DL3008  (should probably do this at some point)
+          - --ignore
+          - DL3013      # https://github.com/hadolint/hadolint/wiki/DL3013  (should probably do this too at some point)
+          - --ignore
+          - DL3003      # https://github.com/hadolint/hadolint/wiki/DL3003  (seems excessive to use WORKDIR so much)
   # Shell script hooks
   - repo: https://github.com/lovesegfault/beautysh
     rev: v6.2.1

Dockerfile

@@ -1,12 +1,36 @@
-FROM node:16 AS compile-frontend
+# syntax=docker/dockerfile:1.4
 
-COPY . /src
+# Pull the installer images from the library
+# These are all built previously
+# They provide either a .deb or .whl
+
+ARG JBIG2ENC_VERSION
+ARG QPDF_VERSION
+ARG PIKEPDF_VERSION
+ARG PSYCOPG2_VERSION
+
+FROM ghcr.io/paperless-ngx/paperless-ngx/builder/jbig2enc:${JBIG2ENC_VERSION} as jbig2enc-builder
+FROM ghcr.io/paperless-ngx/paperless-ngx/builder/qpdf:${QPDF_VERSION} as qpdf-builder
+FROM ghcr.io/paperless-ngx/paperless-ngx/builder/pikepdf:${PIKEPDF_VERSION} as pikepdf-builder
+FROM ghcr.io/paperless-ngx/paperless-ngx/builder/psycopg2:${PSYCOPG2_VERSION} as psycopg2-builder
+
+FROM --platform=$BUILDPLATFORM node:16-bullseye-slim AS compile-frontend
+
+# This stage compiles the frontend
+# This stage runs once for the native platform, as the outputs are not
+# dependent on target arch
+# Inputs: None
+
+COPY ./src-ui /src/src-ui
 
 WORKDIR /src/src-ui
-RUN npm update npm -g && npm ci --no-optional
-RUN ./node_modules/.bin/ng build --configuration production
+RUN set -eux \
+  && npm update npm -g \
+  && npm ci --no-optional
+RUN set -eux \
+  && ./node_modules/.bin/ng build --configuration production
 
-FROM ghcr.io/paperless-ngx/builder/ngx-base:1.7.0 as main-app
+FROM python:3.9-slim-bullseye as main-app
 
 LABEL org.opencontainers.image.authors="paperless-ngx team <hello@paperless-ngx.com>"
 LABEL org.opencontainers.image.documentation="https://paperless-ngx.readthedocs.io/en/latest/"
@@ -14,45 +38,167 @@ LABEL org.opencontainers.image.source="https://github.com/paperless-ngx/paperles
 LABEL org.opencontainers.image.url="https://github.com/paperless-ngx/paperless-ngx"
 LABEL org.opencontainers.image.licenses="GPL-3.0-only"
 
-WORKDIR /usr/src/paperless/src/
+ARG DEBIAN_FRONTEND=noninteractive
 
-COPY requirements.txt ../
+#
+# Begin installation and configuration
+# Order the steps below from least often changed to most
+#
 
-# Python dependencies
-RUN apt-get update \
-  # python-Levenshtein still needs to be compiled here
-  && apt-get -y --no-install-recommends install \
-    build-essential \
-    && python3 -m pip install --upgrade --no-cache-dir pip wheel \
-  && python3 -m pip install --default-timeout=1000 --upgrade --no-cache-dir supervisor \
-  && python3 -m pip install --default-timeout=1000 --no-cache-dir -r ../requirements.txt \
-  && apt-get -y purge build-essential \
-  && apt-get -y autoremove --purge \
-  && rm -rf /var/lib/apt/lists/*
+# copy jbig2enc
+# Basically will never change again
+COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/.libs/libjbig2enc* /usr/local/lib/
+COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/jbig2 /usr/local/bin/
+COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/*.h /usr/local/include/
+
+# Packages need for running
+ARG RUNTIME_PACKAGES="\
+  curl \
+  file \
+  # fonts for text file thumbnail generation
+  fonts-liberation \
+  gettext \
+  ghostscript \
+  gnupg \
+  gosu \
+  icc-profiles-free \
+  imagemagick \
+  media-types \
+  liblept5 \
+  libpq5 \
+  libxml2 \
+  liblcms2-2 \
+  libtiff5 \
+  libxslt1.1 \
+  libfreetype6 \
+  libwebp6 \
+  libopenjp2-7 \
+  libimagequant0 \
+  libraqm0 \
+  libgnutls30 \
+  libjpeg62-turbo \
+  optipng \
+  python3 \
+  python3-pip \
+  python3-setuptools \
+  postgresql-client \
+  # For Numpy
+  libatlas3-base \
+  # thumbnail size reduction
+  pngquant \
+  # OCRmyPDF dependencies
+  tesseract-ocr \
+  tesseract-ocr-eng \
+  tesseract-ocr-deu \
+  tesseract-ocr-fra \
+  tesseract-ocr-ita \
+  tesseract-ocr-spa \
+  tzdata \
+  unpaper \
+  # Mime type detection
+  zlib1g \
+  # Barcode splitter
+  libzbar0 \
+  poppler-utils"
+
+# Install basic runtime packages.
+# These change very infrequently
+RUN set -eux \
+  echo "Installing system packages" \
+    && apt-get update \
+    && apt-get install --yes --quiet --no-install-recommends ${RUNTIME_PACKAGES} \
+    && rm -rf /var/lib/apt/lists/* \
+  && echo "Installing supervisor" \
+    && python3 -m pip install --default-timeout=1000 --upgrade --no-cache-dir supervisor==4.2.4
+
+# Copy gunicorn config
+# Changes very infrequently
+WORKDIR /usr/src/paperless/
+
+COPY gunicorn.conf.py .
 
 # setup docker-specific things
-COPY docker/ ./docker/
+# Use mounts to avoid copying installer files into the image
+# These change sometimes, but rarely
+WORKDIR /usr/src/paperless/src/docker/
 
-RUN cd docker \
+RUN --mount=type=bind,readwrite,source=docker,target=./ \
+  set -eux \
+  && echo "Configuring ImageMagick" \
     && cp imagemagick-policy.xml /etc/ImageMagick-6/policy.xml \
-  && mkdir /var/log/supervisord /var/run/supervisord \
-  && cp supervisord.conf /etc/supervisord.conf \
-  && cp docker-entrypoint.sh /sbin/docker-entrypoint.sh \
-  && chmod 755 /sbin/docker-entrypoint.sh \
-  && cp docker-prepare.sh /sbin/docker-prepare.sh \
-  && chmod 755 /sbin/docker-prepare.sh \
-  && chmod +x install_management_commands.sh \
-  && ./install_management_commands.sh \
-  && cd .. \
-  && rm -rf docker/
+  && echo "Configuring supervisord" \
+    && mkdir /var/log/supervisord /var/run/supervisord \
+    && cp supervisord.conf /etc/supervisord.conf \
+  && echo "Setting up Docker scripts" \
+    && cp docker-entrypoint.sh /sbin/docker-entrypoint.sh \
+    && chmod 755 /sbin/docker-entrypoint.sh \
+    && cp docker-prepare.sh /sbin/docker-prepare.sh \
+    && chmod 755 /sbin/docker-prepare.sh \
+    && cp wait-for-redis.py /sbin/wait-for-redis.py \
+    && chmod 755 /sbin/wait-for-redis.py \
+  && echo "Installing managment commands" \
+    && chmod +x install_management_commands.sh \
+    && ./install_management_commands.sh
 
-COPY gunicorn.conf.py ../
+# Install the built packages from the installer library images
+# Use mounts to avoid copying installer files into the image
+# These change sometimes
+RUN --mount=type=bind,from=qpdf-builder,target=/qpdf \
+    --mount=type=bind,from=psycopg2-builder,target=/psycopg2 \
+    --mount=type=bind,from=pikepdf-builder,target=/pikepdf \
+  set -eux \
+  && echo "Installing qpdf" \
+    && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/libqpdf28_*.deb \
+    && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/qpdf_*.deb \
+  && echo "Installing pikepdf and dependencies" \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/packaging*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/lxml*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/Pillow*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/pyparsing*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/pikepdf*.whl \
+    && python -m pip list \
+  && echo "Installing psycopg2" \
+    && python3 -m pip install --no-cache-dir /psycopg2/usr/src/psycopg2/wheels/psycopg2*.whl \
+    && python -m pip list
 
-# copy app
-COPY --from=compile-frontend /src/src/ ./
+# Python dependencies
+# Change pretty frequently
+COPY requirements.txt ../
+
+# Packages needed only for building a few quick Python
+# dependencies
+ARG BUILD_PACKAGES="\
+  build-essential \
+  python3-dev"
+
+RUN set -eux \
+  && echo "Installing build system packages" \
+    && apt-get update \
+    && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
+    && python3 -m pip install --no-cache-dir --upgrade wheel \
+  && echo "Installing Python requirements" \
+    && python3 -m pip install --default-timeout=1000 --no-cache-dir -r ../requirements.txt \
+  && echo "Cleaning up image" \
+    && apt-get -y purge ${BUILD_PACKAGES} \
+    && apt-get -y autoremove --purge \
+    && apt-get clean --yes \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /tmp/* \
+    && rm -rf /var/tmp/* \
+    && rm -rf /var/cache/apt/archives/* \
+    && truncate -s 0 /var/log/*log
+
+WORKDIR /usr/src/paperless/src/
+
+# copy backend
+COPY ./src ./
+
+# copy frontend
+COPY --from=compile-frontend /src/src/documents/static/frontend/ ./documents/static/frontend/
 
 # add users, setup scripts
-RUN addgroup --gid 1000 paperless \
+RUN set -eux \
+  && addgroup --gid 1000 paperless \
   && useradd --uid 1000 --gid paperless --home-dir /usr/src/paperless paperless \
   && chown -R paperless:paperless ../ \
   && gosu paperless python3 manage.py collectstatic --clear --no-input \

Pipfile

@@ -19,7 +19,7 @@ djangorestframework = "~=3.13"
 filelock = "*"
 fuzzywuzzy = {extras = ["speedup"], version = "*"}
 gunicorn = "*"
-imap-tools = "*"
+imap-tools = "~=0.54.0"
 langdetect = "*"
 pathvalidate = "*"
 pillow = "~=9.1"

Pipfile.lock

@@ -44,11 +44,11 @@
         },
         "asgiref": {
             "hashes": [
-                "sha256:2f8abc20f7248433085eda803936d98992f1343ddb022065779f37c5da0181d0",
-                "sha256:88d59c13d634dcffe0510be048210188edd79aeccb6a6c9028cdad6f31d730a9"
+                "sha256:45a429524fba18aba9d512498b19d220c4d628e75b40cf5c627524dbaebc5cc1",
+                "sha256:fddeea3c53fa99d0cdb613c3941cc6e52d822491fc2753fba25768fb5bf4e865"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.5.0"
+            "version": "==3.5.1"
         },
         "async-timeout": {
             "hashes": [
@@ -99,6 +99,7 @@
                 "sha256:f04e857b59d9d1ccc39ce2da1021d196e47234873820cbeaad210724b1ee28ac",
                 "sha256:fadbfe37f74051d024037f223b8e001611eac868b5c5b06144ef4d8b799862f2"
             ],
+            "index": "pypi",
             "markers": "python_version < '3.9'",
             "version": "==0.2.1"
         },
@@ -206,11 +207,11 @@
         },
         "click": {
             "hashes": [
-                "sha256:24e1a4a9ec5bf6299411369b208c1df2188d9eb8d916302fe6bf03faed227f1e",
-                "sha256:479707fe14d9ec9a0757618b7a100a0ae4c4e236fac5b7f80ca68028141a1a72"
+                "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e",
+                "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==8.1.2"
+            "version": "==8.1.3"
         },
         "coloredlogs": {
             "hashes": [
@@ -237,29 +238,31 @@
         },
         "cryptography": {
             "hashes": [
-                "sha256:0a3bf09bb0b7a2c93ce7b98cb107e9170a90c51a0162a20af1c61c765b90e60b",
-                "sha256:1f64a62b3b75e4005df19d3b5235abd43fa6358d5516cfc43d87aeba8d08dd51",
-                "sha256:32db5cc49c73f39aac27574522cecd0a4bb7384e71198bc65a0d23f901e89bb7",
-                "sha256:4881d09298cd0b669bb15b9cfe6166f16fc1277b4ed0d04a22f3d6430cb30f1d",
-                "sha256:4e2dddd38a5ba733be6a025a1475a9f45e4e41139d1321f412c6b360b19070b6",
-                "sha256:53e0285b49fd0ab6e604f4c5d9c5ddd98de77018542e88366923f152dbeb3c29",
-                "sha256:70f8f4f7bb2ac9f340655cbac89d68c527af5bb4387522a8413e841e3e6628c9",
-                "sha256:7b2d54e787a884ffc6e187262823b6feb06c338084bbe80d45166a1cb1c6c5bf",
-                "sha256:7be666cc4599b415f320839e36367b273db8501127b38316f3b9f22f17a0b815",
-                "sha256:8241cac0aae90b82d6b5c443b853723bcc66963970c67e56e71a2609dc4b5eaf",
-                "sha256:82740818f2f240a5da8dfb8943b360e4f24022b093207160c77cadade47d7c85",
-                "sha256:8897b7b7ec077c819187a123174b645eb680c13df68354ed99f9b40a50898f77",
-                "sha256:c2c5250ff0d36fd58550252f54915776940e4e866f38f3a7866d92b32a654b86",
-                "sha256:ca9f686517ec2c4a4ce930207f75c00bf03d94e5063cbc00a1dc42531511b7eb",
-                "sha256:d2b3d199647468d410994dbeb8cec5816fb74feb9368aedf300af709ef507e3e",
-                "sha256:da73d095f8590ad437cd5e9faf6628a218aa7c387e1fdf67b888b47ba56a17f0",
-                "sha256:e167b6b710c7f7bc54e67ef593f8731e1f45aa35f8a8a7b72d6e42ec76afd4b3",
-                "sha256:ea634401ca02367c1567f012317502ef3437522e2fc44a3ea1844de028fa4b84",
-                "sha256:ec6597aa85ce03f3e507566b8bcdf9da2227ec86c4266bd5e6ab4d9e0cc8dab2",
-                "sha256:f64b232348ee82f13aac22856515ce0195837f6968aeaa94a3d0353ea2ec06a6"
+                "sha256:06bfafa6e53ccbfb7a94be4687b211a025ce0625e3f3c60bb15cd048a18f3ed8",
+                "sha256:0db5cf21bd7d092baacb576482b0245102cea2d3cf09f09271ce9f69624ecb6f",
+                "sha256:125702572be12bcd318e3a14e9e70acd4be69a43664a75f0397e8650fe3c6cc3",
+                "sha256:1858eff6246bb8bbc080eee78f3dd1528739e3f416cba5f9914e8631b8df9871",
+                "sha256:315af6268de72bcfa0bb3401350ce7d921f216e6b60de12a363dad128d9d459f",
+                "sha256:451aaff8b8adf2dd0597cbb1fdcfc8a7d580f33f843b7cce75307a7f20112dd8",
+                "sha256:58021d6e9b1d88b1105269d0da5e60e778b37dfc0e824efc71343dd003726831",
+                "sha256:618391152147a1221c87b1b0b7f792cafcfd4b5a685c5c72eeea2ddd29aeceff",
+                "sha256:6d4daf890e674d191757d8d7d60dc3a29c58c72c7a76a05f1c0a326013f47e8b",
+                "sha256:74b55f67f4cf026cb84da7a1b04fc2a1d260193d4ad0ea5e9897c8b74c1e76ac",
+                "sha256:7ceae26f876aabe193b13a0c36d1bb8e3e7e608d17351861b437bd882f617e9f",
+                "sha256:930b829e8a2abaf43a19f38277ae3c5e1ffcf547b936a927d2587769ae52c296",
+                "sha256:a18ff4bfa9d64914a84d7b06c46eb86e0cc03113470b3c111255aceb6dcaf81d",
+                "sha256:ae1cd29fbe6b716855454e44f4bf743465152e15d2d317303fe3b58ee9e5af7a",
+                "sha256:b1ee5c82cf03b30f6ae4e32d2bcb1e167ef74d6071cbb77c2af30f101d0b360b",
+                "sha256:bf585476fcbcd37bed08072e8e2db3954ce1bfc68087a2dc9c19cfe0b90979ca",
+                "sha256:c4a58eeafbd7409054be41a377e726a7904a17c26f45abf18125d21b1215b08b",
+                "sha256:cce90609e01e1b192fae9e13665058ab46b2ea53a3c05a3ea74a3eb8c3af8857",
+                "sha256:d610d0ee14dd9109006215c7c0de15eee91230b70a9bce2263461cf7c3720b83",
+                "sha256:e69a0e36e62279120e648e787b76d79b41e0f9e86c1c636a4f38d415595c722e",
+                "sha256:f095988548ec5095e3750cdb30e6962273d239b1998ba1aac66c0d5bee7111c1",
+                "sha256:faf0f5456c059c7b1c29441bdd5e988f0ba75bdc3eea776520d8dcb1e30e1b5c"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==36.0.2"
+            "version": "==37.0.1"
         },
         "daphne": {
             "hashes": [
@@ -474,16 +477,16 @@
                 "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff",
                 "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"
             ],
-            "markers": "python_version >= '3.5'",
+            "markers": "python_version >= '3'",
             "version": "==3.3"
         },
         "imap-tools": {
             "hashes": [
-                "sha256:119f1a60ea4048a4c5d72d9e9fa47c295685e340c730cb0b71fdf0ad3b7e53f8",
-                "sha256:3648bac835657b1c56ba856452c8a28bdbe3689d3730f95a4ad20d4c39f1c2d0"
+                "sha256:15d20ac8695fc4978a913c2186f482a802f5229c41c6e0c66c7bad8f1f590cf1",
+                "sha256:606b73a1b5ecc4c72eea5ad19231e07a88bf9ba9adbdd4acb8cf71a359dd43ec"
             ],
             "index": "pypi",
-            "version": "==0.53.0"
+            "version": "==0.54.0"
         },
         "img2pdf": {
             "hashes": [
@@ -493,11 +496,12 @@
         },
         "importlib-resources": {
             "hashes": [
-                "sha256:1b93238cbf23b4cde34240dd8321d99e9bf2eb4bc91c0c99b2886283e7baad85",
-                "sha256:a9dd72f6cc106aeb50f6e66b86b69b454766dd6e39b69ac68450253058706bcc"
+                "sha256:b6062987dfc51f0fcb809187cffbd60f35df7acb4589091f154214af6d0d49d3",
+                "sha256:e447dc01619b1e951286f3929be820029d48c75eb25d265c28b92a16548212b8"
             ],
+            "index": "pypi",
             "markers": "python_version < '3.9'",
-            "version": "==5.6.0"
+            "version": "==5.7.1"
         },
         "incremental": {
             "hashes": [
@@ -671,11 +675,11 @@
         },
         "ocrmypdf": {
             "hashes": [
-                "sha256:7f0a6165b80ba1b37ce5943cf5b4faf93bf98c04c8f5157ef83c5f292491485f",
-                "sha256:d52410bc38cf5b66da27668e38c66ac41fd3136457c1ec388b311f0a78ee213c"
+                "sha256:0c1cc0a7596fa9da1bfde67141227eeb813aba5e954f88199eacc5f51f1d67d9",
+                "sha256:48bbdd5d15b76f34aa3a91910918e51f91bb3833b4e86da45f8542afda118404"
             ],
             "index": "pypi",
-            "version": "==13.4.2"
+            "version": "==13.4.3"
         },
         "packaging": {
             "hashes": [
@@ -711,36 +715,36 @@
         },
         "pikepdf": {
             "hashes": [
-                "sha256:01be838a44430c4be84b748a33950fed09892472934a8041596c11189f365f7f",
-                "sha256:0cc95ef470169dfa5acc9196299bdba236716234a0d8b2746e2a563bc6f1f456",
-                "sha256:13e72d0aeeb3fc452569a3f7994acdd007de9aad804ced734d57cec269261b8b",
-                "sha256:2873503522ef26a09a6020c29c2efd221fa2ddc31e83bd902be27d317144cf63",
-                "sha256:2d5d6d3248b33ca5961d84bc3121a299cd27237fad56868d815e381c9a98d3d1",
-                "sha256:2f62e6c7bcf5d631e6ea74cf861f3e816f587c6ccb4ecbf6ac862e088ba2e4ac",
-                "sha256:51694d3d2f90510da6a8d7a4d07313ca868b373fffec6de270d9bbff1ce37180",
-                "sha256:5c23cbd7ae71f08fb5b5d9660eb0bc61abf345ada01bea6e1b6884c4261e17d6",
-                "sha256:6371bf02a436be2b7c63322b83a8e47523f2cd16438b2e93d546c7caf9ae308d",
-                "sha256:657293b74af8c7cf03f9905218a7935b26a4f3006803016b40b3db78e04cb35c",
-                "sha256:680d47377bb9fd6a36b6a81464ee269b4b29cbf29a84ae4f2ab8f6ea3665bf69",
-                "sha256:710535c679ab0d7b8249f72247832773e7a9a121dfbe9cad7f6465bd9bb45fae",
-                "sha256:7b4d7c09036d863915cb01007ca183d6fe64e2d57c0472453097bc9e029a58fb",
-                "sha256:978b6388ae99a024bdcae5a322c68e90c187cb568d09d43e6586b3479267121d",
-                "sha256:9917a03d500aab72715a9236136af7a5c8c7b26c034bf71ebdf028e177f0d25f",
-                "sha256:996faa6b119488f96d7271672a22af86e56e5544ec6b8eae6cd7d4432c70ae2d",
-                "sha256:9bac9e9d6b28dc0cc5a554051f183fbd070d0f9fe63c4e9aca939b8c44a5bb4d",
-                "sha256:aac14061de06843759ea6f5777fd8d7b71af808ed9264f57483a3311a09788ab",
-                "sha256:ad5361c3669fc0c8dbaf8fa0a590bddf59fad256bb2c527d5ce5cf991743a240",
-                "sha256:bc40b30c37f8f7c5bef873eca1f04e91ce34b6b74507d8d0019238a17d281fdc",
-                "sha256:bd9faae19787a5d05b9fcbe84d7cfe4d44e318068e06eca18906b9dba45425b6",
-                "sha256:c64e7905ec438b7a6c12626f2859df87f471892fab75b65b1441d9e1b38b4dde",
-                "sha256:d4db409b21a8ec0d3a79d2bbd894b997b13223c9ccf341cdc31b64360f1ee4c7",
-                "sha256:e0b635d6d9faefb4d0d32722279b8eb4e4d5d7b596c426f3433343de65e0c772",
-                "sha256:e62e9e8afe77fe2f06715faf10f38a4810d282d66f1e9e05208bb8d9723e6acf",
-                "sha256:f85d309bcfeeb3e2d344346a5050bfc41e332f19d390f79c20e4fc7de4b10a17",
-                "sha256:fe3fc2efe498aba6204b85c17c6a5d54ab7303354ecc5c3da624a6b6af0b3406"
+                "sha256:101ec256a8d312c17decae52226cf32a3e7dc834583134300c2f4e60b70e6e91",
+                "sha256:12b5b3cfc649e2542576a7e55c11e245278f14f727f116904893e54329102867",
+                "sha256:1b8f68a75c0a6f6d4d102d0821365ae2aaa9ab635c6eb6c840569a56b1a266f4",
+                "sha256:1bef3512be59fe0f481375b7eb415ca51ee7c80555031401f5f17ee3392e4add",
+                "sha256:1d3141916dc9efc433fd22beba544f67a53a805800c3ff902baffa398ef4c85e",
+                "sha256:3052df8514d26b676c50e65afc49a1d260c43a08c322c75cc2592c10a9a5b26d",
+                "sha256:356d5554516a295fc10db3f25cfde4e92326f6d015da55d71b84f5ced2a07a5a",
+                "sha256:55330c24b8e04ee09f1bc514c2b6107bb03a5eeb0b74929a61100cd6be22ae29",
+                "sha256:553cf11933fdfe07fdd357ab40b9732db102e921b27c1065239308d42b7b858f",
+                "sha256:5626312990a894c5db3a269455f7eb98df5f59188dde1797c0e352d60fdf89af",
+                "sha256:59c24a65c94693ab4a7e92f4809f847b57461120256c083054e61c99c4952e84",
+                "sha256:607deb1181a7cf5369cf70edfc41574d46c0a17c0cac1f6234272bd4cb3487e4",
+                "sha256:60bdd49e6251f8c99989e6769d4ad29b209c1eaf88090f49d4b30fba98442e40",
+                "sha256:73a7cc3c42609e00393b9d4e1b9ee132f528060254a174bf18ef31a154be0386",
+                "sha256:75f1e2917b4d2d6573fe3d1c3b2ec70829b64515b2f723f5c3bebcdd65761e6c",
+                "sha256:92ca9191680eccc21697e9e9c218e600ab31e7c24f6125749738c10ae2dc7c07",
+                "sha256:9bcaf96e2f571f0fc7e3178cdf1bcca7c13e5c68128e8246031226d47ecf23f1",
+                "sha256:a8f3e2229e2683497fe4ccc4af06050c125160a11bb3562b6c4ddaee4d0cc5c5",
+                "sha256:c277066938ca0ddb2bfe75874ef8dd3aa259936fe15c4cf7d4282f89ba82ab3a",
+                "sha256:c532542a99757d9f41df0cf1fc8f64a044d0eb822822cc069c80be35731df275",
+                "sha256:dfa89bd86e01413531c1d7d201fb01f0e62b52ea926a8e8ca6f99f86ed761e95",
+                "sha256:e064010b733b0a2ec4ec97982cd2887f9025292f2d228c6d5e6eca9d84851e53",
+                "sha256:ea927afe7cb04cc7ade30b961f528ef53e8d9cf467dcc4639cf944fef872a1a1",
+                "sha256:f40703b6267aa43d7f72468fa0a3b505ffff74ece2a4c69cfd3c90e023c41381",
+                "sha256:f45cc4544bbd4c308a525a6bb8e2e29b3f849803ee557c6e35c684447f0a92e5",
+                "sha256:f8ccda5ee992c73f647bcd96c9aa30f5eb9e8a6c5bdd6e3dcb29ebbffbe01a69",
+                "sha256:fe386d93345c9b5a9690f7a7bfb789a5ec5467c34402628e10bda8a4f5bac73e"
             ],
             "index": "pypi",
-            "version": "==5.1.1"
+            "version": "==5.1.2"
         },
         "pillow": {
             "hashes": [
@@ -1265,11 +1269,11 @@
         },
         "typing-extensions": {
             "hashes": [
-                "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42",
-                "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2"
+                "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708",
+                "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==4.1.1"
+            "markers": "python_version >= '3.7'",
+            "version": "==4.2.0"
         },
         "tzdata": {
             "hashes": [
@@ -1373,56 +1377,56 @@
         },
         "websockets": {
             "hashes": [
-                "sha256:038afef2a05893578d10dadbdbb5f112bd115c46347e1efe99f6a356ff062138",
-                "sha256:05f6e9757017270e7a92a2975e2ae88a9a582ffc4629086fd6039aa80e99cd86",
-                "sha256:0b66421f9f13d4df60cd48ab977ed2c2b6c9147ae1a33caf5a9f46294422fda1",
-                "sha256:0cd02f36d37e503aca88ab23cc0a1a0e92a263d37acf6331521eb38040dcf77b",
-                "sha256:0f73cb2526d6da268e86977b2c4b58f2195994e53070fe567d5487c6436047e6",
-                "sha256:117383d0a17a0dda349f7a8790763dde75c1508ff8e4d6e8328b898b7df48397",
-                "sha256:1c1f3b18c8162e3b09761d0c6a0305fd642934202541cc511ef972cb9463261e",
-                "sha256:1c9031e90ebfc486e9cdad532b94004ade3aa39a31d3c46c105bb0b579cd2490",
-                "sha256:2349fa81b6b959484bb2bda556ccb9eb70ba68987646a0f8a537a1a18319fb03",
-                "sha256:24b879ba7db12bb525d4e58089fcbe6a3df3ce4666523183654170e86d372cbe",
-                "sha256:2aa9b91347ecd0412683f28aabe27f6bad502d89bd363b76e0a3508b1596402e",
-                "sha256:56d48eebe9e39ce0d68701bce3b21df923aa05dcc00f9fd8300de1df31a7c07c",
-                "sha256:5a38a0175ae82e4a8c4bac29fc01b9ee26d7d5a614e5ee11e7813c68a7d938ce",
-                "sha256:5b04270b5613f245ec84bb2c6a482a9d009aefad37c0575f6cda8499125d5d5c",
-                "sha256:6193bbc1ee63aadeb9a4d81de0e19477401d150d506aee772d8380943f118186",
-                "sha256:669e54228a4d9457abafed27cbf0e2b9f401445c4dfefc12bf8e4db9751703b8",
-                "sha256:6a009eb551c46fd79737791c0c833fc0e5b56bcd1c3057498b262d660b92e9cd",
-                "sha256:71a4491cfe7a9f18ee57d41163cb6a8a3fa591e0f0564ca8b0ed86b2a30cced4",
-                "sha256:7b38a5c9112e3dbbe45540f7b60c5204f49b3cb501b40950d6ab34cd202ab1d0",
-                "sha256:7bb9d8a6beca478c7e9bdde0159bd810cc1006ad6a7cb460533bae39da692ca2",
-                "sha256:82bc33db6d8309dc27a3bee11f7da2288ad925fcbabc2a4bb78f7e9c56249baf",
-                "sha256:8351c3c86b08156337b0e4ece0e3c5ec3e01fcd14e8950996832a23c99416098",
-                "sha256:8beac786a388bb99a66c3be4ab0fb38273c0e3bc17f612a4e0a47c4fc8b9c045",
-                "sha256:97950c7c844ec6f8d292440953ae18b99e3a6a09885e09d20d5e7ecd9b914cf8",
-                "sha256:98f57b3120f8331cd7440dbe0e776474f5e3632fdaa474af1f6b754955a47d71",
-                "sha256:9ca2ca05a4c29179f06cf6727b45dba5d228da62623ec9df4184413d8aae6cb9",
-                "sha256:a03a25d95cc7400bd4d61a63460b5d85a7761c12075ee2f51de1ffe73aa593d3",
-                "sha256:a10c0c1ee02164246f90053273a42d72a3b2452a7e7486fdae781138cf7fbe2d",
-                "sha256:a72b92f96e5e540d5dda99ee3346e199ade8df63152fa3c737260da1730c411f",
-                "sha256:ac081aa0307f263d63c5ff0727935c736c8dad51ddf2dc9f5d0c4759842aefaa",
-                "sha256:b22bdc795e62e71118b63e14a08bacfa4f262fd2877de7e5b950f5ac16b0348f",
-                "sha256:b4059e2ccbe6587b6dc9a01db5fc49ead9a884faa4076eea96c5ec62cb32f42a",
-                "sha256:b7fe45ae43ac814beb8ca09d6995b56800676f2cfa8e23f42839dc69bba34a42",
-                "sha256:bef03a51f9657fb03d8da6ccd233fe96e04101a852f0ffd35f5b725b28221ff3",
-                "sha256:bffc65442dd35c473ca9790a3fa3ba06396102a950794f536783f4b8060af8dd",
-                "sha256:c21a67ab9a94bd53e10bba21912556027fea944648a09e6508415ad14e37c325",
-                "sha256:c67d9cacb3f6537ca21e9b224d4fd08481538e43bcac08b3d93181b0816def39",
-                "sha256:c6e56606842bb24e16e36ae7eb308d866b4249cf0be8f63b212f287eeb76b124",
-                "sha256:cb316b87cbe3c0791c2ad92a5a36bf6adc87c457654335810b25048c1daa6fd5",
-                "sha256:cef40a1b183dcf39d23b392e9dd1d9b07ab9c46aadf294fff1350fb79146e72b",
-                "sha256:cf931c33db9c87c53d009856045dd524e4a378445693382a920fa1e0eb77c36c",
-                "sha256:d4d110a84b63c5cfdd22485acc97b8b919aefeecd6300c0c9d551e055b9a88ea",
-                "sha256:d5396710f86a306cf52f87fd8ea594a0e894ba0cc5a36059eaca3a477dc332aa",
-                "sha256:f09f46b1ff6d09b01c7816c50bd1903cf7d02ebbdb63726132717c2fcda835d5",
-                "sha256:f14bd10e170abc01682a9f8b28b16e6f20acf6175945ef38db6ffe31b0c72c3f",
-                "sha256:f5c335dc0e7dc271ef36df3f439868b3c790775f345338c2f61a562f1074187b",
-                "sha256:f8296b8408ec6853b26771599990721a26403e62b9de7e50ac0a056772ac0b5e",
-                "sha256:fa35c5d1830d0fb7b810324e9eeab9aa92e8f273f11fdbdc0741dcded6d72b9f"
+                "sha256:07cdc0a5b2549bcfbadb585ad8471ebdc7bdf91e32e34ae3889001c1c106a6af",
+                "sha256:210aad7fdd381c52e58777560860c7e6110b6174488ef1d4b681c08b68bf7f8c",
+                "sha256:28dd20b938a57c3124028680dc1600c197294da5db4292c76a0b48efb3ed7f76",
+                "sha256:2f94fa3ae454a63ea3a19f73b95deeebc9f02ba2d5617ca16f0bbdae375cda47",
+                "sha256:31564a67c3e4005f27815634343df688b25705cccb22bc1db621c781ddc64c69",
+                "sha256:347974105bbd4ea068106ec65e8e8ebd86f28c19e529d115d89bd8cc5cda3079",
+                "sha256:379e03422178436af4f3abe0aa8f401aa77ae2487843738542a75faf44a31f0c",
+                "sha256:3eda1cb7e9da1b22588cefff09f0951771d6ee9fa8dbe66f5ae04cc5f26b2b55",
+                "sha256:51695d3b199cd03098ae5b42833006a0f43dc5418d3102972addc593a783bc02",
+                "sha256:54c000abeaff6d8771a4e2cef40900919908ea7b6b6a30eae72752607c6db559",
+                "sha256:5b936bf552e4f6357f5727579072ff1e1324717902127ffe60c92d29b67b7be3",
+                "sha256:6075fd24df23133c1b078e08a9b04a3bc40b31a8def4ee0b9f2c8865acce913e",
+                "sha256:661f641b44ed315556a2fa630239adfd77bd1b11cb0b9d96ed8ad90b0b1e4978",
+                "sha256:6ea6b300a6bdd782e49922d690e11c3669828fe36fc2471408c58b93b5535a98",
+                "sha256:6ed1d6f791eabfd9808afea1e068f5e59418e55721db8b7f3bfc39dc831c42ae",
+                "sha256:7934e055fd5cd9dee60f11d16c8d79c4567315824bacb1246d0208a47eca9755",
+                "sha256:7ab36e17af592eec5747c68ef2722a74c1a4a70f3772bc661079baf4ae30e40d",
+                "sha256:7f6d96fdb0975044fdd7953b35d003b03f9e2bcf85f2d2cf86285ece53e9f991",
+                "sha256:83e5ca0d5b743cde3d29fda74ccab37bdd0911f25bd4cdf09ff8b51b7b4f2fa1",
+                "sha256:85506b3328a9e083cc0a0fb3ba27e33c8db78341b3eb12eb72e8afd166c36680",
+                "sha256:8af75085b4bc0b5c40c4a3c0e113fa95e84c60f4ed6786cbb675aeb1ee128247",
+                "sha256:8b1359aba0ff810d5830d5ab8e2c4a02bebf98a60aa0124fb29aa78cfdb8031f",
+                "sha256:8fbd7d77f8aba46d43245e86dd91a8970eac4fb74c473f8e30e9c07581f852b2",
+                "sha256:907e8247480f287aa9bbc9391bd6de23c906d48af54c8c421df84655eef66af7",
+                "sha256:93d5ea0b5da8d66d868b32c614d2b52d14304444e39e13a59566d4acb8d6e2e4",
+                "sha256:97bc9d41e69a7521a358f9b8e44871f6cdeb42af31815c17aed36372d4eec667",
+                "sha256:994cdb1942a7a4c2e10098d9162948c9e7b235df755de91ca33f6e0481366fdb",
+                "sha256:a141de3d5a92188234afa61653ed0bbd2dde46ad47b15c3042ffb89548e77094",
+                "sha256:a1e15b230c3613e8ea82c9fc6941b2093e8eb939dd794c02754d33980ba81e36",
+                "sha256:aad5e300ab32036eb3fdc350ad30877210e2f51bceaca83fb7fef4d2b6c72b79",
+                "sha256:b529fdfa881b69fe563dbd98acce84f3e5a67df13de415e143ef053ff006d500",
+                "sha256:b9c77f0d1436ea4b4dc089ed8335fa141e6a251a92f75f675056dac4ab47a71e",
+                "sha256:bb621ec2dbbbe8df78a27dbd9dd7919f9b7d32a73fafcb4d9252fc4637343582",
+                "sha256:c7250848ce69559756ad0086a37b82c986cd33c2d344ab87fea596c5ac6d9442",
+                "sha256:c8d1d14aa0f600b5be363077b621b1b4d1eb3fbf90af83f9281cda668e6ff7fd",
+                "sha256:d1655a6fc7aecd333b079d00fb3c8132d18988e47f19740c69303bf02e9883c6",
+                "sha256:d6353ba89cfc657a3f5beabb3b69be226adbb5c6c7a66398e17809b0ce3c4731",
+                "sha256:da4377904a3379f0c1b75a965fff23b28315bcd516d27f99a803720dfebd94d4",
+                "sha256:e49ea4c1a9543d2bd8a747ff24411509c29e4bdcde05b5b0895e2120cb1a761d",
+                "sha256:e4e08305bfd76ba8edab08dcc6496f40674f44eb9d5e23153efa0a35750337e8",
+                "sha256:e6fa05a680e35d0fcc1470cb070b10e6fe247af54768f488ed93542e71339d6f",
+                "sha256:e7e6f2d6fd48422071cc8a6f8542016f350b79cc782752de531577d35e9bd677",
+                "sha256:e904c0381c014b914136c492c8fa711ca4cced4e9b3d110e5e7d436d0fc289e8",
+                "sha256:ec2b0ab7edc8cd4b0eb428b38ed89079bdc20c6bdb5f889d353011038caac2f9",
+                "sha256:ef5ce841e102278c1c2e98f043db99d6755b1c58bde475516aef3a008ed7f28e",
+                "sha256:f351c7d7d92f67c0609329ab2735eee0426a03022771b00102816a72715bb00b",
+                "sha256:fab7c640815812ed5f10fbee7abbf58788d602046b7bb3af9b1ac753a6d5e916",
+                "sha256:fc06cc8073c8e87072138ba1e431300e2d408f054b27047d047b549455066ff4"
             ],
-            "version": "==10.2"
+            "version": "==10.3"
         },
         "whitenoise": {
             "hashes": [
@@ -1446,6 +1450,7 @@
                 "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad",
                 "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"
             ],
+            "index": "pypi",
             "markers": "python_version < '3.9'",
             "version": "==3.8.0"
         },
@@ -1525,11 +1530,11 @@
         },
         "babel": {
             "hashes": [
-                "sha256:ab49e12b91d937cd11f0b67cb259a57ab4ad2b59ac7a3b41d6c06c0ac5b0def9",
-                "sha256:bc0c176f9f6a994582230df350aa6e05ba2ebe4b3ac317eab29d9be5d2768da0"
+                "sha256:3f349e85ad3154559ac4930c3918247d319f21910d5ce4b25d439ed8693b98d2",
+                "sha256:98aeaca086133efb3e1e2aad0396987490c8425929ddbcfe0550184fdc54cd13"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==2.9.1"
+            "markers": "python_version >= '3.6'",
+            "version": "==2.10.1"
         },
         "black": {
             "hashes": [
@@ -1585,16 +1590,14 @@
         },
         "click": {
             "hashes": [
-                "sha256:24e1a4a9ec5bf6299411369b208c1df2188d9eb8d916302fe6bf03faed227f1e",
-                "sha256:479707fe14d9ec9a0757618b7a100a0ae4c4e236fac5b7f80ca68028141a1a72"
+                "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e",
+                "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==8.1.2"
+            "version": "==8.1.3"
         },
         "coverage": {
-            "extras": [
-
-            ],
+            "extras": [],
             "hashes": [
                 "sha256:03e2a7826086b91ef345ff18742ee9fc47a6839ccd517061ef8fa1976e652ce9",
                 "sha256:07e6db90cd9686c767dcc593dff16c8c09f9814f5e9c51034066cad3373b914d",
@@ -1688,11 +1691,11 @@
         },
         "faker": {
             "hashes": [
-                "sha256:188961065fb5c78ea639f42176f55100f72c90c3a3179ac6c955c4bd712b0511",
-                "sha256:7758ece2593ce603db117db3d27393c31f4af03f783e176f3f0e14839a4f3426"
+                "sha256:0301ace8365d98f3d0bf6e9a40200c8548e845d3812402ae1daf589effe3fb01",
+                "sha256:b1903db92175d78051858128ada397c7dc76f376f6967975419da232b3ebd429"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==13.3.4"
+            "version": "==13.7.0"
         },
         "filelock": {
             "hashes": [
@@ -1704,18 +1707,18 @@
         },
         "identify": {
             "hashes": [
-                "sha256:3f3244a559290e7d3deb9e9adc7b33594c1bc85a9dd82e0f1be519bf12a1ec17",
-                "sha256:5f06b14366bd1facb88b00540a1de05b69b310cbc2654db3c7e07fa3a4339323"
+                "sha256:3acfe15a96e4272b4ec5662ee3e231ceba976ef63fd9980ed2ce9cc415df393f",
+                "sha256:c83af514ea50bf2be2c4a3f2fb349442b59dc87284558ae9ff54191bff3541d2"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.4.12"
+            "version": "==2.5.0"
         },
         "idna": {
             "hashes": [
                 "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff",
                 "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"
             ],
-            "markers": "python_version >= '3.5'",
+            "markers": "python_version >= '3'",
             "version": "==3.3"
         },
         "imagesize": {
@@ -1726,6 +1729,14 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.3.0"
         },
+        "importlib-metadata": {
+            "hashes": [
+                "sha256:1208431ca90a8cca1a6b8af391bb53c1a2db74e5d1cef6ddced95d4b2062edc6",
+                "sha256:ea4c597ebf37142f827b8f39299579e31685c31d3a438b59f469406afd0f2539"
+            ],
+            "markers": "python_version < '3.10'",
+            "version": "==4.11.3"
+        },
         "iniconfig": {
             "hashes": [
                 "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
@@ -1735,11 +1746,11 @@
         },
         "jinja2": {
             "hashes": [
-                "sha256:539835f51a74a69f41b848a9645dbdc35b4f20a3b601e2d9a7e22947b15ff119",
-                "sha256:640bed4bb501cbd17194b3cace1dc2126f5b619cf068a726b98192a0fde74ae9"
+                "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852",
+                "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.1.1"
+            "version": "==3.1.2"
         },
         "markupsafe": {
             "hashes": [
@@ -1818,11 +1829,11 @@
         },
         "platformdirs": {
             "hashes": [
-                "sha256:7535e70dfa32e84d4b34996ea99c5e432fa29a708d0f4e394bbcb2a8faa4f16d",
-                "sha256:bcae7cab893c2d310a711b70b24efb93334febe65f8de776ee320b517471e227"
+                "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788",
+                "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.5.1"
+            "version": "==2.5.2"
         },
         "pluggy": {
             "hashes": [
@@ -1858,11 +1869,11 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:44238f1b60a76d78fc8ca0528ee429702aae011c265fe6a8dd8b63049ae41c65",
-                "sha256:4e426f72023d88d03b2fa258de560726ce890ff3b630f88c21cbb8b2503b8c6a"
+                "sha256:5eb116118f9612ff1ee89ac96437bb6b49e8f04d8a13b514ba26f620208e26eb",
+                "sha256:dc9c10fb40944260f6ed4c688ece0cd2048414940f1cea51b8b226318411c519"
             ],
-            "markers": "python_version >= '3.5'",
-            "version": "==2.11.2"
+            "markers": "python_version >= '3.6'",
+            "version": "==2.12.0"
         },
         "pyparsing": {
             "hashes": [
@@ -1874,11 +1885,11 @@
         },
         "pytest": {
             "hashes": [
-                "sha256:841132caef6b1ad17a9afde46dc4f6cfa59a05f9555aae5151f73bdf2820ca63",
-                "sha256:92f723789a8fdd7180b6b06483874feca4c48a5c76968e03bb3e7f806a1869ea"
+                "sha256:13d0e3ccfc2b6e26be000cb6568c832ba67ba32e719443bfe725814d3c42433c",
+                "sha256:a06a0425453864a270bc45e71f783330a7428defb4230fb5e6a731fde06ecd45"
             ],
             "index": "pypi",
-            "version": "==7.1.1"
+            "version": "==7.1.2"
         },
         "pytest-cov": {
             "hashes": [
@@ -2090,11 +2101,19 @@
         },
         "tox": {
             "hashes": [
-                "sha256:67e0e32c90e278251fea45b696d0fef3879089ccbe979b0c556d35d5a70e2993",
-                "sha256:be3362472a33094bce26727f5f771ca0facf6dafa217f65875314e9a6600c95c"
+                "sha256:0805727eb4d6b049de304977dfc9ce315a1938e6619c3ab9f38682bb04662a5a",
+                "sha256:37888f3092aa4e9f835fc8cc6dadbaaa0782651c41ef359e3a5743fcb0308160"
             ],
             "index": "pypi",
-            "version": "==3.24.5"
+            "version": "==3.25.0"
+        },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708",
+                "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==4.2.0"
         },
         "urllib3": {
             "hashes": [
@@ -2111,6 +2130,15 @@
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==20.14.1"
+        },
+        "zipp": {
+            "hashes": [
+                "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad",
+                "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"
+            ],
+            "index": "pypi",
+            "markers": "python_version < '3.9'",
+            "version": "==3.8.0"
         }
     }
 }

build-docker-image.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+# Helper script for building the Docker image locally.
+# Parses and provides the nessecary versions of other images to Docker
+# before passing in the rest of script args.
+
+# First Argument: The Dockerfile to build
+# Other Arguments: Additional arguments to docker build
+
+# Example Usage:
+#	./build-docker-image.sh Dockerfile -t paperless-ngx:my-awesome-feature
+
+set -eux
+
+if ! command -v jq;  then
+	echo "jq required"
+	exit 1
+elif [ ! -f "$1" ]; then
+	echo "$1 is not a file, please provide the Dockerfile"
+	exit 1
+fi
+
+# Parse what we can from Pipfile.lock
+pikepdf_version=$(jq ".default.pikepdf.version" Pipfile.lock  | sed 's/=//g' | sed 's/"//g')
+psycopg2_version=$(jq ".default.psycopg2.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
+# Read this from the other config file
+qpdf_version=$(jq ".qpdf.version" .build-config.json | sed 's/"//g')
+jbig2enc_version=$(jq ".jbig2enc.version" .build-config.json | sed 's/"//g')
+# Get the branch name (used for caching)
+branch_name=$(git rev-parse --abbrev-ref HEAD)
+
+# https://docs.docker.com/develop/develop-images/build_enhancements/
+# Required to use cache-from
+export DOCKER_BUILDKIT=1
+
+docker build --file "$1" \
+	--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:"${branch_name}" \
+	--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:dev \
+	--build-arg JBIG2ENC_VERSION="${jbig2enc_version}" \
+	--build-arg QPDF_VERSION="${qpdf_version}" \
+	--build-arg PIKEPDF_VERSION="${pikepdf_version}" \
+	--build-arg PSYCOPG2_VERSION="${psycopg2_version}" "${@:2}" .

docker-builders/Dockerfile.frontend

@@ -0,0 +1,14 @@
+# This Dockerfile compiles the frontend
+# Inputs: None
+
+FROM node:16-bullseye-slim AS compile-frontend
+
+COPY ./src /src/src
+COPY ./src-ui /src/src-ui
+
+WORKDIR /src/src-ui
+RUN set -eux \
+  && npm update npm -g \
+  && npm ci --no-optional
+RUN set -eux \
+  && ./node_modules/.bin/ng build --configuration production

docker-builders/Dockerfile.jbig2enc

@@ -0,0 +1,39 @@
+# This Dockerfile compiles the jbig2enc library
+# Inputs:
+#    - JBIG2ENC_VERSION - the Git tag to checkout and build
+
+FROM debian:bullseye-slim as main
+
+LABEL org.opencontainers.image.description="A intermediate image with jbig2enc built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+  build-essential \
+  automake \
+  libtool \
+  libleptonica-dev \
+  zlib1g-dev \
+  git \
+  ca-certificates"
+
+WORKDIR /usr/src/jbig2enc
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+RUN apt-get update --quiet \
+  && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
+  && rm -rf /var/lib/apt/lists/*
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+ARG JBIG2ENC_VERSION
+
+RUN set -eux \
+  && git clone --quiet --branch $JBIG2ENC_VERSION https://github.com/agl/jbig2enc .
+RUN set -eux \
+  && ./autogen.sh
+RUN set -eux \
+  && ./configure && make

docker-builders/Dockerfile.pikepdf

@@ -0,0 +1,92 @@
+# This Dockerfile builds the pikepdf wheel
+# Inputs:
+#    - REPO - Docker repository to pull qpdf from
+#    - QPDF_VERSION - The image qpdf version to copy .deb files from
+#    - PIKEPDF_GIT_TAG - The Git tag to clone and build from
+#    - PIKEPDF_VERSION - Used to force the built pikepdf version to match
+
+# Default to pulling from the main repo registry when manually building
+ARG REPO="paperless-ngx/paperless-ngx"
+
+ARG QPDF_VERSION
+FROM ghcr.io/${REPO}/builder/qpdf:${QPDF_VERSION} as qpdf-builder
+
+# This does nothing, except provide a name for a copy below
+
+FROM python:3.9-slim-bullseye as main
+
+LABEL org.opencontainers.image.description="A intermediate image with pikepdf wheel built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+  build-essential \
+  python3-dev \
+  python3-pip \
+  git \
+  # qpdf requirement - https://github.com/qpdf/qpdf#crypto-providers
+  libgnutls28-dev \
+  # lxml requrements - https://lxml.de/installation.html
+  libxml2-dev \
+  libxslt1-dev \
+  # Pillow requirements - https://pillow.readthedocs.io/en/stable/installation.html#external-libraries
+  # JPEG functionality
+  libjpeg62-turbo-dev \
+  # conpressed PNG
+  zlib1g-dev \
+  # compressed TIFF
+  libtiff-dev \
+  # type related services
+  libfreetype-dev \
+  # color management
+  liblcms2-dev \
+  # WebP format
+  libwebp-dev \
+  # JPEG 2000
+  libopenjp2-7-dev \
+  # improved color quantization
+  libimagequant-dev \
+  # complex text layout support
+  libraqm-dev"
+
+WORKDIR /usr/src
+
+COPY --from=qpdf-builder /usr/src/qpdf/*.deb ./
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+
+RUN set -eux \
+  && apt-get update --quiet \
+  && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
+  && dpkg --install libqpdf28_*.deb \
+  && dpkg --install libqpdf-dev_*.deb \
+  && python3 -m pip install --no-cache-dir --upgrade \
+    pip \
+    wheel \
+    # https://pikepdf.readthedocs.io/en/latest/installation.html#requirements
+    pybind11 \
+  && rm -rf /var/lib/apt/lists/*
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+ARG PIKEPDF_GIT_TAG
+ARG PIKEPDF_VERSION
+
+RUN set -eux \
+  && echo "building pikepdf wheel" \
+  # Note the v in the tag name here
+  && git clone --quiet --depth 1 --branch "${PIKEPDF_GIT_TAG}" https://github.com/pikepdf/pikepdf.git \
+  && cd pikepdf \
+  # pikepdf seems to specifciy either a next version when built OR
+  # a post release tag.
+  # In either case, this won't match what we want from requirements.txt
+  # Directly modify the setup.py to set the version we just checked out of Git
+  && sed -i "s/use_scm_version=True/version=\"${PIKEPDF_VERSION}\"/g" setup.py \
+  # https://github.com/pikepdf/pikepdf/issues/323
+  && rm pyproject.toml \
+  && mkdir wheels \
+  && python3 -m pip wheel . --wheel-dir wheels \
+  && ls -ahl wheels

docker-builders/Dockerfile.psycopg2

@@ -0,0 +1,45 @@
+# This Dockerfile builds the psycopg2 wheel
+# Inputs:
+#    - PSYCOPG2_GIT_TAG - The Git tag to clone and build from
+#    - PSYCOPG2_VERSION - Unused, kept for future possible usage
+
+FROM python:3.9-slim-bullseye as main
+
+LABEL org.opencontainers.image.description="A intermediate image with psycopg2 wheel built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+  build-essential \
+  git \
+  python3-dev \
+  python3-pip \
+  # https://www.psycopg.org/docs/install.html#prerequisites
+  libpq-dev"
+
+WORKDIR /usr/src
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+
+RUN set -eux \
+  && apt-get update --quiet \
+  && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
+  && rm -rf /var/lib/apt/lists/* \
+  && python3 -m pip install --no-cache-dir --upgrade pip wheel
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+ARG PSYCOPG2_GIT_TAG
+ARG PSYCOPG2_VERSION
+
+RUN set -eux \
+  && echo "Building psycopg2 wheel" \
+  && cd /usr/src \
+  && git clone --quiet --depth 1 --branch ${PSYCOPG2_GIT_TAG} https://github.com/psycopg/psycopg2.git \
+  && cd psycopg2 \
+  && mkdir wheels \
+  && python3 -m pip wheel . --wheel-dir wheels \
+  && ls -ahl wheels/

docker-builders/Dockerfile.qpdf

@@ -0,0 +1,53 @@
+FROM debian:bullseye-slim as main
+
+LABEL org.opencontainers.image.description="A intermediate image with qpdf built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+  build-essential \
+  debhelper \
+  debian-keyring \
+  devscripts \
+  equivs  \
+  libtool \
+  # https://qpdf.readthedocs.io/en/stable/installation.html#system-requirements
+  libjpeg62-turbo-dev \
+  libgnutls28-dev \
+  packaging-dev \
+  zlib1g-dev"
+
+WORKDIR /usr/src
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+
+RUN set -eux \
+  && apt-get update --quiet \
+  && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
+  && rm -rf /var/lib/apt/lists/*
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+# This must match to pikepdf's minimum at least
+ARG QPDF_VERSION
+
+# In order to get the required version of qpdf, it is backported from bookwork
+# and then built from source
+RUN set -eux \
+  && echo "Building qpdf" \
+  && echo "deb-src http://deb.debian.org/debian/ bookworm main" > /etc/apt/sources.list.d/bookworm-src.list \
+  && apt-get update \
+  && mkdir qpdf \
+  && cd qpdf \
+  && apt-get source --yes --quiet qpdf=${QPDF_VERSION}-1/bookworm \
+  && rm -rf /var/lib/apt/lists/* \
+  && cd qpdf-$QPDF_VERSION \
+  # We don't need to build the tests (also don't run them)
+  && rm -rf libtests \
+  && DEBEMAIL=hello@paperless-ngx.com debchange --bpo \
+  && export DEB_BUILD_OPTIONS="terse nocheck nodoc parallel=2" \
+  && dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes \
+  && ls -ahl ../*.deb

docker/compose/docker-compose.portainer.yml

@@ -55,7 +55,7 @@ services:
     ports:
       - 8010:8000
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s
       timeout: 10s
       retries: 5

docker/compose/docker-compose.postgres-tika.yml

@@ -1,7 +1,6 @@
 # docker-compose file for running paperless from the docker container registry.
 # This file contains everything paperless needs to run.
-# Paperless supports amd64, arm and arm64 hardware. The apache/tika image
-# does not support arm or arm64, however.
+# Paperless supports amd64, arm and arm64 hardware.
 #
 # All compose files of paperless configure paperless in the following way:
 #
@@ -60,7 +59,7 @@ services:
     ports:
       - 8000:8000
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s
       timeout: 10s
       retries: 5
@@ -78,14 +77,14 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: gotenberg/gotenberg:7
+    image: gotenberg/gotenberg:7.4
     restart: unless-stopped
     command:
       - "gotenberg"
       - "--chromium-disable-routes=true"
 
   tika:
-    image: apache/tika
+    image: ghcr.io/paperless-ngx/tika:latest
     restart: unless-stopped
 
 volumes:

docker/compose/docker-compose.postgres.yml

@@ -53,7 +53,7 @@ services:
     ports:
       - 8000:8000
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s
       timeout: 10s
       retries: 5

docker/compose/docker-compose.sqlite-tika.arm.yml

@@ -1,84 +0,0 @@
-# docker-compose file for running paperless from the docker container registry.
-# This file contains everything paperless needs to run.
-# Paperless supports amd64, arm and arm64 hardware.
-#
-# All compose files of paperless configure paperless in the following way:
-#
-# - Paperless is (re)started on system boot, if it was running before shutdown.
-# - Docker volumes for storing data are managed by Docker.
-# - Folders for importing and exporting files are created in the same directory
-#   as this file and mounted to the correct folders inside the container.
-# - Paperless listens on port 8000.
-#
-# SQLite is used as the database. The SQLite file is stored in the data volume.
-#
-# iwishiwasaneagle/apache-tika-arm docker image is used to enable arm64 arch
-# which apache/tika does not currently support.
-#
-# In addition to that, this docker-compose file adds the following optional
-# configurations:
-#
-# - Apache Tika and Gotenberg servers are started with paperless and paperless
-#   is configured to use these services. These provide support for consuming
-#   Office documents (Word, Excel, Power Point and their LibreOffice counter-
-#   parts.
-#
-# To install and update paperless with this file, do the following:
-#
-# - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env'
-#   and '.env' into a folder.
-# - Run 'docker-compose pull'.
-# - Run 'docker-compose run --rm webserver createsuperuser' to create a user.
-# - Run 'docker-compose up -d'.
-#
-# For more extensive installation and update instructions, refer to the
-# documentation.
-
-version: "3.4"
-services:
-  broker:
-    image: redis:6.0
-    restart: unless-stopped
-    volumes:
-      - redisdata:/data
-
-  webserver:
-    image: ghcr.io/paperless-ngx/paperless-ngx:latest
-    restart: unless-stopped
-    depends_on:
-      - broker
-      - gotenberg
-      - tika
-    ports:
-      - 8000:8000
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000"]
-      interval: 30s
-      timeout: 10s
-      retries: 5
-    volumes:
-      - data:/usr/src/paperless/data
-      - media:/usr/src/paperless/media
-      - ./export:/usr/src/paperless/export
-      - ./consume:/usr/src/paperless/consume
-    env_file: docker-compose.env
-    environment:
-      PAPERLESS_REDIS: redis://broker:6379
-      PAPERLESS_TIKA_ENABLED: 1
-      PAPERLESS_TIKA_GOTENBERG_ENDPOINT: http://gotenberg:3000
-      PAPERLESS_TIKA_ENDPOINT: http://tika:9998
-
-  gotenberg:
-    image: thecodingmachine/gotenberg
-    restart: unless-stopped
-    environment:
-      DISABLE_GOOGLE_CHROME: 1
-
-  tika:
-    image: iwishiwasaneagle/apache-tika-arm@sha256:a78c25ffe57ecb1a194b2859d42a61af46e9e845191512b8f1a4bf90578ffdfd
-    restart: unless-stopped
-
-volumes:
-  data:
-  media:
-  redisdata:

docker/compose/docker-compose.sqlite-tika.yml

@@ -1,8 +1,6 @@
 # docker-compose file for running paperless from the docker container registry.
 # This file contains everything paperless needs to run.
-# Paperless supports amd64, arm and arm64 hardware. The apache/tika image
-# does not support arm or arm64, however.
-#
+# Paperless supports amd64, arm and arm64 hardware.
 # All compose files of paperless configure paperless in the following way:
 #
 # - Paperless is (re)started on system boot, if it was running before shutdown.
@@ -50,7 +48,7 @@ services:
     ports:
       - 8000:8000
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s
       timeout: 10s
       retries: 5
@@ -67,14 +65,14 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: gotenberg/gotenberg:7
+    image: gotenberg/gotenberg:7.4
     restart: unless-stopped
     command:
       - "gotenberg"
       - "--chromium-disable-routes=true"
 
   tika:
-    image: apache/tika
+    image: ghcr.io/paperless-ngx/tika:latest
     restart: unless-stopped
 
 volumes:

docker/compose/docker-compose.sqlite.yml

@@ -39,7 +39,7 @@ services:
     ports:
       - 8000:8000
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s
       timeout: 10s
       retries: 5

docker/docker-prepare.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -e
+
 wait_for_postgres() {
 	attempt_num=1
 	max_attempts=5
@@ -7,7 +9,7 @@ wait_for_postgres() {
 	echo "Waiting for PostgreSQL to start..."
 
 	host="${PAPERLESS_DBHOST:=localhost}"
-	port="${PAPERLESS_DBPORT:=5342}"
+	port="${PAPERLESS_DBPORT:=5432}"
 
 
 	while [ ! "$(pg_isready -h $host -p $port)" ]; do
@@ -25,6 +27,14 @@ wait_for_postgres() {
 	done
 }
 
+wait_for_redis() {
+	# We use a Python script to send the Redis ping
+	# instead of installing redis-tools just for 1 thing
+	if ! python3 /sbin/wait-for-redis.py; then
+		exit 1
+	fi
+}
+
 migrations() {
 	(
 		# flock is in place to prevent multiple containers from doing migrations
@@ -58,6 +68,8 @@ do_work() {
 		wait_for_postgres
 	fi
 
+	wait_for_redis
+
 	migrations
 
 	search_index

docker/install_management_commands.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -eu
+
 for command in document_archiver document_exporter document_importer mail_fetcher document_create_classifier document_index document_renamer document_retagger document_thumbnails document_sanity_checker manage_superuser;
 do
 	echo "installing $command..."

docker/supervisord.conf

@@ -28,6 +28,7 @@ stderr_logfile_maxbytes=0
 [program:scheduler]
 command=python3 manage.py qcluster
 user=paperless
+stopasgroup = true
 
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0

docker/wait-for-redis.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+"""
+Simple script which attempts to ping the Redis broker as set in the environment for
+a certain number of times, waiting a little bit in between
+
+"""
+import os
+import sys
+import time
+from typing import Final
+
+from redis import Redis
+
+if __name__ == "__main__":
+
+    MAX_RETRY_COUNT: Final[int] = 5
+    RETRY_SLEEP_SECONDS: Final[int] = 5
+
+    REDIS_URL: Final[str] = os.getenv("PAPERLESS_REDIS", "redis://localhost:6379")
+
+    print(f"Waiting for Redis: {REDIS_URL}", flush=True)
+
+    attempt = 0
+    with Redis.from_url(url=REDIS_URL) as client:
+        while attempt < MAX_RETRY_COUNT:
+            try:
+                client.ping()
+                break
+            except Exception:
+                print(
+                    f"Redis ping #{attempt} failed, waiting {RETRY_SLEEP_SECONDS}s",
+                    flush=True,
+                )
+                time.sleep(RETRY_SLEEP_SECONDS)
+                attempt += 1
+
+    if attempt >= MAX_RETRY_COUNT:
+        print(f"Failed to connect to: {REDIS_URL}")
+        sys.exit(os.EX_UNAVAILABLE)
+    else:
+        print(f"Connected to Redis broker: {REDIS_URL}")
+        sys.exit(os.EX_OK)

docs/Dockerfile

@@ -1,10 +1,10 @@
 FROM python:3.5.1
 
 # Install Sphinx and Pygments
-RUN pip install Sphinx Pygments
+RUN pip install --no-cache-dir Sphinx Pygments \
+  # Setup directories, copy data
+  && mkdir /build
 
-# Setup directories, copy data
-RUN mkdir /build
 COPY . /build
 WORKDIR /build/docs
 

docs/_static/css/custom.css

@@ -0,0 +1,592 @@
+/* Variables */
+:root {
+  --color-text-body: #5c5962;
+  --color-text-body-light: #fcfcfc;
+  --color-text-anchor: #7253ed;
+  --color-text-alt: rgba(0, 0, 0, 0.3);
+  --color-text-title: #27262b;
+  --color-text-code-inline: #e74c3c;
+  --color-text-code-nt: #062873;
+  --color-text-selection: #b19eff;
+  --color-bg-body: #fcfcfc;
+  --color-bg-body-alt: #f3f6f6;
+  --color-bg-side-nav: #f5f6fa;
+  --color-bg-side-nav-hover: #ebedf5;
+  --color-bg-code-block: var(--color-bg-side-nav);
+  --color-border: #eeebee;
+  --color-btn-neutral-bg: #f3f6f6;
+  --color-btn-neutral-bg-hover: #e5ebeb;
+  --color-success-title: #1abc9c;
+  --color-success-body: #dbfaf4;
+  --color-warning-title: #f0b37e;
+  --color-warning-body: #ffedcc;
+  --color-danger-title: #f29f97;
+  --color-danger-body: #fdf3f2;
+  --color-info-title: #6ab0de;
+  --color-info-body: #e7f2fa;
+}
+
+.dark-mode {
+  --color-text-body: #abb2bf;
+  --color-text-body-light: #9499a2;
+  --color-text-alt: rgba(0255, 255, 255, 0.5);
+  --color-text-title: var(--color-text-anchor);
+  --color-text-code-inline: #abb2bf;
+  --color-text-code-nt: #2063f3;
+  --color-text-selection: #030303;
+  --color-bg-body: #1d1d20 !important;
+  --color-bg-body-alt: #131315;
+  --color-bg-side-nav: #18181a;
+  --color-bg-side-nav-hover: #101216;
+  --color-bg-code-block: #101216;
+  --color-border: #47494f;
+  --color-btn-neutral-bg: #242529;
+  --color-btn-neutral-bg-hover: #101216;
+  --color-success-title: #02120f;
+  --color-success-body: #041b17;
+  --color-warning-title: #1b0e03;
+  --color-warning-body: #371d06;
+  --color-danger-title: #120902;
+  --color-danger-body: #1b0503;
+  --color-info-title: #020608;
+  --color-info-body: #06141e;
+}
+
+* {
+  transition: background-color 0.3s ease, border-color 0.3s ease;
+}
+
+/* Typography */
+body {
+  font-family: system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif;
+  font-size: inherit;
+  line-height: 1.4;
+  color: var(--color-text-body);
+}
+
+h1, h2, h3, h4, h5, h6 {
+  font-family: inherit;
+}
+
+.rst-content .toctree-wrapper>p.caption, .rst-content h1, .rst-content h2, .rst-content h3, .rst-content h4, .rst-content h5, .rst-content h6 {
+  padding-top: .5em;
+}
+
+p, .main-content-wrap, .rst-content .section ul, .rst-content .toctree-wrapper ul, .rst-content section ul, .wy-plain-list-disc, article ul {
+  line-height: 1.6;
+}
+
+pre, .code, .rst-content .linenodiv pre, .rst-content div[class^=highlight] pre, .rst-content pre.literal-block {
+  font-family: "SFMono-Regular", Menlo,Consolas, Monospace;
+  font-size: 0.75em;
+  line-height: 1.8;
+}
+
+.wy-menu-vertical li.toctree-l3,.wy-menu-vertical li.toctree-l4 {
+  font-size: 1rem
+}
+
+.rst-versions {
+  font-family: inherit;
+  line-height: 1;
+}
+
+footer, footer p {
+  font-size: .8rem;
+}
+
+footer .rst-footer-buttons {
+  font-size: 1rem;
+}
+
+@media (max-width: 400px) {
+  /* break code lines on mobile */
+  pre, code {
+    word-break: break-word;
+  }
+}
+
+
+/* Layout */
+.wy-side-nav-search, .wy-menu-vertical {
+  width: auto;
+}
+
+.wy-nav-side {
+  z-index: 0;
+  display: flex;
+  flex-wrap: wrap;
+  background-color: var(--color-bg-side-nav);
+}
+
+.wy-side-scroll {
+  width: 100%;
+  overflow-y: auto;
+}
+
+@media (min-width: 66.5rem) {
+  .wy-side-scroll {
+      width:264px
+  }
+}
+
+@media (min-width: 50rem) {
+  .wy-nav-side {
+      flex-wrap: nowrap;
+      position: fixed;
+      width: 248px;
+      height: 100%;
+      flex-direction: column;
+      border-right: 1px solid var(--color-border);
+      align-items:flex-end
+  }
+}
+
+@media (min-width: 66.5rem) {
+  .wy-nav-side {
+      width: calc((100% - 1064px) / 2 + 264px);
+      min-width:264px
+  }
+}
+
+@media (min-width: 50rem) {
+  .wy-nav-content-wrap {
+      position: relative;
+      max-width: 800px;
+      margin-left:248px
+  }
+}
+
+@media (min-width: 66.5rem) {
+  .wy-nav-content-wrap {
+      margin-left:calc((100% - 1064px) / 2 + 264px)
+  }
+}
+
+
+/* Colors */
+body.wy-body-for-nav,
+.wy-nav-content {
+  background: var(--color-bg-body);
+}
+
+.wy-nav-side {
+  border-right: 1px solid var(--color-border);
+}
+
+.wy-side-nav-search, .wy-nav-top {
+  background: var(--color-bg-side-nav);
+  border-bottom: 1px solid var(--color-border);
+}
+
+.wy-nav-content-wrap {
+  background: inherit;
+}
+
+.wy-side-nav-search > a, .wy-nav-top a, .wy-nav-top i {
+  color: var(--color-text-title);
+}
+
+.wy-side-nav-search > a:hover, .wy-nav-top a:hover {
+  background: transparent;
+}
+
+.wy-side-nav-search > div.version {
+  color: var(--color-text-alt)
+}
+
+.wy-side-nav-search > div[role="search"] {
+  border-top: 1px solid var(--color-border);
+}
+
+.wy-menu-vertical li.toctree-l2.current>a, .wy-menu-vertical li.toctree-l2.current li.toctree-l3>a,
+.wy-menu-vertical li.toctree-l3.current>a, .wy-menu-vertical li.toctree-l3.current li.toctree-l4>a {
+  background: var(--color-bg-side-nav);
+}
+
+.rst-content .highlighted {
+  background: #eedd85;
+  box-shadow: 0 0 0 2px #eedd85;
+  font-weight: 600;
+}
+
+.wy-side-nav-search input[type=text],
+html.writer-html5 .rst-content table.docutils th {
+  color: var(--color-text-body);
+}
+
+.rst-content table.docutils:not(.field-list) tr:nth-child(2n-1) td,
+.wy-table-backed,
+.wy-table-odd td,
+.wy-table-striped tr:nth-child(2n-1) td {
+  background-color: var(--color-bg-body-alt);
+}
+
+.rst-content table.docutils,
+.wy-table-bordered-all,
+html.writer-html5 .rst-content table.docutils th,
+.rst-content table.docutils td,
+.wy-table-bordered-all td,
+hr {
+  border-color: var(--color-border) !important;
+}
+
+::selection {
+  background: var(--color-text-selection);
+}
+
+/* Ridiculous rules are taken from sphinx_rtd */
+.rst-content .admonition-title,
+.wy-alert-title {
+  color: var(--color-text-body-light);
+}
+
+.rst-content .hint,
+.rst-content .important,
+.rst-content .tip,
+.rst-content .wy-alert-success,
+.wy-alert.wy-alert-success {
+  background: var(--color-success-body);
+}
+
+.rst-content .hint .admonition-title,
+.rst-content .hint .wy-alert-title,
+.rst-content .important .admonition-title,
+.rst-content .important .wy-alert-title,
+.rst-content .tip .admonition-title,
+.rst-content .tip .wy-alert-title,
+.rst-content .wy-alert-success .admonition-title,
+.rst-content .wy-alert-success .wy-alert-title,
+.wy-alert.wy-alert-success .rst-content .admonition-title,
+.wy-alert.wy-alert-success .wy-alert-title {
+  background-color: var(--color-success-title);
+}
+
+.rst-content .admonition-todo,
+.rst-content .attention,
+.rst-content .caution,
+.rst-content .warning,
+.rst-content .wy-alert-warning,
+.wy-alert.wy-alert-warning {
+  background: var(--color-warning-body);
+}
+
+.rst-content .admonition-todo .admonition-title,
+.rst-content .admonition-todo .wy-alert-title,
+.rst-content .attention .admonition-title,
+.rst-content .attention .wy-alert-title,
+.rst-content .caution .admonition-title,
+.rst-content .caution .wy-alert-title,
+.rst-content .warning .admonition-title,
+.rst-content .warning .wy-alert-title,
+.rst-content .wy-alert-warning .admonition-title,
+.rst-content .wy-alert-warning .wy-alert-title,
+.rst-content .wy-alert.wy-alert-warning .admonition-title,
+.wy-alert.wy-alert-warning .rst-content .admonition-title,
+.wy-alert.wy-alert-warning .wy-alert-title {
+  background: var(--color-warning-title);
+}
+
+.rst-content .danger,
+.rst-content .error,
+.rst-content .wy-alert-danger,
+.wy-alert.wy-alert-danger {
+  background: var(--color-danger-body);
+}
+
+.rst-content .danger .admonition-title,
+.rst-content .danger .wy-alert-title,
+.rst-content .error .admonition-title,
+.rst-content .error .wy-alert-title,
+.rst-content .wy-alert-danger .admonition-title,
+.rst-content .wy-alert-danger .wy-alert-title,
+.wy-alert.wy-alert-danger .rst-content .admonition-title,
+.wy-alert.wy-alert-danger .wy-alert-title {
+  background: var(--color-danger-title);
+}
+
+.rst-content .note,
+.rst-content .seealso,
+.rst-content .wy-alert-info,
+.wy-alert.wy-alert-info {
+  background: var(--color-info-body);
+}
+
+.rst-content .note .admonition-title,
+.rst-content .note .wy-alert-title,
+.rst-content .seealso .admonition-title,
+.rst-content .seealso .wy-alert-title,
+.rst-content .wy-alert-info .admonition-title,
+.rst-content .wy-alert-info .wy-alert-title,
+.wy-alert.wy-alert-info .rst-content .admonition-title,
+.wy-alert.wy-alert-info .wy-alert-title {
+  background: var(--color-info-title);
+}
+
+
+
+/* Links */
+a, a:visited,
+.wy-menu-vertical a,
+a.icon.icon-home,
+.wy-menu-vertical li.toctree-l1.current > a.current {
+  color: var(--color-text-anchor);
+  text-decoration: none;
+}
+
+a:hover, .wy-breadcrumbs-aside a {
+  color: var(--color-text-anchor); /* reset */
+}
+
+.rst-versions a, .rst-versions .rst-current-version {
+  color: #var(--color-text-anchor);
+}
+
+.wy-nav-content a.reference, .wy-nav-content a:not([class]) {
+  background-image: linear-gradient(var(--color-border) 0%, var(--color-border) 100%);
+  background-repeat: repeat-x;
+  background-position: 0 100%;
+  background-size: 1px 1px;
+}
+
+.wy-nav-content a.reference:hover, .wy-nav-content a:not([class]):hover {
+  background-image: linear-gradient(rgba(114,83,237,0.45) 0%, rgba(114,83,237,0.45) 100%);
+  background-size: 1px 1px;
+}
+
+.wy-menu-vertical a:hover,
+.wy-menu-vertical li.current a:hover,
+.wy-menu-vertical a:active {
+  background: var(--color-bg-side-nav-hover) !important;
+  color: var(--color-text-body);
+}
+
+.wy-menu-vertical li.toctree-l1.current>a,
+.wy-menu-vertical li.current>a,
+.wy-menu-vertical li.on a {
+  background-color: var(--color-bg-side-nav-hover);
+  border: none;
+  font-weight: normal;
+}
+
+.wy-menu-vertical li.current {
+  background-color: inherit;
+}
+
+.wy-menu-vertical li.current a {
+  border-right: none;
+}
+
+.wy-menu-vertical li.toctree-l2 a,
+.wy-menu-vertical li.toctree-l3 a,
+.wy-menu-vertical li.toctree-l4 a,
+.wy-menu-vertical li.toctree-l5 a,
+.wy-menu-vertical li.toctree-l6 a,
+.wy-menu-vertical li.toctree-l7 a,
+.wy-menu-vertical li.toctree-l8 a,
+.wy-menu-vertical li.toctree-l9 a,
+.wy-menu-vertical li.toctree-l10 a {
+  color: var(--color-text-body);
+}
+
+a.image-reference, a.image-reference:hover {
+  background: none !important;
+}
+
+a.image-reference img {
+  cursor: zoom-in;
+}
+
+
+/* Code blocks */
+.rst-content code, .rst-content tt, code {
+  padding: 0.25em;
+  font-weight: 400;
+  background-color: var(--color-bg-code-block);
+  border: 1px solid var(--color-border);
+  border-radius: 4px;
+}
+
+.rst-content div[class^=highlight], .rst-content pre.literal-block {
+  padding: 0.7rem;
+  margin-top: 0;
+  margin-bottom: 0.75rem;
+  overflow-x: auto;
+  background-color: var(--color-bg-side-nav);
+  border-color: var(--color-border);
+  border-radius: 4px;
+  box-shadow: none;
+}
+
+.rst-content .admonition-title,
+.rst-content div.admonition,
+.wy-alert-title {
+  padding: 10px 12px;
+  border-top-left-radius: 4px;
+  border-top-right-radius: 4px;
+}
+
+.highlight .go {
+  color: inherit;
+}
+
+.highlight .nt {
+  color: var(--color-text-code-nt);
+}
+
+.rst-content code.literal,
+.rst-content tt.literal {
+  border-color: var(--color-border);
+  background-color: var(--color-border);
+  color: var(--color-text-code-inline)
+}
+
+
+/* Search */
+.wy-side-nav-search input[type=text] {
+  border: none;
+  border-radius: 0;
+  background-color: transparent;
+  font-family: inherit;
+  font-size: .85rem;
+  box-shadow: none;
+  padding: .7rem 1rem .7rem 2.8rem;
+  margin: 0;
+}
+
+#rtd-search-form {
+  position: relative;
+}
+
+#rtd-search-form:before {
+  font: normal normal normal 14px/1 FontAwesome;
+  font-size: inherit;
+  text-rendering: auto;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+  content: "\f002";
+  color: var(--color-text-alt);
+  position: absolute;
+  left: 1.5rem;
+  top: .7rem;
+}
+
+/* Side nav */
+.wy-side-nav-search {
+  padding: 1rem 0 0 0;
+}
+
+.wy-menu-vertical li a button.toctree-expand {
+  float: right;
+  margin-right: -1.5em;
+  padding: 0 .5em;
+}
+
+.wy-menu-vertical a,
+.wy-menu-vertical li.current>a,
+.wy-menu-vertical li.current li>a {
+  padding-right: 1.5em !important;
+}
+
+.wy-menu-vertical li.current li>a.current {
+  font-weight: 600;
+}
+
+/* Misc spacing */
+.rst-content .admonition-title, .wy-alert-title {
+  padding: 10px 12px;
+}
+
+/* Buttons */
+.btn {
+  display: inline-block;
+  box-sizing: border-box;
+  padding: 0.3em 1em;
+  margin: 0;
+  font-family: inherit;
+  font-size: inherit;
+  font-weight: 500;
+  line-height: 1.5;
+  color: #var(--color-text-anchor);
+  text-decoration: none;
+  vertical-align: baseline;
+  background-color: #f7f7f7;
+  border-width: 0;
+  border-radius: 4px;
+  box-shadow: 0 1px 2px rgba(0,0,0,0.12),0 3px 10px rgba(0,0,0,0.08);
+  appearance: none;
+}
+
+.btn:active {
+  padding: 0.3em 1em;
+}
+
+.rst-content .btn:focus {
+  outline: 1px solid #ccc;
+}
+
+.rst-content .btn-neutral, .rst-content .btn span.fa {
+  color: var(--color-text-body) !important;
+}
+
+.btn-neutral {
+  background-color: var(--color-btn-neutral-bg) !important;
+  color: var(--color-btn-neutral-text) !important;
+  border: 1px solid var(--color-btn-neutral-bg);
+}
+
+.btn:hover, .btn-neutral:hover {
+  background-color: var(--color-btn-neutral-bg-hover) !important;
+}
+
+
+/* Icon overrides */
+.wy-side-nav-search a.icon-home:before {
+  display: none;
+}
+
+.fa-minus-square-o:before,.wy-menu-vertical li.current>a button.toctree-expand:before,.wy-menu-vertical li.on a button.toctree-expand:before {
+  content: "\f106"; /* fa-angle-up */
+}
+
+.fa-plus-square-o:before, .wy-menu-vertical li button.toctree-expand:before {
+  content: "\f107"; /* fa-angle-down */
+}
+
+
+/* Misc */
+.wy-nav-top {
+  line-height: 36px;
+}
+
+.wy-nav-top > i {
+  font-size: 24px;
+  padding: 8px 0 0 2px;
+  color:#var(--color-text-anchor);
+}
+
+.rst-content table.docutils td,
+.rst-content table.docutils th,
+.rst-content table.field-list td,
+.rst-content table.field-list th,
+.wy-table td,
+.wy-table th {
+  padding: 8px 14px;
+}
+
+.dark-mode-toggle {
+  position: absolute;
+  top: 14px;
+  right: 12px;
+  height: 20px;
+  width: 24px;
+  z-index: 10;
+  border: none;
+  background-color: transparent;
+  color: inherit;
+  opacity: 0.7;
+}
+
+.wy-nav-content-wrap {
+  z-index: 20;
+}

docs/_static/custom.css

@@ -1,14 +0,0 @@
-/* override table width restrictions */
-@media screen and (min-width: 767px) {
-
-  .wy-table-responsive table td {
-    /* !important prevents the common CSS stylesheets from
-       overriding this as on RTD they are loaded after this stylesheet */
-    white-space: normal !important;
-  }
-
-  .wy-table-responsive {
-    overflow: visible !important;
-  }
-
-}

docs/_static/js/darkmode.js

@@ -0,0 +1,47 @@
+let toggleButton;
+let icon;
+
+function load() {
+	"use strict";
+
+	toggleButton = document.createElement("button");
+	toggleButton.setAttribute("title", "Toggle dark mode");
+	toggleButton.classList.add("dark-mode-toggle");
+	icon = document.createElement("i");
+	icon.classList.add("fa", darkModeState ? "fa-sun-o" : "fa-moon-o");
+	toggleButton.appendChild(icon);
+	document.body.prepend(toggleButton);
+
+	// Listen for changes in the OS settings
+	// addListener is used because older versions of Safari don't support addEventListener
+	// prefersDarkQuery set in <head>
+	if (prefersDarkQuery) {
+		prefersDarkQuery.addListener(function (evt) {
+			toggleDarkMode(evt.matches);
+		});
+	}
+
+	// Initial setting depending on the prefers-color-mode or localstorage
+	// darkModeState should be set in the document <head> to prevent flash
+	if (darkModeState == undefined) darkModeState = false;
+	toggleDarkMode(darkModeState);
+
+	// Toggles the "dark-mode" class on click and sets localStorage state
+	toggleButton.addEventListener("click", () => {
+		darkModeState = !darkModeState;
+
+		toggleDarkMode(darkModeState);
+		localStorage.setItem("dark-mode", darkModeState);
+	});
+}
+
+function toggleDarkMode(state) {
+	document.documentElement.classList.toggle("dark-mode", state);
+	document.documentElement.classList.toggle("light-mode", !state);
+	icon.classList.remove("fa-sun-o");
+	icon.classList.remove("fa-moon-o");
+	icon.classList.add(state ? "fa-sun-o" : "fa-moon-o");
+	darkModeState = state;
+}
+
+document.addEventListener("DOMContentLoaded", load);

docs/_templates/layout.html

@@ -0,0 +1,13 @@
+{% extends "!layout.html" %}
+{% block extrahead %}
+    <script>
+        // MediaQueryList object
+        const prefersDarkQuery = window.matchMedia("(prefers-color-scheme: dark)");
+        const lsDark = localStorage.getItem("dark-mode");
+        let darkModeState = lsDark !== null ? lsDark == "true" : prefersDarkQuery.matches;
+
+        document.documentElement.classList.toggle("dark-mode", darkModeState);
+        document.documentElement.classList.toggle("light-mode", !darkModeState);
+    </script>
+    {{ super() }}
+{% endblock %}

docs/administration.rst

@@ -117,6 +117,23 @@ Then you can start paperless-ngx with ``-d`` to have it run in the background.
 
                 image: ghcr.io/paperless-ngx/paperless-ngx:latest
 
+    .. note::
+        In version 1.7.1 and onwards, the Docker image can now pinned to a release series.
+        This is often combined with automatic updaters such as Watchtower to allow safer
+        unattended upgrading to new bugfix releases only.  It is still recommended to always
+        review release notes before upgrading.  To ping your install to a release series, edit
+        the ``docker-compose.yml`` find the line that says
+
+            .. code::
+
+                image: ghcr.io/paperless-ngx/paperless-ngx:latest
+
+        and replace the version with the series you want to track, for example:
+
+            .. code::
+
+                image: ghcr.io/paperless-ngx/paperless-ngx:1.7
+
 Bare Metal Route
 ================
 
@@ -369,7 +386,7 @@ the naming scheme.
 
 .. warning::
 
-    Since this command moves you documents around alot, it is advised to to
+    Since this command moves you documents around a lot, it is advised to to
     a backup before. The renaming logic is robust and will never overwrite
     or delete a file, but you can't ever be careful enough.
 

docs/advanced_usage.rst

@@ -243,7 +243,7 @@ will create a directory structure as follows:
     last filename a document was stored as. If you do rename a file, paperless will
     report your files as missing and won't be able to find them.
 
-Paperless provides the following placeholders withing filenames:
+Paperless provides the following placeholders within filenames:
 
 * ``{asn}``: The archive serial number of the document, or "none".
 * ``{correspondent}``: The name of the correspondent, or "none".

docs/changelog.rst

@@ -8,7 +8,83 @@ Changelog
 paperless-ngx 1.7.0
 ###################
 
-Changelog pending
+Breaking Changes
+
+* ``PAPERLESS_URL`` is now required when using a reverse proxy. See `#674`_.
+
+Features
+
+* Allow setting more than one tag in mail rules `@jonasc`_ (#270)
+* global drag'n'drop `@shamoon`_ (#283).
+* Fix: download buttons should disable while waiting `@shamoon`_ (#630).
+* Update checker `@shamoon`_ (#591).
+* Show prompt on password-protected pdfs `@shamoon`_ (#564).
+* Filtering query params aka browser navigation for filtering `@shamoon`_ (#540).
+* Clickable tags in dashboard widgets `@shamoon`_ (#515).
+* Add bottom pagination `@shamoon`_ (#372).
+* Feature barcode splitter `@gador`_ (#532).
+* App loading screen `@shamoon`_ (#298).
+* Use progress bar for delayed buttons `@shamoon`_ (#415).
+* Add minimum length for documents text filter `@shamoon`_ (#401).
+* Added nav buttons in the document detail view `@GruberViktor`_ (#273).
+* Improve date keyboard input `@shamoon`_ (#253).
+* Color theming `@shamoon`_ (#243).
+* Parse dates when entered without separators `@GruberViktor`_ (#250).
+
+Bug Fixes
+
+* add "localhost" to ALLOWED_HOSTS `@gador`_ (#700).
+* Fix: scanners table `@qcasey`_ (#690).
+* Adds wait for file before consuming `@stumpylog`_ (#483).
+* Fix: frontend document editing erases time data `@shamoon`_ (#654).
+* Increase length of SavedViewFilterRule `@stumpylog`_ (#612).
+* Fixes attachment filename matching during mail fetching `@stumpylog`_ (#680).
+* Add ``PAPERLESS_URL`` env variable & CSRF var `@shamoon`_ (#674).
+* Fix: download buttons should disable while waiting `@shamoon`_ (#630).
+* Fixes downloaded filename, add more consumer ignore settings `@stumpylog`_ (#599).
+* FIX BUG: case-sensitive matching was not possible `@danielBreitlauch`_ (#594).
+* uses shutil.move instead of rename `@gador`_ (#617).
+* Fix npm deps 01.02.22 2 `@shamoon`_ (#610).
+* Fix npm dependencies 01.02.22 `@shamoon`_ (#600).
+* fix issue 416: implement PAPERLESS_OCR_MAX_IMAGE_PIXELS `@hacker-h`_ (#441).
+* fix: exclude cypress from build in Dockerfile `@FrankStrieter`_ (#526).
+* Corrections to pass pre-commit hooks `@schnuffle`_ (#454).
+* Fix 311 unable to click checkboxes in document list `@shamoon`_ (#313).
+* Fix imap tools bug `@stumpylog`_ (#393).
+* Fix filterable dropdown buttons arent translated `@shamoon`_ (#366).
+* Fix 224: "Auto-detected date is day before receipt date" `@a17t`_ (#246).
+* Fix minor sphinx errors `@shamoon`_ (#322).
+* Fix page links hidden `@shamoon`_ (#314).
+* Fix: Include excluded items in dropdown count `@shamoon`_ (#263).
+
+Translation
+
+* `@miku323`_ contributed to Slovenian translation.
+* `@FaintGhost`_ contributed to Chinese Simplified translation.
+* `@DarkoBG79`_ contributed to Serbian translation.
+* `Kemal Secer`_ contributed to Turkish translation.
+* `@Prominence`_ contributed to Belarusian translation.
+
+Documentation
+
+* Fix: scanners table `@qcasey`_ (#690).
+* Add `PAPERLESS_URL` env variable & CSRF var `@shamoon`_ (#674).
+* Fixes downloaded filename, add more consumer ignore settings `@stumpylog`_ (#599).
+* fix issue 416: implement ``PAPERLESS_OCR_MAX_IMAGE_PIXELS`` `@hacker-h`_ (#441).
+* Fix minor sphinx errors `@shamoon`_ (#322).
+
+Maintenance
+
+* Add ``PAPERLESS_URL`` env variable & CSRF var `@shamoon`_ (#674).
+* Chore: Implement release-drafter action for Changelogs `@qcasey`_ (#669).
+* Chore: Add CODEOWNERS `@qcasey`_ (#667).
+* Support docker-compose v2 in install `@stumpylog`_ (#611).
+* Add Belarusian localization `@shamoon`_ (#588).
+* Add Turkish localization `@shamoon`_ (#536).
+* Add Serbian localization `@shamoon`_ (#504).
+* Create PULL_REQUEST_TEMPLATE.md `@shamoon`_ (#304).
+* Add Chinese localization `@shamoon`_ (#247).
+* Add Slovenian language for frontend `@shamoon`_ (#315).
 
 paperless-ngx 1.6.0
 ###################
@@ -40,6 +116,10 @@ Version 1.6.0 merges several pending PRs from jonaswinkler's repo and includes n
 * `@shamoon`_ created a slick new logo (#165).
 * `@tim-vogel`_ fixed exports missing groups (#193).
 
+Known issues:
+
+* 1.6.0 included a malformed package-lock.json, as a result users who want to build the docker image themselves need to change line 6 of the ``Dockerfile`` to ``RUN npm update npm -g && npm install --legacy-peer-deps``.
+
 Thank you to the following people for their documentation updates, fixes, and comprehensive testing:
 
 `@m0veax`_, `@a17t`_, `@fignew`_, `@muued`_, `@bauerj`_, `@isigmund`_, `@denilsonsa`_, `@mweimerskirch`_, `@alexander-bauer`_, `@apeltzer`_, `@tribut`_, `@yschroeder`_, `@gador`_, `@sAksham-Ar`_, `@sbrunner`_, `@philpagel`_, `@davemachado`_, `@2600box`_, `@qcasey`_, `@Nicarim`_, `@kpj`_, `@filcuk`_, `@Timoms`_, `@mattlamb99`_, `@padraigkitterick`_, `@ajkavanagh`_, `@Tooa`_, `@Unkn0wnCat`_, `@pewter77`_, `@stumpylog`_, `@Toxix`_, `@azapater`_, `@jschpp`_
@@ -145,7 +225,7 @@ paperless-ng 1.4.0
 
   * New URL pattern for accessing documents by ASN directly (http://<paperless>/asn/123)
 
-  * Added logging when executing pre- and post-consume scripts.
+  * Added logging when executing pre* and post-consume scripts.
 
   * Better error logging during document consumption.
 
@@ -1581,6 +1661,16 @@ bulk of the work on this big change.
 .. _@azapater: https://github.com/azapater
 .. _@tim-vogel: https://github.com/tim-vogel
 .. _@jschpp: https://github.com/jschpp
+.. _@schnuffle: https://github.com/schnuffle
+.. _@GruberViktor: https://github.com/gruberviktor
+.. _@hacker-h: https://github.com/hacker-h
+.. _@danielBreitlauch: https://github.com/danielbreitlauch
+.. _@miku323: https://github.com/miku323
+.. _@FaintGhost: https://github.com/FaintGhost
+.. _@DarkoBG79: https://github.com/DarkoBG79
+.. _Kemal Secer: https://crowdin.com/profile/kemal.secer
+.. _@Prominence: https://github.com/Prominence
+.. _@jonasc: https://github.com/jonasc
 
 .. _#20: https://github.com/the-paperless-project/paperless/issues/20
 .. _#44: https://github.com/the-paperless-project/paperless/issues/44
@@ -1689,6 +1779,7 @@ bulk of the work on this big change.
 .. _#488: https://github.com/the-paperless-project/paperless/pull/488
 .. _#489: https://github.com/the-paperless-project/paperless/pull/489
 .. _#492: https://github.com/the-paperless-project/paperless/pull/492
+.. _#674: https://github.com/paperless-ngx/paperless-ngx/pull/674
 
 .. _a new home on Docker Hub: https://hub.docker.com/r/danielquinn/paperless/
 .. _optipng: http://optipng.sourceforge.net/

docs/conf.py

@@ -2,6 +2,8 @@ import sphinx_rtd_theme
 
 
 __version__ = None
+__full_version_str__ = None
+__major_minor_version_str__ = None
 exec(open("../src/paperless/version.py").read())
 
 
@@ -15,7 +17,7 @@ extensions = [
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-# templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix of source filenames.
 source_suffix = ".rst"
@@ -41,9 +43,9 @@ copyright = "2015-2022, Daniel Quinn, Jonas Winkler, and the paperless-ngx team"
 #
 
 # The short X.Y version.
-version = ".".join([str(_) for _ in __version__[:2]])
+version = __major_minor_version_str__
 # The full version, including alpha/beta/rc tags.
-release = ".".join([str(_) for _ in __version__[:3]])
+release = __full_version_str__
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -119,6 +121,16 @@ html_theme_path = []
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ["_static"]
 
+# These paths are either relative to html_static_path
+# or fully qualified paths (eg. https://...)
+html_css_files = [
+    "css/custom.css",
+]
+
+html_js_files = [
+    "js/darkmode.js",
+]
+
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the documentation.

docs/configuration.rst

@@ -130,6 +130,8 @@ PAPERLESS_LOGROTATE_MAX_BACKUPS=<num>
 
     Defaults to 20.
 
+.. _hosting-and-security:
+
 Hosting & Security
 ##################
 
@@ -170,6 +172,9 @@ PAPERLESS_ALLOWED_HOSTS=<comma-separated-list>
 
     Can also be set using PAPERLESS_URL (see above).
 
+    If manually set, please remember to include "localhost". Otherwise docker
+    healthcheck will fail.
+
     Defaults to "*", which is all hosts.
 
 PAPERLESS_CORS_ALLOWED_HOSTS=<comma-separated-list>
@@ -206,7 +211,7 @@ PAPERLESS_AUTO_LOGIN_USERNAME=<username>
 PAPERLESS_ADMIN_USER=<username>
     If this environment variable is specified, Paperless automatically creates
     a superuser with the provided username at start. This is useful in cases
-    where you can not run the `createsuperuser` command seperately, such as Kubernetes
+    where you can not run the `createsuperuser` command separately, such as Kubernetes
     or AWS ECS.
 
     Requires `PAPERLESS_ADMIN_PASSWORD` to be set.
@@ -469,7 +474,7 @@ PAPERLESS_TIKA_GOTENBERG_ENDPOINT=<url>
     Defaults to "http://localhost:3000".
 
 If you run paperless on docker, you can add those services to the docker-compose
-file (see the provided ``docker-compose.tika.yml`` file for reference). The changes
+file (see the provided ``docker-compose.sqlite-tika.yml`` file for reference). The changes
 requires are as follows:
 
 .. code:: yaml
@@ -490,14 +495,14 @@ requires are as follows:
         # ...
 
         gotenberg:
-            image: gotenberg/gotenberg:7
+            image: gotenberg/gotenberg:7.4
             restart: unless-stopped
             command:
                 - "gotenberg"
                 - "--chromium-disable-routes=true"
 
         tika:
-            image: apache/tika
+            image: ghcr.io/paperless-ngx/tika:latest
             restart: unless-stopped
 
 Add the configuration variables to the environment of the webserver (alternatively
@@ -624,8 +629,19 @@ PAPERLESS_CONSUMER_ENABLE_BARCODES=<bool>
     If no barcodes are detected in the uploaded file, no page separation
     will happen.
 
+    The original document will be removed and the separated pages will be
+    saved as pdf.
+
     Defaults to false.
 
+PAPERLESS_CONSUMER_BARCODE_TIFF_SUPPORT=<bool>
+    Whether TIFF image files should be scanned for barcodes.
+    This will automatically convert any TIFF image(s) to pdfs for later
+    processing.
+    This only has an effect, if PAPERLESS_CONSUMER_ENABLE_BARCODES has been
+    enabled.
+
+    Defaults to false.
 
 PAPERLESS_CONSUMER_BARCODE_STRING=PATCHT
   Defines the string to be detected as a separator barcode.

docs/extending.rst

@@ -334,11 +334,17 @@ directory.
 Building the Docker image
 =========================
 
+The docker image is primarily built by the GitHub actions workflow, but it can be
+faster when developing to build and tag an image locally.
+
+To provide the build arguments automatically, build the image using the helper
+script ``build-docker-image.sh``.
+
 Building the docker image from source:
 
     .. code:: shell-session
 
-        docker build . -t <your-tag>
+        ./build-docker-image.sh Dockerfile -t <your-tag>
 
 Extending Paperless
 ===================

docs/faq.rst

@@ -7,7 +7,7 @@ Frequently asked questions
 
 **A:** While Paperless-ngx is already considered largely "feature-complete" it is a community-driven
 project and development will be guided in this way. New features can be submitted via
-GitHub discussions and "up-voted" by the community but this is not a garauntee the feature
+GitHub discussions and "up-voted" by the community but this is not a guarantee the feature
 will be implemented. This project will always be open to collaboration in the form of PRs,
 ideas etc.
 

docs/screenshots.rst

@@ -4,41 +4,60 @@
 Screenshots
 ***********
 
-This is what paperless-ngx looks like. You shouldn't use paperless to index
-research papers though, its a horrible tool for that job.
+This is what Paperless-ngx looks like.
 
 The dashboard shows customizable views on your document and allows document uploads:
 
 .. image:: _static/screenshots/dashboard.png
+    :target: _static/screenshots/dashboard.png
 
 The document list provides three different styles to scroll through your documents:
 
 .. image:: _static/screenshots/documents-table.png
+    :target: _static/screenshots/documents-table.png
 .. image:: _static/screenshots/documents-smallcards.png
+    :target: _static/screenshots/documents-smallcards.png
 .. image:: _static/screenshots/documents-largecards.png
+    :target: _static/screenshots/documents-largecards.png
+
+Paperless-ngx also supports "dark mode":
+
+.. image:: _static/screenshots/documents-smallcards-dark.png
+    :target: _static/screenshots/documents-smallcards-dark.png
 
 Extensive filtering mechanisms:
 
 .. image:: _static/screenshots/documents-filter.png
+    :target: _static/screenshots/documents-filter.png
 
-Side-by-side editing of documents. Optimized for 1080p.
+Bulk editing of document tags, correspondents, etc.:
+
+.. image:: _static/screenshots/bulk-edit.png
+    :target: _static/screenshots/bulk-edit.png
+
+Side-by-side editing of documents:
 
 .. image:: _static/screenshots/editing.png
+    :target: _static/screenshots/editing.png
 
 Tag editing. This looks about the same for correspondents and document types.
 
 .. image:: _static/screenshots/new-tag.png
+    :target: _static/screenshots/new-tag.png
 
 Searching provides auto complete and highlights the results.
 
 .. image:: _static/screenshots/search-preview.png
+    :target: _static/screenshots/search-preview.png
 .. image:: _static/screenshots/search-results.png
+    :target: _static/screenshots/search-results.png
 
 Fancy mail filters!
 
 .. image:: _static/screenshots/mail-rules-edited.png
+    :target: _static/screenshots/mail-rules-edited.png
 
-Mobile support in the future? This kinda works, however some layouts are still
-too wide.
+Mobile devices are supported.
 
 .. image:: _static/screenshots/mobile.png
+    :target: _static/screenshots/mobile.png

docs/setup.rst

@@ -291,12 +291,14 @@ writing. Windows is not and will never be supported.
     *   ``libpq-dev`` for PostgreSQL
     *   ``libmagic-dev`` for mime type detection
     *   ``mime-support`` for mime type detection
+    *   ``libzbar0`` for barcode detection
+    *   ``poppler-utils`` for barcode detection
 
     Use this list for your preferred package management:
 
     .. code::
 
-        python3 python3-pip python3-dev imagemagick fonts-liberation optipng gnupg libpq-dev libmagic-dev mime-support
+        python3 python3-pip python3-dev imagemagick fonts-liberation optipng gnupg libpq-dev libmagic-dev mime-support libzbar0 poppler-utils
 
     These dependencies are required for OCRmyPDF, which is used for text recognition.
 
@@ -345,6 +347,8 @@ writing. Windows is not and will never be supported.
         paperless stores its data. If you like, you can point both to the same directory.
     *   ``PAPERLESS_SECRET_KEY`` should be a random sequence of characters. It's used for authentication. Failure
         to do so allows third parties to forge authentication credentials.
+    *   ``PAPERLESS_URL`` if you are behind a reverse proxy. This should point to your domain. Please see
+        :ref:`configuration` for more information.
 
     Many more adjustments can be made to paperless, especially the OCR part. The following options are recommended
     for everyone:
@@ -724,8 +728,6 @@ configuring some options in paperless can help improve performance immensely:
     times. Thumbnails will be about 20% larger.
 *   If using docker, consider setting ``PAPERLESS_WEBSERVER_WORKERS`` to
     1. This will save some memory.
-*   Use the arm compatible docker-compose if you're wanting to use Tika on something like
-		a raspberry pi. The official apache/tika image does not support the arm architecture.
 
 For details, refer to :ref:`configuration`.
 
@@ -784,4 +786,6 @@ the following configuration is required for paperless to operate:
         }
     }
 
+The ``PAPERLESS_URL`` configuration variable is also required when using a reverse proxy. Please refer to the :ref:`hosting-and-security` docs.
+
 Also read `this <https://channels.readthedocs.io/en/stable/deploying.html#nginx-supervisor-ubuntu>`__, towards the end of the section.

docs/troubleshooting.rst

@@ -125,7 +125,7 @@ If using docker-compose, this is achieved by the following configuration change
 .. code:: yaml
 
     gotenberg:
-        image: gotenberg/gotenberg:7
+        image: gotenberg/gotenberg:7.4
         restart: unless-stopped
         command:
             - "gotenberg"

docs/usage_overview.rst

@@ -62,7 +62,7 @@ your documents:
 
 1.  OCR the document, if it has no text. Digital documents usually have text,
     and this step will be skipped for those documents.
-2.  Paperless will create an archiveable PDF/A document from your document.
+2.  Paperless will create an archivable PDF/A document from your document.
     If this document is coming from your scanner, it will have embedded selectable text.
 3.  Paperless performs automatic matching of tags, correspondents and types on the
     document before storing it in the database.
@@ -102,12 +102,14 @@ files from the scanner.  Typically, you're looking at an FTP server like
 
 .. TODO: hyperref to configuration of the location of this magic folder.
 
-Dashboard upload
-================
+Web UI Upload
+=============
 
 The dashboard has a file drop field to upload documents to paperless. Simply drag a file
 onto this field or select a file with the file dialog. Multiple files are supported.
 
+You can also upload documents on any other page of the web UI by dragging-and-dropping
+files into your browser window.
 
 .. _usage-mobile_upload:
 
@@ -182,9 +184,10 @@ These are as follows:
 
     When defining a mail rule with a folder, you may need to try different characters to
     define how the sub-folders are separated.  Common values include ".", "/" or "|", but
-    this varies by the mail server.  Unfortunately, this isn't a value we can determine
-    automatically.  Either check the documentation for your mail server, or check for
-    errors in the logs and try different folder separator values.
+    this varies by the mail server.  Check the documentation for your mail server.  In the
+    event of an error fetching mail from a certain folder, check the Paperless logs.  When
+    a folder is not located, Paperless will attempt to list all folders found in the account
+    to the Paperless logs.
 
 .. note::
 

gunicorn.conf.py

@@ -24,7 +24,7 @@ def worker_int(worker):
     ## get traceback info
     import threading, sys, traceback
 
-    id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
+    id2name = {th.ident: th.name for th in threading.enumerate()}
     code = []
     for threadId, stack in sys._current_frames().items():
         code.append("\n# Thread: %s(%d)" % (id2name.get(threadId, ""), threadId))

install-paperless-ngx.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 ask() {
 	while true ; do
@@ -319,7 +319,10 @@ wget "https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/main/docker/
 
 SECRET_KEY=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 64 | head -n 1)
 
-DEFAULT_LANGUAGES="deu eng fra ita spa"
+DEFAULT_LANGUAGES=("deu eng fra ita spa")
+
+_split_langs="${OCR_LANGUAGE//+/ }"
+read -r -a OCR_LANGUAGES_ARRAY <<< "${_split_langs}"
 
 {
 	if [[ ! $URL == "" ]] ; then
@@ -334,8 +337,8 @@ DEFAULT_LANGUAGES="deu eng fra ita spa"
 	echo "PAPERLESS_TIME_ZONE=$TIME_ZONE"
 	echo "PAPERLESS_OCR_LANGUAGE=$OCR_LANGUAGE"
 	echo "PAPERLESS_SECRET_KEY=$SECRET_KEY"
-	if [[ ! " ${DEFAULT_LANGUAGES[*]} " =~ ${OCR_LANGUAGE} ]] ; then
-		echo "PAPERLESS_OCR_LANGUAGES=$OCR_LANGUAGE"
+	if [[ ! ${DEFAULT_LANGUAGES[*]} =~ ${OCR_LANGUAGES_ARRAY[*]} ]] ; then
+		echo "PAPERLESS_OCR_LANGUAGES=${OCR_LANGUAGES_ARRAY[*]}"
 	fi
 } > docker-compose.env
 

requirements.txt

@@ -5,12 +5,12 @@
 #    pipenv lock --requirements
 #
 
--i https://pypi.python.org/simple/
---extra-index-url https://www.piwheels.org/simple/
+-i https://pypi.python.org/simple
+--extra-index-url https://www.piwheels.org/simple
 aioredis==1.3.1
 anyio==3.5.0; python_full_version >= '3.6.2'
 arrow==1.2.2; python_version >= '3.6'
-asgiref==3.5.0; python_version >= '3.7'
+asgiref==3.5.1; python_version >= '3.7'
 async-timeout==4.0.2; python_version >= '3.6'
 attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 autobahn==22.3.2; python_version >= '3.7'
@@ -23,11 +23,11 @@ channels-redis==3.4.0
 channels==3.0.4
 chardet==4.0.0; python_version >= '3.1'
 charset-normalizer==2.0.12; python_version >= '3'
-click==8.1.2; python_version >= '3.7'
+click==8.1.3; python_version >= '3.7'
 coloredlogs==15.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 concurrent-log-handler==0.9.20
 constantly==15.1.0
-cryptography==36.0.2; python_version >= '3.6'
+cryptography==37.0.1; python_version >= '3.6'
 daphne==3.0.2; python_version >= '3.6'
 dateparser==1.1.1
 django-cors-headers==3.11.0
@@ -45,10 +45,10 @@ hiredis==2.0.0; python_version >= '3.6'
 httptools==0.4.0
 humanfriendly==10.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 hyperlink==21.0.0
-idna==3.3; python_version >= '3.5'
-imap-tools==0.53.0
+idna==3.3; python_version >= '3'
+imap-tools==0.54.0
 img2pdf==0.4.4
-importlib-resources==5.6.0; python_version < '3.9'
+importlib-resources==5.7.1; python_version < '3.9'
 incremental==21.3.0
 inotify-simple==1.3.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 inotifyrecursive==0.3.5
@@ -57,12 +57,12 @@ langdetect==1.0.9
 lxml==4.8.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 msgpack==1.0.3
 numpy==1.22.3; python_version >= '3.8'
-ocrmypdf==13.4.2
+ocrmypdf==13.4.3
 packaging==21.3; python_version >= '3.6'
 pathvalidate==2.5.0
 pdf2image==1.16.0
 pdfminer.six==20220319
-pikepdf==5.1.1
+pikepdf==5.1.2
 pillow==9.1.0
 pluggy==1.0.0; python_version >= '3.6'
 portalocker==2.4.0; python_version >= '3'
@@ -97,7 +97,7 @@ tika==1.24
 tqdm==4.64.0
 twisted[tls]==22.4.0; python_full_version >= '3.6.7'
 txaio==22.2.1; python_version >= '3.6'
-typing-extensions==4.1.1; python_version >= '3.6'
+typing-extensions==4.2.0; python_version >= '3.7'
 tzdata==2022.1; python_version >= '3.6'
 tzlocal==4.2; python_version >= '3.6'
 urllib3==1.26.9; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
@@ -106,7 +106,7 @@ uvloop==0.16.0
 watchdog==2.1.7
 watchgod==0.8.2
 wcwidth==0.2.5
-websockets==10.2
+websockets==10.3
 whitenoise==6.0.0
 whoosh==2.7.4
 zipp==3.8.0; python_version < '3.9'

scripts/start_services.sh

@@ -2,5 +2,5 @@
 
 docker run -p 5432:5432 -e POSTGRES_PASSWORD=password -v paperless_pgdata:/var/lib/postgresql/data -d postgres:13
 docker run -d -p 6379:6379 redis:latest
-docker run -p 3000:3000 -d gotenberg/gotenberg:7
-docker run -p 9998:9998 -d apache/tika
+docker run -p 3000:3000 -d gotenberg/gotenberg:7.4
+docker run -p 9998:9998 -d ghcr.io/paperless-ngx/tika:latest

src-ui/cypress/integration/manage.spec.ts

@@ -26,7 +26,7 @@ describe('manage', () => {
         req.reply({ count: 3, next: null, previous: null, results: [] })
     })
     cy.visit('/tags')
-    cy.get('tbody').find('button').contains('Documents').first().click() // id = 4
+    cy.get('tbody').find('button:visible').contains('Documents').first().click() // id = 4
     cy.contains('3 documents')
   })
 })

src-ui/package.json

@@ -13,15 +13,15 @@
   },
   "private": true,
   "dependencies": {
-    "@angular/common": "~13.3.1",
-    "@angular/compiler": "~13.3.1",
-    "@angular/core": "~13.3.1",
-    "@angular/forms": "~13.3.1",
-    "@angular/localize": "~13.3.1",
-    "@angular/platform-browser": "~13.3.1",
-    "@angular/platform-browser-dynamic": "~13.3.1",
-    "@angular/router": "~13.3.1",
-    "@ng-bootstrap/ng-bootstrap": "^12.0.1",
+    "@angular/common": "~13.3.5",
+    "@angular/compiler": "~13.3.5",
+    "@angular/core": "~13.3.5",
+    "@angular/forms": "~13.3.5",
+    "@angular/localize": "~13.3.5",
+    "@angular/platform-browser": "~13.3.5",
+    "@angular/platform-browser-dynamic": "~13.3.5",
+    "@angular/router": "~13.3.5",
+    "@ng-bootstrap/ng-bootstrap": "^12.1.1",
     "@ng-select/ng-select": "^8.1.1",
     "@ngneat/dirty-check-forms": "^3.0.2",
     "@popperjs/core": "^2.11.4",
@@ -38,21 +38,23 @@
   },
   "devDependencies": {
     "@angular-builders/jest": "13.0.3",
-    "@angular-devkit/build-angular": "~13.3.1",
-    "@angular/cli": "~13.3.1",
-    "@angular/compiler-cli": "~13.3.1",
+    "@angular-devkit/build-angular": "~13.3.4",
+    "@angular/cli": "~13.3.4",
+    "@angular/compiler-cli": "~13.3.5",
     "@types/jest": "27.4.1",
-    "@types/node": "^17.0.23",
+    "@types/node": "^17.0.30",
     "codelyzer": "^6.0.2",
-    "concurrently": "7.0.0",
-    "jest": "27.5.1",
+    "concurrently": "7.1.0",
+    "jest": "28.0.3",
+    "jest-environment-jsdom": "^28.0.2",
+    "jest-preset-angular": "^12.0.0-next.1",
     "ts-node": "~10.7.0",
     "tslint": "~6.1.3",
     "typescript": "~4.6.3",
     "wait-on": "~6.0.1"
   },
   "optionalDependencies": {
-    "cypress": "~9.5.3",
-    "@cypress/schematic": "^1.6.0"
+    "@cypress/schematic": "^1.6.0",
+    "cypress": "~9.6.0"
   }
 }

src-ui/setup-jest.ts

@@ -1,4 +1,4 @@
-import 'jest-preset-angular/setup-jest'
+import { jest } from '@jest/globals'
 
 /* global mocks for jsdom */
 const mock = () => {
@@ -26,5 +26,6 @@ Object.defineProperty(document.body.style, 'transform', {
   },
 })
 
-/* output shorter and more meaningful Zone error stack traces */
-// Error.stackTraceLimit = 2
+HTMLCanvasElement.prototype.getContext = <
+  typeof HTMLCanvasElement.prototype.getContext
+>jest.fn()

src-ui/src/app/components/app-frame/app-frame.component.scss

@@ -1,4 +1,3 @@
-@import "/src/theme";
 /*
  * Sidebar
  */
@@ -36,10 +35,15 @@
 .sidebar .nav-link {
   font-weight: 500;
 
-  &:hover, &.active {
+  &:hover, &.active, &:focus {
     color: var(--bs-primary);
   }
 
+  &:focus-visible {
+    outline: none;
+    background-color: var(--bs-body-bg);
+  }
+
   &.active {
     font-weight: bold;
   }

src-ui/src/app/components/app-frame/app-frame.component.ts

@@ -22,6 +22,7 @@ import {
   RemoteVersionService,
   AppRemoteVersion,
 } from 'src/app/services/rest/remote-version.service'
+import { QueryParamsService } from 'src/app/services/query-params.service'
 
 @Component({
   selector: 'app-app-frame',
@@ -37,7 +38,8 @@ export class AppFrameComponent {
     public savedViewService: SavedViewService,
     private list: DocumentListViewService,
     private meta: Meta,
-    private remoteVersionService: RemoteVersionService
+    private remoteVersionService: RemoteVersionService,
+    private queryParamsService: QueryParamsService
   ) {
     this.remoteVersionService
       .checkForUpdates()
@@ -92,7 +94,7 @@ export class AppFrameComponent {
 
   search() {
     this.closeMenu()
-    this.list.quickFilter([
+    this.queryParamsService.navigateWithFilterRules([
       {
         rule_type: FILTER_FULLTEXT_QUERY,
         value: (this.searchField.value as string).trim(),

src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.scss

@@ -1,5 +1,3 @@
-@import "/src/theme";
-
 .badge-corner {
   position: absolute;
   top: -8px;

src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts

@@ -3,11 +3,11 @@ import { Router } from '@angular/router'
 import { Subscription } from 'rxjs'
 import { PaperlessDocument } from 'src/app/data/paperless-document'
 import { PaperlessSavedView } from 'src/app/data/paperless-saved-view'
-import { DocumentListViewService } from 'src/app/services/document-list-view.service'
 import { ConsumerStatusService } from 'src/app/services/consumer-status.service'
 import { DocumentService } from 'src/app/services/rest/document.service'
 import { PaperlessTag } from 'src/app/data/paperless-tag'
 import { FILTER_HAS_TAGS_ALL } from 'src/app/data/filter-rule-type'
+import { QueryParamsService } from 'src/app/services/query-params.service'
 
 @Component({
   selector: 'app-saved-view-widget',
@@ -18,7 +18,7 @@ export class SavedViewWidgetComponent implements OnInit, OnDestroy {
   constructor(
     private documentService: DocumentService,
     private router: Router,
-    private list: DocumentListViewService,
+    private queryParamsService: QueryParamsService,
     private consumerStatusService: ConsumerStatusService
   ) {}
 
@@ -60,13 +60,14 @@ export class SavedViewWidgetComponent implements OnInit, OnDestroy {
     if (this.savedView.show_in_sidebar) {
       this.router.navigate(['view', this.savedView.id])
     } else {
-      this.list.loadSavedView(this.savedView, true)
-      this.router.navigate(['documents'])
+      this.router.navigate(['documents'], {
+        queryParams: { view: this.savedView.id },
+      })
     }
   }
 
   clickTag(tag: PaperlessTag) {
-    this.list.quickFilter([
+    this.queryParamsService.navigateWithFilterRules([
       { rule_type: FILTER_HAS_TAGS_ALL, value: tag.id.toString() },
     ])
   }

src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss

@@ -1,5 +1,3 @@
-@import "/src/theme";
-
 form {
   position: relative;
 }

src-ui/src/app/components/document-detail/document-detail.component.html

@@ -162,8 +162,8 @@
             <div [ngbNavOutlet]="nav" class="mt-2"></div>
 
             <button type="button" class="btn btn-outline-secondary" (click)="discard()" i18n [disabled]="networkActive || !(isDirty$ | async)">Discard</button>&nbsp;
-            <button type="button" class="btn btn-outline-primary" (click)="saveEditNext()" *ngIf="hasNext()" i18n [disabled]="networkActive || !(isDirty$ | async)">Save & next</button>&nbsp;
-            <button type="submit" class="btn btn-primary" i18n [disabled]="networkActive || !(isDirty$ | async)">Save</button>&nbsp;
+            <button type="button" class="btn btn-outline-primary" (click)="saveEditNext()" *ngIf="hasNext()" i18n [disabled]="networkActive || !(isDirty$ | async) || error">Save & next</button>&nbsp;
+            <button type="submit" class="btn btn-primary" i18n [disabled]="networkActive || !(isDirty$ | async) || error">Save</button>&nbsp;
         </form>
     </div>
 

src-ui/src/app/components/document-detail/document-detail.component.ts

@@ -34,6 +34,8 @@ import {
 } from 'rxjs/operators'
 import { PaperlessDocumentSuggestions } from 'src/app/data/paperless-document-suggestions'
 import { FILTER_FULLTEXT_MORELIKE } from 'src/app/data/filter-rule-type'
+import { normalizeDateStr } from 'src/app/utils/date'
+import { QueryParamsService } from 'src/app/services/query-params.service'
 
 @Component({
   selector: 'app-document-detail',
@@ -113,7 +115,8 @@ export class DocumentDetailComponent
     private documentListViewService: DocumentListViewService,
     private documentTitlePipe: DocumentTitlePipe,
     private toastService: ToastService,
-    private settings: SettingsService
+    private settings: SettingsService,
+    private queryParamsService: QueryParamsService
   ) {
     this.titleSubject
       .pipe(
@@ -145,18 +148,24 @@ export class DocumentDetailComponent
     this.documentForm.valueChanges
       .pipe(takeUntil(this.unsubscribeNotifier))
       .subscribe((changes) => {
+        this.error = null
         if (this.ogDate) {
-          let newDate = new Date(changes['created'])
-          newDate.setHours(
-            this.ogDate.getHours(),
-            this.ogDate.getMinutes(),
-            this.ogDate.getSeconds(),
-            this.ogDate.getMilliseconds()
-          )
-          this.documentForm.patchValue(
-            { created: this.formatDate(newDate) },
-            { emitEvent: false }
-          )
+          try {
+            let newDate = new Date(normalizeDateStr(changes['created']))
+            newDate.setHours(
+              this.ogDate.getHours(),
+              this.ogDate.getMinutes(),
+              this.ogDate.getSeconds(),
+              this.ogDate.getMilliseconds()
+            )
+            this.documentForm.patchValue(
+              { created: newDate.toISOString() },
+              { emitEvent: false }
+            )
+          } catch (e) {
+            // catch this before we try to save and simulate an api error
+            this.error = { created: e.message }
+          }
         }
 
         Object.assign(this.document, this.documentForm.value)
@@ -199,22 +208,22 @@ export class DocumentDetailComponent
             this.updateComponent(doc)
           }
 
-          this.ogDate = new Date(doc.created)
+          this.ogDate = new Date(normalizeDateStr(doc.created.toString()))
 
           // Initialize dirtyCheck
           this.store = new BehaviorSubject({
             title: doc.title,
             content: doc.content,
-            created: this.formatDate(this.ogDate),
+            created: this.ogDate.toISOString(),
             correspondent: doc.correspondent,
             document_type: doc.document_type,
             archive_serial_number: doc.archive_serial_number,
             tags: [...doc.tags],
           })
 
-          // ensure we're always starting with 24-char ISO8601 string
+          // start with ISO8601 string
           this.documentForm.patchValue(
-            { created: this.formatDate(this.ogDate) },
+            { created: this.ogDate.toISOString() },
             { emitEvent: false }
           )
 
@@ -244,6 +253,7 @@ export class DocumentDetailComponent
 
   updateComponent(doc: PaperlessDocument) {
     this.document = doc
+    this.requiresPassword = false
     this.documentsService
       .getMetadata(doc.id)
       .pipe(first())
@@ -318,16 +328,17 @@ export class DocumentDetailComponent
     this.documentsService
       .get(this.documentId)
       .pipe(first())
-      .subscribe(
-        (doc) => {
+      .subscribe({
+        next: (doc) => {
           Object.assign(this.document, doc)
           this.title = doc.title
           this.documentForm.patchValue(doc)
+          this.openDocumentService.setDirty(doc.id, false)
         },
-        (error) => {
+        error: () => {
           this.router.navigate(['404'])
-        }
-      )
+        },
+      })
   }
 
   save() {
@@ -437,7 +448,7 @@ export class DocumentDetailComponent
   }
 
   moreLike() {
-    this.documentListViewService.quickFilter([
+    this.queryParamsService.navigateWithFilterRules([
       {
         rule_type: FILTER_FULLTEXT_MORELIKE,
         value: this.documentId.toString(),
@@ -485,8 +496,4 @@ export class DocumentDetailComponent
       this.password = (event.target as HTMLInputElement).value
     }
   }
-
-  formatDate(date: Date): string {
-    return date.toISOString().split('.')[0] + 'Z'
-  }
 }

src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html

@@ -42,7 +42,7 @@
                 <path fill-rule="evenodd" d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5L13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5zm-9.761 5.175l-.106.106-1.528 3.821 3.821-1.528.106-.106A.5.5 0 0 1 5 12.5V12h-.5a.5.5 0 0 1-.5-.5V11h-.5a.5.5 0 0 1-.468-.325z"/>
               </svg>&nbsp;<span class="d-block d-md-inline" i18n>Edit</span>
             </a>
-            <a class="btn btn-sm btn-outline-secondary" [href]="previewUrl"
+            <a class="btn btn-sm btn-outline-secondary" target="_blank" [href]="previewUrl"
             [ngbPopover]="previewContent" [popoverTitle]="document.title | documentTitle"
             autoClose="true" popoverClass="shadow" (mouseenter)="mouseEnterPreview()" (mouseleave)="mouseLeavePreview()" #popover="ngbPopover">
               <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-eye" viewBox="0 0 16 16">

src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss

@@ -1,5 +1,3 @@
-@import "/src/theme";
-
 .result-content {
   overflow-wrap: anywhere;
 }

src-ui/src/app/components/document-list/document-card-small/document-card-small.component.scss

@@ -1,5 +1,3 @@
-@import "/src/theme";
-
 .card-text {
   font-size: 90%;
 }

src-ui/src/app/components/document-list/document-list.component.html

@@ -38,7 +38,7 @@
   <div ngbDropdown class="btn-group ms-2 flex-fill">
     <button class="btn btn-outline-primary btn-sm" id="dropdownBasic1" ngbDropdownToggle i18n>Sort</button>
     <div ngbDropdownMenu aria-labelledby="dropdownBasic1" class="shadow dropdown-menu-right">
-      <div class="w-100 d-flex btn-group-toggle pb-2 mb-1 border-bottom" ngbRadioGroup [(ngModel)]="list.sortReverse">
+      <div class="w-100 d-flex btn-group-toggle pb-2 mb-1 border-bottom" ngbRadioGroup [(ngModel)]="listSort">
         <label ngbButtonLabel class="btn-outline-primary btn-sm mx-2 flex-fill">
           <input ngbButton type="radio" class="btn btn-check btn-sm" [value]="false">
           <svg class="toolbaricon" fill="currentColor">
@@ -53,7 +53,7 @@
         </label>
       </div>
       <div>
-        <button *ngFor="let f of getSortFields()" ngbDropdownItem (click)="list.sortField = f.field"
+        <button *ngFor="let f of getSortFields()" ngbDropdownItem (click)="setSortField(f.field)"
           [class.active]="list.sortField == f.field">{{f.name}}
         </button>
       </div>
@@ -64,7 +64,7 @@
     <button class="btn btn-sm btn-outline-primary dropdown-toggle flex-fill" ngbDropdownToggle i18n>Views</button>
     <div class="dropdown-menu shadow dropdown-menu-right" ngbDropdownMenu>
       <ng-container *ngIf="!list.activeSavedViewId">
-        <button ngbDropdownItem *ngFor="let view of savedViewService.allViews" (click)="loadViewConfig(view)">{{view.name}}</button>
+        <button ngbDropdownItem *ngFor="let view of savedViewService.allViews" (click)="loadViewConfig(view.id)">{{view.name}}</button>
         <div class="dropdown-divider" *ngIf="savedViewService.allViews.length > 0"></div>
       </ng-container>
 

src-ui/src/app/components/document-list/document-list.component.scss

@@ -1,5 +1,3 @@
-@import "/src/theme";
-
 ::ng-deep app-document-list app-page-header > div.mb-3 {
   margin-bottom: 0 !important;
 }

src-ui/src/app/components/document-list/document-list.component.ts

@@ -9,20 +9,9 @@ import {
 } from '@angular/core'
 import { ActivatedRoute, Router } from '@angular/router'
 import { NgbModal } from '@ng-bootstrap/ng-bootstrap'
-import {
-  filter,
-  first,
-  map,
-  Subject,
-  Subscription,
-  switchMap,
-  takeUntil,
-} from 'rxjs'
+import { filter, first, map, Subject, switchMap, takeUntil } from 'rxjs'
 import { FilterRule, isFullTextFilterRule } from 'src/app/data/filter-rule'
-import {
-  FILTER_FULLTEXT_MORELIKE,
-  FILTER_RULE_TYPES,
-} from 'src/app/data/filter-rule-type'
+import { FILTER_FULLTEXT_MORELIKE } from 'src/app/data/filter-rule-type'
 import { PaperlessDocument } from 'src/app/data/paperless-document'
 import { PaperlessSavedView } from 'src/app/data/paperless-saved-view'
 import {
@@ -32,7 +21,10 @@ import {
 import { ConsumerStatusService } from 'src/app/services/consumer-status.service'
 import { DocumentListViewService } from 'src/app/services/document-list-view.service'
 import {
-  DocumentService,
+  filterRulesFromQueryParams,
+  QueryParamsService,
+} from 'src/app/services/query-params.service'
+import {
   DOCUMENT_SORT_FIELDS,
   DOCUMENT_SORT_FIELDS_FULLTEXT,
 } from 'src/app/services/rest/document.service'
@@ -49,13 +41,13 @@ import { SaveViewConfigDialogComponent } from './save-view-config-dialog/save-vi
 export class DocumentListComponent implements OnInit, OnDestroy, AfterViewInit {
   constructor(
     public list: DocumentListViewService,
-    private documentService: DocumentService,
     public savedViewService: SavedViewService,
     public route: ActivatedRoute,
     private router: Router,
     private toastService: ToastService,
     private modalService: NgbModal,
-    private consumerStatusService: ConsumerStatusService
+    private consumerStatusService: ConsumerStatusService,
+    private queryParamsService: QueryParamsService
   ) {}
 
   @ViewChild('filterEditor')
@@ -83,8 +75,26 @@ export class DocumentListComponent implements OnInit, OnDestroy, AfterViewInit {
       : DOCUMENT_SORT_FIELDS
   }
 
+  set listSort(reverse: boolean) {
+    this.list.sortReverse = reverse
+    this.queryParamsService.sortField = this.list.sortField
+    this.queryParamsService.sortReverse = reverse
+  }
+
+  get listSort(): boolean {
+    return this.list.sortReverse
+  }
+
+  setSortField(field: string) {
+    this.list.sortField = field
+    this.queryParamsService.sortField = field
+    this.queryParamsService.sortReverse = this.listSort
+  }
+
   onSort(event: SortEvent) {
     this.list.setSort(event.column, event.reverse)
+    this.queryParamsService.sortField = event.column
+    this.queryParamsService.sortReverse = event.reverse
   }
 
   get isBulkEditing(): boolean {
@@ -109,60 +119,39 @@ export class DocumentListComponent implements OnInit, OnDestroy, AfterViewInit {
 
     this.route.paramMap
       .pipe(
-        filter((params) => params.has('id')), // only on saved view
+        filter((params) => params.has('id')), // only on saved view e.g. /view/id
         switchMap((params) => {
           return this.savedViewService
             .getCached(+params.get('id'))
-            .pipe(map((view) => ({ params, view })))
+            .pipe(map((view) => ({ view })))
         })
       )
       .pipe(takeUntil(this.unsubscribeNotifier))
-      .subscribe(({ view, params }) => {
+      .subscribe(({ view }) => {
         if (!view) {
           this.router.navigate(['404'])
           return
         }
         this.list.activateSavedView(view)
         this.list.reload()
+        this.queryParamsService.updateFromView(view)
         this.unmodifiedFilterRules = view.filter_rules
       })
 
-    const allFilterRuleQueryParams: string[] = FILTER_RULE_TYPES.map(
-      (rt) => rt.filtervar
-    )
-
     this.route.queryParamMap
       .pipe(
-        filter(() => !this.route.snapshot.paramMap.has('id')), // only when not on saved view
+        filter(() => !this.route.snapshot.paramMap.has('id')), // only when not on /view/id
         takeUntil(this.unsubscribeNotifier)
       )
       .subscribe((queryParams) => {
-        // transform query params to filter rules
-        let filterRulesFromQueryParams: FilterRule[] = []
-        allFilterRuleQueryParams
-          .filter((frqp) => queryParams.has(frqp))
-          .forEach((filterQueryParamName) => {
-            const filterQueryParamValues: string[] = queryParams
-              .get(filterQueryParamName)
-              .split(',')
-
-            filterRulesFromQueryParams = filterRulesFromQueryParams.concat(
-              // map all values to filter rules
-              filterQueryParamValues.map((val) => {
-                return {
-                  rule_type: FILTER_RULE_TYPES.find(
-                    (rt) => rt.filtervar == filterQueryParamName
-                  ).id,
-                  value: val,
-                }
-              })
-            )
-          })
-
-        this.list.activateSavedView(null)
-        this.list.filterRules = filterRulesFromQueryParams
-        this.list.reload()
-        this.unmodifiedFilterRules = []
+        if (queryParams.has('view')) {
+          // loading a saved view on /documents
+          this.loadViewConfig(parseInt(queryParams.get('view')))
+        } else {
+          this.list.activateSavedView(null)
+          this.queryParamsService.parseQueryParams(queryParams)
+          this.unmodifiedFilterRules = []
+        }
       })
   }
 
@@ -171,17 +160,7 @@ export class DocumentListComponent implements OnInit, OnDestroy, AfterViewInit {
       .pipe(takeUntil(this.unsubscribeNotifier))
       .subscribe({
         next: (filterRules) => {
-          const params =
-            this.documentService.filterRulesToQueryParams(filterRules)
-
-          // if we were on a saved view we navigate 'away' to /documents
-          let base = []
-          if (this.route.snapshot.paramMap.has('id')) base = ['/documents']
-
-          this.router.navigate(base, {
-            relativeTo: this.route,
-            queryParams: params,
-          })
+          this.queryParamsService.updateFilterRules(filterRules)
         },
       })
   }
@@ -192,9 +171,15 @@ export class DocumentListComponent implements OnInit, OnDestroy, AfterViewInit {
     this.unsubscribeNotifier.complete()
   }
 
-  loadViewConfig(view: PaperlessSavedView) {
-    this.list.loadSavedView(view)
-    this.list.reload()
+  loadViewConfig(viewId: number) {
+    this.savedViewService
+      .getCached(viewId)
+      .pipe(first())
+      .subscribe((view) => {
+        this.list.loadSavedView(view)
+        this.list.reload()
+        this.queryParamsService.updateFromView(view)
+      })
   }
 
   saveViewConfig() {
@@ -282,7 +267,7 @@ export class DocumentListComponent implements OnInit, OnDestroy, AfterViewInit {
   }
 
   clickMoreLike(documentID: number) {
-    this.list.quickFilter([
+    this.queryParamsService.navigateWithFilterRules([
       { rule_type: FILTER_FULLTEXT_MORELIKE, value: documentID.toString() },
     ])
   }

src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.ts

@@ -3,7 +3,7 @@ import { NgbModal } from '@ng-bootstrap/ng-bootstrap'
 import { FILTER_CORRESPONDENT } from 'src/app/data/filter-rule-type'
 import { PaperlessCorrespondent } from 'src/app/data/paperless-correspondent'
 import { CustomDatePipe } from 'src/app/pipes/custom-date.pipe'
-import { DocumentListViewService } from 'src/app/services/document-list-view.service'
+import { QueryParamsService } from 'src/app/services/query-params.service'
 import { CorrespondentService } from 'src/app/services/rest/correspondent.service'
 import { ToastService } from 'src/app/services/toast.service'
 import { CorrespondentEditDialogComponent } from '../../common/edit-dialog/correspondent-edit-dialog/correspondent-edit-dialog.component'
@@ -20,7 +20,7 @@ export class CorrespondentListComponent extends ManagementListComponent<Paperles
     correspondentsService: CorrespondentService,
     modalService: NgbModal,
     toastService: ToastService,
-    list: DocumentListViewService,
+    queryParamsService: QueryParamsService,
     private datePipe: CustomDatePipe
   ) {
     super(
@@ -28,13 +28,13 @@ export class CorrespondentListComponent extends ManagementListComponent<Paperles
       modalService,
       CorrespondentEditDialogComponent,
       toastService,
-      list,
+      queryParamsService,
       FILTER_CORRESPONDENT,
       $localize`correspondent`,
       [
         {
           key: 'last_correspondence',
-          name: $localize`Last correspondence`,
+          name: $localize`Last used`,
           valueFn: (c: PaperlessCorrespondent) => {
             return this.datePipe.transform(c.last_correspondence)
           },

src-ui/src/app/components/manage/document-type-list/document-type-list.component.ts

@@ -2,7 +2,7 @@ import { Component } from '@angular/core'
 import { NgbModal } from '@ng-bootstrap/ng-bootstrap'
 import { FILTER_DOCUMENT_TYPE } from 'src/app/data/filter-rule-type'
 import { PaperlessDocumentType } from 'src/app/data/paperless-document-type'
-import { DocumentListViewService } from 'src/app/services/document-list-view.service'
+import { QueryParamsService } from 'src/app/services/query-params.service'
 import { DocumentTypeService } from 'src/app/services/rest/document-type.service'
 import { ToastService } from 'src/app/services/toast.service'
 import { DocumentTypeEditDialogComponent } from '../../common/edit-dialog/document-type-edit-dialog/document-type-edit-dialog.component'
@@ -18,14 +18,14 @@ export class DocumentTypeListComponent extends ManagementListComponent<Paperless
     documentTypeService: DocumentTypeService,
     modalService: NgbModal,
     toastService: ToastService,
-    list: DocumentListViewService
+    queryParamsService: QueryParamsService
   ) {
     super(
       documentTypeService,
       modalService,
       DocumentTypeEditDialogComponent,
       toastService,
-      list,
+      queryParamsService,
       FILTER_DOCUMENT_TYPE,
       $localize`document type`,
       []

src-ui/src/app/components/manage/management-list/management-list.component.html

@@ -17,7 +17,7 @@
   <thead>
     <tr>
       <th scope="col" sortable="name" [currentSortField]="sortField" [currentSortReverse]="sortReverse" (sort)="onSort($event)" i18n>Name</th>
-      <th scope="col" sortable="matching_algorithm" [currentSortField]="sortField" [currentSortReverse]="sortReverse" (sort)="onSort($event)" i18n>Matching</th>
+      <th scope="col" class="d-none d-sm-table-cell" sortable="matching_algorithm" [currentSortField]="sortField" [currentSortReverse]="sortReverse" (sort)="onSort($event)" i18n>Matching</th>
       <th scope="col" sortable="document_count" [currentSortField]="sortField" [currentSortReverse]="sortReverse" (sort)="onSort($event)" i18n>Document count</th>
       <th scope="col" *ngFor="let column of extraColumns" sortable="{{column.key}}" [currentSortField]="sortField" [currentSortReverse]="sortReverse" (sort)="onSort($event)">{{column.name}}</th>
       <th scope="col" i18n>Actions</th>
@@ -26,14 +26,28 @@
   <tbody>
     <tr *ngFor="let object of data">
       <td scope="row">{{ object.name }}</td>
-      <td scope="row">{{ getMatching(object) }}</td>
+      <td scope="row" class="d-none d-sm-table-cell">{{ getMatching(object) }}</td>
       <td scope="row">{{ object.document_count }}</td>
       <td scope="row" *ngFor="let column of extraColumns">
         <div *ngIf="column.rendersHtml; else colValue" [innerHtml]="column.valueFn.call(null, object) | safeHtml"></div>
         <ng-template #colValue>{{ column.valueFn.call(null, object) }}</ng-template>
       </td>
       <td scope="row">
-        <div class="btn-group">
+        <div class="btn-group d-block d-sm-none">
+          <div ngbDropdown class="d-inline-block">
+            <button type="button" class="btn btn-link" id="actionsMenuMobile" ngbDropdownToggle>
+              <svg class="toolbaricon" fill="currentColor">
+                <use xlink:href="assets/bootstrap-icons.svg#three-dots-vertical" />
+              </svg>
+            </button>
+            <div ngbDropdownMenu aria-labelledby="actionsMenuMobile">
+              <button (click)="filterDocuments(object)" ngbDropdownItem i18n>Filter Documents</button>
+              <button (click)="openEditDialog(object)" ngbDropdownItem i18n>Edit</button>
+              <button class="text-danger" (click)="openDeleteDialog(object)" ngbDropdownItem i18n>Delete</button>
+            </div>
+          </div>
+        </div>
+        <div class="btn-group d-none d-sm-block">
           <button class="btn btn-sm btn-outline-secondary" (click)="filterDocuments(object)">
             <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-funnel" viewBox="0 0 16 16">
               <path fill-rule="evenodd" d="M1.5 1.5A.5.5 0 0 1 2 1h12a.5.5 0 0 1 .5.5v2a.5.5 0 0 1-.128.334L10 8.692V13.5a.5.5 0 0 1-.342.474l-3 1A.5.5 0 0 1 6 14.5V8.692L1.628 3.834A.5.5 0 0 1 1.5 3.5v-2zm1 .5v1.308l4.372 4.858A.5.5 0 0 1 7 8.5v5.306l2-.666V8.5a.5.5 0 0 1 .128-.334L13.5 3.308V2h-11z"/>

src-ui/src/app/components/manage/management-list/management-list.component.scss

@@ -0,0 +1,4 @@
+// hide caret on mobile dropdown
+.d-block.d-sm-none .dropdown-toggle::after {
+    display: none;
+}

src-ui/src/app/components/manage/management-list/management-list.component.ts

@@ -18,7 +18,7 @@ import {
   SortableDirective,
   SortEvent,
 } from 'src/app/directives/sortable.directive'
-import { DocumentListViewService } from 'src/app/services/document-list-view.service'
+import { QueryParamsService } from 'src/app/services/query-params.service'
 import { AbstractNameFilterService } from 'src/app/services/rest/abstract-name-filter-service'
 import { ToastService } from 'src/app/services/toast.service'
 import { ConfirmDialogComponent } from '../../common/confirm-dialog/confirm-dialog.component'
@@ -42,7 +42,7 @@ export abstract class ManagementListComponent<T extends ObjectWithId>
     private modalService: NgbModal,
     private editDialogComponent: any,
     private toastService: ToastService,
-    private list: DocumentListViewService,
+    private queryParamsService: QueryParamsService,
     protected filterRuleType: number,
     public typeName: string,
     public extraColumns: ManagementListColumn[]
@@ -140,7 +140,7 @@ export abstract class ManagementListComponent<T extends ObjectWithId>
   }
 
   filterDocuments(object: ObjectWithId) {
-    this.list.quickFilter([
+    this.queryParamsService.navigateWithFilterRules([
       { rule_type: this.filterRuleType, value: object.id.toString() },
     ])
   }

src-ui/src/app/components/manage/settings/settings.component.html

@@ -7,7 +7,7 @@
 
   <ul ngbNav #nav="ngbNav" class="nav-tabs">
     <li [ngbNavItem]="1">
-      <a ngbNavLink i18n>General settings</a>
+      <a ngbNavLink i18n>General</a>
       <ng-template ngbNavContent>
 
         <h4 i18n>Appearance</h4>
@@ -104,7 +104,7 @@
           <div class="col-md-3 col-form-label">
             <span i18n>Theme Color</span>
           </div>
-          <div class="col-3">
+          <div class="col col-md-3">
             <app-input-color i18n-title formControlName="themeColor" [error]="error?.color"></app-input-color>
           </div>
           <div class="col-2">

src-ui/src/app/components/manage/tag-list/tag-list.component.ts

@@ -2,7 +2,7 @@ import { Component } from '@angular/core'
 import { NgbModal } from '@ng-bootstrap/ng-bootstrap'
 import { FILTER_HAS_TAGS_ALL } from 'src/app/data/filter-rule-type'
 import { PaperlessTag } from 'src/app/data/paperless-tag'
-import { DocumentListViewService } from 'src/app/services/document-list-view.service'
+import { QueryParamsService } from 'src/app/services/query-params.service'
 import { TagService } from 'src/app/services/rest/tag.service'
 import { ToastService } from 'src/app/services/toast.service'
 import { TagEditDialogComponent } from '../../common/edit-dialog/tag-edit-dialog/tag-edit-dialog.component'
@@ -18,14 +18,14 @@ export class TagListComponent extends ManagementListComponent<PaperlessTag> {
     tagService: TagService,
     modalService: NgbModal,
     toastService: ToastService,
-    list: DocumentListViewService
+    queryParamsService: QueryParamsService
   ) {
     super(
       tagService,
       modalService,
       TagEditDialogComponent,
       toastService,
-      list,
+      queryParamsService,
       FILTER_HAS_TAGS_ALL,
       $localize`tag`,
       [

src-ui/src/app/pipes/custom-date.pipe.ts

@@ -1,6 +1,7 @@
 import { DatePipe } from '@angular/common'
 import { Inject, LOCALE_ID, Pipe, PipeTransform } from '@angular/core'
 import { SettingsService, SETTINGS_KEYS } from '../services/settings.service'
+import { normalizeDateStr } from '../utils/date'
 
 const FORMAT_TO_ISO_FORMAT = {
   longDate: 'y-MM-dd',
@@ -33,6 +34,7 @@ export class CustomDatePipe implements PipeTransform {
       this.settings.get(SETTINGS_KEYS.DATE_LOCALE) ||
       this.defaultLocale
     let f = format || this.settings.get(SETTINGS_KEYS.DATE_FORMAT)
+    if (typeof value == 'string') value = normalizeDateStr(value)
     if (l == 'iso-8601') {
       return this.datePipe.transform(value, FORMAT_TO_ISO_FORMAT[f], timezone)
     } else {

src-ui/src/app/services/document-list-view.service.ts

@@ -1,5 +1,5 @@
 import { Injectable } from '@angular/core'
-import { ActivatedRoute, Router } from '@angular/router'
+import { ActivatedRoute, Params, Router } from '@angular/router'
 import { Observable } from 'rxjs'
 import {
   cloneFilterRules,
@@ -220,6 +220,13 @@ export class DocumentListViewService {
     return this.activeListViewState.sortReverse
   }
 
+  get sortParams(): Params {
+    return {
+      sortField: this.sortField,
+      sortReverse: this.sortReverse,
+    }
+  }
+
   get collectionSize(): number {
     return this.activeListViewState.collectionSize
   }
@@ -265,14 +272,6 @@ export class DocumentListViewService {
     }
   }
 
-  quickFilter(filterRules: FilterRule[]) {
-    const params = this.documentService.filterRulesToQueryParams(filterRules)
-    this.router.navigate(['/documents'], {
-      relativeTo: this.route,
-      queryParams: params,
-    })
-  }
-
   getLastPage(): number {
     return Math.ceil(this.collectionSize / this.currentPageSize)
   }
@@ -434,9 +433,7 @@ export class DocumentListViewService {
 
   constructor(
     private documentService: DocumentService,
-    private settings: SettingsService,
-    private router: Router,
-    private route: ActivatedRoute
+    private settings: SettingsService
   ) {
     let documentListViewConfigJson = localStorage.getItem(
       DOCUMENT_LIST_SERVICE.CURRENT_VIEW_CONFIG

src-ui/src/app/services/query-params.service.ts

@@ -0,0 +1,156 @@
+import { Injectable } from '@angular/core'
+import { ParamMap, Params, Router } from '@angular/router'
+import { FilterRule } from '../data/filter-rule'
+import { FILTER_RULE_TYPES } from '../data/filter-rule-type'
+import { PaperlessSavedView } from '../data/paperless-saved-view'
+import { DocumentListViewService } from './document-list-view.service'
+
+const SORT_FIELD_PARAMETER = 'sort'
+const SORT_REVERSE_PARAMETER = 'reverse'
+
+@Injectable({
+  providedIn: 'root',
+})
+export class QueryParamsService {
+  constructor(private router: Router, private list: DocumentListViewService) {}
+
+  private filterParams: Params = {}
+  private sortParams: Params = {}
+
+  updateFilterRules(
+    filterRules: FilterRule[],
+    updateQueryParams: boolean = true
+  ) {
+    this.filterParams = filterRulesToQueryParams(filterRules)
+    if (updateQueryParams) this.updateQueryParams()
+  }
+
+  set sortField(field: string) {
+    this.sortParams[SORT_FIELD_PARAMETER] = field
+    this.updateQueryParams()
+  }
+
+  set sortReverse(reverse: boolean) {
+    if (!reverse) this.sortParams[SORT_REVERSE_PARAMETER] = undefined
+    else this.sortParams[SORT_REVERSE_PARAMETER] = reverse
+    this.updateQueryParams()
+  }
+
+  get params(): Params {
+    return {
+      ...this.sortParams,
+      ...this.filterParams,
+    }
+  }
+
+  private updateQueryParams() {
+    // if we were on a saved view we navigate 'away' to /documents
+    let base = []
+    if (this.router.routerState.snapshot.url.includes('/view/'))
+      base = ['/documents']
+
+    this.router.navigate(base, {
+      queryParams: this.params,
+    })
+  }
+
+  public parseQueryParams(queryParams: ParamMap) {
+    let filterRules = filterRulesFromQueryParams(queryParams)
+    if (
+      filterRules.length ||
+      queryParams.has(SORT_FIELD_PARAMETER) ||
+      queryParams.has(SORT_REVERSE_PARAMETER)
+    ) {
+      this.list.filterRules = filterRules
+      this.list.sortField = queryParams.get(SORT_FIELD_PARAMETER)
+      this.list.sortReverse =
+        queryParams.has(SORT_REVERSE_PARAMETER) ||
+        (!queryParams.has(SORT_FIELD_PARAMETER) &&
+          !queryParams.has(SORT_REVERSE_PARAMETER))
+      this.list.reload()
+    } else if (
+      filterRules.length == 0 &&
+      !queryParams.has(SORT_FIELD_PARAMETER)
+    ) {
+      // this is navigating to /documents so we need to update the params from the list
+      this.updateFilterRules(this.list.filterRules, false)
+      this.sortParams[SORT_FIELD_PARAMETER] = this.list.sortField
+      this.sortParams[SORT_REVERSE_PARAMETER] = this.list.sortReverse
+      this.router.navigate([], {
+        queryParams: this.params,
+        replaceUrl: true,
+      })
+    }
+  }
+
+  updateFromView(view: PaperlessSavedView) {
+    if (!this.router.routerState.snapshot.url.includes('/view/')) {
+      // navigation for /documents?view=
+      this.router.navigate([], {
+        queryParams: { view: view.id },
+      })
+    }
+    // make sure params are up-to-date
+    this.updateFilterRules(view.filter_rules, false)
+    this.sortParams[SORT_FIELD_PARAMETER] = this.list.sortField
+    this.sortParams[SORT_REVERSE_PARAMETER] = this.list.sortReverse
+  }
+
+  navigateWithFilterRules(filterRules: FilterRule[]) {
+    this.updateFilterRules(filterRules)
+    this.router.navigate(['/documents'], {
+      queryParams: this.params,
+    })
+  }
+}
+
+export function filterRulesToQueryParams(filterRules: FilterRule[]): Object {
+  if (filterRules) {
+    let params = {}
+    for (let rule of filterRules) {
+      let ruleType = FILTER_RULE_TYPES.find((t) => t.id == rule.rule_type)
+      if (ruleType.multi) {
+        params[ruleType.filtervar] = params[ruleType.filtervar]
+          ? params[ruleType.filtervar] + ',' + rule.value
+          : rule.value
+      } else if (ruleType.isnull_filtervar && rule.value == null) {
+        params[ruleType.isnull_filtervar] = true
+      } else {
+        params[ruleType.filtervar] = rule.value
+      }
+    }
+    return params
+  } else {
+    return null
+  }
+}
+
+export function filterRulesFromQueryParams(queryParams: ParamMap) {
+  const allFilterRuleQueryParams: string[] = FILTER_RULE_TYPES.map(
+    (rt) => rt.filtervar
+  )
+
+  // transform query params to filter rules
+  let filterRulesFromQueryParams: FilterRule[] = []
+  allFilterRuleQueryParams
+    .filter((frqp) => queryParams.has(frqp))
+    .forEach((filterQueryParamName) => {
+      const filterQueryParamValues: string[] = queryParams
+        .get(filterQueryParamName)
+        .split(',')
+
+      filterRulesFromQueryParams = filterRulesFromQueryParams.concat(
+        // map all values to filter rules
+        filterQueryParamValues.map((val) => {
+          return {
+            rule_type: FILTER_RULE_TYPES.find(
+              (rt) => rt.filtervar == filterQueryParamName
+            ).id,
+            value: val,
+          }
+        })
+      )
+    })
+
+  return filterRulesFromQueryParams
+}

src-ui/src/app/services/rest/document.service.ts

@@ -10,8 +10,8 @@ import { map } from 'rxjs/operators'
 import { CorrespondentService } from './correspondent.service'
 import { DocumentTypeService } from './document-type.service'
 import { TagService } from './tag.service'
-import { FILTER_RULE_TYPES } from 'src/app/data/filter-rule-type'
 import { PaperlessDocumentSuggestions } from 'src/app/data/paperless-document-suggestions'
+import { filterRulesToQueryParams } from '../query-params.service'
 
 export const DOCUMENT_SORT_FIELDS = [
   { field: 'archive_serial_number', name: $localize`ASN` },
@@ -57,27 +57,6 @@ export class DocumentService extends AbstractPaperlessService<PaperlessDocument>
     super(http, 'documents')
   }
 
-  public filterRulesToQueryParams(filterRules: FilterRule[]): Object {
-    if (filterRules) {
-      let params = {}
-      for (let rule of filterRules) {
-        let ruleType = FILTER_RULE_TYPES.find((t) => t.id == rule.rule_type)
-        if (ruleType.multi) {
-          params[ruleType.filtervar] = params[ruleType.filtervar]
-            ? params[ruleType.filtervar] + ',' + rule.value
-            : rule.value
-        } else if (ruleType.isnull_filtervar && rule.value == null) {
-          params[ruleType.isnull_filtervar] = true
-        } else {
-          params[ruleType.filtervar] = rule.value
-        }
-      }
-      return params
-    } else {
-      return null
-    }
-  }
-
   addObservablesToDocument(doc: PaperlessDocument) {
     if (doc.correspondent) {
       doc.correspondent$ = this.correspondentService.getCached(
@@ -106,7 +85,7 @@ export class DocumentService extends AbstractPaperlessService<PaperlessDocument>
       pageSize,
       sortField,
       sortReverse,
-      Object.assign(extraParams, this.filterRulesToQueryParams(filterRules))
+      Object.assign(extraParams, filterRulesToQueryParams(filterRules))
     ).pipe(
       map((results) => {
         results.results.forEach((doc) => this.addObservablesToDocument(doc))