From 5d17df3a1832e6cdfb34d2959b77adbd0851c5c1 Mon Sep 17 00:00:00 2001 From: Philipp Date: Tue, 12 Dec 2023 22:04:29 +0000 Subject: [PATCH] add devcontainer for vscode --- .devcontainer/Dockerfile | 192 ++++++++++++++++++ .devcontainer/devcontainer.json | 12 ++ ...ocker-compose.devcontainer.sqlite-tika.yml | 91 +++++++++ .devcontainer/vscode/launch.json | 15 ++ .devcontainer/vscode/settings.json | 11 + .devcontainer/vscode/tasks.json | 81 ++++++++ .gitignore | 2 + 7 files changed, 404 insertions(+) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/docker-compose.devcontainer.sqlite-tika.yml create mode 100644 .devcontainer/vscode/launch.json create mode 100644 .devcontainer/vscode/settings.json create mode 100644 .devcontainer/vscode/tasks.json diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000..124139333 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,192 @@ +# syntax=docker/dockerfile:1 +# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md + +# Stage: compile-frontend +# Purpose: Compiles the frontend +# Notes: +# - Does NPM stuff with Typescript and such +FROM --platform=$BUILDPLATFORM docker.io/node:20-bookworm-slim AS compile-frontend + +COPY ./src-ui /src/src-ui + +WORKDIR /src/src-ui +RUN set -eux \ + && npm update npm -g \ + && npm ci --omit=optional +RUN set -eux \ + && ./node_modules/.bin/ng build --configuration production + +# Stage: main-app +# Purpose: The final image +FROM docker.io/python:3.11-bookworm as main-app + +ARG DEBIAN_FRONTEND=noninteractive + +# Buildx provided, must be defined to use though +ARG TARGETARCH + +# Can be workflow provided, defaults set for manual building +ARG JBIG2ENC_VERSION=0.29 +ARG QPDF_VERSION=11.6.3 +ARG GS_VERSION=10.02.0 + +# +# Begin installation and configuration +# Order the steps below from least often changed to most +# + +# Packages need for running +ARG RUNTIME_PACKAGES="\ + # General utils + curl \ + # Docker specific + gosu \ + # Timezones support + tzdata \ + # fonts for text file thumbnail generation + fonts-liberation \ + gettext \ + ghostscript \ + gnupg \ + icc-profiles-free \ + imagemagick \ + # PostgreSQL + libpq5 \ + postgresql-client \ + # MySQL / MariaDB + mariadb-client \ + # OCRmyPDF dependencies + tesseract-ocr \ + tesseract-ocr-eng \ + tesseract-ocr-deu \ + tesseract-ocr-fra \ + tesseract-ocr-ita \ + tesseract-ocr-spa \ + unpaper \ + pngquant \ + jbig2dec \ + # lxml + libxml2 \ + libxslt1.1 \ + # itself + qpdf \ + # Mime type detection + file \ + libmagic1 \ + media-types \ + zlib1g \ + # Barcode splitter + libzbar0 \ + poppler-utils" + +# Install Pipenv +RUN set -eux \ + && echo "Installing pipenv" \ + && python3 -m pip install --no-cache-dir --upgrade pipenv==2023.10.24 + +# Install basic runtime packages. +# These change very infrequently +RUN set -eux \ + echo "Installing system packages" \ + && apt-get update \ + && apt-get install --yes --quiet --no-install-recommends ${RUNTIME_PACKAGES} \ + && echo "Installing pre-built updates" \ + && echo "Installing qpdf ${QPDF_VERSION}" \ + && curl --fail --silent --show-error --location \ + --output libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \ + https://github.com/paperless-ngx/builder/releases/download/qpdf-${QPDF_VERSION}/libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \ + && curl --fail --silent --show-error --location \ + --output qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \ + https://github.com/paperless-ngx/builder/releases/download/qpdf-${QPDF_VERSION}/qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \ + && dpkg --install ./libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \ + && dpkg --install ./qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \ + && echo "Installing Ghostscript ${GS_VERSION}" \ + && curl --fail --silent --show-error --location \ + --output libgs10_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \ + https://github.com/paperless-ngx/builder/releases/download/ghostscript-${GS_VERSION}/libgs10_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \ + && curl --fail --silent --show-error --location \ + --output ghostscript_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \ + https://github.com/paperless-ngx/builder/releases/download/ghostscript-${GS_VERSION}/ghostscript_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \ + && curl --fail --silent --show-error --location \ + --output libgs10-common_${GS_VERSION}.dfsg-2_all.deb \ + https://github.com/paperless-ngx/builder/releases/download/ghostscript-${GS_VERSION}/libgs10-common_${GS_VERSION}.dfsg-2_all.deb \ + && dpkg --install ./libgs10-common_${GS_VERSION}.dfsg-2_all.deb \ + && dpkg --install ./libgs10_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \ + && dpkg --install ./ghostscript_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \ + && echo "Installing jbig2enc" \ + && curl --fail --silent --show-error --location \ + --output jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \ + https://github.com/paperless-ngx/builder/releases/download/jbig2enc-${JBIG2ENC_VERSION}/jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \ + && dpkg --install ./jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb + +# setup docker-specific things +# These change sometimes, but rarely +WORKDIR /usr/src/paperless/src/docker/ + +COPY [ \ + "docker/imagemagick-policy.xml", \ + "docker/wait-for-redis.py", \ + "docker/env-from-file.sh", \ + "docker/flower-conditional.sh", \ + "/usr/src/paperless/src/docker/" \ +] + +RUN set -eux \ + && echo "Configuring ImageMagick" \ + && mv imagemagick-policy.xml /etc/ImageMagick-6/policy.xml \ + && echo "Setting up Docker scripts" \ + && mv wait-for-redis.py /sbin/wait-for-redis.py \ + && chmod 755 /sbin/wait-for-redis.py \ + && mv env-from-file.sh /sbin/env-from-file.sh \ + && chmod 755 /sbin/env-from-file.sh \ + && mv flower-conditional.sh /usr/local/bin/flower-conditional.sh \ + && chmod 755 /usr/local/bin/flower-conditional.sh + +# Packages needed only for building a few quick Python +# dependencies +ARG BUILD_PACKAGES="\ + build-essential \ + git \ + # https://www.psycopg.org/docs/install.html#prerequisites + libpq-dev \ + # https://github.com/PyMySQL/mysqlclient#linux + default-libmysqlclient-dev \ + pkg-config" + +# hadolint ignore=DL3042 +RUN --mount=type=cache,target=/root/.cache/pip/,id=pip-cache \ + set -eux \ + && echo "Installing build system packages" \ + && apt-get update \ + && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \ + && python3 -m pip install --no-cache-dir --upgrade wheel + + +WORKDIR /usr/src/paperless/paperless/ + +# copy frontend +COPY --from=compile-frontend --chown=1000:1000 /src/src/documents/static ./static + +# add users, setup scripts +# Mount the compiled frontend to expected location +RUN set -eux \ + && echo "Setting up user/group" \ + && addgroup --gid 1000 paperless \ + && useradd --uid 1000 --gid paperless --home-dir /usr/src/paperless paperless \ + && usermod -s /bin/bash paperless \ + && echo "Creating volume directories" \ + && mkdir --parents --verbose /usr/src/paperless/data \ + && mkdir --parents --verbose /usr/src/paperless/media \ + && mkdir --parents --verbose /usr/src/paperless/consume \ + && mkdir --parents --verbose /usr/src/paperless/export \ + && mkdir --parents --verbose /usr/src/paperless/paperless/.venv \ + && echo "Adjusting all permissions" \ + && chown --from root:root --changes --recursive paperless:paperless /usr/src/paperless +# && echo "Collecting static files" \ +# && gosu paperless python3 manage.py collectstatic --clear --no-input --link \ +# && gosu paperless python3 manage.py compilemessages + +VOLUME ["/usr/src/paperless/data", \ + "/usr/src/paperless/media", \ + "/usr/src/paperless/consume", \ + "/usr/src/paperless/export"] diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..fd81ae4da --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,12 @@ +{ + "name": "Paperless Development", + "dockerComposeFile": "docker-compose.devcontainer.sqlite-tika.yml", + "service": "paperless-development", + "workspaceFolder": "/usr/src/paperless/paperless", + "postCreateCommand": "/bin/bash -c pre-commit install && pipenv install --dev", + "extensions": [ + "mhutchie.git-graph", + "ms-python.python" + ], + "remoteUser": "paperless" + } diff --git a/.devcontainer/docker-compose.devcontainer.sqlite-tika.yml b/.devcontainer/docker-compose.devcontainer.sqlite-tika.yml new file mode 100644 index 000000000..ac4a5066b --- /dev/null +++ b/.devcontainer/docker-compose.devcontainer.sqlite-tika.yml @@ -0,0 +1,91 @@ +# docker-compose file for running paperless from the docker container registry. +# This file contains everything paperless needs to run. +# Paperless supports amd64, arm and arm64 hardware. +# All compose files of paperless configure paperless in the following way: +# +# - Paperless is (re)started on system boot, if it was running before shutdown. +# - Docker volumes for storing data are managed by Docker. +# - Folders for importing and exporting files are created in the same directory +# as this file and mounted to the correct folders inside the container. +# - Paperless listens on port 8000. +# +# SQLite is used as the database. The SQLite file is stored in the data volume. +# +# In addition to that, this docker-compose file adds the following optional +# configurations: +# +# - Apache Tika and Gotenberg servers are started with paperless and paperless +# is configured to use these services. These provide support for consuming +# Office documents (Word, Excel, Power Point and their LibreOffice counter- +# parts. +# +# To install and update paperless with this file, do the following: +# +# - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env' +# and '.env' into a folder. +# - Run 'docker-compose pull'. +# - Run 'docker-compose run --rm webserver createsuperuser' to create a user. +# - Run 'docker-compose up -d'. +# +# For more extensive installation and update instructions, refer to the +# documentation. + +version: "3.4" +services: + broker: + image: docker.io/library/redis:7 + restart: unless-stopped + volumes: + - redisdata:/data + + # No ports need to be exposed, the vscode devcontainer plugin manages them. + paperless-development: + image: paperless-ngx + build: + context: ../ # Dockerfile can not access files from parent directories if context is not set. + dockerfile: ./.devcontainer/Dockerfile + restart: unless-stopped + depends_on: + - broker + - gotenberg + - tika + volumes: + - ..:/usr/src/paperless/paperless:cached + - ../.devcontainer/vscode:/usr/src/paperless/paperless/.vscode # vscode config files + - pipenv:/usr/src/paperless/paperless/.venv # pipenv environment persisted in volume + - /usr/src/paperless/paperless/static/frontend # static frontend files exist only in container + - data:/usr/src/paperless/data + - media:/usr/src/paperless/media + - ./export:/usr/src/paperless/export + - ./consume:/usr/src/paperless/consume + environment: + PAPERLESS_REDIS: redis://broker:6379 + PAPERLESS_TIKA_ENABLED: 1 + PAPERLESS_TIKA_GOTENBERG_ENDPOINT: http://gotenberg:3000 + PAPERLESS_TIKA_ENDPOINT: http://tika:9998 + PAPERLESS_STATICDIR: ./src/documents/static + PAPERLESS_DEBUG: true + + # Overrides default command so things don't shut down after the process ends. + command: /bin/sh -c "while sleep 1000; do :; done" + + gotenberg: + image: docker.io/gotenberg/gotenberg:7.8 + restart: unless-stopped + + # The gotenberg chromium route is used to convert .eml files. We do not + # want to allow external content like tracking pixels or even javascript. + command: + - "gotenberg" + - "--chromium-disable-javascript=true" + - "--chromium-allow-list=file:///tmp/.*" + + tika: + image: ghcr.io/paperless-ngx/tika:latest + restart: unless-stopped + +volumes: + data: + media: + redisdata: + pipenv: diff --git a/.devcontainer/vscode/launch.json b/.devcontainer/vscode/launch.json new file mode 100644 index 000000000..431c13d32 --- /dev/null +++ b/.devcontainer/vscode/launch.json @@ -0,0 +1,15 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "paperless backend", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/src/manage.py", + "console": "integratedTerminal", + "justMyCode": true, + "args": ["runserver"], + "django": true + } + ] +} diff --git a/.devcontainer/vscode/settings.json b/.devcontainer/vscode/settings.json new file mode 100644 index 000000000..86c718cad --- /dev/null +++ b/.devcontainer/vscode/settings.json @@ -0,0 +1,11 @@ +{ + "python.testing.pytestArgs": [ + "src" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "files.watcherExclude": { + "**/.venv/**": true, + "**/pytest_cache/**": true + } +} diff --git a/.devcontainer/vscode/tasks.json b/.devcontainer/vscode/tasks.json new file mode 100644 index 000000000..696ee28c2 --- /dev/null +++ b/.devcontainer/vscode/tasks.json @@ -0,0 +1,81 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Document Consumer", + "type": "shell", + "command": "pipenv run python manage.py document_consumer", + "group": "build", + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "shared", + "showReuseMessage": false, + "clear": true, + "revealProblems": "onProblem" + }, + "options": { + "cwd": "${workspaceFolder}/src" + } + + }, + { + "label": "Maintenance: Migrate", + "type": "shell", + "command": "pipenv run python manage.py migrate", + "group": "none", + "presentation": { + "echo": true, + "reveal": "always", + "focus": true, + "panel": "shared", + "showReuseMessage": false, + "clear": true, + "revealProblems": "onProblem" + }, + "options": { + "cwd": "${workspaceFolder}/src" + } + }, + { + "label": "Maintenance: createsuperuser", + "type": "shell", + "command": "pipenv run python manage.py createsuperuser", + "group": "none", + "presentation": { + "echo": true, + "reveal": "always", + "focus": true, + "panel": "shared", + "showReuseMessage": false, + "clear": true, + "revealProblems": "onProblem" + }, + "options": { + "cwd": "${workspaceFolder}/src" + } + }, + { + "label": "Celery Worker", + "type": "shell", + "command": "pipenv run celery --app paperless worker -l DEBUG", + "group": { + "kind": "build", + "isDefault": true + }, + "presentation": { + "echo": true, + "reveal": "always", + "focus": true, + "panel": "shared", + "showReuseMessage": false, + "clear": true, + "revealProblems": "onProblem" + }, + "options": { + "cwd": "${workspaceFolder}/src" + } + } + ] + } diff --git a/.gitignore b/.gitignore index 3ac5cc0dc..cc5630dcd 100644 --- a/.gitignore +++ b/.gitignore @@ -65,6 +65,8 @@ target/ .vscode /src-ui/.vscode /docs/.vscode +.vscode-server +*CommandMarker # Other stuff that doesn't belong .virtualenv