Merge pull request #902 from paperless-ngx/v1.7.1

v1.7.1

.build-config.json

@@ -0,0 +1,9 @@
+{
+  "qpdf": {
+      "version": "10.6.3"
+    },
+  "jbig2enc": {
+      "version": "0.29",
+      "git_tag": "0.29"
+    }
+}

.editorconfig

@@ -33,5 +33,5 @@ indent_style = space
 [**/test_*.py]
 max_line_length = off
 
-[Dockerfile]
+[Dockerfile*]
 indent_style = space

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -0,0 +1,86 @@
+name: Bug report
+description: Something is not working
+title: "[BUG] Concise description of the issue"
+labels: ["bug", "unconfirmed"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Have a question? 👉 [Start a new discussion](https://github.com/paperless-ngx/paperless-ngx/discussions/new) or [ask in chat](https://matrix.to/#/#paperless:adnidor.de).
+
+        Before opening an issue, please double check:
+
+        - [The troubleshooting documentation](https://paperless-ngx.readthedocs.io/en/latest/troubleshooting.html).
+        - [The installation instructions](https://paperless-ngx.readthedocs.io/en/latest/setup.html#installation).
+        - [Existing issues and discussions](https://github.com/paperless-ngx/paperless-ngx/search?q=&type=issues).
+
+        If you encounter issues while installing or configuring Paperless-ngx, please post in the ["Support" section of the discussions](https://github.com/paperless-ngx/paperless-ngx/discussions/new?category=support).
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: A clear and concise description of what the bug is. If applicable, add screenshots to help explain your problem.
+      placeholder: |
+        Currently Paperless does not work when...
+
+        [Screenshot if applicable]
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Steps to reproduce
+      description: Steps to reproduce the behavior.
+      placeholder: |
+        1. Go to '...'
+        2. Click on '....'
+        3. See error
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Webserver logs
+      description: If available, post any logs from the web server related to your issue.
+      render: bash
+  - type: input
+    id: version
+    attributes:
+      label: Paperless-ngx version
+      placeholder: e.g. 1.6.0
+    validations:
+      required: true
+  - type: input
+    id: host-os
+    attributes:
+      label: Host OS
+      description: Host OS of the machine running paperless-ngx. Please add the architecture (uname -m) if applicable.
+      placeholder: e.g. Archlinux / Ubuntu 20.04 / Raspberry Pi `arm64`
+    validations:
+      required: true
+  - type: dropdown
+    id: install-method
+    attributes:
+      label: Installation method
+      options:
+        - Docker
+        - Bare metal
+        - Other (please describe above)
+    validations:
+      required: true
+  - type: input
+    id: browser
+    attributes:
+      label: Browser
+      description: Which browser you are using, if relevant.
+      placeholder: e.g. Chrome, Safari
+  - type: input
+    id: config-changes
+    attributes:
+      label: Configuration changes
+      description: Any configuration changes you made in `docker-compose.yml`, `docker-compose.env` or `paperless.conf`.
+  - type: input
+    id: other
+    attributes:
+      label: Other
+      description: Any other relevant details.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,50 +0,0 @@
----
-name: Bug report
-about: Something is not working
-title: '[BUG] Concise description of the issue'
-labels: ''
-assignees: ''
----
-
-<!---
-=> Before opening an issue, please check the documentation and see if it helps you resolve your issue: https://paperless-ngx.readthedocs.io/en/latest/troubleshooting.html
-=> Please also make sure that you followed the installation instructions.
-=> Please search the issues and look for similar issues before opening a bug report.
-
-=> If you would like to submit a feature request please submit one under https://github.com/paperless-ngx/paperless-ngx/discussions/categories/feature-requests
-
-=> If you encounter issues while installing of configuring Paperless-ngx, please post that in the "Support" section of the discussions. Remember that Paperless successfully runs on a variety of different systems. If paperless does not start, it's probably an issue with your system, and not an issue of paperless.
-
-=> Don't remove the [BUG] prefix from the title.
--->
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**To Reproduce**
-Steps to reproduce the behavior:
-
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-**Webserver logs**
-
-```
-If available, post any logs from the web server related to your issue.
-```
-
-**Relevant information**
-
-- Host OS of the machine running paperless: [e.g. Archlinux / Ubuntu 20.04]
-- Browser [e.g. chrome, safari]
-- Version [e.g. 1.0.0]
-- Installation method: [docker / bare metal]
-- Any configuration changes you made in `docker-compose.yml`, `docker-compose.env` or `paperless.conf`.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,11 @@
+blank_issues_enabled: false
+contact_links:
+  - name: 🤔 Questions and Help
+    url: https://github.com/paperless-ngx/paperless-ngx/discussions
+    about: This issue tracker is not for support questions. Please refer to our Discussions.
+  - name: 💬 Chat
+    url: https://matrix.to/#/#paperless:adnidor.de
+    about: Want to discuss Paperless-ngx with others? Check out our chat.
+  - name: 🚀 Feature Request
+    url: https://github.com/paperless-ngx/paperless-ngx/discussions/new?category=feature-requests
+    about: Remember to search for existing feature requests and "up-vote" any you like

.github/ISSUE_TEMPLATE/other.md

@@ -1,19 +0,0 @@
----
-name: Other
-about: Anything that is not a feature request or bug.
-title: '[Other] Title of your issue'
-labels: ''
-assignees: ''
----
-
-<!--
-
-=> Discussions, Feedback and other suggestions belong in the "Discussion" section and not on the issue tracker.
-
-=> If you would like to submit a feature request please submit one under https://github.com/paperless-ngx/paperless-ngx/discussions/categories/feature-requests
-
-=> If you encounter issues while installing of configuring Paperless-ngx, please post that in the "Support" section of the discussions. Remember that Paperless successfully runs on a variety of different systems. If paperless does not start, it's probably is an issue with your system, and not an issue of paperless.
-
-=> Don't remove the [Other] prefix from the title.
-
--->

.github/PULL_REQUEST_TEMPLATE.md

@@ -5,19 +5,11 @@ Note: All PRs with code changes should be targeted to the `dev` branch, pure doc
 ## Proposed change
 
 <!--
-Please include a summary of the change and which issue is fixed (if any) and any relevant motivation / context. List any dependencies that are required for this change. If appropriate, please include an explanation of how your poposed change can be tested. Screenshots and / or videos can also be helpful if appropriate.
+Please include a summary of the change and which issue is fixed (if any) and any relevant motivation / context. List any dependencies that are required for this change. If appropriate, please include an explanation of how your proposed change can be tested. Screenshots and / or videos can also be helpful if appropriate.
 -->
 
 Fixes # (issue)
 
-<!--
-Please also tag the relevant team to help with review. You can tag any of the following:
-@paperless-ngx/backend (Python / django, database, etc.)
-@paperless-ngx/frontend (JavaScript/Typescript, HTML, CSS, etc.)
-@paperless-ngx/ci-cd (GitHub Actions, deployment)
-@paperless-ngx/test (General testing for larger PRs)
--->
-
 ## Type of change
 
 <!--

.github/dependabot.yml

@@ -6,11 +6,14 @@ updates:
   # Enable version updates for npm
   - package-ecosystem: "npm"
     target-branch: "dev"
-    # Look for `package.json` and `lock` files in the `root` directory
+    # Look for `package.json` and `lock` files in the `/src-ui` directory
     directory: "/src-ui"
     # Check the npm registry for updates every month
     schedule:
       interval: "monthly"
+    labels:
+      - "frontend"
+      - "dependencies"
     # Add reviewers
     reviewers:
       - "paperless-ngx/frontend"
@@ -26,9 +29,13 @@ updates:
     labels:
       - "backend"
       - "dependencies"
+    # Add reviewers
+    reviewers:
+      - "paperless-ngx/backend"
 
   # Enable updates for Github Actions
   - package-ecosystem: "github-actions"
+    target-branch: "dev"
     directory: "/"
     schedule:
       # Check for updates to GitHub Actions every month
@@ -38,4 +45,4 @@ updates:
       - "dependencies"
     # Add reviewers
     reviewers:
-      - "paperless-ngx/backend"
+      - "paperless-ngx/ci-cd"

.github/release-drafter.yml

@@ -0,0 +1,36 @@
+categories:
+  - title: 'Breaking Changes'
+    labels:
+      - 'breaking-change'
+  - title: 'Features'
+    labels:
+      - 'enhancement'
+  - title: 'Bug Fixes'
+    labels:
+      - 'bug'
+  - title: 'Documentation'
+    label: 'documentation'
+  - title: 'Maintenance'
+    labels:
+      - 'chore'
+      - 'deployment'
+      - 'translation'
+  - title: 'Dependencies'
+    collapse-after: 3
+    label: 'dependencies'
+include-labels:
+  - 'enhancement'
+  - 'bug'
+  - 'chore'
+  - 'deployment'
+  - 'translation'
+  - 'dependencies'
+replacers: # Changes "Feature: Update checker" to "Update checker"
+  - search: '/Feature:|Feat:|\[feature\]/gi'
+    replace: ''
+change-template: '- $TITLE @$AUTHOR (#$NUMBER)'
+change-title-escapes: '\<*_&#@'
+template: |
+  # Changelog
+
+  $CHANGES

.github/scripts/common.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+
+
+def get_image_tag(
+    repo_name: str,
+    pkg_name: str,
+    pkg_version: str,
+) -> str:
+    """
+    Returns a string representing the normal image for a given package
+    """
+    return f"ghcr.io/{repo_name}/builder/{pkg_name}:{pkg_version}"
+
+
+def get_cache_image_tag(
+    repo_name: str,
+    pkg_name: str,
+    pkg_version: str,
+    branch_name: str,
+) -> str:
+    """
+    Returns a string representing the expected image cache tag for a given package
+
+    Registry type caching is utilized for the builder images, to allow fast
+    rebuilds, generally almost instant for the same version
+    """
+    return f"ghcr.io/{repo_name}/builder/cache/{pkg_name}:{pkg_version}"

.github/scripts/get-build-json.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+"""
+This is a helper script for the mutli-stage Docker image builder.
+It provides a single point of configuration for package version control.
+The output JSON object is used by the CI workflow to determine what versions
+to build and pull into the final Docker image.
+
+Python package information is obtained from the Pipfile.lock.  As this is
+kept updated by dependabot, it usually will need no further configuration.
+The sole exception currently is pikepdf, which has a dependency on qpdf,
+and is configured here to use the latest version of qpdf built by the workflow.
+
+Other package version information is configured directly below, generally by
+setting the version and Git information, if any.
+
+"""
+import argparse
+import json
+import os
+from pathlib import Path
+from typing import Final
+
+from common import get_cache_image_tag
+from common import get_image_tag
+
+
+def _main():
+    parser = argparse.ArgumentParser(
+        description="Generate a JSON object of information required to build the given package, based on the Pipfile.lock",
+    )
+    parser.add_argument(
+        "package",
+        help="The name of the package to generate JSON for",
+    )
+
+    PIPFILE_LOCK_PATH: Final[Path] = Path("Pipfile.lock")
+    BUILD_CONFIG_PATH: Final[Path] = Path(".build-config.json")
+
+    # Read the main config file
+    build_json: Final = json.loads(BUILD_CONFIG_PATH.read_text())
+
+    # Read Pipfile.lock file
+    pipfile_data: Final = json.loads(PIPFILE_LOCK_PATH.read_text())
+
+    args: Final = parser.parse_args()
+
+    # Read from environment variables set by GitHub Actions
+    repo_name: Final[str] = os.environ["GITHUB_REPOSITORY"]
+    branch_name: Final[str] = os.environ["GITHUB_REF_NAME"]
+
+    # Default output values
+    version = None
+    git_tag = None
+    extra_config = {}
+
+    if args.package in pipfile_data["default"]:
+        # Read the version from Pipfile.lock
+        pkg_data = pipfile_data["default"][args.package]
+        pkg_version = pkg_data["version"].split("==")[-1]
+        version = pkg_version
+
+        # Based on the package, generate the expected Git tag name
+        if args.package == "pikepdf":
+            git_tag = f"v{pkg_version}"
+        elif args.package == "psycopg2":
+            git_tag = pkg_version.replace(".", "_")
+
+        # Any extra/special values needed
+        if args.package == "pikepdf":
+            extra_config["qpdf_version"] = build_json["qpdf"]["version"]
+
+    elif args.package in build_json:
+        version = build_json[args.package]["version"]
+
+        if "git_tag" in build_json[args.package]:
+            git_tag = build_json[args.package]["git_tag"]
+    else:
+        raise NotImplementedError(args.package)
+
+    # The JSON object we'll output
+    output = {
+        "name": args.package,
+        "version": version,
+        "git_tag": git_tag,
+        "image_tag": get_image_tag(repo_name, args.package, version),
+        "cache_tag": get_cache_image_tag(
+            repo_name,
+            args.package,
+            version,
+            branch_name,
+        ),
+    }
+
+    # Add anything special a package may need
+    output.update(extra_config)
+
+    # Output the JSON info to stdout
+    print(json.dumps(output))
+
+
+if __name__ == "__main__":
+    _main()

.github/stale.yml

@@ -2,11 +2,8 @@
 daysUntilStale: 30
 # Number of days of inactivity before a stale issue is closed
 daysUntilClose: 7
-# Issues with these labels will never be considered stale
-exemptLabels:
-  - pinned
-  - security
-  - fixpending
+onlyLabels:
+  - unconfirmed
 # Label to use when marking an issue as stale
 staleLabel: stale
 # Comment to post when marking an issue as stale. Set to `false` to disable

.github/workflows/ci.yml

@@ -3,8 +3,10 @@ name: ci
 on:
   push:
     tags:
-      - ngx-*
-      - beta-*
+      # https://semver.org/#spec-item-2
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+      # https://semver.org/#spec-item-9
+      - 'v[0-9]+.[0-9]+.[0-9]+-beta.rc[0-9]+'
     branches-ignore:
       - 'translations**'
   pull_request:
@@ -45,160 +47,164 @@ jobs:
           name: documentation
           path: docs/_build/html/
 
-  code-checks-backend:
-    name: "Backend Code Checks"
-    runs-on: ubuntu-20.04
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      -
-        name: Install checkers
-        run: |
-          pipx install reorder-python-imports
-          pipx install yesqa
-          pipx install add-trailing-comma
-          pipx install flake8
-      -
-        name: Run reorder-python-imports
-        run: |
-          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs reorder-python-imports
-      -
-        name: Run yesqa
-        run: |
-          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs yesqa
-      -
-        name: Run add-trailing-comma
-        run: |
-          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs add-trailing-comma
-      # black is placed after add-trailing-comma because it may format differently
-      # if a trailing comma is added
-      -
-        name: Run black
-        uses: psf/black@stable
-        with:
-          options: "--check --diff"
-          version: "22.3.0"
-      -
-        name: Run flake8 checks
-        run: |
-          cd src/
-          flake8 --max-line-length=88 --ignore=E203,W503
+  ci-backend:
+    uses: ./.github/workflows/reusable-ci-backend.yml
 
-  code-checks-frontend:
-    name: "Frontend Code Checks"
-    runs-on: ubuntu-20.04
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-        with:
-          node-version: '16'
-      -
-        name: Install prettier
-        run: |
-          npm install prettier
-      -
-        name: Run prettier
-        run:
-          npx prettier --check --ignore-path Pipfile.lock **/*.js **/*.ts *.md **/*.md
+  ci-frontend:
+    uses: ./.github/workflows/reusable-ci-frontend.yml
 
-  tests-backend:
-    needs: [code-checks-backend]
-    name: "Backend Tests (${{ matrix.python-version }})"
+  prepare-docker-build:
+    name: Prepare Docker Pipeline Data
+    if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v'))
     runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version: ['3.8', '3.9']
-      fail-fast: false
+    needs:
+      - documentation
+      - ci-backend
+      - ci-frontend
     steps:
       -
         name: Checkout
         uses: actions/checkout@v3
-        with:
-          fetch-depth: 2
-      -
-        name: Install pipenv
-        run: pipx install pipenv
       -
         name: Set up Python
         uses: actions/setup-python@v3
         with:
-          python-version: "${{ matrix.python-version }}"
-          cache: "pipenv"
-          cache-dependency-path: 'Pipfile.lock'
+          python-version: "3.9"
       -
-        name: Install system dependencies
+        name: Setup qpdf image
+        id: qpdf-setup
         run: |
-          sudo apt-get update -qq
-          sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript optipng
-      -
-        name: Install Python dependencies
-        run: |
-          pipenv sync --dev
-      -
-        name: Tests
-        run: |
-          cd src/
-          pipenv run pytest
-      -
-        name: Get changed files
-        id: changed-files-specific
-        uses: tj-actions/changed-files@v18.1
-        with:
-          files: |
-            src/**
-      -
-        name: List all changed files
-        run: |
-          for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
-            echo "${file} was changed"
-          done
-      -
-        name: Publish coverage results
-        if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        # https://github.com/coveralls-clients/coveralls-python/issues/251
-        run: |
-          cd src/
-          pipenv run coveralls --service=github
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py qpdf)
 
-  tests-frontend:
-    needs: [code-checks-frontend]
-    name: "Frontend Tests"
-    runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        node-version: [16.x]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+          echo ${build_json}
+
+          echo ::set-output name=qpdf-json::${build_json}
+      -
+        name: Setup psycopg2 image
+        id: psycopg2-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py psycopg2)
+
+          echo ${build_json}
+
+          echo ::set-output name=psycopg2-json::${build_json}
+      -
+        name: Setup pikepdf image
+        id: pikepdf-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py pikepdf)
+
+          echo ${build_json}
+
+          echo ::set-output name=pikepdf-json::${build_json}
+      -
+        name: Setup jbig2enc image
+        id: jbig2enc-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py jbig2enc)
+
+          echo ${build_json}
+
+          echo ::set-output name=jbig2enc-json::${build_json}
+
+    outputs:
+
+      qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }}
+
+      pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }}
+
+      psycopg2-json: ${{ steps.psycopg2-setup.outputs.psycopg2-json }}
+
+      jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}}
+
+  build-qpdf-debs:
+    name: qpdf
+    needs:
+      - prepare-docker-build
+    uses: ./.github/workflows/reusable-workflow-builder.yml
     with:
-          node-version: ${{ matrix.node-version }}
-      - run: cd src-ui && npm ci
-      - run: cd src-ui && npm run test
-      - run: cd src-ui && npm run e2e:ci
+      dockerfile: ./docker-builders/Dockerfile.qpdf
+      build-json: ${{ needs.prepare-docker-build.outputs.qpdf-json }}
+      build-args: |
+        QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
+
+  build-jbig2enc:
+    name: jbig2enc
+    needs:
+      - prepare-docker-build
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.jbig2enc
+      build-json: ${{ needs.prepare-docker-build.outputs.jbig2enc-json }}
+      build-args: |
+        JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
+
+  build-psycopg2-wheel:
+    name: psycopg2
+    needs:
+      - prepare-docker-build
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.psycopg2
+      build-json: ${{ needs.prepare-docker-build.outputs.psycopg2-json }}
+      build-args: |
+        PSYCOPG2_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).git_tag }}
+        PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
+
+  build-pikepdf-wheel:
+    name: pikepdf
+    needs:
+      - prepare-docker-build
+      - build-qpdf-debs
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.pikepdf
+      build-json: ${{ needs.prepare-docker-build.outputs.pikepdf-json }}
+      build-args: |
+        REPO=${{ github.repository }}
+        QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
+        PIKEPDF_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).git_tag }}
+        PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
 
   # build and push image to docker hub.
   build-docker-image:
-    if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || startsWith(github.ref, 'refs/tags/ngx-') || startsWith(github.ref, 'refs/tags/beta-'))
     runs-on: ubuntu-20.04
-    needs: [tests-backend, tests-frontend]
+    concurrency:
+      group: ${{ github.workflow }}-build-docker-image-${{ github.ref_name }}
+      cancel-in-progress: true
+    needs:
+      - prepare-docker-build
+      - build-psycopg2-wheel
+      - build-jbig2enc
+      - build-qpdf-debs
+      - build-pikepdf-wheel
     steps:
+      -
+        name: Check pushing to Docker Hub
+        id: docker-hub
+        # Only push to Dockerhub from the main repo
+        # Otherwise forks would require a Docker Hub account and secrets setup
+        run: |
+          if [[ ${{ github.repository }} == "paperless-ngx/paperless-ngx" ]] ; then
+            echo ::set-output name=enable::"true"
+          else
+            echo ::set-output name=enable::"false"
+          fi
       -
         name: Gather Docker metadata
         id: docker-meta
         uses: docker/metadata-action@v3
         with:
-          images: ghcr.io/${{ github.repository }}
+          images: |
+            ghcr.io/${{ github.repository }}
+            name=paperlessngx/paperless-ngx,enable=${{ steps.docker-hub.outputs.enable }}
           tags: |
-            type=match,pattern=ngx-(\d.\d.\d),group=1
-            type=semver,pattern=ngx-{{version}}
-            type=semver,pattern=ngx-{{major}}.{{minor}}
+            # Tag branches with branch name
             type=ref,event=branch
+            # Process semver tags
+            # For a tag x.y.z or vX.Y.Z, output an x.y.z and x.y image tag
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
       -
         name: Checkout
         uses: actions/checkout@v3
@@ -215,6 +221,14 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Login to Docker Hub
+        uses: docker/login-action@v1
+        # Don't attempt to login is not pushing to Docker Hub
+        if: steps.docker-hub.outputs.enable == 'true'
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
       -
         name: Build and push
         uses: docker/build-push-action@v2
@@ -225,8 +239,19 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.docker-meta.outputs.tags }}
           labels: ${{ steps.docker-meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          build-args: |
+            JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
+            QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
+            PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
+            PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
+          # Get cache layers from this branch, then dev, then main
+          # This allows new branches to get at least some cache benefits, generally from dev
+          cache-from: |
+            type=registry,ref=ghcr.io/${{ github.repository }}/builder/cache/app:${{ github.ref_name }}
+            type=registry,ref=ghcr.io/${{ github.repository }}/builder/cache/app:dev
+            type=registry,ref=ghcr.io/${{ github.repository }}/builder/cache/app:main
+          cache-to: |
+            type=registry,mode=max,ref=ghcr.io/${{ github.repository }}/builder/cache/app:${{ github.ref_name }}
       -
         name: Inspect image
         run: |
@@ -244,7 +269,8 @@ jobs:
           path: src/documents/static/frontend/
 
   build-release:
-    needs: [build-docker-image, documentation]
+    needs:
+      - build-docker-image
     runs-on: ubuntu-20.04
     steps:
       -
@@ -312,8 +338,9 @@ jobs:
 
   publish-release:
     runs-on: ubuntu-20.04
-    needs: build-release
-    if: contains(github.ref, 'refs/tags/ngx-') || contains(github.ref, 'refs/tags/beta-')
+    needs:
+      - build-release
+    if: github.ref_type == 'tag' && (startsWith(github.ref_name, 'v') || contains(github.ref_name, '-beta.rc'))
     steps:
       -
         name: Download release artifact
@@ -325,27 +352,24 @@ jobs:
         name: Get version
         id: get_version
         run: |
-          if [[ $GITHUB_REF == refs/tags/ngx-* ]]; then
-            echo ::set-output name=version::${GITHUB_REF#refs/tags/ngx-}
-            echo ::set-output name=prerelease::false
-            echo ::set-output name=body::"For a complete list of changes, see the changelog at https://paperless-ngx.readthedocs.io/en/latest/changelog.html"
-          elif [[ $GITHUB_REF == refs/tags/beta-* ]]; then
-            echo ::set-output name=version::${GITHUB_REF#refs/tags/beta-}
+          echo ::set-output name=version::${{ github.ref_name }}
+          if [[ ${{ contains(github.ref_name, '-beta.rc') }} == 'true' ]]; then
             echo ::set-output name=prerelease::true
-            echo ::set-output name=body::"For a complete list of changes, see the changelog at https://github.com/paperless-ngx/paperless-ngx/blob/beta/docs/changelog.rst"
+          else
+            echo ::set-output name=prerelease::false
           fi
       -
-        name: Create release
-        id: create_release
-        uses: actions/create-release@v1
+        name: Create Release and Changelog
+        id: create-release
+        uses: release-drafter/release-drafter@v5
+        with:
+          name: Paperless-ngx ${{ steps.get_version.outputs.version }}
+          tag: ${{ steps.get_version.outputs.version }}
+          version: ${{ steps.get_version.outputs.version }}
+          prerelease: ${{ steps.get_version.outputs.prerelease }}
+          publish: true # ensures release is not marked as draft
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          tag_name: ngx-${{ steps.get_version.outputs.version }}
-          release_name: Paperless-ngx ${{ steps.get_version.outputs.version }}
-          draft: false
-          prerelease: ${{ steps.get_version.outputs.prerelease }}
-          body: ${{ steps.get_version.outputs.body }}
       -
         name: Upload release archive
         id: upload-release-asset
@@ -353,7 +377,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
+          upload_url: ${{ steps.create-release.outputs.upload_url }}
           asset_path: ./paperless-ngx.tar.xz
           asset_name: paperless-ngx-${{ steps.get_version.outputs.version }}.tar.xz
           asset_content_type: application/x-xz

.github/workflows/codeql-analysis.yml

@@ -42,7 +42,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -51,4 +51,4 @@ jobs:
         # queries: ./path/to/local/query, your-org/your-repo/queries@main
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

.github/workflows/reusable-ci-backend.yml

@@ -0,0 +1,108 @@
+name: Backend CI Jobs
+
+on:
+  workflow_call:
+
+jobs:
+
+  code-checks-backend:
+    name: "Code Style Checks"
+    runs-on: ubuntu-20.04
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Install checkers
+        run: |
+          pipx install reorder-python-imports
+          pipx install yesqa
+          pipx install add-trailing-comma
+          pipx install flake8
+      -
+        name: Run reorder-python-imports
+        run: |
+          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs reorder-python-imports
+      -
+        name: Run yesqa
+        run: |
+          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs yesqa
+      -
+        name: Run add-trailing-comma
+        run: |
+          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs add-trailing-comma
+      # black is placed after add-trailing-comma because it may format differently
+      # if a trailing comma is added
+      -
+        name: Run black
+        uses: psf/black@stable
+        with:
+          options: "--check --diff"
+          version: "22.3.0"
+      -
+        name: Run flake8 checks
+        run: |
+          cd src/
+          flake8 --max-line-length=88 --ignore=E203,W503
+
+  tests-backend:
+    name: "Tests (${{ matrix.python-version }})"
+    runs-on: ubuntu-20.04
+    needs:
+      - code-checks-backend
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.9', '3.10']
+      fail-fast: false
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      -
+        name: Install pipenv
+        run: pipx install pipenv
+      -
+        name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: "${{ matrix.python-version }}"
+          cache: "pipenv"
+          cache-dependency-path: 'Pipfile.lock'
+      -
+        name: Install system dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript optipng libzbar0 poppler-utils
+      -
+        name: Install Python dependencies
+        run: |
+          pipenv sync --dev
+      -
+        name: Tests
+        run: |
+          cd src/
+          pipenv run pytest
+      -
+        name: Get changed files
+        id: changed-files-specific
+        uses: tj-actions/changed-files@v19
+        with:
+          files: |
+            src/**
+      -
+        name: List all changed files
+        run: |
+          for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
+            echo "${file} was changed"
+          done
+      -
+        name: Publish coverage results
+        if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        # https://github.com/coveralls-clients/coveralls-python/issues/251
+        run: |
+          cd src/
+          pipenv run coveralls --service=github

.github/workflows/reusable-ci-frontend.yml

@@ -0,0 +1,42 @@
+name: Frontend CI Jobs
+
+on:
+  workflow_call:
+
+jobs:
+
+  code-checks-frontend:
+    name: "Code Style Checks"
+    runs-on: ubuntu-20.04
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '16'
+      -
+        name: Install prettier
+        run: |
+          npm install prettier
+      -
+        name: Run prettier
+        run:
+          npx prettier --check --ignore-path Pipfile.lock **/*.js **/*.ts *.md **/*.md
+  tests-frontend:
+    name: "Tests"
+    runs-on: ubuntu-20.04
+    needs:
+      - code-checks-frontend
+    strategy:
+      matrix:
+        node-version: [16.x]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+      - run: cd src-ui && npm ci
+      - run: cd src-ui && npm run test
+      - run: cd src-ui && npm run e2e:ci

.github/workflows/reusable-workflow-builder.yml

@@ -0,0 +1,53 @@
+name: Reusable Image Builder
+
+on:
+  workflow_call:
+    inputs:
+      dockerfile:
+        required: true
+        type: string
+      build-json:
+        required: true
+        type: string
+      build-args:
+        required: false
+        default: ""
+        type: string
+
+concurrency:
+  group: ${{ github.workflow }}-${{ fromJSON(inputs.build-json).name }}-${{ fromJSON(inputs.build-json).version }}
+  cancel-in-progress: false
+
+jobs:
+  build-image:
+    name: Build ${{ fromJSON(inputs.build-json).name }} @ ${{ fromJSON(inputs.build-json).version }}
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Login to Github Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Build ${{ fromJSON(inputs.build-json).name }}
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ${{ inputs.dockerfile }}
+          tags: ${{ fromJSON(inputs.build-json).image_tag }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          build-args: ${{ inputs.build-args }}
+          push: true
+          cache-from: type=registry,ref=${{ fromJSON(inputs.build-json).cache_tag }}
+          cache-to: type=registry,mode=max,ref=${{ fromJSON(inputs.build-json).cache_tag }}

.pre-commit-config.yaml

@@ -5,7 +5,7 @@
 repos:
   # General hooks
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.1.0
+    rev: v4.2.0
     hooks:
       - id: check-docstring-first
       - id: check-json
@@ -27,7 +27,7 @@ repos:
       - id: check-case-conflict
       - id: detect-private-key
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: "v2.6.1"
+    rev: "v2.6.2"
     hooks:
       - id: prettier
         types_or:
@@ -37,7 +37,7 @@ repos:
         exclude: "(^Pipfile\\.lock$)"
   # Python hooks
   - repo: https://github.com/asottile/reorder_python_imports
-    rev: v3.0.1
+    rev: v3.1.0
     hooks:
       - id: reorder-python-imports
         exclude: "(migrations)"
@@ -47,7 +47,7 @@ repos:
       - id: yesqa
         exclude: "(migrations)"
   - repo: https://github.com/asottile/add-trailing-comma
-    rev: "v2.2.1"
+    rev: "v2.2.3"
     hooks:
       - id: add-trailing-comma
         exclude: "(migrations)"
@@ -62,11 +62,25 @@ repos:
     rev: 22.3.0
     hooks:
       - id: black
-  # Dockerfile hooks
-  - repo: https://github.com/pryorda/dockerfilelint-precommit-hooks
-    rev: "v0.1.0"
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v2.32.1
     hooks:
-      - id: dockerfilelint
+      - id: pyupgrade
+        exclude: "(migrations)"
+        args:
+          - "--py38-plus"
+  # Dockerfile hooks
+  - repo: https://github.com/AleksaC/hadolint-py
+    rev: v2.10.0
+    hooks:
+      - id: hadolint
+        args:
+          - --ignore
+          - DL3008      # https://github.com/hadolint/hadolint/wiki/DL3008  (should probably do this at some point)
+          - --ignore
+          - DL3013      # https://github.com/hadolint/hadolint/wiki/DL3013  (should probably do this too at some point)
+          - --ignore
+          - DL3003      # https://github.com/hadolint/hadolint/wiki/DL3003  (seems excessive to use WORKDIR so much)
   # Shell script hooks
   - repo: https://github.com/lovesegfault/beautysh
     rev: v6.2.1

CODEOWNERS

@@ -0,0 +1,10 @@
+/.github/workflows/ @paperless-ngx/ci-cd
+/docker/ @paperless-ngx/ci-cd
+/scripts/ @paperless-ngx/ci-cd
+
+/src-ui/ @paperless-ngx/frontend
+
+/src/ @paperless-ngx/backend
+Pipfile* @paperless-ngx/backend
+*.py @paperless-ngx/backend
+requirements.txt @paperless-ngx/backend

Dockerfile

@@ -1,12 +1,36 @@
-FROM node:16 AS compile-frontend
+# syntax=docker/dockerfile:1.4
 
-COPY . /src
+# Pull the installer images from the library
+# These are all built previously
+# They provide either a .deb or .whl
+
+ARG JBIG2ENC_VERSION
+ARG QPDF_VERSION
+ARG PIKEPDF_VERSION
+ARG PSYCOPG2_VERSION
+
+FROM ghcr.io/paperless-ngx/paperless-ngx/builder/jbig2enc:${JBIG2ENC_VERSION} as jbig2enc-builder
+FROM ghcr.io/paperless-ngx/paperless-ngx/builder/qpdf:${QPDF_VERSION} as qpdf-builder
+FROM ghcr.io/paperless-ngx/paperless-ngx/builder/pikepdf:${PIKEPDF_VERSION} as pikepdf-builder
+FROM ghcr.io/paperless-ngx/paperless-ngx/builder/psycopg2:${PSYCOPG2_VERSION} as psycopg2-builder
+
+FROM --platform=$BUILDPLATFORM node:16-bullseye-slim AS compile-frontend
+
+# This stage compiles the frontend
+# This stage runs once for the native platform, as the outputs are not
+# dependent on target arch
+# Inputs: None
+
+COPY ./src-ui /src/src-ui
 
 WORKDIR /src/src-ui
-RUN npm update npm -g && npm ci --no-optional
-RUN ./node_modules/.bin/ng build --configuration production
+RUN set -eux \
+  && npm update npm -g \
+  && npm ci --no-optional
+RUN set -eux \
+  && ./node_modules/.bin/ng build --configuration production
 
-FROM ghcr.io/paperless-ngx/builder/ngx-base:1.0 as main-app
+FROM python:3.9-slim-bullseye as main-app
 
 LABEL org.opencontainers.image.authors="paperless-ngx team <hello@paperless-ngx.com>"
 LABEL org.opencontainers.image.documentation="https://paperless-ngx.readthedocs.io/en/latest/"
@@ -14,45 +38,167 @@ LABEL org.opencontainers.image.source="https://github.com/paperless-ngx/paperles
 LABEL org.opencontainers.image.url="https://github.com/paperless-ngx/paperless-ngx"
 LABEL org.opencontainers.image.licenses="GPL-3.0-only"
 
-WORKDIR /usr/src/paperless/src/
+ARG DEBIAN_FRONTEND=noninteractive
 
-COPY requirements.txt ../
+#
+# Begin installation and configuration
+# Order the steps below from least often changed to most
+#
 
-# Python dependencies
-RUN apt-get update \
-  # python-Levenshtein still needs to be compiled here
-  && apt-get -y --no-install-recommends install \
-    build-essential \
-    && python3 -m pip install --upgrade --no-cache-dir pip wheel \
-  && python3 -m pip install --default-timeout=1000 --upgrade --no-cache-dir supervisor \
-  && python3 -m pip install --default-timeout=1000 --no-cache-dir -r ../requirements.txt \
-  && apt-get -y purge build-essential \
-  && apt-get -y autoremove --purge \
-  && rm -rf /var/lib/apt/lists/*
+# copy jbig2enc
+# Basically will never change again
+COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/.libs/libjbig2enc* /usr/local/lib/
+COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/jbig2 /usr/local/bin/
+COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/*.h /usr/local/include/
+
+# Packages need for running
+ARG RUNTIME_PACKAGES="\
+  curl \
+  file \
+  # fonts for text file thumbnail generation
+  fonts-liberation \
+  gettext \
+  ghostscript \
+  gnupg \
+  gosu \
+  icc-profiles-free \
+  imagemagick \
+  media-types \
+  liblept5 \
+  libpq5 \
+  libxml2 \
+  liblcms2-2 \
+  libtiff5 \
+  libxslt1.1 \
+  libfreetype6 \
+  libwebp6 \
+  libopenjp2-7 \
+  libimagequant0 \
+  libraqm0 \
+  libgnutls30 \
+  libjpeg62-turbo \
+  optipng \
+  python3 \
+  python3-pip \
+  python3-setuptools \
+  postgresql-client \
+  # For Numpy
+  libatlas3-base \
+  # thumbnail size reduction
+  pngquant \
+  # OCRmyPDF dependencies
+  tesseract-ocr \
+  tesseract-ocr-eng \
+  tesseract-ocr-deu \
+  tesseract-ocr-fra \
+  tesseract-ocr-ita \
+  tesseract-ocr-spa \
+  tzdata \
+  unpaper \
+  # Mime type detection
+  zlib1g \
+  # Barcode splitter
+  libzbar0 \
+  poppler-utils"
+
+# Install basic runtime packages.
+# These change very infrequently
+RUN set -eux \
+  echo "Installing system packages" \
+    && apt-get update \
+    && apt-get install --yes --quiet --no-install-recommends ${RUNTIME_PACKAGES} \
+    && rm -rf /var/lib/apt/lists/* \
+  && echo "Installing supervisor" \
+    && python3 -m pip install --default-timeout=1000 --upgrade --no-cache-dir supervisor==4.2.4
+
+# Copy gunicorn config
+# Changes very infrequently
+WORKDIR /usr/src/paperless/
+
+COPY gunicorn.conf.py .
 
 # setup docker-specific things
-COPY docker/ ./docker/
+# Use mounts to avoid copying installer files into the image
+# These change sometimes, but rarely
+WORKDIR /usr/src/paperless/src/docker/
 
-RUN cd docker \
+RUN --mount=type=bind,readwrite,source=docker,target=./ \
+  set -eux \
+  && echo "Configuring ImageMagick" \
     && cp imagemagick-policy.xml /etc/ImageMagick-6/policy.xml \
+  && echo "Configuring supervisord" \
     && mkdir /var/log/supervisord /var/run/supervisord \
     && cp supervisord.conf /etc/supervisord.conf \
+  && echo "Setting up Docker scripts" \
     && cp docker-entrypoint.sh /sbin/docker-entrypoint.sh \
     && chmod 755 /sbin/docker-entrypoint.sh \
     && cp docker-prepare.sh /sbin/docker-prepare.sh \
     && chmod 755 /sbin/docker-prepare.sh \
+    && cp wait-for-redis.py /sbin/wait-for-redis.py \
+    && chmod 755 /sbin/wait-for-redis.py \
+  && echo "Installing managment commands" \
     && chmod +x install_management_commands.sh \
-  && ./install_management_commands.sh \
-  && cd .. \
-  && rm -rf docker/
+    && ./install_management_commands.sh
 
-COPY gunicorn.conf.py ../
+# Install the built packages from the installer library images
+# Use mounts to avoid copying installer files into the image
+# These change sometimes
+RUN --mount=type=bind,from=qpdf-builder,target=/qpdf \
+    --mount=type=bind,from=psycopg2-builder,target=/psycopg2 \
+    --mount=type=bind,from=pikepdf-builder,target=/pikepdf \
+  set -eux \
+  && echo "Installing qpdf" \
+    && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/libqpdf28_*.deb \
+    && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/qpdf_*.deb \
+  && echo "Installing pikepdf and dependencies" \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/packaging*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/lxml*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/Pillow*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/pyparsing*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/pikepdf*.whl \
+    && python -m pip list \
+  && echo "Installing psycopg2" \
+    && python3 -m pip install --no-cache-dir /psycopg2/usr/src/psycopg2/wheels/psycopg2*.whl \
+    && python -m pip list
 
-# copy app
-COPY --from=compile-frontend /src/src/ ./
+# Python dependencies
+# Change pretty frequently
+COPY requirements.txt ../
+
+# Packages needed only for building a few quick Python
+# dependencies
+ARG BUILD_PACKAGES="\
+  build-essential \
+  python3-dev"
+
+RUN set -eux \
+  && echo "Installing build system packages" \
+    && apt-get update \
+    && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
+    && python3 -m pip install --no-cache-dir --upgrade wheel \
+  && echo "Installing Python requirements" \
+    && python3 -m pip install --default-timeout=1000 --no-cache-dir -r ../requirements.txt \
+  && echo "Cleaning up image" \
+    && apt-get -y purge ${BUILD_PACKAGES} \
+    && apt-get -y autoremove --purge \
+    && apt-get clean --yes \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /tmp/* \
+    && rm -rf /var/tmp/* \
+    && rm -rf /var/cache/apt/archives/* \
+    && truncate -s 0 /var/log/*log
+
+WORKDIR /usr/src/paperless/src/
+
+# copy backend
+COPY ./src ./
+
+# copy frontend
+COPY --from=compile-frontend /src/src/documents/static/frontend/ ./documents/static/frontend/
 
 # add users, setup scripts
-RUN addgroup --gid 1000 paperless \
+RUN set -eux \
+  && addgroup --gid 1000 paperless \
   && useradd --uid 1000 --gid paperless --home-dir /usr/src/paperless paperless \
   && chown -R paperless:paperless ../ \
   && gosu paperless python3 manage.py collectstatic --clear --no-input \

Pipfile

@@ -19,10 +19,10 @@ djangorestframework = "~=3.13"
 filelock = "*"
 fuzzywuzzy = {extras = ["speedup"], version = "*"}
 gunicorn = "*"
-imap-tools = "*"
+imap-tools = "~=0.54.0"
 langdetect = "*"
 pathvalidate = "*"
-pillow = "~=9.0"
+pillow = "~=9.1"
 # Any version update to pikepdf requires a base image update
 pikepdf = "~=5.1"
 python-gnupg = "*"
@@ -51,6 +51,8 @@ concurrent-log-handler = "*"
 "backports.zoneinfo" = {version = "*", markers = "python_version < '3.9'"}
 "importlib-resources" = {version = "*", markers = "python_version < '3.9'"}
 zipp = {version = "*", markers = "python_version < '3.9'"}
+pyzbar = "*"
+pdf2image = "*"
 
 [dev-packages]
 coveralls = "*"
@@ -62,7 +64,7 @@ pytest-django = "*"
 pytest-env = "*"
 pytest-sugar = "*"
 pytest-xdist = "*"
-sphinx = "~=4.4.0"
+sphinx = "~=4.5.0"
 sphinx_rtd_theme = "*"
 tox = "*"
 black = "*"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "f33c0aeb0cb880b444efd367ad5b47358dde2a9b18b55dcd6943dd81f6c7e2a6"
+            "sha256": "9573af313c811561d467d814c52c6bd1439bc48e3b31d7f56afed5f0ebe4b648"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -44,11 +44,11 @@
         },
         "asgiref": {
             "hashes": [
-                "sha256:2f8abc20f7248433085eda803936d98992f1343ddb022065779f37c5da0181d0",
-                "sha256:88d59c13d634dcffe0510be048210188edd79aeccb6a6c9028cdad6f31d730a9"
+                "sha256:45a429524fba18aba9d512498b19d220c4d628e75b40cf5c627524dbaebc5cc1",
+                "sha256:fddeea3c53fa99d0cdb613c3941cc6e52d822491fc2753fba25768fb5bf4e865"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.5.0"
+            "version": "==3.5.1"
         },
         "async-timeout": {
             "hashes": [
@@ -68,10 +68,10 @@
         },
         "autobahn": {
             "hashes": [
-                "sha256:60e1f4c602aacd052ffe3d46ae40b6b75f8286b3c46922c213b523162e58c17e"
+                "sha256:58a887c7a196bb08d8b6624cb3695f493a9e5c9f00fd350d8d6f829b47ff9036"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==22.2.2"
+            "version": "==22.3.2"
         },
         "automat": {
             "hashes": [
@@ -207,11 +207,11 @@
         },
         "click": {
             "hashes": [
-                "sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1",
-                "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb"
+                "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e",
+                "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==8.0.4"
+            "markers": "python_version >= '3.7'",
+            "version": "==8.1.3"
         },
         "coloredlogs": {
             "hashes": [
@@ -238,29 +238,31 @@
         },
         "cryptography": {
             "hashes": [
-                "sha256:0a3bf09bb0b7a2c93ce7b98cb107e9170a90c51a0162a20af1c61c765b90e60b",
-                "sha256:1f64a62b3b75e4005df19d3b5235abd43fa6358d5516cfc43d87aeba8d08dd51",
-                "sha256:32db5cc49c73f39aac27574522cecd0a4bb7384e71198bc65a0d23f901e89bb7",
-                "sha256:4881d09298cd0b669bb15b9cfe6166f16fc1277b4ed0d04a22f3d6430cb30f1d",
-                "sha256:4e2dddd38a5ba733be6a025a1475a9f45e4e41139d1321f412c6b360b19070b6",
-                "sha256:53e0285b49fd0ab6e604f4c5d9c5ddd98de77018542e88366923f152dbeb3c29",
-                "sha256:70f8f4f7bb2ac9f340655cbac89d68c527af5bb4387522a8413e841e3e6628c9",
-                "sha256:7b2d54e787a884ffc6e187262823b6feb06c338084bbe80d45166a1cb1c6c5bf",
-                "sha256:7be666cc4599b415f320839e36367b273db8501127b38316f3b9f22f17a0b815",
-                "sha256:8241cac0aae90b82d6b5c443b853723bcc66963970c67e56e71a2609dc4b5eaf",
-                "sha256:82740818f2f240a5da8dfb8943b360e4f24022b093207160c77cadade47d7c85",
-                "sha256:8897b7b7ec077c819187a123174b645eb680c13df68354ed99f9b40a50898f77",
-                "sha256:c2c5250ff0d36fd58550252f54915776940e4e866f38f3a7866d92b32a654b86",
-                "sha256:ca9f686517ec2c4a4ce930207f75c00bf03d94e5063cbc00a1dc42531511b7eb",
-                "sha256:d2b3d199647468d410994dbeb8cec5816fb74feb9368aedf300af709ef507e3e",
-                "sha256:da73d095f8590ad437cd5e9faf6628a218aa7c387e1fdf67b888b47ba56a17f0",
-                "sha256:e167b6b710c7f7bc54e67ef593f8731e1f45aa35f8a8a7b72d6e42ec76afd4b3",
-                "sha256:ea634401ca02367c1567f012317502ef3437522e2fc44a3ea1844de028fa4b84",
-                "sha256:ec6597aa85ce03f3e507566b8bcdf9da2227ec86c4266bd5e6ab4d9e0cc8dab2",
-                "sha256:f64b232348ee82f13aac22856515ce0195837f6968aeaa94a3d0353ea2ec06a6"
+                "sha256:06bfafa6e53ccbfb7a94be4687b211a025ce0625e3f3c60bb15cd048a18f3ed8",
+                "sha256:0db5cf21bd7d092baacb576482b0245102cea2d3cf09f09271ce9f69624ecb6f",
+                "sha256:125702572be12bcd318e3a14e9e70acd4be69a43664a75f0397e8650fe3c6cc3",
+                "sha256:1858eff6246bb8bbc080eee78f3dd1528739e3f416cba5f9914e8631b8df9871",
+                "sha256:315af6268de72bcfa0bb3401350ce7d921f216e6b60de12a363dad128d9d459f",
+                "sha256:451aaff8b8adf2dd0597cbb1fdcfc8a7d580f33f843b7cce75307a7f20112dd8",
+                "sha256:58021d6e9b1d88b1105269d0da5e60e778b37dfc0e824efc71343dd003726831",
+                "sha256:618391152147a1221c87b1b0b7f792cafcfd4b5a685c5c72eeea2ddd29aeceff",
+                "sha256:6d4daf890e674d191757d8d7d60dc3a29c58c72c7a76a05f1c0a326013f47e8b",
+                "sha256:74b55f67f4cf026cb84da7a1b04fc2a1d260193d4ad0ea5e9897c8b74c1e76ac",
+                "sha256:7ceae26f876aabe193b13a0c36d1bb8e3e7e608d17351861b437bd882f617e9f",
+                "sha256:930b829e8a2abaf43a19f38277ae3c5e1ffcf547b936a927d2587769ae52c296",
+                "sha256:a18ff4bfa9d64914a84d7b06c46eb86e0cc03113470b3c111255aceb6dcaf81d",
+                "sha256:ae1cd29fbe6b716855454e44f4bf743465152e15d2d317303fe3b58ee9e5af7a",
+                "sha256:b1ee5c82cf03b30f6ae4e32d2bcb1e167ef74d6071cbb77c2af30f101d0b360b",
+                "sha256:bf585476fcbcd37bed08072e8e2db3954ce1bfc68087a2dc9c19cfe0b90979ca",
+                "sha256:c4a58eeafbd7409054be41a377e726a7904a17c26f45abf18125d21b1215b08b",
+                "sha256:cce90609e01e1b192fae9e13665058ab46b2ea53a3c05a3ea74a3eb8c3af8857",
+                "sha256:d610d0ee14dd9109006215c7c0de15eee91230b70a9bce2263461cf7c3720b83",
+                "sha256:e69a0e36e62279120e648e787b76d79b41e0f9e86c1c636a4f38d415595c722e",
+                "sha256:f095988548ec5095e3750cdb30e6962273d239b1998ba1aac66c0d5bee7111c1",
+                "sha256:faf0f5456c059c7b1c29441bdd5e988f0ba75bdc3eea776520d8dcb1e30e1b5c"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==36.0.2"
+            "version": "==37.0.1"
         },
         "daphne": {
             "hashes": [
@@ -280,11 +282,11 @@
         },
         "django": {
             "hashes": [
-                "sha256:1239218849e922033a35d2a2f777cb8bee18bd725416744074f455f34ff50d0c",
-                "sha256:77ff2e7050e3324c9b67e29b6707754566f58514112a9ac73310f60cd5261930"
+                "sha256:07c8638e7a7f548dc0acaaa7825d84b7bd42b10e8d22268b3d572946f1e9b687",
+                "sha256:4e8177858524417563cc0430f29ea249946d831eacb0068a1455686587df40b5"
             ],
             "index": "pypi",
-            "version": "==4.0.3"
+            "version": "==4.0.4"
         },
         "django-cors-headers": {
             "hashes": [
@@ -475,31 +477,31 @@
                 "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff",
                 "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"
             ],
-            "markers": "python_version >= '3.5'",
+            "markers": "python_version >= '3'",
             "version": "==3.3"
         },
         "imap-tools": {
             "hashes": [
-                "sha256:119f1a60ea4048a4c5d72d9e9fa47c295685e340c730cb0b71fdf0ad3b7e53f8",
-                "sha256:3648bac835657b1c56ba856452c8a28bdbe3689d3730f95a4ad20d4c39f1c2d0"
+                "sha256:15d20ac8695fc4978a913c2186f482a802f5229c41c6e0c66c7bad8f1f590cf1",
+                "sha256:606b73a1b5ecc4c72eea5ad19231e07a88bf9ba9adbdd4acb8cf71a359dd43ec"
             ],
             "index": "pypi",
-            "version": "==0.53.0"
+            "version": "==0.54.0"
         },
         "img2pdf": {
             "hashes": [
-                "sha256:8e51c5043efa95d751481b516071a006f87c2a4059961a9ac43ec238915de09f"
+                "sha256:8ec898a9646523fd3862b154f3f47cd52609c24cc3e2dc1fb5f0168f0cbe793c"
             ],
-            "version": "==0.4.3"
+            "version": "==0.4.4"
         },
         "importlib-resources": {
             "hashes": [
-                "sha256:1b93238cbf23b4cde34240dd8321d99e9bf2eb4bc91c0c99b2886283e7baad85",
-                "sha256:a9dd72f6cc106aeb50f6e66b86b69b454766dd6e39b69ac68450253058706bcc"
+                "sha256:b6062987dfc51f0fcb809187cffbd60f35df7acb4589091f154214af6d0d49d3",
+                "sha256:e447dc01619b1e951286f3929be820029d48c75eb25d265c28b92a16548212b8"
             ],
             "index": "pypi",
             "markers": "python_version < '3.9'",
-            "version": "==5.6.0"
+            "version": "==5.7.1"
         },
         "incremental": {
             "hashes": [
@@ -673,11 +675,11 @@
         },
         "ocrmypdf": {
             "hashes": [
-                "sha256:201ed2f589f851be73908fce35fbb6fb05e4739289d3cd8765f9519f49ea1cd9",
-                "sha256:f42e60bc2b6534634dd08928584275b1c556dc875c849650afcc38f7da9e2856"
+                "sha256:0c1cc0a7596fa9da1bfde67141227eeb813aba5e954f88199eacc5f51f1d67d9",
+                "sha256:48bbdd5d15b76f34aa3a91910918e51f91bb3833b4e86da45f8542afda118404"
             ],
             "index": "pypi",
-            "version": "==13.4.1"
+            "version": "==13.4.3"
         },
         "packaging": {
             "hashes": [
@@ -695,87 +697,98 @@
             "index": "pypi",
             "version": "==2.5.0"
         },
-        "pdfminer.six": {
+        "pdf2image": {
             "hashes": [
-                "sha256:0351f17d362ee2d48b158be52bcde6576d96460efd038a3e89a043fba6d634d7",
-                "sha256:d3efb75c0249b51c1bf795e3a8bddf1726b276c77bf75fb136adea471ee2825b"
+                "sha256:84f79f2b8fad943e36323ea4e937fcb05f26ded0caa0a01181df66049e42fb65",
+                "sha256:d58ed94d978a70c73c2bb7fdf8acbaf2a7089c29ff8141be5f45433c0c4293bb"
             ],
             "index": "pypi",
-            "version": "==20211012"
+            "version": "==1.16.0"
+        },
+        "pdfminer.six": {
+            "hashes": [
+                "sha256:af0630f98a292bad4170f54e80f82ca81b916dd0b2c996437ec45c02f11d8762",
+                "sha256:eff2ce0abeaa4df94dc3461f70eab104487c7b4a2b3c7e9fd0aeec6c5f44d6a6"
+            ],
+            "index": "pypi",
+            "version": "==20220319"
         },
         "pikepdf": {
             "hashes": [
-                "sha256:1567b74d15c16d1bef56a6d5f56fb6a35f4cc022ae252d35f33f56bb16b87966",
-                "sha256:179f24b4a4d0e89c7f592d85ba4daf7d34c709eb0a691425b881413fdce70734",
-                "sha256:22201b06db627a86cc91c1b76491dd9c57fce2df4e3bc8ba700ff66f7f7da04a",
-                "sha256:3e1a0b9ecf5d4aa106c3c0db558952f9a15f343f812c3bba6d6e1a56e25224ed",
-                "sha256:45ce2479e5ba74896ef389abf92831d7fbb34f25e6557adb9115710223a0bb13",
-                "sha256:4fa5c8494b011b19bd198dd9c3cd94676b905360b2231ad35171ae586644b823",
-                "sha256:559b3d502cc1a6813cbcb0766b0797fec034303f8f9b0734cf938fb1734e2b74",
-                "sha256:61731fceaab99850bc7045232301c2332bba727f78b53f7038fcbdcaf3d64309",
-                "sha256:6460d489341e7f8dc3f6b0dbf1f5a75a918ebd1e0ecb4c2b00877264a68ee1f4",
-                "sha256:716ca6fc8947502cb73a517c884066afa132ca998e085a309b8fb8c5796d6277",
-                "sha256:726450eb9baaad5697687c2621d481c80f868b68c06d2cff4be3f6a7ce28cfee",
-                "sha256:7e9c247ca384ad1606281eda4d841bc8cbff90875979ac3b520bcc5404bf9b26",
-                "sha256:8333d813b452daa4a066e135fd7ab6f7c07ccc02cb8381455d61d74f0a0ad0fe",
-                "sha256:83d0af374b103934de033f096205143fa9d6f78e789ba78c8aa6dfb0e5b73bc3",
-                "sha256:878c1c95298486d8cea7e8236c70613e7eae1426cbf362c3883ecd06e8f9c2d9",
-                "sha256:9abef24d929c4a08292dc4be4d6c4e5bf93832e747eef5f39e854348a332f46a",
-                "sha256:a474241dbeda246356b6448f607f4fb9fee5b9f5cb19511a768b88b471325865",
-                "sha256:a553fd06e5f6e78c5e840d066e7c8b1a988e16489fe0bd4a143ca601809ea4dd",
-                "sha256:a6154c6bb7606ef534444f54271a410a6337cee54dcdef20f0fa0686f622cf50",
-                "sha256:ac0082379cf6aa6c0c682bee4a3d2adbe6d60b9125e3632876d7c5e9665c07ab",
-                "sha256:c22e3fbfb76ad7838dde82c8d9fafd4c09fd419cee531b31d1b48a07344ac2b4",
-                "sha256:d42c52ff2e8fb00fad14182f67a8f076f38c75a874123b6776aa5c6af09e1126",
-                "sha256:d6ef14b722f80351e15c9163e0aa1f1df84f065c080765f8232ec51af6bd3368",
-                "sha256:d71a38445c80972572248815ebe61f9c814c53925a6b83b6596f3482a98a5f25",
-                "sha256:e81ebbdd53257f411827bbb301900cdbdca34ca60b4f7248f80c6e6980062498",
-                "sha256:f52f4ed655c25c408e8454cbcf0e7223b62f635d1e8ab738fd0c2d46531d28ed",
-                "sha256:fb407ae5820ee0bf71022d5a8f539d709dac590443270a13baf6a8872c76d46f"
+                "sha256:101ec256a8d312c17decae52226cf32a3e7dc834583134300c2f4e60b70e6e91",
+                "sha256:12b5b3cfc649e2542576a7e55c11e245278f14f727f116904893e54329102867",
+                "sha256:1b8f68a75c0a6f6d4d102d0821365ae2aaa9ab635c6eb6c840569a56b1a266f4",
+                "sha256:1bef3512be59fe0f481375b7eb415ca51ee7c80555031401f5f17ee3392e4add",
+                "sha256:1d3141916dc9efc433fd22beba544f67a53a805800c3ff902baffa398ef4c85e",
+                "sha256:3052df8514d26b676c50e65afc49a1d260c43a08c322c75cc2592c10a9a5b26d",
+                "sha256:356d5554516a295fc10db3f25cfde4e92326f6d015da55d71b84f5ced2a07a5a",
+                "sha256:55330c24b8e04ee09f1bc514c2b6107bb03a5eeb0b74929a61100cd6be22ae29",
+                "sha256:553cf11933fdfe07fdd357ab40b9732db102e921b27c1065239308d42b7b858f",
+                "sha256:5626312990a894c5db3a269455f7eb98df5f59188dde1797c0e352d60fdf89af",
+                "sha256:59c24a65c94693ab4a7e92f4809f847b57461120256c083054e61c99c4952e84",
+                "sha256:607deb1181a7cf5369cf70edfc41574d46c0a17c0cac1f6234272bd4cb3487e4",
+                "sha256:60bdd49e6251f8c99989e6769d4ad29b209c1eaf88090f49d4b30fba98442e40",
+                "sha256:73a7cc3c42609e00393b9d4e1b9ee132f528060254a174bf18ef31a154be0386",
+                "sha256:75f1e2917b4d2d6573fe3d1c3b2ec70829b64515b2f723f5c3bebcdd65761e6c",
+                "sha256:92ca9191680eccc21697e9e9c218e600ab31e7c24f6125749738c10ae2dc7c07",
+                "sha256:9bcaf96e2f571f0fc7e3178cdf1bcca7c13e5c68128e8246031226d47ecf23f1",
+                "sha256:a8f3e2229e2683497fe4ccc4af06050c125160a11bb3562b6c4ddaee4d0cc5c5",
+                "sha256:c277066938ca0ddb2bfe75874ef8dd3aa259936fe15c4cf7d4282f89ba82ab3a",
+                "sha256:c532542a99757d9f41df0cf1fc8f64a044d0eb822822cc069c80be35731df275",
+                "sha256:dfa89bd86e01413531c1d7d201fb01f0e62b52ea926a8e8ca6f99f86ed761e95",
+                "sha256:e064010b733b0a2ec4ec97982cd2887f9025292f2d228c6d5e6eca9d84851e53",
+                "sha256:ea927afe7cb04cc7ade30b961f528ef53e8d9cf467dcc4639cf944fef872a1a1",
+                "sha256:f40703b6267aa43d7f72468fa0a3b505ffff74ece2a4c69cfd3c90e023c41381",
+                "sha256:f45cc4544bbd4c308a525a6bb8e2e29b3f849803ee557c6e35c684447f0a92e5",
+                "sha256:f8ccda5ee992c73f647bcd96c9aa30f5eb9e8a6c5bdd6e3dcb29ebbffbe01a69",
+                "sha256:fe386d93345c9b5a9690f7a7bfb789a5ec5467c34402628e10bda8a4f5bac73e"
             ],
             "index": "pypi",
-            "version": "==5.1.0"
+            "version": "==5.1.2"
         },
         "pillow": {
             "hashes": [
-                "sha256:011233e0c42a4a7836498e98c1acf5e744c96a67dd5032a6f666cc1fb97eab97",
-                "sha256:0f29d831e2151e0b7b39981756d201f7108d3d215896212ffe2e992d06bfe049",
-                "sha256:12875d118f21cf35604176872447cdb57b07126750a33748bac15e77f90f1f9c",
-                "sha256:14d4b1341ac07ae07eb2cc682f459bec932a380c3b122f5540432d8977e64eae",
-                "sha256:1c3c33ac69cf059bbb9d1a71eeaba76781b450bc307e2291f8a4764d779a6b28",
-                "sha256:1d19397351f73a88904ad1aee421e800fe4bbcd1aeee6435fb62d0a05ccd1030",
-                "sha256:253e8a302a96df6927310a9d44e6103055e8fb96a6822f8b7f514bb7ef77de56",
-                "sha256:2632d0f846b7c7600edf53c48f8f9f1e13e62f66a6dbc15191029d950bfed976",
-                "sha256:335ace1a22325395c4ea88e00ba3dc89ca029bd66bd5a3c382d53e44f0ccd77e",
-                "sha256:413ce0bbf9fc6278b2d63309dfeefe452835e1c78398efb431bab0672fe9274e",
-                "sha256:5100b45a4638e3c00e4d2320d3193bdabb2d75e79793af7c3eb139e4f569f16f",
-                "sha256:514ceac913076feefbeaf89771fd6febde78b0c4c1b23aaeab082c41c694e81b",
-                "sha256:528a2a692c65dd5cafc130de286030af251d2ee0483a5bf50c9348aefe834e8a",
-                "sha256:6295f6763749b89c994fcb6d8a7f7ce03c3992e695f89f00b741b4580b199b7e",
-                "sha256:6c8bc8238a7dfdaf7a75f5ec5a663f4173f8c367e5a39f87e720495e1eed75fa",
-                "sha256:718856856ba31f14f13ba885ff13874be7fefc53984d2832458f12c38205f7f7",
-                "sha256:7f7609a718b177bf171ac93cea9fd2ddc0e03e84d8fa4e887bdfc39671d46b00",
-                "sha256:80ca33961ced9c63358056bd08403ff866512038883e74f3a4bf88ad3eb66838",
-                "sha256:80fe64a6deb6fcfdf7b8386f2cf216d329be6f2781f7d90304351811fb591360",
-                "sha256:81c4b81611e3a3cb30e59b0cf05b888c675f97e3adb2c8672c3154047980726b",
-                "sha256:855c583f268edde09474b081e3ddcd5cf3b20c12f26e0d434e1386cc5d318e7a",
-                "sha256:9bfdb82cdfeccec50aad441afc332faf8606dfa5e8efd18a6692b5d6e79f00fd",
-                "sha256:a5d24e1d674dd9d72c66ad3ea9131322819ff86250b30dc5821cbafcfa0b96b4",
-                "sha256:a9f44cd7e162ac6191491d7249cceb02b8116b0f7e847ee33f739d7cb1ea1f70",
-                "sha256:b5b3f092fe345c03bca1e0b687dfbb39364b21ebb8ba90e3fa707374b7915204",
-                "sha256:b9618823bd237c0d2575283f2939655f54d51b4527ec3972907a927acbcc5bfc",
-                "sha256:cef9c85ccbe9bee00909758936ea841ef12035296c748aaceee535969e27d31b",
-                "sha256:d21237d0cd37acded35154e29aec853e945950321dd2ffd1a7d86fe686814669",
-                "sha256:d3c5c79ab7dfce6d88f1ba639b77e77a17ea33a01b07b99840d6ed08031cb2a7",
-                "sha256:d9d7942b624b04b895cb95af03a23407f17646815495ce4547f0e60e0b06f58e",
-                "sha256:db6d9fac65bd08cea7f3540b899977c6dee9edad959fa4eaf305940d9cbd861c",
-                "sha256:ede5af4a2702444a832a800b8eb7f0a7a1c0eed55b644642e049c98d589e5092",
-                "sha256:effb7749713d5317478bb3acb3f81d9d7c7f86726d41c1facca068a04cf5bb4c",
-                "sha256:f154d173286a5d1863637a7dcd8c3437bb557520b01bddb0be0258dcb72696b5",
-                "sha256:f25ed6e28ddf50de7e7ea99d7a976d6a9c415f03adcaac9c41ff6ff41b6d86ac"
+                "sha256:01ce45deec9df310cbbee11104bae1a2a43308dd9c317f99235b6d3080ddd66e",
+                "sha256:0c51cb9edac8a5abd069fd0758ac0a8bfe52c261ee0e330f363548aca6893595",
+                "sha256:17869489de2fce6c36690a0c721bd3db176194af5f39249c1ac56d0bb0fcc512",
+                "sha256:21dee8466b42912335151d24c1665fcf44dc2ee47e021d233a40c3ca5adae59c",
+                "sha256:25023a6209a4d7c42154073144608c9a71d3512b648a2f5d4465182cb93d3477",
+                "sha256:255c9d69754a4c90b0ee484967fc8818c7ff8311c6dddcc43a4340e10cd1636a",
+                "sha256:35be4a9f65441d9982240e6966c1eaa1c654c4e5e931eaf580130409e31804d4",
+                "sha256:3f42364485bfdab19c1373b5cd62f7c5ab7cc052e19644862ec8f15bb8af289e",
+                "sha256:3fddcdb619ba04491e8f771636583a7cc5a5051cd193ff1aa1ee8616d2a692c5",
+                "sha256:463acf531f5d0925ca55904fa668bb3461c3ef6bc779e1d6d8a488092bdee378",
+                "sha256:4fe29a070de394e449fd88ebe1624d1e2d7ddeed4c12e0b31624561b58948d9a",
+                "sha256:55dd1cf09a1fd7c7b78425967aacae9b0d70125f7d3ab973fadc7b5abc3de652",
+                "sha256:5a3ecc026ea0e14d0ad7cd990ea7f48bfcb3eb4271034657dc9d06933c6629a7",
+                "sha256:5cfca31ab4c13552a0f354c87fbd7f162a4fafd25e6b521bba93a57fe6a3700a",
+                "sha256:66822d01e82506a19407d1afc104c3fcea3b81d5eb11485e593ad6b8492f995a",
+                "sha256:69e5ddc609230d4408277af135c5b5c8fe7a54b2bdb8ad7c5100b86b3aab04c6",
+                "sha256:6b6d4050b208c8ff886fd3db6690bf04f9a48749d78b41b7a5bf24c236ab0165",
+                "sha256:7a053bd4d65a3294b153bdd7724dce864a1d548416a5ef61f6d03bf149205160",
+                "sha256:82283af99c1c3a5ba1da44c67296d5aad19f11c535b551a5ae55328a317ce331",
+                "sha256:8782189c796eff29dbb37dd87afa4ad4d40fc90b2742704f94812851b725964b",
+                "sha256:8d79c6f468215d1a8415aa53d9868a6b40c4682165b8cb62a221b1baa47db458",
+                "sha256:97bda660702a856c2c9e12ec26fc6d187631ddfd896ff685814ab21ef0597033",
+                "sha256:a325ac71914c5c043fa50441b36606e64a10cd262de12f7a179620f579752ff8",
+                "sha256:a336a4f74baf67e26f3acc4d61c913e378e931817cd1e2ef4dfb79d3e051b481",
+                "sha256:a598d8830f6ef5501002ae85c7dbfcd9c27cc4efc02a1989369303ba85573e58",
+                "sha256:a5eaf3b42df2bcda61c53a742ee2c6e63f777d0e085bbc6b2ab7ed57deb13db7",
+                "sha256:aea7ce61328e15943d7b9eaca87e81f7c62ff90f669116f857262e9da4057ba3",
+                "sha256:af79d3fde1fc2e33561166d62e3b63f0cc3e47b5a3a2e5fea40d4917754734ea",
+                "sha256:c24f718f9dd73bb2b31a6201e6db5ea4a61fdd1d1c200f43ee585fc6dcd21b34",
+                "sha256:c5b0ff59785d93b3437c3703e3c64c178aabada51dea2a7f2c5eccf1bcf565a3",
+                "sha256:c7110ec1701b0bf8df569a7592a196c9d07c764a0a74f65471ea56816f10e2c8",
+                "sha256:c870193cce4b76713a2b29be5d8327c8ccbe0d4a49bc22968aa1e680930f5581",
+                "sha256:c9efef876c21788366ea1f50ecb39d5d6f65febe25ad1d4c0b8dff98843ac244",
+                "sha256:de344bcf6e2463bb25179d74d6e7989e375f906bcec8cb86edb8b12acbc7dfef",
+                "sha256:eb1b89b11256b5b6cad5e7593f9061ac4624f7651f7a8eb4dfa37caa1dfaa4d0",
+                "sha256:ed742214068efa95e9844c2d9129e209ed63f61baa4d54dbf4cf8b5e2d30ccf2",
+                "sha256:f401ed2bbb155e1ade150ccc63db1a4f6c1909d3d378f7d1235a44e90d75fb97",
+                "sha256:fb89397013cf302f282f0fc998bb7abf11d49dcff72c8ecb320f76ea6e2c5717"
             ],
             "index": "pypi",
-            "version": "==9.0.1"
+            "version": "==9.1.0"
         },
         "pluggy": {
             "hashes": [
@@ -862,11 +875,11 @@
         },
         "pyparsing": {
             "hashes": [
-                "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea",
-                "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484"
+                "sha256:7bf433498c016c4314268d95df76c81b842a4cb2b276fa3312cfb1e1d85f6954",
+                "sha256:ef7b523f6356f763771559412c0d7134753f037822dad1b16945b7b846f7ad06"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==3.0.7"
+            "markers": "python_full_version >= '3.6.8'",
+            "version": "==3.0.8"
         },
         "python-dateutil": {
             "hashes": [
@@ -959,6 +972,15 @@
             ],
             "version": "==6.0"
         },
+        "pyzbar": {
+            "hashes": [
+                "sha256:13e3ee5a2f3a545204a285f41814d5c0db571967e8d4af8699a03afc55182a9c",
+                "sha256:4559628b8192feb25766d954b36a3753baaf5c97c03135aec7e4a026036b475d",
+                "sha256:8f4c5264c9c7c6b9f20d01efc52a4eba1ded47d9ba857a94130afe33703eb518"
+            ],
+            "index": "pypi",
+            "version": "==0.1.9"
+        },
         "redis": {
             "hashes": [
                 "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2",
@@ -1173,11 +1195,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:89eef7b71423ab7fccc7dfafdc145410ef170c4a89567427f932448135e08cdf",
-                "sha256:92b15f45ab164eb0c410d2bf661a6e9d15e3b78c0dffb0325f2bf0f313071cae"
+                "sha256:26ead7d1f93efc0f8c804d9fafafbe4a44b179580a7105754b245155f9af05a8",
+                "sha256:47c7b0c0f8fc10eec4cf1e71c6fdadf8decaa74ffa087e68cd1c20db7ad6a592"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==61.1.1"
+            "version": "==62.1.0"
         },
         "six": {
             "hashes": [
@@ -1220,22 +1242,22 @@
         },
         "tqdm": {
             "hashes": [
-                "sha256:4230a49119a416c88cc47d0d2d32d5d90f1a282d5e497d49801950704e49863d",
-                "sha256:6461b009d6792008d0000e1b0c7ca50195ec78c0e808a3a6b668a56a3236c3a5"
+                "sha256:40be55d30e200777a307a7585aee69e4eabb46b4ec6a4b4a5f2d9f11e7d5408d",
+                "sha256:74a2cdefe14d11442cedf3ba4e21a3b84ff9a2dbdc6cfae2c34addb2a14a5ea6"
             ],
             "index": "pypi",
-            "version": "==4.63.1"
+            "version": "==4.64.0"
         },
         "twisted": {
             "extras": [
                 "tls"
             ],
             "hashes": [
-                "sha256:57f32b1f6838facb8c004c89467840367ad38e9e535f8252091345dba500b4f2",
-                "sha256:5c63c149eb6b8fe1e32a0215b1cef96fabdba04f705d8efb9174b1ccf5b49d49"
+                "sha256:a047990f57dfae1e0bd2b7df2526d4f16dcdc843774dc108b78c52f2a5f13680",
+                "sha256:f9f7a91f94932477a9fc3b169d57f54f96c6e74a23d78d9ce54039a7f48928a2"
             ],
             "markers": "python_full_version >= '3.6.7'",
-            "version": "==22.2.0"
+            "version": "==22.4.0"
         },
         "txaio": {
             "hashes": [
@@ -1247,11 +1269,11 @@
         },
         "typing-extensions": {
             "hashes": [
-                "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42",
-                "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2"
+                "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708",
+                "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==4.1.1"
+            "markers": "python_version >= '3.7'",
+            "version": "==4.2.0"
         },
         "tzdata": {
             "hashes": [
@@ -1263,11 +1285,11 @@
         },
         "tzlocal": {
             "hashes": [
-                "sha256:0f28015ac68a5c067210400a9197fc5d36ba9bc3f8eaf1da3cbd59acdfed9e09",
-                "sha256:28ba8d9fcb6c9a782d6e0078b4f6627af1ea26aeaa32b4eab5324abc7df4149f"
+                "sha256:89885494684c929d9191c57aa27502afc87a579be5cdd3225c77c463ea043745",
+                "sha256:ee5842fa3a795f023514ac2d801c4a81d1743bbe642e3940143326b3a00addd7"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==4.1"
+            "version": "==4.2"
         },
         "urllib3": {
             "hashes": [
@@ -1341,10 +1363,10 @@
         },
         "watchgod": {
             "hashes": [
-                "sha256:4ba20c2fa3e63df706ab50e694b9453b05395fadb7cbbfd984d71fb1547d485d",
-                "sha256:c12d15f3df7d11e740704e45398277f75f1d78f46ad59ca9d7505bfd8b8d3086"
+                "sha256:2f3e8137d98f493ff58af54ea00f4d1433a6afe2ed08ab331a657df468c6bfce",
+                "sha256:cb11ff66657befba94d828e3b622d5fb76f22fbda1376f355f3e6e51e97d9450"
             ],
-            "version": "==0.8.1"
+            "version": "==0.8.2"
         },
         "wcwidth": {
             "hashes": [
@@ -1355,56 +1377,56 @@
         },
         "websockets": {
             "hashes": [
-                "sha256:038afef2a05893578d10dadbdbb5f112bd115c46347e1efe99f6a356ff062138",
-                "sha256:05f6e9757017270e7a92a2975e2ae88a9a582ffc4629086fd6039aa80e99cd86",
-                "sha256:0b66421f9f13d4df60cd48ab977ed2c2b6c9147ae1a33caf5a9f46294422fda1",
-                "sha256:0cd02f36d37e503aca88ab23cc0a1a0e92a263d37acf6331521eb38040dcf77b",
-                "sha256:0f73cb2526d6da268e86977b2c4b58f2195994e53070fe567d5487c6436047e6",
-                "sha256:117383d0a17a0dda349f7a8790763dde75c1508ff8e4d6e8328b898b7df48397",
-                "sha256:1c1f3b18c8162e3b09761d0c6a0305fd642934202541cc511ef972cb9463261e",
-                "sha256:1c9031e90ebfc486e9cdad532b94004ade3aa39a31d3c46c105bb0b579cd2490",
-                "sha256:2349fa81b6b959484bb2bda556ccb9eb70ba68987646a0f8a537a1a18319fb03",
-                "sha256:24b879ba7db12bb525d4e58089fcbe6a3df3ce4666523183654170e86d372cbe",
-                "sha256:2aa9b91347ecd0412683f28aabe27f6bad502d89bd363b76e0a3508b1596402e",
-                "sha256:56d48eebe9e39ce0d68701bce3b21df923aa05dcc00f9fd8300de1df31a7c07c",
-                "sha256:5a38a0175ae82e4a8c4bac29fc01b9ee26d7d5a614e5ee11e7813c68a7d938ce",
-                "sha256:5b04270b5613f245ec84bb2c6a482a9d009aefad37c0575f6cda8499125d5d5c",
-                "sha256:6193bbc1ee63aadeb9a4d81de0e19477401d150d506aee772d8380943f118186",
-                "sha256:669e54228a4d9457abafed27cbf0e2b9f401445c4dfefc12bf8e4db9751703b8",
-                "sha256:6a009eb551c46fd79737791c0c833fc0e5b56bcd1c3057498b262d660b92e9cd",
-                "sha256:71a4491cfe7a9f18ee57d41163cb6a8a3fa591e0f0564ca8b0ed86b2a30cced4",
-                "sha256:7b38a5c9112e3dbbe45540f7b60c5204f49b3cb501b40950d6ab34cd202ab1d0",
-                "sha256:7bb9d8a6beca478c7e9bdde0159bd810cc1006ad6a7cb460533bae39da692ca2",
-                "sha256:82bc33db6d8309dc27a3bee11f7da2288ad925fcbabc2a4bb78f7e9c56249baf",
-                "sha256:8351c3c86b08156337b0e4ece0e3c5ec3e01fcd14e8950996832a23c99416098",
-                "sha256:8beac786a388bb99a66c3be4ab0fb38273c0e3bc17f612a4e0a47c4fc8b9c045",
-                "sha256:97950c7c844ec6f8d292440953ae18b99e3a6a09885e09d20d5e7ecd9b914cf8",
-                "sha256:98f57b3120f8331cd7440dbe0e776474f5e3632fdaa474af1f6b754955a47d71",
-                "sha256:9ca2ca05a4c29179f06cf6727b45dba5d228da62623ec9df4184413d8aae6cb9",
-                "sha256:a03a25d95cc7400bd4d61a63460b5d85a7761c12075ee2f51de1ffe73aa593d3",
-                "sha256:a10c0c1ee02164246f90053273a42d72a3b2452a7e7486fdae781138cf7fbe2d",
-                "sha256:a72b92f96e5e540d5dda99ee3346e199ade8df63152fa3c737260da1730c411f",
-                "sha256:ac081aa0307f263d63c5ff0727935c736c8dad51ddf2dc9f5d0c4759842aefaa",
-                "sha256:b22bdc795e62e71118b63e14a08bacfa4f262fd2877de7e5b950f5ac16b0348f",
-                "sha256:b4059e2ccbe6587b6dc9a01db5fc49ead9a884faa4076eea96c5ec62cb32f42a",
-                "sha256:b7fe45ae43ac814beb8ca09d6995b56800676f2cfa8e23f42839dc69bba34a42",
-                "sha256:bef03a51f9657fb03d8da6ccd233fe96e04101a852f0ffd35f5b725b28221ff3",
-                "sha256:bffc65442dd35c473ca9790a3fa3ba06396102a950794f536783f4b8060af8dd",
-                "sha256:c21a67ab9a94bd53e10bba21912556027fea944648a09e6508415ad14e37c325",
-                "sha256:c67d9cacb3f6537ca21e9b224d4fd08481538e43bcac08b3d93181b0816def39",
-                "sha256:c6e56606842bb24e16e36ae7eb308d866b4249cf0be8f63b212f287eeb76b124",
-                "sha256:cb316b87cbe3c0791c2ad92a5a36bf6adc87c457654335810b25048c1daa6fd5",
-                "sha256:cef40a1b183dcf39d23b392e9dd1d9b07ab9c46aadf294fff1350fb79146e72b",
-                "sha256:cf931c33db9c87c53d009856045dd524e4a378445693382a920fa1e0eb77c36c",
-                "sha256:d4d110a84b63c5cfdd22485acc97b8b919aefeecd6300c0c9d551e055b9a88ea",
-                "sha256:d5396710f86a306cf52f87fd8ea594a0e894ba0cc5a36059eaca3a477dc332aa",
-                "sha256:f09f46b1ff6d09b01c7816c50bd1903cf7d02ebbdb63726132717c2fcda835d5",
-                "sha256:f14bd10e170abc01682a9f8b28b16e6f20acf6175945ef38db6ffe31b0c72c3f",
-                "sha256:f5c335dc0e7dc271ef36df3f439868b3c790775f345338c2f61a562f1074187b",
-                "sha256:f8296b8408ec6853b26771599990721a26403e62b9de7e50ac0a056772ac0b5e",
-                "sha256:fa35c5d1830d0fb7b810324e9eeab9aa92e8f273f11fdbdc0741dcded6d72b9f"
+                "sha256:07cdc0a5b2549bcfbadb585ad8471ebdc7bdf91e32e34ae3889001c1c106a6af",
+                "sha256:210aad7fdd381c52e58777560860c7e6110b6174488ef1d4b681c08b68bf7f8c",
+                "sha256:28dd20b938a57c3124028680dc1600c197294da5db4292c76a0b48efb3ed7f76",
+                "sha256:2f94fa3ae454a63ea3a19f73b95deeebc9f02ba2d5617ca16f0bbdae375cda47",
+                "sha256:31564a67c3e4005f27815634343df688b25705cccb22bc1db621c781ddc64c69",
+                "sha256:347974105bbd4ea068106ec65e8e8ebd86f28c19e529d115d89bd8cc5cda3079",
+                "sha256:379e03422178436af4f3abe0aa8f401aa77ae2487843738542a75faf44a31f0c",
+                "sha256:3eda1cb7e9da1b22588cefff09f0951771d6ee9fa8dbe66f5ae04cc5f26b2b55",
+                "sha256:51695d3b199cd03098ae5b42833006a0f43dc5418d3102972addc593a783bc02",
+                "sha256:54c000abeaff6d8771a4e2cef40900919908ea7b6b6a30eae72752607c6db559",
+                "sha256:5b936bf552e4f6357f5727579072ff1e1324717902127ffe60c92d29b67b7be3",
+                "sha256:6075fd24df23133c1b078e08a9b04a3bc40b31a8def4ee0b9f2c8865acce913e",
+                "sha256:661f641b44ed315556a2fa630239adfd77bd1b11cb0b9d96ed8ad90b0b1e4978",
+                "sha256:6ea6b300a6bdd782e49922d690e11c3669828fe36fc2471408c58b93b5535a98",
+                "sha256:6ed1d6f791eabfd9808afea1e068f5e59418e55721db8b7f3bfc39dc831c42ae",
+                "sha256:7934e055fd5cd9dee60f11d16c8d79c4567315824bacb1246d0208a47eca9755",
+                "sha256:7ab36e17af592eec5747c68ef2722a74c1a4a70f3772bc661079baf4ae30e40d",
+                "sha256:7f6d96fdb0975044fdd7953b35d003b03f9e2bcf85f2d2cf86285ece53e9f991",
+                "sha256:83e5ca0d5b743cde3d29fda74ccab37bdd0911f25bd4cdf09ff8b51b7b4f2fa1",
+                "sha256:85506b3328a9e083cc0a0fb3ba27e33c8db78341b3eb12eb72e8afd166c36680",
+                "sha256:8af75085b4bc0b5c40c4a3c0e113fa95e84c60f4ed6786cbb675aeb1ee128247",
+                "sha256:8b1359aba0ff810d5830d5ab8e2c4a02bebf98a60aa0124fb29aa78cfdb8031f",
+                "sha256:8fbd7d77f8aba46d43245e86dd91a8970eac4fb74c473f8e30e9c07581f852b2",
+                "sha256:907e8247480f287aa9bbc9391bd6de23c906d48af54c8c421df84655eef66af7",
+                "sha256:93d5ea0b5da8d66d868b32c614d2b52d14304444e39e13a59566d4acb8d6e2e4",
+                "sha256:97bc9d41e69a7521a358f9b8e44871f6cdeb42af31815c17aed36372d4eec667",
+                "sha256:994cdb1942a7a4c2e10098d9162948c9e7b235df755de91ca33f6e0481366fdb",
+                "sha256:a141de3d5a92188234afa61653ed0bbd2dde46ad47b15c3042ffb89548e77094",
+                "sha256:a1e15b230c3613e8ea82c9fc6941b2093e8eb939dd794c02754d33980ba81e36",
+                "sha256:aad5e300ab32036eb3fdc350ad30877210e2f51bceaca83fb7fef4d2b6c72b79",
+                "sha256:b529fdfa881b69fe563dbd98acce84f3e5a67df13de415e143ef053ff006d500",
+                "sha256:b9c77f0d1436ea4b4dc089ed8335fa141e6a251a92f75f675056dac4ab47a71e",
+                "sha256:bb621ec2dbbbe8df78a27dbd9dd7919f9b7d32a73fafcb4d9252fc4637343582",
+                "sha256:c7250848ce69559756ad0086a37b82c986cd33c2d344ab87fea596c5ac6d9442",
+                "sha256:c8d1d14aa0f600b5be363077b621b1b4d1eb3fbf90af83f9281cda668e6ff7fd",
+                "sha256:d1655a6fc7aecd333b079d00fb3c8132d18988e47f19740c69303bf02e9883c6",
+                "sha256:d6353ba89cfc657a3f5beabb3b69be226adbb5c6c7a66398e17809b0ce3c4731",
+                "sha256:da4377904a3379f0c1b75a965fff23b28315bcd516d27f99a803720dfebd94d4",
+                "sha256:e49ea4c1a9543d2bd8a747ff24411509c29e4bdcde05b5b0895e2120cb1a761d",
+                "sha256:e4e08305bfd76ba8edab08dcc6496f40674f44eb9d5e23153efa0a35750337e8",
+                "sha256:e6fa05a680e35d0fcc1470cb070b10e6fe247af54768f488ed93542e71339d6f",
+                "sha256:e7e6f2d6fd48422071cc8a6f8542016f350b79cc782752de531577d35e9bd677",
+                "sha256:e904c0381c014b914136c492c8fa711ca4cced4e9b3d110e5e7d436d0fc289e8",
+                "sha256:ec2b0ab7edc8cd4b0eb428b38ed89079bdc20c6bdb5f889d353011038caac2f9",
+                "sha256:ef5ce841e102278c1c2e98f043db99d6755b1c58bde475516aef3a008ed7f28e",
+                "sha256:f351c7d7d92f67c0609329ab2735eee0426a03022771b00102816a72715bb00b",
+                "sha256:fab7c640815812ed5f10fbee7abbf58788d602046b7bb3af9b1ac753a6d5e916",
+                "sha256:fc06cc8073c8e87072138ba1e431300e2d408f054b27047d047b549455066ff4"
             ],
-            "version": "==10.2"
+            "version": "==10.3"
         },
         "whitenoise": {
             "hashes": [
@@ -1425,12 +1447,12 @@
         },
         "zipp": {
             "hashes": [
-                "sha256:9f50f446828eb9d45b267433fd3e9da8d801f614129124863f9c51ebceafb87d",
-                "sha256:b47250dd24f92b7dd6a0a8fc5244da14608f3ca90a5efcd37a3b1642fac9a375"
+                "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad",
+                "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"
             ],
             "index": "pypi",
             "markers": "python_version < '3.9'",
-            "version": "==3.7.0"
+            "version": "==3.8.0"
         },
         "zope.interface": {
             "hashes": [
@@ -1508,40 +1530,40 @@
         },
         "babel": {
             "hashes": [
-                "sha256:ab49e12b91d937cd11f0b67cb259a57ab4ad2b59ac7a3b41d6c06c0ac5b0def9",
-                "sha256:bc0c176f9f6a994582230df350aa6e05ba2ebe4b3ac317eab29d9be5d2768da0"
+                "sha256:3f349e85ad3154559ac4930c3918247d319f21910d5ce4b25d439ed8693b98d2",
+                "sha256:98aeaca086133efb3e1e2aad0396987490c8425929ddbcfe0550184fdc54cd13"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==2.9.1"
+            "markers": "python_version >= '3.6'",
+            "version": "==2.10.1"
         },
         "black": {
             "hashes": [
-                "sha256:07e5c049442d7ca1a2fc273c79d1aecbbf1bc858f62e8184abe1ad175c4f7cc2",
-                "sha256:0e21e1f1efa65a50e3960edd068b6ae6d64ad6235bd8bfea116a03b21836af71",
-                "sha256:1297c63b9e1b96a3d0da2d85d11cd9bf8664251fd69ddac068b98dc4f34f73b6",
-                "sha256:228b5ae2c8e3d6227e4bde5920d2fc66cc3400fde7bcc74f480cb07ef0b570d5",
-                "sha256:2d6f331c02f0f40aa51a22e479c8209d37fcd520c77721c034517d44eecf5912",
-                "sha256:2ff96450d3ad9ea499fc4c60e425a1439c2120cbbc1ab959ff20f7c76ec7e866",
-                "sha256:3524739d76b6b3ed1132422bf9d82123cd1705086723bc3e235ca39fd21c667d",
-                "sha256:35944b7100af4a985abfcaa860b06af15590deb1f392f06c8683b4381e8eeaf0",
-                "sha256:373922fc66676133ddc3e754e4509196a8c392fec3f5ca4486673e685a421321",
-                "sha256:5fa1db02410b1924b6749c245ab38d30621564e658297484952f3d8a39fce7e8",
-                "sha256:6f2f01381f91c1efb1451998bd65a129b3ed6f64f79663a55fe0e9b74a5f81fd",
-                "sha256:742ce9af3086e5bd07e58c8feb09dbb2b047b7f566eb5f5bc63fd455814979f3",
-                "sha256:7835fee5238fc0a0baf6c9268fb816b5f5cd9b8793423a75e8cd663c48d073ba",
-                "sha256:8871fcb4b447206904932b54b567923e5be802b9b19b744fdff092bd2f3118d0",
-                "sha256:a7c0192d35635f6fc1174be575cb7915e92e5dd629ee79fdaf0dcfa41a80afb5",
-                "sha256:b1a5ed73ab4c482208d20434f700d514f66ffe2840f63a6252ecc43a9bc77e8a",
-                "sha256:c8226f50b8c34a14608b848dc23a46e5d08397d009446353dad45e04af0c8e28",
-                "sha256:ccad888050f5393f0d6029deea2a33e5ae371fd182a697313bdbd835d3edaf9c",
-                "sha256:dae63f2dbf82882fa3b2a3c49c32bffe144970a573cd68d247af6560fc493ae1",
-                "sha256:e2f69158a7d120fd641d1fa9a921d898e20d52e44a74a6fbbcc570a62a6bc8ab",
-                "sha256:efbadd9b52c060a8fc3b9658744091cb33c31f830b3f074422ed27bad2b18e8f",
-                "sha256:f5660feab44c2e3cb24b2419b998846cbb01c23c7fe645fee45087efa3da2d61",
-                "sha256:fdb8754b453fb15fad3f72cd9cad3e16776f0964d67cf30ebcbf10327a3777a3"
+                "sha256:06f9d8846f2340dfac80ceb20200ea5d1b3f181dd0556b47af4e8e0b24fa0a6b",
+                "sha256:10dbe6e6d2988049b4655b2b739f98785a884d4d6b85bc35133a8fb9a2233176",
+                "sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09",
+                "sha256:30d78ba6bf080eeaf0b7b875d924b15cd46fec5fd044ddfbad38c8ea9171043a",
+                "sha256:328efc0cc70ccb23429d6be184a15ce613f676bdfc85e5fe8ea2a9354b4e9015",
+                "sha256:35020b8886c022ced9282b51b5a875b6d1ab0c387b31a065b84db7c33085ca79",
+                "sha256:5795a0375eb87bfe902e80e0c8cfaedf8af4d49694d69161e5bd3206c18618bb",
+                "sha256:5891ef8abc06576985de8fa88e95ab70641de6c1fca97e2a15820a9b69e51b20",
+                "sha256:637a4014c63fbf42a692d22b55d8ad6968a946b4a6ebc385c5505d9625b6a464",
+                "sha256:67c8301ec94e3bcc8906740fe071391bce40a862b7be0b86fb5382beefecd968",
+                "sha256:6d2fc92002d44746d3e7db7cf9313cf4452f43e9ea77a2c939defce3b10b5c82",
+                "sha256:6ee227b696ca60dd1c507be80a6bc849a5a6ab57ac7352aad1ffec9e8b805f21",
+                "sha256:863714200ada56cbc366dc9ae5291ceb936573155f8bf8e9de92aef51f3ad0f0",
+                "sha256:9b542ced1ec0ceeff5b37d69838106a6348e60db7b8fdd245294dc1d26136265",
+                "sha256:a6342964b43a99dbc72f72812bf88cad8f0217ae9acb47c0d4f141a6416d2d7b",
+                "sha256:ad4efa5fad66b903b4a5f96d91461d90b9507a812b3c5de657d544215bb7877a",
+                "sha256:bc58025940a896d7e5356952228b68f793cf5fcb342be703c3a2669a1488cb72",
+                "sha256:cc1e1de68c8e5444e8f94c3670bb48a2beef0e91dddfd4fcc29595ebd90bb9ce",
+                "sha256:cee3e11161dde1b2a33a904b850b0899e0424cc331b7295f2a9698e79f9a69a0",
+                "sha256:e3556168e2e5c49629f7b0f377070240bd5511e45e25a4497bb0073d9dda776a",
+                "sha256:e8477ec6bbfe0312c128e74644ac8a02ca06bcdb8982d4ee06f209be28cdf163",
+                "sha256:ee8f1f7228cce7dffc2b464f07ce769f478968bfb3dd1254a4c2eeed84928aad",
+                "sha256:fd57160949179ec517d32ac2ac898b5f20d68ed1a9c977346efbac9c2f1e779d"
             ],
             "index": "pypi",
-            "version": "==22.1.0"
+            "version": "==22.3.0"
         },
         "certifi": {
             "hashes": [
@@ -1568,16 +1590,14 @@
         },
         "click": {
             "hashes": [
-                "sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1",
-                "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb"
+                "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e",
+                "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==8.0.4"
+            "markers": "python_version >= '3.7'",
+            "version": "==8.1.3"
         },
         "coverage": {
-            "extras": [
-                "toml"
-            ],
+            "extras": [],
             "hashes": [
                 "sha256:03e2a7826086b91ef345ff18742ee9fc47a6839ccd517061ef8fa1976e652ce9",
                 "sha256:07e6db90cd9686c767dcc593dff16c8c09f9814f5e9c51034066cad3373b914d",
@@ -1671,11 +1691,11 @@
         },
         "faker": {
             "hashes": [
-                "sha256:5536ceb63380f0d598c026b7c330c17d719a19d1a495e9397ee8f5259420a696",
-                "sha256:85ed0cb379e3b7414bdb6b484994beaf9bddc74472c8c35f743c16cf5fc0c314"
+                "sha256:0301ace8365d98f3d0bf6e9a40200c8548e845d3812402ae1daf589effe3fb01",
+                "sha256:b1903db92175d78051858128ada397c7dc76f376f6967975419da232b3ebd429"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==13.3.3"
+            "version": "==13.7.0"
         },
         "filelock": {
             "hashes": [
@@ -1687,18 +1707,18 @@
         },
         "identify": {
             "hashes": [
-                "sha256:3f3244a559290e7d3deb9e9adc7b33594c1bc85a9dd82e0f1be519bf12a1ec17",
-                "sha256:5f06b14366bd1facb88b00540a1de05b69b310cbc2654db3c7e07fa3a4339323"
+                "sha256:3acfe15a96e4272b4ec5662ee3e231ceba976ef63fd9980ed2ce9cc415df393f",
+                "sha256:c83af514ea50bf2be2c4a3f2fb349442b59dc87284558ae9ff54191bff3541d2"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.4.12"
+            "version": "==2.5.0"
         },
         "idna": {
             "hashes": [
                 "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff",
                 "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"
             ],
-            "markers": "python_version >= '3.5'",
+            "markers": "python_version >= '3'",
             "version": "==3.3"
         },
         "imagesize": {
@@ -1726,11 +1746,11 @@
         },
         "jinja2": {
             "hashes": [
-                "sha256:539835f51a74a69f41b848a9645dbdc35b4f20a3b601e2d9a7e22947b15ff119",
-                "sha256:640bed4bb501cbd17194b3cace1dc2126f5b619cf068a726b98192a0fde74ae9"
+                "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852",
+                "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.1.1"
+            "version": "==3.1.2"
         },
         "markupsafe": {
             "hashes": [
@@ -1809,11 +1829,11 @@
         },
         "platformdirs": {
             "hashes": [
-                "sha256:7535e70dfa32e84d4b34996ea99c5e432fa29a708d0f4e394bbcb2a8faa4f16d",
-                "sha256:bcae7cab893c2d310a711b70b24efb93334febe65f8de776ee320b517471e227"
+                "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788",
+                "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.5.1"
+            "version": "==2.5.2"
         },
         "pluggy": {
             "hashes": [
@@ -1825,11 +1845,11 @@
         },
         "pre-commit": {
             "hashes": [
-                "sha256:725fa7459782d7bec5ead072810e47351de01709be838c2ce1726b9591dad616",
-                "sha256:c1a8040ff15ad3d648c70cc3e55b93e4d2d5b687320955505587fd79bbaed06a"
+                "sha256:02226e69564ebca1a070bd1f046af866aa1c318dbc430027c50ab832ed2b73f2",
+                "sha256:5d445ee1fa8738d506881c5d84f83c62bb5be6b2838e32207433647e8e5ebe10"
             ],
             "index": "pypi",
-            "version": "==2.17.0"
+            "version": "==2.18.1"
         },
         "py": {
             "hashes": [
@@ -1849,27 +1869,27 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:44238f1b60a76d78fc8ca0528ee429702aae011c265fe6a8dd8b63049ae41c65",
-                "sha256:4e426f72023d88d03b2fa258de560726ce890ff3b630f88c21cbb8b2503b8c6a"
+                "sha256:5eb116118f9612ff1ee89ac96437bb6b49e8f04d8a13b514ba26f620208e26eb",
+                "sha256:dc9c10fb40944260f6ed4c688ece0cd2048414940f1cea51b8b226318411c519"
             ],
-            "markers": "python_version >= '3.5'",
-            "version": "==2.11.2"
+            "markers": "python_version >= '3.6'",
+            "version": "==2.12.0"
         },
         "pyparsing": {
             "hashes": [
-                "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea",
-                "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484"
+                "sha256:7bf433498c016c4314268d95df76c81b842a4cb2b276fa3312cfb1e1d85f6954",
+                "sha256:ef7b523f6356f763771559412c0d7134753f037822dad1b16945b7b846f7ad06"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==3.0.7"
+            "markers": "python_full_version >= '3.6.8'",
+            "version": "==3.0.8"
         },
         "pytest": {
             "hashes": [
-                "sha256:841132caef6b1ad17a9afde46dc4f6cfa59a05f9555aae5151f73bdf2820ca63",
-                "sha256:92f723789a8fdd7180b6b06483874feca4c48a5c76968e03bb3e7f806a1869ea"
+                "sha256:13d0e3ccfc2b6e26be000cb6568c832ba67ba32e719443bfe725814d3c42433c",
+                "sha256:a06a0425453864a270bc45e71f783330a7428defb4230fb5e6a731fde06ecd45"
             ],
             "index": "pypi",
-            "version": "==7.1.1"
+            "version": "==7.1.2"
         },
         "pytest-cov": {
             "hashes": [
@@ -1995,11 +2015,11 @@
         },
         "sphinx": {
             "hashes": [
-                "sha256:5da895959511473857b6d0200f56865ed62c31e8f82dd338063b84ec022701fe",
-                "sha256:6caad9786055cb1fa22b4a365c1775816b876f91966481765d7d50e9f0dd35cc"
+                "sha256:7bf8ca9637a4ee15af412d1a1d9689fec70523a68ca9bb9127c2f3eeb344e2e6",
+                "sha256:ebf612653238bcc8f4359627a9b7ce44ede6fdd75d9d30f68255c7383d3a6226"
             ],
             "index": "pypi",
-            "version": "==4.4.0"
+            "version": "==4.5.0"
         },
         "sphinx-rtd-theme": {
             "hashes": [
@@ -2081,19 +2101,19 @@
         },
         "tox": {
             "hashes": [
-                "sha256:67e0e32c90e278251fea45b696d0fef3879089ccbe979b0c556d35d5a70e2993",
-                "sha256:be3362472a33094bce26727f5f771ca0facf6dafa217f65875314e9a6600c95c"
+                "sha256:0805727eb4d6b049de304977dfc9ce315a1938e6619c3ab9f38682bb04662a5a",
+                "sha256:37888f3092aa4e9f835fc8cc6dadbaaa0782651c41ef359e3a5743fcb0308160"
             ],
             "index": "pypi",
-            "version": "==3.24.5"
+            "version": "==3.25.0"
         },
         "typing-extensions": {
             "hashes": [
-                "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42",
-                "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2"
+                "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708",
+                "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==4.1.1"
+            "markers": "python_version >= '3.7'",
+            "version": "==4.2.0"
         },
         "urllib3": {
             "hashes": [
@@ -2105,20 +2125,20 @@
         },
         "virtualenv": {
             "hashes": [
-                "sha256:1e8588f35e8b42c6ec6841a13c5e88239de1e6e4e4cedfd3916b306dc826ec66",
-                "sha256:8e5b402037287126e81ccde9432b95a8be5b19d36584f64957060a3488c11ca8"
+                "sha256:e617f16e25b42eb4f6e74096b9c9e37713cf10bf30168fb4a739f3fa8f898a3a",
+                "sha256:ef589a79795589aada0c1c5b319486797c03b67ac3984c48c669c0e4f50df3a5"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==20.14.0"
+            "version": "==20.14.1"
         },
         "zipp": {
             "hashes": [
-                "sha256:9f50f446828eb9d45b267433fd3e9da8d801f614129124863f9c51ebceafb87d",
-                "sha256:b47250dd24f92b7dd6a0a8fc5244da14608f3ca90a5efcd37a3b1642fac9a375"
+                "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad",
+                "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"
             ],
             "index": "pypi",
             "markers": "python_version < '3.9'",
-            "version": "==3.7.0"
+            "version": "==3.8.0"
         }
     }
 }

build-docker-image.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+# Helper script for building the Docker image locally.
+# Parses and provides the nessecary versions of other images to Docker
+# before passing in the rest of script args.
+
+# First Argument: The Dockerfile to build
+# Other Arguments: Additional arguments to docker build
+
+# Example Usage:
+#	./build-docker-image.sh Dockerfile -t paperless-ngx:my-awesome-feature
+
+set -eux
+
+if ! command -v jq;  then
+	echo "jq required"
+	exit 1
+elif [ ! -f "$1" ]; then
+	echo "$1 is not a file, please provide the Dockerfile"
+	exit 1
+fi
+
+# Parse what we can from Pipfile.lock
+pikepdf_version=$(jq ".default.pikepdf.version" Pipfile.lock  | sed 's/=//g' | sed 's/"//g')
+psycopg2_version=$(jq ".default.psycopg2.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
+# Read this from the other config file
+qpdf_version=$(jq ".qpdf.version" .build-config.json | sed 's/"//g')
+jbig2enc_version=$(jq ".jbig2enc.version" .build-config.json | sed 's/"//g')
+# Get the branch name (used for caching)
+branch_name=$(git rev-parse --abbrev-ref HEAD)
+
+# https://docs.docker.com/develop/develop-images/build_enhancements/
+# Required to use cache-from
+export DOCKER_BUILDKIT=1
+
+docker build --file "$1" \
+	--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:"${branch_name}" \
+	--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:dev \
+	--build-arg JBIG2ENC_VERSION="${jbig2enc_version}" \
+	--build-arg QPDF_VERSION="${qpdf_version}" \
+	--build-arg PIKEPDF_VERSION="${pikepdf_version}" \
+	--build-arg PSYCOPG2_VERSION="${psycopg2_version}" "${@:2}" .

docker-builders/Dockerfile.frontend

@@ -0,0 +1,14 @@
+# This Dockerfile compiles the frontend
+# Inputs: None
+
+FROM node:16-bullseye-slim AS compile-frontend
+
+COPY ./src /src/src
+COPY ./src-ui /src/src-ui
+
+WORKDIR /src/src-ui
+RUN set -eux \
+  && npm update npm -g \
+  && npm ci --no-optional
+RUN set -eux \
+  && ./node_modules/.bin/ng build --configuration production

docker-builders/Dockerfile.jbig2enc

@@ -0,0 +1,39 @@
+# This Dockerfile compiles the jbig2enc library
+# Inputs:
+#    - JBIG2ENC_VERSION - the Git tag to checkout and build
+
+FROM debian:bullseye-slim as main
+
+LABEL org.opencontainers.image.description="A intermediate image with jbig2enc built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+  build-essential \
+  automake \
+  libtool \
+  libleptonica-dev \
+  zlib1g-dev \
+  git \
+  ca-certificates"
+
+WORKDIR /usr/src/jbig2enc
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+RUN apt-get update --quiet \
+  && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
+  && rm -rf /var/lib/apt/lists/*
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+ARG JBIG2ENC_VERSION
+
+RUN set -eux \
+  && git clone --quiet --branch $JBIG2ENC_VERSION https://github.com/agl/jbig2enc .
+RUN set -eux \
+  && ./autogen.sh
+RUN set -eux \
+  && ./configure && make

docker-builders/Dockerfile.pikepdf

@@ -0,0 +1,92 @@
+# This Dockerfile builds the pikepdf wheel
+# Inputs:
+#    - REPO - Docker repository to pull qpdf from
+#    - QPDF_VERSION - The image qpdf version to copy .deb files from
+#    - PIKEPDF_GIT_TAG - The Git tag to clone and build from
+#    - PIKEPDF_VERSION - Used to force the built pikepdf version to match
+
+# Default to pulling from the main repo registry when manually building
+ARG REPO="paperless-ngx/paperless-ngx"
+
+ARG QPDF_VERSION
+FROM ghcr.io/${REPO}/builder/qpdf:${QPDF_VERSION} as qpdf-builder
+
+# This does nothing, except provide a name for a copy below
+
+FROM python:3.9-slim-bullseye as main
+
+LABEL org.opencontainers.image.description="A intermediate image with pikepdf wheel built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+  build-essential \
+  python3-dev \
+  python3-pip \
+  git \
+  # qpdf requirement - https://github.com/qpdf/qpdf#crypto-providers
+  libgnutls28-dev \
+  # lxml requrements - https://lxml.de/installation.html
+  libxml2-dev \
+  libxslt1-dev \
+  # Pillow requirements - https://pillow.readthedocs.io/en/stable/installation.html#external-libraries
+  # JPEG functionality
+  libjpeg62-turbo-dev \
+  # conpressed PNG
+  zlib1g-dev \
+  # compressed TIFF
+  libtiff-dev \
+  # type related services
+  libfreetype-dev \
+  # color management
+  liblcms2-dev \
+  # WebP format
+  libwebp-dev \
+  # JPEG 2000
+  libopenjp2-7-dev \
+  # improved color quantization
+  libimagequant-dev \
+  # complex text layout support
+  libraqm-dev"
+
+WORKDIR /usr/src
+
+COPY --from=qpdf-builder /usr/src/qpdf/*.deb ./
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+
+RUN set -eux \
+  && apt-get update --quiet \
+  && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
+  && dpkg --install libqpdf28_*.deb \
+  && dpkg --install libqpdf-dev_*.deb \
+  && python3 -m pip install --no-cache-dir --upgrade \
+    pip \
+    wheel \
+    # https://pikepdf.readthedocs.io/en/latest/installation.html#requirements
+    pybind11 \
+  && rm -rf /var/lib/apt/lists/*
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+ARG PIKEPDF_GIT_TAG
+ARG PIKEPDF_VERSION
+
+RUN set -eux \
+  && echo "building pikepdf wheel" \
+  # Note the v in the tag name here
+  && git clone --quiet --depth 1 --branch "${PIKEPDF_GIT_TAG}" https://github.com/pikepdf/pikepdf.git \
+  && cd pikepdf \
+  # pikepdf seems to specifciy either a next version when built OR
+  # a post release tag.
+  # In either case, this won't match what we want from requirements.txt
+  # Directly modify the setup.py to set the version we just checked out of Git
+  && sed -i "s/use_scm_version=True/version=\"${PIKEPDF_VERSION}\"/g" setup.py \
+  # https://github.com/pikepdf/pikepdf/issues/323
+  && rm pyproject.toml \
+  && mkdir wheels \
+  && python3 -m pip wheel . --wheel-dir wheels \
+  && ls -ahl wheels

docker-builders/Dockerfile.psycopg2

@@ -0,0 +1,45 @@
+# This Dockerfile builds the psycopg2 wheel
+# Inputs:
+#    - PSYCOPG2_GIT_TAG - The Git tag to clone and build from
+#    - PSYCOPG2_VERSION - Unused, kept for future possible usage
+
+FROM python:3.9-slim-bullseye as main
+
+LABEL org.opencontainers.image.description="A intermediate image with psycopg2 wheel built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+  build-essential \
+  git \
+  python3-dev \
+  python3-pip \
+  # https://www.psycopg.org/docs/install.html#prerequisites
+  libpq-dev"
+
+WORKDIR /usr/src
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+
+RUN set -eux \
+  && apt-get update --quiet \
+  && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
+  && rm -rf /var/lib/apt/lists/* \
+  && python3 -m pip install --no-cache-dir --upgrade pip wheel
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+ARG PSYCOPG2_GIT_TAG
+ARG PSYCOPG2_VERSION
+
+RUN set -eux \
+  && echo "Building psycopg2 wheel" \
+  && cd /usr/src \
+  && git clone --quiet --depth 1 --branch ${PSYCOPG2_GIT_TAG} https://github.com/psycopg/psycopg2.git \
+  && cd psycopg2 \
+  && mkdir wheels \
+  && python3 -m pip wheel . --wheel-dir wheels \
+  && ls -ahl wheels/

docker-builders/Dockerfile.qpdf

@@ -0,0 +1,53 @@
+FROM debian:bullseye-slim as main
+
+LABEL org.opencontainers.image.description="A intermediate image with qpdf built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+  build-essential \
+  debhelper \
+  debian-keyring \
+  devscripts \
+  equivs  \
+  libtool \
+  # https://qpdf.readthedocs.io/en/stable/installation.html#system-requirements
+  libjpeg62-turbo-dev \
+  libgnutls28-dev \
+  packaging-dev \
+  zlib1g-dev"
+
+WORKDIR /usr/src
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+
+RUN set -eux \
+  && apt-get update --quiet \
+  && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
+  && rm -rf /var/lib/apt/lists/*
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+# This must match to pikepdf's minimum at least
+ARG QPDF_VERSION
+
+# In order to get the required version of qpdf, it is backported from bookwork
+# and then built from source
+RUN set -eux \
+  && echo "Building qpdf" \
+  && echo "deb-src http://deb.debian.org/debian/ bookworm main" > /etc/apt/sources.list.d/bookworm-src.list \
+  && apt-get update \
+  && mkdir qpdf \
+  && cd qpdf \
+  && apt-get source --yes --quiet qpdf=${QPDF_VERSION}-1/bookworm \
+  && rm -rf /var/lib/apt/lists/* \
+  && cd qpdf-$QPDF_VERSION \
+  # We don't need to build the tests (also don't run them)
+  && rm -rf libtests \
+  && DEBEMAIL=hello@paperless-ngx.com debchange --bpo \
+  && export DEB_BUILD_OPTIONS="terse nocheck nodoc parallel=2" \
+  && dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes \
+  && ls -ahl ../*.deb

docker/compose/docker-compose.env

@@ -22,6 +22,10 @@
 # Docker setup does not use the configuration file.
 # A few commonly adjusted settings are provided below.
 
+# This is required if you will be exposing Paperless-ngx on a public domain
+# (if doing so please consider security measures such as reverse proxy)
+#PAPERLESS_URL=https://paperless.example.com
+
 # Adjust this key if you plan to make paperless available publicly. It should
 # be a very long sequence of random characters. You don't need to remember it.
 #PAPERLESS_SECRET_KEY=change-me

docker/compose/docker-compose.portainer.yml

@@ -55,7 +55,7 @@ services:
     ports:
       - 8010:8000
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s
       timeout: 10s
       retries: 5

docker/compose/docker-compose.postgres-tika.yml

@@ -1,7 +1,6 @@
 # docker-compose file for running paperless from the docker container registry.
 # This file contains everything paperless needs to run.
-# Paperless supports amd64, arm and arm64 hardware. The apache/tika image
-# does not support arm or arm64, however.
+# Paperless supports amd64, arm and arm64 hardware.
 #
 # All compose files of paperless configure paperless in the following way:
 #
@@ -60,7 +59,7 @@ services:
     ports:
       - 8000:8000
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s
       timeout: 10s
       retries: 5
@@ -78,14 +77,14 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: gotenberg/gotenberg:7
+    image: gotenberg/gotenberg:7.4
     restart: unless-stopped
     command:
       - "gotenberg"
       - "--chromium-disable-routes=true"
 
   tika:
-    image: apache/tika
+    image: ghcr.io/paperless-ngx/tika:latest
     restart: unless-stopped
 
 volumes:

docker/compose/docker-compose.postgres.yml

@@ -53,7 +53,7 @@ services:
     ports:
       - 8000:8000
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s
       timeout: 10s
       retries: 5

docker/compose/docker-compose.sqlite-tika.arm.yml

@@ -1,84 +0,0 @@
-# docker-compose file for running paperless from the docker container registry.
-# This file contains everything paperless needs to run.
-# Paperless supports amd64, arm and arm64 hardware.
-#
-# All compose files of paperless configure paperless in the following way:
-#
-# - Paperless is (re)started on system boot, if it was running before shutdown.
-# - Docker volumes for storing data are managed by Docker.
-# - Folders for importing and exporting files are created in the same directory
-#   as this file and mounted to the correct folders inside the container.
-# - Paperless listens on port 8000.
-#
-# SQLite is used as the database. The SQLite file is stored in the data volume.
-#
-# iwishiwasaneagle/apache-tika-arm docker image is used to enable arm64 arch
-# which apache/tika does not currently support.
-#
-# In addition to that, this docker-compose file adds the following optional
-# configurations:
-#
-# - Apache Tika and Gotenberg servers are started with paperless and paperless
-#   is configured to use these services. These provide support for consuming
-#   Office documents (Word, Excel, Power Point and their LibreOffice counter-
-#   parts.
-#
-# To install and update paperless with this file, do the following:
-#
-# - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env'
-#   and '.env' into a folder.
-# - Run 'docker-compose pull'.
-# - Run 'docker-compose run --rm webserver createsuperuser' to create a user.
-# - Run 'docker-compose up -d'.
-#
-# For more extensive installation and update instructions, refer to the
-# documentation.
-
-version: "3.4"
-services:
-  broker:
-    image: redis:6.0
-    restart: unless-stopped
-    volumes:
-      - redisdata:/data
-
-  webserver:
-    image: ghcr.io/paperless-ngx/paperless-ngx:latest
-    restart: unless-stopped
-    depends_on:
-      - broker
-      - gotenberg
-      - tika
-    ports:
-      - 8000:8000
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000"]
-      interval: 30s
-      timeout: 10s
-      retries: 5
-    volumes:
-      - data:/usr/src/paperless/data
-      - media:/usr/src/paperless/media
-      - ./export:/usr/src/paperless/export
-      - ./consume:/usr/src/paperless/consume
-    env_file: docker-compose.env
-    environment:
-      PAPERLESS_REDIS: redis://broker:6379
-      PAPERLESS_TIKA_ENABLED: 1
-      PAPERLESS_TIKA_GOTENBERG_ENDPOINT: http://gotenberg:3000
-      PAPERLESS_TIKA_ENDPOINT: http://tika:9998
-
-  gotenberg:
-    image: thecodingmachine/gotenberg
-    restart: unless-stopped
-    environment:
-      DISABLE_GOOGLE_CHROME: 1
-
-  tika:
-    image: iwishiwasaneagle/apache-tika-arm@sha256:a78c25ffe57ecb1a194b2859d42a61af46e9e845191512b8f1a4bf90578ffdfd
-    restart: unless-stopped
-
-volumes:
-  data:
-  media:
-  redisdata:

docker/compose/docker-compose.sqlite-tika.yml

@@ -1,8 +1,6 @@
 # docker-compose file for running paperless from the docker container registry.
 # This file contains everything paperless needs to run.
-# Paperless supports amd64, arm and arm64 hardware. The apache/tika image
-# does not support arm or arm64, however.
-#
+# Paperless supports amd64, arm and arm64 hardware.
 # All compose files of paperless configure paperless in the following way:
 #
 # - Paperless is (re)started on system boot, if it was running before shutdown.
@@ -50,7 +48,7 @@ services:
     ports:
       - 8000:8000
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s
       timeout: 10s
       retries: 5
@@ -67,14 +65,14 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: gotenberg/gotenberg:7
+    image: gotenberg/gotenberg:7.4
     restart: unless-stopped
     command:
       - "gotenberg"
       - "--chromium-disable-routes=true"
 
   tika:
-    image: apache/tika
+    image: ghcr.io/paperless-ngx/tika:latest
     restart: unless-stopped
 
 volumes:

docker/compose/docker-compose.sqlite.yml

@@ -39,7 +39,7 @@ services:
     ports:
       - 8000:8000
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s
       timeout: 10s
       retries: 5

docker/docker-prepare.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -e
+
 wait_for_postgres() {
 	attempt_num=1
 	max_attempts=5
@@ -7,7 +9,7 @@ wait_for_postgres() {
 	echo "Waiting for PostgreSQL to start..."
 
 	host="${PAPERLESS_DBHOST:=localhost}"
-	port="${PAPERLESS_DBPORT:=5342}"
+	port="${PAPERLESS_DBPORT:=5432}"
 
 
 	while [ ! "$(pg_isready -h $host -p $port)" ]; do
@@ -25,6 +27,14 @@ wait_for_postgres() {
 	done
 }
 
+wait_for_redis() {
+	# We use a Python script to send the Redis ping
+	# instead of installing redis-tools just for 1 thing
+	if ! python3 /sbin/wait-for-redis.py; then
+		exit 1
+	fi
+}
+
 migrations() {
 	(
 		# flock is in place to prevent multiple containers from doing migrations
@@ -58,6 +68,8 @@ do_work() {
 		wait_for_postgres
 	fi
 
+	wait_for_redis
+
 	migrations
 
 	search_index

docker/install_management_commands.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -eu
+
 for command in document_archiver document_exporter document_importer mail_fetcher document_create_classifier document_index document_renamer document_retagger document_thumbnails document_sanity_checker manage_superuser;
 do
 	echo "installing $command..."

docker/supervisord.conf

@@ -28,6 +28,7 @@ stderr_logfile_maxbytes=0
 [program:scheduler]
 command=python3 manage.py qcluster
 user=paperless
+stopasgroup = true
 
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0

docker/wait-for-redis.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+"""
+Simple script which attempts to ping the Redis broker as set in the environment for
+a certain number of times, waiting a little bit in between
+
+"""
+import os
+import sys
+import time
+from typing import Final
+
+from redis import Redis
+
+if __name__ == "__main__":
+
+    MAX_RETRY_COUNT: Final[int] = 5
+    RETRY_SLEEP_SECONDS: Final[int] = 5
+
+    REDIS_URL: Final[str] = os.getenv("PAPERLESS_REDIS", "redis://localhost:6379")
+
+    print(f"Waiting for Redis: {REDIS_URL}", flush=True)
+
+    attempt = 0
+    with Redis.from_url(url=REDIS_URL) as client:
+        while attempt < MAX_RETRY_COUNT:
+            try:
+                client.ping()
+                break
+            except Exception:
+                print(
+                    f"Redis ping #{attempt} failed, waiting {RETRY_SLEEP_SECONDS}s",
+                    flush=True,
+                )
+                time.sleep(RETRY_SLEEP_SECONDS)
+                attempt += 1
+
+    if attempt >= MAX_RETRY_COUNT:
+        print(f"Failed to connect to: {REDIS_URL}")
+        sys.exit(os.EX_UNAVAILABLE)
+    else:
+        print(f"Connected to Redis broker: {REDIS_URL}")
+        sys.exit(os.EX_OK)

docs/Dockerfile

@@ -1,10 +1,10 @@
 FROM python:3.5.1
 
 # Install Sphinx and Pygments
-RUN pip install Sphinx Pygments
-
+RUN pip install --no-cache-dir Sphinx Pygments \
   # Setup directories, copy data
-RUN mkdir /build
+  && mkdir /build
+
 COPY . /build
 WORKDIR /build/docs
 

docs/_static/css/custom.css

@@ -0,0 +1,592 @@
+/* Variables */
+:root {
+  --color-text-body: #5c5962;
+  --color-text-body-light: #fcfcfc;
+  --color-text-anchor: #7253ed;
+  --color-text-alt: rgba(0, 0, 0, 0.3);
+  --color-text-title: #27262b;
+  --color-text-code-inline: #e74c3c;
+  --color-text-code-nt: #062873;
+  --color-text-selection: #b19eff;
+  --color-bg-body: #fcfcfc;
+  --color-bg-body-alt: #f3f6f6;
+  --color-bg-side-nav: #f5f6fa;
+  --color-bg-side-nav-hover: #ebedf5;
+  --color-bg-code-block: var(--color-bg-side-nav);
+  --color-border: #eeebee;
+  --color-btn-neutral-bg: #f3f6f6;
+  --color-btn-neutral-bg-hover: #e5ebeb;
+  --color-success-title: #1abc9c;
+  --color-success-body: #dbfaf4;
+  --color-warning-title: #f0b37e;
+  --color-warning-body: #ffedcc;
+  --color-danger-title: #f29f97;
+  --color-danger-body: #fdf3f2;
+  --color-info-title: #6ab0de;
+  --color-info-body: #e7f2fa;
+}
+
+.dark-mode {
+  --color-text-body: #abb2bf;
+  --color-text-body-light: #9499a2;
+  --color-text-alt: rgba(0255, 255, 255, 0.5);
+  --color-text-title: var(--color-text-anchor);
+  --color-text-code-inline: #abb2bf;
+  --color-text-code-nt: #2063f3;
+  --color-text-selection: #030303;
+  --color-bg-body: #1d1d20 !important;
+  --color-bg-body-alt: #131315;
+  --color-bg-side-nav: #18181a;
+  --color-bg-side-nav-hover: #101216;
+  --color-bg-code-block: #101216;
+  --color-border: #47494f;
+  --color-btn-neutral-bg: #242529;
+  --color-btn-neutral-bg-hover: #101216;
+  --color-success-title: #02120f;
+  --color-success-body: #041b17;
+  --color-warning-title: #1b0e03;
+  --color-warning-body: #371d06;
+  --color-danger-title: #120902;
+  --color-danger-body: #1b0503;
+  --color-info-title: #020608;
+  --color-info-body: #06141e;
+}
+
+* {
+  transition: background-color 0.3s ease, border-color 0.3s ease;
+}
+
+/* Typography */
+body {
+  font-family: system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif;
+  font-size: inherit;
+  line-height: 1.4;
+  color: var(--color-text-body);
+}
+
+h1, h2, h3, h4, h5, h6 {
+  font-family: inherit;
+}
+
+.rst-content .toctree-wrapper>p.caption, .rst-content h1, .rst-content h2, .rst-content h3, .rst-content h4, .rst-content h5, .rst-content h6 {
+  padding-top: .5em;
+}
+
+p, .main-content-wrap, .rst-content .section ul, .rst-content .toctree-wrapper ul, .rst-content section ul, .wy-plain-list-disc, article ul {
+  line-height: 1.6;
+}
+
+pre, .code, .rst-content .linenodiv pre, .rst-content div[class^=highlight] pre, .rst-content pre.literal-block {
+  font-family: "SFMono-Regular", Menlo,Consolas, Monospace;
+  font-size: 0.75em;
+  line-height: 1.8;
+}
+
+.wy-menu-vertical li.toctree-l3,.wy-menu-vertical li.toctree-l4 {
+  font-size: 1rem
+}
+
+.rst-versions {
+  font-family: inherit;
+  line-height: 1;
+}
+
+footer, footer p {
+  font-size: .8rem;
+}
+
+footer .rst-footer-buttons {
+  font-size: 1rem;
+}
+
+@media (max-width: 400px) {
+  /* break code lines on mobile */
+  pre, code {
+    word-break: break-word;
+  }
+}
+
+
+/* Layout */
+.wy-side-nav-search, .wy-menu-vertical {
+  width: auto;
+}
+
+.wy-nav-side {
+  z-index: 0;
+  display: flex;
+  flex-wrap: wrap;
+  background-color: var(--color-bg-side-nav);
+}
+
+.wy-side-scroll {
+  width: 100%;
+  overflow-y: auto;
+}
+
+@media (min-width: 66.5rem) {
+  .wy-side-scroll {
+      width:264px
+  }
+}
+
+@media (min-width: 50rem) {
+  .wy-nav-side {
+      flex-wrap: nowrap;
+      position: fixed;
+      width: 248px;
+      height: 100%;
+      flex-direction: column;
+      border-right: 1px solid var(--color-border);
+      align-items:flex-end
+  }
+}
+
+@media (min-width: 66.5rem) {
+  .wy-nav-side {
+      width: calc((100% - 1064px) / 2 + 264px);
+      min-width:264px
+  }
+}
+
+@media (min-width: 50rem) {
+  .wy-nav-content-wrap {
+      position: relative;
+      max-width: 800px;
+      margin-left:248px
+  }
+}
+
+@media (min-width: 66.5rem) {
+  .wy-nav-content-wrap {
+      margin-left:calc((100% - 1064px) / 2 + 264px)
+  }
+}
+
+
+/* Colors */
+body.wy-body-for-nav,
+.wy-nav-content {
+  background: var(--color-bg-body);
+}
+
+.wy-nav-side {
+  border-right: 1px solid var(--color-border);
+}
+
+.wy-side-nav-search, .wy-nav-top {
+  background: var(--color-bg-side-nav);
+  border-bottom: 1px solid var(--color-border);
+}
+
+.wy-nav-content-wrap {
+  background: inherit;
+}
+
+.wy-side-nav-search > a, .wy-nav-top a, .wy-nav-top i {
+  color: var(--color-text-title);
+}
+
+.wy-side-nav-search > a:hover, .wy-nav-top a:hover {
+  background: transparent;
+}
+
+.wy-side-nav-search > div.version {
+  color: var(--color-text-alt)
+}
+
+.wy-side-nav-search > div[role="search"] {
+  border-top: 1px solid var(--color-border);
+}
+
+.wy-menu-vertical li.toctree-l2.current>a, .wy-menu-vertical li.toctree-l2.current li.toctree-l3>a,
+.wy-menu-vertical li.toctree-l3.current>a, .wy-menu-vertical li.toctree-l3.current li.toctree-l4>a {
+  background: var(--color-bg-side-nav);
+}
+
+.rst-content .highlighted {
+  background: #eedd85;
+  box-shadow: 0 0 0 2px #eedd85;
+  font-weight: 600;
+}
+
+.wy-side-nav-search input[type=text],
+html.writer-html5 .rst-content table.docutils th {
+  color: var(--color-text-body);
+}
+
+.rst-content table.docutils:not(.field-list) tr:nth-child(2n-1) td,
+.wy-table-backed,
+.wy-table-odd td,
+.wy-table-striped tr:nth-child(2n-1) td {
+  background-color: var(--color-bg-body-alt);
+}
+
+.rst-content table.docutils,
+.wy-table-bordered-all,
+html.writer-html5 .rst-content table.docutils th,
+.rst-content table.docutils td,
+.wy-table-bordered-all td,
+hr {
+  border-color: var(--color-border) !important;
+}
+
+::selection {
+  background: var(--color-text-selection);
+}
+
+/* Ridiculous rules are taken from sphinx_rtd */
+.rst-content .admonition-title,
+.wy-alert-title {
+  color: var(--color-text-body-light);
+}
+
+.rst-content .hint,
+.rst-content .important,
+.rst-content .tip,
+.rst-content .wy-alert-success,
+.wy-alert.wy-alert-success {
+  background: var(--color-success-body);
+}
+
+.rst-content .hint .admonition-title,
+.rst-content .hint .wy-alert-title,
+.rst-content .important .admonition-title,
+.rst-content .important .wy-alert-title,
+.rst-content .tip .admonition-title,
+.rst-content .tip .wy-alert-title,
+.rst-content .wy-alert-success .admonition-title,
+.rst-content .wy-alert-success .wy-alert-title,
+.wy-alert.wy-alert-success .rst-content .admonition-title,
+.wy-alert.wy-alert-success .wy-alert-title {
+  background-color: var(--color-success-title);
+}
+
+.rst-content .admonition-todo,
+.rst-content .attention,
+.rst-content .caution,
+.rst-content .warning,
+.rst-content .wy-alert-warning,
+.wy-alert.wy-alert-warning {
+  background: var(--color-warning-body);
+}
+
+.rst-content .admonition-todo .admonition-title,
+.rst-content .admonition-todo .wy-alert-title,
+.rst-content .attention .admonition-title,
+.rst-content .attention .wy-alert-title,
+.rst-content .caution .admonition-title,
+.rst-content .caution .wy-alert-title,
+.rst-content .warning .admonition-title,
+.rst-content .warning .wy-alert-title,
+.rst-content .wy-alert-warning .admonition-title,
+.rst-content .wy-alert-warning .wy-alert-title,
+.rst-content .wy-alert.wy-alert-warning .admonition-title,
+.wy-alert.wy-alert-warning .rst-content .admonition-title,
+.wy-alert.wy-alert-warning .wy-alert-title {
+  background: var(--color-warning-title);
+}
+
+.rst-content .danger,
+.rst-content .error,
+.rst-content .wy-alert-danger,
+.wy-alert.wy-alert-danger {
+  background: var(--color-danger-body);
+}
+
+.rst-content .danger .admonition-title,
+.rst-content .danger .wy-alert-title,
+.rst-content .error .admonition-title,
+.rst-content .error .wy-alert-title,
+.rst-content .wy-alert-danger .admonition-title,
+.rst-content .wy-alert-danger .wy-alert-title,
+.wy-alert.wy-alert-danger .rst-content .admonition-title,
+.wy-alert.wy-alert-danger .wy-alert-title {
+  background: var(--color-danger-title);
+}
+
+.rst-content .note,
+.rst-content .seealso,
+.rst-content .wy-alert-info,
+.wy-alert.wy-alert-info {
+  background: var(--color-info-body);
+}
+
+.rst-content .note .admonition-title,
+.rst-content .note .wy-alert-title,
+.rst-content .seealso .admonition-title,
+.rst-content .seealso .wy-alert-title,
+.rst-content .wy-alert-info .admonition-title,
+.rst-content .wy-alert-info .wy-alert-title,
+.wy-alert.wy-alert-info .rst-content .admonition-title,
+.wy-alert.wy-alert-info .wy-alert-title {
+  background: var(--color-info-title);
+}
+
+
+
+/* Links */
+a, a:visited,
+.wy-menu-vertical a,
+a.icon.icon-home,
+.wy-menu-vertical li.toctree-l1.current > a.current {
+  color: var(--color-text-anchor);
+  text-decoration: none;
+}
+
+a:hover, .wy-breadcrumbs-aside a {
+  color: var(--color-text-anchor); /* reset */
+}
+
+.rst-versions a, .rst-versions .rst-current-version {
+  color: #var(--color-text-anchor);
+}
+
+.wy-nav-content a.reference, .wy-nav-content a:not([class]) {
+  background-image: linear-gradient(var(--color-border) 0%, var(--color-border) 100%);
+  background-repeat: repeat-x;
+  background-position: 0 100%;
+  background-size: 1px 1px;
+}
+
+.wy-nav-content a.reference:hover, .wy-nav-content a:not([class]):hover {
+  background-image: linear-gradient(rgba(114,83,237,0.45) 0%, rgba(114,83,237,0.45) 100%);
+  background-size: 1px 1px;
+}
+
+.wy-menu-vertical a:hover,
+.wy-menu-vertical li.current a:hover,
+.wy-menu-vertical a:active {
+  background: var(--color-bg-side-nav-hover) !important;
+  color: var(--color-text-body);
+}
+
+.wy-menu-vertical li.toctree-l1.current>a,
+.wy-menu-vertical li.current>a,
+.wy-menu-vertical li.on a {
+  background-color: var(--color-bg-side-nav-hover);
+  border: none;
+  font-weight: normal;
+}
+
+.wy-menu-vertical li.current {
+  background-color: inherit;
+}
+
+.wy-menu-vertical li.current a {
+  border-right: none;
+}
+
+.wy-menu-vertical li.toctree-l2 a,
+.wy-menu-vertical li.toctree-l3 a,
+.wy-menu-vertical li.toctree-l4 a,
+.wy-menu-vertical li.toctree-l5 a,
+.wy-menu-vertical li.toctree-l6 a,
+.wy-menu-vertical li.toctree-l7 a,
+.wy-menu-vertical li.toctree-l8 a,
+.wy-menu-vertical li.toctree-l9 a,
+.wy-menu-vertical li.toctree-l10 a {
+  color: var(--color-text-body);
+}
+
+a.image-reference, a.image-reference:hover {
+  background: none !important;
+}
+
+a.image-reference img {
+  cursor: zoom-in;
+}
+
+
+/* Code blocks */
+.rst-content code, .rst-content tt, code {
+  padding: 0.25em;
+  font-weight: 400;
+  background-color: var(--color-bg-code-block);
+  border: 1px solid var(--color-border);
+  border-radius: 4px;
+}
+
+.rst-content div[class^=highlight], .rst-content pre.literal-block {
+  padding: 0.7rem;
+  margin-top: 0;
+  margin-bottom: 0.75rem;
+  overflow-x: auto;
+  background-color: var(--color-bg-side-nav);
+  border-color: var(--color-border);
+  border-radius: 4px;
+  box-shadow: none;
+}
+
+.rst-content .admonition-title,
+.rst-content div.admonition,
+.wy-alert-title {
+  padding: 10px 12px;
+  border-top-left-radius: 4px;
+  border-top-right-radius: 4px;
+}
+
+.highlight .go {
+  color: inherit;
+}
+
+.highlight .nt {
+  color: var(--color-text-code-nt);
+}
+
+.rst-content code.literal,
+.rst-content tt.literal {
+  border-color: var(--color-border);
+  background-color: var(--color-border);
+  color: var(--color-text-code-inline)
+}
+
+
+/* Search */
+.wy-side-nav-search input[type=text] {
+  border: none;
+  border-radius: 0;
+  background-color: transparent;
+  font-family: inherit;
+  font-size: .85rem;
+  box-shadow: none;
+  padding: .7rem 1rem .7rem 2.8rem;
+  margin: 0;
+}
+
+#rtd-search-form {
+  position: relative;
+}
+
+#rtd-search-form:before {
+  font: normal normal normal 14px/1 FontAwesome;
+  font-size: inherit;
+  text-rendering: auto;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+  content: "\f002";
+  color: var(--color-text-alt);
+  position: absolute;
+  left: 1.5rem;
+  top: .7rem;
+}
+
+/* Side nav */
+.wy-side-nav-search {
+  padding: 1rem 0 0 0;
+}
+
+.wy-menu-vertical li a button.toctree-expand {
+  float: right;
+  margin-right: -1.5em;
+  padding: 0 .5em;
+}
+
+.wy-menu-vertical a,
+.wy-menu-vertical li.current>a,
+.wy-menu-vertical li.current li>a {
+  padding-right: 1.5em !important;
+}
+
+.wy-menu-vertical li.current li>a.current {
+  font-weight: 600;
+}
+
+/* Misc spacing */
+.rst-content .admonition-title, .wy-alert-title {
+  padding: 10px 12px;
+}
+
+/* Buttons */
+.btn {
+  display: inline-block;
+  box-sizing: border-box;
+  padding: 0.3em 1em;
+  margin: 0;
+  font-family: inherit;
+  font-size: inherit;
+  font-weight: 500;
+  line-height: 1.5;
+  color: #var(--color-text-anchor);
+  text-decoration: none;
+  vertical-align: baseline;
+  background-color: #f7f7f7;
+  border-width: 0;
+  border-radius: 4px;
+  box-shadow: 0 1px 2px rgba(0,0,0,0.12),0 3px 10px rgba(0,0,0,0.08);
+  appearance: none;
+}
+
+.btn:active {
+  padding: 0.3em 1em;
+}
+
+.rst-content .btn:focus {
+  outline: 1px solid #ccc;
+}
+
+.rst-content .btn-neutral, .rst-content .btn span.fa {
+  color: var(--color-text-body) !important;
+}
+
+.btn-neutral {
+  background-color: var(--color-btn-neutral-bg) !important;
+  color: var(--color-btn-neutral-text) !important;
+  border: 1px solid var(--color-btn-neutral-bg);
+}
+
+.btn:hover, .btn-neutral:hover {
+  background-color: var(--color-btn-neutral-bg-hover) !important;
+}
+
+
+/* Icon overrides */
+.wy-side-nav-search a.icon-home:before {
+  display: none;
+}
+
+.fa-minus-square-o:before,.wy-menu-vertical li.current>a button.toctree-expand:before,.wy-menu-vertical li.on a button.toctree-expand:before {
+  content: "\f106"; /* fa-angle-up */
+}
+
+.fa-plus-square-o:before, .wy-menu-vertical li button.toctree-expand:before {
+  content: "\f107"; /* fa-angle-down */
+}
+
+
+/* Misc */
+.wy-nav-top {
+  line-height: 36px;
+}
+
+.wy-nav-top > i {
+  font-size: 24px;
+  padding: 8px 0 0 2px;
+  color:#var(--color-text-anchor);
+}
+
+.rst-content table.docutils td,
+.rst-content table.docutils th,
+.rst-content table.field-list td,
+.rst-content table.field-list th,
+.wy-table td,
+.wy-table th {
+  padding: 8px 14px;
+}
+
+.dark-mode-toggle {
+  position: absolute;
+  top: 14px;
+  right: 12px;
+  height: 20px;
+  width: 24px;
+  z-index: 10;
+  border: none;
+  background-color: transparent;
+  color: inherit;
+  opacity: 0.7;
+}
+
+.wy-nav-content-wrap {
+  z-index: 20;
+}

docs/_static/custom.css

@@ -1,14 +0,0 @@
-/* override table width restrictions */
-@media screen and (min-width: 767px) {
-
-  .wy-table-responsive table td {
-    /* !important prevents the common CSS stylesheets from
-       overriding this as on RTD they are loaded after this stylesheet */
-    white-space: normal !important;
-  }
-
-  .wy-table-responsive {
-    overflow: visible !important;
-  }
-
-}

docs/_static/js/darkmode.js

@@ -0,0 +1,47 @@
+let toggleButton;
+let icon;
+
+function load() {
+	"use strict";
+
+	toggleButton = document.createElement("button");
+	toggleButton.setAttribute("title", "Toggle dark mode");
+	toggleButton.classList.add("dark-mode-toggle");
+	icon = document.createElement("i");
+	icon.classList.add("fa", darkModeState ? "fa-sun-o" : "fa-moon-o");
+	toggleButton.appendChild(icon);
+	document.body.prepend(toggleButton);
+
+	// Listen for changes in the OS settings
+	// addListener is used because older versions of Safari don't support addEventListener
+	// prefersDarkQuery set in <head>
+	if (prefersDarkQuery) {
+		prefersDarkQuery.addListener(function (evt) {
+			toggleDarkMode(evt.matches);
+		});
+	}
+
+	// Initial setting depending on the prefers-color-mode or localstorage
+	// darkModeState should be set in the document <head> to prevent flash
+	if (darkModeState == undefined) darkModeState = false;
+	toggleDarkMode(darkModeState);
+
+	// Toggles the "dark-mode" class on click and sets localStorage state
+	toggleButton.addEventListener("click", () => {
+		darkModeState = !darkModeState;
+
+		toggleDarkMode(darkModeState);
+		localStorage.setItem("dark-mode", darkModeState);
+	});
+}
+
+function toggleDarkMode(state) {
+	document.documentElement.classList.toggle("dark-mode", state);
+	document.documentElement.classList.toggle("light-mode", !state);
+	icon.classList.remove("fa-sun-o");
+	icon.classList.remove("fa-moon-o");
+	icon.classList.add(state ? "fa-sun-o" : "fa-moon-o");
+	darkModeState = state;
+}
+
+document.addEventListener("DOMContentLoaded", load);

docs/_templates/layout.html

@@ -0,0 +1,13 @@
+{% extends "!layout.html" %}
+{% block extrahead %}
+    <script>
+        // MediaQueryList object
+        const prefersDarkQuery = window.matchMedia("(prefers-color-scheme: dark)");
+        const lsDark = localStorage.getItem("dark-mode");
+        let darkModeState = lsDark !== null ? lsDark == "true" : prefersDarkQuery.matches;
+
+        document.documentElement.classList.toggle("dark-mode", darkModeState);
+        document.documentElement.classList.toggle("light-mode", !darkModeState);
+    </script>
+    {{ super() }}
+{% endblock %}

docs/administration.rst

@@ -117,6 +117,23 @@ Then you can start paperless-ngx with ``-d`` to have it run in the background.
 
                 image: ghcr.io/paperless-ngx/paperless-ngx:latest
 
+    .. note::
+        In version 1.7.1 and onwards, the Docker image can now pinned to a release series.
+        This is often combined with automatic updaters such as Watchtower to allow safer
+        unattended upgrading to new bugfix releases only.  It is still recommended to always
+        review release notes before upgrading.  To ping your install to a release series, edit
+        the ``docker-compose.yml`` find the line that says
+
+            .. code::
+
+                image: ghcr.io/paperless-ngx/paperless-ngx:latest
+
+        and replace the version with the series you want to track, for example:
+
+            .. code::
+
+                image: ghcr.io/paperless-ngx/paperless-ngx:1.7
+
 Bare Metal Route
 ================
 

docs/advanced_usage.rst

@@ -200,7 +200,7 @@ Troubleshooting:
 - Check your script's permission e.g. in case of permission error ``sudo chmod 755 post-consumption-example.sh``
 - Pipe your scripts's output to a log file e.g. ``echo "${DOCUMENT_ID}" | tee --append /usr/src/paperless/scripts/post-consumption-example.log``
 
-.. _post-consumption-example.sh: https://github.com/jonaswinkler/paperless-ngx/blob/master/scripts/post-consumption-example.sh
+.. _post-consumption-example.sh: https://github.com/paperless-ngx/paperless-ngx/blob/main/scripts/post-consumption-example.sh
 
 .. _advanced-file_name_handling:
 
@@ -243,7 +243,7 @@ will create a directory structure as follows:
     last filename a document was stored as. If you do rename a file, paperless will
     report your files as missing and won't be able to find them.
 
-Paperless provides the following placeholders withing filenames:
+Paperless provides the following placeholders within filenames:
 
 * ``{asn}``: The archive serial number of the document, or "none".
 * ``{correspondent}``: The name of the correspondent, or "none".

docs/changelog.rst

@@ -5,6 +5,87 @@
 Changelog
 *********
 
+paperless-ngx 1.7.0
+###################
+
+Breaking Changes
+
+* ``PAPERLESS_URL`` is now required when using a reverse proxy. See `#674`_.
+
+Features
+
+* Allow setting more than one tag in mail rules `@jonasc`_ (#270)
+* global drag'n'drop `@shamoon`_ (#283).
+* Fix: download buttons should disable while waiting `@shamoon`_ (#630).
+* Update checker `@shamoon`_ (#591).
+* Show prompt on password-protected pdfs `@shamoon`_ (#564).
+* Filtering query params aka browser navigation for filtering `@shamoon`_ (#540).
+* Clickable tags in dashboard widgets `@shamoon`_ (#515).
+* Add bottom pagination `@shamoon`_ (#372).
+* Feature barcode splitter `@gador`_ (#532).
+* App loading screen `@shamoon`_ (#298).
+* Use progress bar for delayed buttons `@shamoon`_ (#415).
+* Add minimum length for documents text filter `@shamoon`_ (#401).
+* Added nav buttons in the document detail view `@GruberViktor`_ (#273).
+* Improve date keyboard input `@shamoon`_ (#253).
+* Color theming `@shamoon`_ (#243).
+* Parse dates when entered without separators `@GruberViktor`_ (#250).
+
+Bug Fixes
+
+* add "localhost" to ALLOWED_HOSTS `@gador`_ (#700).
+* Fix: scanners table `@qcasey`_ (#690).
+* Adds wait for file before consuming `@stumpylog`_ (#483).
+* Fix: frontend document editing erases time data `@shamoon`_ (#654).
+* Increase length of SavedViewFilterRule `@stumpylog`_ (#612).
+* Fixes attachment filename matching during mail fetching `@stumpylog`_ (#680).
+* Add ``PAPERLESS_URL`` env variable & CSRF var `@shamoon`_ (#674).
+* Fix: download buttons should disable while waiting `@shamoon`_ (#630).
+* Fixes downloaded filename, add more consumer ignore settings `@stumpylog`_ (#599).
+* FIX BUG: case-sensitive matching was not possible `@danielBreitlauch`_ (#594).
+* uses shutil.move instead of rename `@gador`_ (#617).
+* Fix npm deps 01.02.22 2 `@shamoon`_ (#610).
+* Fix npm dependencies 01.02.22 `@shamoon`_ (#600).
+* fix issue 416: implement PAPERLESS_OCR_MAX_IMAGE_PIXELS `@hacker-h`_ (#441).
+* fix: exclude cypress from build in Dockerfile `@FrankStrieter`_ (#526).
+* Corrections to pass pre-commit hooks `@schnuffle`_ (#454).
+* Fix 311 unable to click checkboxes in document list `@shamoon`_ (#313).
+* Fix imap tools bug `@stumpylog`_ (#393).
+* Fix filterable dropdown buttons arent translated `@shamoon`_ (#366).
+* Fix 224: "Auto-detected date is day before receipt date" `@a17t`_ (#246).
+* Fix minor sphinx errors `@shamoon`_ (#322).
+* Fix page links hidden `@shamoon`_ (#314).
+* Fix: Include excluded items in dropdown count `@shamoon`_ (#263).
+
+Translation
+
+* `@miku323`_ contributed to Slovenian translation.
+* `@FaintGhost`_ contributed to Chinese Simplified translation.
+* `@DarkoBG79`_ contributed to Serbian translation.
+* `Kemal Secer`_ contributed to Turkish translation.
+* `@Prominence`_ contributed to Belarusian translation.
+
+Documentation
+
+* Fix: scanners table `@qcasey`_ (#690).
+* Add `PAPERLESS_URL` env variable & CSRF var `@shamoon`_ (#674).
+* Fixes downloaded filename, add more consumer ignore settings `@stumpylog`_ (#599).
+* fix issue 416: implement ``PAPERLESS_OCR_MAX_IMAGE_PIXELS`` `@hacker-h`_ (#441).
+* Fix minor sphinx errors `@shamoon`_ (#322).
+
+Maintenance
+
+* Add ``PAPERLESS_URL`` env variable & CSRF var `@shamoon`_ (#674).
+* Chore: Implement release-drafter action for Changelogs `@qcasey`_ (#669).
+* Chore: Add CODEOWNERS `@qcasey`_ (#667).
+* Support docker-compose v2 in install `@stumpylog`_ (#611).
+* Add Belarusian localization `@shamoon`_ (#588).
+* Add Turkish localization `@shamoon`_ (#536).
+* Add Serbian localization `@shamoon`_ (#504).
+* Create PULL_REQUEST_TEMPLATE.md `@shamoon`_ (#304).
+* Add Chinese localization `@shamoon`_ (#247).
+* Add Slovenian language for frontend `@shamoon`_ (#315).
+
 paperless-ngx 1.6.0
 ###################
 
@@ -35,6 +116,10 @@ Version 1.6.0 merges several pending PRs from jonaswinkler's repo and includes n
 * `@shamoon`_ created a slick new logo (#165).
 * `@tim-vogel`_ fixed exports missing groups (#193).
 
+Known issues:
+
+* 1.6.0 included a malformed package-lock.json, as a result users who want to build the docker image themselves need to change line 6 of the ``Dockerfile`` to ``RUN npm update npm -g && npm install --legacy-peer-deps``.
+
 Thank you to the following people for their documentation updates, fixes, and comprehensive testing:
 
 `@m0veax`_, `@a17t`_, `@fignew`_, `@muued`_, `@bauerj`_, `@isigmund`_, `@denilsonsa`_, `@mweimerskirch`_, `@alexander-bauer`_, `@apeltzer`_, `@tribut`_, `@yschroeder`_, `@gador`_, `@sAksham-Ar`_, `@sbrunner`_, `@philpagel`_, `@davemachado`_, `@2600box`_, `@qcasey`_, `@Nicarim`_, `@kpj`_, `@filcuk`_, `@Timoms`_, `@mattlamb99`_, `@padraigkitterick`_, `@ajkavanagh`_, `@Tooa`_, `@Unkn0wnCat`_, `@pewter77`_, `@stumpylog`_, `@Toxix`_, `@azapater`_, `@jschpp`_
@@ -140,7 +225,7 @@ paperless-ng 1.4.0
 
   * New URL pattern for accessing documents by ASN directly (http://<paperless>/asn/123)
 
-  * Added logging when executing pre- and post-consume scripts.
+  * Added logging when executing pre* and post-consume scripts.
 
   * Better error logging during document consumption.
 
@@ -1576,6 +1661,16 @@ bulk of the work on this big change.
 .. _@azapater: https://github.com/azapater
 .. _@tim-vogel: https://github.com/tim-vogel
 .. _@jschpp: https://github.com/jschpp
+.. _@schnuffle: https://github.com/schnuffle
+.. _@GruberViktor: https://github.com/gruberviktor
+.. _@hacker-h: https://github.com/hacker-h
+.. _@danielBreitlauch: https://github.com/danielbreitlauch
+.. _@miku323: https://github.com/miku323
+.. _@FaintGhost: https://github.com/FaintGhost
+.. _@DarkoBG79: https://github.com/DarkoBG79
+.. _Kemal Secer: https://crowdin.com/profile/kemal.secer
+.. _@Prominence: https://github.com/Prominence
+.. _@jonasc: https://github.com/jonasc
 
 .. _#20: https://github.com/the-paperless-project/paperless/issues/20
 .. _#44: https://github.com/the-paperless-project/paperless/issues/44
@@ -1684,6 +1779,7 @@ bulk of the work on this big change.
 .. _#488: https://github.com/the-paperless-project/paperless/pull/488
 .. _#489: https://github.com/the-paperless-project/paperless/pull/489
 .. _#492: https://github.com/the-paperless-project/paperless/pull/492
+.. _#674: https://github.com/paperless-ngx/paperless-ngx/pull/674
 
 .. _a new home on Docker Hub: https://hub.docker.com/r/danielquinn/paperless/
 .. _optipng: http://optipng.sourceforge.net/

docs/conf.py

@@ -2,6 +2,8 @@ import sphinx_rtd_theme
 
 
 __version__ = None
+__full_version_str__ = None
+__major_minor_version_str__ = None
 exec(open("../src/paperless/version.py").read())
 
 
@@ -15,7 +17,7 @@ extensions = [
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-# templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix of source filenames.
 source_suffix = ".rst"
@@ -41,9 +43,9 @@ copyright = "2015-2022, Daniel Quinn, Jonas Winkler, and the paperless-ngx team"
 #
 
 # The short X.Y version.
-version = ".".join([str(_) for _ in __version__[:2]])
+version = __major_minor_version_str__
 # The full version, including alpha/beta/rc tags.
-release = ".".join([str(_) for _ in __version__[:3]])
+release = __full_version_str__
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -119,6 +121,16 @@ html_theme_path = []
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ["_static"]
 
+# These paths are either relative to html_static_path
+# or fully qualified paths (eg. https://...)
+html_css_files = [
+    "css/custom.css",
+]
+
+html_js_files = [
+    "js/darkmode.js",
+]
+
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the documentation.

docs/configuration.rst

@@ -130,6 +130,8 @@ PAPERLESS_LOGROTATE_MAX_BACKUPS=<num>
 
     Defaults to 20.
 
+.. _hosting-and-security:
+
 Hosting & Security
 ##################
 
@@ -142,7 +144,24 @@ PAPERLESS_SECRET_KEY=<key>
 
     Default is listed in the file ``src/paperless/settings.py``.
 
-PAPERLESS_ALLOWED_HOSTS<comma-separated-list>
+PAPERLESS_URL=<url>
+    This setting can be used to set the three options below (ALLOWED_HOSTS,
+    CORS_ALLOWED_HOSTS and CSRF_TRUSTED_ORIGINS). If the other options are
+    set the values will be combined with this one. Do not include a trailing
+    slash. E.g. https://paperless.domain.com
+
+    Defaults to empty string, leaving the other settings unaffected.
+
+PAPERLESS_CSRF_TRUSTED_ORIGINS=<comma-separated-list>
+    A list of trusted origins for unsafe requests (e.g. POST). As of Django 4.0
+    this is required to access the Django admin via the web.
+    See https://docs.djangoproject.com/en/4.0/ref/settings/#csrf-trusted-origins
+
+    Can also be set using PAPERLESS_URL (see above).
+
+    Defaults to empty string, which does not add any origins to the trusted list.
+
+PAPERLESS_ALLOWED_HOSTS=<comma-separated-list>
     If you're planning on putting Paperless on the open internet, then you
     really should set this value to the domain name you're using.  Failing to do
     so leaves you open to HTTP host header attacks:
@@ -151,12 +170,19 @@ PAPERLESS_ALLOWED_HOSTS<comma-separated-list>
     Just remember that this is a comma-separated list, so "example.com" is fine,
     as is "example.com,www.example.com", but NOT " example.com" or "example.com,"
 
+    Can also be set using PAPERLESS_URL (see above).
+
+    If manually set, please remember to include "localhost". Otherwise docker
+    healthcheck will fail.
+
     Defaults to "*", which is all hosts.
 
-PAPERLESS_CORS_ALLOWED_HOSTS<comma-separated-list>
+PAPERLESS_CORS_ALLOWED_HOSTS=<comma-separated-list>
     You need to add your servers to the list of allowed hosts that can do CORS
     calls. Set this to your public domain name.
 
+    Can also be set using PAPERLESS_URL (see above).
+
     Defaults to "http://localhost:8000".
 
 PAPERLESS_FORCE_SCRIPT_NAME=<path>
@@ -185,7 +211,7 @@ PAPERLESS_AUTO_LOGIN_USERNAME=<username>
 PAPERLESS_ADMIN_USER=<username>
     If this environment variable is specified, Paperless automatically creates
     a superuser with the provided username at start. This is useful in cases
-    where you can not run the `createsuperuser` command seperately, such as Kubernetes
+    where you can not run the `createsuperuser` command separately, such as Kubernetes
     or AWS ECS.
 
     Requires `PAPERLESS_ADMIN_PASSWORD` to be set.
@@ -448,7 +474,7 @@ PAPERLESS_TIKA_GOTENBERG_ENDPOINT=<url>
     Defaults to "http://localhost:3000".
 
 If you run paperless on docker, you can add those services to the docker-compose
-file (see the provided ``docker-compose.tika.yml`` file for reference). The changes
+file (see the provided ``docker-compose.sqlite-tika.yml`` file for reference). The changes
 requires are as follows:
 
 .. code:: yaml
@@ -469,14 +495,14 @@ requires are as follows:
         # ...
 
         gotenberg:
-            image: gotenberg/gotenberg:7
+            image: gotenberg/gotenberg:7.4
             restart: unless-stopped
             command:
                 - "gotenberg"
                 - "--chromium-disable-routes=true"
 
         tika:
-            image: apache/tika
+            image: ghcr.io/paperless-ngx/tika:latest
             restart: unless-stopped
 
 Add the configuration variables to the environment of the webserver (alternatively
@@ -540,6 +566,10 @@ PAPERLESS_WORKER_TIMEOUT=<num>
     large documents within the default 1800 seconds. So extending this timeout
     may prove to be useful on weak hardware setups.
 
+PAPERLESS_WORKER_RETRY=<num>
+    If PAPERLESS_WORKER_TIMEOUT has been configured, the retry time for a task can
+    also be configured.  By default, this value will be set to 10s more than the
+    worker timeout.  This value should never be set less than the worker timeout.
 
 PAPERLESS_TIME_ZONE=<timezone>
     Set the time zone here.
@@ -588,6 +618,38 @@ PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS=<bool>
 
     Defaults to false.
 
+PAPERLESS_CONSUMER_ENABLE_BARCODES=<bool>
+    Enables the scanning and page separation based on detected barcodes.
+    This allows for scanning and adding multiple documents per uploaded
+    file, which are separated by one or multiple barcode pages.
+
+    For ease of use, it is suggested to use a standardized separation page,
+    e.g. `here <https://www.alliancegroup.co.uk/patch-codes.htm>`_.
+
+    If no barcodes are detected in the uploaded file, no page separation
+    will happen.
+
+    The original document will be removed and the separated pages will be
+    saved as pdf.
+
+    Defaults to false.
+
+PAPERLESS_CONSUMER_BARCODE_TIFF_SUPPORT=<bool>
+    Whether TIFF image files should be scanned for barcodes.
+    This will automatically convert any TIFF image(s) to pdfs for later
+    processing.
+    This only has an effect, if PAPERLESS_CONSUMER_ENABLE_BARCODES has been
+    enabled.
+
+    Defaults to false.
+
+PAPERLESS_CONSUMER_BARCODE_STRING=PATCHT
+  Defines the string to be detected as a separator barcode.
+  If paperless is used with the PATCH-T separator pages, users
+  shouldn't change this.
+
+  Defaults to "PATCHT"
+
 
 PAPERLESS_CONVERT_MEMORY_LIMIT=<num>
     On smaller systems, or even in the case of Very Large Documents, the consumer
@@ -671,7 +733,7 @@ PAPERLESS_CONSUMER_IGNORE_PATTERNS=<json>
 
     This can be adjusted by configuring a custom json array with patterns to exclude.
 
-    Defautls to ``[".DS_STORE/*", "._*", ".stfolder/*"]``.
+    Defaults to ``[".DS_STORE/*", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini"]``.
 
 Binaries
 ########
@@ -764,3 +826,26 @@ PAPERLESS_OCR_LANGUAGES=<list>
         PAPERLESS_OCR_LANGUAGE=tur
 
     Defaults to none, which does not install any additional languages.
+
+
+.. _configuration-update-checking:
+
+Update Checking
+###############
+
+PAPERLESS_ENABLE_UPDATE_CHECK=<bool>
+    Enable (or disable) the automatic check for available updates. This feature is disabled
+    by default but if it is not explicitly set Paperless-ngx will show a message about this.
+
+    If enabled, the feature works by pinging the the Github API for the latest release e.g.
+    https://api.github.com/repos/paperless-ngx/paperless-ngx/releases/latest
+    to determine whether a new version is available.
+
+    Actual updating of the app must still be performed manually.
+
+    Note that for users of thirdy-party containers e.g. linuxserver.io this notification
+    may be 'ahead' of a new release from the third-party maintainers.
+
+    In either case, no tracking data is collected by the app in any way.
+
+    Defaults to none, which disables the feature.

docs/extending.rst

@@ -334,11 +334,17 @@ directory.
 Building the Docker image
 =========================
 
+The docker image is primarily built by the GitHub actions workflow, but it can be
+faster when developing to build and tag an image locally.
+
+To provide the build arguments automatically, build the image using the helper
+script ``build-docker-image.sh``.
+
 Building the docker image from source:
 
     .. code:: shell-session
 
-        docker build . -t <your-tag>
+        ./build-docker-image.sh Dockerfile -t <your-tag>
 
 Extending Paperless
 ===================

docs/faq.rst

@@ -7,7 +7,7 @@ Frequently asked questions
 
 **A:** While Paperless-ngx is already considered largely "feature-complete" it is a community-driven
 project and development will be guided in this way. New features can be submitted via
-GitHub discussions and "up-voted" by the community but this is not a garauntee the feature
+GitHub discussions and "up-voted" by the community but this is not a guarantee the feature
 will be implemented. This project will always be open to collaboration in the form of PRs,
 ideas etc.
 

docs/scanners.rst

@@ -13,43 +13,43 @@ that works right for you based on recommendations from other Paperless users.
 Physical scanners
 =================
 
-+---------+----------------+-----+-----+-----+------+----------+----------------+
++---------+----------------+-----+------+-----+-----+------+----------+----------------+
 | Brand   | Model          | Supports                                 | Recommended By |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-|         |                | FTP | NFS | SMB | SMTP | API [1]_ |                |
-+=========+================+=====+=====+=====+======+==========+================+
-| Brother | `ADS-1700W`_   | yes |     | yes | yes  |          |`holzhannes`_   |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Brother | `ADS-1600W`_   | yes |     | yes | yes  |          |`holzhannes`_   |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Brother | `ADS-1500W`_   | yes |     | yes | yes  |          |`danielquinn`_  |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Brother | `ADS-1100W`_   | yes |     |     |      |          |`ytzelf`_       |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Brother | `ADS-2800W`_   | yes | yes |     | yes  | yes      |`philpagel`_    |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Brother | `MFC-J6930DW`_ | yes |     |     |      |          |`ayounggun`_    |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Brother | `MFC-L5850DW`_ | yes |     |     | yes  |          |`holzhannes`_   |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Brother | `MFC-L2750DW`_ | yes |     | yes | yes  |          |`muued`_        |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Brother | `MFC-J5910DW`_ | yes |     |     |      |          |`bmsleight`_    |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Brother | `MFC-8950DW`_  | yes |     |     | yes  | yes      |`philpagel`_    |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Brother | `MFC-9142CDN`_ | yes |     | yes |      |          |`REOLDEV`_      |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Fujitsu | `ix500`_       | yes |     | yes |      |          |`eonist`_       |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Epson   | `ES-580W`_     | yes |     | yes | yes  |          |`fignew`_       |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Epson   | `WF-7710DWF`_  | yes |     | yes |      |          |`Skylinar`_     |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Fujitsu | `S1300i`_      | yes |     | yes |      |          |`jonaswinkler`_ |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Doxie   | `Q2`_          |     |     |     |      | yes      |`Unkn0wnCat`_   |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
++---------+----------------+-----+------+-----+-----+------+----------+----------------+
+|         |                | FTP | SFTP | NFS | SMB | SMTP | API [1]_ |                |
++=========+================+=====+======+=====+=====+======+==========+================+
+| Brother | `ADS-1700W`_   | yes |      |     | yes | yes  |          |`holzhannes`_   |
++---------+----------------+-----+------+-----+-----+------+----------+----------------+
+| Brother | `ADS-1600W`_   | yes |      |     | yes | yes  |          |`holzhannes`_   |
++---------+----------------+-----+------+-----+-----+------+----------+----------------+
+| Brother | `ADS-1500W`_   | yes |      |     | yes | yes  |          |`danielquinn`_  |
++---------+----------------+-----+------+-----+-----+------+----------+----------------+
+| Brother | `ADS-1100W`_   | yes |      |     |     |      |          |`ytzelf`_       |
++---------+----------------+-----+------+-----+-----+------+----------+----------------+
+| Brother | `ADS-2800W`_   | yes | yes  |     | yes | yes  |          |`philpagel`_    |
++---------+----------------+-----+------+-----+-----+------+----------+----------------+
+| Brother | `MFC-J6930DW`_ | yes |      |     |     |      |          |`ayounggun`_    |
++---------+----------------+-----+------+-----+-----+------+----------+----------------+
+| Brother | `MFC-L5850DW`_ | yes |      |     |     | yes  |          |`holzhannes`_   |
++---------+----------------+-----+------+-----+-----+------+----------+----------------+
+| Brother | `MFC-L2750DW`_ | yes |      |     | yes | yes  |          |`muued`_        |
++---------+----------------+-----+------+-----+-----+------+----------+----------------+
+| Brother | `MFC-J5910DW`_ | yes |      |     |     |      |          |`bmsleight`_    |
++---------+----------------+-----+------+-----+-----+------+----------+----------------+
+| Brother | `MFC-8950DW`_  | yes |      |     | yes | yes  |          |`philpagel`_    |
++---------+----------------+-----+------+-----+-----+------+----------+----------------+
+| Brother | `MFC-9142CDN`_ | yes |      |     | yes |      |          |`REOLDEV`_      |
++---------+----------------+-----+------+-----+-----+------+----------+----------------+
+| Fujitsu | `ix500`_       | yes |      |     | yes |      |          |`eonist`_       |
++---------+----------------+-----+------+-----+-----+------+----------+----------------+
+| Epson   | `ES-580W`_     | yes |      |     | yes | yes  |          |`fignew`_       |
++---------+----------------+-----+------+-----+-----+------+----------+----------------+
+| Epson   | `WF-7710DWF`_  | yes |      |     | yes |      |          |`Skylinar`_     |
++---------+----------------+-----+------+-----+-----+------+----------+----------------+
+| Fujitsu | `S1300i`_      | yes |      |     | yes |      |          |`jonaswinkler`_ |
++---------+----------------+-----+------+-----+-----+------+----------+----------------+
+| Doxie   | `Q2`_          |     |      |     |     |      | yes      |`Unkn0wnCat`_   |
++---------+----------------+-----+------+-----+-----+------+----------+----------------+
 
 .. _MFC-L5850DW: https://www.brother-usa.com/products/mfcl5850dw
 .. _MFC-L2750DW: https://www.brother.de/drucker/laserdrucker/mfc-l2750dw
@@ -131,4 +131,3 @@ This part assumes your Doxie is connected to WiFi and you know its IP.
 6. Click *Submit* at the bottom of the page
 
 Congrats, you can now scan directly from your Doxie to your Paperless-ngx instance!
-

docs/screenshots.rst

@@ -4,41 +4,60 @@
 Screenshots
 ***********
 
-This is what paperless-ngx looks like. You shouldn't use paperless to index
-research papers though, its a horrible tool for that job.
+This is what Paperless-ngx looks like.
 
 The dashboard shows customizable views on your document and allows document uploads:
 
 .. image:: _static/screenshots/dashboard.png
+    :target: _static/screenshots/dashboard.png
 
 The document list provides three different styles to scroll through your documents:
 
 .. image:: _static/screenshots/documents-table.png
+    :target: _static/screenshots/documents-table.png
 .. image:: _static/screenshots/documents-smallcards.png
+    :target: _static/screenshots/documents-smallcards.png
 .. image:: _static/screenshots/documents-largecards.png
+    :target: _static/screenshots/documents-largecards.png
+
+Paperless-ngx also supports "dark mode":
+
+.. image:: _static/screenshots/documents-smallcards-dark.png
+    :target: _static/screenshots/documents-smallcards-dark.png
 
 Extensive filtering mechanisms:
 
 .. image:: _static/screenshots/documents-filter.png
+    :target: _static/screenshots/documents-filter.png
 
-Side-by-side editing of documents. Optimized for 1080p.
+Bulk editing of document tags, correspondents, etc.:
+
+.. image:: _static/screenshots/bulk-edit.png
+    :target: _static/screenshots/bulk-edit.png
+
+Side-by-side editing of documents:
 
 .. image:: _static/screenshots/editing.png
+    :target: _static/screenshots/editing.png
 
 Tag editing. This looks about the same for correspondents and document types.
 
 .. image:: _static/screenshots/new-tag.png
+    :target: _static/screenshots/new-tag.png
 
 Searching provides auto complete and highlights the results.
 
 .. image:: _static/screenshots/search-preview.png
+    :target: _static/screenshots/search-preview.png
 .. image:: _static/screenshots/search-results.png
+    :target: _static/screenshots/search-results.png
 
 Fancy mail filters!
 
 .. image:: _static/screenshots/mail-rules-edited.png
+    :target: _static/screenshots/mail-rules-edited.png
 
-Mobile support in the future? This kinda works, however some layouts are still
-too wide.
+Mobile devices are supported.
 
 .. image:: _static/screenshots/mobile.png
+    :target: _static/screenshots/mobile.png

docs/setup.rst

@@ -291,12 +291,14 @@ writing. Windows is not and will never be supported.
     *   ``libpq-dev`` for PostgreSQL
     *   ``libmagic-dev`` for mime type detection
     *   ``mime-support`` for mime type detection
+    *   ``libzbar0`` for barcode detection
+    *   ``poppler-utils`` for barcode detection
 
     Use this list for your preferred package management:
 
     .. code::
 
-        python3 python3-pip python3-dev imagemagick fonts-liberation optipng gnupg libpq-dev libmagic-dev mime-support
+        python3 python3-pip python3-dev imagemagick fonts-liberation optipng gnupg libpq-dev libmagic-dev mime-support libzbar0 poppler-utils
 
     These dependencies are required for OCRmyPDF, which is used for text recognition.
 
@@ -345,6 +347,8 @@ writing. Windows is not and will never be supported.
         paperless stores its data. If you like, you can point both to the same directory.
     *   ``PAPERLESS_SECRET_KEY`` should be a random sequence of characters. It's used for authentication. Failure
         to do so allows third parties to forge authentication credentials.
+    *   ``PAPERLESS_URL`` if you are behind a reverse proxy. This should point to your domain. Please see
+        :ref:`configuration` for more information.
 
     Many more adjustments can be made to paperless, especially the OCR part. The following options are recommended
     for everyone:
@@ -724,8 +728,6 @@ configuring some options in paperless can help improve performance immensely:
     times. Thumbnails will be about 20% larger.
 *   If using docker, consider setting ``PAPERLESS_WEBSERVER_WORKERS`` to
     1. This will save some memory.
-*   Use the arm compatible docker-compose if you're wanting to use Tika on something like
-		a raspberry pi. The official apache/tika image does not support the arm architecture.
 
 For details, refer to :ref:`configuration`.
 
@@ -784,4 +786,6 @@ the following configuration is required for paperless to operate:
         }
     }
 
+The ``PAPERLESS_URL`` configuration variable is also required when using a reverse proxy. Please refer to the :ref:`hosting-and-security` docs.
+
 Also read `this <https://channels.readthedocs.io/en/stable/deploying.html#nginx-supervisor-ubuntu>`__, towards the end of the section.

docs/troubleshooting.rst

@@ -125,7 +125,7 @@ If using docker-compose, this is achieved by the following configuration change
 .. code:: yaml
 
     gotenberg:
-        image: gotenberg/gotenberg:7
+        image: gotenberg/gotenberg:7.4
         restart: unless-stopped
         command:
             - "gotenberg"

docs/usage_overview.rst

@@ -62,7 +62,7 @@ your documents:
 
 1.  OCR the document, if it has no text. Digital documents usually have text,
     and this step will be skipped for those documents.
-2.  Paperless will create an archiveable PDF/A document from your document.
+2.  Paperless will create an archivable PDF/A document from your document.
     If this document is coming from your scanner, it will have embedded selectable text.
 3.  Paperless performs automatic matching of tags, correspondents and types on the
     document before storing it in the database.
@@ -102,12 +102,14 @@ files from the scanner.  Typically, you're looking at an FTP server like
 
 .. TODO: hyperref to configuration of the location of this magic folder.
 
-Dashboard upload
-================
+Web UI Upload
+=============
 
 The dashboard has a file drop field to upload documents to paperless. Simply drag a file
 onto this field or select a file with the file dialog. Multiple files are supported.
 
+You can also upload documents on any other page of the web UI by dragging-and-dropping
+files into your browser window.
 
 .. _usage-mobile_upload:
 
@@ -178,6 +180,15 @@ These are as follows:
     automatically or manually and tell paperless to move them to yet another folder
     after consumption. It's up to you.
 
+.. note::
+
+    When defining a mail rule with a folder, you may need to try different characters to
+    define how the sub-folders are separated.  Common values include ".", "/" or "|", but
+    this varies by the mail server.  Check the documentation for your mail server.  In the
+    event of an error fetching mail from a certain folder, check the Paperless logs.  When
+    a folder is not located, Paperless will attempt to list all folders found in the account
+    to the Paperless logs.
+
 .. note::
 
     Paperless will process the rules in the order defined in the admin page.

gunicorn.conf.py

@@ -24,7 +24,7 @@ def worker_int(worker):
     ## get traceback info
     import threading, sys, traceback
 
-    id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
+    id2name = {th.ident: th.name for th in threading.enumerate()}
     code = []
     for threadId, stack in sys._current_frames().items():
         code.append("\n# Thread: %s(%d)" % (id2name.get(threadId, ""), threadId))

install-paperless-ngx.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 ask() {
 	while true ; do
@@ -47,24 +47,29 @@ if [[ $(id -u) == "0" ]] ; then
 	exit 1
 fi
 
-if [[ -z $(which wget) ]] ; then
+if ! command -v wget &> /dev/null ; then
 	echo "wget executable not found. Is wget installed?"
 	exit 1
 fi
 
-if [[ -z $(which docker) ]] ; then
+if ! command -v docker &> /dev/null ; then
 	echo "docker executable not found. Is docker installed?"
 	exit 1
 fi
 
-if [[ -z $(which docker-compose) ]] ; then
+DOCKER_COMPOSE_CMD="docker-compose"
+if ! command -v ${DOCKER_COMPOSE_CMD} ; then
+	if docker compose version &> /dev/null ; then
+		DOCKER_COMPOSE_CMD="docker compose"
+	else
 		echo "docker-compose executable not found. Is docker-compose installed?"
 		exit 1
 	fi
+fi
 
 # Check if user has permissions to run Docker by trying to get the status of Docker (docker status).
 # If this fails, the user probably does not have permissions for Docker.
-if [ ! "$(docker stats --no-stream 2>/dev/null 1>&2)" ] ; then
+if ! docker stats --no-stream &> /dev/null ; then
 	echo ""
 	echo "WARN: It look like the current user does not have Docker permissions."
 	echo "WARN: Use 'sudo usermod -aG docker $USER' to assign Docker permissions to the user."
@@ -87,6 +92,14 @@ echo ""
 echo "1. Application configuration"
 echo "============================"
 
+echo ""
+echo "The URL paperless will be available at. This is required if the"
+echo "installation will be accessible via the web, otherwise can be left blank."
+echo ""
+
+ask "URL" ""
+URL=$ask_result
+
 echo ""
 echo "The port on which the paperless webserver will listen for incoming"
 echo "connections."
@@ -273,6 +286,7 @@ if [[ "$DATABASE_BACKEND" == "postgres" ]] ; then
 	fi
 fi
 echo ""
+echo "URL: $URL"
 echo "Port: $PORT"
 echo "Database: $DATABASE_BACKEND"
 echo "Tika enabled: $TIKA_ENABLED"
@@ -305,9 +319,15 @@ wget "https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/main/docker/
 
 SECRET_KEY=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 64 | head -n 1)
 
-DEFAULT_LANGUAGES="deu eng fra ita spa"
+DEFAULT_LANGUAGES=("deu eng fra ita spa")
+
+_split_langs="${OCR_LANGUAGE//+/ }"
+read -r -a OCR_LANGUAGES_ARRAY <<< "${_split_langs}"
 
 {
+	if [[ ! $URL == "" ]] ; then
+		echo "PAPERLESS_URL=$URL"
+	fi
 	if [[ ! $USERMAP_UID == "1000" ]] ; then
 		echo "USERMAP_UID=$USERMAP_UID"
 	fi
@@ -317,8 +337,8 @@ DEFAULT_LANGUAGES="deu eng fra ita spa"
 	echo "PAPERLESS_TIME_ZONE=$TIME_ZONE"
 	echo "PAPERLESS_OCR_LANGUAGE=$OCR_LANGUAGE"
 	echo "PAPERLESS_SECRET_KEY=$SECRET_KEY"
-	if [[ ! " ${DEFAULT_LANGUAGES[*]} " =~ ${OCR_LANGUAGE} ]] ; then
-		echo "PAPERLESS_OCR_LANGUAGES=$OCR_LANGUAGE"
+	if [[ ! ${DEFAULT_LANGUAGES[*]} =~ ${OCR_LANGUAGES_ARRAY[*]} ]] ; then
+		echo "PAPERLESS_OCR_LANGUAGES=${OCR_LANGUAGES_ARRAY[*]}"
 	fi
 } > docker-compose.env
 
@@ -351,8 +371,8 @@ if [ "$l1" -eq "$l2" ] ; then
 fi
 
 
-docker-compose pull
+${DOCKER_COMPOSE_CMD} pull
 
-docker-compose run --rm -e DJANGO_SUPERUSER_PASSWORD="$PASSWORD" webserver createsuperuser --noinput --username "$USERNAME" --email "$EMAIL"
+${DOCKER_COMPOSE_CMD} run --rm -e DJANGO_SUPERUSER_PASSWORD="$PASSWORD" webserver createsuperuser --noinput --username "$USERNAME" --email "$EMAIL"
 
-docker-compose up -d
+${DOCKER_COMPOSE_CMD} up -d

paperless.conf.example

@@ -27,8 +27,10 @@
 # Security and hosting
 
 #PAPERLESS_SECRET_KEY=change-me
-#PAPERLESS_ALLOWED_HOSTS=example.com,www.example.com
-#PAPERLESS_CORS_ALLOWED_HOSTS=http://example.com,http://localhost:8000
+#PAPERLESS_URL=https://example.com
+#PAPERLESS_CSRF_TRUSTED_ORIGINS=https://example.com # can be set using PAPERLESS_URL
+#PAPERLESS_ALLOWED_HOSTS=example.com,www.example.com # can be set using PAPERLESS_URL
+#PAPERLESS_CORS_ALLOWED_HOSTS=https://localhost:8080,https://example.com # can be set using PAPERLESS_URL
 #PAPERLESS_FORCE_SCRIPT_NAME=
 #PAPERLESS_STATIC_URL=/static/
 #PAPERLESS_AUTO_LOGIN_USERNAME=
@@ -58,8 +60,10 @@
 #PAPERLESS_CONSUMER_POLLING=10
 #PAPERLESS_CONSUMER_DELETE_DUPLICATES=false
 #PAPERLESS_CONSUMER_RECURSIVE=false
-#PAPERLESS_CONSUMER_IGNORE_PATTERNS=[".DS_STORE/*", "._*", ".stfolder/*"]
+#PAPERLESS_CONSUMER_IGNORE_PATTERNS=[".DS_STORE/*", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini"]
 #PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS=false
+#PAPERLESS_CONSUMER_ENABLE_BARCODES=false
+#PAPERLESS_CONSUMER_ENABLE_BARCODES=PATCHT
 #PAPERLESS_OPTIMIZE_THUMBNAILS=true
 #PAPERLESS_PRE_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh
 #PAPERLESS_POST_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh
@@ -67,6 +71,7 @@
 #PAPERLESS_FILENAME_PARSE_TRANSFORMS=[]
 #PAPERLESS_THUMBNAIL_FONT_NAME=
 #PAPERLESS_IGNORE_DATES=
+#PAPERLESS_ENABLE_UPDATE_CHECK=
 
 # Tika settings
 

requirements.txt

@@ -10,10 +10,10 @@
 aioredis==1.3.1
 anyio==3.5.0; python_full_version >= '3.6.2'
 arrow==1.2.2; python_version >= '3.6'
-asgiref==3.5.0; python_version >= '3.7'
+asgiref==3.5.1; python_version >= '3.7'
 async-timeout==4.0.2; python_version >= '3.6'
 attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-autobahn==22.2.2; python_version >= '3.7'
+autobahn==22.3.2; python_version >= '3.7'
 automat==20.2.0
 backports.zoneinfo==0.2.1; python_version < '3.9'
 blessed==1.19.1; python_version >= '2.7'
@@ -23,11 +23,11 @@ channels-redis==3.4.0
 channels==3.0.4
 chardet==4.0.0; python_version >= '3.1'
 charset-normalizer==2.0.12; python_version >= '3'
-click==8.0.4; python_version >= '3.6'
+click==8.1.3; python_version >= '3.7'
 coloredlogs==15.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 concurrent-log-handler==0.9.20
 constantly==15.1.0
-cryptography==36.0.2; python_version >= '3.6'
+cryptography==37.0.1; python_version >= '3.6'
 daphne==3.0.2; python_version >= '3.6'
 dateparser==1.1.1
 django-cors-headers==3.11.0
@@ -35,7 +35,7 @@ django-extensions==3.1.5
 django-filter==21.1
 django-picklefield==3.0.1; python_version >= '3'
 django-q==1.3.9
-django==4.0.3
+django==4.0.4
 djangorestframework==3.13.1
 filelock==3.6.0
 fuzzywuzzy[speedup]==0.18.0
@@ -45,10 +45,10 @@ hiredis==2.0.0; python_version >= '3.6'
 httptools==0.4.0
 humanfriendly==10.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 hyperlink==21.0.0
-idna==3.3; python_version >= '3.5'
-imap-tools==0.53.0
-img2pdf==0.4.3
-importlib-resources==5.6.0; python_version < '3.9'
+idna==3.3; python_version >= '3'
+imap-tools==0.54.0
+img2pdf==0.4.4
+importlib-resources==5.7.1; python_version < '3.9'
 incremental==21.3.0
 inotify-simple==1.3.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 inotifyrecursive==0.3.5
@@ -57,12 +57,13 @@ langdetect==1.0.9
 lxml==4.8.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 msgpack==1.0.3
 numpy==1.22.3; python_version >= '3.8'
-ocrmypdf==13.4.1
+ocrmypdf==13.4.3
 packaging==21.3; python_version >= '3.6'
 pathvalidate==2.5.0
-pdfminer.six==20211012
-pikepdf==5.1.0
-pillow==9.0.1
+pdf2image==1.16.0
+pdfminer.six==20220319
+pikepdf==5.1.2
+pillow==9.1.0
 pluggy==1.0.0; python_version >= '3.6'
 portalocker==2.4.0; python_version >= '3'
 psycopg2==2.9.3
@@ -70,7 +71,7 @@ pyasn1-modules==0.2.8
 pyasn1==0.4.8
 pycparser==2.21
 pyopenssl==22.0.0
-pyparsing==3.0.7; python_version >= '3.6'
+pyparsing==3.0.8; python_full_version >= '3.6.8'
 python-dateutil==2.8.2
 python-dotenv==0.20.0
 python-gnupg==0.4.8
@@ -79,6 +80,7 @@ python-magic==0.4.25
 pytz-deprecation-shim==0.1.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
 pytz==2022.1
 pyyaml==6.0
+pyzbar==0.1.9
 redis==3.5.3
 regex==2022.3.2; python_version >= '3.6'
 reportlab==3.6.9; python_version >= '3.7' and python_version < '4'
@@ -86,26 +88,26 @@ requests==2.27.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3
 scikit-learn==1.0.2
 scipy==1.8.0; python_version < '3.11' and python_version >= '3.8'
 service-identity==21.1.0
-setuptools==61.1.1; python_version >= '3.7'
+setuptools==62.1.0; python_version >= '3.7'
 six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 sniffio==1.2.0; python_version >= '3.5'
 sqlparse==0.4.2; python_version >= '3.5'
 threadpoolctl==3.1.0; python_version >= '3.6'
 tika==1.24
-tqdm==4.63.1
-twisted[tls]==22.2.0; python_full_version >= '3.6.7'
+tqdm==4.64.0
+twisted[tls]==22.4.0; python_full_version >= '3.6.7'
 txaio==22.2.1; python_version >= '3.6'
-typing-extensions==4.1.1; python_version >= '3.6'
+typing-extensions==4.2.0; python_version >= '3.7'
 tzdata==2022.1; python_version >= '3.6'
-tzlocal==4.1; python_version >= '3.6'
+tzlocal==4.2; python_version >= '3.6'
 urllib3==1.26.9; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
 uvicorn[standard]==0.17.6
 uvloop==0.16.0
 watchdog==2.1.7
-watchgod==0.8.1
+watchgod==0.8.2
 wcwidth==0.2.5
-websockets==10.2
+websockets==10.3
 whitenoise==6.0.0
 whoosh==2.7.4
-zipp==3.7.0; python_version < '3.9'
+zipp==3.8.0; python_version < '3.9'
 zope.interface==5.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'

scripts/start_services.sh

@@ -2,5 +2,5 @@
 
 docker run -p 5432:5432 -e POSTGRES_PASSWORD=password -v paperless_pgdata:/var/lib/postgresql/data -d postgres:13
 docker run -d -p 6379:6379 redis:latest
-docker run -p 3000:3000 -d gotenberg/gotenberg:7
-docker run -p 9998:9998 -d apache/tika
+docker run -p 3000:3000 -d gotenberg/gotenberg:7.4
+docker run -p 9998:9998 -d ghcr.io/paperless-ngx/tika:latest

src-ui/cypress/integration/manage.spec.ts

@@ -26,7 +26,7 @@ describe('manage', () => {
         req.reply({ count: 3, next: null, previous: null, results: [] })
     })
     cy.visit('/tags')
-    cy.get('tbody').find('button').contains('Documents').first().click() // id = 4
+    cy.get('tbody').find('button:visible').contains('Documents').first().click() // id = 4
     cy.contains('3 documents')
   })
 })

src-ui/package.json

@@ -13,15 +13,15 @@
   },
   "private": true,
   "dependencies": {
-    "@angular/common": "~13.3.1",
-    "@angular/compiler": "~13.3.1",
-    "@angular/core": "~13.3.1",
-    "@angular/forms": "~13.3.1",
-    "@angular/localize": "~13.3.1",
-    "@angular/platform-browser": "~13.3.1",
-    "@angular/platform-browser-dynamic": "~13.3.1",
-    "@angular/router": "~13.3.1",
-    "@ng-bootstrap/ng-bootstrap": "^12.0.1",
+    "@angular/common": "~13.3.5",
+    "@angular/compiler": "~13.3.5",
+    "@angular/core": "~13.3.5",
+    "@angular/forms": "~13.3.5",
+    "@angular/localize": "~13.3.5",
+    "@angular/platform-browser": "~13.3.5",
+    "@angular/platform-browser-dynamic": "~13.3.5",
+    "@angular/router": "~13.3.5",
+    "@ng-bootstrap/ng-bootstrap": "^12.1.1",
     "@ng-select/ng-select": "^8.1.1",
     "@ngneat/dirty-check-forms": "^3.0.2",
     "@popperjs/core": "^2.11.4",
@@ -38,21 +38,23 @@
   },
   "devDependencies": {
     "@angular-builders/jest": "13.0.3",
-    "@angular-devkit/build-angular": "~13.3.1",
-    "@angular/cli": "~13.3.1",
-    "@angular/compiler-cli": "~13.3.1",
+    "@angular-devkit/build-angular": "~13.3.4",
+    "@angular/cli": "~13.3.4",
+    "@angular/compiler-cli": "~13.3.5",
     "@types/jest": "27.4.1",
-    "@types/node": "^17.0.23",
+    "@types/node": "^17.0.30",
     "codelyzer": "^6.0.2",
-    "concurrently": "7.0.0",
-    "jest": "27.5.1",
+    "concurrently": "7.1.0",
+    "jest": "28.0.3",
+    "jest-environment-jsdom": "^28.0.2",
+    "jest-preset-angular": "^12.0.0-next.1",
     "ts-node": "~10.7.0",
     "tslint": "~6.1.3",
     "typescript": "~4.6.3",
     "wait-on": "~6.0.1"
   },
   "optionalDependencies": {
-    "cypress": "~9.5.3",
-    "@cypress/schematic": "^1.6.0"
+    "@cypress/schematic": "^1.6.0",
+    "cypress": "~9.6.0"
   }
 }

src-ui/setup-jest.ts

@@ -1,4 +1,4 @@
-import 'jest-preset-angular/setup-jest'
+import { jest } from '@jest/globals'
 
 /* global mocks for jsdom */
 const mock = () => {
@@ -26,5 +26,6 @@ Object.defineProperty(document.body.style, 'transform', {
   },
 })
 
-/* output shorter and more meaningful Zone error stack traces */
-// Error.stackTraceLimit = 2
+HTMLCanvasElement.prototype.getContext = <
+  typeof HTMLCanvasElement.prototype.getContext
+>jest.fn()

src-ui/src/app/app.component.html

@@ -1,3 +1,13 @@
 <app-toasts></app-toasts>
 
+<ngx-file-drop dropZoneClassName="main-dropzone" contentClassName="main-content" [disabled]="!dragDropEnabled"
+(onFileDrop)="dropped($event)" (onFileOver)="fileOver()" (onFileLeave)="fileLeave()">
+    <ng-template ngx-file-drop-content-tmp>
+        <div class="global-dropzone-overlay fade" [class.show]="fileIsOver" [class.hide]="hidden">
+            <h2 i18n>Drop files to begin upload</h2>
+        </div>
+        <div [class.inert]="fileIsOver">
             <router-outlet></router-outlet>
+        </div>
+    </ng-template>
+</ngx-file-drop>

src-ui/src/app/app.component.ts

@@ -4,6 +4,8 @@ import { Router } from '@angular/router'
 import { Subscription } from 'rxjs'
 import { ConsumerStatusService } from './services/consumer-status.service'
 import { ToastService } from './services/toast.service'
+import { NgxFileDropEntry } from 'ngx-file-drop'
+import { UploadDocumentsService } from './services/upload-documents.service'
 
 @Component({
   selector: 'app-root',
@@ -15,11 +17,16 @@ export class AppComponent implements OnInit, OnDestroy {
   successSubscription: Subscription
   failedSubscription: Subscription
 
+  private fileLeaveTimeoutID: any
+  fileIsOver: boolean = false
+  hidden: boolean = true
+
   constructor(
     private settings: SettingsService,
     private consumerStatusService: ConsumerStatusService,
     private toastService: ToastService,
-    private router: Router
+    private router: Router,
+    private uploadDocumentsService: UploadDocumentsService
   ) {
     let anyWindow = window as any
     anyWindow.pdfWorkerSrc = 'assets/js/pdf.worker.min.js'
@@ -100,4 +107,36 @@ export class AppComponent implements OnInit, OnDestroy {
         }
       })
   }
+
+  public get dragDropEnabled(): boolean {
+    return !this.router.url.includes('dashboard')
+  }
+
+  public fileOver() {
+    // allows transition
+    setTimeout(() => {
+      this.fileIsOver = true
+    }, 1)
+    this.hidden = false
+    // stop fileLeave timeout
+    clearTimeout(this.fileLeaveTimeoutID)
+  }
+
+  public fileLeave(immediate: boolean = false) {
+    const ms = immediate ? 0 : 500
+
+    this.fileLeaveTimeoutID = setTimeout(() => {
+      this.fileIsOver = false
+      // await transition completed
+      setTimeout(() => {
+        this.hidden = true
+      }, 150)
+    }, ms)
+  }
+
+  public dropped(files: NgxFileDropEntry[]) {
+    this.fileLeave(true)
+    this.uploadDocumentsService.uploadFiles(files)
+    this.toastService.showInfo($localize`Initiating upload...`, 3000)
+  }
 }

src-ui/src/app/components/app-frame/app-frame.component.html

@@ -12,7 +12,7 @@
   </a>
   <div class="search-form-container flex-grow-1 py-2 pb-3 pb-sm-2 px-3 ps-md-4 me-sm-auto order-3 order-sm-1">
     <form (ngSubmit)="search()" class="form-inline flex-grow-1">
-      <svg width="1em" height="1em">
+      <svg width="1em" height="1em" fill="currentColor">
         <use xlink:href="assets/bootstrap-icons.svg#search"/>
       </svg>
       <input class="form-control form-control-sm" type="text" placeholder="Search documents" aria-label="Search"
@@ -25,7 +25,7 @@
         <span *ngIf="displayName" class="navbar-text small me-2 text-light d-none d-sm-inline">
           {{displayName}}
         </span>
-        <svg width="1.3em" height="1.3em">
+        <svg width="1.3em" height="1.3em" fill="currentColor">
           <use xlink:href="assets/bootstrap-icons.svg#person-circle"/>
         </svg>
       </button>
@@ -170,21 +170,46 @@
           <li class="nav-item">
             <div class="d-flex w-100 flex-wrap">
               <a class="nav-link pe-2 pb-1" target="_blank" rel="noopener noreferrer" href="https://github.com/paperless-ngx/paperless-ngx">
-                <svg xmlns="http://www.w3.org/2000/svg" fill="currentColor" class="sidebaricon bi bi-github" viewBox="0 0 16 16">
-                  <path d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.012 8.012 0 0 0 16 8c0-4.42-3.58-8-8-8z"/>
+                <svg xmlns="http://www.w3.org/2000/svg" fill="currentColor" class="sidebaricon" viewBox="0 0 16 16">
+                  <use xlink:href="assets/bootstrap-icons.svg#github" />
                 </svg>&nbsp;<ng-container i18n>GitHub</ng-container>
               </a>
               <a class="nav-link-additional small text-muted ms-3" target="_blank" rel="noopener noreferrer" href="https://github.com/paperless-ngx/paperless-ngx/discussions/categories/feature-requests" title="Suggest an idea">
-                <svg xmlns="http://www.w3.org/2000/svg" width="1.3em" height="1.3em" fill="currentColor" class="bi bi-lightbulb pe-1" viewBox="0 0 16 16">
-                  <path d="M2 6a6 6 0 1 1 10.174 4.31c-.203.196-.359.4-.453.619l-.762 1.769A.5.5 0 0 1 10.5 13a.5.5 0 0 1 0 1 .5.5 0 0 1 0 1l-.224.447a1 1 0 0 1-.894.553H6.618a1 1 0 0 1-.894-.553L5.5 15a.5.5 0 0 1 0-1 .5.5 0 0 1 0-1 .5.5 0 0 1-.46-.302l-.761-1.77a1.964 1.964 0 0 0-.453-.618A5.984 5.984 0 0 1 2 6zm6-5a5 5 0 0 0-3.479 8.592c.263.254.514.564.676.941L5.83 12h4.342l.632-1.467c.162-.377.413-.687.676-.941A5 5 0 0 0 8 1z"/>
+                <svg xmlns="http://www.w3.org/2000/svg" width="1.1em" height="1.1em" fill="currentColor" class="me-1" viewBox="0 0 16 16">
+                  <use xlink:href="assets/bootstrap-icons.svg#lightbulb" />
                 </svg>
                 <ng-container i18n>Suggest an idea</ng-container>
               </a>
             </div>
           </li>
           <li class="nav-item mt-2">
-            <div class="px-3 py-2 text-muted small">
-              {{versionString}}
+            <div class="px-3 py-2 text-muted small d-flex align-items-center flex-wrap">
+              <div class="me-3">{{ versionString }}</div>
+              <div *ngIf="appRemoteVersion" class="version-check">
+                <ng-template #updateAvailablePopContent>
+                  <span class="small">Paperless-ngx v{{ appRemoteVersion.version }} <ng-container i18n>is available.</ng-container><br/><ng-container i18n>Click to view.</ng-container></span>
+                </ng-template>
+                <ng-template #updateCheckingNotEnabledPopContent>
+                  <span class="small"><ng-container i18n>Checking for updates is disabled.</ng-container><br/><ng-container i18n>Click for more information.</ng-container></span>
+                </ng-template>
+                <ng-container *ngIf="appRemoteVersion.feature_is_set; else updateCheckNotSet">
+                  <a *ngIf="appRemoteVersion.update_available" class="small text-decoration-none" target="_blank" rel="noopener noreferrer" href="https://github.com/paperless-ngx/paperless-ngx/releases"
+                  [ngbPopover]="updateAvailablePopContent" popoverClass="shadow" triggers="mouseenter:mouseleave" container="body">
+                    <svg fill="currentColor" class="me-1" width="1.2em" height="1.2em" style="vertical-align: text-top;" viewBox="0 0 16 16">
+                      <use xlink:href="assets/bootstrap-icons.svg#info-circle" />
+                    </svg>
+                    <ng-container *ngIf="appRemoteVersion?.update_available" i18n>Update available</ng-container>
+                  </a>
+                </ng-container>
+                <ng-template #updateCheckNotSet>
+                  <a class="small text-decoration-none" target="_blank" rel="noopener noreferrer" href="https://paperless-ngx.readthedocs.io/en/latest/configuration.html#update-checking"
+                  [ngbPopover]="updateCheckingNotEnabledPopContent" popoverClass="shadow" triggers="mouseenter:mouseleave" container="body">
+                    <svg fill="currentColor" class="me-1" width="1.2em" height="1.2em" style="vertical-align: text-top;" viewBox="0 0 16 16">
+                      <use xlink:href="assets/bootstrap-icons.svg#info-circle" />
+                    </svg>
+                  </a>
+                </ng-template>
+              </div>
             </div>
           </li>
         </ul>

src-ui/src/app/components/app-frame/app-frame.component.scss

@@ -1,4 +1,3 @@
-@import "/src/theme";
 /*
  * Sidebar
  */
@@ -35,21 +34,25 @@
 
 .sidebar .nav-link {
   font-weight: 500;
-}
 
-.sidebar .nav-link .sidebaricon {
-  margin-right: 4px;
-}
-
-.sidebar .nav-link.active {
+  &:hover, &.active, &:focus {
     color: var(--bs-primary);
+  }
+
+  &:focus-visible {
+    outline: none;
+    background-color: var(--bs-body-bg);
+  }
+
+  &.active {
     font-weight: bold;
   }
 
-.sidebar .nav-link.active .sidebaricon,
-.sidebar .nav-link:hover .sidebaricon {
+  .sidebaricon {
+    margin-right: 4px;
     color: inherit;
   }
+}
 
 .sidebar-heading {
   font-size: 0.75rem;
@@ -171,8 +174,28 @@
 
     &:focus {
       background-color: rgba(0, 0, 0, 0.3);
+      color: var(--bs-light);
       flex-grow: 1;
       padding-left: 0.5rem;
     }
   }
 }
+
+.version-check {
+  animation: pulse 2s ease-in-out 0s 1;
+}
+
+@keyframes pulse {
+  0% {
+    opacity: 0;
+  }
+  25% {
+    opacity: 100%;
+  }
+  75% {
+    opacity: 0;
+  }
+  100% {
+    opacity: 100%;
+  }
+}

src-ui/src/app/components/app-frame/app-frame.component.ts

@@ -18,6 +18,11 @@ import { DocumentDetailComponent } from '../document-detail/document-detail.comp
 import { Meta } from '@angular/platform-browser'
 import { DocumentListViewService } from 'src/app/services/document-list-view.service'
 import { FILTER_FULLTEXT_QUERY } from 'src/app/data/filter-rule-type'
+import {
+  RemoteVersionService,
+  AppRemoteVersion,
+} from 'src/app/services/rest/remote-version.service'
+import { QueryParamsService } from 'src/app/services/query-params.service'
 
 @Component({
   selector: 'app-app-frame',
@@ -32,10 +37,19 @@ export class AppFrameComponent {
     private searchService: SearchService,
     public savedViewService: SavedViewService,
     private list: DocumentListViewService,
-    private meta: Meta
-  ) {}
+    private meta: Meta,
+    private remoteVersionService: RemoteVersionService,
+    private queryParamsService: QueryParamsService
+  ) {
+    this.remoteVersionService
+      .checkForUpdates()
+      .subscribe((appRemoteVersion: AppRemoteVersion) => {
+        this.appRemoteVersion = appRemoteVersion
+      })
+  }
 
   versionString = `${environment.appTitle} ${environment.version}`
+  appRemoteVersion
 
   isMenuCollapsed: boolean = true
 
@@ -80,7 +94,7 @@ export class AppFrameComponent {
 
   search() {
     this.closeMenu()
-    this.list.quickFilter([
+    this.queryParamsService.navigateWithFilterRules([
       {
         rule_type: FILTER_FULLTEXT_QUERY,
         value: (this.searchField.value as string).trim(),

src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.scss

@@ -1,5 +1,3 @@
-@import "/src/theme";
-
 .badge-corner {
   position: absolute;
   top: -8px;
@@ -42,7 +40,7 @@
   filter: brightness(0.5);
 
   &.active {
-    background-color: var(--ngx-primary-lighten-30);
+    background-color: var(--pngx-primary-lighten-30);
   }
 }
 

src-ui/src/app/components/common/toasts/toasts.component.html

@@ -2,7 +2,7 @@
   *ngFor="let toast of toasts"
   [header]="toast.title" [autohide]="true" [delay]="toast.delay"
   [class]="toast.classname"
-  (hide)="toastService.closeToast(toast)">
+  (hidden)="toastService.closeToast(toast)">
   <p>{{toast.content}}</p>
   <p *ngIf="toast.action"><button class="btn btn-sm btn-outline-secondary" (click)="toastService.closeToast(toast); toast.action()">{{toast.actionName}}</button></p>
 </ngb-toast>

src-ui/src/app/components/common/toasts/toasts.component.scss

@@ -5,3 +5,7 @@
   margin: 0.5em;
   z-index: 1200;
 }
+
+.toast:not(.show) {
+  display: block; // this corrects an ng-bootstrap bug that prevented animations
+}

src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts

@@ -3,11 +3,11 @@ import { Router } from '@angular/router'
 import { Subscription } from 'rxjs'
 import { PaperlessDocument } from 'src/app/data/paperless-document'
 import { PaperlessSavedView } from 'src/app/data/paperless-saved-view'
-import { DocumentListViewService } from 'src/app/services/document-list-view.service'
 import { ConsumerStatusService } from 'src/app/services/consumer-status.service'
 import { DocumentService } from 'src/app/services/rest/document.service'
 import { PaperlessTag } from 'src/app/data/paperless-tag'
 import { FILTER_HAS_TAGS_ALL } from 'src/app/data/filter-rule-type'
+import { QueryParamsService } from 'src/app/services/query-params.service'
 
 @Component({
   selector: 'app-saved-view-widget',
@@ -18,7 +18,7 @@ export class SavedViewWidgetComponent implements OnInit, OnDestroy {
   constructor(
     private documentService: DocumentService,
     private router: Router,
-    private list: DocumentListViewService,
+    private queryParamsService: QueryParamsService,
     private consumerStatusService: ConsumerStatusService
   ) {}
 
@@ -60,13 +60,14 @@ export class SavedViewWidgetComponent implements OnInit, OnDestroy {
     if (this.savedView.show_in_sidebar) {
       this.router.navigate(['view', this.savedView.id])
     } else {
-      this.list.loadSavedView(this.savedView, true)
-      this.router.navigate(['documents'])
+      this.router.navigate(['documents'], {
+        queryParams: { view: this.savedView.id },
+      })
     }
   }
 
   clickTag(tag: PaperlessTag) {
-    this.list.quickFilter([
+    this.queryParamsService.navigateWithFilterRules([
       { rule_type: FILTER_HAS_TAGS_ALL, value: tag.id.toString() },
     ])
   }

src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss

@@ -1,5 +1,3 @@
-@import "/src/theme";
-
 form {
   position: relative;
 }
@@ -33,3 +31,7 @@ form {
   mix-blend-mode: soft-light;
   pointer-events: none;
 }
+
+::ng-deep .ngx-file-drop__drop-zone--over {
+  background-color: var(--pngx-primary-faded) !important;
+}

src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts

@@ -6,7 +6,7 @@ import {
   FileStatus,
   FileStatusPhase,
 } from 'src/app/services/consumer-status.service'
-import { DocumentService } from 'src/app/services/rest/document.service'
+import { UploadDocumentsService } from 'src/app/services/upload-documents.service'
 
 const MAX_ALERTS = 5
 
@@ -19,8 +19,8 @@ export class UploadFileWidgetComponent implements OnInit {
   alertsExpanded = false
 
   constructor(
-    private documentService: DocumentService,
-    private consumerStatusService: ConsumerStatusService
+    private consumerStatusService: ConsumerStatusService,
+    private uploadDocumentsService: UploadDocumentsService
   ) {}
 
   getStatus() {
@@ -116,48 +116,6 @@ export class UploadFileWidgetComponent implements OnInit {
   public fileLeave(event) {}
 
   public dropped(files: NgxFileDropEntry[]) {
-    for (const droppedFile of files) {
-      if (droppedFile.fileEntry.isFile) {
-        const fileEntry = droppedFile.fileEntry as FileSystemFileEntry
-        fileEntry.file((file: File) => {
-          let formData = new FormData()
-          formData.append('document', file, file.name)
-          let status = this.consumerStatusService.newFileUpload(file.name)
-
-          status.message = $localize`Connecting...`
-
-          this.documentService.uploadDocument(formData).subscribe(
-            (event) => {
-              if (event.type == HttpEventType.UploadProgress) {
-                status.updateProgress(
-                  FileStatusPhase.UPLOADING,
-                  event.loaded,
-                  event.total
-                )
-                status.message = $localize`Uploading...`
-              } else if (event.type == HttpEventType.Response) {
-                status.taskId = event.body['task_id']
-                status.message = $localize`Upload complete, waiting...`
-              }
-            },
-            (error) => {
-              switch (error.status) {
-                case 400: {
-                  this.consumerStatusService.fail(status, error.error.document)
-                  break
-                }
-                default: {
-                  this.consumerStatusService.fail(
-                    status,
-                    $localize`HTTP error: ${error.status} ${error.statusText}`
-                  )
-                  break
-                }
-              }
-            }
-          )
-        })
-      }
-    }
+    this.uploadDocumentsService.uploadFiles(files)
   }
 }

src-ui/src/app/components/document-detail/document-detail.component.html

@@ -137,9 +137,10 @@
                 <li [ngbNavItem]="4" class="d-md-none">
                     <a ngbNavLink>Preview</a>
                     <ng-template ngbNavContent *ngIf="pdfPreview.offsetParent == undefined">
+                        <div class="position-relative">
                             <ng-container *ngIf="getContentType() == 'application/pdf'">
                                 <div class="preview-sticky pdf-viewer-container" *ngIf="!useNativePdfViewer ; else nativePdfViewer">
-                            <pdf-viewer [src]="previewUrl" [original-size]="false" [show-borders]="true" [show-all]="true" [render-text-mode]="2"></pdf-viewer>
+                                    <pdf-viewer [src]="{ url: previewUrl, password: password }" [original-size]="false" [show-borders]="true" [show-all]="true" [(page)]="previewCurrentPage" [render-text-mode]="2" (error)="onError($event)" (after-load-complete)="pdfPreviewLoaded($event)"></pdf-viewer>
                                 </div>
                                 <ng-template #nativePdfViewer>
                                     <object [data]="previewUrl | safeUrl" class="preview-sticky" width="100%"></object>
@@ -148,6 +149,12 @@
                             <ng-container *ngIf="getContentType() == 'text/plain'">
                                 <object [data]="previewUrl | safeUrl" type="text/plain" class="preview-sticky bg-white" width="100%"></object>
                             </ng-container>
+                            <div *ngIf="requiresPassword" class="password-prompt">
+                                <form>
+                                    <input autocomplete="" class="form-control" i18n-placeholder placeholder="Enter Password" type="password" (keyup)="onPasswordKeyUp($event)" />
+                                </form>
+                            </div>
+                        </div>
                     </ng-template>
                 </li>
             </ul>
@@ -155,15 +162,15 @@
             <div [ngbNavOutlet]="nav" class="mt-2"></div>
 
             <button type="button" class="btn btn-outline-secondary" (click)="discard()" i18n [disabled]="networkActive || !(isDirty$ | async)">Discard</button>&nbsp;
-            <button type="button" class="btn btn-outline-primary" (click)="saveEditNext()" *ngIf="hasNext()" i18n [disabled]="networkActive || !(isDirty$ | async)">Save & next</button>&nbsp;
-            <button type="submit" class="btn btn-primary" i18n [disabled]="networkActive || !(isDirty$ | async)">Save</button>&nbsp;
+            <button type="button" class="btn btn-outline-primary" (click)="saveEditNext()" *ngIf="hasNext()" i18n [disabled]="networkActive || !(isDirty$ | async) || error">Save & next</button>&nbsp;
+            <button type="submit" class="btn btn-primary" i18n [disabled]="networkActive || !(isDirty$ | async) || error">Save</button>&nbsp;
         </form>
     </div>
 
-    <div class="col-md-6 col-xl-8 mb-3 d-none d-md-block" #pdfPreview>
+    <div class="col-md-6 col-xl-8 mb-3 d-none d-md-block position-relative" #pdfPreview>
         <ng-container *ngIf="getContentType() == 'application/pdf'">
             <div class="preview-sticky pdf-viewer-container" *ngIf="!useNativePdfViewer ; else nativePdfViewer">
-                <pdf-viewer [src]="previewUrl" [original-size]="false" [show-borders]="true" [show-all]="true" [(page)]="previewCurrentPage" [render-text-mode]="2" (after-load-complete)="pdfPreviewLoaded($event)"></pdf-viewer>
+                <pdf-viewer [src]="{ url: previewUrl, password: password }" [original-size]="false" [show-borders]="true" [show-all]="true" [(page)]="previewCurrentPage" [render-text-mode]="2" (error)="onError($event)" (after-load-complete)="pdfPreviewLoaded($event)"></pdf-viewer>
             </div>
             <ng-template #nativePdfViewer>
                 <object [data]="previewUrl | safeUrl" class="preview-sticky" width="100%"></object>
@@ -172,5 +179,11 @@
         <ng-container *ngIf="getContentType() == 'text/plain'">
             <object [data]="previewUrl | safeUrl" type="text/plain" class="preview-sticky bg-white" width="100%"></object>
         </ng-container>
+        <div *ngIf="requiresPassword" class="password-prompt">
+            <form>
+                <input autocomplete="" class="form-control" i18n-placeholder placeholder="Enter Password" type="password" (keyup)="onPasswordKeyUp($event)" />
+            </form>
         </div>
     </div>
+
+</div>

src-ui/src/app/components/document-detail/document-detail.component.scss

@@ -17,3 +17,10 @@
   --page-margin: 1px 0 -8px;
   width: 100% !important;
 }
+
+.password-prompt {
+  position: absolute;
+  top: 30%;
+  left: 30%;
+  right: 30%;
+}

src-ui/src/app/components/document-detail/document-detail.component.ts

@@ -1,10 +1,4 @@
-import {
-  Component,
-  OnInit,
-  OnDestroy,
-  ViewChild,
-  ElementRef,
-} from '@angular/core'
+import { Component, OnInit, OnDestroy, ViewChild } from '@angular/core'
 import { FormControl, FormGroup } from '@angular/forms'
 import { ActivatedRoute, Router } from '@angular/router'
 import { NgbModal, NgbNav } from '@ng-bootstrap/ng-bootstrap'
@@ -40,6 +34,8 @@ import {
 } from 'rxjs/operators'
 import { PaperlessDocumentSuggestions } from 'src/app/data/paperless-document-suggestions'
 import { FILTER_FULLTEXT_MORELIKE } from 'src/app/data/filter-rule-type'
+import { normalizeDateStr } from 'src/app/utils/date'
+import { QueryParamsService } from 'src/app/services/query-params.service'
 
 @Component({
   selector: 'app-document-detail',
@@ -90,6 +86,11 @@ export class DocumentDetailComponent
   isDirty$: Observable<boolean>
   unsubscribeNotifier: Subject<any> = new Subject()
 
+  requiresPassword: boolean = false
+  password: string
+
+  ogDate: Date
+
   @ViewChild('nav') nav: NgbNav
   @ViewChild('pdfPreview') set pdfPreview(element) {
     // this gets called when compontent added or removed from DOM
@@ -114,7 +115,8 @@ export class DocumentDetailComponent
     private documentListViewService: DocumentListViewService,
     private documentTitlePipe: DocumentTitlePipe,
     private toastService: ToastService,
-    private settings: SettingsService
+    private settings: SettingsService,
+    private queryParamsService: QueryParamsService
   ) {
     this.titleSubject
       .pipe(
@@ -145,7 +147,27 @@ export class DocumentDetailComponent
   ngOnInit(): void {
     this.documentForm.valueChanges
       .pipe(takeUntil(this.unsubscribeNotifier))
-      .subscribe((wow) => {
+      .subscribe((changes) => {
+        this.error = null
+        if (this.ogDate) {
+          try {
+            let newDate = new Date(normalizeDateStr(changes['created']))
+            newDate.setHours(
+              this.ogDate.getHours(),
+              this.ogDate.getMinutes(),
+              this.ogDate.getSeconds(),
+              this.ogDate.getMilliseconds()
+            )
+            this.documentForm.patchValue(
+              { created: newDate.toISOString() },
+              { emitEvent: false }
+            )
+          } catch (e) {
+            // catch this before we try to save and simulate an api error
+            this.error = { created: e.message }
+          }
+        }
+
         Object.assign(this.document, this.documentForm.value)
       })
 
@@ -186,17 +208,25 @@ export class DocumentDetailComponent
             this.updateComponent(doc)
           }
 
+          this.ogDate = new Date(normalizeDateStr(doc.created.toString()))
+
           // Initialize dirtyCheck
           this.store = new BehaviorSubject({
             title: doc.title,
             content: doc.content,
-            created: doc.created,
+            created: this.ogDate.toISOString(),
             correspondent: doc.correspondent,
             document_type: doc.document_type,
             archive_serial_number: doc.archive_serial_number,
             tags: [...doc.tags],
           })
 
+          // start with ISO8601 string
+          this.documentForm.patchValue(
+            { created: this.ogDate.toISOString() },
+            { emitEvent: false }
+          )
+
           this.isDirty$ = dirtyCheck(
             this.documentForm,
             this.store.asObservable()
@@ -223,6 +253,7 @@ export class DocumentDetailComponent
 
   updateComponent(doc: PaperlessDocument) {
     this.document = doc
+    this.requiresPassword = false
     this.documentsService
       .getMetadata(doc.id)
       .pipe(first())
@@ -297,16 +328,17 @@ export class DocumentDetailComponent
     this.documentsService
       .get(this.documentId)
       .pipe(first())
-      .subscribe(
-        (doc) => {
+      .subscribe({
+        next: (doc) => {
           Object.assign(this.document, doc)
           this.title = doc.title
           this.documentForm.patchValue(doc)
+          this.openDocumentService.setDirty(doc.id, false)
         },
-        (error) => {
+        error: () => {
           this.router.navigate(['404'])
-        }
-      )
+        },
+      })
   }
 
   save() {
@@ -416,7 +448,7 @@ export class DocumentDetailComponent
   }
 
   moreLike() {
-    this.documentListViewService.quickFilter([
+    this.queryParamsService.navigateWithFilterRules([
       {
         rule_type: FILTER_FULLTEXT_MORELIKE,
         value: this.documentId.toString(),
@@ -450,5 +482,18 @@ export class DocumentDetailComponent
 
   pdfPreviewLoaded(pdf: PDFDocumentProxy) {
     this.previewNumPages = pdf.numPages
+    if (this.password) this.requiresPassword = false
+  }
+
+  onError(event) {
+    if (event.name == 'PasswordException') {
+      this.requiresPassword = true
+    }
+  }
+
+  onPasswordKeyUp(event: KeyboardEvent) {
+    if ('Enter' == event.key) {
+      this.password = (event.target as HTMLInputElement).value
+    }
   }
 }

src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.html

@@ -57,13 +57,18 @@
   </div>
   <div class="col-auto ms-auto mb-2 mb-xl-0 d-flex">
     <div class="btn-group btn-group-sm me-2">
-      <button type="button" class="btn btn-outline-primary btn-sm" (click)="downloadSelected()">
-        <svg width="1em" height="1em" viewBox="0 0 16 16" fill="currentColor">
+      <button type="button" [disabled]="awaitingDownload" class="btn btn-outline-primary btn-sm" (click)="downloadSelected()">
+        <svg *ngIf="!awaitingDownload" width="1em" height="1em" viewBox="0 0 16 16" fill="currentColor">
           <use xlink:href="assets/bootstrap-icons.svg#download" />
-        </svg>&nbsp;<ng-container i18n>Download</ng-container>
+        </svg>
+        <div *ngIf="awaitingDownload" class="spinner-border spinner-border-sm" role="status">
+          <span class="visually-hidden">Preparing download...</span>
+        </div>
+        &nbsp;
+        <ng-container i18n>Download</ng-container>
       </button>
       <div class="btn-group" ngbDropdown role="group" aria-label="Button group with nested dropdown">
-        <button class="btn btn-outline-primary btn-sm dropdown-toggle-split" ngbDropdownToggle></button>
+        <button [disabled]="awaitingDownload" class="btn btn-outline-primary btn-sm dropdown-toggle-split" ngbDropdownToggle></button>
         <div class="dropdown-menu shadow" ngbDropdownMenu>
           <button ngbDropdownItem i18n (click)="downloadSelected('originals')">Download originals</button>
         </div>

src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.ts

@@ -39,6 +39,7 @@ export class BulkEditorComponent {
   tagSelectionModel = new FilterableDropdownSelectionModel()
   correspondentSelectionModel = new FilterableDropdownSelectionModel()
   documentTypeSelectionModel = new FilterableDropdownSelectionModel()
+  awaitingDownload: boolean
 
   constructor(
     private documentTypeService: DocumentTypeService,
@@ -317,10 +318,12 @@ export class BulkEditorComponent {
   }
 
   downloadSelected(content = 'archive') {
+    this.awaitingDownload = true
     this.documentService
       .bulkDownload(Array.from(this.list.selected), content)
       .subscribe((result: any) => {
         saveAs(result, 'documents.zip')
+        this.awaitingDownload = false
       })
   }
 }

src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html

@@ -42,7 +42,7 @@
                 <path fill-rule="evenodd" d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5L13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5zm-9.761 5.175l-.106.106-1.528 3.821 3.821-1.528.106-.106A.5.5 0 0 1 5 12.5V12h-.5a.5.5 0 0 1-.5-.5V11h-.5a.5.5 0 0 1-.468-.325z"/>
               </svg>&nbsp;<span class="d-block d-md-inline" i18n>Edit</span>
             </a>
-            <a class="btn btn-sm btn-outline-secondary" [href]="previewUrl"
+            <a class="btn btn-sm btn-outline-secondary" target="_blank" [href]="previewUrl"
             [ngbPopover]="previewContent" [popoverTitle]="document.title | documentTitle"
             autoClose="true" popoverClass="shadow" (mouseenter)="mouseEnterPreview()" (mouseleave)="mouseLeavePreview()" #popover="ngbPopover">
               <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-eye" viewBox="0 0 16 16">

src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss

@@ -1,5 +1,3 @@
-@import "/src/theme";
-
 .result-content {
   overflow-wrap: anywhere;
 }
@@ -60,7 +58,7 @@
 }
 
 .doc-img-background-selected {
-  background-color: var(--ngx-primary-faded);
+  background-color: var(--pngx-primary-faded);
 }
 
 .card-info {

src-ui/src/app/components/document-list/document-card-small/document-card-small.component.scss

@@ -1,5 +1,3 @@
-@import "/src/theme";
-
 .card-text {
   font-size: 90%;
 }
@@ -45,7 +43,7 @@
 }
 
 .doc-img-background-selected {
-  background-color: var(--ngx-primary-faded);
+  background-color: var(--pngx-primary-faded);
 }
 
 .card-info {