Rework Paperless-ngx around Caddy, network, and storage roles

paperless-ngx/files/nginx.conf.j2

@@ -1,27 +0,0 @@
-server {
-
-  listen 80;
-  server_name {{ http_host }};
-
-  # Allow large attachments
-  client_max_body_size 128M;
-
-  location / {
-
-    # Adjust host and port as required.
-    proxy_pass http://localhost:{{ web_port }}/;
-
-    # These configuration options are required for WebSockets to work.
-    proxy_http_version 1.1;
-    proxy_set_header Upgrade $http_upgrade;
-    proxy_set_header Connection "upgrade";
-
-    proxy_redirect off;
-    proxy_set_header Host $host;
-    proxy_set_header X-Real-IP $remote_addr;
-    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-    proxy_set_header X-Forwarded-Host $server_name;
-    add_header Referrer-Policy "strict-origin-when-cross-origin";
-  }
-}
-

paperless-ngx/playbook.yml

@@ -1,9 +1,11 @@
 ---
-- name: Setup Paperless-NGX and Nginx reverse proxy
+- name: Setup Paperless-NGX behind Caddy
-  hosts: gondor
+  hosts: monoco
   become: true
   vars_files:
     - vars/default.yml
   roles:
+    - network
+    - storage
     - paperless
-    - nginx
+    - caddy

paperless-ngx/readme.md

@@ -4,3 +4,16 @@ Create a super user inititally:
 ```shell
 docker exec -it paperless-ngx python3 manage.py createsuperuser
 ```
+
+## Storage (AWS only)
+
+The VPS holds only paperless's small `data` dir. Everything else is mounted with rclone
+(systemd units `paperless-*.service`, config in `/etc/rclone-paperless.conf`):
+
+- `consume`: Nextcloud WebDAV folder (`nextcloud_consume_path`). Drop files in it from Nextcloud and
+  paperless ingests them (polled every 60s, as inotify does not work on WebDAV).
+- `export`: Nextcloud WebDAV folder (`nextcloud_export_path`), filled nightly by `document_exporter`.
+- `media`: an S3 bucket (`s3_bucket`), cached locally up to 2G. Use a different bucket from Nextcloud's.
+
+Set `nextcloud_app_password` and the S3 keys in `vars/default.yml` (ideally via ansible-vault).
+paperless does not officially support S3 for media, so expect it to be slower, and keep S3 versioning on.

paperless-ngx/roles/caddy/handlers/main.yaml

@@ -0,0 +1,5 @@
+---
+# Handlers for Caddy role
+- name: Reload Caddy configuration
+  ansible.builtin.command: "docker exec -w /etc/caddy {{ caddy_container }} caddy reload"
+  listen: "Reload Caddy"

paperless-ngx/roles/caddy/tasks/main.yaml

@@ -0,0 +1,22 @@
+---
+- name: Read current Caddyfile
+  ansible.builtin.slurp:
+    path: "{{ caddyfile_path }}"
+  register: caddyfile_raw
+
+- name: Decode Caddyfile content
+  set_fact:
+    caddyfile_content: "{{ caddyfile_raw['content'] | b64decode }}"
+
+- name: Check if snippet already exists
+  set_fact:
+    snippet_present: "{{ snippet_block in caddyfile_content }}"
+
+- name: Add snippet if missing
+  ansible.builtin.blockinfile:
+    path: "{{ caddyfile_path }}"
+    marker: "# {mark} Ansible managed Caddy snippet for {{ host }}"
+    block: "{{ snippet_block }}"
+    create: yes
+  when: not snippet_present
+  notify: "Reload Caddy"

paperless-ngx/roles/network/tasks/main.yaml

@@ -0,0 +1,10 @@
+---
+- name: "Ensure {{ container_name }} Docker network exists"
+  docker_network:
+    name: "{{ docker_network }}"
+    state: present
+    connected:
+      - "{{ caddy_container }}"
+      - "{{ postgres_container }}"
+      - "{{ redis_container }}"
+    appends: yes

paperless-ngx/roles/nginx/handlers/main.yaml

@@ -1,6 +0,0 @@
----
-# Handlers for Nginx role
-- name: Reload Nginx
-  service:
-    name: nginx
-    state: reloaded

paperless-ngx/roles/nginx/tasks/main.yaml

@@ -1,24 +0,0 @@
----
-# Tasks for Nginx role
-- name: Check if Nginx conf file exists
-  stat:
-    path: "/etc/nginx/sites-available/{{ http_conf }}"
-  register: status
-
-- name: No need to reload Nginx
-  debug:
-    msg: "No need to reload Nginx as sites-available entries have already been created"
-
-- name: Set Nginx conf file
-  when: status.stat.exists == false
-  template:
-    src: "files/nginx.conf.j2"
-    dest: "/etc/nginx/sites-available/{{ http_conf }}"
-
-- name: Enable new site
-  when: status.stat.exists == false
-  file:
-    src: "/etc/nginx/sites-available/{{ http_conf }}"
-    dest: "/etc/nginx/sites-enabled/{{ http_conf }}"
-    state: link
-  notify: Reload Nginx

paperless-ngx/roles/paperless/tasks/main.yml

@@ -5,41 +5,46 @@
     source: pull
     force_source: yes
 
-
 - name: Ensure Paperless Directories exists
   file:
       path: "{{ item }}"
       state: directory
-      mode: 0755
+      mode: "0755"
   loop:
     - "/var/lib/{{ host_directory }}"
     - "/var/lib/{{ host_directory }}/data"
-    - "/var/lib/{{ host_directory }}/media"
-    - "/var/lib/{{ host_directory }}/export"
-    - "/var/lib/{{ host_directory }}/consume"
 
 - name: Run Paperless-NGX Docker container
   community.docker.docker_container:
     name: "{{ container_name }}"
     image: "{{ image_name }}:{{ image_version }}"
     state: started
-    restart_policy: always
+    restart_policy: unless-stopped
-    links:
+    networks:
-      - postgres:db
+      - name: "{{ docker_network }}"
-      - redis:redis
     env:
-      PAPERLESS_URL: "https://{{ http_host }}"
+      PAPERLESS_URL: "https://{{ host }}"
-      PAPERLESS_REDIS: redis://redis:6379
+      PAPERLESS_REDIS: redis://{{ redis_container }}:6379
-      PAPERLESS_DBHOST: db
+      PAPERLESS_DBHOST: "{{ postgres_container }}"
       PAPERLESS_DBUSER: "{{ postgres_user }}"
       PAPERLESS_DBPASS: "{{ postgres_password }}"
       PAPERLESS_DBNAME: "{{ postgres_db }}"
       PAPERLESS_TIME_ZONE: "{{ paperless_time_zone }}"
+      USERMAP_UID: "{{ paperless_uid }}"
+      USERMAP_GID: "{{ paperless_uid }}"
+      PAPERLESS_CONSUMER_RECURSIVE: "true"
+      # inotify cannot see changes made remotely through WebDAV, so poll
+      PAPERLESS_CONSUMER_POLLING: "60"
       PAPERLESS_OCR_USER_ARGS: '{"invalidate_digital_signatures": true}'
     volumes:
       - "/var/lib/{{ host_directory }}/data:/usr/src/paperless/data"
-      - "/var/lib/{{ host_directory }}/media:/usr/src/paperless/media"
+      - "/mnt/paperless/media:/usr/src/paperless/media:rslave"
-      - "/var/lib/{{ host_directory }}/export:/usr/src/paperless/export"
+      - "/mnt/paperless/export:/usr/src/paperless/export:rslave"
-      - "/var/lib/{{ host_directory }}/consume:/usr/src/paperless/consume"
+      - "/mnt/paperless/consume:/usr/src/paperless/consume:rslave"
-    ports:
+
-      - "127.0.0.1:{{ web_port }}:8000"
+- name: Export documents nightly to the Nextcloud export folder
+  cron:
+    name: Paperless export
+    minute: "30"
+    hour: "2"
+    job: "docker exec {{ container_name }} document_exporter ../export -d"

paperless-ngx/roles/storage/handlers/main.yaml

@@ -0,0 +1,7 @@
+---
+- name: Restart rclone mounts
+  systemd:
+    name: "paperless-{{ item.name }}.service"
+    state: restarted
+    daemon_reload: true
+  loop: "{{ rclone_mounts }}"

paperless-ngx/roles/storage/tasks/main.yaml

@@ -0,0 +1,59 @@
+---
+# rclone FUSE mounts so consume/export (Nextcloud WebDAV) and media (S3)
+# take no space on the VPS beyond a small cache.
+- name: Install rclone and fuse
+  apt:
+    name:
+      - rclone
+      - fuse3
+    state: present
+    update_cache: true
+
+- name: Obscure the Nextcloud password for rclone
+  command: rclone obscure "{{ nextcloud_app_password }}"
+  register: obscured_pass
+  changed_when: false
+  no_log: true
+
+- name: Write rclone config
+  template:
+    src: rclone.conf.j2
+    dest: /etc/rclone-paperless.conf
+    owner: root
+    group: root
+    mode: "0600"
+  no_log: true
+  notify: Restart rclone mounts
+
+- name: Ensure mount points and cache exist
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: "0755"
+  loop:
+    - /mnt/paperless
+    - "{{ rclone_cache_dir }}"
+    - /mnt/paperless/consume
+    - /mnt/paperless/export
+    - /mnt/paperless/media
+
+- name: Ensure the Nextcloud folders exist
+  command: rclone --config /etc/rclone-paperless.conf mkdir "{{ item.remote }}"
+  loop: "{{ rclone_mounts }}"
+  changed_when: false
+
+- name: Install mount units
+  template:
+    src: rclone-mount.service.j2
+    dest: "/etc/systemd/system/paperless-{{ item.name }}.service"
+    mode: "0644"
+  loop: "{{ rclone_mounts }}"
+  notify: Restart rclone mounts
+
+- name: Enable and start mount units
+  systemd:
+    name: "paperless-{{ item.name }}.service"
+    enabled: true
+    state: started
+    daemon_reload: true
+  loop: "{{ rclone_mounts }}"

paperless-ngx/roles/storage/templates/rclone-mount.service.j2

@@ -0,0 +1,19 @@
+[Unit]
+Description=rclone mount for Paperless {{ item.name }}
+After=network-online.target
+Wants=network-online.target
+Before=docker.service
+
+[Service]
+Type=notify
+ExecStart=/usr/bin/rclone mount {{ item.remote }} /mnt/paperless/{{ item.name }} \
+  --config /etc/rclone-paperless.conf \
+  --allow-other --uid {{ paperless_uid }} --gid {{ paperless_uid }} --umask 022 \
+  --vfs-cache-mode {{ item.cache }} --cache-dir {{ rclone_cache_dir }} \
+  --vfs-cache-max-size 2G --dir-cache-time 30s
+ExecStop=/bin/fusermount3 -u /mnt/paperless/{{ item.name }}
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target

paperless-ngx/roles/storage/templates/rclone.conf.j2

@@ -0,0 +1,13 @@
+[nextcloud]
+type = webdav
+url = {{ nextcloud_url }}/remote.php/dav/files/{{ nextcloud_user }}
+vendor = nextcloud
+user = {{ nextcloud_user }}
+pass = {{ obscured_pass.stdout }}
+
+[s3]
+type = s3
+provider = AWS
+access_key_id = {{ s3_access_key }}
+secret_access_key = {{ s3_secret_key }}
+region = {{ s3_region }}

paperless-ngx/vars/default.yml

@@ -1,21 +1,56 @@
 ---
-
+container_name: paperless-ngx
 image_name: ghcr.io/paperless-ngx/paperless-ngx
 image_version: "latest"
-container_name: paperless-ngx
 
-web_port: 8000          # Configurable port
+docker_network: paperless_net
+
+postgres_container: postgres
+caddy_container: caddy
+redis_container: valkey
+
+host: paperless.tanshu.com
+
+caddyfile_path: /var/lib/caddy/conf/Caddyfile
+snippet_block: |
+  {{ host }} {
+      request_body {
+          max_size 128MB
+      }
+
+      reverse_proxy {{ container_name }}:8000
+
+      header {
+          Strict-Transport-Security "max-age=15552000; includeSubDomains; preload"
+      }
+  }
 
 host_directory: paperless
-http_host: paperless.tanshu.com  # Replace with your domain
-http_conf: "{{ http_host }}.conf"
 
 paperless_time_zone: Etc/UTC
 postgres_user: "postgres"
 postgres_password: "123456"
 postgres_db: "paperless"
 
+# Storage: nothing bulky lives on the VPS disk. rclone mounts (see the storage role):
+#   consume/export -> Nextcloud WebDAV (Nextcloud itself is backed by S3)
+#   media          -> S3 bucket directly
+paperless_uid: 1000  # uid/gid paperless runs as; the mounts are presented as this user
+rclone_cache_dir: /var/cache/rclone
 
+nextcloud_url: https://cloud.hopsngrains.com
+nextcloud_user: admin
+nextcloud_app_password: "xxxx"  # Nextcloud > Settings > Security > App password
+nextcloud_consume_path: Paperless/Consume
+nextcloud_export_path: Paperless/Export
 
-# db_url: postgresql://postgres:123456@db:5432/paperless
+s3_bucket: "xxxx"  # use a bucket separate from Nextcloud's
+s3_region: ap-south-1
+s3_access_key: "xxxx"
+s3_secret_key: "xxxx"
+s3_media_prefix: media
 
+rclone_mounts:
+  - { name: consume, remote: "nextcloud:{{ nextcloud_consume_path }}", cache: "off" }
+  - { name: export, remote: "nextcloud:{{ nextcloud_export_path }}", cache: "writes" }
+  - { name: media, remote: "s3:{{ s3_bucket }}/{{ s3_media_prefix }}", cache: "full" }