diff --git a/paperless-ngx/files/nginx.conf.j2 b/paperless-ngx/files/nginx.conf.j2 deleted file mode 100644 index 9d866c2..0000000 --- a/paperless-ngx/files/nginx.conf.j2 +++ /dev/null @@ -1,27 +0,0 @@ -server { - - listen 80; - server_name {{ http_host }}; - - # Allow large attachments - client_max_body_size 128M; - - location / { - - # Adjust host and port as required. - proxy_pass http://localhost:{{ web_port }}/; - - # These configuration options are required for WebSockets to work. - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - - proxy_redirect off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Host $server_name; - add_header Referrer-Policy "strict-origin-when-cross-origin"; - } -} - diff --git a/paperless-ngx/playbook.yml b/paperless-ngx/playbook.yml index 1b955a0..a5e6924 100755 --- a/paperless-ngx/playbook.yml +++ b/paperless-ngx/playbook.yml @@ -1,9 +1,11 @@ --- -- name: Setup Paperless-NGX and Nginx reverse proxy - hosts: gondor +- name: Setup Paperless-NGX behind Caddy + hosts: monoco become: true vars_files: - vars/default.yml roles: + - network + - storage - paperless - - nginx + - caddy diff --git a/paperless-ngx/readme.md b/paperless-ngx/readme.md index d4f8281..260a63d 100644 --- a/paperless-ngx/readme.md +++ b/paperless-ngx/readme.md @@ -4,3 +4,16 @@ Create a super user inititally: ```shell docker exec -it paperless-ngx python3 manage.py createsuperuser ``` + +## Storage (AWS only) + +The VPS holds only paperless's small `data` dir. Everything else is mounted with rclone +(systemd units `paperless-*.service`, config in `/etc/rclone-paperless.conf`): + +- `consume`: Nextcloud WebDAV folder (`nextcloud_consume_path`). Drop files in it from Nextcloud and + paperless ingests them (polled every 60s, as inotify does not work on WebDAV). +- `export`: Nextcloud WebDAV folder (`nextcloud_export_path`), filled nightly by `document_exporter`. +- `media`: an S3 bucket (`s3_bucket`), cached locally up to 2G. Use a different bucket from Nextcloud's. + +Set `nextcloud_app_password` and the S3 keys in `vars/default.yml` (ideally via ansible-vault). +paperless does not officially support S3 for media, so expect it to be slower, and keep S3 versioning on. diff --git a/paperless-ngx/roles/caddy/handlers/main.yaml b/paperless-ngx/roles/caddy/handlers/main.yaml new file mode 100644 index 0000000..bc77d4f --- /dev/null +++ b/paperless-ngx/roles/caddy/handlers/main.yaml @@ -0,0 +1,5 @@ +--- +# Handlers for Caddy role +- name: Reload Caddy configuration + ansible.builtin.command: "docker exec -w /etc/caddy {{ caddy_container }} caddy reload" + listen: "Reload Caddy" diff --git a/paperless-ngx/roles/caddy/tasks/main.yaml b/paperless-ngx/roles/caddy/tasks/main.yaml new file mode 100644 index 0000000..2f514e4 --- /dev/null +++ b/paperless-ngx/roles/caddy/tasks/main.yaml @@ -0,0 +1,22 @@ +--- +- name: Read current Caddyfile + ansible.builtin.slurp: + path: "{{ caddyfile_path }}" + register: caddyfile_raw + +- name: Decode Caddyfile content + set_fact: + caddyfile_content: "{{ caddyfile_raw['content'] | b64decode }}" + +- name: Check if snippet already exists + set_fact: + snippet_present: "{{ snippet_block in caddyfile_content }}" + +- name: Add snippet if missing + ansible.builtin.blockinfile: + path: "{{ caddyfile_path }}" + marker: "# {mark} Ansible managed Caddy snippet for {{ host }}" + block: "{{ snippet_block }}" + create: yes + when: not snippet_present + notify: "Reload Caddy" diff --git a/paperless-ngx/roles/network/tasks/main.yaml b/paperless-ngx/roles/network/tasks/main.yaml new file mode 100644 index 0000000..dc5cf7f --- /dev/null +++ b/paperless-ngx/roles/network/tasks/main.yaml @@ -0,0 +1,10 @@ +--- +- name: "Ensure {{ container_name }} Docker network exists" + docker_network: + name: "{{ docker_network }}" + state: present + connected: + - "{{ caddy_container }}" + - "{{ postgres_container }}" + - "{{ redis_container }}" + appends: yes diff --git a/paperless-ngx/roles/nginx/handlers/main.yaml b/paperless-ngx/roles/nginx/handlers/main.yaml deleted file mode 100644 index 622fd36..0000000 --- a/paperless-ngx/roles/nginx/handlers/main.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# Handlers for Nginx role -- name: Reload Nginx - service: - name: nginx - state: reloaded diff --git a/paperless-ngx/roles/nginx/tasks/main.yaml b/paperless-ngx/roles/nginx/tasks/main.yaml deleted file mode 100644 index 4441fe7..0000000 --- a/paperless-ngx/roles/nginx/tasks/main.yaml +++ /dev/null @@ -1,24 +0,0 @@ ---- -# Tasks for Nginx role -- name: Check if Nginx conf file exists - stat: - path: "/etc/nginx/sites-available/{{ http_conf }}" - register: status - -- name: No need to reload Nginx - debug: - msg: "No need to reload Nginx as sites-available entries have already been created" - -- name: Set Nginx conf file - when: status.stat.exists == false - template: - src: "files/nginx.conf.j2" - dest: "/etc/nginx/sites-available/{{ http_conf }}" - -- name: Enable new site - when: status.stat.exists == false - file: - src: "/etc/nginx/sites-available/{{ http_conf }}" - dest: "/etc/nginx/sites-enabled/{{ http_conf }}" - state: link - notify: Reload Nginx diff --git a/paperless-ngx/roles/paperless/tasks/main.yml b/paperless-ngx/roles/paperless/tasks/main.yml index 8374691..15016a4 100644 --- a/paperless-ngx/roles/paperless/tasks/main.yml +++ b/paperless-ngx/roles/paperless/tasks/main.yml @@ -5,41 +5,46 @@ source: pull force_source: yes - - name: Ensure Paperless Directories exists file: path: "{{ item }}" state: directory - mode: 0755 + mode: "0755" loop: - "/var/lib/{{ host_directory }}" - "/var/lib/{{ host_directory }}/data" - - "/var/lib/{{ host_directory }}/media" - - "/var/lib/{{ host_directory }}/export" - - "/var/lib/{{ host_directory }}/consume" - name: Run Paperless-NGX Docker container community.docker.docker_container: name: "{{ container_name }}" image: "{{ image_name }}:{{ image_version }}" state: started - restart_policy: always - links: - - postgres:db - - redis:redis + restart_policy: unless-stopped + networks: + - name: "{{ docker_network }}" env: - PAPERLESS_URL: "https://{{ http_host }}" - PAPERLESS_REDIS: redis://redis:6379 - PAPERLESS_DBHOST: db + PAPERLESS_URL: "https://{{ host }}" + PAPERLESS_REDIS: redis://{{ redis_container }}:6379 + PAPERLESS_DBHOST: "{{ postgres_container }}" PAPERLESS_DBUSER: "{{ postgres_user }}" PAPERLESS_DBPASS: "{{ postgres_password }}" PAPERLESS_DBNAME: "{{ postgres_db }}" PAPERLESS_TIME_ZONE: "{{ paperless_time_zone }}" + USERMAP_UID: "{{ paperless_uid }}" + USERMAP_GID: "{{ paperless_uid }}" + PAPERLESS_CONSUMER_RECURSIVE: "true" + # inotify cannot see changes made remotely through WebDAV, so poll + PAPERLESS_CONSUMER_POLLING: "60" PAPERLESS_OCR_USER_ARGS: '{"invalidate_digital_signatures": true}' volumes: - "/var/lib/{{ host_directory }}/data:/usr/src/paperless/data" - - "/var/lib/{{ host_directory }}/media:/usr/src/paperless/media" - - "/var/lib/{{ host_directory }}/export:/usr/src/paperless/export" - - "/var/lib/{{ host_directory }}/consume:/usr/src/paperless/consume" - ports: - - "127.0.0.1:{{ web_port }}:8000" \ No newline at end of file + - "/mnt/paperless/media:/usr/src/paperless/media:rslave" + - "/mnt/paperless/export:/usr/src/paperless/export:rslave" + - "/mnt/paperless/consume:/usr/src/paperless/consume:rslave" + +- name: Export documents nightly to the Nextcloud export folder + cron: + name: Paperless export + minute: "30" + hour: "2" + job: "docker exec {{ container_name }} document_exporter ../export -d" diff --git a/paperless-ngx/roles/storage/handlers/main.yaml b/paperless-ngx/roles/storage/handlers/main.yaml new file mode 100644 index 0000000..32d4bd6 --- /dev/null +++ b/paperless-ngx/roles/storage/handlers/main.yaml @@ -0,0 +1,7 @@ +--- +- name: Restart rclone mounts + systemd: + name: "paperless-{{ item.name }}.service" + state: restarted + daemon_reload: true + loop: "{{ rclone_mounts }}" diff --git a/paperless-ngx/roles/storage/tasks/main.yaml b/paperless-ngx/roles/storage/tasks/main.yaml new file mode 100644 index 0000000..ea9b128 --- /dev/null +++ b/paperless-ngx/roles/storage/tasks/main.yaml @@ -0,0 +1,59 @@ +--- +# rclone FUSE mounts so consume/export (Nextcloud WebDAV) and media (S3) +# take no space on the VPS beyond a small cache. +- name: Install rclone and fuse + apt: + name: + - rclone + - fuse3 + state: present + update_cache: true + +- name: Obscure the Nextcloud password for rclone + command: rclone obscure "{{ nextcloud_app_password }}" + register: obscured_pass + changed_when: false + no_log: true + +- name: Write rclone config + template: + src: rclone.conf.j2 + dest: /etc/rclone-paperless.conf + owner: root + group: root + mode: "0600" + no_log: true + notify: Restart rclone mounts + +- name: Ensure mount points and cache exist + file: + path: "{{ item }}" + state: directory + mode: "0755" + loop: + - /mnt/paperless + - "{{ rclone_cache_dir }}" + - /mnt/paperless/consume + - /mnt/paperless/export + - /mnt/paperless/media + +- name: Ensure the Nextcloud folders exist + command: rclone --config /etc/rclone-paperless.conf mkdir "{{ item.remote }}" + loop: "{{ rclone_mounts }}" + changed_when: false + +- name: Install mount units + template: + src: rclone-mount.service.j2 + dest: "/etc/systemd/system/paperless-{{ item.name }}.service" + mode: "0644" + loop: "{{ rclone_mounts }}" + notify: Restart rclone mounts + +- name: Enable and start mount units + systemd: + name: "paperless-{{ item.name }}.service" + enabled: true + state: started + daemon_reload: true + loop: "{{ rclone_mounts }}" diff --git a/paperless-ngx/roles/storage/templates/rclone-mount.service.j2 b/paperless-ngx/roles/storage/templates/rclone-mount.service.j2 new file mode 100644 index 0000000..b7c6d37 --- /dev/null +++ b/paperless-ngx/roles/storage/templates/rclone-mount.service.j2 @@ -0,0 +1,19 @@ +[Unit] +Description=rclone mount for Paperless {{ item.name }} +After=network-online.target +Wants=network-online.target +Before=docker.service + +[Service] +Type=notify +ExecStart=/usr/bin/rclone mount {{ item.remote }} /mnt/paperless/{{ item.name }} \ + --config /etc/rclone-paperless.conf \ + --allow-other --uid {{ paperless_uid }} --gid {{ paperless_uid }} --umask 022 \ + --vfs-cache-mode {{ item.cache }} --cache-dir {{ rclone_cache_dir }} \ + --vfs-cache-max-size 2G --dir-cache-time 30s +ExecStop=/bin/fusermount3 -u /mnt/paperless/{{ item.name }} +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/paperless-ngx/roles/storage/templates/rclone.conf.j2 b/paperless-ngx/roles/storage/templates/rclone.conf.j2 new file mode 100644 index 0000000..0531d6f --- /dev/null +++ b/paperless-ngx/roles/storage/templates/rclone.conf.j2 @@ -0,0 +1,13 @@ +[nextcloud] +type = webdav +url = {{ nextcloud_url }}/remote.php/dav/files/{{ nextcloud_user }} +vendor = nextcloud +user = {{ nextcloud_user }} +pass = {{ obscured_pass.stdout }} + +[s3] +type = s3 +provider = AWS +access_key_id = {{ s3_access_key }} +secret_access_key = {{ s3_secret_key }} +region = {{ s3_region }} diff --git a/paperless-ngx/vars/default.yml b/paperless-ngx/vars/default.yml index 7abd94f..1073593 100644 --- a/paperless-ngx/vars/default.yml +++ b/paperless-ngx/vars/default.yml @@ -1,21 +1,56 @@ --- - +container_name: paperless-ngx image_name: ghcr.io/paperless-ngx/paperless-ngx image_version: "latest" -container_name: paperless-ngx -web_port: 8000 # Configurable port +docker_network: paperless_net + +postgres_container: postgres +caddy_container: caddy +redis_container: valkey + +host: paperless.tanshu.com + +caddyfile_path: /var/lib/caddy/conf/Caddyfile +snippet_block: | + {{ host }} { + request_body { + max_size 128MB + } + + reverse_proxy {{ container_name }}:8000 + + header { + Strict-Transport-Security "max-age=15552000; includeSubDomains; preload" + } + } host_directory: paperless -http_host: paperless.tanshu.com # Replace with your domain -http_conf: "{{ http_host }}.conf" paperless_time_zone: Etc/UTC postgres_user: "postgres" postgres_password: "123456" postgres_db: "paperless" +# Storage: nothing bulky lives on the VPS disk. rclone mounts (see the storage role): +# consume/export -> Nextcloud WebDAV (Nextcloud itself is backed by S3) +# media -> S3 bucket directly +paperless_uid: 1000 # uid/gid paperless runs as; the mounts are presented as this user +rclone_cache_dir: /var/cache/rclone +nextcloud_url: https://cloud.hopsngrains.com +nextcloud_user: admin +nextcloud_app_password: "xxxx" # Nextcloud > Settings > Security > App password +nextcloud_consume_path: Paperless/Consume +nextcloud_export_path: Paperless/Export -# db_url: postgresql://postgres:123456@db:5432/paperless +s3_bucket: "xxxx" # use a bucket separate from Nextcloud's +s3_region: ap-south-1 +s3_access_key: "xxxx" +s3_secret_key: "xxxx" +s3_media_prefix: media +rclone_mounts: + - { name: consume, remote: "nextcloud:{{ nextcloud_consume_path }}", cache: "off" } + - { name: export, remote: "nextcloud:{{ nextcloud_export_path }}", cache: "writes" } + - { name: media, remote: "s3:{{ s3_bucket }}/{{ s3_media_prefix }}", cache: "full" }