From 17c64cf2f82930f93214861dae9386d184ad1ae8 Mon Sep 17 00:00:00 2001 From: Amritanshu Date: Fri, 16 Oct 2026 16:15:38 +0530 Subject: [PATCH] Fix the Vaultwarden readiness probe The first deploy failed its wait. The app was up -- Rocket had launched on port 80 and systemd reported the unit active -- but `podman healthcheck run` kept returning non-zero. Podman's healthcheck plumbing is unreliable for Quadlet-managed rootless containers: the check is registered against the *user* systemd manager, and when that fails the check never runs, leaving `podman healthcheck run` to report that the container has no healthcheck at all even though the image declares one (upstream issues #26066, #25034, #22804). Exec the image's own /healthcheck.sh instead: the identical probe (curl http://localhost:80/alive) with no plumbing in between, and still nothing the play has to invent. The cost is that the path is hardcoded, which the image's Dockerfile pins, so a moved script fails as a plain 127 rather than hanging. The wait is now a block with a rescue that prints the probe's own stdout/stderr, the container's health status and the last 20 log lines. The bare "non-zero return code" the plain task reported is exactly the message that hides the reason. The readme's verification commands and a new gotcha record why the probe is exec'd, so nobody "fixes" it back to `podman healthcheck run`. --- bitwarden/readme.md | 15 +++++- bitwarden/roles/bitwarden/tasks/main.yaml | 60 ++++++++++++++++++----- 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/bitwarden/readme.md b/bitwarden/readme.md index f37e4fe..b39713d 100644 --- a/bitwarden/readme.md +++ b/bitwarden/readme.md @@ -292,7 +292,7 @@ journalctl --user -u bitwarden -f podman logs bitwarden # What the playbook checks for you: -podman healthcheck run bitwarden # the image's own probe +podman exec bitwarden /healthcheck.sh # the image's own probe podman exec bitwarden getent hosts postgres podman exec bitwarden getent hosts db @@ -325,6 +325,19 @@ Then reboot the host and confirm the unit returns unattended — without ## Notes and gotchas +- **The readiness probe is exec'd, not `podman healthcheck run`.** Podman's + healthcheck plumbing is unreliable for Quadlet-managed rootless containers: the + check is registered against the *user* systemd manager, and when that fails the + check never runs and `podman healthcheck run` reports that the container has no + healthcheck at all, even though the image declares one + ([podman#26066](https://github.com/containers/podman/issues/26066), + [podman#22804](https://github.com/containers/podman/issues/22804)). That is what + the first deploy here hit — the app was serving and the probe wrapper was the + only thing failing. The play therefore executes the image's own + `/healthcheck.sh` directly, which is the same probe with no plumbing in + between. One cosmetic consequence: `podman inspect bitwarden --format + '{{.State.Health.Status}}'` can stay `starting` forever, which is not a sign + that the app is unhealthy. - **The stale SQLite file.** If `~/podman-data/bitwarden/data/db.sqlite3` exists, the play prints a warning on every run rather than failing, because the file is inert while `DATABASE_URL` is set. It is not a backup. Delete it once the diff --git a/bitwarden/roles/bitwarden/tasks/main.yaml b/bitwarden/roles/bitwarden/tasks/main.yaml index 070e95a..a013f8c 100644 --- a/bitwarden/roles/bitwarden/tasks/main.yaml +++ b/bitwarden/roles/bitwarden/tasks/main.yaml @@ -85,17 +85,55 @@ # answering, and `Restart=always` means even a crash-looping container is # reported active. # -# `podman healthcheck run` executes the image's *own* HEALTHCHECK — which is -# /healthcheck.sh curling http://localhost:80/alive — so this depends on nothing -# the play has to know about: no curl, wget or python3 inside the image, and no -# hardcoded path that a later image tag could move. -- name: Wait for Vaultwarden to pass its own health check - ansible.builtin.command: "podman healthcheck run {{ container_name }}" - register: bitwarden_health - until: bitwarden_health.rc == 0 - retries: 24 - delay: 5 - changed_when: false +# The probe is the image's own /healthcheck.sh — the script its HEALTHCHECK runs, +# curling http://localhost:80/alive — but it is exec'd directly rather than via +# `podman healthcheck run`. Podman's healthcheck plumbing is unreliable for +# Quadlet-managed rootless containers: the check is registered against the *user* +# systemd manager, and when that fails the check never runs, and `podman +# healthcheck run` reports that the container has no healthcheck at all even +# though the image declares one (upstream issues #26066, #25034, #22804). That is +# exactly what happened on the first deploy here: the app was serving, and the +# probe wrapper was the only thing failing. +# +# Exec'ing the script is the identical probe with no plumbing in between, and it +# still depends on nothing the play has to invent — no curl/wget/python3 of its +# own, just the script the image already ships. The cost is that the path is +# hardcoded; the image's Dockerfile pins it (`COPY docker/healthcheck.sh +# docker/start.sh /`), and if a future tag moves it the failure is a plain 127, +# whose stderr the rescue below prints. +- name: Wait for Vaultwarden to answer its own health check + block: + - name: Wait for Vaultwarden to answer its own health check + ansible.builtin.command: "podman exec {{ container_name }} /healthcheck.sh" + register: bitwarden_health + until: bitwarden_health.rc == 0 + retries: 24 + delay: 5 + changed_when: false + rescue: + # A bare "non-zero return code" hides the one line that matters — curl's + # reason, or "no such file or directory" for a moved script — so pull the + # probe's own output and what the app is saying. + - name: Collect readiness diagnostics + ansible.builtin.command: "{{ item }}" + loop: + - "podman exec {{ container_name }} curl -sS http://localhost:80/alive" + - "podman inspect {{ container_name }} --format '{% raw %}{{ .State.Health.Status }}{% endraw %}'" + - "podman logs --tail 20 {{ container_name }}" + register: bitwarden_readiness_diag + changed_when: false + failed_when: false + + - name: Show readiness diagnostics + ansible.builtin.debug: + msg: + - "probe stdout: {{ bitwarden_health.stdout | default('(none)') }}" + - "probe stderr: {{ bitwarden_health.stderr | default('(none)') }}" + - "{{ bitwarden_readiness_diag.results | map(attribute='stdout') | list }}" + + - name: Fail + ansible.builtin.fail: + msg: "Vaultwarden is running but did not answer /alive — see the diagnostics above." # The database is the whole vault, and it arrives over the network that # roles/postgres attached Postgres to. Check the name the environment file