diff --git a/ansible/README.md b/ansible/README.md index c90c926..4915e1c 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -52,6 +52,22 @@ central list: Removing a service is the reverse: delete the drop-in, delete the Caddyfile block, `systemctl --user daemon-reload`, and the network is no longer Caddy's. +`mozimo-network.service` is a `oneshot` with `RemainAfterExit=yes`, so systemd +keeps reporting it as `active (exited)` even if the network underneath it was +removed out of band (`podman network rm`, `podman network prune`, a wiped +`/run/user/`); restarting Caddy then starts it against a network that is +not there, and Caddy's own restart loop takes *every* site on the host down +with it. The playbook therefore asks podman (`podman network exists`) rather +than systemd, and recreates the network before Caddy is restarted. To recover +by hand: + +```bash +ssh emily +systemctl --user restart mozimo-network.service # recreate mozimo_net +systemctl --user reset-failed caddy.service +systemctl --user start caddy.service +``` + ## Migrating monoco → emily monoco is the current host: rootful Docker, data in `/var/lib/mozimo`, its own diff --git a/ansible/roles/network/tasks/main.yaml b/ansible/roles/network/tasks/main.yaml index 7d5f7c7..f563bc1 100644 --- a/ansible/roles/network/tasks/main.yaml +++ b/ansible/roles/network/tasks/main.yaml @@ -13,6 +13,44 @@ register: mozimo_network_unit notify: Restart Caddy +# The generator only turns the .network file into a service on a reload, and +# the recreate below has to be able to start that service on a first deploy. +- name: Generate the user service from the network Quadlet + ansible.builtin.systemd_service: + daemon_reload: true + scope: user + +# systemd reports a finished oneshot as "active (exited)" for as long as its +# unit file exists, so the unit's state says nothing about the network +# underneath it: a manual `podman network rm`, a prune, or a wiped runtime +# directory all leave mozimo-network.service looking healthy. Ask podman. +# +# This matters because Caddy *requires* this network. Starting Caddy against a +# network that is not there kills it, and its restart loop then takes every +# other site on this host down with it — so a missing network has to be +# repaired here, before the handler below is flushed. +- name: Check whether the mozimo network still exists + ansible.builtin.command: "podman network exists {{ network_name }}" + register: mozimo_network_check + changed_when: false + failed_when: false + check_mode: false + +- name: Recreate the mozimo network if it went missing + ansible.builtin.systemd_service: + name: "{{ network_quadlet }}-network.service" + state: restarted + scope: user + when: mozimo_network_check.rc != 0 and not ansible_check_mode + +# Fail here, with the network named, rather than one task later in the Caddy +# restart, where the same failure shows up as the host's other sites going down. +- name: Confirm the mozimo network is there before Caddy restarts onto it + ansible.builtin.command: "podman network exists {{ network_name }}" + changed_when: false + check_mode: false + when: not ansible_check_mode + - name: Register the network with Caddy ansible.builtin.template: src: caddy-network.conf.j2 @@ -24,6 +62,12 @@ # Caddy has to be on the network before the site block starts proxying to it, # otherwise the first requests after a reload are 502s. Restarting here also # flushes before the mozimo unit comes up. +# +# A recreated network keeps its name but not its identity, so Caddy has to be +# restarted onto it even when neither template changed. - name: Restart Caddy now if its network membership changed ansible.builtin.meta: flush_handlers - when: mozimo_network_unit.changed or mozimo_caddy_dropin.changed + when: >- + mozimo_network_unit.changed + or mozimo_caddy_dropin.changed + or mozimo_network_check.rc != 0