Fix phantom verify failure: exclude secrets/ from stack detection + errexit-proof verify's stack-ps

Root cause of PR #16's pipeline failure (#398-#401): secrets/ is a
tooling folder (secrets-map.yaml, *.secrets.example), not a stack, but
folder-detection (cut -d/ -f1, excluding only deploy/) computed
ALL_STACKS="ai secrets" the first time a commit touched secrets/. The
deploy step survived only because 'secrets' coincidentally sits in the
bootstrap-tier skip list; verify had no guard — `docker stack ps
secrets` exited non-zero and, under errexit, the OUTPUT=$(...)
assignment killed the step before the designed WARNING path could run.
The ai deploy itself was fully successful (complete 21-key render, all
6 services healthy) — the failure was pure phantom. Hazard first
flagged July 2026 in PR #3 (closed unmerged).

Changes:
- grep -v '^deploy$'  ->  grep -vE '^(deploy|secrets)$' in ALL five
  folder-detection sites (validate, provision-secrets x2, deploy,
  verify)
- verify: `|| true` inside the stack-ps command substitution so a
  genuinely missing stack warns instead of aborting (comment added)
- deploy step: also rsync secrets/ tooling to the host mirror
  unconditionally, same treatment as deploy/
- header: 2026-09-08 incident note

Everything else byte-identical to main. Dollar-escape audit done.

.gitignore

@@ -2,8 +2,17 @@
 *.swp
 *.swo
 .DS_Store
+
+# Secrets / stack env files - never commit these.
+# deploy/global.env and immich.env-example etc. are the intended committed
+# templates/examples; actual secret-bearing .env files must stay untracked.
 .env
 .env.local
+*.env
+!*.env-example
+!deploy/global.env
+*.env.bak
+*.env.backup
 secrets/
 keys/
 *.key
@@ -15,3 +24,7 @@ keys/
 # Local overrides
 docker-compose.override.yml
 local/
+
+# Editor/adhoc backups that sometimes get left in the tree
+*.bak
+*.bak-*

.woodpecker.yml

@@ -1,300 +0,0 @@
-when:
-  - event: push
-    branch: main
-  - event: manual
-    branch: main
-
-steps:
-  validate:
-    image: alpine:latest
-    commands:
-      - apk add --no-cache docker-cli docker-cli-compose
-      - |
-        # Collect changed stacks — both flat files and folder-based
-        CHANGED_FILES=$(echo "${CI_COMMIT_CHANGED_FILES}" | tr ',' '\n')
-
-        # Flat: any root-level *.yaml
-        FLAT=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.ya?ml$' || true)
-
-        # Folder: any file under a subfolder (e.g. immich/immich.yml)
-        FOLDERS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+/' | cut -d/ -f1 | sort -u || true)
-
-        [ -z "$FLAT" ] && [ -z "$FOLDERS" ] && echo "No stacks changed" && exit 0
-
-        for f in $FLAT; do
-          [ -f "$f" ] || continue
-          docker compose -f "$f" --no-interpolate config -q \
-            && echo "  OK $f" || { echo "  FAIL $f"; exit 1; }
-        done
-
-        for d in $FOLDERS; do
-          [ -d "$d" ] || continue
-          # Build -f flags for all compose files in folder, main first
-          MAIN=""
-          EXTRAS=""
-          for ext in yml yaml; do
-            [ -f "$d/$d.$ext" ] && MAIN="$d/$d.$ext" && break
-          done
-          [ -z "$MAIN" ] && echo "  SKIP $d (no main compose file)" && continue
-          for cf in $(find "$d" -maxdepth 1 \( -name '*.yml' -o -name '*.yaml' \) | sort); do
-            [ "$cf" = "$MAIN" ] && continue
-            EXTRAS="$EXTRAS -f $cf"
-          done
-          docker compose -f "$MAIN" $EXTRAS --no-interpolate config -q \
-            && echo "  OK $d/" || { echo "  FAIL $d/"; exit 1; }
-        done
-
-  provision-secrets:
-    image: alpine:latest
-    environment:
-      SSH_KEY:
-        from_secret: ssh_key
-      SWARM_MANAGER_IP:
-        from_secret: swarm_manager_ip
-      3DPRINT_DB_PASSWORD:
-        from_secret: 3dprint_db_password
-      GAMMA_AUTH_TOKEN:
-        from_secret: gamma_auth_token
-      GIT_DB_PASSWORD:
-        from_secret: git_db_password
-      GIT_RUNNER_TOKEN:
-        from_secret: git_runner_token
-      GIT_MCP_ACCESS_TOKEN:
-        from_secret: git_mcp_access_token
-      HOMEASSISTANT_FRIGATE_RTSP_PASSWORD:
-        from_secret: homeassistant_frigate_rtsp_password
-      HOMEASSISTANT_IMMICH_API_KEY:
-        from_secret: homeassistant_immich_api_key
-      IMMICH_DB_PASSWORD:
-        from_secret: immich_db_password
-      IMMICH_KIOSK_BASICAUTH:
-        from_secret: immich_kiosk_basicauth
-      MEALIE_DB_PASSWORD:
-        from_secret: mealie_db_password
-      MEALIE_LDAP_QUERY_PASSWORD:
-        from_secret: mealie_ldap_query_password
-      MESHCENTRAL_BACKUP_PASSWORD:
-        from_secret: meshcentral_backup_password
-      N8N_DB_PASSWORD:
-        from_secret: n8n_db_password
-      N8N_ENCRYPTION_KEY:
-        from_secret: n8n_encryption_key
-      POSTGRESQL_PASSWORD:
-        from_secret: postgresql_password
-      POSTGRESQL_PGADMIN_PASSWORD:
-        from_secret: postgresql_pgadmin_password
-      PRODUCTIVITY_PAPERLESS_SECRET_KEY:
-        from_secret: productivity_paperless_secret_key
-      PRODUCTIVITY_DB_PASSWORD:
-        from_secret: productivity_db_password
-      PRODUCTIVITY_OIDC_PROVIDERS:
-        from_secret: productivity_oidc_providers
-      VAULTWARDEN_ADMIN_TOKEN:
-        from_secret: vaultwarden_admin_token
-      VAULTWARDEN_DATABASE_URL:
-        from_secret: vaultwarden_database_url
-      ENTERTAINMENT_DISCORD_TOKEN:
-        from_secret: entertainment_discord_token
-      ENTERTAINMENT_DISCORD_CLIENT_SECRET:
-        from_secret: entertainment_discord_client_secret
-      ENTERTAINMENT_SECRET_KEY_BASE:
-        from_secret: entertainment_secret_key_base
-      ENTERTAINMENT_BASIC_AUTH_PASSWORD:
-        from_secret: entertainment_basic_auth_password
-      ENTERTAINMENT_SPARKY_DB_PASSWORD:
-        from_secret: entertainment_sparky_db_password
-      ENTERTAINMENT_SPARKY_APP_DB_PASSWORD:
-        from_secret: entertainment_sparky_app_db_password
-      ENTERTAINMENT_SPARKY_ENCRYPTION_KEY:
-        from_secret: entertainment_sparky_encryption_key
-      ENTERTAINMENT_BETTER_AUTH_SECRET:
-        from_secret: entertainment_better_auth_secret
-    commands:
-      - apk add --no-cache openssh-client
-      - mkdir -p ~/.ssh
-      - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa
-      - chmod 600 ~/.ssh/id_rsa
-      - ssh-keyscan -H ${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
-      - |
-        CHANGED_FILES=$(echo "${CI_COMMIT_CHANGED_FILES}" | tr ',' '\n')
-        FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
-        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+/' | cut -d/ -f1 | sort -u || true)
-        ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
-        [ -z "$ALL_STACKS" ] && echo "No stacks changed, skipping" && exit 0
-      - scp -o StrictHostKeyChecking=no deploy/create-secrets.sh root@${SWARM_MANAGER_IP}:/tmp/cs.sh
-      - |
-        CHANGED_FILES=$(echo "${CI_COMMIT_CHANGED_FILES}" | tr ',' '\n')
-        FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
-        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+/' | cut -d/ -f1 | sort -u || true)
-        ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
-
-        for STACK in $ALL_STACKS; do
-          echo "Provisioning: $STACK"
-          case "$STACK" in
-            maintenance|media|unifi|guacamole|security|auth|traefik|meshcentral|ddm)
-              echo "  No Docker secrets for $STACK — secrets in host .env";;
-            immich)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'immich_db_password'     '${IMMICH_DB_PASSWORD}'
-                create_or_update_secret 'immich_kiosk_basicauth' '${IMMICH_KIOSK_BASICAUTH}'";;
-            woodpecker)
-              echo "  Manual only — skipping";;
-            3dprint)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret '3dprint_db_password' '${3DPRINT_DB_PASSWORD}'";;
-            gamma)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'gamma_auth_token' '${GAMMA_AUTH_TOKEN}'";;
-            git)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'git_db_password'       '${GIT_DB_PASSWORD}'
-                create_or_update_secret 'git_runner_token'      '${GIT_RUNNER_TOKEN}'
-                create_or_update_secret 'git_mcp_access_token'  '${GIT_MCP_ACCESS_TOKEN}'";;
-            homeassistant)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'homeassistant_frigate_rtsp_password' '${HOMEASSISTANT_FRIGATE_RTSP_PASSWORD}'
-                create_or_update_secret 'homeassistant_immich_api_key'        '${HOMEASSISTANT_IMMICH_API_KEY}'";;
-            mealie)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'mealie_db_password'         '${MEALIE_DB_PASSWORD}'
-                create_or_update_secret 'mealie_ldap_query_password' '${MEALIE_LDAP_QUERY_PASSWORD}'";;
-            n8n)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'n8n_db_password'     '${N8N_DB_PASSWORD}'
-                create_or_update_secret 'n8n_encryption_key'  '${N8N_ENCRYPTION_KEY}'";;
-            postgresql)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'postgresql_password'         '${POSTGRESQL_PASSWORD}'
-                create_or_update_secret 'postgresql_pgadmin_password' '${POSTGRESQL_PGADMIN_PASSWORD}'";;
-            productivity)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'productivity_paperless_secret_key' '${PRODUCTIVITY_PAPERLESS_SECRET_KEY}'
-                create_or_update_secret 'productivity_db_password'          '${PRODUCTIVITY_DB_PASSWORD}'
-                create_or_update_secret 'productivity_oidc_providers'       '${PRODUCTIVITY_OIDC_PROVIDERS}'";;
-            vaultwarden)
-              printf '%s' "${VAULTWARDEN_ADMIN_TOKEN}"  | ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "docker secret rm vaultwarden_admin_token  2>/dev/null; docker secret create vaultwarden_admin_token -"
-              printf '%s' "${VAULTWARDEN_DATABASE_URL}" | ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "docker secret rm vaultwarden_database_url 2>/dev/null; docker secret create vaultwarden_database_url -";;
-            ai)
-              echo "  No Docker secrets for ai -- secrets in host .env";;
-            entertainment)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'entertainment_discord_token'          '${ENTERTAINMENT_DISCORD_TOKEN}'
-                create_or_update_secret 'entertainment_discord_client_secret'  '${ENTERTAINMENT_DISCORD_CLIENT_SECRET}'
-                create_or_update_secret 'entertainment_secret_key_base'        '${ENTERTAINMENT_SECRET_KEY_BASE}'
-                create_or_update_secret 'entertainment_basic_auth_password'    '${ENTERTAINMENT_BASIC_AUTH_PASSWORD}'
-                create_or_update_secret 'entertainment_sparky_db_password'     '${ENTERTAINMENT_SPARKY_DB_PASSWORD}'
-                create_or_update_secret 'entertainment_sparky_app_db_password' '${ENTERTAINMENT_SPARKY_APP_DB_PASSWORD}'
-                create_or_update_secret 'entertainment_sparky_encryption_key'  '${ENTERTAINMENT_SPARKY_ENCRYPTION_KEY}'
-                create_or_update_secret 'entertainment_better_auth_secret'     '${ENTERTAINMENT_BETTER_AUTH_SECRET}'";;
-            *)
-              echo "  No secrets case for $STACK";;
-          esac
-        done
-
-  deploy:
-    image: alpine:latest
-    environment:
-      SSH_KEY:
-        from_secret: ssh_key
-      SWARM_MANAGER_IP:
-        from_secret: swarm_manager_ip
-    commands:
-      - apk add --no-cache openssh-client rsync
-      - mkdir -p ~/.ssh
-      - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa
-      - chmod 600 ~/.ssh/id_rsa
-      - ssh-keyscan -H ${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
-      - |
-        CHANGED_FILES=$(echo "${CI_COMMIT_CHANGED_FILES}" | tr ',' '\n')
-        FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
-        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+/' | cut -d/ -f1 | sort -u || true)
-        ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
-        [ -z "$ALL_STACKS" ] && echo "No stacks changed" && exit 0
-
-        # Always sync deploy/ scripts first
-        rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
-          deploy/ root@${SWARM_MANAGER_IP}:/volume1/docker/compose-files/deploy/
-
-        for STACK in $ALL_STACKS; do
-          echo "--- Deploying: $STACK ---"
-          # Sync flat file or entire folder
-          if [ -d "$STACK" ]; then
-            rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
-              "$STACK/" root@${SWARM_MANAGER_IP}:/volume1/docker/compose-files/$STACK/
-          elif [ -f "${STACK}.yaml" ]; then
-            rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
-              "${STACK}.yaml" root@${SWARM_MANAGER_IP}:/volume1/docker/compose-files/
-          fi
-          ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} \
-            "bash /volume1/docker/compose-files/deploy/stack-deploy.sh $STACK" \
-            && echo "  OK $STACK" || { echo "  FAIL $STACK"; exit 1; }
-        done
-
-  verify:
-    image: alpine:latest
-    environment:
-      SSH_KEY:
-        from_secret: ssh_key
-      SWARM_MANAGER_IP:
-        from_secret: swarm_manager_ip
-    commands:
-      - apk add --no-cache openssh-client
-      - mkdir -p ~/.ssh
-      - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa
-      - chmod 600 ~/.ssh/id_rsa
-      - ssh-keyscan -H ${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
-      - |
-        CHANGED_FILES=$(echo "${CI_COMMIT_CHANGED_FILES}" | tr ',' '\n')
-        FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
-        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+/' | cut -d/ -f1 | sort -u || true)
-        ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
-        [ -z "$ALL_STACKS" ] && exit 0
-
-        # Give Swarm time to schedule tasks after deploy
-        echo "Waiting 15s for Swarm to schedule tasks..."
-        sleep 15
-
-        FAILED=0
-        for STACK in $ALL_STACKS; do
-          echo "--- $STACK ---"
-          RUNNING=$(ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} \
-            "docker stack ps $STACK --filter desired-state=running \
-            --format '  {{.Name}} {{.CurrentState}}'" 2>&1)
-          echo "$RUNNING"
-          COUNT=$(echo "$RUNNING" | grep -c 'Running\|Starting\|Preparing' || true)
-          if [ "$COUNT" -eq 0 ]; then
-            echo "  ERROR: No running tasks found for stack '$STACK' — deploy may have failed!"
-            FAILED=1
-          fi
-        done
-
-        [ "$FAILED" -eq 0 ] || exit 1
-
-  notify-success:
-    image: alpine:latest
-    environment:
-      TEAMS_WEBHOOK:
-        from_secret: teams_webhook
-    commands:
-      - apk add --no-cache curl
-      - |
-        CHANGED=$(echo "${CI_COMMIT_CHANGED_FILES}" | tr ',' ' ')
-        PAYLOAD=$(printf '{"title":"Swarm Deploy OK","text":"Stacks: %s Commit: %s","themeColor":"00aa00"}' "$CHANGED" "$CI_COMMIT_MESSAGE")
-        curl -sf -X POST -H 'Content-Type: application/json' -d "$PAYLOAD" "$TEAMS_WEBHOOK" || true
-    when:
-      - status: success
-
-  notify-failure:
-    image: alpine:latest
-    environment:
-      TEAMS_WEBHOOK:
-        from_secret: teams_webhook
-      WOODPECKER_HOST:
-        from_secret: woodpecker_host
-    commands:
-      - apk add --no-cache curl
-      - |
-        PAYLOAD=$(printf '{"title":"Swarm Deploy FAILED","text":"Commit: %s See: https://%s","themeColor":"ff0000"}' "$CI_COMMIT_MESSAGE" "$WOODPECKER_HOST")
-        curl -sf -X POST -H 'Content-Type: application/json' -d "$PAYLOAD" "$TEAMS_WEBHOOK" || true
-    when:
-      - status: failure

.woodpecker/deploy.yml

@@ -0,0 +1,501 @@
+when:
+  - event: push
+    branch: main
+
+# ─────────────────────────────────────────────────────────────────────────────
+# IMPORTANT — Woodpecker variable substitution rules (learned the hard way):
+#
+#   Woodpecker pre-processes the ENTIRE yaml text (comments included!) before
+#   the shell runs anything:
+#     - A dollar sign followed by a braced variable name is substituted at
+#       pipeline-compile time from Woodpecker's own metadata (the CI_* vars).
+#       Secrets DO NOT exist in that map, so a braced reference to a
+#       secret-backed env var silently becomes an EMPTY STRING.
+#     - A double dollar sign is unescaped to a single dollar sign and passed
+#       through to the shell untouched.
+#
+#   Therefore:
+#     - Braced, single-dollar form: ONLY for Woodpecker CI_* metadata vars.
+#     - Double-dollar braced form: for everything that must be resolved by the
+#       shell at runtime (i.e., every from_secret-backed environment variable).
+#     - Bare single-dollar VAR (no braces) also passes through to the shell.
+#     - NEVER write a literal dollar-brace sequence in comments either — the
+#       substitution engine parses comments too and will fail the pipeline
+#       with "missing closing brace" on anything it cannot parse.
+#
+#   This was the root cause of a long-running "SWARM_MANAGER_IP secret is
+#   empty" failure: braced references were blanked at compile time before the
+#   shell ever saw them.
+#
+#   2026-08-26 HOTFIX: a full-file rewrite (AI secrets migration PR) dropped
+#   one $ from every $${VAR} occurrence throughout this file, re-introducing
+#   exactly the bug described above for EVERY secret-backed var, not just the
+#   new AI ones. The if-empty guards caught it immediately (SWARM_MANAGER_IP
+#   came back blank) and aborted before any ssh/scp/rsync ran, so no live
+#   secret or service was touched — but no CI provisioning/deploy could run
+#   until this was restored. Lesson: grep for the literal string '$${' and
+#   diff the count against the previous version before ever committing a
+#   full-file rewrite of this pipeline.
+#
+#   2026-08-26 FIX: FOLDER_STACKS/FOLDERS detection (grep -E '^[^/.][^/]*/')
+#   matches ANY non-dot top-level folder in the changed-files list, including
+#   deploy/ — the shared tooling folder, not a stack. A PR touching only
+#   deploy/envparse.py caused stack-deploy.sh to be invoked with "deploy" as
+#   a stack name, which correctly errored ("No main compose file... in
+#   .../deploy") since deploy/ has no deploy.yaml. No live service was
+#   affected (the error occurs before any redeploy), but it produced a
+#   confusing pipeline failure on an otherwise-correct change. deploy/ is
+#   already unconditionally rsynced at the top of the deploy step regardless
+#   of which stacks changed, so it's safe to exclude it from the stack list
+#   everywhere folders are detected below.
+#
+#   2026-09-03/07 INCIDENT + REDESIGN: hand-maintained grep -v + printf
+#   line-surgery cases repeatedly drifted (missing '=', duplicated keys,
+#   mismatched env var names rendering EMPTY secrets) and broke live
+#   services. Root cause: no authoritative key list and shell heredocs
+#   hostile to hand-editing. Stacks migrate one at a time to a data-driven
+#   model: secrets/secrets-map.yaml (data only) + per-stack .env.template
+#   (authoritative FULL file) + deploy/provision-stack.py (whole-file
+#   render, hard failure naming any missing value). Migrated stacks call
+#   the script; unmigrated stacks keep legacy case entries until their own
+#   PR. See the "Secrets & Deployment Architecture — Global Direction" note.
+#
+#   2026-09-08 FIX: secrets/ is a tooling/docs folder (secrets-map.yaml +
+#   *.secrets.example), not a stack — but folder-detection treated it as one
+#   the first time a commit touched it (PR #16). deploy survived only because
+#   'secrets' sits in the bootstrap-tier skip list; verify had no guard and
+#   died on `docker stack ps secrets` failing under errexit (assignment from
+#   a failing command substitution aborts the step). Fixed by excluding
+#   secrets/ alongside deploy/ in ALL folder-detection sites, and by
+#   tolerating a failing stack-ps in verify (|| true) so a genuinely missing
+#   stack produces the designed WARNING instead of killing the step. This
+#   hazard was first flagged in July (PR #3, closed unmerged).
+# ─────────────────────────────────────────────────────────────────────────────
+
+steps:
+  validate:
+    image: alpine:latest
+    commands:
+      - apk add --no-cache docker-cli docker-cli-compose
+      - |
+        # Collect changed stacks — both flat files and folder-based
+        # NOTE: Woodpecker 3.16 exposes changed files as CI_PIPELINE_FILES, a JSON
+        # array string. CI_* metadata vars are correctly substituted at compile time.
+        CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
+
+        # Flat: any root-level *.yaml
+        FLAT=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.ya?ml$' || true)
+
+        # Folder: any file under a subfolder (e.g. immich/immich.yml).
+        # Exclude dotfolders (.woodpecker, .git, .gitea, etc.) and the
+        # non-stack tooling folders deploy/ and secrets/ (see notes above).
+        FOLDERS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -vE '^(deploy|secrets)$' | sort -u || true)
+
+        [ -z "$FLAT" ] && [ -z "$FOLDERS" ] && echo "No stacks changed" && exit 0
+
+        for f in $FLAT; do
+          [ -f "$f" ] || continue
+          docker compose -f "$f" config --no-interpolate -q \
+            && echo "  OK $f" || { echo "  FAIL $f"; exit 1; }
+        done
+
+        for d in $FOLDERS; do
+          [ -d "$d" ] || continue
+          # Build -f flags for all compose files in folder, main first
+          MAIN=""
+          EXTRAS=""
+          for ext in yml yaml; do
+            [ -f "$d/$d.$ext" ] && MAIN="$d/$d.$ext" && break
+          done
+          [ -z "$MAIN" ] && echo "  SKIP $d (no main compose file)" && continue
+          for cf in $(find "$d" -maxdepth 1 \( -name '*.yml' -o -name '*.yaml' \) | sort); do
+            [ "$cf" = "$MAIN" ] && continue
+            EXTRAS="$EXTRAS -f $cf"
+          done
+          docker compose -f "$MAIN" $EXTRAS config --no-interpolate -q \
+            && echo "  OK $d/" || { echo "  FAIL $d/"; exit 1; }
+        done
+
+  provision-secrets:
+    image: alpine:latest
+    environment:
+      SSH_KEY:
+        from_secret: ssh_key
+      SWARM_MANAGER_IP:
+        from_secret: swarm_manager_ip
+      PRINT3D_DB_PASSWORD:
+        from_secret: 3dprint_db_password
+      GAMMA_AUTH_TOKEN:
+        from_secret: gamma_auth_token
+      GIT_DB_PASSWORD:
+        from_secret: git_db_password
+      GIT_RUNNER_TOKEN:
+        from_secret: git_runner_token
+      GIT_MCP_ACCESS_TOKEN:
+        from_secret: git_mcp_access_token
+      HOMEASSISTANT_FRIGATE_RTSP_PASSWORD:
+        from_secret: homeassistant_frigate_rtsp_password
+      HOMEASSISTANT_IMMICH_API_KEY:
+        from_secret: homeassistant_immich_api_key
+      IMMICH_DB_PASSWORD:
+        from_secret: immich_db_password
+      IMMICH_KIOSK_BASICAUTH:
+        from_secret: immich_kiosk_basicauth
+      MEALIE_DB_PASSWORD:
+        from_secret: mealie_db_password
+      MEALIE_LDAP_QUERY_PASSWORD:
+        from_secret: mealie_ldap_query_password
+      MESHCENTRAL_BACKUP_PASSWORD:
+        from_secret: meshcentral_backup_password
+      N8N_DB_PASSWORD:
+        from_secret: n8n_db_password
+      N8N_ENCRYPTION_KEY:
+        from_secret: n8n_encryption_key
+      POSTGRESQL_PASSWORD:
+        from_secret: postgresql_password
+      POSTGRESQL_PGADMIN_PASSWORD:
+        from_secret: postgresql_pgadmin_password
+      POSTGRESQL_REPLICATION_PASSWORD:
+        from_secret: postgresql_replication_password
+      POSTGRESQL_PATRONI_PASSWORD:
+        from_secret: postgresql_patroni_password
+      PRODUCTIVITY_PAPERLESS_SECRET_KEY:
+        from_secret: productivity_paperless_secret_key
+      PRODUCTIVITY_DB_PASSWORD:
+        from_secret: productivity_db_password
+      PRODUCTIVITY_OIDC_PROVIDERS:
+        from_secret: productivity_oidc_providers
+      VAULTWARDEN_ADMIN_TOKEN:
+        from_secret: vaultwarden_admin_token
+      VAULTWARDEN_DATABASE_URL:
+        from_secret: vaultwarden_database_url
+      ENTERTAINMENT_DISCORD_TOKEN:
+        from_secret: entertainment_discord_token
+      ENTERTAINMENT_DISCORD_CLIENT_SECRET:
+        from_secret: entertainment_discord_client_secret
+      ENTERTAINMENT_SECRET_KEY_BASE:
+        from_secret: entertainment_secret_key_base
+      ENTERTAINMENT_BASIC_AUTH_PASSWORD:
+        from_secret: entertainment_basic_auth_password
+      ENTERTAINMENT_SPARKY_DB_PASSWORD:
+        from_secret: entertainment_sparky_db_password
+      ENTERTAINMENT_SPARKY_APP_DB_PASSWORD:
+        from_secret: entertainment_sparky_app_db_password
+      ENTERTAINMENT_SPARKY_ENCRYPTION_KEY:
+        from_secret: entertainment_sparky_encryption_key
+      ENTERTAINMENT_BETTER_AUTH_SECRET:
+        from_secret: entertainment_better_auth_secret
+      # ── ai stack (manifest-driven — secrets/secrets-map.yaml +
+      #    ai/ai.env.template + deploy/provision-stack.py). Env var names
+      #    below match the template placeholders EXACTLY; this block is the
+      #    only per-secret touchpoint left in this file for migrated stacks
+      #    (Woodpecker v3 requires explicit from_secret declarations). ──
+      AI_AWS_ACCESS_KEY_ID:
+        from_secret: ai_aws_access_key_id
+      AI_AWS_SECRET_ACCESS_KEY:
+        from_secret: ai_aws_secret_access_key
+      AI_LITELLM_MASTER_KEY:
+        from_secret: ai_litellm_master_key
+      AI_LITELLM_SALT_KEY:
+        from_secret: ai_litellm_salt_key
+      AI_LITELLM_DATABASE_URL:
+        from_secret: ai_litellm_database_url
+      AI_LITELLM_POSTGRES_PASSWORD:
+        from_secret: ai_litellm_postgres_password
+      AI_OPEN_WEBUI_SECRET_KEY:
+        from_secret: ai_webui_secret_key
+      AI_OPEN_WEBUI_DATABASE_URL:
+        from_secret: ai_open_webui_database_url
+      AI_OPEN_WEBUI_OAUTH_CLIENT_SECRET:
+        from_secret: ai_oauth_client_secret
+      AI_MCPO_API_KEY:
+        from_secret: ai_mcpo_api_key
+      FLOWAGENT_AZURE_CLIENT_ID:
+        from_secret: flowagent_azure_client_id
+      FLOWAGENT_AZURE_TENANT_ID:
+        from_secret: flowagent_azure_tenant_id
+      FLOWAGENT_AZURE_CLIENT_SECRET:
+        from_secret: flowagent_azure_client_secret
+    commands:
+      - apk add --no-cache openssh-client python3 py3-yaml
+      - mkdir -p ~/.ssh
+      - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa
+      - chmod 600 ~/.ssh/id_rsa
+      - |
+        if [ -z "$${SWARM_MANAGER_IP}" ]; then
+          echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets."
+          exit 1
+        fi
+      - ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
+      - |
+        CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
+        FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
+        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -vE '^(deploy|secrets)$' | sort -u || true)
+        ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
+        [ -z "$ALL_STACKS" ] && echo "No stacks changed, skipping" && exit 0
+      - scp -o StrictHostKeyChecking=no deploy/create-secrets.sh root@$${SWARM_MANAGER_IP}:/tmp/cs.sh
+      - |
+        CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
+        FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
+        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -vE '^(deploy|secrets)$' | sort -u || true)
+        ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
+
+        for STACK in $ALL_STACKS; do
+          echo "Provisioning: $STACK"
+          case "$STACK" in
+            maintenance|media|unifi|guacamole|security|auth|traefik|meshcentral|ddm)
+              echo "  No Docker secrets for $STACK — secrets in host .env";;
+            immich)
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'immich_db_password'     '$${IMMICH_DB_PASSWORD}'
+                create_or_update_secret 'immich_kiosk_basicauth' '$${IMMICH_KIOSK_BASICAUTH}'";;
+            woodpecker)
+              echo "  Manual only — skipping";;
+            3dprint)
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret '3dprint_db_password' '$${PRINT3D_DB_PASSWORD}'";;
+            gamma)
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'gamma_auth_token' '$${GAMMA_AUTH_TOKEN}'";;
+            git)
+              # PATTERN B, DELIBERATE (see decision notes) — git hosts the source
+              # of truth for every other stack's compose files, so it must be
+              # restorable from a flat git.yaml + git.env backup alone, with zero
+              # dependency on a running Swarm's Docker secret store. Native Docker
+              # secrets (Pattern C) can't satisfy that: they only exist inside an
+              # already-running Swarm, which is exactly the circular dependency
+              # this stack can't have. Mirrors the retired ai) case's grep -v +
+              # printf rewrite-in-place approach, never sed (values may contain
+              # slash, dollar sign, ampersand).
+              #
+              # TEST PHASE: target is git.env.pipelinetest, NOT the real git.env.
+              # The real file is never opened for writing by this step. First run
+              # seeds the test file from the real git.env (carries over all
+              # non-secret lines untouched); every push after that only refreshes
+              # the 3 secret lines below. Also strips the legacy GITEA_ACCESS_TOKEN
+              # key name so the test file converges on the git.env.example-
+              # documented GITEA_MCP_ACCESS_TOKEN key. Cutover to the real file —
+              # and pointing git.yaml/stack-deploy at it — is a deliberate,
+              # separate follow-up after manually diffing this render.
+              # (Candidate for the secrets-map.yaml/provision-stack.py migration
+              # in its own PR; kept legacy for now.)
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "FILE=/volume1/docker/compose-files/git.env.pipelinetest
+                TMP=\$FILE.tmp.\$\$
+                [ -f \$FILE ] || cp /volume1/docker/compose-files/git.env \$FILE
+                grep -vE '^(GITEA__database__PASSWD|GITEA_RUNNER_REGISTRATION_TOKEN|GITEA_MCP_ACCESS_TOKEN|GITEA_ACCESS_TOKEN)=' \$FILE > \$TMP 2>/dev/null || touch \$TMP
+                { cat \$TMP
+                  printf 'GITEA__database__PASSWD=%s\n' '$${GIT_DB_PASSWORD}'
+                  printf 'GITEA_RUNNER_REGISTRATION_TOKEN=%s\n' '$${GIT_RUNNER_TOKEN}'
+                  printf 'GITEA_MCP_ACCESS_TOKEN=%s\n' '$${GIT_MCP_ACCESS_TOKEN}'
+                } > \$FILE
+                rm -f \$TMP
+                echo '  [OK] git.env.pipelinetest updated - real git.env untouched'";;
+            homeassistant)
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'homeassistant_frigate_rtsp_password' '$${HOMEASSISTANT_FRIGATE_RTSP_PASSWORD}'
+                create_or_update_secret 'homeassistant_immich_api_key'        '$${HOMEASSISTANT_IMMICH_API_KEY}'";;
+            mealie)
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'mealie_db_password'         '$${MEALIE_DB_PASSWORD}'
+                create_or_update_secret 'mealie_ldap_query_password' '$${MEALIE_LDAP_QUERY_PASSWORD}'";;
+            n8n)
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'n8n_db_password'     '$${N8N_DB_PASSWORD}'
+                create_or_update_secret 'n8n_encryption_key'  '$${N8N_ENCRYPTION_KEY}'";;
+            postgresql)
+              # NOTE: bootstrap-tier stack (manual deploy only via stack-deploy.sh).
+              # Secrets are still auto-provisioned here so they exist on the host
+              # before the manual `stack-deploy.sh postgresql` run picks them up.
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'postgresql_password'             '$${POSTGRESQL_PASSWORD}'
+                create_or_update_secret 'postgresql_pgadmin_password'     '$${POSTGRESQL_PGADMIN_PASSWORD}'
+                create_or_update_secret 'postgresql_replication_password' '$${POSTGRESQL_REPLICATION_PASSWORD}'
+                create_or_update_secret 'postgresql_patroni_password'     '$${POSTGRESQL_PATRONI_PASSWORD}'";;
+            productivity)
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'productivity_paperless_secret_key' '$${PRODUCTIVITY_PAPERLESS_SECRET_KEY}'
+                create_or_update_secret 'productivity_db_password'          '$${PRODUCTIVITY_DB_PASSWORD}'
+                create_or_update_secret 'productivity_oidc_providers'       '$${PRODUCTIVITY_OIDC_PROVIDERS}'";;
+            vaultwarden)
+              # NOTE: vaultwarden_database_url cannot be rotated in-place — Swarm refuses
+              # to remove a secret referenced by a running service's spec. We provision
+              # under a versioned name instead; vaultwarden.yaml maps it back to the same
+              # in-container filename via target. The old secret is removed manually once
+              # the compose file cutover is confirmed healthy.
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'vaultwarden_admin_token'     '$${VAULTWARDEN_ADMIN_TOKEN}'
+                create_or_update_secret 'vaultwarden_database_url_v2' '$${VAULTWARDEN_DATABASE_URL}'";;
+            ai)
+              # MIGRATED (2026-09-07) to manifest-driven provisioning after the
+              # line-surgery approach repeatedly drifted (dropped keys 2026-09-03;
+              # duplicate AI_LITELLM_POSTGRES_PASSWORD + stray legacy keys +
+              # WEB_UI/WEBUI env-name mismatch rendering an EMPTY OAuth client
+              # secret, found 2026-09-07). All logic lives in
+              # deploy/provision-stack.py; the authoritative key list lives in
+              # ai/ai.env.template; the mapping lives in secrets/secrets-map.yaml.
+              # This case is intentionally one line.
+              python3 deploy/provision-stack.py ai;;
+            entertainment)
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'entertainment_discord_token'          '$${ENTERTAINMENT_DISCORD_TOKEN}'
+                create_or_update_secret 'entertainment_discord_client_secret'  '$${ENTERTAINMENT_DISCORD_CLIENT_SECRET}'
+                create_or_update_secret 'entertainment_secret_key_base'        '$${ENTERTAINMENT_SECRET_KEY_BASE}'
+                create_or_update_secret 'entertainment_basic_auth_password'    '$${ENTERTAINMENT_BASIC_AUTH_PASSWORD}'
+                create_or_update_secret 'entertainment_sparky_db_password'     '$${ENTERTAINMENT_SPARKY_DB_PASSWORD}'
+                create_or_update_secret 'entertainment_sparky_app_db_password' '$${ENTERTAINMENT_SPARKY_APP_DB_PASSWORD}'
+                create_or_update_secret 'entertainment_sparky_encryption_key'  '$${ENTERTAINMENT_SPARKY_ENCRYPTION_KEY}'
+                create_or_update_secret 'entertainment_better_auth_secret'     '$${ENTERTAINMENT_BETTER_AUTH_SECRET}'";;
+            *)
+              echo "  No secrets case for $STACK";;
+          esac
+        done
+
+  deploy:
+    image: alpine:latest
+    environment:
+      SSH_KEY:
+        from_secret: ssh_key
+      SWARM_MANAGER_IP:
+        from_secret: swarm_manager_ip
+    commands:
+      - apk add --no-cache openssh-client rsync
+      - mkdir -p ~/.ssh
+      - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa
+      - chmod 600 ~/.ssh/id_rsa
+      - |
+        if [ -z "$${SWARM_MANAGER_IP}" ]; then
+          echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets."
+          exit 1
+        fi
+      - ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
+      - |
+        CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
+        FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
+        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -vE '^(deploy|secrets)$' | sort -u || true)
+        ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
+        [ -z "$ALL_STACKS" ] && echo "No stacks changed" && exit 0
+
+        # Always sync deploy/ scripts first
+        rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
+          deploy/ root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/deploy/
+
+        # Sync secrets/ tooling (manifest + examples) alongside deploy/ —
+        # provision-stack.py reads secrets/secrets-map.yaml from the CI
+        # checkout, but the host mirror should stay complete for emergency
+        # manual provisioning runs.
+        rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
+          secrets/ root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/secrets/
+
+        for STACK in $ALL_STACKS; do
+          echo "--- Deploying: $STACK ---"
+          # Sync files to host first (always, even for bootstrap stacks)
+          if [ -d "$STACK" ]; then
+            rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
+              "$STACK/" root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/$STACK/
+          elif [ -f "$STACK.yaml" ]; then
+            rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
+              "$STACK.yaml" root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/
+          fi
+
+          # Bootstrap-tier guard: file synced to host, deploy is MANUAL
+          case "$STACK" in
+            traefik|woodpecker|postgresql|secrets|git)
+              echo "  [BOOTSTRAP] $STACK: file synced. Deploy is MANUAL."
+              echo "  Run: bash /volume1/docker/compose-files/deploy/stack-deploy.sh $STACK"
+              continue ;;
+          esac
+
+          # Tier 2: auto-deploy
+          ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} \
+            "bash /volume1/docker/compose-files/deploy/stack-deploy.sh $STACK" \
+            && echo "  OK $STACK" || { echo "  FAIL $STACK"; exit 1; }
+        done
+
+  verify:
+    image: alpine:latest
+    environment:
+      SSH_KEY:
+        from_secret: ssh_key
+      SWARM_MANAGER_IP:
+        from_secret: swarm_manager_ip
+    commands:
+      - apk add --no-cache openssh-client
+      - mkdir -p ~/.ssh
+      - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa
+      - chmod 600 ~/.ssh/id_rsa
+      - |
+        if [ -z "$${SWARM_MANAGER_IP}" ]; then
+          echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets."
+          exit 1
+        fi
+      - ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
+      - |
+        CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
+        FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
+        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -vE '^(deploy|secrets)$' | sort -u || true)
+        ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
+        [ -z "$ALL_STACKS" ] && exit 0
+
+        # NOTE: `docker stack deploy` briefly tears down and recreates tasks in
+        # Swarm's internal bookkeeping, so `docker stack ps` can transiently
+        # return nothing right after deploy even when the service is healthy.
+        # A single `sleep 5` + one-shot check produced false-alarm-looking
+        # "nothing found in stack" output on ordinary deploys (e.g. vaultwarden,
+        # 2026-08-25). Retry with backoff instead of a single fixed sleep, and
+        # only warn (don't fail the pipeline) if tasks never show up.
+        # 2026-09-08: `|| true` inside the command substitution is REQUIRED —
+        # this step runs under errexit, and an assignment from a failing
+        # command substitution (e.g. `docker stack ps` on a stack that doesn't
+        # exist) kills the whole step before the WARNING path can run.
+        ATTEMPTS=6
+        DELAY=5
+        for STACK in $ALL_STACKS; do
+          echo "--- $STACK ---"
+          i=1
+          while [ "$i" -le "$ATTEMPTS" ]; do
+            OUTPUT=$(ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} \
+              "docker stack ps $STACK --filter desired-state=running \
+              --format '  {{.Name}} {{.CurrentState}}'" 2>/dev/null || true)
+            if [ -n "$OUTPUT" ]; then
+              echo "$OUTPUT"
+              break
+            fi
+            if [ "$i" -eq "$ATTEMPTS" ]; then
+              echo "  WARNING: no running tasks found for $STACK after $((ATTEMPTS * DELAY))s."
+              echo "  This may be transient Swarm settle time, or a real problem — check manually:"
+              echo "  ssh root@$${SWARM_MANAGER_IP} 'docker stack ps $STACK --no-trunc'"
+            else
+              sleep "$DELAY"
+            fi
+            i=$((i + 1))
+          done
+        done
+
+  notify-success:
+    image: alpine:latest
+    environment:
+      TEAMS_WEBHOOK:
+        from_secret: teams_webhook
+    commands:
+      - apk add --no-cache curl
+      - |
+        CHANGED=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' ' ')
+        PAYLOAD=$(printf '{"title":"Swarm Deploy OK","text":"Stacks: %s Commit: %s","themeColor":"00aa00"}' "$CHANGED" "$CI_COMMIT_MESSAGE")
+        curl -sf -X POST -H 'Content-Type: application/json' -d "$PAYLOAD" "$TEAMS_WEBHOOK" || true
+    when:
+      - status: success
+
+  notify-failure:
+    image: alpine:latest
+    environment:
+      TEAMS_WEBHOOK:
+        from_secret: teams_webhook
+      WOODPECKER_HOST:
+        from_secret: woodpecker_host
+    commands:
+      - apk add --no-cache curl
+      - |
+        PAYLOAD=$(printf '{"title":"Swarm Deploy FAILED","text":"Commit: %s See: https://%s","themeColor":"ff0000"}' "$CI_COMMIT_MESSAGE" "$WOODPECKER_HOST")
+        curl -sf -X POST -H 'Content-Type: application/json' -d "$PAYLOAD" "$TEAMS_WEBHOOK" || true
+    when:
+      - status: failure

.woodpecker/renovate.yml

@@ -0,0 +1,44 @@
+# ================================================================
+# Renovate Bot - Cron Pipeline
+# ================================================================
+# WOODPECKER v3 REQUIREMENTS - DO NOT DOWNGRADE SYNTAX:
+#
+#   CORRECT (v3):
+#     environment:
+#       MY_VAR:
+#         from_secret: my_secret
+#
+#   WRONG (v1/v2 - BREAKS in v3):
+#     secrets: [my_secret]
+#     secrets:
+#       - my_secret
+#
+# This pipeline ONLY runs on the "renovate" cron schedule.
+# It does NOT run on push events - that is handled by .woodpecker.yml
+# Teams notifications are delivered via Gitea webhook → Power Automate
+# ================================================================
+
+when:
+  - event: cron
+    cron: renovate
+
+steps:
+  run-renovate:
+    image: renovate/renovate:latest
+    # Pull policy: always ensure latest renovate image
+    pull: true
+    environment:
+      # ⚠️ Woodpecker v3 syntax - from_secret map form required
+      RENOVATE_TOKEN:
+        from_secret: renovate_token
+      # Renovate platform config - self-hosted Gitea
+      RENOVATE_PLATFORM: gitea
+      RENOVATE_ENDPOINT: https://git.bryanmail.net
+      RENOVATE_GIT_AUTHOR: "Renovate Bot <renovate-bot@bryanmail.net>"
+      # Repos to scan - space-separated for multiple
+      RENOVATE_REPOSITORIES: "homelab/compose-files"
+      # Use the renovate.json in the repo root for all other config
+      RENOVATE_CONFIG_FILE: renovate.json
+      LOG_LEVEL: info
+    commands:
+      - renovate

SECRETS.md

@@ -14,3 +14,8 @@
 # 2. Select repository: homelab/compose-files
 # 3. Settings → Secrets
 # 4. Add SSH_KEY and TEAMS_WEBHOOK
+
+# ai stack secrets (added 2026-08-25, see PR #4):
+#   ai_aws_access_key_id, ai_aws_secret_access_key, ai_litellm_master_key,
+#   ai_litellm_salt_key, ai_litellm_db_password, ai_webui_secret_key,
+#   ai_open_webui_database_url, ai_oauth_client_secret

ai/ai.env.template

@@ -0,0 +1,55 @@
+# ─────────────────────────────────────────────────────────────────────────────
+# ai.env.template — AUTHORITATIVE template for ai/ai.env (rendered by
+# deploy/provision-stack.py per secrets/secrets-map.yaml).
+#
+#   - This file IS the complete key list for ai.env. The whole file is
+#     rendered on every provisioning run — no line surgery, so a key can
+#     never silently go missing again (root cause of the 2026-09-03 outage).
+#   - Key names match EXACTLY what ai/ai.yaml references (AI_<SERVICE>_*
+#     naming adopted on main 2026-09-06).
+#   - Non-secret config lives here as LITERAL values (visible, reviewable).
+#   - Secret values are dollar-brace placeholders resolved from the CI env
+#     (Woodpecker from_secret vars) at provisioning time. provision-stack.py
+#     FAILS HARD if any placeholder is missing/empty.
+#   - The rendered ai/ai.env exists only on the host (gitignored).
+#   - Rendered by provision-stack.py, NOT Woodpecker's yaml preprocessor —
+#     single-dollar placeholders are safe here (deploy.yml's double-dollar
+#     rule does NOT apply to this file).
+#
+# Consumed by ai/ai.yaml. DOMAIN_NAME comes from deploy/global.env, not here.
+# ─────────────────────────────────────────────────────────────────────────────
+
+# ── LiteLLM (non-secret config) ──────────────────────────────────────────────
+AI_AWS_REGION_NAME=us-east-2
+AI_LITELLM_MODIFY_PARAMS=False
+AI_LITELLM_DATABASE_MIGRATIONS=True
+
+# ── LiteLLM (secrets) ────────────────────────────────────────────────────────
+AI_AWS_ACCESS_KEY_ID=${AI_AWS_ACCESS_KEY_ID}
+AI_AWS_SECRET_ACCESS_KEY=${AI_AWS_SECRET_ACCESS_KEY}
+AI_LITELLM_MASTER_KEY=${AI_LITELLM_MASTER_KEY}
+AI_LITELLM_SALT_KEY=${AI_LITELLM_SALT_KEY}
+# Full connection URL is itself a secret (ai_litellm_database_url) — the
+# URL structure never appears in git.
+AI_LITELLM_DATABASE_URL=${AI_LITELLM_DATABASE_URL}
+AI_LITELLM_POSTGRES_PASSWORD=${AI_LITELLM_POSTGRES_PASSWORD}
+
+# ── Open WebUI (non-secret config) ───────────────────────────────────────────
+AI_OPEN_WEBUI_URL=https://ai.bryanmail.net
+AI_OPEN_WEBUI_ENABLE_OAUTH_SIGNUP=true
+AI_OPEN_WEBUI_OAUTH_MERGE_ACCOUNTS_BY_EMAIL=true
+AI_OPEN_WEBUI_OAUTH_PROVIDER_NAME=Authentik
+AI_OPEN_WEBUI_OPENID_PROVIDER_URL=https://auth.bryanmail.net/application/o/open-web-ui/.well-known/openid-configuration
+# OAuth client ID is a public identifier by OAuth2 design (it is sent to the
+# browser); the client SECRET below is the protected credential.
+AI_OPEN_WEBUI_OAUTH_CLIENT_ID=hVmhi1dS3TnG2cUw5QwLOx5FDLSWtnQUdZyeB5zK
+AI_OPEN_WEBUI_OAUTH_SCOPES=openid email profile
+AI_OPEN_WEBUI_OPENID_REDIRECT_URI=https://ai.bryanmail.net/oauth/oidc/callback
+
+# ── Open WebUI (secrets) ─────────────────────────────────────────────────────
+AI_OPEN_WEBUI_SECRET_KEY=${AI_OPEN_WEBUI_SECRET_KEY}
+AI_OPEN_WEBUI_DATABASE_URL=${AI_OPEN_WEBUI_DATABASE_URL}
+AI_OPEN_WEBUI_OAUTH_CLIENT_SECRET=${AI_OPEN_WEBUI_OAUTH_CLIENT_SECRET}
+
+# ── mcpo / mcpo-critical (secrets) ───────────────────────────────────────────
+MCPO_API_KEY=${AI_MCPO_API_KEY}

ai/ai.yaml

@@ -1,6 +1,6 @@
 services:
   open-webui:
-    image: ghcr.io/open-webui/open-webui:main
+    image: ghcr.io/open-webui/open-webui:0.11.3 #
     entrypoint:
     - /bin/bash
     - /app/tools/startup.sh
@@ -16,20 +16,20 @@ services:
       start_period: 60s
     environment:
     - OLLAMA_BASE_URL=http://ollama-intel-arc:11434
-    - WEBUI_SECRET_KEY=${WEBUI_SECRET_KEY}
+    - WEBUI_SECRET_KEY=${AI_OPEN_WEBUI_SECRET_KEY}
     - WEBUI_DB_HOST=postgresql
-    - DATABASE_URL=${OPEN_WEBUI_DATABASE_URL}
+    - DATABASE_URL=${AI_OPEN_WEBUI_DATABASE_URL}
     - ENABLE_TOOL_SERVER_CALLING=true
     - TOOL_SERVER_CALLING=true
-    - WEBUI_URL=${WEBUI_URL}
+    - WEBUI_URL=${AI_OPEN_WEBUI_URL}
-    - ENABLE_OAUTH_SIGNUP=${ENABLE_OAUTH_SIGNUP}
+    - ENABLE_OAUTH_SIGNUP=${AI_OPEN_WEBUI_ENABLE_OAUTH_SIGNUP}
-    - OAUTH_MERGE_ACCOUNTS_BY_EMAIL=${OAUTH_MERGE_ACCOUNTS_BY_EMAIL}
+    - OAUTH_MERGE_ACCOUNTS_BY_EMAIL=${AI_OPEN_WEBUI_OAUTH_MERGE_ACCOUNTS_BY_EMAIL}
-    - OAUTH_PROVIDER_NAME=${OAUTH_PROVIDER_NAME}
+    - OAUTH_PROVIDER_NAME=${AI_OPEN_WEBUI_OAUTH_PROVIDER_NAME}
-    - OPENID_PROVIDER_URL=${OPENID_PROVIDER_URL}
+    - OPENID_PROVIDER_URL=${AI_OPEN_WEBUI_OPENID_PROVIDER_URL}
-    - OAUTH_CLIENT_ID=${OAUTH_CLIENT_ID}
+    - OAUTH_CLIENT_ID=${AI_OPEN_WEBUI_OAUTH_CLIENT_ID}
-    - OAUTH_CLIENT_SECRET=${OAUTH_CLIENT_SECRET}
+    - OAUTH_CLIENT_SECRET=${AI_OPEN_WEBUI_OAUTH_CLIENT_SECRET}
-    - OAUTH_SCOPES=${OAUTH_SCOPES}
+    - OAUTH_SCOPES=${AI_OPEN_WEBUI_OAUTH_SCOPES}
-    - OPENID_REDIRECT_URI=${OPENID_REDIRECT_URI}
+    - OPENID_REDIRECT_URI=${AI_OPEN_WEBUI_OPENID_REDIRECT_URI}
     networks:
     - traefik_backend
     - postgresql_db-backend
@@ -56,26 +56,22 @@ services:
       - traefik.http.services.open-webui.loadbalancer.sticky.cookie.secure=true
       - traefik.http.services.open-webui.loadbalancer.sticky.cookie.httponly=true
       - traefik.swarm.network=traefik_backend
-  litellm: #
+  litellm:
-    image: docker.litellm.ai/berriai/litellm-database:v1.90.3 #main-latest
+    image: docker.litellm.ai/berriai/litellm-database:v1.93.0  # upgraded from v1.92.1; fixes Bedrock parallel tool call truncation bug
     command: --config /app/config.yaml --port 4000
     environment:
-    - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+    - AWS_ACCESS_KEY_ID=${AI_AWS_ACCESS_KEY_ID}
-    - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+    - AWS_SECRET_ACCESS_KEY=${AI_AWS_SECRET_ACCESS_KEY}
-    - AWS_REGION_NAME=${AWS_REGION_NAME}
+    - AWS_REGION_NAME=${AI_AWS_REGION_NAME}
-    - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY}
+    - LITELLM_MASTER_KEY=${AI_LITELLM_MASTER_KEY}
-    - LITELLM_SALT_KEY=${LITELLM_SALT_KEY}
+    - LITELLM_SALT_KEY=${AI_LITELLM_SALT_KEY}
-    - DATABASE_URL=${DATABASE_URL}
+    - DATABASE_URL=${AI_LITELLM_DATABASE_URL}
-    - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+    - POSTGRES_PASSWORD=${AI_LITELLM_POSTGRES_PASSWORD}
-    - LITELLM_MODIFY_PARAMS=${LITELLM_MODIFY_PARAMS}
+    - LITELLM_MODIFY_PARAMS=${AI_LITELLM_MODIFY_PARAMS}
+    - LITELLM_DATABASE_MIGRATIONS=${AI_LITELLM_DATABASE_MIGRATIONS}
     networks:
     - traefik_backend
     - postgresql_db-backend
-#    ports:
-#    - target: 4000
-#      published: 4000
-#      protocol: tcp
-#      mode: ingress
     volumes:
     - type: bind
       source: /volume1/docker/litellm/app/config.yaml
@@ -119,28 +115,19 @@ services:
     - ${MCPO_API_KEY}
     - --hot-reload
     environment:
-    - PROXMOX_ALLOW_UNSAFE_COMMANDS=false #true
+    - PROXMOX_ALLOW_UNSAFE_COMMANDS=false
     volumes:
     - /volume1/docker/mcpo/config-critical.json:/app/config/config.json:ro
     - /volume1/docker/mcpo/data:/mcpo_data
     - /volume1/docker/cronicle/ssh_keys:/app/ssh_keys:ro
-#    - /volume1/docker/mcpo/teams-mcp-auth:/app/teams-mcp-auth
     - /volume1/docker/mcpo/uv-cache:/app/uv-cache
     networks:
     - traefik_backend
     deploy:
       labels:
       - traefik.enable=false
-#      - traefik.http.routers.mcp.rule=Host(`mcpc.bryanmail.net`)
-#      - traefik.http.routers.mcp.tls=true
-#      - traefik.http.routers.mcp.tls.certresolver=letsencrypt
-#      - traefik.http.routers.mcp.entrypoints=websecure
-#      - traefik.http.services.mcp.loadbalancer.server.port=8000
-#      - traefik.http.services.mcp.loadbalancer.server.scheme=http
-#      - traefik.http.routers.mcp.middlewares=forwardAuth-authentik@file, crowdsec@file
-#      - traefik.swarm.network=traefik_backend
   mcpo:
-    image: ghcr.io/open-webui/mcpo:main
+    image: git.bryanmail.net/homelab/flowagent-mcpo:c5b22618404a
     command:
     - --config
     - /app/config/config.json
@@ -153,28 +140,24 @@ services:
     - /volume1/docker/mcpo/config.json:/app/config/config.json:ro
     - /volume1/docker/mcpo/data:/mcpo_data
     - /volume1/docker/cronicle/ssh_keys:/app/ssh_keys:ro
-#    - /volume1/docker/mcpo/teams-mcp-auth:/app/teams-mcp-auth
     - /volume1/docker/mcpo/uv-cache:/app/uv-cache
+    secrets:
+    - source: flowagent_azure_client_id
+      target: flowagent_azure_client_id
+    - source: flowagent_azure_tenant_id
+      target: flowagent_azure_tenant_id
+    - source: flowagent_azure_client_secret
+      target: flowagent_azure_client_secret
     networks:
     - traefik_backend
     deploy:
       labels:
       - traefik.enable=false
-#      - traefik.http.routers.mcp.rule=Host(`mcp.bryanmail.net`)
-#      - traefik.http.routers.mcp.tls=true
-#      - traefik.http.routers.mcp.tls.certresolver=letsencrypt
-#      - traefik.http.routers.mcp.entrypoints=websecure
-#      - traefik.http.services.mcp.loadbalancer.server.port=8000
-#      - traefik.http.services.mcp.loadbalancer.server.scheme=http
-#      - traefik.http.routers.mcp.middlewares=forwardAuth-authentik@file, crowdsec@file
-#      - traefik.swarm.network=traefik_backend
   n8n:
     image: docker.n8n.io/n8nio/n8n
     networks:
     - traefik_backend
     - postgresql_db-backend
-#    ports:
-#    - 5678:5678
     volumes:
     - /volume1/docker/n8n/data:/home/node/.n8n
     - /volume1/docker/n8n/files:/files
@@ -188,7 +171,7 @@ services:
       - traefik.http.services.n8n.loadbalancer.server.port=5678
       - traefik.http.services.n8n.loadbalancer.server.scheme=http
       - traefik.http.middlewares.n8n.headers.SSLRedirect=true
-      - traefik.http.middlewares.n8n.headers.STSSeconds=315360000
+      - traefik.http.middlewares.n8n.headers.STSSeconds=31536000
       - traefik.http.middlewares.n8n.headers.browserXSSFilter=true
       - traefik.http.middlewares.n8n.headers.contentTypeNosniff=true
       - traefik.http.middlewares.n8n.headers.forceSTSHeader=true
@@ -197,6 +180,13 @@ services:
       - traefik.http.middlewares.n8n.headers.STSPreload=true
       - traefik.http.routers.n8n.middlewares=forwardAuth-authentik@file, crowdsec@file
       - traefik.swarm.network=traefik_backend
+secrets:
+  flowagent_azure_client_id:
+    external: true
+  flowagent_azure_tenant_id:
+    external: true
+  flowagent_azure_client_secret:
+    external: true
 networks:
   traefik_backend:
     external: true

deploy/envparse.py

@@ -22,12 +22,229 @@ def merge_envs(base_path, override_path):
     merged = {**base, **override}
     return list(merged.items())
 
+# ──────────────────────────────────────────────────────────────────────────
+# 2026-08-26 FIX — double-interpolation truncation bug (Pattern B stacks):
+#
+#   stack-deploy.sh's single-file/no-extras path is:
+#     envsubst "$VARS" < ai.yaml | docker stack deploy -c - ai
+#
+#   envsubst substitutes ${VAR} placeholders in the compose YAML with the
+#   literal, raw value of each shell-exported variable. If that raw value
+#   itself contains a literal '$' followed by word characters (e.g. a
+#   randomly-generated secret like "...i*Edu$RyAVYTqr4yzSS##..."), the
+#   resulting YAML text now contains what LOOKS like a second variable
+#   reference. `docker stack deploy -c -` runs Compose's own interpolation
+#   pass on that YAML text before creating the service — and Compose sees
+#   that leftover "$RyAVYTqr4yzSS", finds no such env var, and silently
+#   substitutes empty string. The secret gets truncated in the running
+#   container with NO error or warning.
+#
+#   Confirmed impact (2026-08-26): LITELLM_MASTER_KEY and LITELLM_SALT_KEY
+#   in the `ai` stack were both truncated at their first literal '$' after
+#   a real deploy — 87-char secret arrived in the container as 73 chars.
+#
+#   This affects every stack using Pattern B (host .env + envsubst, not
+#   native Docker secrets): ai, maintenance, media, unifi, guacamole,
+#   security, auth, traefik, meshcentral, ddm — any of them could have a
+#   '$'-containing value silently truncating right now without detection,
+#   since the failure is silent and only visible by diffing the source
+#   value against the live container env.
+#
+#   Fix: escape every literal '$' in a value as '$$' at export time, BEFORE
+#   envsubst ever sees it. envsubst does not interpret '$' in the
+#   replacement text (only in the template), so the doubled dollar survives
+#   envsubst untouched. Compose's interpolation pass then consumes exactly
+#   one level of escaping ('$$' -> literal '$'), landing on the correct
+#   original single '$' with no leftover variable-reference lookalike.
+#
+#   ONLY applies to the single-file path (no `docker compose config` step
+#   downstream). See export_raw/export_raw_merged below for why the
+#   folder+extras path must NOT use this.
+# ──────────────────────────────────────────────────────────────────────────
+
+def escape_dollar(v):
+    return v.replace('$', '$$')
+
+# ──────────────────────────────────────────────────────────────────────────
+# 2026-08-26 FIX #2 — SECOND double-interpolation bug, folder+extras path
+# (discovered fixing IMMICH_KIOSK_BASICAUTH, a bcrypt hash full of '$'):
+#
+#   The folder+extras render path is:
+#     docker compose <files> config | strip | envsubst "$VARS"
+#     -> docker stack deploy -c -
+#
+#   `docker compose config` performs its OWN ${VAR} interpolation AND its
+#   own re-escaping of the output: any literal '$' character that ends up
+#   in the rendered YAML — escaped or not — gets doubled to '$$' by
+#   `docker compose config` itself, unconditionally, as part of producing
+#   spec-safe output. Confirmed by isolated test:
+#     raw MYVAR='a$b$c'   -> docker compose config -> "a$$b$$c"   (correct,
+#                                                        one level added)
+#     escaped MYVAR='a$$b$$c' (i.e. pre-doubled by escape_dollar) ->
+#       docker compose config -> "a$$$$b$$$$c"        (WRONG, doubled twice)
+#
+#   `docker stack deploy -c -` still only removes exactly ONE level of
+#   escaping on its way in (confirmed: "a$$b$$c" -> container label
+#   "a$b$c", correct). So across the whole folder+extras pipeline there is
+#   exactly ONE implicit escaping step (`docker compose config`) and ONE
+#   implicit un-escaping step (`docker stack deploy`) already built in —
+#   pre-escaping the exported value on top of that leaves one extra,
+#   uncollapsed level of '$$' in the final container label/env value.
+#
+#   Confirmed impact (2026-08-26): IMMICH_KIOSK_BASICAUTH
+#   ("BabyBryan:$2y$05$...") rendered as "BabyBryan:$$2y$$05$$..." in the
+#   final container label — Traefik basic auth would never match the real
+#   password hash, silently locking out the kiosk with no error.
+#
+#   Fix: use export_raw / export_raw_merged (NO escape_dollar) whenever the
+#   render path goes through `docker compose config` — i.e. any stack with
+#   extension files. Use export / export_merged (WITH escape_dollar) only
+#   for the single-file path, which has no `docker compose config` step and
+#   therefore only Swarm's own interpolation pass to protect against.
+# ──────────────────────────────────────────────────────────────────────────
+
+# ──────────────────────────────────────────────────────────────────────────
+# 2026-08-26 FIX — depends_on long-form vs Swarm short-form:
+#
+#   Folder-based stacks with extension files (e.g. immich + hwaccel.*.yml)
+#   go through the `docker compose <files> config | strip | docker stack
+#   deploy -c -` merge path. Modern `docker compose config` normalizes the
+#   short-form list syntax:
+#     depends_on: [database, redis]
+#   into the long-form condition mapping:
+#     depends_on:
+#       database:
+#         condition: service_started
+#         required: true
+#       redis:
+#         condition: service_started
+#         required: true
+#
+#   `docker stack deploy` (Swarm mode) does NOT understand the long-form
+#   mapping and rejects it with: "services.<svc>.depends_on must be a list".
+#   Single-file stacks (no extras) never hit this because they skip the
+#   `docker compose config` step entirely and go straight through envsubst.
+#
+#   Fix: collapse any long-form depends_on mapping back into the Swarm-
+#   compatible short-form list, purely as a text transform on the rendered
+#   YAML, right alongside the existing env_file stripping.
+# ──────────────────────────────────────────────────────────────────────────
+
+_DEP_RE = re.compile(r'^([ \t]*)depends_on:\s*$')
+_CHILD_KEY_RE = re.compile(r'^([ \t]*)([\w.\-]+):\s*$')
+_CHILD_LIST_RE = re.compile(r'^([ \t]*)-\s*(\S+)\s*$')
+
+def collapse_depends_on(text):
+    lines = text.split('\n')
+    out = []
+    i = 0
+    n = len(lines)
+    while i < n:
+        line = lines[i]
+        m = _DEP_RE.match(line)
+        if not m:
+            out.append(line)
+            i += 1
+            continue
+
+        indent = m.group(1)
+        base_indent = len(indent)
+        out.append(line)
+        i += 1
+
+        services = []
+        parse_ok = True
+        while i < n:
+            l = lines[i]
+            if l.strip() == '':
+                i += 1
+                continue
+            cur_indent = len(l) - len(l.lstrip(' \t'))
+            if cur_indent <= base_indent:
+                break  # dedent — end of this depends_on block
+
+            lm = _CHILD_LIST_RE.match(l)
+            if lm and cur_indent == base_indent + 2:
+                services.append(lm.group(2))
+                i += 1
+                continue
+
+            km = _CHILD_KEY_RE.match(l)
+            if km and cur_indent == base_indent + 2:
+                services.append(km.group(2))
+                i += 1
+                # skip nested condition/required/restart lines under this service
+                while i < n:
+                    nl = lines[i]
+                    if nl.strip() == '':
+                        i += 1
+                        continue
+                    nl_indent = len(nl) - len(nl.lstrip(' \t'))
+                    if nl_indent > base_indent + 2:
+                        i += 1
+                        continue
+                    break
+                continue
+
+            # Unexpected shape — bail without transforming to avoid corrupting output
+            parse_ok = False
+            break
+
+        if not parse_ok or not services:
+            # Re-emit whatever we consumed as-is (best effort: fall back to
+            # original slice) rather than risk mangling an unfamiliar shape.
+            pass
+
+        for svc in services:
+            out.append(indent + '  - ' + svc)
+
+    return '\n'.join(out)
+
+# ──────────────────────────────────────────────────────────────────────────
+# 2026-08-26 FIX — top-level `name:` property rejected by Swarm:
+#
+#   `docker compose config` (Compose Spec output) emits a top-level
+#   `name: <project>` key as the first line of the rendered document, e.g.:
+#     name: immich
+#     services:
+#       ...
+#
+#   This is valid Compose Spec but `docker stack deploy` (Swarm mode) uses
+#   a stricter schema that does NOT allow a top-level `name` property, and
+#   rejects the whole document with: "(root) Additional property name is
+#   not allowed". Single-file stacks (no extras) never hit this because
+#   they skip `docker compose config` and go straight through envsubst on
+#   the raw source YAML, which never had a top-level `name:` to begin with.
+#
+#   Fix: drop ONLY the top-level `name:` line (column 0, i.e. no leading
+#   whitespace) at the START of the document. Nested `name:` fields under
+#   networks/volumes/secrets (e.g. `name: postgresql_db-backend`, always
+#   indented) are legitimate and must NOT be touched.
+# ──────────────────────────────────────────────────────────────────────────
+
+_TOP_NAME_RE = re.compile(r'^name:\s*\S+\s*$')
+
+def strip_top_level_name(text):
+    lines = text.split('\n')
+    if lines and _TOP_NAME_RE.match(lines[0]):
+        lines = lines[1:]
+    return '\n'.join(lines)
+
 mode = sys.argv[1]
 if mode == 'export':
     for k, v in parse_env(sys.argv[2]):
-        print('export {}={}'.format(k, repr(v)))
+        print('export {}={}'.format(k, repr(escape_dollar(v))))
 elif mode == 'export_merged':
     # export_merged <global.env> <stack.env>
+    for k, v in merge_envs(sys.argv[2], sys.argv[3]):
+        print('export {}={}'.format(k, repr(escape_dollar(v))))
+elif mode == 'export_raw':
+    # export_raw <env-file> — NO escape_dollar. Use for the docker-compose-
+    # config render path (folder+extras), which does its own '$' escaping.
+    for k, v in parse_env(sys.argv[2]):
+        print('export {}={}'.format(k, repr(v)))
+elif mode == 'export_raw_merged':
+    # export_raw_merged <global.env> <stack.env> — NO escape_dollar.
     for k, v in merge_envs(sys.argv[2], sys.argv[3]):
         print('export {}={}'.format(k, repr(v)))
 elif mode == 'vars':
@@ -38,4 +255,6 @@ elif mode == 'vars_merged':
 elif mode == 'strip':
     t = sys.stdin.read()
     t = re.sub(r'[ \t]*env_file:[ \t]*\n([ \t]+-[^\n]*\n)+', '', t)
+    t = collapse_depends_on(t)
+    t = strip_top_level_name(t)
     sys.stdout.write(t)

deploy/git-guard.sh

@@ -0,0 +1,142 @@
+#!/usr/bin/env bash
+# git-guard.sh — Ensures the compose-files working tree is in sync with Gitea
+# before any deploy proceeds. Called automatically by stack-deploy.sh.
+#
+# Behavior:
+#   - Clean + up to date        -> pass silently
+#   - Clean + behind (ff-only)  -> auto `git pull --ff-only`, then pass
+#   - Ahead only (unpushed)     -> interactive: offer to push; non-interactive: BLOCK
+#   - Dirty tracked changes     -> offer to commit + push right now
+#                                   (auto in non-interactive/CI runs, after a
+#                                   secret-pattern scan of the staged diff)
+#   - Diverged (local AND       -> REFUSE. Never auto-resolves. Prints the
+#     remote both moved)           backup/stash/reset recovery steps and exits.
+#
+# Exit codes: 0 = safe to deploy, 1 = blocked, needs human intervention
+
+set -euo pipefail
+
+DIR="/volume1/docker/compose-files"
+cd "$DIR"
+
+# Non-interactive detection (Woodpecker/cron have no TTY on stdin)
+INTERACTIVE=0
+[ -t 0 ] && INTERACTIVE=1
+
+echo "==> git-guard: checking repo sync state"
+
+git fetch origin --quiet
+
+LOCAL="$(git rev-parse main)"
+REMOTE="$(git rev-parse origin/main)"
+BASE="$(git merge-base main origin/main)"
+
+DIRTY=0
+git status --porcelain | grep -q . && DIRTY=1
+
+SECRET_PATTERN='(-----BEGIN [A-Z]+ PRIVATE KEY-----|AKIA[0-9A-Z]{16}|xox[baprs]-[0-9a-zA-Z-]+|password[[:space:]]*[:=][[:space:]]*[^$ ]|api[_-]?key[[:space:]]*[:=][[:space:]]*[^$ ])'
+
+# ---- Case: dirty tracked changes ----
+if [ "$DIRTY" -eq 1 ]; then
+  echo "!! WORKING TREE DIRTY — uncommitted changes detected:"
+  git status --short
+  echo
+
+  if [ "$INTERACTIVE" -eq 1 ]; then
+    read -rp "Commit and push these changes to origin/main now? [y/N] " ans
+  else
+    ans="y"
+    echo "(non-interactive session — auto-committing and pushing)"
+  fi
+
+  if [[ "$ans" =~ ^[Yy]$ ]]; then
+    git add -A
+
+    if git diff --cached | grep -Eiq "$SECRET_PATTERN"; then
+      echo "ERROR: possible secret detected in staged changes. Refusing to auto-commit."
+      echo "Review manually: git diff --cached"
+      git reset
+      exit 1
+    fi
+
+    git commit -m "chore(auto): git-guard autofix - commit local changes before deploy $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+
+    if git push origin main; then
+      echo "==> Pushed. Re-checking sync state..."
+      exec "$0" "$@"
+    else
+      echo "ERROR: push failed (likely diverged from origin). Aborting deploy."
+      echo "Run: cd $DIR && git status"
+      exit 1
+    fi
+  else
+    echo "Aborting deploy - commit or stash changes manually, then retry."
+    exit 1
+  fi
+fi
+
+# ---- Case: fully in sync ----
+if [ "$LOCAL" = "$REMOTE" ]; then
+  echo "==> In sync with origin/main ($LOCAL). OK to deploy."
+  exit 0
+fi
+
+# ---- Case: behind only (fast-forwardable) ----
+if [ "$LOCAL" = "$BASE" ]; then
+  echo "!! Local main is behind origin/main."
+  if [ "$INTERACTIVE" -eq 1 ]; then
+    read -rp "Fast-forward pull now? [y/N] " ans
+  else
+    ans="y"
+    echo "(non-interactive session — auto fast-forwarding)"
+  fi
+  if [[ "$ans" =~ ^[Yy]$ ]]; then
+    git pull --ff-only origin main
+    echo "==> Fast-forwarded to $(git rev-parse --short main). OK to deploy."
+    exit 0
+  else
+    echo "Aborting deploy - pull manually, then retry."
+    exit 1
+  fi
+fi
+
+# ---- Case: ahead only (local commits not yet pushed) ----
+if [ "$REMOTE" = "$BASE" ]; then
+  echo "!! Local main is AHEAD of origin/main (unpushed commits):"
+  git log --oneline "origin/main..main"
+  echo
+  if [ "$INTERACTIVE" -eq 1 ]; then
+    read -rp "Push local commits to origin/main now? [y/N] " ans
+  else
+    ans="n"
+    echo "(non-interactive session — will NOT auto-push ahead commits; needs human review)"
+  fi
+  if [[ "$ans" =~ ^[Yy]$ ]]; then
+    git push origin main
+    echo "==> Pushed. OK to deploy."
+    exit 0
+  else
+    echo "Aborting deploy. Review with: git log origin/main..main"
+    exit 1
+  fi
+fi
+
+# ---- Case: true divergence (both ahead and behind) — NEVER auto-fix ----
+echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+echo "!! DIVERGED: local main and origin/main have both moved independently."
+echo "!!"
+echo "!!   Local-only commits:"
+git log --oneline "$BASE..main" | sed 's/^/!!     /'
+echo "!!"
+echo "!!   Remote-only commits:"
+git log --oneline "$BASE..origin/main" | sed 's/^/!!     /'
+echo "!!"
+echo "!! This requires a human decision - git-guard will NOT auto-resolve this."
+echo "!! Recommended recovery:"
+echo "!!   1. tar backup:      tar czf /volume1/docker/compose-files-backup-\$(date +%Y%m%d-%H%M%S).tar.gz -C /volume1/docker compose-files"
+echo "!!   2. name the branch: git branch backup/pre-reset-\$(date +%Y%m%d)"
+echo "!!   3. stash all state: git stash push -u -m 'pre-reset-snapshot'"
+echo "!!   4. reset to origin: git reset --hard origin/main"
+echo "!!   5. selectively restore needed files from the stash/backup branch"
+echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+exit 1

deploy/mount-guard.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+"""
+mount-guard.py — Pre-flight check for bind mount paths before `docker stack deploy`.
+
+Usage: mount-guard.py <rendered-compose.yml>
+
+Reads the FINAL rendered compose YAML (after env substitution, right before
+it would be handed to `docker stack deploy -c -`) and checks:
+
+  1. MISSING PATHS — every bind-mount source path referenced by any service
+     actually exists on disk. Swarm rejects the task at schedule time if not
+     (see: "bind source path does not exist"), but catching it here is
+     faster and clearer, and gives the option to create it on the spot.
+
+  2. SUSPICIOUS POSTGRES DATA DIRS — for any bind mount whose *target*
+     looks like a Postgres data directory, warn if the *source* exists but
+     is empty or missing a PG_VERSION file. This is the class of near-miss
+     from the 2026-08-26 Immich incident: a wrong-but-existing empty path
+     would have let Postgres silently initialize a brand-new database while
+     the real data sat orphaned elsewhere, with no error at all.
+
+Exit codes: 0 = safe to deploy, 1 = blocked / aborted.
+
+Interactive sessions get a prompt with remediation options. Non-interactive
+sessions (Woodpecker, cron, CI) NEVER auto-proceed past a finding here —
+this is a data-safety check, not a convenience autofix.
+"""
+
+import sys
+import os
+import re
+import yaml
+
+
+def is_bind_like(path):
+    return isinstance(path, str) and (
+        path.startswith('/') or path.startswith('./') or path.startswith('../')
+    )
+
+
+def gather_bind_mounts(doc):
+    """Return list of (service_name, target, source) for host-path bind mounts.
+
+    Handles both compose-file syntaxes that can appear in a rendered stack:
+      - short form:  "hostpath:containerpath[:mode]"        (raw single-file path)
+      - long form:   {type: bind, source: ..., target: ...} (post `docker compose config`)
+    Named volumes (source has no leading '/', './', '../') are intentionally skipped.
+    """
+    results = []
+    services = doc.get('services') or {}
+    for svc_name, svc in services.items():
+        if not isinstance(svc, dict):
+            continue
+
+        for v in (svc.get('volumes') or []):
+            if isinstance(v, str):
+                parts = v.split(':')
+                if len(parts) >= 2 and is_bind_like(parts[0]):
+                    results.append((svc_name, parts[1], parts[0]))
+            elif isinstance(v, dict):
+                if v.get('type') == 'bind':
+                    src = v.get('source')
+                    tgt = v.get('target')
+                    if is_bind_like(src):
+                        results.append((svc_name, tgt, src))
+
+        # devices: short form "host:container[:mode]" (rarely used in this repo —
+        # devices are conventionally expressed as bind-mounted volumes instead,
+        # for Docker Swarm / DDM compatibility — but handle it if present).
+        for d in (svc.get('devices') or []):
+            if isinstance(d, str):
+                parts = d.split(':')
+                if parts and is_bind_like(parts[0]):
+                    tgt = parts[1] if len(parts) > 1 else parts[0]
+                    results.append((svc_name, tgt, parts[0]))
+
+    return results
+
+
+_PG_TARGET_RE = re.compile(r'postgres', re.IGNORECASE)
+
+
+def looks_like_postgres_target(target):
+    return bool(target) and bool(_PG_TARGET_RE.search(target))
+
+
+def check_postgres_data(path):
+    """Return a warning string if `path` looks like an empty/uninitialized
+    Postgres data directory. Returns None if it looks fine. Caller must
+    ensure `path` already exists before calling this."""
+    if not os.path.isdir(path):
+        # Exists but isn't a directory (e.g. a file) — a different, separate
+        # problem; the missing-path style check doesn't apply, but this is
+        # clearly wrong too.
+        return "exists but is not a directory"
+    try:
+        entries = os.listdir(path)
+    except PermissionError:
+        return "cannot list directory contents (permission denied) — unable to verify"
+    if not entries:
+        return "directory exists but is EMPTY — looks like an uninitialized/wrong Postgres data dir"
+    if 'PG_VERSION' not in entries:
+        return "directory exists and is non-empty but has no PG_VERSION file — does not look like a real Postgres data directory"
+    return None
+
+
+def prompt(question, choices):
+    """choices: dict of key -> description. Returns the chosen key (lowercase)."""
+    print("Options:")
+    for k, desc in choices.items():
+        print(f"  [{k}] {desc}")
+    while True:
+        ans = input(f"{question} [{'/'.join(choices.keys())}]: ").strip().lower()
+        if ans in choices:
+            return ans
+        print(f"Please enter one of: {', '.join(choices.keys())}")
+
+
+def main():
+    if len(sys.argv) != 2:
+        print("Usage: mount-guard.py <rendered-compose.yml>", file=sys.stderr)
+        return 1
+
+    rendered_path = sys.argv[1]
+    with open(rendered_path) as f:
+        doc = yaml.safe_load(f)
+
+    if not doc or 'services' not in doc:
+        print("!! mount-guard: rendered YAML has no 'services' key — refusing to guess, blocking.")
+        return 1
+
+    mounts = gather_bind_mounts(doc)
+
+    missing = []
+    pg_warnings = []
+
+    for svc_name, target, src in mounts:
+        if not os.path.exists(src):
+            missing.append((svc_name, target, src))
+            continue
+        if looks_like_postgres_target(target):
+            warn = check_postgres_data(src)
+            if warn:
+                pg_warnings.append((svc_name, target, src, warn))
+
+    if not missing and not pg_warnings:
+        print(f"==> mount-guard: {len(mounts)} bind mount path(s) checked, all present and sane. OK to deploy.")
+        return 0
+
+    print("!! mount-guard found potential issues with bind mount paths:\n")
+
+    if missing:
+        print("MISSING PATHS (Swarm will reject the task if these aren't created first):")
+        for svc_name, target, src in missing:
+            print(f"  - service '{svc_name}': {src}  (mounted at {target})")
+        print()
+
+    if pg_warnings:
+        print("SUSPICIOUS POSTGRES DATA DIRECTORIES:")
+        print("  Path exists, but contents don't look like a real Postgres data dir.")
+        print("  This is the exact shape of the 2026-08-26 Immich near-miss: a wrong")
+        print("  bind path that HAPPENED to exist (empty) would have let Postgres")
+        print("  silently init a new empty DB while the real data sat orphaned")
+        print("  elsewhere — with no error or warning from Docker at all.")
+        for svc_name, target, src, warn in pg_warnings:
+            print(f"  - service '{svc_name}': {src}  (mounted at {target})")
+            print(f"      {warn}")
+        print()
+
+    interactive = sys.stdin.isatty()
+
+    if not interactive:
+        print("(non-interactive session — mount-guard will NOT auto-proceed on any finding above)")
+        print("Re-run interactively to review and decide, or fix the paths and retry.")
+        return 1
+
+    if missing:
+        choices = {
+            'a': 'Abort deploy (recommended if any path above is unexpected)',
+            'm': 'mkdir -p the missing path(s) now, then continue',
+            'c': 'Continue anyway without creating them (will likely fail at Swarm scheduling)',
+        }
+        ans = prompt("Missing bind mount paths found. Choice", choices)
+        if ans == 'a':
+            print("Aborting deploy.")
+            return 1
+        if ans == 'm':
+            for _, _, src in missing:
+                print(f"  mkdir -p {src}")
+                os.makedirs(src, exist_ok=True)
+            print("==> Created missing path(s). Continuing.")
+        # 'c' falls through and continues without creating
+
+    if pg_warnings:
+        choices = {
+            'a': 'Abort deploy (recommended unless you intended a fresh Postgres init here)',
+            'c': 'Continue anyway (I have verified this is expected, e.g. legitimate first-time init)',
+        }
+        ans = prompt("Suspicious Postgres data directory found. Choice", choices)
+        if ans == 'a':
+            print("Aborting deploy.")
+            return 1
+        print("==> Continuing deploy despite Postgres data warning, per operator confirmation.")
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

deploy/provision-stack.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+"""provision-stack.py — manifest-driven env-file rendering + Docker secret
+provisioning for one stack.
+
+Usage (from the CI workspace root, inside the provision-secrets step):
+    python3 deploy/provision-stack.py <stack>
+
+Reads secrets/secrets-map.yaml (data only — no code, no values) and, for the
+named stack:
+
+  1. env_template -> renders the COMPLETE env file. Placeholders of the form
+     dollar-brace VARNAME are resolved from this process's environment (the
+     Woodpecker from_secret-backed vars). The whole file is rendered every
+     run; nothing is line-edited in place, so keys can never silently go
+     missing (root cause of the 2026-09-03 ai.env incident).
+  2. env_dest -> ships the rendered file to the Swarm manager over ssh stdin
+     (write-to-temp + atomic mv, mode 600). The rendered file never touches
+     the CI workspace disk under the repo (no chance of being committed) and
+     never appears on a command line.
+  3. docker_secrets -> for each swarm-secret-name -> ENV_VAR mapping, creates
+     or rotates the Docker secret. Values are passed via ssh stdin only.
+     Rotation uses the same sha256-checksum-label convention as
+     deploy/create-secrets.sh (unchanged secrets are skipped silently).
+
+Safety properties:
+  - FAILS HARD (non-zero) if any referenced env var is missing or empty, and
+    lists the missing NAMES. A partial/broken render can never ship.
+  - FAILS HARD if any unresolved placeholder remains after rendering.
+  - NEVER prints a secret value — names and counts only.
+  - Requires SWARM_MANAGER_IP in the environment and a usable ssh identity
+    (both already set up by the provision-secrets step).
+
+Stacks not present in the manifest exit 0 with a notice, so this script is
+safe to call unconditionally; legacy case-entries in deploy.yml keep handling
+unmigrated stacks.
+"""
+
+import hashlib
+import os
+import re
+import subprocess
+import sys
+
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+MANIFEST_PATH = os.path.join(REPO_ROOT, "secrets", "secrets-map.yaml")
+REMOTE_BASE = "/volume1/docker/compose-files"
+PLACEHOLDER_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
+
+SSH_OPTS = ["-o", "StrictHostKeyChecking=no"]
+
+
+def die(msg: str) -> None:
+    print(f"ERROR: {msg}", file=sys.stderr)
+    sys.exit(1)
+
+
+def load_manifest() -> dict:
+    try:
+        import yaml  # py3-yaml, installed by the provision-secrets step
+    except ImportError:
+        die("PyYAML not available — provision-secrets step must apk add py3-yaml")
+    if not os.path.isfile(MANIFEST_PATH):
+        die(f"manifest not found: {MANIFEST_PATH}")
+    with open(MANIFEST_PATH, "r", encoding="utf-8") as fh:
+        data = yaml.safe_load(fh) or {}
+    stacks = data.get("stacks")
+    if not isinstance(stacks, dict):
+        die("manifest has no 'stacks:' mapping")
+    return stacks
+
+
+def ssh_target() -> str:
+    ip = os.environ.get("SWARM_MANAGER_IP", "").strip()
+    if not ip:
+        die("SWARM_MANAGER_IP is empty — check Woodpecker repo secrets")
+    return f"root@{ip}"
+
+
+def ssh_run(target: str, remote_cmd: str, stdin_data: bytes | None = None,
+            check: bool = True) -> subprocess.CompletedProcess:
+    proc = subprocess.run(
+        ["ssh", *SSH_OPTS, target, remote_cmd],
+        input=stdin_data, capture_output=True,
+    )
+    if check and proc.returncode != 0:
+        # stderr may be verbose but must never contain our secret values —
+        # we only ever send values via stdin, never embed them in remote_cmd.
+        die(f"remote command failed (rc={proc.returncode}): {remote_cmd}\n"
+            f"{proc.stderr.decode(errors='replace').strip()}")
+    return proc
+
+
+def render_template(template_path: str) -> str:
+    if not os.path.isfile(template_path):
+        die(f"env_template not found: {template_path}")
+    with open(template_path, "r", encoding="utf-8") as fh:
+        raw = fh.read()
+
+    referenced = sorted(set(PLACEHOLDER_RE.findall(raw)))
+    missing = [v for v in referenced
+               if not os.environ.get(v, "").strip()]
+    if missing:
+        die("template references vars that are MISSING or EMPTY in the CI "
+            "environment (add them via from_secret in "
+            ".woodpecker/deploy.yml provision-secrets, and as Woodpecker "
+            "secrets):\n  " + "\n  ".join(missing))
+
+    rendered = PLACEHOLDER_RE.sub(lambda m: os.environ[m.group(1)], raw)
+
+    # Belt-and-braces: nothing placeholder-shaped may survive the render.
+    leftover = sorted(set(PLACEHOLDER_RE.findall(rendered)))
+    if leftover:
+        die("unresolved placeholders remain after rendering: "
+            + ", ".join(leftover))
+
+    print(f"  [render] {template_path}: {len(referenced)} secret placeholder(s) "
+          f"resolved: {', '.join(referenced)}")
+    return rendered
+
+
+def ship_env_file(target: str, rendered: str, dest_rel: str) -> None:
+    dest = f"{REMOTE_BASE}/{dest_rel}"
+    tmp = f"{dest}.provision-tmp"
+    # Value travels over ssh stdin; never on a command line; atomic mv.
+    ssh_run(target,
+            f"umask 077 && cat > {tmp} && chmod 600 {tmp} && mv {tmp} {dest}",
+            stdin_data=rendered.encode())
+    keys = [ln.split("=", 1)[0] for ln in rendered.splitlines()
+            if "=" in ln and not ln.lstrip().startswith("#") and ln.strip()]
+    print(f"  [env] shipped {dest} ({len(keys)} keys): {', '.join(keys)}")
+
+
+def provision_docker_secret(target: str, name: str, env_var: str) -> None:
+    value = os.environ.get(env_var, "")
+    if not value.strip():
+        die(f"docker secret '{name}': env var {env_var} is missing/empty")
+
+    new_hash = hashlib.sha256(value.encode()).hexdigest()
+    probe = ssh_run(
+        target,
+        f"docker secret inspect {name} "
+        "--format '{{index .Spec.Labels \"checksum\"}}' 2>/dev/null || true",
+        check=True)
+    old_hash = probe.stdout.decode().strip()
+
+    if old_hash == new_hash:
+        print(f"  [skip]   docker secret {name} (unchanged)")
+        return
+
+    if old_hash:
+        rm = ssh_run(target, f"docker secret rm {name}", check=False)
+        if rm.returncode != 0:
+            die(f"docker secret {name}: value changed but removal failed — "
+                "it is probably referenced by a running service. Provision "
+                "under a versioned name (see vaultwarden_database_url_v2 "
+                "precedent) or scale the service down first.")
+        action = "update"
+    else:
+        action = "create"
+
+    ssh_run(target,
+            f"docker secret create --label checksum={new_hash} "
+            f"--label managed-by=woodpecker {name} -",
+            stdin_data=value.encode())
+    print(f"  [{action}] docker secret {name} (value via stdin)")
+
+
+def main() -> None:
+    if len(sys.argv) != 2:
+        die("usage: provision-stack.py <stack>")
+    stack = sys.argv[1]
+
+    stacks = load_manifest()
+    cfg = stacks.get(stack)
+    if cfg is None:
+        print(f"  [info] stack '{stack}' not in secrets-map.yaml — "
+              "legacy provisioning (deploy.yml case-entry) applies. Nothing to do.")
+        return
+
+    target = ssh_target()
+    print(f"==> provision-stack: {stack}")
+
+    template_rel = cfg.get("env_template")
+    dest_rel = cfg.get("env_dest")
+    if template_rel and not dest_rel:
+        die("env_template set but env_dest missing in manifest")
+    if template_rel:
+        rendered = render_template(os.path.join(REPO_ROOT, template_rel))
+        ship_env_file(target, rendered, dest_rel)
+
+    for name, env_var in (cfg.get("docker_secrets") or {}).items():
+        provision_docker_secret(target, name, env_var)
+
+    print(f"==> provision-stack: {stack} done")
+
+
+if __name__ == "__main__":
+    main()

deploy/stack-deploy.sh

@@ -20,14 +20,38 @@
 # All files in the folder matching *.yml or *.yaml are included.
 # Main file (<stack>.yml or <stack>.yaml) is always passed FIRST.
 # Remaining files are sorted and appended.
+#
+# Render pipeline (unified for all modes as of 2026-08-26):
+#   1. Render full compose YAML (with extras merged + env substituted) to a
+#      temp file.
+#   2. Run mount-guard.py against that temp file — checks every bind mount
+#      source path exists, and flags suspicious-looking empty Postgres data
+#      dirs, before anything touches Swarm.
+#   3. docker stack deploy -c <tempfile> <stack>
+#
+# Dollar-escaping (2026-08-26, see envparse.py comments for full detail):
+#   - Single-file path (no extras): envsubst does no '$' escaping of its
+#     own, and only Swarm's `docker stack deploy` interpolation pass runs
+#     downstream -> use export/export_merged (escapes '$' -> '$$' once).
+#   - Folder+extras path: `docker compose config` ALSO does its own '$'
+#     escaping on top of Swarm's -> use export_raw/export_raw_merged (no
+#     pre-escaping) or values get doubled twice. Getting this wrong
+#     silently corrupts any secret/hash containing '$' (confirmed impact:
+#     LITELLM keys truncated, IMMICH_KIOSK_BASICAUTH bcrypt hash mismatched).
 
 set -euo pipefail
 
 STACK="${1:?Usage: stack-deploy.sh <stack-name>}"
 DIR="/volume1/docker/compose-files"
 PY="$DIR/deploy/envparse.py"
+MOUNT_GUARD="$DIR/deploy/mount-guard.py"
 GLOBAL_ENV="$DIR/deploy/global.env"
 
+# ── Pre-flight: ensure local checkout is in sync with Gitea ─────────────────
+# Prevents deploying from a stale/diverged local tree (see incident 2026-08-26).
+# Invoked via `bash` explicitly so the tracked file's exec bit doesn't matter.
+bash "$DIR/deploy/git-guard.sh" || { echo "ERROR: git-guard check failed. Deploy aborted."; exit 1; }
+
 # ── Locate compose file(s) ──────────────────────────────────────────────────
 
 FOLDER="$DIR/$STACK"
@@ -80,6 +104,12 @@ for f in "${EXTRAS[@]:-}"; do
   [ -n "$f" ] && F_FLAGS+=(-f "$f")
 done
 
+# Whether the render will go through `docker compose config` (folder mode
+# with 1+ extras). This determines which escaping mode is correct — see
+# header comment and envparse.py for why these must differ.
+USES_COMPOSE_CONFIG=0
+[ "${#F_FLAGS[@]}" -gt 2 ] && USES_COMPOSE_CONFIG=1
+
 # ── Load env (global base + optional stack override) ─────────────────────────
 
 HINT="  Hint: 'secret not found' means Woodpecker hasn't provisioned secrets yet.\n  Trigger the pipeline: https://woodpecker.bryanmail.net\n"
@@ -89,19 +119,25 @@ STACK_EXISTS=0
 [ -f "$GLOBAL_ENV" ] && GLOBAL_EXISTS=1
 [ -f "$ENVFILE"    ] && STACK_EXISTS=1
 
+if [ "$USES_COMPOSE_CONFIG" -eq 1 ]; then
+  EXPORT_MODE="export_raw"; EXPORT_MERGED_MODE="export_raw_merged"
+else
+  EXPORT_MODE="export"; EXPORT_MERGED_MODE="export_merged"
+fi
+
 if [ "$GLOBAL_EXISTS" -eq 1 ] && [ "$STACK_EXISTS" -eq 1 ]; then
-  echo "    Env:     $GLOBAL_ENV + $ENVFILE (stack overrides global)"
+  echo "    Env:     $GLOBAL_ENV + $ENVFILE (stack overrides global) [$EXPORT_MERGED_MODE]"
-  eval "$(python3 "$PY" export_merged "$GLOBAL_ENV" "$ENVFILE")"
+  eval "$(python3 "$PY" "$EXPORT_MERGED_MODE" "$GLOBAL_ENV" "$ENVFILE")"
   VARS="$(python3 "$PY" vars_merged "$GLOBAL_ENV" "$ENVFILE")"
 
 elif [ "$GLOBAL_EXISTS" -eq 1 ]; then
-  echo "    Env:     $GLOBAL_ENV (no stack env)"
+  echo "    Env:     $GLOBAL_ENV (no stack env) [$EXPORT_MODE]"
-  eval "$(python3 "$PY" export "$GLOBAL_ENV")"
+  eval "$(python3 "$PY" "$EXPORT_MODE" "$GLOBAL_ENV")"
   VARS="$(python3 "$PY" vars "$GLOBAL_ENV")"
 
 elif [ "$STACK_EXISTS" -eq 1 ]; then
-  echo "    Env:     $ENVFILE (no global env)"
+  echo "    Env:     $ENVFILE (no global env) [$EXPORT_MODE]"
-  eval "$(python3 "$PY" export "$ENVFILE")"
+  eval "$(python3 "$PY" "$EXPORT_MODE" "$ENVFILE")"
   VARS="$(python3 "$PY" vars "$ENVFILE")"
 
 else
@@ -111,33 +147,44 @@ fi
 
 echo "    Vars:    $VARS"
 
-# ── Deploy ───────────────────────────────────────────────────────────────────
+# ── Render final compose YAML to a temp file ──────────────────────────────────
+
+RENDERED="$(mktemp /tmp/stack-deploy.XXXXXX.yml)"
+trap 'rm -f "$RENDERED"' EXIT
 
 if [ -n "$VARS" ]; then
-  if [ "${#F_FLAGS[@]}" -gt 2 ]; then
+  if [ "$USES_COMPOSE_CONFIG" -eq 1 ]; then
     # Folder mode with extras: merge via docker compose config
     docker compose "${F_FLAGS[@]}" config \
       | python3 "$PY" strip \
       | envsubst "$VARS" \
-      | docker stack deploy -c - "$STACK" \
+      > "$RENDERED"
-      || { printf "\n%b" "$HINT"; exit 1; }
   else
     # Single file
     envsubst "$VARS" < "$MAIN" \
       | python3 "$PY" strip \
-      | docker stack deploy -c - "$STACK" \
+      > "$RENDERED"
-      || { printf "\n%b" "$HINT"; exit 1; }
   fi
 else
-  if [ "${#F_FLAGS[@]}" -gt 2 ]; then
+  if [ "$USES_COMPOSE_CONFIG" -eq 1 ]; then
     docker compose "${F_FLAGS[@]}" config \
       | python3 "$PY" strip \
-      | docker stack deploy -c - "$STACK" \
+      > "$RENDERED"
-      || { printf "\n%b" "$HINT"; exit 1; }
   else
-    docker stack deploy -c "$MAIN" "$STACK" \
+    python3 "$PY" strip < "$MAIN" > "$RENDERED"
-      || { printf "\n%b" "$HINT"; exit 1; }
   fi
 fi
 
+# ── Pre-flight: bind mount paths exist + Postgres-data sanity ───────────────
+# See deploy/mount-guard.py for details. Blocks on missing paths or
+# suspicious-looking empty/uninitialized Postgres data directories (see
+# incident 2026-08-26: a wrong-but-existing empty bind path would have let
+# Postgres silently init a fresh DB while real data sat orphaned elsewhere).
+python3 "$MOUNT_GUARD" "$RENDERED" || { echo "ERROR: mount-guard check failed. Deploy aborted."; exit 1; }
+
+# ── Deploy ───────────────────────────────────────────────────────────────────
+
+docker stack deploy -c "$RENDERED" "$STACK" \
+  || { printf "\n%b" "$HINT"; exit 1; }
+
 echo "==> Done: $STACK"

deploy/update-woodpecker-ai.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+"""
+Temporary script to update .woodpecker.yml with PA_MCP_CLIENT_ID provisioning.
+Run once, then delete this file.
+"""
+import re
+import sys
+
+def main():
+    # Read the current .woodpecker.yml
+    with open('.woodpecker.yml', 'r') as f:
+        content = f.read()
+
+    # 1. Add PA_MCP_CLIENT_ID to the provision-secrets environment section
+    # Find VAULTWARDEN_DATABASE_URL and add PA_MCP_CLIENT_ID after it
+    if 'PA_MCP_CLIENT_ID:' not in content:
+        env_pattern = r'(VAULTWARDEN_DATABASE_URL:\s+from_secret: vaultwarden_database_url)'
+        env_replacement = r'\1\n      PA_MCP_CLIENT_ID:\n        from_secret: pa_mcp_client_id'
+        content = re.sub(env_pattern, env_replacement, content)
+        print("✓ Added PA_MCP_CLIENT_ID to provision-secrets environment")
+    else:
+        print("⊘ PA_MCP_CLIENT_ID already in environment section")
+
+    # 2. Replace the ai) case in provision-secrets commands
+    # Find "ai) echo ..." and replace with actual provisioning
+    if 'No Docker secrets for ai' in content:
+        ai_case_pattern = r'(\s+)(ai\))\s+echo\s+"[^"]*No Docker secrets for ai[^"]*";;'
+        ai_replacement = r'\1\2\n\1  ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh\n\1    create_or_update_secret \'pa_mcp_client_id\' \'${PA_MCP_CLIENT_ID}\'";;'
+        content = re.sub(ai_case_pattern, ai_replacement, content)
+        print("✓ Updated ai) case to provision pa_mcp_client_id secret")
+    else:
+        print("⊘ ai) case already updated")
+
+    # 3. Add mcpo/config.json sync in the deploy step
+    # Find the deploy scripts sync and add mcpo sync after it
+    if 'mcpo/config.json' not in content:
+        # Find the "Always sync deploy/ scripts first" section
+        deploy_pattern = r'(# Always sync deploy/ scripts first\s+rsync[^\n]+\n\s+deploy/[^\n]+\n)'
+        mcpo_sync = (
+            r'\1\n'
+            r'        # Sync mcpo/config.json to host\n'
+            r'        if [ -f "mcpo/config.json" ]; then\n'
+            r'          rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \\\n'
+            r'            mcpo/config.json root@${SWARM_MANAGER_IP}:/volume1/docker/mcpo/config.json\n'
+            r'        fi\n'
+        )
+        content = re.sub(deploy_pattern, mcpo_sync, content, flags=re.DOTALL)
+        print("✓ Added mcpo/config.json sync to deploy step")
+    else:
+        print("⊘ mcpo/config.json sync already present")
+
+    # Write the updated content
+    with open('.woodpecker.yml', 'w') as f:
+        f.write(content)
+
+    print("\n✓ .woodpecker.yml updated successfully")
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

guacamole/guacamole.env

@@ -0,0 +1,11 @@
+# Guacamole - Pattern B env (folder-mode)
+LDAP_HOST=homehub.bryanmail.net
+GUACAMOLE_LDAP_SEARCH_BIND_DN=AVB@bryanmail.net
+GUACAMOLE_LDAP_BIND_PASSWORD=***REDACTED***
+GUACAMOLE_OIDC_AUTHORIZATION_ENDPOINT=https://auth.bryanmail.net/application/o/authorize/
+GUACAMOLE_OIDC_JWKS_ENDPOINT=https://auth.bryanmail.net/application/o/guacamole/jwks/
+GUACAMOLE_OIDC_ISSUER=https://auth.bryanmail.net/application/o/guacamole/
+GUACAMOLE_OIDC_CLIENT_ID=***REDACTED***
+GUACAMOLE_OIDC_CLIENT_SECRET=***REDACTED***
+GUACAMOLE_OIDC_REDIRECT_URI=https://remote.bryanmail.net/
+GUACAMOLE_DB_PASSWORD=***REDACTED***

homeassistant/homeassistant.env

@@ -0,0 +1,2 @@
+# Home Assistant - Pattern B env (folder-mode)
+FRIGATE_RTSP_PASSWORD=***REDACTED***

homeassistant/homeassistant.yaml

@@ -40,7 +40,7 @@ services:
 
   frigate-nvr:
     hostname: frigate
-    image: ghcr.io/blakeblackshear/frigate:0.17.1
+    image: ghcr.io/blakeblackshear/frigate:0.17.2
     cap_add:
       - CAP_PERFMON
       - SYS_ADMIN

immich/hwaccel.ml.yml

@@ -1,18 +1,2 @@
-# Hardware-accelerated machine learning extension for immich-machine-learning
+# This file is deprecated - hardware acceleration volumes are now in immich.yml
-# Active backend: openvino
+# Keeping for reference only
-# See https://immich.app/docs/features/ml-hardware-acceleration
-
-services:
-  openvino:
-    volumes:
-      - /dev/bus/usb:/dev/bus/usb
-      - /dev/dri:/dev/dri
-    hostname: immich-ml
-    networks:
-      - traefik_backend
-
-networks:
-  postgresql_db-backend:
-    external: true
-  traefik_backend:
-    external: true

immich/hwaccel.transcoding.yml

@@ -1,17 +1,2 @@
-# Hardware-accelerated transcoding extension for immich-server
+# This file is deprecated - hardware acceleration volumes are now in immich.yml
-# Active backend: quicksync
+# Keeping for reference only
-# See https://immich.app/docs/features/hardware-transcoding
-
-services:
-  quicksync:
-    devices:
-      - /dev/dri:/dev/dri
-    hostname: immich-transcode
-    networks:
-      - traefik_backend
-
-networks:
-  postgresql_db-backend:
-    external: true
-  traefik_backend:
-    external: true

immich/immich.env-example

@@ -0,0 +1,37 @@
+# Immich stack environment variables - EXAMPLE
+# Copy this to immich.env (untracked, see .gitignore) and fill in
+# IMMICH_KIOSK_BASICAUTH with a real value. Do NOT commit immich.env.
+#
+# Pattern C (partial): DB_PASSWORD as Docker secret (_FILE), IMMICH_KIOSK_BASICAUTH
+# in host .env (Traefik label).
+#
+# NOTE 2026-08-26: the paths below are the REAL, confirmed-correct values for
+# this homelab (verified via `docker service inspect immich_<svc> --format
+# '{{json .PreviousSpec.TaskTemplate.ContainerSpec.Mounts}}'` against actual
+# on-disk data). An earlier version of this file had plausible-looking but
+# WRONG generic paths (/volume1/docker/immich/db, /volume1/docker/immich/uploads),
+# which caused a real incident when immich.env was regenerated from this
+# template without checking against the live services first. If you ever
+# need to regenerate immich.env from this file, these paths should still be
+# correct — but if in doubt, re-verify with the PreviousSpec command above
+# before deploying.
+
+IMMICH_VERSION=release
+UPLOAD_LOCATION=/volume1/Immich-Photos/
+BULK_UPLOAD_LOCATION=/volume1/Immich-Photos/bulk-upload
+DB_DATA_LOCATION=/volume1/docker/immich-postgresql
+REDIS_DATA_LOCATION=/volume1/docker/immich/redis
+
+DB_USERNAME=immich
+DB_DATABASE_NAME=immich
+DB_HOSTNAME=immich_postgresql
+DB_PORT=5432
+
+REDIS_HOSTNAME=immich_redis
+REDIS_PORT=6379
+REDIS_DBINDEX=0
+
+IMMICH_TRUSTED_PROXIES=172.16.0.0/12
+IMMICH_TRAEFIK_HOST=immich.bryanmail.net
+IMMICH_KIOSK_HOST=immich-kiosk.bryanmail.net
+IMMICH_KIOSK_BASICAUTH=user:changeme

immich/immich.yml

@@ -10,6 +10,7 @@ services:
       - ${UPLOAD_LOCATION}:/data
       - /etc/localtime:/etc/localtime:ro
       - ${BULK_UPLOAD_LOCATION}:/bulk-upload
+      - /dev/dri:/dev/dri
     networks:
       - traefik_backend
       - postgresql_db-backend
@@ -39,7 +40,7 @@ services:
         - traefik.swarm.network=traefik_backend
       resources:
         limits:
-          cpus: '8'
+#          cpus: 8
           memory: 8G
       placement:
         constraints:
@@ -71,7 +72,7 @@ services:
         - traefik.enable=false
       resources:
         limits:
-          cpus: '8'
+#          cpus: 8
           memory: 8G
       placement:
         constraints:
@@ -92,7 +93,7 @@ services:
         - traefik.enable=false
       resources:
         limits:
-          cpus: '2'
+#          cpus: 2
           memory: 256M
       placement:
         constraints:
@@ -122,7 +123,7 @@ services:
     deploy:
       resources:
         limits:
-          cpus: '4'
+#          cpus: 4
           memory: 2G
       placement:
         constraints:
@@ -152,7 +153,7 @@ services:
         - traefik.http.middlewares.immich-kiosk-auth.basicauth.users=${IMMICH_KIOSK_BASICAUTH}
       resources:
         limits:
-          cpus: '1'
+ #         cpus: 1
           memory: 256M
 
 volumes:

maintenance/maintenance.env

@@ -0,0 +1 @@
+# Maintenance (Cronicle + Uptime Kuma) - Pattern B env (folder-mode)

maintenance/maintenance.yaml

@@ -66,6 +66,8 @@ services:
         max-size: "10m"
         max-file: "3"
     deploy:
+      restart_policy:
+        delay: 30s
       labels:
         - traefik.enable=true
         - traefik.http.routers.uptime.rule=Host(`uptime.${DOMAIN}`)

mealie/mealie.env

@@ -0,0 +1,7 @@
+# Mealie - Pattern B env (folder-mode); DB/LDAP bind passwords are Docker secrets
+POSTGRES_HOST=postgresql
+POSTGRES_PORT=5432
+LDAP_HOST=homehub.bryanmail.net
+LDAP_BASE_DN=DC=bryanmail,DC=net
+LDAP_QUERY_BIND=CN=AVB,OU=AAD Sync,OU=House Members,OU=Topside Pinnacle,DC=bryanmail,DC=net
+LDAP_ADMIN_GROUP=OU=AAD Sync,OU=House Members,OU=Topside Pinnacle,DC=bryanmail,DC=net

postgresql.yaml

@@ -2,8 +2,6 @@ version: "3.6"
 services:
   postgresql:
     stop_grace_period: 60s
-    env_file:
-      - postgresql.env
     command:
       - postgres
     environment:
@@ -68,8 +66,6 @@ services:
 
 
   pgadmin:
-    env_file:
-      - postgresql.env
     image: dpage/pgadmin4:latest
     environment:
       PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL}

postgresql/cutover/RUNBOOK.md

@@ -0,0 +1,244 @@
+# ADR-0001 Phase 3 Cutover — RUNBOOK
+
+**Status:** Ready to execute. All prerequisites (standby role, pg_hba.conf,
+architecture validation, dry-run drills) are complete — see companion note
+"ADR-0001 Dry-Run Debugging Log — Resume Point" for full history.
+
+**Scope:** Cut production traffic over from the single-instance legacy
+`postgresql` container (stack: `postgresql`, service: `postgresql`) to the
+new Patroni/etcd/HAProxy HA stack (`postgresqlha`), using Patroni's
+`standby_cluster` continuous-streaming mode — NOT a one-shot basebackup —
+so the pre-cutover write gap is near-zero.
+
+**Critical constraint:** This chat (Open WebUI + LiteLLM, stack `ai`) is
+itself a consumer of the database being migrated. **The chat will not
+survive the cutover if anything goes wrong that requires interactive
+debugging mid-procedure.** Every phase from "legacy read-only" onward is
+scripted with automatic rollback specifically so no human intervention via
+this chat is required to reach a safe state. `cutover.sh` and
+`rollback.sh` are both directly runnable on docker-2 via SSH/LXC-exec,
+independent of Woodpecker — **do not rely on a CI pipeline run to execute
+or recover from this**, since Woodpecker's own DB (via `WOODPECKER_DATABASE_DATASOURCE`)
+is itself a consumer and could be part of what's disrupted.
+
+---
+
+## 1. Dependency map (confirmed against live compose files)
+
+All consumers below reach the DB via the Swarm-DNS name `postgresql`
+(alias `db` also exists on the legacy service) on the external overlay
+network `postgresql_db-backend`. Confirmed by reading each stack's
+compose file directly — not assumed.
+
+| Stack | Service | How it depends on `postgresql` | Blast radius if broken |
+|---|---|---|---|
+| `ai` | `open-webui` | `WEBUI_DB_HOST=postgresql`, `DATABASE_URL=${OPEN_WEBUI_DATABASE_URL}` | **This chat.** Session/user/chat state. |
+| `ai` | `litellm` | `DATABASE_URL=${DATABASE_URL}` | LLM routing layer this chat calls through. |
+| `ai` | `n8n` | attached to network, no explicit `DB_*` vars found in `ai.yaml` | Unconfirmed — likely SQLite. **Preflight must verify live, not assume.** |
+| `ai` | `mcpo` / `mcpo-critical` | **NOT attached to `postgresql_db-backend`** | None. Confirmed no dependency — off-limits stack unaffected either way. |
+| `auth` | `authentik-server`, `authentik-worker` | `AUTHENTIK_POSTGRESQL__HOST=postgresql`, DB name `authentik` | SSO — breaks all authenticated services cluster-wide. |
+| `git` | `gitea-server` | attached to network; actual DB config lives in un-committed `git.env` | Preflight must inspect live env, don't assume driver. |
+| `woodpecker` | `woodpecker-server` | explicit DSN `postgres://${DB_USER}:${DB_PASS}@${POSTGRES_HOST}:${POSTGRES_PORT}/woodpecker` | CI/CD pipeline itself. **See special-case note above — do not depend on Woodpecker to run/recover this cutover.** |
+| `postgresql` (same stack) | `pgadmin`, `databasus` | same-stack, same network | Lower risk, co-located. |
+
+**Not yet individually confirmed by direct compose-file read** (ADR
+mentions ~13 total consumers): `immich`, `mealie`, `vaultwarden`,
+`guacamole`, `homeassistant`, `meshcentral`, `maintenance` (Cronicle /
+Uptime Kuma may only monitor, not store, in Postgres — verify). Do not
+trust this table as exhaustive — `preflight.sh` enumerates consumers
+**authoritatively** via `docker network inspect postgresql_db-backend`
+(live truth: every container actually attached right now) cross-checked
+against a repo-wide grep for `postgresql_db-backend` / `POSTGRES` /
+`DATABASE_URL` patterns (catches anything not currently running). Treat
+any mismatch between the two as a hard stop, not a warning.
+
+---
+
+## 2. Architecture summary
+
+- **New stack:** `postgresqlha` — 3x etcd (Raft quorum, one per node,
+  hostname-pinned), 2x Patroni/Spilo (`patroni-0` primary-labeled,
+  `patroni-1` replica-labeled, via `node.labels.pg-role` constraints),
+  1x HAProxy (`deploy.mode: global`, TCP health-routes to whichever
+  Patroni node answers `GET /primary` with 200).
+- **Bootstrap method:** `standby_cluster` (continuous streaming from live
+  legacy primary) — NOT `CLONE_WITH_BASEBACKUP`. Validated end-to-end in
+  `pgha-test` dry run: bootstrap, cascade replication, live write
+  propagation (<3s lag), promotion (zero data loss), and both graceful +
+  hard-crash failover drills post-promotion. See ADR-0001 note for full
+  validation detail.
+- **Auth prerequisite (already live in production):** `standby` role
+  (REPLICATION LOGIN) + matching `pg_hba.conf` line
+  (`host replication standby 10.0.4.0/24 scram-sha-256`) — created and
+  network-proven against the live `postgresql_postgresql` container.
+  Nothing further needed here.
+- **File:** `postgresql/cutover/postgresql-ha-staging.yaml` — the exact,
+  already-rewritten (standby_cluster, no CLONE_*) compose file this
+  cutover deploys. Not yet deployed for real prior to this procedure.
+
+### Why the phased order looks the way it does (the actual fix over the failed attempt)
+
+The prior failed attempt scaled legacy to 0 **before** HAProxy could
+answer to the `postgresql`/`db` aliases — a DNS gap where no backend
+answered at all for a window. The fix is not simply "add the alias
+first" in isolation — that alone introduces two *worse* failure modes:
+
+1. If HAProxy joins the alias before Patroni is genuinely promoted,
+   HAProxy's own healthcheck (`GET /primary`, expects 200) correctly
+   reports the standby_leader as unhealthy (it's read-only, not a real
+   primary) — so any connection Swarm's round-robin DNS routes to
+   HAProxy during that window hits **zero healthy backends** and fails
+   outright. Worse than the original gap, not better.
+2. If legacy is still accepting writes at the exact moment of promotion,
+   any write landing on legacy after the replica's last synced LSN is
+   **silently lost** the instant Patroni becomes the new source of
+   truth — a split-brain data-loss risk, not just a DNS problem.
+
+The corrected order below closes both gaps: legacy is flipped read-only
+*before* promotion (closing gap #2, fully reversible with one command),
+and the alias is only added *after* HAProxy has something genuinely
+healthy to route to (closing gap #1, and incidentally also fixing the
+original DNS-gap bug since legacy is never removed from DNS until a
+healthy replacement already answers).
+
+---
+
+## 3. Phased cutover procedure
+
+Executed by `cutover.sh`, run directly on docker-2. Each phase has an
+explicit, automatic check; **any failed check triggers immediate
+invocation of `rollback.sh`** and the script exits nonzero — no manual
+intervention required to reach a safe state.
+
+| Phase | Action | Verification before proceeding | Rollback if this phase fails |
+|---|---|---|---|
+| 0 | Run `preflight.sh` (backup, catalog sanity, disk space, consumer enumeration) | All checks pass | Abort before touching anything live — nothing to roll back |
+| 1 | Deploy `postgresqlha` stack (`postgresql-ha-staging.yaml`) | All 6 services (`etcd-1/2/3`, `patroni-0`, `patroni-1`, `haproxy`) running; `patroni-1` reaches `standby_leader` via `/cluster` REST | `docker stack rm postgresqlha`, wipe fresh data dirs it created |
+| 2 | Wait for streaming lag = 0 (`patroni-0` replica of `patroni-1`, both streaming) | `/cluster` REST shows `lag: 0` on both nodes for 3 consecutive polls | Same as Phase 1 — nothing on legacy touched yet |
+| 3 | **Canary write/propagation check (pre-promotion, sanity only)** — insert a timestamped row into a disposable scratch table on legacy, confirm it appears on `patroni-0` (far cascade replica) within 5s | Row appears | Same as Phase 1 |
+| 4 | **Flip legacy read-only**: `ALTER SYSTEM SET default_transaction_read_only = on; SELECT pg_reload_conf();` against legacy | Confirm via `SHOW default_transaction_read_only;` returns `on`; confirm one more streaming-lag poll = 0 (drains any last in-flight write) | **Reverse immediately**: `ALTER SYSTEM SET default_transaction_read_only = off; SELECT pg_reload_conf();` — legacy resumes normal read-write service, consumers were never touched, no visible outage |
+| 5 | **Promote Patroni**: remove `standby_cluster` key from DCS via `etcdctl` (validated method — no dedicated `patronictl` subcommand exists) | `patroni-1` transitions `standby_leader` → `leader` within ~30s (dry run: ~15s); `pg_is_in_recovery()` returns `false` on `patroni-1` | Reverse Phase 4 (read-only off) — legacy is still the only thing anything is talking to, so this is a clean abort |
+| 6 | Confirm HAProxy backend health | `curl` the HAProxy stats page (`:7000/`, internal-only) or `GET /primary` directly against each Patroni node shows exactly one healthy backend (the new leader) | Reverse Phase 4 — legacy resumes r/w, new HA stack is simply torn down (it never took live traffic) |
+| 7 | **Add `postgresql`/`db` aliases to HAProxy** on `postgresql_db-backend` (deploy HAProxy with the alias config now, or `docker network connect --alias` if already running) | `getent hosts postgresql` from a disposable probe container on that network resolves to **both** legacy's IP and HAProxy's IP | Remove the alias (`docker network disconnect`/redeploy without it) — legacy is still answering `postgresql` alone the whole time, zero consumer impact |
+| 8 | **Canary write/propagation check through the alias** — connect to `postgresql:5432` (not directly to a Patroni node), write a timestamped row, confirm visible on both `patroni-0` and `patroni-1` | Round-trip succeeds, row visible on both nodes | Remove alias (Phase 7 rollback) — legacy still primary of record |
+| 9 | Verify each known consumer reconnects and can write through the alias — for each stack in the dependency map, run its lightest real health check (e.g., Authentik `ak healthcheck`, Gitea `/api/healthz`, Woodpecker `/healthz`, Open WebUI `/health`, LiteLLM `/health`) | All pass | Full rollback via `rollback.sh` (see below) — treat any consumer failure here as cutover failure, not a per-consumer issue to patch around |
+| 10 | **Stop legacy**: `docker service scale postgresql_postgresql=0` | HAProxy backend still healthy post-legacy-stop; `getent hosts postgresql` now resolves to HAProxy only; repeat consumer health checks from Phase 9 | Restore legacy: `docker service scale postgresql_postgresql=1`, confirm it rejoins DNS; since it was read-only since Phase 4 and never diverged from the new leader's data, no data reconciliation needed at this point — **but past this phase, treat rollback as symmetric-but-not-free**: HAProxy has been serving real production writes, so rolling back here means picking legacy back up as primary of record and tearing down `postgresqlha`, not simply reversing a flag. Use `rollback.sh`, do not improvise. |
+| 11 | Final verification pass — all consumers, disk space, `docker stack ps postgresqlha` clean, HAProxy stats page shows steady state | All green | N/A — cutover complete |
+
+**Design note on why the canary check happens twice (Phase 3 and Phase
+8):** Phase 3 is a pre-promotion sanity check that streaming genuinely
+works (matches the dry-run validation methodology). Phase 8 is the actual
+point of the exercise — proving a write through the *real client path*
+(the alias, exactly as every consumer will use it) round-trips
+correctly, not just that Patroni's internal replication works. Do not
+skip either on the assumption the other covers it.
+
+---
+
+## 4. Rollback procedure
+
+`rollback.sh` is standalone and callable independently **at any point**,
+not just as a `cutover.sh` failure handler. It detects current state and
+takes the minimum action to restore the pre-cutover configuration:
+
+1. If legacy is scaled to 0 → scale back to 1, wait for healthy.
+2. If `default_transaction_read_only` is `on` on legacy → set back to
+   `off`, reload conf.
+3. If HAProxy is holding the `postgresql`/`db` aliases → remove them
+   (redeploy HAProxy without the alias, or disconnect/reconnect the
+   network attachment).
+4. Tear down the `postgresqlha` stack entirely (`docker stack rm
+   postgresqlha`), wait for task shutdown (verify via `docker stack ps`
+   — don't assume immediate), then wipe its data dirs so a future
+   attempt starts clean (mirrors the `pgha-test` teardown precedent from
+   the dry run — data dirs must be wiped, not just the parent dir left
+   behind, since Swarm bind mounts don't auto-create missing host parent
+   dirs; use `mkdir -p` per-path afterward, no brace expansion in this
+   exec environment).
+5. Re-run the Phase 9-equivalent consumer health checks to confirm
+   everything is back on legacy and functioning.
+6. Print a clear final state summary (which phase it rolled back from,
+   what's now running, what if anything still needs manual follow-up).
+
+**If `rollback.sh` itself fails partway** (e.g., legacy won't scale back
+up): this is the one scenario requiring manual intervention. In that
+case:
+- Do **not** attempt further automation blind — check
+  `docker service logs postgresql_postgresql --tail 100` and `docker
+  stack ps postgresql --no-trunc` first.
+- The legacy data directory (`/volume1/docker/PostgreSQL/data-17`) is
+  never deleted by any script in this procedure — only `postgresqlha`'s
+  own dirs are wiped. Worst case, legacy's data is intact and it's a
+  matter of getting the container/service healthy again, not data
+  recovery.
+- If this chat (Open WebUI/LiteLLM) is unreachable because of this,
+  fall back to direct SSH: `ssh root@192.168.4.32`, then work from
+  `docker service ls`, `docker stack ps postgresql --no-trunc`, `docker
+  service logs postgresql_postgresql --tail 100 -f`.
+
+---
+
+## 5. Known gotchas carried forward from the dry run (still apply)
+
+- **`$(...)` vs `$$(...)`** in any `command:` block — Compose parses
+  `$(` as attempted `${VAR}` interpolation and fails. Already fixed in
+  `postgresql-ha-staging.yaml`; don't reintroduce if editing.
+- **`ETCD3_HOSTS` not `ETCD_HOSTS`** — our etcd v3.5.9 has the v2 API
+  disabled.
+- **Docker secrets are immutable** — rotating a secret's value does not
+  propagate to running containers or to Woodpecker's provisioning logic
+  until an actual pipeline run re-executes `create_or_update_secret`.
+  Not expected to matter for this cutover (no secret rotation planned),
+  flagged in case a rollback attempt considers regenerating one.
+- **LXC exec is scoped to docker-2 only** and has a ~30s timeout for
+  long-running commands (e.g. `docker service create` may exceed it even
+  though the operation succeeds) — check `docker service ps`/`docker
+  service logs --raw` separately rather than trusting a timeout as
+  failure. To reach containers on docker-1/docker-3, proxy through
+  another container on the same overlay network (e.g. `docker exec
+  <legacy> psql -h patroni-1 ...`).
+- **Shell globs inside `docker exec <container> <cmd>`** get expanded by
+  the *outer* shell — wrap in `sh -c '...'` with the glob inside quotes
+  if one is ever needed.
+- **`docker stack rm` returns immediately; tasks take several seconds to
+  actually exit** — always re-check `docker stack ps <name>` after a
+  brief wait before wiping any bind-mount data dirs.
+
+---
+
+## 6. File manifest
+
+All in `postgresql/cutover/` unless noted:
+
+- `RUNBOOK.md` — this file.
+- `preflight.sh` — pre-flight checks (backup, catalog sanity, disk
+  space, authoritative consumer enumeration). Run standalone before
+  `cutover.sh`; `cutover.sh` also invokes it as Phase 0.
+- `cutover.sh` — the phased script described above, Phases 1–11,
+  automatic rollback on any failed check.
+- `rollback.sh` — standalone, idempotent, callable independently at any
+  point (also invoked automatically by `cutover.sh` on failure).
+- `postgresql-ha-staging.yaml` — the HA stack compose file this
+  procedure deploys (already written, standby_cluster design, not yet
+  deployed for real prior to this procedure).
+- `post_init_wrapper.sh` — Spilo post_init override (superuser
+  placeholder role fix), referenced by `postgresql-ha-staging.yaml`.
+- `test/pgha-dryrun.yaml` — the disposable dry-run design, kept for
+  reference/future re-testing. Not part of the real cutover path.
+- `../haproxy.cfg` — HAProxy config (Patroni-aware TCP router), shared
+  between steady-state and this cutover; no changes needed for the
+  cutover itself beyond the alias timing described above.
+
+---
+
+## 7. Post-cutover follow-up (manual, not scripted)
+
+- Confirm `postgresql_postgresql` (legacy service) can eventually be
+  fully decommissioned (remove from `postgresql.yaml`) once the new
+  stack has run stable for a reasonable burn-in period — not part of
+  this procedure, a deliberate later step.
+- Re-point `postgresql.yaml`'s `pgadmin`/`databasus` services at the new
+  topology if they need updating (they connect via the same
+  `postgresql` alias, so likely no change needed — verify post-cutover).
+- Revisit the deferred PG18/Spilo major-version upgrade project (see
+  ADR-0001 note) — explicitly out of scope here.

postgresql/cutover/cutover.sh

@@ -0,0 +1,921 @@
+#!/usr/bin/env bash
+# ADR-0001 Phase 3 Cutover — cutover.sh
+#
+# Run DIRECTLY on docker-2 via SSH (NOT via the LXC-exec MCP tool — several
+# phases here run well past its ~30s timeout):
+#
+#   ssh root@192.168.4.32
+#   bash /volume1/docker/compose-files/postgresql/cutover/cutover.sh
+#
+# Implements RUNBOOK.md's 11 phases. Any failed check triggers immediate,
+# automatic invocation of rollback.sh and the script exits nonzero — no
+# manual intervention should be required to reach a safe state. Designed
+# to be run unattended, but given the blast radius (production DB, this
+# chat's own dependency on it, ~13 consumers) watching it live the first
+# time is strongly recommended anyway.
+#
+# ── Design notes on HOW checks are implemented (read before editing) ──────
+#
+# Node locality: plain `docker exec` only works for containers whose task
+# is actually scheduled on the node you run it from — Swarm does NOT proxy
+# exec across nodes. patroni-0/1/haproxy/etcd-1/3 may land on docker-1 or
+# docker-3, not docker-2. Two techniques are used throughout to work
+# around this, both already validated in the ADR-0001 dry run:
+#   1. Legacy's own container IS pinned to docker-2 (see postgresql.yaml:
+#      node.hostname == docker-2) and shares postgresql_db-backend with
+#      every HA-stack container — so `docker exec <legacy_cid> ...` is
+#      always available locally and can reach any HA container by its
+#      Swarm-DNS hostname (patroni-0, patroni-1, haproxy, etcd-2, etc.).
+#      This is the SAME proxy pattern documented in the ADR-0001 note
+#      ("docker exec <legacy container> psql -h patroni-1 -p 5432 ...").
+#   2. `docker service ps <service>` and `docker service logs <service>`
+#      work cluster-wide from any manager regardless of which node the
+#      task landed on — used for stability/health signal checks that
+#      don't need to reach inside the container.
+# Patroni REST calls (port 8008) are made via bash's `/dev/tcp` raw-socket
+# trick executed inside the legacy container, NOT via curl — this avoids
+# assuming curl is present in the (Debian-slim) postgres:17 legacy image.
+#
+# ⚠️  DEVIATION FLAGGED FOR OPERATOR REVIEW — Phase 5 (promotion):
+# The ADR-0001 dry run promoted by removing the standby_cluster key
+# directly via etcdctl (patronictl had an unrelated config-loader quirk
+# in that environment). This script instead uses Patroni's own official,
+# documented REST API mechanism: `PATCH /config` with `{"standby_cluster":
+# null}`, which is Patroni's sanctioned way to remove a dynamic-config
+# key (see Patroni docs, REST API section). This avoids needing jq/python
+# inside the minimal etcd image to hand-edit YAML, and avoids the
+# patronictl quirk entirely — but it was NOT the exact mechanism proven
+# in the dry run. A manual fallback (raw etcdctl, matching the dry-run
+# method) is documented below and in RUNBOOK.md in case this PATCH
+# doesn't behave as expected. Recommend a low-stakes smoke test (e.g. a
+# GET /config call, done automatically below before the real PATCH) and
+# operator attention specifically during this phase.
+#
+# Manual fallback for Phase 5 if the PATCH approach ever needs bypassing:
+#   docker exec <etcd-2_cid> etcdctl --endpoints=http://etcd-1:2379,http://etcd-2:2379,http://etcd-3:2379 \
+#     get /service/postgres-ha/config --print-value-only
+#   (hand-edit the returned YAML to remove the standby_cluster block, then)
+#   docker exec <etcd-2_cid> etcdctl --endpoints=... put /service/postgres-ha/config -- "<edited YAML>"
+# etcd-2 is guaranteed to be on docker-2 (node.hostname == docker-2
+# constraint in both staging/final yaml), so this exec always works
+# locally if needed.
+#
+# ── FIX LOG (post-first-real-run) ──────────────────────────────────────
+# 2026 run #1 aborted at Phase 1's node-label check with a false negative
+# despite labels genuinely being set (docker-2=primary, docker-3=replica,
+# confirmed via `docker node inspect --format '{{json .Spec.Labels}}'`).
+# Root cause: the original check used
+#   docker node inspect {} --format '{{.Spec.Labels.pg-role}}'
+# — Go's template engine cannot parse a bare `.pg-role` field reference
+# (the hyphen is invalid in that position), so the command errored
+# silently (stderr redirected to /dev/null) and always returned empty,
+# regardless of actual label state. FIXED below by switching to
+# `docker node ls --filter "node.label=pg-role=primary"`, which uses
+# Docker's own filter syntax instead of Go template field access and
+# sidesteps the hyphen problem entirely.
+# Separately, rollback.sh's automatic consumer re-verification (triggered
+# by this false-negative rollback) flagged woodpecker's healthz as
+# "failed" because it returned HTTP 204 — a legitimate, common
+# "healthy, no content" response for that kind of endpoint. The health
+# checks below (and the matching ones in rollback.sh) now accept any 2xx
+# status code, not just literal 200.
+#
+# 2026 run #2 hit TWO further real bugs in Phase 1/2, both fixed below:
+#   a) Phase 1 hardcoded patroni-1 as the expected standby_leader. The
+#      Patroni/etcd bootstrap-race winner is actually nondeterministic —
+#      it's a lock race, not something the compose file controls. The
+#      ADR-0001 dry run happened to have patroni-1 win it, but the
+#      earlier real production attempt (see ADR-0001 note) had patroni-0
+#      win instead — proof both outcomes genuinely occur. A check that
+#      only ever accepts patroni-1 fails outright half the time even when
+#      the cluster is perfectly healthy. FIXED by polling BOTH patroni-0
+#      AND patroni-1 every iteration; whichever reports standby_leader
+#      first is captured as $LEADER_HOST, the other becomes
+#      $REPLICA_HOST. Every phase from here on (2,3,4,5,6,8,10) uses
+#      these two variables instead of a hardcoded hostname.
+#   b) Phase 1's wait window (240s) and Phase 2's (300s) were both far
+#      shorter than the real observed basebackup duration for this
+#      cluster's ~42GB dataset — prior polling (both the dry run and the
+#      earlier real attempt) showed a full basebackup_fast_xlog copy
+#      takes roughly 8-11 minutes. The role only flips to
+#      standby_leader/replica AFTER the copy fully completes, so both
+#      timeouts could fire (and did) while a completely legitimate
+#      basebackup was still in progress, triggering a false-negative
+#      auto-rollback mid-copy. FIXED: both windows extended (and later
+#      re-tuned further by direct operator edits in git — see commit
+#      history for exact current values of PHASE1_MAX_WAIT/
+#      PHASE2_MAX_WAIT) with a per-poll `du -sh` (timeout-guarded, since
+#      du has been observed to hang on CephFS under heavy concurrent
+#      write load) logged on every iteration, so progress is directly
+#      observable instead of a silent binary wait.
+#
+# 2026 run #3 follow-up (operator request, no new failure observed yet):
+#   Phase 2's cascade-replica bootstrap adds an EXTRA network/streaming
+#   hop on top of the same basebackup workload Phase 1 already does
+#   (patroni-0/1's loser bootstraps FROM the winner, not from legacy
+#   directly) — plausibly slower than Phase 1's copy, not just equal to
+#   it. Phase 2's window was extended further than Phase 1's for this
+#   reason (see current PHASE2_MAX_WAIT value below).
+#
+# 2026 run #4 — Phase 2's lag check was a SILENT NO-OP; found after a
+# real run passed Phase 1/2 cleanly (patroni-0 reached standby_leader,
+# patroni-1 basebackup completed, role flipped to "replica") but then
+# failed Phase 3's canary write with "Canary row did not propagate to
+# patroni-1 within 5s" moments later. Root cause: `patroni_lag()` queried
+# the REPLICA's OWN `/patroni` endpoint for lag/replication_state — but
+# Patroni only exposes per-connection replication lag (derived from
+# pg_stat_replication) on the LEADER/standby_leader side; a plain
+# replica's own `/patroni` response has no such field at all. So
+# $LAG_INFO was ALWAYS empty, and Phase 2's gate:
+#   [ "$REPLICA_ROLE" = "replica" ] && { [ -z "$LAG_INFO" ] || ... ; }
+# short-circuited permanently true on the `[ -z "$LAG_INFO" ]` branch —
+# the lag check never actually ran. Phase 2 degraded to "did the role
+# flip to replica 3 times in a row?", which can be true BEFORE the
+# replica has actually caught up on WAL backlog from its own basebackup,
+# especially right after a fresh cascade connection. Phase 3's real
+# canary write then landed in exactly that still-catching-up window.
+# FIXED: replaced patroni_lag() with cluster_member_lag_state(), which
+# queries the LEADER's `/cluster` endpoint (works from either node,
+# reflects live DCS state — same endpoint/shape directly verified in the
+# original ADR-0001 dry run: member entries look like
+# {"name": "patroni-1", "role": "replica", "state": "streaming", "lag": 0})
+# and extracts the SPECIFIC replica's "state"/"lag" fields from its own
+# member object (members are flat JSON objects with no nested braces, so
+# a bounded grep safely isolates exactly one member's fragment). Phase
+# 2's gate now requires literal state=streaming AND (lag=0 or lag field
+# absent — some responses omit it when lag is genuinely 0, per the
+# ADR-0001 note), not just "field happened to be empty because we asked
+# the wrong node." Phase 4's (informational, non-gating) lag log line
+# updated to use the same corrected function for consistency.
+#
+# 2026 run #5 — BOTH nodes stuck forever on "waiting for standby_leader
+# to bootstrap", never even attempting to race for the role, on a run
+# that started completely fresh (operator confirmed no other changes).
+# Root cause: a PRIOR run had been interrupted (operator stopped it
+# short) without a clean rollback.sh pass, leaving the etcd data
+# directories (etcd-1/2/3-data) non-empty on disk even though the
+# postgresqlha STACK itself was already gone. etcd's own persisted raft
+# state still had /service/postgres-ha/initialize set (a real system
+# identifier from the earlier attempt) plus a /service/postgres-ha/status
+# key listing old replication slots — so when the new run's patroni-0/1
+# containers (with genuinely fresh, empty PGDATA) came up against this
+# NOT-actually-fresh etcd, Patroni correctly concluded "this cluster
+# already exists, someone else must already be the leader" and both
+# nodes deferred forever waiting for a leader that could never appear
+# (nobody actually holds the lock). rollback.sh's own data-dir wipe only
+# runs when it detects the postgresqlha stack IS currently present (its
+# Case D/E paths) — if the stack was already gone by the time rollback
+# logic ran (e.g. an operator-initiated stop before rollback.sh got a
+# chance to run), its Case A path does NOT wipe data dirs, leaving
+# exactly this trap for the next run. FIXED: added a "Pre-Phase-0" check
+# (below, before preflight.sh even runs) that detects a still-present
+# postgresqlha stack OR non-empty etcd-*/patroni-*-data directories left
+# over from a prior run, and — since this is destructive (wipes data) and
+# the operator explicitly asked for this to be an interactive decision,
+# not silently automatic — PROMPTS before cleaning up. See that section
+# for the exact detection logic, the prompt itself, and the
+# AUTO_CLEANUP=yes escape hatch for unattended re-runs where the operator
+# has already decided cleanup is always wanted.
+#
+# 2026 run #6 (addition, no new run yet) — SESSION LOGGING added. Every
+# prior investigation into a failed run depended entirely on someone
+# having a live terminal open and scrolled back far enough, or on the
+# operator's own memory. This is fragile, especially for the one
+# genuinely dangerous scenario every phase above is built to avoid ever
+# reaching manually: rollback.sh itself failing partway (see
+# trigger_rollback() below — this is the one case RUNBOOK.md section 4
+# says requires manual intervention). Without a persisted transcript,
+# investigating THAT scenario after the fact means reconstructing what
+# happened from partial memory of a scrollback buffer that may already be
+# gone. FIXED: this script now writes its ENTIRE stdout/stderr transcript
+# (all 11 phases, plus Pre-Phase-0 and Phase 0's preflight.sh output, plus
+# any triggered rollback.sh output) to a single timestamped file in
+# BACKUP_DIR — the SAME directory the pg_dumpall backup itself lands in,
+# so the backup and the transcript of the run that produced/needed it are
+# always sitting right next to each other. See "SESSION LOGGING" comment
+# below for implementation detail. See ADR-0001 note, Session Update 9.
+#
+# 2026 run #7 — After the underlying Ceph IOPS problem (tracked/fixed
+# separately, not a script issue) was resolved, a real run finally got
+# CLEANLY past Phase 1 AND Phase 2 (basebackup ~690s, streaming lag=0
+# confirmed) but failed Phase 3 within 6 seconds of Phase 2 passing:
+# "Canary row did not propagate to patroni-1 within 5s." This is the
+# SECOND time this exact Phase 3 failure has been seen with Phase 1/2
+# genuinely healthy beforehand (see 2026 run #4 above for the first,
+# which had a different — since-fixed — root cause in the lag check
+# itself). Investigating this run's session log revealed a diagnostic
+# gap: the per-attempt `psql -h ${REPLICA_HOST} ... SELECT 1 FROM
+# _cutover_canary ...` check discarded stderr entirely (`2>/dev/null`),
+# so when the row failed to appear, the log showed only "row not found"
+# with NO indication of WHY — a genuine multi-second replication delay
+# on this specific DDL+INSERT and an outright connection/auth/permission
+# error to patroni-1 look IDENTICAL in that log, and there was no way to
+# tell them apart after the fact. FIXED: each attempt's stderr is now
+# captured to /tmp/cutover_phase3_attempt_<N>.stderr (mirrors the
+# existing Phase 8 pattern below) AND echoed into the log per-attempt
+# alongside the (possibly empty) result, so a future failure's log alone
+# should show definitively whether this is a genuine propagation delay
+# or a real connection-level error. The final trigger_rollback() message
+# on failure also now includes the last non-empty stderr seen, so it's
+# visible directly in the FAIL line without needing to scroll back.
+#
+# 2026 run #8 — The 2026 run #7 stderr-capture fix immediately paid off:
+# the very next run's Phase 3 failure log showed, on every single one of
+# 5 attempts:
+#   Password for user PGadmin:
+#   psql: error: connection to server at "patroni-1" (...), port 5432
+#   failed: fe_sendauth: no password supplied
+# This was NEVER a replication/timing issue at all. ROOT CAUSE: legacy's
+# pg_hba.conf requires scram-sha-256 for any non-local connection ("host
+# all all all scram-sha-256"), and patroni-0/patroni-1 (per
+# postgresql-ha-staging.yaml) use PGUSER_SUPERUSER=PGadmin with the SAME
+# postgresql_password Docker secret as legacy's own PGadmin. Every remote
+# `psql -h <patroni-node> -U "$POSTGRES_USER" ...` call in this script
+# had ALWAYS omitted a password entirely — this bug existed from the
+# script's very first version, it simply had never been reached with
+# working diagnostics until the run #7 fix exposed it. Checked the WHOLE
+# script for this pattern, not just Phase 3 — found the IDENTICAL missing
+# password in Phase 5 (the post-promotion `pg_is_in_recovery()` check on
+# $LEADER_HOST) and Phase 8 (the alias write + both visibility checks on
+# $LEADER_HOST/$REPLICA_HOST) — all three would have failed the exact
+# same way if ever reached. FIXED: read the postgresql_password secret
+# ONCE (via `docker exec "$LEGACY_CID" cat /run/secrets/postgresql_password`,
+# same secret/mechanism legacy itself already uses) into $ADMIN_PGPASSWORD
+# early in Phase 3, then pass it to every remote `psql -h <node>` call via
+# `docker exec -e PGPASSWORD="$ADMIN_PGPASSWORD" ...` — using docker exec's
+# own -e flag rather than splicing the password into the bash -c string,
+# specifically to avoid quoting/escaping hazards and to avoid the value
+# ever needing to appear inside this script's own string-construction
+# logic. NOT yet re-validated by an actual successful run past Phase 3.
+
+set -uo pipefail   # NOTE: deliberately not -e — every phase below checks
+                   # its own command's exit status explicitly so we can
+                   # invoke rollback with a specific, accurate reason
+                   # rather than an opaque `set -e` abort mid-phase.
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="/volume1/docker/compose-files"
+CUTOVER_DIR="${REPO_ROOT}/postgresql/cutover"
+HA_STACK="postgresqlha"
+LEGACY_STACK="postgresql"
+LEGACY_SERVICE="postgresql_postgresql"
+NETWORK_NAME="postgresql_db-backend"
+PATRONI_SCOPE="postgres-ha"
+BACKUP_DIR="/volume1/SMB-docker/backup"
+
+# ── SESSION LOGGING (added — see ADR-0001 note, Session Update 9) ────────
+# Persists this run's ENTIRE multi-phase stdout/stderr transcript to a
+# single timestamped file in BACKUP_DIR — the SAME location preflight.sh
+# writes the pg_dumpall backup to, per explicit operator request: the
+# backup and the transcript of the run that needed it always end up
+# sitting right next to each other. CUTOVER_SESSION_LOG is exported so
+# preflight.sh and rollback.sh (both invoked below as plain
+# `bash <script>` child processes, which inherit exported env vars)
+# automatically detect it's already open and APPEND to this same file
+# instead of opening their own — one merged transcript per session,
+# covering Pre-Phase-0 through Phase 11 AND any automatically-triggered
+# rollback.sh output, not three separate fragments. `tee` mirrors
+# everything to the real stdout/stderr too, so live console/SSH-visible
+# output is completely unchanged — this is purely additive persistence.
+mkdir -p "$BACKUP_DIR"
+CUTOVER_SESSION_TS="$(date +%Y%m%d-%H%M%S)"
+CUTOVER_SESSION_LOG="${BACKUP_DIR}/cutover-session-${CUTOVER_SESSION_TS}.log"
+export CUTOVER_SESSION_LOG
+exec > >(tee -a "$CUTOVER_SESSION_LOG") 2>&1
+
+# Always print exactly where the transcript lives and what the final exit
+# code was, even on an unexpected/unhandled exit (e.g. a bug in this
+# script itself, not one of the deliberate trigger_rollback() paths) —
+# the exit code and log location should never be a mystery to whoever
+# investigates later.
+_cutover_exit_trap() {
+  local rc=$?
+  echo "" >&2
+  echo "[cutover $(date +%H:%M:%S)] Session ending, exit code ${rc}. Full transcript: ${CUTOVER_SESSION_LOG}" >&2
+  if [ "$rc" -ne 0 ]; then
+    echo "[cutover $(date +%H:%M:%S)] Non-zero exit — if this did NOT go through a clean trigger_rollback() path (check the log above for a matching FAIL line), treat current state as unverified and consult RUNBOOK.md section 4 before assuming it's safe." >&2
+  fi
+}
+trap _cutover_exit_trap EXIT
+
+# Consumer services checked in Phase 9 — (service_name, external health URL
+# or empty if internal-only). Derived from the dependency map in
+# RUNBOOK.md section 1. Update this list if the dependency map changes.
+declare -A CONSUMER_HEALTH_URLS=(
+  ["auth_authentik-server"]="https://auth.bryanmail.net/-/health/live/"
+  ["git_gitea-server"]="https://git.bryanmail.net/api/healthz"
+  ["woodpecker_woodpecker-server"]="https://woodpecker.bryanmail.net/healthz"
+)
+# Services checked via Swarm task-stability only (no forwardAuth-free
+# external endpoint known / internal-only) — open-webui and litellm sit
+# behind Authentik forwardAuth on their public routes, so an external curl
+# would just hit a login redirect, not real app health.
+CONSUMER_STABILITY_ONLY=(
+  "ai_open-webui"
+  "ai_litellm"
+  "auth_authentik-worker"
+)
+
+log()  { echo "[cutover $(date +%H:%M:%S)] $*"; }
+warn() { echo "[cutover WARN $(date +%H:%M:%S)] $*" >&2; }
+
+# Returns success (0) if $1 looks like a 2xx HTTP status code.
+is_2xx() { [[ "$1" =~ ^2[0-9][0-9]$ ]]; }
+
+# Any hard failure calls this — invokes rollback.sh, then exits nonzero.
+# No manual step should be required after this runs.
+trigger_rollback() {
+  local reason="$1"
+  echo "" >&2
+  echo "[cutover FAIL $(date +%H:%M:%S)] ${reason}" >&2
+  echo "[cutover] Invoking rollback.sh automatically — no manual action should be required." >&2
+  bash "${CUTOVER_DIR}/rollback.sh" "${reason}"
+  local rc=$?
+  if [ $rc -ne 0 ]; then
+    echo "" >&2
+    echo "[cutover] ⚠️  rollback.sh ITSELF returned nonzero (${rc}). This is the one" >&2
+    echo "[cutover] scenario requiring manual intervention — see RUNBOOK.md section 4," >&2
+    echo "[cutover] 'If rollback.sh itself fails partway'. Do not blindly re-run scripts." >&2
+    echo "[cutover] Full transcript of everything up to this point (including rollback.sh's" >&2
+    echo "[cutover] own attempted output above) is preserved at: ${CUTOVER_SESSION_LOG}" >&2
+  fi
+  exit 1
+}
+
+# Get the container ID for the (always docker-2-local) legacy container.
+legacy_cid() {
+  docker ps -q --filter "name=${LEGACY_SERVICE}" | head -1
+}
+
+# Maps a Patroni hostname to its bind-mounted data directory on the host,
+# used for progress logging during basebackup (a full ~42GB copy has been
+# observed to take 8-11 minutes — see Phase 1/2 comments above and below).
+patroni_data_dir() {
+  case "$1" in
+    patroni-0) echo "/volume1/docker/PostgreSQL/patroni-0-data" ;;
+    patroni-1) echo "/volume1/docker/PostgreSQL/patroni-1-data" ;;
+    *) echo "" ;;
+  esac
+}
+
+# timeout-guarded `du -sh` — du has been observed to hang on CephFS while
+# a basebackup is writing heavily to the same volume. Returns "?" rather
+# than blocking the whole polling loop if it doesn't return quickly.
+data_dir_size() {
+  local dir="$1" out
+  out=$(timeout 10 du -sh "$dir" 2>/dev/null | awk '{print $1}')
+  echo "${out:-?}"
+}
+
+# Raw-socket HTTP GET via /dev/tcp, executed inside the legacy container.
+# Avoids assuming curl exists in the legacy postgres:17 image.
+# Args: host port path
+http_get_via_legacy() {
+  local host="$1" port="$2" path="$3" cid
+  cid="$(legacy_cid)"
+  docker exec "$cid" bash -c "
+    exec 3<>/dev/tcp/${host}/${port} || exit 99
+    printf 'GET ${path} HTTP/1.0\r\nHost: ${host}\r\nConnection: close\r\n\r\n' >&3
+    cat <&3
+  " 2>/dev/null
+}
+
+# Raw-socket HTTP PATCH with basic auth + JSON body, via /dev/tcp inside
+# the legacy container. Args: host port path username password json_body
+http_patch_via_legacy() {
+  local host="$1" port="$2" path="$3" user="$4" pass="$5" body="$6" cid auth
+  cid="$(legacy_cid)"
+  auth="$(docker exec "$cid" bash -c "printf '%s' '${user}:${pass}' | base64 -w0")"
+  local body_len=${#body}
+  docker exec "$cid" bash -c "
+    exec 3<>/dev/tcp/${host}/${port} || exit 99
+    printf 'PATCH ${path} HTTP/1.0\r\nHost: ${host}\r\nAuthorization: Basic ${auth}\r\nContent-Type: application/json\r\nContent-Length: ${body_len}\r\nConnection: close\r\n\r\n${body}' >&3
+    cat <&3
+  " 2>/dev/null
+}
+
+# Poll a Patroni node's /patroni endpoint for its reported role.
+# Args: hostname (e.g. patroni-1) -> prints role string (leader/replica/standby_leader/etc)
+patroni_role() {
+  local host="$1"
+  http_get_via_legacy "$host" 8008 "/patroni" | grep -o '"role"[^,}]*' | sed 's/.*: *"//;s/"$//'
+}
+
+# FIXED (see header "FIX LOG", 2026 run #4): a plain replica's OWN
+# /patroni endpoint has NO lag/replication_state field — that data only
+# exists on the LEADER/standby_leader side (derived from
+# pg_stat_replication). Query the LEADER's /cluster endpoint instead and
+# extract ONE SPECIFIC member's "state"/"lag" fields from its member
+# object. Member objects are flat (no nested braces), so a bounded grep
+# safely isolates exactly one member's JSON fragment before parsing.
+# Args: query_host (should be the LEADER/standby_leader) member_name
+#       (the node whose lag/state you actually want, e.g. the replica)
+# Prints: "state=<value> lag=<value>" (either may be <none> if absent)
+cluster_member_lag_state() {
+  local query_host="$1" member_name="$2" cluster_json member_json state lag
+  cluster_json="$(http_get_via_legacy "$query_host" 8008 "/cluster")"
+  member_json="$(echo "$cluster_json" | grep -o "{\"name\": *\"${member_name}\"[^}]*}")"
+  if [ -z "$member_json" ]; then
+    echo "state=<none> lag=<none>"
+    return
+  fi
+  state="$(echo "$member_json" | grep -o '"state": *"[^"]*"' | sed 's/.*"state": *"//;s/"$//')"
+  lag="$(echo "$member_json" | grep -o '"lag": *[0-9][0-9]*' | sed 's/.*: *//')"
+  echo "state=${state:-<none>} lag=${lag:-<none>}"
+}
+
+wait_for_stack_running() {
+  local stack="$1" expected_count="$2" max_wait="${3:-120}"
+  local waited=0
+  while [ "$waited" -lt "$max_wait" ]; do
+    local running
+    running=$(docker stack ps "$stack" --filter "desired-state=running" --format '{{.CurrentState}}' 2>/dev/null | grep -c '^Running')
+    if [ "$running" -ge "$expected_count" ]; then
+      return 0
+    fi
+    sleep 5
+    waited=$((waited + 5))
+  done
+  return 1
+}
+
+# ═══════════════════════════════════════════════════════════════════════
+log "=== ADR-0001 Phase 3 real cutover starting ==="
+log "Full session transcript (this run, all phases + any triggered rollback.sh output): ${CUTOVER_SESSION_LOG}"
+if [ -d "${REPO_ROOT}/.git" ]; then
+  GIT_HEAD="$(git -C "$REPO_ROOT" rev-parse HEAD 2>/dev/null || echo unknown)"
+  log "Repo HEAD at ${REPO_ROOT}: ${GIT_HEAD} (confirm this matches origin/main — see ADR-0001 note's local-checkout-drift lesson, Session Update 8, if unsure)"
+fi
+log "Chat dependency reminder: this chat (ai_open-webui/litellm) is itself"
+log "a consumer of the DB being migrated. If this script exits nonzero"
+log "before completing rollback, follow RUNBOOK.md section 4 manually via"
+log "direct SSH — do not assume this chat will remain usable to help."
+
+# ── Pre-Phase-0: stale-state detection & optional interactive cleanup ──
+# See header "FIX LOG", 2026 run #5, for the full incident this addresses:
+# a prior interrupted run can leave etcd-*/patroni-*-data non-empty on
+# disk (or the postgresqlha stack itself still deployed) even after the
+# stack is gone — and a fresh Patroni bootstrap against non-fresh etcd
+# state hangs BOTH nodes forever waiting for a leader that will never
+# appear, since etcd already "remembers" the cluster as initialized.
+# This check runs BEFORE preflight.sh's ~6+ minute pg_dumpall so a
+# doomed run doesn't waste that time before the operator gets a chance
+# to intervene. Destructive (wipes data dirs) — per explicit operator
+# request, this is an INTERACTIVE prompt, not silent automatic cleanup.
+# For unattended re-runs where cleanup should always happen without a
+# human present, set AUTO_CLEANUP=yes in the environment beforehand.
+log "--- Pre-Phase-0: checking for stale state from a prior interrupted run ---"
+
+STALE_FOUND=0
+STALE_DETAILS=()
+
+if docker stack ls --format '{{.Name}}' 2>/dev/null | grep -qx "${HA_STACK}"; then
+  STALE_FOUND=1
+  STALE_DETAILS+=("the ${HA_STACK} stack is already deployed (a previous run may still be in progress, or was stopped without a clean rollback.sh pass)")
+fi
+
+for d in etcd-1-data etcd-2-data etcd-3-data patroni-0-data patroni-1-data; do
+  DIR="/volume1/docker/PostgreSQL/${d}"
+  if [ -d "$DIR" ] && [ -n "$(ls -A "$DIR" 2>/dev/null)" ]; then
+    STALE_FOUND=1
+    STALE_DETAILS+=("${DIR} is non-empty (leftover from a prior run — etcd in particular will remember the old cluster as already-initialized and cause both Patroni nodes to hang forever waiting for a leader)")
+  fi
+done
+
+if [ "$STALE_FOUND" -eq 1 ]; then
+  warn "Detected possible stale state from a previous cutover attempt:"
+  for detail in "${STALE_DETAILS[@]}"; do
+    warn "  - ${detail}"
+  done
+  warn "Proceeding without cleaning this up is very likely to reproduce the"
+  warn "2026-08-05 incident where BOTH patroni-0 and patroni-1 sat forever"
+  warn "on 'waiting for standby_leader to bootstrap', never even attempting"
+  warn "to race for the role — see the ADR-0001 note, Session Update 9."
+
+  AUTO_CLEANUP="${AUTO_CLEANUP:-}"
+  DO_CLEANUP=""
+  if [ "$AUTO_CLEANUP" = "yes" ]; then
+    log "AUTO_CLEANUP=yes set in environment — cleaning up automatically, no prompt."
+    DO_CLEANUP="y"
+  elif [ -t 0 ]; then
+    read -rp "[cutover] Clean up this stale state now before proceeding? This tears down any existing ${HA_STACK} stack and wipes etcd-*/patroni-*-data. Legacy production data (postgresql_postgresql, data-17) is NEVER touched by this. [y/N] " DO_CLEANUP
+  else
+    trigger_rollback "Stale state detected from a prior run and this session is non-interactive (no tty to prompt) — refusing to guess. Re-run interactively to be prompted, set AUTO_CLEANUP=yes beforehand to skip the prompt, or clean up manually first (see ADR-0001 note, Session Update 9)."
+  fi
+
+  if [[ "$DO_CLEANUP" =~ ^[Yy] ]]; then
+    log "Operator confirmed — cleaning up stale state..."
+    if docker stack ls --format '{{.Name}}' 2>/dev/null | grep -qx "${HA_STACK}"; then
+      log "Tearing down existing ${HA_STACK} stack..."
+      docker stack rm "${HA_STACK}"
+      log "Waiting for tasks to actually finish shutting down (docker stack rm returns immediately)..."
+      for i in $(seq 1 24); do
+        REMAINING=$(docker stack ps "${HA_STACK}" 2>/dev/null | grep -c . || true)
+        if [ "$REMAINING" -eq 0 ]; then
+          break
+        fi
+        sleep 5
+      done
+      REMAINING=$(docker stack ps "${HA_STACK}" 2>/dev/null | grep -c . || true)
+      if [ "$REMAINING" -gt 0 ]; then
+        trigger_rollback "${HA_STACK} still shows ${REMAINING} task(s) after 120s wait during stale-state cleanup — unsafe to wipe data dirs while tasks may still be shutting down."
+      fi
+      log "Confirmed: ${HA_STACK} stack fully removed."
+    fi
+    for d in etcd-1-data etcd-2-data etcd-3-data patroni-0-data patroni-1-data; do
+      DIR="/volume1/docker/PostgreSQL/${d}"
+      if [ -d "$DIR" ]; then
+        rm -rf "$DIR"
+        mkdir -p "$DIR"
+        log "  wiped and recreated: ${DIR}"
+      fi
+    done
+    log "Stale-state cleanup complete — proceeding with a genuinely clean slate."
+  else
+    warn "Proceeding WITHOUT cleanup, per operator choice."
+    warn "If Phase 1 hangs with both nodes stuck on 'waiting for standby_leader"
+    warn "to bootstrap' and never attempting to race for the role, this stale"
+    warn "state is almost certainly why — stop this run and re-run choosing"
+    warn "cleanup, or run manually: docker stack rm ${HA_STACK}; then wipe"
+    warn "etcd-*/patroni-*-data under /volume1/docker/PostgreSQL/ before retrying."
+  fi
+else
+  log "No stale state detected from a prior run — proceeding."
+fi
+
+# ── Phase 0: preflight ────────────────────────────────────────────────
+log "--- Phase 0: preflight ---"
+if ! bash "${CUTOVER_DIR}/preflight.sh"; then
+  echo "[cutover] preflight.sh failed. Nothing has been touched — safe to stop here, no rollback needed." >&2
+  exit 1
+fi
+log "PASS: Phase 0 preflight"
+
+# ── Phase 1: deploy postgresqlha staging stack ─────────────────────────
+log "--- Phase 1: deploy postgresqlha (staging) ---"
+
+# FIXED (see header "FIX LOG"): use docker's own --filter syntax instead
+# of Go template field access on a hyphenated label name (which silently
+# fails to parse and always returns empty).
+PRIMARY_LABELED=$(docker node ls --filter "node.label=pg-role=primary" --format '{{.Hostname}}' | grep -c . || true)
+REPLICA_LABELED=$(docker node ls --filter "node.label=pg-role=replica" --format '{{.Hostname}}' | grep -c . || true)
+if [ "$PRIMARY_LABELED" -lt 1 ] || [ "$REPLICA_LABELED" -lt 1 ]; then
+  trigger_rollback "No node(s) found with pg-role=primary/replica labels — patroni-0/1 placement constraints cannot schedule. Set via: docker node update --label-add pg-role=primary <node> (and =replica on another). Nothing deployed yet, rollback is a no-op safety call."
+fi
+log "Node labels confirmed: ${PRIMARY_LABELED} node(s) with pg-role=primary, ${REPLICA_LABELED} node(s) with pg-role=replica."
+
+docker stack deploy -c "${CUTOVER_DIR}/postgresql-ha-staging.yaml" "${HA_STACK}"
+if [ $? -ne 0 ]; then
+  trigger_rollback "docker stack deploy of postgresql-ha-staging.yaml failed."
+fi
+
+log "Waiting for postgresqlha services to be running (etcd-1/2/3, patroni-0/1, haproxy = 6)..."
+if ! wait_for_stack_running "${HA_STACK}" 6 180; then
+  trigger_rollback "postgresqlha stack did not reach 6 running tasks within 180s."
+fi
+
+# FIXED (see header "FIX LOG" item a): the standby_leader bootstrap race
+# winner is nondeterministic — poll BOTH nodes, whichever wins becomes
+# $LEADER_HOST, the other becomes $REPLICA_HOST. Every later phase uses
+# these two variables instead of a hardcoded hostname.
+# FIXED (see header "FIX LOG" item b): window extended — a full ~42GB
+# basebackup has been observed to take 8-11 minutes, and the role only
+# flips AFTER it fully completes. Each poll logs both nodes' role AND
+# data-dir size so progress is directly observable instead of a silent
+# binary wait.
+log "Confirming standby_leader bootstrap completes (winner is nondeterministic — polling BOTH patroni-0 and patroni-1; a full ~42GB basebackup has been observed to take 8-11 minutes, so this window is generous)..."
+LEADER_HOST=""
+REPLICA_HOST=""
+PHASE1_MAX_WAIT=3600
+PHASE1_POLL_INTERVAL=15
+waited=0
+ROLE0=""
+ROLE1=""
+while [ "$waited" -lt "$PHASE1_MAX_WAIT" ]; do
+  ROLE0="$(patroni_role patroni-0)"
+  ROLE1="$(patroni_role patroni-1)"
+  SIZE0="$(data_dir_size "$(patroni_data_dir patroni-0)")"
+  SIZE1="$(data_dir_size "$(patroni_data_dir patroni-1)")"
+  log "patroni-0 role=${ROLE0:-<none>} data_dir=${SIZE0} | patroni-1 role=${ROLE1:-<none>} data_dir=${SIZE1} (${waited}s/${PHASE1_MAX_WAIT}s elapsed)"
+  if [ "$ROLE0" = "standby_leader" ]; then
+    LEADER_HOST="patroni-0"; REPLICA_HOST="patroni-1"; break
+  fi
+  if [ "$ROLE1" = "standby_leader" ]; then
+    LEADER_HOST="patroni-1"; REPLICA_HOST="patroni-0"; break
+  fi
+  sleep "$PHASE1_POLL_INTERVAL"
+  waited=$((waited + PHASE1_POLL_INTERVAL))
+done
+if [ -z "$LEADER_HOST" ]; then
+  trigger_rollback "Neither patroni-0 nor patroni-1 reached standby_leader role within ${PHASE1_MAX_WAIT}s. Last observed: patroni-0='${ROLE0}' patroni-1='${ROLE1}'."
+fi
+log "PASS: Phase 1 — ${LEADER_HOST} is standby_leader (bootstrap winner determined dynamically); ${REPLICA_HOST} will be the cascade replica"
+
+# ── Phase 2: wait for streaming lag = 0 ────────────────────────────────
+log "--- Phase 2: confirm streaming lag = 0 (cascade replica: ${REPLICA_HOST}) ---"
+# FIXED (see header "FIX LOG" item b): ${REPLICA_HOST} is itself
+# bootstrapping via a full basebackup from ${LEADER_HOST} — same
+# 8-11-min-for-~42GB precedent as Phase 1.
+# FURTHER EXTENDED (see header "FIX LOG", 2026 run #3 follow-up, operator
+# request): this cascade-replica bootstrap adds an EXTRA network/streaming
+# hop on top of the same basebackup workload, so its window is now longer
+# than Phase 1's rather than merely equal to it.
+# FIXED (see header "FIX LOG", 2026 run #4): the lag check itself was a
+# silent no-op — see cluster_member_lag_state() above for the real fix.
+# This gate now queries ${LEADER_HOST}'s /cluster view of ${REPLICA_HOST}
+# and requires literal state=streaming AND (lag=0 or lag absent).
+log "Note: ${REPLICA_HOST} is bootstrapping via a full basebackup from ${LEADER_HOST} — an extra hop beyond Phase 1's basebackup, so this window is deliberately longer than Phase 1's."
+CONSECUTIVE_ZERO=0
+PHASE2_MAX_WAIT=2400
+PHASE2_POLL_INTERVAL=15
+waited=0
+REPLICA_ROLE=""
+LAG_INFO=""
+while [ "$waited" -lt "$PHASE2_MAX_WAIT" ]; do
+  REPLICA_ROLE="$(patroni_role "$REPLICA_HOST")"
+  LAG_INFO="$(cluster_member_lag_state "$LEADER_HOST" "$REPLICA_HOST")"
+  SIZE="$(data_dir_size "$(patroni_data_dir "$REPLICA_HOST")")"
+  log "${REPLICA_HOST} role=${REPLICA_ROLE:-<none>} ${LAG_INFO} data_dir=${SIZE} (${waited}s/${PHASE2_MAX_WAIT}s elapsed)"
+  if [ "$REPLICA_ROLE" = "replica" ] && echo "$LAG_INFO" | grep -q "state=streaming" && ! echo "$LAG_INFO" | grep -qE "lag=[1-9]"; then
+    CONSECUTIVE_ZERO=$((CONSECUTIVE_ZERO + 1))
+  else
+    CONSECUTIVE_ZERO=0
+  fi
+  [ "$CONSECUTIVE_ZERO" -ge 3 ] && break
+  sleep "$PHASE2_POLL_INTERVAL"
+  waited=$((waited + PHASE2_POLL_INTERVAL))
+done
+if [ "$CONSECUTIVE_ZERO" -lt 3 ]; then
+  trigger_rollback "Streaming lag never reached 3 consecutive zero/streaming polls on ${REPLICA_HOST} within ${PHASE2_MAX_WAIT}s. Last state: role=${REPLICA_ROLE} ${LAG_INFO}"
+fi
+log "PASS: Phase 2 — streaming confirmed, lag=0/streaming for 3 consecutive polls on ${REPLICA_HOST}"
+
+# ── Phase 3: pre-promotion canary write/propagation check ─────────────
+log "--- Phase 3: pre-promotion canary (legacy -> ${REPLICA_HOST} cascade replica) ---"
+LEGACY_CID="$(legacy_cid)"
+if [ -z "$LEGACY_CID" ]; then
+  trigger_rollback "Could not find legacy container for canary write."
+fi
+
+# FIXED (see header "FIX LOG", 2026 run #8): every remote `psql -h
+# <patroni-node>` call in Phases 3/5/8 below needs a password — legacy's
+# pg_hba.conf requires scram-sha-256 for any non-local connection, and
+# patroni-0/patroni-1 use the SAME PGadmin superuser + SAME
+# postgresql_password secret as legacy itself (per
+# postgresql-ha-staging.yaml). This was previously never supplied at all,
+# so every one of these remote calls had always failed with
+# "fe_sendauth: no password supplied" — masked until the run #7
+# stderr-capture fix exposed it. Read the secret ONCE here (same
+# secret/mechanism legacy already uses) and pass it to every remote call
+# below via `docker exec -e PGPASSWORD=...` (NOT spliced into the bash -c
+# string, to avoid quoting hazards).
+ADMIN_PGPASSWORD=$(docker exec "$LEGACY_CID" cat /run/secrets/postgresql_password 2>/dev/null)
+if [ -z "$ADMIN_PGPASSWORD" ]; then
+  trigger_rollback "Could not read postgresql_password secret needed for remote psql -h <patroni-node> calls in Phases 3/5/8. Nothing changed yet."
+fi
+
+CANARY_VAL="cutover-$(date +%s)"
+# FIXED (see header "FIX LOG", 2026 run #7): this write's stderr used to
+# be discarded entirely; now captured and logged on failure so a genuine
+# write error to LEGACY (as opposed to a downstream propagation issue)
+# is visible instead of only surfacing as a generic "failed to write".
+CANARY_WRITE_OUT=$(docker exec "$LEGACY_CID" bash -c "psql -U \"\$POSTGRES_USER\" -c \"CREATE TABLE IF NOT EXISTS _cutover_canary (val text, ts timestamptz default now()); INSERT INTO _cutover_canary(val) VALUES ('${CANARY_VAL}');\"" 2>&1)
+CANARY_WRITE_RC=$?
+if [ "$CANARY_WRITE_RC" -ne 0 ]; then
+  log "Canary write to legacy — psql output: ${CANARY_WRITE_OUT}"
+  trigger_rollback "Failed to write pre-promotion canary row to legacy. psql output logged above this FAIL line."
+fi
+# FIXED (see header "FIX LOG", 2026 run #7): the propagation check below
+# used to discard stderr entirely (2>/dev/null) — this made a genuine
+# replication delay and an outright connection/auth error to
+# ${REPLICA_HOST} indistinguishable in the log, and this exact failure
+# ("Canary row did not propagate ... within 5s") had already been seen
+# TWICE (2026 run #4, root-caused as the lag-check bug; 2026 run #7,
+# root-caused as the missing PGPASSWORD below — see run #8) with no way
+# to tell which kind of failure either one actually was after the fact.
+# Each attempt's stderr is now captured to a numbered file (same pattern
+# already used by Phase 8 below) AND echoed into the log per-attempt, so
+# a future failure should be self-explanatory from the log alone.
+# FIXED (see header "FIX LOG", 2026 run #8): added
+# `-e PGPASSWORD="$ADMIN_PGPASSWORD"` to the docker exec — this remote
+# psql -h call had NEVER had a password supplied before.
+FOUND=0
+LAST_CANARY_CHECK_ERR=""
+for i in $(seq 1 5); do
+  rm -f "/tmp/cutover_phase3_attempt_${i}.stderr"
+  RESULT=$(docker exec -e PGPASSWORD="$ADMIN_PGPASSWORD" "$LEGACY_CID" bash -c "psql -h ${REPLICA_HOST} -U \"\$POSTGRES_USER\" -tAc \"SELECT 1 FROM _cutover_canary WHERE val = '${CANARY_VAL}';\"" 2>"/tmp/cutover_phase3_attempt_${i}.stderr")
+  CANARY_CHECK_ERR="$(cat "/tmp/cutover_phase3_attempt_${i}.stderr" 2>/dev/null)"
+  if [ -n "$CANARY_CHECK_ERR" ]; then
+    log "  attempt ${i}/5 on ${REPLICA_HOST}: result='${RESULT:-<empty>}' stderr: ${CANARY_CHECK_ERR}"
+    LAST_CANARY_CHECK_ERR="$CANARY_CHECK_ERR"
+  else
+    log "  attempt ${i}/5 on ${REPLICA_HOST}: result='${RESULT:-<empty>}' (no stderr)"
+  fi
+  if [ "$RESULT" = "1" ]; then
+    FOUND=1
+    rm -f /tmp/cutover_phase3_attempt_*.stderr
+    break
+  fi
+  sleep 1
+done
+if [ "$FOUND" -ne 1 ]; then
+  trigger_rollback "Canary row did not propagate to ${REPLICA_HOST} within 5s. standby_cluster streaming is not working as expected. Last psql stderr seen (empty means every attempt connected cleanly and simply found no row yet — a genuine propagation delay, not a connection error): '${LAST_CANARY_CHECK_ERR}'. Full per-attempt detail logged above; raw files also left at /tmp/cutover_phase3_attempt_*.stderr on docker-2 for inspection."
+fi
+log "PASS: Phase 3 — canary write propagated to ${REPLICA_HOST} within 5s"
+
+# ── Phase 4: flip legacy to read-only ───────────────────────────────────
+log "--- Phase 4: flip legacy read-only ---"
+docker exec "$LEGACY_CID" bash -c 'psql -U "$POSTGRES_USER" -c "ALTER SYSTEM SET default_transaction_read_only = on; SELECT pg_reload_conf();"' >/dev/null 2>&1
+if [ $? -ne 0 ]; then
+  trigger_rollback "Failed to set default_transaction_read_only=on on legacy."
+fi
+RO_STATE=$(docker exec "$LEGACY_CID" bash -c 'psql -U "$POSTGRES_USER" -tAc "SHOW default_transaction_read_only;"' 2>/dev/null | tr -d '[:space:]')
+if [ "$RO_STATE" != "on" ]; then
+  trigger_rollback "Verified read-only state is '${RO_STATE}', expected 'on'. Reversing immediately."
+fi
+log "Legacy confirmed read-only. Draining one more lag poll to confirm quiescence..."
+sleep 5
+LAG_INFO="$(cluster_member_lag_state "$LEADER_HOST" "$REPLICA_HOST")"
+log "Post-read-only lag check (${REPLICA_HOST}): ${LAG_INFO}"
+log "PASS: Phase 4 — legacy is read-only, no further writes possible there"
+
+# ── Phase 5: promote Patroni ─────────────────────────────────────────
+log "--- Phase 5: promote ${LEADER_HOST} (remove standby_cluster from DCS config) ---"
+log "⚠️  Highest-scrutiny phase — see script header for the documented deviation from the dry-run's proven method (using Patroni's official REST PATCH /config instead). Manual etcdctl fallback documented above and in RUNBOOK.md if needed."
+
+PATRONI_PASS=$(docker exec "$LEGACY_CID" cat /run/secrets/postgresql_patroni_password 2>/dev/null)
+if [ -z "$PATRONI_PASS" ]; then
+  # legacy container may not have this secret mounted — fall back to reading via the leader host itself if reachable, else hard-fail before touching anything
+  trigger_rollback "Could not read postgresql_patroni_password secret needed for the promotion PATCH call. Nothing promoted yet."
+fi
+
+log "Smoke-test: GET /config on ${LEADER_HOST} before attempting the PATCH..."
+CONFIG_CHECK=$(http_get_via_legacy "$LEADER_HOST" 8008 "/config")
+if ! echo "$CONFIG_CHECK" | grep -qi "standby_cluster"; then
+  trigger_rollback "GET /config on ${LEADER_HOST} did not show a standby_cluster key as expected before promotion — refusing to PATCH against an unexpected state. Raw response logged above for review."
+fi
+log "Smoke-test passed — standby_cluster key confirmed present pre-promotion."
+
+PATCH_RESPONSE=$(http_patch_via_legacy "$LEADER_HOST" 8008 "/config" "patroni" "$PATRONI_PASS" '{"standby_cluster": null}')
+log "PATCH /config response: ${PATCH_RESPONSE}"
+
+log "Waiting for ${LEADER_HOST} to transition standby_leader -> leader (expect ~15-30s per dry-run drills)..."
+PROMOTED=0
+for i in $(seq 1 12); do
+  ROLE="$(patroni_role "$LEADER_HOST")"
+  log "${LEADER_HOST} role: ${ROLE}"
+  if [ "$ROLE" = "leader" ]; then
+    PROMOTED=1
+    break
+  fi
+  sleep 5
+done
+if [ "$PROMOTED" -ne 1 ]; then
+  trigger_rollback "${LEADER_HOST} did not reach 'leader' role within 60s of the promotion PATCH. Last observed role: '${ROLE}'. Manual etcdctl fallback is documented in this script's header / RUNBOOK.md if the operator wants to investigate before accepting the automatic rollback."
+fi
+
+# FIXED (see header "FIX LOG", 2026 run #8): added
+# `-e PGPASSWORD="$ADMIN_PGPASSWORD"` — this remote psql -h call had
+# NEVER had a password supplied before (would have failed with
+# "fe_sendauth: no password supplied" exactly like Phase 3's did, had it
+# ever been reached).
+IN_RECOVERY=$(docker exec -e PGPASSWORD="$ADMIN_PGPASSWORD" "$LEGACY_CID" bash -c "psql -h ${LEADER_HOST} -U \"\$POSTGRES_USER\" -tAc \"SELECT pg_is_in_recovery();\"" 2>/dev/null | tr -d '[:space:]')
+if [ "$IN_RECOVERY" != "f" ]; then
+  trigger_rollback "${LEADER_HOST} reports role=leader but pg_is_in_recovery() = '${IN_RECOVERY}' (expected 'f'). Not trusting this as a genuine promotion."
+fi
+log "PASS: Phase 5 — ${LEADER_HOST} is genuine writable leader, pg_is_in_recovery=f"
+
+# ── Phase 6: confirm HAProxy would see a healthy backend ──────────────
+log "--- Phase 6: confirm healthy backend before touching DNS ---"
+PRIMARY_CHECK=$(http_get_via_legacy "$LEADER_HOST" 8008 "/primary")
+if ! echo "$PRIMARY_CHECK" | grep -q "200 OK"; then
+  trigger_rollback "${LEADER_HOST} does not answer 200 on GET /primary post-promotion. Raw response: ${PRIMARY_CHECK}"
+fi
+log "PASS: Phase 6 — ${LEADER_HOST} answers 200 on /primary, HAProxy's healthcheck will pass"
+
+# ── Phase 7: deploy final HAProxy with postgresql/db aliases ──────────
+log "--- Phase 7: deploy postgresql-ha-final.yaml (adds postgresql/db aliases) ---"
+docker stack deploy -c "${CUTOVER_DIR}/postgresql-ha-final.yaml" "${HA_STACK}"
+if [ $? -ne 0 ]; then
+  trigger_rollback "docker stack deploy of postgresql-ha-final.yaml failed."
+fi
+log "Waiting for haproxy service update to roll out..."
+sleep 15
+if ! wait_for_stack_running "${HA_STACK}" 6 90; then
+  trigger_rollback "postgresqlha stack did not return to 6 running tasks after haproxy alias update."
+fi
+
+log "Confirming DNS resolution of 'postgresql' now shows BOTH legacy and haproxy..."
+RESOLVE_OUTPUT=$(docker exec "$LEGACY_CID" bash -c "getent hosts postgresql" 2>/dev/null)
+log "Resolution: ${RESOLVE_OUTPUT}"
+NUM_IPS=$(echo "$RESOLVE_OUTPUT" | awk '{print $1}' | sort -u | wc -l)
+if [ "$NUM_IPS" -lt 2 ]; then
+  trigger_rollback "Expected 2 distinct IPs for 'postgresql' (legacy + haproxy) after alias deploy, found ${NUM_IPS}. Resolution output: ${RESOLVE_OUTPUT}"
+fi
+log "PASS: Phase 7 — 'postgresql' resolves to ${NUM_IPS} IPs (legacy + haproxy), alias handoff live"
+
+# ── Phase 8: canary write through the alias (retry-tolerant of legacy read-only hits) ──
+log "--- Phase 8: canary write/propagation THROUGH the shared alias ---"
+log "Note: during this window 'postgresql' round-robins between legacy (read-only)"
+log "and haproxy (writable). Write attempts landing on legacy will correctly fail"
+log "with a read-only error — that is EXPECTED, not a hard failure by itself."
+# FIXED (see header "FIX LOG", 2026 run #8): added
+# `-e PGPASSWORD="$ADMIN_PGPASSWORD"` to all three docker exec calls in
+# this phase — writes/reads that round-robin onto haproxy (i.e. land on
+# the promoted leader) are remote, non-local connections and had NEVER
+# had a password supplied before. (Writes/reads that happen to land on
+# legacy itself go through the "local ... trust" pg_hba.conf rule and
+# don't strictly need this, but supplying it is harmless either way.)
+CANARY2_VAL="cutover-alias-$(date +%s)"
+ALIAS_WRITE_OK=0
+for i in $(seq 1 10); do
+  docker exec -e PGPASSWORD="$ADMIN_PGPASSWORD" "$LEGACY_CID" bash -c "psql -h postgresql -U \"\$POSTGRES_USER\" -c \"INSERT INTO _cutover_canary(val) VALUES ('${CANARY2_VAL}');\"" >/tmp/cutover_phase8_attempt_$i.log 2>&1
+  if [ $? -eq 0 ]; then
+    ALIAS_WRITE_OK=1
+    break
+  fi
+  sleep 2
+done
+if [ "$ALIAS_WRITE_OK" -ne 1 ]; then
+  trigger_rollback "Never succeeded writing through the 'postgresql' alias after 10 attempts (~20s). See /tmp/cutover_phase8_attempt_*.log on docker-2 for the read-only/connection errors encountered."
+fi
+log "Alias write succeeded on attempt ${i}. Confirming visibility on BOTH ${LEADER_HOST} and ${REPLICA_HOST}..."
+VLEADER=$(docker exec -e PGPASSWORD="$ADMIN_PGPASSWORD" "$LEGACY_CID" bash -c "psql -h ${LEADER_HOST} -U \"\$POSTGRES_USER\" -tAc \"SELECT 1 FROM _cutover_canary WHERE val = '${CANARY2_VAL}';\"" 2>/dev/null)
+VREPLICA=$(docker exec -e PGPASSWORD="$ADMIN_PGPASSWORD" "$LEGACY_CID" bash -c "psql -h ${REPLICA_HOST} -U \"\$POSTGRES_USER\" -tAc \"SELECT 1 FROM _cutover_canary WHERE val = '${CANARY2_VAL}';\"" 2>/dev/null)
+if [ "$VLEADER" != "1" ] || [ "$VREPLICA" != "1" ]; then
+  trigger_rollback "Alias-written canary row not visible on both nodes (${LEADER_HOST}='${VLEADER}' ${REPLICA_HOST}='${VREPLICA}')."
+fi
+log "PASS: Phase 8 — real client-path write through the alias succeeded and is visible on both nodes"
+
+# ── Phase 9: verify all known consumers ────────────────────────────────
+log "--- Phase 9: verify known consumers reconnect/function ---"
+CONSUMER_FAIL=0
+for svc in "${!CONSUMER_HEALTH_URLS[@]}"; do
+  url="${CONSUMER_HEALTH_URLS[$svc]}"
+  CODE=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 10 "$url")
+  log "  ${svc} -> ${url} -> HTTP ${CODE}"
+  if ! is_2xx "$CODE"; then
+    warn "  ${svc} did not return a 2xx status (got ${CODE})"
+    CONSUMER_FAIL=1
+  fi
+done
+for svc in "${CONSUMER_STABILITY_ONLY[@]}"; do
+  RESTARTS=$(docker service ps "$svc" --filter "desired-state=running" --format '{{.CurrentState}}' 2>/dev/null | grep -c 'seconds ago\|Starting')
+  log "  ${svc} -> recent-restart signal count: ${RESTARTS} (0 expected if stable)"
+  if [ "$RESTARTS" -gt 0 ]; then
+    warn "  ${svc} shows a recent restart — may just be catching up post-cutover, but flagging."
+    CONSUMER_FAIL=1
+  fi
+done
+if [ "$CONSUMER_FAIL" -ne 0 ]; then
+  trigger_rollback "One or more consumers failed verification in Phase 9. See warnings above. Treating any consumer failure here as a cutover failure, not something to patch around live."
+fi
+log "PASS: Phase 9 — all known consumers verified"
+
+# ── Phase 10: stop legacy ──────────────────────────────────────────────
+log "--- Phase 10: stop legacy service ---"
+docker service scale "${LEGACY_SERVICE}=0"
+if [ $? -ne 0 ]; then
+  trigger_rollback "Failed to scale ${LEGACY_SERVICE} to 0."
+fi
+sleep 10
+log "Confirming HAProxy backend still healthy post-legacy-stop..."
+PRIMARY_CHECK2=$(http_get_via_legacy "$LEADER_HOST" 8008 "/primary" 2>/dev/null || true)
+# Note: legacy is now stopped, so http_get_via_legacy (which proxies via
+# the legacy container) will start failing itself shortly — this is
+# EXPECTED and is why this is the last check that uses that proxy method.
+log "Last legacy-proxied check (legacy is now down, this method retires after this call): ${PRIMARY_CHECK2}"
+
+log "Confirming 'postgresql' now resolves ONLY to haproxy..."
+# Use a disposable probe container instead of the (now-stopping) legacy container.
+PROBE_OUT=$(docker run --rm --network "${NETWORK_NAME}" alpine:3 sh -c "getent hosts postgresql" 2>/dev/null)
+log "Resolution post-legacy-stop: ${PROBE_OUT}"
+NUM_IPS2=$(echo "$PROBE_OUT" | awk '{print $1}' | sort -u | wc -l)
+if [ "$NUM_IPS2" -ne 1 ]; then
+  warn "Expected exactly 1 IP for 'postgresql' now that legacy is stopped, found ${NUM_IPS2}. This may just be DNS cache settling — re-check manually before treating as fatal, but NOT auto-rolling-back here since legacy has already been cleanly stopped (scale=1 would just resume it)."
+fi
+
+log "Re-running Phase 9 consumer checks post-legacy-stop..."
+CONSUMER_FAIL2=0
+for svc in "${!CONSUMER_HEALTH_URLS[@]}"; do
+  url="${CONSUMER_HEALTH_URLS[$svc]}"
+  CODE=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 10 "$url")
+  log "  ${svc} -> ${url} -> HTTP ${CODE}"
+  is_2xx "$CODE" || CONSUMER_FAIL2=1
+done
+if [ "$CONSUMER_FAIL2" -ne 0 ]; then
+  trigger_rollback "Consumer check failed AFTER stopping legacy. Per RUNBOOK.md, rollback past this point is asymmetric (restores legacy as primary of record, not just a flag flip) — invoking rollback.sh now rather than leaving this in a broken state."
+fi
+log "PASS: Phase 10 — legacy stopped, consumers still healthy"
+
+# ── Phase 11: final verification ────────────────────────────────────────
+log "--- Phase 11: final verification pass ---"
+docker stack ps "${HA_STACK}" --no-trunc | tail -n +1
+AVAIL_GB=$(df --output=avail -BG /volume1/docker/PostgreSQL 2>/dev/null | tail -1 | tr -dc '0-9')
+log "Disk space remaining at /volume1/docker/PostgreSQL: ${AVAIL_GB}GB"
+log "=== CUTOVER COMPLETE ==="
+log "Backup taken this run: $(cat ${BACKUP_DIR}/LATEST 2>/dev/null || echo 'unknown')"
+log "Full session transcript: ${CUTOVER_SESSION_LOG}"
+log "See RUNBOOK.md section 7 for manual post-cutover follow-up (legacy"
+log "decommission, external port 5430 / Traefik SNI handoff, burn-in period)."
+exit 0

postgresql/cutover/post_init_wrapper.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+# ─────────────────────────────────────────────────────────────────────────
+# postgresql/cutover/post_init_wrapper.sh
+#
+# Wraps Spilo's own /scripts/post_init.sh to work around a hardcoded
+# assumption in that (unmodified, upstream) script: it contains the line
+#   ALTER VIEW public.failed_authentication_${i} OWNER TO postgres;
+# with "postgres" as a literal, non-parameterized role name. This fails
+# with "role postgres does not exist" whenever PGUSER_SUPERUSER is set to
+# anything other than "postgres" — which we do (PGUSER_SUPERUSER=PGadmin,
+# to match this environment's actual production superuser name).
+#
+# This wrapper is invoked in place of /scripts/post_init.sh via the
+# SPILO_CONFIGURATION override (bootstrap.post_init), which Spilo's
+# configure_spilo.py deep-merges on top of its own generated Patroni
+# config, with user-supplied values taking precedence. Spilo's real
+# post_init.sh is NOT modified or forked — this script only ensures a
+# "postgres" role exists first, then hands off to the original script
+# unchanged, with its original arguments intact.
+#
+# Arguments (passed through from Patroni's bootstrap.post_init call,
+# unchanged): $1 = HUMAN_ROLE (e.g. "zalandos"), $2 = database name to
+# connect to for the initial connection (see Spilo's own post_init.sh
+# usage for exact semantics — not altered here).
+# ─────────────────────────────────────────────────────────────────────────
+set -euo pipefail
+
+echo "post_init_wrapper: ensuring role 'postgres' exists before running Spilo's post_init.sh"
+
+psql -d "$2" -v ON_ERROR_STOP=1 <<'SQL'
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = 'postgres') THEN
+        CREATE ROLE postgres;
+    END IF;
+END;
+$$;
+SQL
+
+echo "post_init_wrapper: role check complete, handing off to Spilo's post_init.sh"
+
+exec /scripts/post_init.sh "$1" "$2"

postgresql/cutover/postgresql-ha-final.yaml

@@ -0,0 +1,261 @@
+# ─────────────────────────────────────────────────────────────────────────
+# postgresql/cutover/postgresql-ha-final.yaml — ADR-0001 Phase 3, Stage 3
+#
+# Deployed by cutover.sh (Phase 7) as an UPDATE to the SAME "postgresqlha"
+# stack that postgresql-ha-staging.yaml created in Stage 2:
+#   docker stack deploy -c postgresql/cutover/postgresql-ha-final.yaml postgresqlha
+#
+# ⚠️  Do NOT deploy this before Patroni has been genuinely promoted
+#     (cutover.sh Phase 5) and confirmed healthy (Phase 6). Deploying it
+#     earlier adds the postgresql/db aliases to a backend HAProxy's own
+#     healthcheck (GET /primary) will correctly report as unhealthy (a
+#     standby_leader is read-only, not a real primary) — any client that
+#     Swarm's round-robin DNS routes to HAProxy during that window would
+#     hit zero healthy servers and fail outright. See RUNBOOK.md for the
+#     full rationale on phase ordering.
+#
+# What changed vs. postgresql-ha-staging.yaml (Stage 2):
+#   - etcd-1/2/3 and patroni-0/patroni-1 service specs are BYTE-IDENTICAL
+#     to the staging file, deliberately. `docker stack deploy` diffs each
+#     service's spec independently — since these five specs are unchanged,
+#     Swarm will NOT recreate/restart them. This matters enormously here:
+#     patroni-1 is the live, newly-promoted PRIMARY carrying real
+#     production traffic by the time this file is deployed, and
+#     patroni-0 is its live replica. Neither may be disrupted by this
+#     step. Only the haproxy service (below) has a spec change, so only
+#     it gets updated.
+#   - haproxy service: adds `postgresql` and `db` as network aliases on
+#     postgresql_db-backend — this is the actual DNS cutover mechanism.
+#     Still deliberately WITHOUT the external port publish (5430) or
+#     traefik_backend network/labels that the STEADY-STATE production
+#     postgresql.yaml's haproxy block eventually needs — taking over
+#     that external/Traefik-SNI path is a documented MANUAL, POST-cutover
+#     step (see RUNBOOK.md "Post-cutover follow-up"), not part of this
+#     automated procedure. Confirmed with the operator: no external
+#     consumer depends on port 5430 during the cutover window, so this
+#     is deferred deliberately, not an oversight.
+#   - Uses the same postgresql/haproxy.cfg already committed to git
+#     (Patroni-aware TCP router, GET /primary healthcheck, no changes
+#     needed for this stage).
+#
+# Once this is live and all consumers have verified (cutover.sh Phase 9),
+# legacy is scaled to 0 (Phase 10) and this becomes the sole answer for
+# the `postgresql`/`db` names on postgresql_db-backend.
+#
+# ── primary_slot_name (2026-08-04) ──────────────────────────────────────
+# patroni-0/patroni-1's standby_cluster blocks below include
+# `primary_slot_name: standby_leader_slot`, added to fix a real bootstrap
+# failure (basebackup succeeded but the resulting Postgres process got
+# stuck in "starting" forever because legacy had no replication slot
+# protecting the WAL it needed to resume streaming — recycled by normal
+# checkpoint activity during the 8-11 min basebackup window). Full
+# incident detail and the matching production-side fix (a physical slot
+# named standby_leader_slot created on legacy, plus wal_keep_size bumped
+# to 4GB as defense-in-depth) are documented in postgresql-ha-staging.yaml's
+# header and in the "ADR-0001 Dry-Run Debugging Log" note. This value MUST
+# stay identical to the staging file's per the byte-identical requirement
+# noted above — if you ever change one, change both.
+# ─────────────────────────────────────────────────────────────────────────
+version: "3.6"
+
+services:
+
+  etcd-1:
+    image: quay.io/coreos/etcd:v3.5.9
+    hostname: etcd-1
+    command:
+      - etcd
+      - --name=etcd-1
+      - --data-dir=/etcd-data
+      - --initial-advertise-peer-urls=http://etcd-1:2380
+      - --listen-peer-urls=http://0.0.0.0:2380
+      - --listen-client-urls=http://0.0.0.0:2379
+      - --advertise-client-urls=http://etcd-1:2379
+      - --initial-cluster=etcd-1=http://etcd-1:2380,etcd-2=http://etcd-2:2380,etcd-3=http://etcd-3:2380
+      - --initial-cluster-state=new
+      - --initial-cluster-token=postgresql-ha-etcd
+    volumes:
+      - /volume1/docker/PostgreSQL/etcd-1-data:/etcd-data
+    networks:
+      - postgresql_db-backend
+    deploy:
+      placement:
+        constraints:
+          - node.hostname == docker-1
+
+  etcd-2:
+    image: quay.io/coreos/etcd:v3.5.9
+    hostname: etcd-2
+    command:
+      - etcd
+      - --name=etcd-2
+      - --data-dir=/etcd-data
+      - --initial-advertise-peer-urls=http://etcd-2:2380
+      - --listen-peer-urls=http://0.0.0.0:2380
+      - --listen-client-urls=http://0.0.0.0:2379
+      - --advertise-client-urls=http://etcd-2:2379
+      - --initial-cluster=etcd-1=http://etcd-1:2380,etcd-2=http://etcd-2:2380,etcd-3=http://etcd-3:2380
+      - --initial-cluster-state=new
+      - --initial-cluster-token=postgresql-ha-etcd
+    volumes:
+      - /volume1/docker/PostgreSQL/etcd-2-data:/etcd-data
+    networks:
+      - postgresql_db-backend
+    deploy:
+      placement:
+        constraints:
+          - node.hostname == docker-2
+
+  etcd-3:
+    image: quay.io/coreos/etcd:v3.5.9
+    hostname: etcd-3
+    command:
+      - etcd
+      - --name=etcd-3
+      - --data-dir=/etcd-data
+      - --initial-advertise-peer-urls=http://etcd-3:2380
+      - --listen-peer-urls=http://0.0.0.0:2380
+      - --listen-client-urls=http://0.0.0.0:2379
+      - --advertise-client-urls=http://etcd-3:2379
+      - --initial-cluster=etcd-1=http://etcd-1:2380,etcd-2=http://etcd-2:2380,etcd-3=http://etcd-3:2380
+      - --initial-cluster-state=new
+      - --initial-cluster-token=postgresql-ha-etcd
+    volumes:
+      - /volume1/docker/PostgreSQL/etcd-3-data:/etcd-data
+    networks:
+      - postgresql_db-backend
+    deploy:
+      placement:
+        constraints:
+          - node.hostname == docker-3
+
+  patroni-0:
+    image: ghcr.io/zalando/spilo-17:4.0-p3
+    hostname: patroni-0
+    command:
+      - /bin/sh
+      - -c
+      - |
+        export PGPASSWORD_SUPERUSER="$$(cat /run/secrets/postgresql_password)"
+        export PGPASSWORD_STANDBY="$$(cat /run/secrets/postgresql_replication_password)"
+        export PATRONI_RESTAPI_PASSWORD="$$(cat /run/secrets/postgresql_patroni_password)"
+        mkdir -p /scripts
+        cat > /scripts/post_init_wrapper.sh <<'WRAP'
+        #!/bin/bash
+        set -e
+        psql -d "$$2" -v ON_ERROR_STOP=1 -c 'DO $$do$$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = $$x$$postgres$$x$$) THEN CREATE ROLE postgres WITH SUPERUSER NOLOGIN; END IF; END $$do$$;'
+        exec /scripts/post_init.sh "$$@"
+        WRAP
+        chmod +x /scripts/post_init_wrapper.sh
+        exec /bin/sh /launch.sh init
+    environment:
+      SCOPE: postgres-ha
+      PATRONI_NAME: patroni-0
+      ETCD3_HOSTS: '"etcd-1:2379","etcd-2:2379","etcd-3:2379"'
+      PGUSER_SUPERUSER: PGadmin
+      PGUSER_STANDBY: standby
+      PATRONI_RESTAPI_USERNAME: patroni
+      PGROOT: /home/postgres/pgdata/pgroot
+      SPILO_CONFIGURATION: |
+        bootstrap:
+          post_init: /scripts/post_init_wrapper.sh "zalandos"
+          dcs:
+            standby_cluster:
+              host: postgresql
+              port: 5432
+              primary_slot_name: standby_leader_slot
+              create_replica_methods:
+              - basebackup_fast_xlog
+    secrets:
+      - postgresql_password
+      - postgresql_replication_password
+      - postgresql_patroni_password
+    volumes:
+      - /volume1/docker/PostgreSQL/patroni-0-data:/home/postgres/pgdata
+    networks:
+      - postgresql_db-backend
+    deploy:
+      placement:
+        constraints:
+          - node.labels.pg-role == primary
+
+  patroni-1:
+    image: ghcr.io/zalando/spilo-17:4.0-p3
+    hostname: patroni-1
+    command:
+      - /bin/sh
+      - -c
+      - |
+        export PGPASSWORD_SUPERUSER="$$(cat /run/secrets/postgresql_password)"
+        export PGPASSWORD_STANDBY="$$(cat /run/secrets/postgresql_replication_password)"
+        export PATRONI_RESTAPI_PASSWORD="$$(cat /run/secrets/postgresql_patroni_password)"
+        mkdir -p /scripts
+        cat > /scripts/post_init_wrapper.sh <<'WRAP'
+        #!/bin/bash
+        set -e
+        psql -d "$$2" -v ON_ERROR_STOP=1 -c 'DO $$do$$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = $$x$$postgres$$x$$) THEN CREATE ROLE postgres WITH SUPERUSER NOLOGIN; END IF; END $$do$$;'
+        exec /scripts/post_init.sh "$$@"
+        WRAP
+        chmod +x /scripts/post_init_wrapper.sh
+        exec /bin/sh /launch.sh init
+    environment:
+      SCOPE: postgres-ha
+      PATRONI_NAME: patroni-1
+      ETCD3_HOSTS: '"etcd-1:2379","etcd-2:2379","etcd-3:2379"'
+      PGUSER_SUPERUSER: PGadmin
+      PGUSER_STANDBY: standby
+      PATRONI_RESTAPI_USERNAME: patroni
+      PGROOT: /home/postgres/pgdata/pgroot
+      SPILO_CONFIGURATION: |
+        bootstrap:
+          post_init: /scripts/post_init_wrapper.sh "zalandos"
+          dcs:
+            standby_cluster:
+              host: postgresql
+              port: 5432
+              primary_slot_name: standby_leader_slot
+              create_replica_methods:
+              - basebackup_fast_xlog
+    secrets:
+      - postgresql_password
+      - postgresql_replication_password
+      - postgresql_patroni_password
+    volumes:
+      - /volume1/docker/PostgreSQL/patroni-1-data:/home/postgres/pgdata
+    networks:
+      - postgresql_db-backend
+    deploy:
+      placement:
+        constraints:
+          - node.labels.pg-role == replica
+
+  # ── Final-stage HAProxy: takes over the postgresql/db aliases ──────────
+  # Internal Swarm-DNS only at this stage — no published port, no
+  # traefik_backend network/labels. External access (port 5430 /
+  # Traefik SNI) is a deliberate, documented MANUAL post-cutover step,
+  # not part of this automated procedure — confirmed no external
+  # consumer depends on it during the cutover window.
+  haproxy:
+    image: haproxy:2.9-alpine
+    hostname: haproxy
+    volumes:
+      - /volume1/docker/compose-files/postgresql/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
+    networks:
+      postgresql_db-backend:
+        aliases:
+          - postgresql
+          - db
+    deploy:
+      mode: global
+
+networks:
+  postgresql_db-backend:
+    external: true
+
+secrets:
+  postgresql_password:
+    external: true
+  postgresql_replication_password:
+    external: true
+  postgresql_patroni_password:
+    external: true

postgresql/cutover/postgresql-ha-staging.yaml

@@ -0,0 +1,363 @@
+# ─────────────────────────────────────────────────────────────────────────
+# postgresql/cutover/postgresql-ha-staging.yaml — ADR-0001 Phase 3, Stage 2
+#
+# Deployed by postgres-ha-cutover.sh as a SEPARATE stack:
+#   docker stack deploy -c postgresql/cutover/postgresql-ha-staging.yaml postgresqlha
+#
+# ⚠️  NEVER deploy this via stack-deploy.sh or under the "postgresql" stack
+#     name. It lives in cutover/ (a subdirectory) deliberately:
+#     stack-deploy.sh folder mode merges all *.yml/yaml at -maxdepth 1 of
+#     postgresql/, and merging this into the production stack would cause
+#     an outage. Subdirectory files are excluded from that merge.
+#
+# What this stage does:
+#   - Joins the EXISTING postgresql_db-backend overlay (external) — the
+#     live single-instance stack is untouched and keeps the "postgresql"
+#     and "db" aliases.
+#   - patroni-0/patroni-1 bootstrap a NEW cluster (scope: postgres-ha) as a
+#     Patroni STANDBY CLUSTER continuously streaming from the live
+#     instance (standby_cluster, host: postgresql — resolves to the OLD
+#     instance during staging). This REPLACES the earlier
+#     CLONE_WITH_BASEBACKUP design (one-shot snapshot) — see "REVISION"
+#     note below for why.
+#   - HAProxy runs WITHOUT production aliases and WITHOUT published ports.
+#     It only gets those in postgresql-ha-final.yaml at promote time.
+#   - Superuser is PGadmin (matches the cloned data's existing superuser,
+#     same password secret), NOT Spilo's default "postgres".
+#
+# ── REVISION (2026-08-01): switched from CLONE_WITH_BASEBACKUP to a
+# Patroni "standby cluster" (continuous streaming) ──
+#
+# WHY: CLONE_WITH_BASEBACKUP is a ONE-SHOT snapshot — once pg_basebackup
+# completes, the new cluster has ZERO further connection to the live
+# database. Any write landing on production between the snapshot and the
+# actual traffic cutover would be silently lost — a real risk given ~13
+# active consumers and a 42GB production database. Patroni's standby_cluster
+# mode (https://patroni.readthedocs.io/en/latest/standby_cluster.html)
+# instead makes the new cluster's leader ("standby leader") continuously
+# stream from the remote primary indefinitely, right up until an explicit
+# promotion (done by removing the standby_cluster key from DCS config —
+# there is no dedicated patronictl subcommand for this) — closing the gap
+# to near-zero. Fully validated end-to-end in the pgha-test dry run
+# (2026-08-01): bootstrap, continuous live-write propagation to the far
+# cascade replica in <3s, and a full promote-with-zero-data-loss cycle —
+# see "ADR-0001 Dry-Run Debugging Log" note for the complete play-by-play.
+#
+# ⚠️ NEW PRODUCTION PREREQUISITE THIS INTRODUCES — the dry run's
+# "legacy" stand-in creates a genuine replication role named "standby"
+# (matching PGUSER_STANDBY below) via a fresh-initdb hook script, since
+# standby_cluster streaming authenticates using the cluster's real
+# replication identity (PGUSER_STANDBY/PGPASSWORD_STANDBY), NOT the
+# PGadmin superuser that CLONE_WITH_BASEBACKUP used. The LIVE
+# `postgresql_postgresql` container has NO such hook mechanism available
+# (it's long since initialized, hooks only run on fresh initdb) and does
+# NOT currently have a "standby" role. This must be created manually
+# against the live database before real cutover, analogous to how the
+# pg_hba.conf replication rule was added manually — e.g.:
+#   CREATE ROLE standby WITH REPLICATION LOGIN PASSWORD '<same value as
+#   the postgresql_replication_password secret>';
+# This is NOT part of this compose file (can't be — it's a one-time SQL
+# statement against already-running production, not something a compose
+# file can express) but MUST be tracked and done as an explicit manual
+# step in the real Phase 3 procedure. STATUS: DONE — see ADR-0001 note,
+# "Production prerequisite" section, for full verification detail
+# (CREATE ROLE + pg_hba.conf line, both proven end-to-end).
+#
+# ── REVISION 2 (2026-08-04): added primary_slot_name to standby_cluster —
+# fixes a real bootstrap failure hit on a live production run ──
+#
+# WHAT HAPPENED: on a real cutover attempt, the standby_leader's OWN
+# basebackup from legacy completed successfully, but the resulting
+# Postgres process then got stuck in "starting"/"rejecting connections"
+# PERMANENTLY (confirmed via the container's own pg_log/*.csv, since
+# docker service logs stops showing Postgres's own LOG lines once the
+# logging collector activates — check
+# <data-dir>/pgroot/pg_log/postgresql-N.csv directly, NOT `docker service
+# logs`, if this needs debugging again). The actual error, repeating
+# every ~5s indefinitely:
+#   LOG:  started streaming WAL from primary at <LSN> on timeline 1
+#   FATAL: could not receive data from WAL stream: ERROR: requested WAL
+#          segment <segment> has already been removed
+#   LOG:  waiting for WAL to become available at <LSN>
+# ROOT CAUSE: legacy had NO replication slot reserving WAL for this
+# connection (confirmed: `select * from pg_replication_slots` = 0 rows,
+# wal_keep_size = 0, max_slot_wal_keep_size = -1/irrelevant with no slot).
+# basebackup_fast_xlog copies data files, then Postgres must replay WAL
+# FORWARD from the backup's start LSN via a normal streaming connection to
+# catch up. With no slot, legacy's normal WAL recycling
+# (checkpoint_timeout=300s) deleted the exact segment needed to resume
+# during the 8-11 minute basebackup window — once gone, gone forever, and
+# the standby_leader can never leave "starting". This ALSO explains why
+# the OTHER node's (the cascade replica's) basebackup then failed with
+# "the database system is starting up" — it targets the standby_leader,
+# which never actually finished starting. Not two bugs — one root cause
+# propagating downstream exactly as it would look if it were two.
+# (Secondary, non-blocking finding from the same run: ~2.2GB of core dump
+# files appeared in the standby_leader's data dir root at the moment of
+# basebackup completion — not yet root-caused, possibly related to
+# bg_mon's repeated port-8080-bind failures inside the container, but NOT
+# confirmed as the same issue and NOT blocking this fix.)
+# THE FIX (this revision): a physical replication slot named
+# "standby_leader_slot" was created on live production legacy:
+#   SELECT pg_create_physical_replication_slot('standby_leader_slot');
+# and `primary_slot_name: standby_leader_slot` was added under
+# bootstrap.dcs.standby_cluster in BOTH patroni-0 and patroni-1's
+# SPILO_CONFIGURATION below (Patroni's standby_cluster feature reads this
+# key and uses it to pin the slot on the external primary automatically —
+# no additional Patroni-side config needed). As defense-in-depth,
+# `wal_keep_size` was ALSO bumped from 0 to 4096MB (4GB) directly on live
+# legacy (`ALTER SYSTEND SET wal_keep_size = '4096MB'; SELECT
+# pg_reload_conf();` — NOTE: ALTER SYSTEM cannot share a transaction with
+# other statements, run as two separate -c calls) — belt-and-suspenders
+# in case the slot itself is ever inadvertently dropped mid-cutover.
+# WHY THIS DIDN'T SURFACE EARLIER: the pgha-test dry run used a tiny
+# disposable database with a near-instant basebackup, leaving no real
+# window for WAL to churn before streaming resumed. It's plausible an even
+# earlier real attempt got lucky on timing. With a real ~42GB dataset and
+# multi-minute basebackups, the gap is real and reproducible.
+# STATUS: slot created and verified on live production
+# (`select slot_name, slot_type, active from pg_replication_slots` shows
+# standby_leader_slot/physical/f — inactive is expected/correct until a
+# standby_cluster actually connects to it). wal_keep_size verified at 4GB.
+# This compose file updated to reference the slot. NOT YET RE-RUN against
+# production as of this revision — see ADR-0001 note's "Not yet done"
+# section for current status.
+#
+# NOTE: command: blocks use $$(...) not $(...) — Compose's own variable
+# interpolation parses $( as an attempted ${VAR} reference and fails with
+# "invalid interpolation format" / "you may need to escape any $ with
+# another $". $$ escapes to a literal $ for the shell at runtime. This
+# applies uniformly to EVERY literal $ character anywhere in a command:
+# block, including inside heredocs and nested quoting — Compose scans the
+# raw text before any shell/heredoc logic ever runs, so quoting context
+# doesn't exempt anything from this escaping requirement. (Confirmed the
+# hard way in the pgha-test dry run — this file originally had the
+# unescaped $(...) bug too, before this standby_cluster rewrite.)
+#
+# NOTE: ETCD3_HOSTS not ETCD_HOSTS — confirmed against zalando/spilo
+# configure_spilo.py: PATRONI_DCS includes both "etcd" (legacy v2 API,
+# python-etcd client) and "etcd3" (v3 API, python-etcd3 client) as
+# distinct DCS backends selected by env var prefix. Our etcd containers
+# (v3.5.9) have the v2 API disabled by default, so ETCD_HOSTS causes
+# Patroni to hit /v2 endpoints that 404. ETCD3_HOSTS selects the correct
+# v3-API client.
+#
+# NOTE: patroni-0/patroni-1 override bootstrap.post_init via
+# SPILO_CONFIGURATION (Spilo's documented, supported mechanism for
+# overriding any generated Patroni config — configure_spilo.py deep-merges
+# user-supplied SPILO_CONFIGURATION on top of its own generated config).
+# This is needed because Spilo's own /scripts/post_init.sh hardcodes
+# "ALTER VIEW ... OWNER TO postgres" with no way to parameterize that role
+# name via env vars. Since our superuser is PGUSER_SUPERUSER=PGadmin, there
+# is no role literally named "postgres" in the cloned data, so Spilo's
+# unmodified script fails with 'ERROR: role "postgres" does not exist'.
+# The override points bootstrap.post_init at our own wrapper script
+# instead, which creates a harmless, idempotent "postgres" role (WITH
+# SUPERUSER NOLOGIN — SUPERUSER is required because Spilo's own
+# _zmon_schema.dump does "SET ROLE TO postgres; CREATE EXTENSION
+# plpython3u", which needs real superuser privileges, not just role
+# existence; NOLOGIN means it can never be used to establish a real
+# connection, so this carries no security exposure), then execs Spilo's
+# real, UNMODIFIED post_init.sh with all original arguments passed through
+# — Zalando's script itself is never patched or forked. Both fixes (role
+# existence + SUPERUSER) were found and validated end-to-end in the
+# pgha-test dry run before being ported here — see the dry-run debugging
+# note for full detail. This is CRITICAL for this file specifically since
+# it streams real production data with the real PGadmin superuser.
+# ─────────────────────────────────────────────────────────────────────────
+version: "3.6"
+
+services:
+
+  etcd-1:
+    image: quay.io/coreos/etcd:v3.5.9
+    hostname: etcd-1
+    command:
+      - etcd
+      - --name=etcd-1
+      - --data-dir=/etcd-data
+      - --initial-advertise-peer-urls=http://etcd-1:2380
+      - --listen-peer-urls=http://0.0.0.0:2380
+      - --listen-client-urls=http://0.0.0.0:2379
+      - --advertise-client-urls=http://etcd-1:2379
+      - --initial-cluster=etcd-1=http://etcd-1:2380,etcd-2=http://etcd-2:2380,etcd-3=http://etcd-3:2380
+      - --initial-cluster-state=new
+      - --initial-cluster-token=postgresql-ha-etcd
+    volumes:
+      - /volume1/docker/PostgreSQL/etcd-1-data:/etcd-data
+    networks:
+      - postgresql_db-backend
+    deploy:
+      placement:
+        constraints:
+          - node.hostname == docker-1
+
+  etcd-2:
+    image: quay.io/coreos/etcd:v3.5.9
+    hostname: etcd-2
+    command:
+      - etcd
+      - --name=etcd-2
+      - --data-dir=/etcd-data
+      - --initial-advertise-peer-urls=http://etcd-2:2380
+      - --listen-peer-urls=http://0.0.0.0:2380
+      - --listen-client-urls=http://0.0.0.0:2379
+      - --advertise-client-urls=http://etcd-2:2379
+      - --initial-cluster=etcd-1=http://etcd-1:2380,etcd-2=http://etcd-2:2380,etcd-3=http://etcd-3:2380
+      - --initial-cluster-state=new
+      - --initial-cluster-token=postgresql-ha-etcd
+    volumes:
+      - /volume1/docker/PostgreSQL/etcd-2-data:/etcd-data
+    networks:
+      - postgresql_db-backend
+    deploy:
+      placement:
+        constraints:
+          - node.hostname == docker-2
+
+  etcd-3:
+    image: quay.io/coreos/etcd:v3.5.9
+    hostname: etcd-3
+    command:
+      - etcd
+      - --name=etcd-3
+      - --data-dir=/etcd-data
+      - --initial-advertise-peer-urls=http://etcd-3:2380
+      - --listen-peer-urls=http://0.0.0.0:2380
+      - --listen-client-urls=http://0.0.0.0:2379
+      - --advertise-client-urls=http://etcd-3:2379
+      - --initial-cluster=etcd-1=http://etcd-1:2380,etcd-2=http://etcd-2:2380,etcd-3=http://etcd-3:2380
+      - --initial-cluster-state=new
+      - --initial-cluster-token=postgresql-ha-etcd
+    volumes:
+      - /volume1/docker/PostgreSQL/etcd-3-data:/etcd-data
+    networks:
+      - postgresql_db-backend
+    deploy:
+      placement:
+        constraints:
+          - node.hostname == docker-3
+
+  patroni-0:
+    image: ghcr.io/zalando/spilo-17:4.0-p3
+    hostname: patroni-0
+    command:
+      - /bin/sh
+      - -c
+      - |
+        export PGPASSWORD_SUPERUSER="$$(cat /run/secrets/postgresql_password)"
+        export PGPASSWORD_STANDBY="$$(cat /run/secrets/postgresql_replication_password)"
+        export PATRONI_RESTAPI_PASSWORD="$$(cat /run/secrets/postgresql_patroni_password)"
+        mkdir -p /scripts
+        cat > /scripts/post_init_wrapper.sh <<'WRAP'
+        #!/bin/bash
+        set -e
+        psql -d "$$2" -v ON_ERROR_STOP=1 -c 'DO $$do$$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = $$x$$postgres$$x$$) THEN CREATE ROLE postgres WITH SUPERUSER NOLOGIN; END IF; END $$do$$;'
+        exec /scripts/post_init.sh "$$@"
+        WRAP
+        chmod +x /scripts/post_init_wrapper.sh
+        exec /bin/sh /launch.sh init
+    environment:
+      SCOPE: postgres-ha
+      PATRONI_NAME: patroni-0
+      ETCD3_HOSTS: '"etcd-1:2379","etcd-2:2379","etcd-3:2379"'
+      PGUSER_SUPERUSER: PGadmin
+      PGUSER_STANDBY: standby
+      PATRONI_RESTAPI_USERNAME: patroni
+      PGROOT: /home/postgres/pgdata/pgroot
+      SPILO_CONFIGURATION: |
+        bootstrap:
+          post_init: /scripts/post_init_wrapper.sh "zalandos"
+          dcs:
+            standby_cluster:
+              host: postgresql
+              port: 5432
+              primary_slot_name: standby_leader_slot
+              create_replica_methods:
+              - basebackup_fast_xlog
+    secrets:
+      - postgresql_password
+      - postgresql_replication_password
+      - postgresql_patroni_password
+    volumes:
+      - /volume1/docker/PostgreSQL/patroni-0-data:/home/postgres/pgdata
+    networks:
+      - postgresql_db-backend
+    deploy:
+      placement:
+        constraints:
+          - node.labels.pg-role == primary
+
+  patroni-1:
+    image: ghcr.io/zalando/spilo-17:4.0-p3
+    hostname: patroni-1
+    command:
+      - /bin/sh
+      - -c
+      - |
+        export PGPASSWORD_SUPERUSER="$$(cat /run/secrets/postgresql_password)"
+        export PGPASSWORD_STANDBY="$$(cat /run/secrets/postgresql_replication_password)"
+        export PATRONI_RESTAPI_PASSWORD="$$(cat /run/secrets/postgresql_patroni_password)"
+        mkdir -p /scripts
+        cat > /scripts/post_init_wrapper.sh <<'WRAP'
+        #!/bin/bash
+        set -e
+        psql -d "$$2" -v ON_ERROR_STOP=1 -c 'DO $$do$$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = $$x$$postgres$$x$$) THEN CREATE ROLE postgres WITH SUPERUSER NOLOGIN; END IF; END $$do$$;'
+        exec /scripts/post_init.sh "$$@"
+        WRAP
+        chmod +x /scripts/post_init_wrapper.sh
+        exec /bin/sh /launch.sh init
+    environment:
+      SCOPE: postgres-ha
+      PATRONI_NAME: patroni-1
+      ETCD3_HOSTS: '"etcd-1:2379","etcd-2:2379","etcd-3:2379"'
+      PGUSER_SUPERUSER: PGadmin
+      PGUSER_STANDBY: standby
+      PATRONI_RESTAPI_USERNAME: patroni
+      PGROOT: /home/postgres/pgdata/pgroot
+      SPILO_CONFIGURATION: |
+        bootstrap:
+          post_init: /scripts/post_init_wrapper.sh "zalandos"
+          dcs:
+            standby_cluster:
+              host: postgresql
+              port: 5432
+              primary_slot_name: standby_leader_slot
+              create_replica_methods:
+              - basebackup_fast_xlog
+    secrets:
+      - postgresql_password
+      - postgresql_replication_password
+      - postgresql_patroni_password
+    volumes:
+      - /volume1/docker/PostgreSQL/patroni-1-data:/home/postgres/pgdata
+    networks:
+      - postgresql_db-backend
+    deploy:
+      placement:
+        constraints:
+          - node.labels.pg-role == replica
+
+  # Staging HAProxy: NO production aliases, NO published ports. Reachable
+  # in-network as "haproxy" for pre-promote testing only.
+  haproxy:
+    image: haproxy:2.9-alpine
+    hostname: haproxy
+    volumes:
+      - /volume1/docker/compose-files/postgresql/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
+    networks:
+      - postgresql_db-backend
+    deploy:
+      mode: global
+
+networks:
+  postgresql_db-backend:
+    external: true
+
+secrets:
+  postgresql_password:
+    external: true
+  postgresql_replication_password:
+    external: true
+  postgresql_patroni_password:
+    external: true

postgresql/cutover/preflight.sh

@@ -0,0 +1,195 @@
+#!/usr/bin/env bash
+# ADR-0001 Phase 3 Cutover — preflight.sh
+#
+# Run DIRECTLY on docker-2 via SSH (NOT via the LXC-exec MCP tool — that
+# has a ~30s timeout and pg_dumpall against a 42GB instance will run far
+# longer than that; this must run as a real, unbounded shell process).
+#
+#   ssh root@192.168.4.32
+#   bash /volume1/docker/compose-files/postgresql/cutover/preflight.sh
+#
+# Exit code 0  = all hard gates passed, safe to proceed to cutover.sh
+# Exit nonzero = hard stop; see output for which gate failed. Nothing is
+#                modified by this script beyond writing a backup file —
+#                it never touches the live legacy service, HA stack, or
+#                any consumer.
+#
+# Hard gates (halt everything on failure):
+#   1. Disk space  — need at least MIN_FREE_GB free before backup starts
+#   2. Backup       — fresh pg_dumpall every run, unconditional, must
+#                      produce a non-empty file with no reported errors
+#
+# Informational checks (logged, do NOT halt the script):
+#   3. Catalog sanity   — enumerate live databases
+#   4. Consumer enumeration — live network + static repo cross-check
+#   5. Reconciliation  — match catalog dbs against known consumers /
+#                          known no-consumer exceptions, flag anything
+#                          truly unrecognized for human review
+#
+# Rationale for treating 3-5 as informational, not hard gates: database
+# name <-> Swarm service/container name mapping is not reliably 1:1
+# (e.g. the `authentik` database vs the `authentik-server` container), so
+# fully automating a hard pass/fail per-database would be fragile and
+# risks either false-positive halts or false-confidence passes. This
+# check exists to surface anomalies for review, not to silently gate
+# cutover on an imperfect heuristic.
+#
+# ── SESSION LOGGING (added — see ADR-0001 note, Session Update 9) ─────────
+# Every run's full stdout/stderr transcript is persisted to a timestamped
+# .log file in BACKUP_DIR — the SAME directory the pg_dumpall backup file
+# itself lands in — so a failed run (or, worse, a failed automatic
+# rollback.sh invoked afterward by cutover.sh) can still be investigated
+# later even if nobody was watching a live terminal at the time. This
+# script either appends to an already-open session log
+# (CUTOVER_SESSION_LOG, exported by a parent cutover.sh so all phases land
+# in ONE merged transcript) or opens its own timestamped log if run
+# standalone. Console/SSH output is unchanged either way — tee mirrors to
+# both the file and the real stdout/stderr.
+
+set -euo pipefail
+
+LEGACY_CONTAINER_FILTER="postgresql_postgresql"
+PG_DATA_ROOT="/volume1/docker/PostgreSQL"
+BACKUP_DIR="/volume1/SMB-docker/backup"
+TS="$(date +%Y%m%d-%H%M%S)"
+BACKUP_FILE="${BACKUP_DIR}/pre-cutover-${TS}.sql"
+MIN_FREE_GB=50
+REPO_ROOT="/volume1/docker/compose-files"
+NETWORK_NAME="postgresql_db-backend"
+
+# Known database(s) that exist in the shared legacy Postgres instance but
+# have NO live Swarm consumer container attached to postgresql_db-backend
+# (e.g. SparkyFitness — confirmed not deployed in prod as a running
+# stack). These are NOT excluded from anything:
+#   - pg_dumpall below captures the WHOLE instance regardless, so this
+#     data is in the backup either way.
+#   - Patroni's standby_cluster streams the entire physical cluster, not
+#     per-database, so this data is replicated into the new HA primary
+#     automatically with zero special handling.
+# This list exists purely so step 5 doesn't flag a KNOWN, already-
+# accepted case as if it were a surprise. Add to this list only for
+# genuinely confirmed no-consumer databases — do not use it to silence
+# a check you haven't actually investigated.
+KNOWN_NO_CONSUMER_DBS=("sparkyfitness")
+
+log()  { echo "[preflight $(date +%H:%M:%S)] $*"; }
+warn() { echo "[preflight WARN $(date +%H:%M:%S)] $*" >&2; }
+fail() { echo "[preflight FAIL $(date +%H:%M:%S)] $*" >&2; exit 1; }
+
+# ── Session logging setup — MUST happen before any other output so the
+# very first log lines are captured too. Deliberately uses the SAME
+# BACKUP_DIR as the pg_dumpall backup file itself (not a separate log
+# location), per explicit operator request: whoever goes looking for "the
+# backup from that failed run" finds the matching transcript sitting
+# right next to it.
+mkdir -p "$BACKUP_DIR"
+if [ -z "${CUTOVER_SESSION_LOG:-}" ]; then
+  # No parent cutover.sh session log already open — standalone invocation,
+  # open our own.
+  CUTOVER_SESSION_LOG="${BACKUP_DIR}/preflight-standalone-${TS}.log"
+  export CUTOVER_SESSION_LOG
+fi
+# Append (not truncate) — if a parent cutover.sh already wrote earlier
+# output to this same file, we must not clobber it. tee mirrors to the
+# real stdout/stderr too, so console/SSH-visible output is unchanged.
+exec > >(tee -a "$CUTOVER_SESSION_LOG") 2>&1
+
+log "=== ADR-0001 Phase 3 cutover preflight starting ==="
+log "Full session transcript: ${CUTOVER_SESSION_LOG}"
+if [ -d "${REPO_ROOT}/.git" ]; then
+  GIT_HEAD="$(git -C "$REPO_ROOT" rev-parse HEAD 2>/dev/null || echo unknown)"
+  log "Repo HEAD at ${REPO_ROOT}: ${GIT_HEAD} (confirm this matches origin/main — see ADR-0001 note's local-checkout-drift lesson, Session Update 8, if unsure)"
+fi
+
+# ── Gate 1: Disk space (checked BEFORE backup; halts before anything is written) ──
+log "Checking free space on ${PG_DATA_ROOT} (resolves through symlinks to real backing filesystem)..."
+if [ ! -d "$PG_DATA_ROOT" ]; then
+  fail "${PG_DATA_ROOT} does not exist on this host — wrong node? This must run on docker-2."
+fi
+AVAIL_GB=$(df --output=avail -BG "$PG_DATA_ROOT" 2>/dev/null | tail -1 | tr -dc '0-9')
+if [ -z "$AVAIL_GB" ]; then
+  fail "Could not determine available disk space at ${PG_DATA_ROOT}."
+fi
+log "Available space: ${AVAIL_GB}GB (minimum required: ${MIN_FREE_GB}GB)"
+if [ "$AVAIL_GB" -lt "$MIN_FREE_GB" ]; then
+  fail "Only ${AVAIL_GB}GB free at ${PG_DATA_ROOT}, need at least ${MIN_FREE_GB}GB. Halting BEFORE backup — nothing has been written or touched."
+fi
+log "PASS: disk space gate (${AVAIL_GB}GB >= ${MIN_FREE_GB}GB)"
+
+mkdir -p "$BACKUP_DIR"
+
+# ── Gate 2: Fresh backup (unconditional every run) ────────────────────────
+log "Taking fresh pg_dumpall backup of legacy instance -> ${BACKUP_FILE}"
+LEGACY_CID=$(docker ps -q --filter "name=${LEGACY_CONTAINER_FILTER}" | head -1)
+if [ -z "$LEGACY_CID" ]; then
+  fail "Could not find a running legacy container matching '${LEGACY_CONTAINER_FILTER}'. Is the postgresql stack up?"
+fi
+log "Legacy container: ${LEGACY_CID}"
+
+# pg_dumpall dumps ALL databases + roles/globals in the instance in one
+# pass, including any database with no live Swarm consumer (e.g.
+# sparkyfitness) — nothing is excluded from this backup regardless of how
+# it's classified in the catalog-sanity step below.
+if ! docker exec "$LEGACY_CID" bash -c 'pg_dumpall -U "$POSTGRES_USER"' > "$BACKUP_FILE" 2> "${BACKUP_FILE}.stderr"; then
+  fail "pg_dumpall exited nonzero. See ${BACKUP_FILE}.stderr for detail. Backup NOT trusted, halting."
+fi
+if [ ! -s "$BACKUP_FILE" ]; then
+  fail "Backup file is empty — pg_dumpall produced no output. See ${BACKUP_FILE}.stderr"
+fi
+if grep -qi "error" "${BACKUP_FILE}.stderr"; then
+  fail "pg_dumpall reported errors on stderr — see ${BACKUP_FILE}.stderr. Backup NOT trusted, halting."
+fi
+BACKUP_SIZE=$(du -h "$BACKUP_FILE" | cut -f1)
+log "PASS: backup gate — ${BACKUP_FILE} (${BACKUP_SIZE})"
+rm -f "${BACKUP_FILE}.stderr"
+
+# Record the backup path so cutover.sh/rollback.sh can reference the most
+# recent one without re-deriving the timestamp.
+echo "$BACKUP_FILE" > "${BACKUP_DIR}/LATEST"
+
+# ── Informational 3: Catalog sanity — enumerate live databases ───────────
+log "Enumerating live databases in legacy instance..."
+LIVE_DBS=$(docker exec "$LEGACY_CID" bash -c 'psql -U "$POSTGRES_USER" -tAc "SELECT datname FROM pg_database WHERE datistemplate = false AND datname NOT IN ('"'"'postgres'"'"');"')
+log "Databases found in catalog:"
+echo "$LIVE_DBS" | sed 's/^/    - /'
+
+# ── Informational 4: Authoritative consumer enumeration ───────────────────
+log "Enumerating live containers attached to ${NETWORK_NAME}..."
+LIVE_CONSUMERS=$(docker network inspect "$NETWORK_NAME" --format '{{range .Containers}}{{.Name}}{{"\n"}}{{end}}' 2>/dev/null || true)
+if [ -z "$LIVE_CONSUMERS" ]; then
+  warn "Could not inspect network ${NETWORK_NAME}, or it has no attached containers. This is unusual — review before proceeding."
+else
+  log "Live containers on ${NETWORK_NAME}:"
+  echo "$LIVE_CONSUMERS" | sed 's/^/    - /'
+fi
+
+log "Cross-checking against repo (static grep for DB references, catches anything not currently running)..."
+if [ -d "$REPO_ROOT" ]; then
+  GREP_HITS=$(grep -rl -E "postgresql_db-backend|DATABASE_URL|POSTGRES_(HOST|USER|PASSWORD)|AUTHENTIK_POSTGRESQL" "$REPO_ROOT" --include="*.yaml" --include="*.yml" 2>/dev/null || true)
+  log "Compose files referencing the DB network/env vars:"
+  echo "$GREP_HITS" | sed 's/^/    - /'
+else
+  warn "Repo root ${REPO_ROOT} not found on this host — skipping static cross-check. Live network enumeration above is still authoritative for 'attached right now', but anything not currently running would be missed."
+fi
+
+# ── Informational 5: Reconcile catalog vs consumers / known exceptions ────
+log "Reconciling catalog databases against known no-consumer exceptions..."
+while IFS= read -r db; do
+  [ -z "$db" ] && continue
+  is_known_exception=false
+  for known in "${KNOWN_NO_CONSUMER_DBS[@]}"; do
+    if [ "$db" == "$known" ]; then
+      is_known_exception=true
+      break
+    fi
+  done
+  if [ "$is_known_exception" = true ]; then
+    log "  '${db}': known no-consumer database (accepted exception, not in Prod as a deployed app). Fully covered by the pg_dumpall backup above and will be fully covered by standby_cluster's whole-instance physical replication — no exclusion, no special handling, just flagged for visibility."
+  else
+    log "  '${db}': present in catalog — cross-reference against the consumer/compose lists above. If this is a database you don't recognize, STOP and investigate before running cutover.sh."
+  fi
+done <<< "$LIVE_DBS"
+
+log "PASS (informational): catalog/consumer enumeration complete — review the lists above."
+log "=== Preflight complete. Backup: ${BACKUP_FILE} (${BACKUP_SIZE}) ==="
+log "=== Hard gates (disk space, backup) both passed. Proceed to cutover.sh after reviewing the catalog/consumer lists above. ==="

postgresql/cutover/rollback.sh

@@ -0,0 +1,317 @@
+#!/usr/bin/env bash
+# ADR-0001 Phase 3 Cutover — rollback.sh
+#
+# STANDALONE and callable independently AT ANY POINT — not just as
+# cutover.sh's failure handler. Detects current state and takes the
+# minimum action to restore the pre-cutover configuration.
+#
+# Run DIRECTLY on docker-2 via SSH:
+#   ssh root@192.168.4.32
+#   bash /volume1/docker/compose-files/postgresql/cutover/rollback.sh ["reason text"]
+#
+# Idempotent: safe to run multiple times, safe to run if nothing is
+# actually mid-cutover (it will detect that and exit cleanly).
+#
+# ⚠️  IMPORTANT ASYMMETRY — READ BEFORE RUNNING MANUALLY LONG AFTER A
+#     CUTOVER YOU BELIEVED SUCCEEDED:
+# Once legacy has been stopped (cutover.sh Phase 10) AND the new leader
+# (patroni-1) has been genuinely serving writes for a meaningful period,
+# "rolling back" is NOT a free flag-flip anymore — it means demoting a
+# leader that may hold real production writes newer than legacy's last
+# known state, and restoring legacy would risk DISCARDING those writes.
+# This script defends against that specific danger with a grace-window
+# check (see "Case E" below): if legacy is stopped and the new leader has
+# been up longer than GRACE_WINDOW_SECONDS, this script REFUSES to
+# automatically restore legacy and instead prints manual guidance. This
+# is deliberate — do not bypass it by hand without actually reconciling
+# data first (e.g. exporting recent writes from patroni-1 before touching
+# anything).
+#
+# ── FIX LOG ─────────────────────────────────────────────────────────────
+# First real run: consumer re-verification below flagged
+# woodpecker_woodpecker-server as failed because its /healthz endpoint
+# returned HTTP 204 — a legitimate "healthy, no content" response, not a
+# failure. FIXED: health checks now accept any 2xx status code, matching
+# the same fix applied to cutover.sh.
+#
+# Later addition (no new incident, proactive) — SESSION LOGGING. This
+# script is explicitly the ONE piece of the whole procedure RUNBOOK.md
+# flags as needing manual intervention if it itself fails partway (see
+# the asymmetry warning above and cutover.sh's trigger_rollback()) — which
+# makes it the single highest-value place in this entire cutover/rollback
+# suite to have a durable transcript, since it's the exact scenario where
+# nobody may be watching a live terminal (rollback.sh is invoked
+# AUTOMATICALLY by cutover.sh on any failed phase) and where getting the
+# post-incident story right matters most. FIXED: every run now persists
+# its full stdout/stderr transcript to BACKUP_DIR (the same directory the
+# pg_dumpall backup lands in) — either appending to an already-open
+# CUTOVER_SESSION_LOG if invoked as a child of cutover.sh (one merged
+# transcript covering the whole session), or opening its own timestamped
+# log if run standalone (e.g. by the operator, long after the fact, per
+# the asymmetry warning above). See ADR-0001 note, Session Update 9.
+set -uo pipefail
+
+REASON="${1:-<no reason given — invoked directly>}"
+CUTOVER_DIR="/volume1/docker/compose-files/postgresql/cutover"
+HA_STACK="postgresqlha"
+LEGACY_STACK="postgresql"
+LEGACY_SERVICE="postgresql_postgresql"
+NETWORK_NAME="postgresql_db-backend"
+BACKUP_DIR="/volume1/SMB-docker/backup"
+GRACE_WINDOW_SECONDS=300   # 5 minutes — matches the kind of near-immediate
+                           # failure cutover.sh's own Phase 10 auto-rollback
+                           # path would trigger. Anything older than this is
+                           # treated as "cutover had already stuck" and
+                           # requires explicit manual handling, not blind
+                           # automation.
+
+# ── SESSION LOGGING (added — see ADR-0001 note, Session Update 9) ────────
+# Same location as preflight.sh's backup and cutover.sh's session log —
+# BACKUP_DIR. If a parent cutover.sh already exported CUTOVER_SESSION_LOG,
+# append to that single merged transcript; otherwise (standalone
+# invocation, including a long-after-the-fact manual run per the
+# asymmetry warning above) open our own timestamped log. tee mirrors to
+# the real stdout/stderr too, so console/SSH-visible output is unchanged.
+mkdir -p "$BACKUP_DIR"
+if [ -z "${CUTOVER_SESSION_LOG:-}" ]; then
+  CUTOVER_SESSION_LOG="${BACKUP_DIR}/rollback-standalone-$(date +%Y%m%d-%H%M%S).log"
+  export CUTOVER_SESSION_LOG
+fi
+exec > >(tee -a "$CUTOVER_SESSION_LOG") 2>&1
+
+# Consumer checks reused from cutover.sh's Phase 9 list, kept in sync
+# manually — see RUNBOOK.md section 1 dependency map if this list changes.
+declare -A CONSUMER_HEALTH_URLS=(
+  ["auth_authentik-server"]="https://auth.bryanmail.net/-/health/live/"
+  ["git_gitea-server"]="https://git.bryanmail.net/api/healthz"
+  ["woodpecker_woodpecker-server"]="https://woodpecker.bryanmail.net/healthz"
+)
+
+log()  { echo "[rollback $(date +%H:%M:%S)] $*"; }
+warn() { echo "[rollback WARN $(date +%H:%M:%S)] $*" >&2; }
+die()  { echo "[rollback FATAL $(date +%H:%M:%S)] $*" >&2; exit 1; }
+
+# Returns success (0) if $1 looks like a 2xx HTTP status code.
+is_2xx() { [[ "$1" =~ ^2[0-9][0-9]$ ]]; }
+
+log "=== rollback.sh invoked ==="
+log "Reason: ${REASON}"
+log "Full session transcript: ${CUTOVER_SESSION_LOG}"
+
+legacy_cid() { docker ps -q --filter "name=${LEGACY_SERVICE}" | head -1; }
+
+http_get_via_probe() {
+  # Uses a disposable probe container rather than assuming legacy is up —
+  # rollback.sh must work even when legacy has already been stopped.
+  local host="$1" port="$2" path="$3"
+  docker run --rm --network "${NETWORK_NAME}" alpine:3 sh -c "
+    apk add -q --no-cache bash >/dev/null 2>&1
+    exec 3<>/dev/tcp/${host}/${port} || exit 99
+    printf 'GET ${path} HTTP/1.0\r\nHost: ${host}\r\nConnection: close\r\n\r\n' >&3
+    cat <&3
+  " 2>/dev/null
+}
+
+# ── Step 0: detect current state ────────────────────────────────────────
+log "--- Detecting current state ---"
+
+HA_STACK_EXISTS=0
+if docker stack ls --format '{{.Name}}' 2>/dev/null | grep -qx "${HA_STACK}"; then
+  HA_STACK_EXISTS=1
+fi
+log "postgresqlha stack exists: ${HA_STACK_EXISTS}"
+
+LEGACY_REPLICAS=$(docker service inspect "${LEGACY_SERVICE}" --format '{{.Spec.Mode.Replicated.Replicas}}' 2>/dev/null || echo "unknown")
+log "Legacy service (${LEGACY_SERVICE}) desired replicas: ${LEGACY_REPLICAS}"
+
+LEGACY_READONLY="unknown"
+LEGACY_CID="$(legacy_cid)"
+if [ -n "$LEGACY_CID" ]; then
+  LEGACY_READONLY=$(docker exec "$LEGACY_CID" bash -c 'psql -U "$POSTGRES_USER" -tAc "SHOW default_transaction_read_only;"' 2>/dev/null | tr -d '[:space:]')
+fi
+log "Legacy read-only state: ${LEGACY_READONLY}"
+
+PATRONI_LEADER_ROLE="unreachable"
+PATRONI_LEADER_SINCE=""
+if [ "$HA_STACK_EXISTS" -eq 1 ]; then
+  RESP=$(http_get_via_probe patroni-1 8008 "/patroni")
+  if echo "$RESP" | grep -q '"role"'; then
+    PATRONI_LEADER_ROLE=$(echo "$RESP" | grep -o '"role"[^,}]*' | sed 's/.*: *"//;s/"$//')
+  fi
+fi
+log "patroni-1 role (if reachable): ${PATRONI_LEADER_ROLE}"
+
+ALIAS_LIVE=0
+if [ -n "$LEGACY_CID" ]; then
+  RESOLVE_OUT=$(docker exec "$LEGACY_CID" bash -c "getent hosts postgresql" 2>/dev/null)
+elif [ "$HA_STACK_EXISTS" -eq 1 ]; then
+  RESOLVE_OUT=$(docker run --rm --network "${NETWORK_NAME}" alpine:3 sh -c "getent hosts postgresql" 2>/dev/null)
+else
+  RESOLVE_OUT=""
+fi
+NUM_IPS=$(echo "$RESOLVE_OUT" | awk 'NF{print $1}' | sort -u | wc -l)
+log "'postgresql' currently resolves to ${NUM_IPS} distinct IP(s): ${RESOLVE_OUT}"
+if [ "$NUM_IPS" -ge 2 ] || { [ "$LEGACY_REPLICAS" = "0" ] && [ "$HA_STACK_EXISTS" -eq 1 ]; }; then
+  ALIAS_LIVE=1
+fi
+
+# ── Step 1: classify and act ─────────────────────────────────────────────
+
+if [ "$HA_STACK_EXISTS" -eq 0 ]; then
+  log "Case A: no postgresqlha stack exists. Nothing to roll back."
+  if [ "$LEGACY_REPLICAS" != "1" ]; then
+    warn "Legacy replicas = ${LEGACY_REPLICAS}, expected 1 even with no HA stack present. Restoring."
+    docker service scale "${LEGACY_SERVICE}=1"
+  fi
+  if [ "$LEGACY_READONLY" = "on" ]; then
+    warn "Legacy is read-only with no HA stack present — reversing (this should not normally happen)."
+    docker exec "$(legacy_cid)" bash -c 'psql -U "$POSTGRES_USER" -c "ALTER SYSTEM SET default_transaction_read_only = off; SELECT pg_reload_conf();"'
+  fi
+  log "=== Rollback complete: nothing was in progress. Pre-cutover state confirmed/restored. ==="
+  exit 0
+fi
+
+if [ "$LEGACY_REPLICAS" = "0" ]; then
+  # Legacy has been stopped — this is Case E territory. Must determine
+  # whether we're in the "just happened, cutover.sh's own auto-rollback"
+  # window, or a genuinely-completed-and-aged cutover.
+  log "Legacy is scaled to 0 — checking how long patroni-1 has held the leader role before deciding how to proceed..."
+  SINCE_INFO=$(http_get_via_probe patroni-1 8008 "/patroni")
+  # Patroni's /patroni response includes a timestamp; if we can't parse
+  # one confidently, fail SAFE (treat as "aged", require manual review)
+  # rather than guessing young.
+  LEADER_TIMESTAMP=$(echo "$SINCE_INFO" | grep -o '"timestamp"[^,}]*' | sed 's/.*: *"//;s/"$//')
+  AGE_SECONDS=999999
+  if [ -n "$LEADER_TIMESTAMP" ]; then
+    LEADER_EPOCH=$(date -d "$LEADER_TIMESTAMP" +%s 2>/dev/null || echo "")
+    NOW_EPOCH=$(date +%s)
+    if [ -n "$LEADER_EPOCH" ]; then
+      AGE_SECONDS=$((NOW_EPOCH - LEADER_EPOCH))
+    fi
+  fi
+  log "Estimated age signal: ${AGE_SECONDS}s (grace window: ${GRACE_WINDOW_SECONDS}s)"
+
+  if [ "$AGE_SECONDS" -gt "$GRACE_WINDOW_SECONDS" ]; then
+    die "
+════════════════════════════════════════════════════════════════════════
+REFUSING to automatically roll back: legacy is stopped and the new
+leader (patroni-1) appears to have been running longer than the
+${GRACE_WINDOW_SECONDS}s grace window. Automatically restoring legacy
+now risks DISCARDING real production writes that landed on patroni-1
+since promotion.
+
+This requires MANUAL handling:
+  1. Confirm whether patroni-1 genuinely has newer data than legacy's
+     last synced point (check application data directly, or compare
+     row counts/timestamps on tables you know are actively written).
+  2. If patroni-1's data is authoritative (i.e. the cutover basically
+     succeeded and this is a LATER problem, not a failed cutover):
+       - Do NOT run this rollback path. Treat this as 'fix forward' —
+         investigate why rollback.sh was invoked and address that
+         specific problem instead (e.g. a single consumer's connection
+         string, not the whole DB layer).
+  3. If you genuinely need to revert to legacy despite this:
+       - Take a fresh pg_dumpall from patroni-1 FIRST (same method as
+         preflight.sh), so no data is lost even if you proceed.
+       - Manually restore that dump into legacy before scaling it back
+         up, OR manually apply just the delta if you can identify it.
+       - Then re-run this script, or reproduce the case-D steps below
+         by hand once legacy has legacy's data superseded correctly.
+
+Reason this rollback.sh run was invoked: ${REASON}
+Full session transcript up to this point: ${CUTOVER_SESSION_LOG}
+════════════════════════════════════════════════════════════════════════
+"
+  fi
+
+  log "Case E (within grace window — treating as an immediate post-Phase-10 failure, matches cutover.sh's own auto-rollback trigger point). Proceeding automatically."
+  log "Restoring legacy service to 1 replica..."
+  docker service scale "${LEGACY_SERVICE}=1"
+  log "Waiting for legacy to become healthy..."
+  for i in $(seq 1 24); do
+    STATE=$(docker service ps "${LEGACY_SERVICE}" --filter "desired-state=running" --format '{{.CurrentState}}' 2>/dev/null | head -1)
+    log "  legacy state: ${STATE}"
+    echo "$STATE" | grep -q "^Running" && break
+    sleep 5
+  done
+  LEGACY_CID="$(legacy_cid)"
+  if [ -z "$LEGACY_CID" ]; then
+    die "Legacy service was scaled back to 1 but no running container found after waiting. MANUAL INTERVENTION REQUIRED — check 'docker service logs ${LEGACY_SERVICE} --tail 100' and 'docker stack ps ${LEGACY_STACK} --no-trunc' directly."
+  fi
+  log "Legacy container back: ${LEGACY_CID}"
+  # fall through to alias-removal and HA-stack-teardown below, shared with case D
+fi
+
+if [ "$LEGACY_READONLY" = "on" ]; then
+  log "Reversing legacy read-only flag..."
+  LEGACY_CID="$(legacy_cid)"
+  docker exec "$LEGACY_CID" bash -c 'psql -U "$POSTGRES_USER" -c "ALTER SYSTEM SET default_transaction_read_only = off; SELECT pg_reload_conf();"'
+  VERIFY=$(docker exec "$LEGACY_CID" bash -c 'psql -U "$POSTGRES_USER" -tAc "SHOW default_transaction_read_only;"' 2>/dev/null | tr -d '[:space:]')
+  if [ "$VERIFY" != "off" ]; then
+    die "Attempted to reverse read-only flag but SHOW still reports '${VERIFY}'. MANUAL INTERVENTION REQUIRED — do not leave legacy read-only if consumers are pointed at it."
+  fi
+  log "Confirmed: legacy is read-write again."
+fi
+
+if [ "$ALIAS_LIVE" -eq 1 ]; then
+  log "postgresql/db aliases are currently held by haproxy — removing by tearing down the HA stack entirely below (simplest reliable removal; redeploying staging-without-alias was considered but full teardown is more certain and this stack has no data worth preserving once we're rolling back)."
+fi
+
+log "--- Tearing down postgresqlha stack ---"
+docker stack rm "${HA_STACK}"
+log "Waiting for tasks to actually finish shutting down (docker stack rm returns immediately, tasks take several seconds)..."
+for i in $(seq 1 24); do
+  REMAINING=$(docker stack ps "${HA_STACK}" 2>/dev/null | grep -c . || true)
+  if [ "$REMAINING" -eq 0 ]; then
+    break
+  fi
+  sleep 5
+done
+REMAINING=$(docker stack ps "${HA_STACK}" 2>/dev/null | grep -c . || true)
+if [ "$REMAINING" -gt 0 ]; then
+  warn "postgresqlha still shows ${REMAINING} task(s) after 120s wait. Proceeding to wipe data dirs anyway is UNSAFE while tasks are still shutting down — stopping here."
+  die "MANUAL INTERVENTION REQUIRED: check 'docker stack ps ${HA_STACK} --no-trunc' before wiping any data dirs by hand."
+fi
+log "Confirmed: postgresqlha stack fully removed."
+
+log "--- Wiping postgresqlha data dirs ---"
+for d in etcd-1-data etcd-2-data etcd-3-data patroni-0-data patroni-1-data; do
+  DIR="/volume1/docker/PostgreSQL/${d}"
+  if [ -d "$DIR" ]; then
+    rm -rf "$DIR"
+    mkdir -p "$DIR"
+    log "  wiped and recreated: ${DIR}"
+  fi
+done
+log "Data dirs wiped (mirrors the pgha-test dry-run teardown precedent — parent dirs recreated with mkdir -p since Swarm bind mounts don't auto-create missing host parent dirs)."
+
+log "--- Re-verifying consumers against restored legacy ---"
+CONSUMER_FAIL=0
+for svc in "${!CONSUMER_HEALTH_URLS[@]}"; do
+  url="${CONSUMER_HEALTH_URLS[$svc]}"
+  CODE=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 10 "$url")
+  log "  ${svc} -> ${url} -> HTTP ${CODE}"
+  if ! is_2xx "$CODE"; then
+    warn "  ${svc} did not return a 2xx status (got ${CODE})"
+    CONSUMER_FAIL=1
+  fi
+done
+if [ "$CONSUMER_FAIL" -ne 0 ]; then
+  warn "One or more consumers did not verify healthy post-rollback. This is now a DIFFERENT problem than the original cutover attempt — legacy itself may need direct attention. Check 'docker service logs <service> --tail 100' for each flagged consumer."
+else
+  log "All checked consumers verified healthy against restored legacy."
+fi
+
+FINAL_RESOLVE=$(docker exec "$(legacy_cid)" bash -c "getent hosts postgresql" 2>/dev/null)
+log "Final 'postgresql' resolution: ${FINAL_RESOLVE}"
+
+log "=== ROLLBACK COMPLETE ==="
+log "Summary:"
+log "  - postgresqlha stack: removed, data dirs wiped and recreated empty"
+log "  - legacy service: running, read-write, sole answer for 'postgresql'/'db'"
+log "  - consumer verification: $([ "$CONSUMER_FAIL" -eq 0 ] && echo 'all passed' || echo 'SOME FAILED — see warnings above, needs manual follow-up')"
+log "  - original invocation reason: ${REASON}"
+log "  - full session transcript: ${CUTOVER_SESSION_LOG}"
+log "Legacy's data directory itself (/volume1/docker/PostgreSQL/data-17) was"
+log "never touched by this script — only postgresqlha's own dirs were wiped."
+exit 0

postgresql/cutover/test/pgha-dryrun.yaml

@@ -0,0 +1,388 @@
+# ─────────────────────────────────────────────────────────────────────────
+# postgresql/cutover/test/pgha-dryrun.yaml — DISPOSABLE dry-run stack
+#
+# TRIGGER NOTE (2026-08-01): touched to force Woodpecker to re-run secret
+# provisioning for the `postgresql` stack case after regenerating
+# postgresql_replication_password to exclude &<>" (see ADR-0001 Dry-Run
+# Debugging Log note for context — Spilo's pystache config templating
+# HTML-escapes double-brace {{PGPASSWORD_STANDBY}} substitutions, corrupting
+# any password containing those characters).
+#
+# Validates the ADR-0001 cutover mechanics WITHOUT touching production:
+#   - own overlay network (pgha-test_db-backend), own stack name
+#   - throwaway "legacy" postgres:17 seeded with marker data, carrying the
+#     in-network alias "postgresql" (mirrors prod CLONE_HOST/standby_cluster
+#     host)
+#   - same etcd/patroni/haproxy topology and images as the real staging file
+#   - reuses the real Docker secrets (read-only mounts; harmless)
+#   - disposable data dirs under /volume1/docker/PostgreSQL/dryrun/
+#
+# Deploy:   docker stack deploy -c <this file> pgha-test
+# Teardown: docker stack rm pgha-test && rm -rf /volume1/docker/PostgreSQL/dryrun
+#
+# ─── REVISION (2026-08-01): switched from CLONE_WITH_BASEBACKUP to a
+# Patroni "standby cluster" (continuous streaming) — see rationale below ───
+#
+# WHY THIS CHANGED: the original CLONE_WITH_BASEBACKUP design (still used by
+# postgresql/postgresql.yaml and postgresql/cutover/postgresql-ha-staging.yaml
+# as of this revision — NOT YET ported here) is a ONE-SHOT snapshot: once
+# pg_basebackup completes, the new cluster has ZERO further connection to
+# the old database. Any write landing on the old DB between the snapshot
+# and the actual traffic cutover is silently lost — a real gap given ~13
+# active consumers and a 42GB production database. Patroni's "standby
+# cluster" mode (https://patroni.readthedocs.io/en/latest/standby_cluster.html)
+# instead makes the new cluster's leader ("standby leader") continuously
+# stream from the remote primary indefinitely, right up until an explicit
+# promotion — closing that gap to near-zero. This file is the dry-run
+# validation of that mode, BEFORE porting it to the real staging/production
+# files. Do not port until this dry run passes end-to-end.
+#
+# KEY DIFFERENCES FROM THE CLONE-BASED DESIGN:
+#   - No CLONE_METHOD/CLONE_SCOPE/CLONE_HOST/CLONE_PORT/CLONE_USER/
+#     CLONE_PASSWORD anywhere. Replaced by a `bootstrap.dcs.standby_cluster`
+#     block inside SPILO_CONFIGURATION on BOTH patroni-0 and patroni-1 —
+#     both get it, not just one, because standby_cluster config is written
+#     ONCE into shared DCS state by whichever node wins the initial
+#     bootstrap race (same non-determinism previously handled the same way
+#     for CLONE_*). Per Patroni docs: "these options will be applied only
+#     once during cluster bootstrap, and the only way to change them
+#     afterwards is through DCS."
+#   - `legacy` now needs a REAL replication role named "standby" (matching
+#     PGUSER_STANDBY), not just the PGadmin superuser. CLONE_WITH_BASEBACKUP
+#     ran pg_basebackup as CLONE_USER=PGadmin (a superuser, so no dedicated
+#     role was needed); standby_cluster streaming instead authenticates
+#     using the cluster's own replication identity (PGUSER_STANDBY/
+#     PGPASSWORD_STANDBY), which never existed on "legacy" before — added
+#     via a second initdb.d hook, reading the password from the SAME
+#     postgresql_replication_password secret already used between
+#     patroni-0/patroni-1 for their own internal replication.
+#   - patroni-1 no longer references CLONE_* either (it never talked to
+#     legacy directly in the old design either — it bootstrapped from
+#     whichever node held the leader lock, exactly as a Patroni "cascade
+#     replica" does in standby-cluster mode too — this is actually simpler
+#     now, not different).
+#   - create_replica_methods references "basebackup_fast_xlog" — this is
+#     Spilo's OWN generated method key (confirmed from configure_spilo.py's
+#     TEMPLATE constant, fetched directly from zalando/spilo source this
+#     project), not the generic "basebackup" name shown in Patroni's own
+#     docs example (that name assumes a plain Patroni method map without
+#     Spilo's wrapper naming).
+#
+# BUG FOUND DURING THIS DRY RUN (2026-08-01, unrelated to standby_cluster
+# itself, but discovered while testing it): Spilo's pystache templating
+# engine renders `password: '{{PGPASSWORD_STANDBY}}'` using DOUBLE braces
+# in its TEMPLATE constant (confirmed from configure_spilo.py source) —
+# pystache HTML-escapes double-brace substitutions by default (only
+# triple-brace {{{...}}} skips escaping, which Spilo deliberately uses for
+# archive_command elsewhere in the same template for exactly this reason).
+# This means any password containing &, <, >, or " gets corrupted into
+# HTML entities (e.g. & becomes &amp;) inside Patroni's own rendered
+# /run/postgres.yml AND /run/postgresql/pgpass — breaking real
+# password-based Patroni-to-Patroni replication auth (confirmed via direct
+# file inspection: the raw secret was verified clean via `cat | xxd`, but
+# the rendered config had "&amp;" baked in). This surfaced here specifically
+# because standby_cluster's cascade-replica bootstrap (patroni-0 replicating
+# from patroni-1) uses genuine hostssl+md5 auth, unlike legacy's disposable
+# "trust" rule which never actually checks the password. Fix: regenerate
+# postgresql_replication_password to exclude &<>" entirely — done
+# 2026-08-01, this file re-tests with the corrected secret.
+#
+# OPEN QUESTIONS THIS DRY RUN IS DESIGNED TO ANSWER (do not assume the
+# answer — observe actual logs):
+#   - Does Patroni require a pre-created replication slot on "legacy" for
+#     the standby_cluster link (primary_slot_name), or does it work
+#     without one since we haven't set that key? ANSWERED (first attempt,
+#     pre-password-fix): works fine without one — patroni-1 successfully
+#     bootstrapped as standby leader and began streaming from legacy with
+#     no slot configured.
+#   - Does "legacy" (vanilla, unmanaged postgres:17) actually need
+#     postgresql.conf discoverable in PGDATA for Patroni's remote checks?
+#     ANSWERED: yes, works fine — vanilla image keeps it there by default,
+#     no complaint logged.
+#   - Does promotion off the standby cluster (detaching from "legacy" and
+#     becoming a normal read-write cluster) work cleanly and pick up the
+#     very latest streamed WAL, closing the gap as intended? STILL TO TEST
+#     — this is the actual feature under test, blocked behind the password
+#     bug above on the first attempt, retesting now.
+#
+# ─── Fixes still carried forward from the CLONE_WITH_BASEBACKUP dry run ───
+#
+# NOTE: command: blocks use $$(...) not $(...) — Compose's own variable
+# interpolation parses $( as an attempted ${VAR} reference and fails with
+# "invalid interpolation format" / "you may need to escape any $ with
+# another $". $$ escapes to a literal $ for the shell at runtime. This
+# applies uniformly to EVERY literal $ character anywhere in a command:
+# block, including inside heredocs and nested quoting — Compose scans the
+# raw text before any shell/heredoc logic ever runs, so quoting context
+# doesn't exempt anything from this escaping requirement.
+#
+# NOTE: ETCD3_HOSTS not ETCD_HOSTS — confirmed against zalando/spilo
+# configure_spilo.py: PATRONI_DCS includes both "etcd" (legacy v2 API,
+# python-etcd client) and "etcd3" (v3 API, python-etcd3 client) as
+# distinct DCS backends selected by env var prefix. Our etcd containers
+# (v3.5.9) have the v2 API disabled by default, so ETCD_HOSTS causes
+# Patroni to hit /v2 endpoints that 404. ETCD3_HOSTS selects the correct
+# v3-API client.
+#
+# NOTE: "legacy" needs a pg_hba.conf rule permitting REPLICATION-type
+# connections (distinct from normal client connections) — this is STILL
+# required under standby_cluster mode, since it's still a genuine
+# replication connection under the hood, just a continuous one instead of
+# one-shot. Fixed via a /docker-entrypoint-initdb.d/ hook script (official
+# extension point). "trust" is acceptable ONLY because this container is
+# fully disposable.
+#
+# NOTE: patroni-0/patroni-1 override bootstrap.post_init via
+# SPILO_CONFIGURATION (Spilo's documented, supported mechanism for
+# overriding any generated Patroni config — configure_spilo.py deep-merges
+# user-supplied SPILO_CONFIGURATION on top of its own generated config).
+# This is needed because Spilo's own /scripts/post_init.sh hardcodes
+# "ALTER VIEW ... OWNER TO postgres" with no way to parameterize that role
+# name via env vars, AND its embedded _zmon_schema.dump does
+# "SET ROLE TO postgres; CREATE EXTENSION plpython3u" which requires
+# genuine superuser. Our wrapper creates a harmless, idempotent
+# NOLOGIN+SUPERUSER "postgres" role first (SUPERUSER carries no real
+# exposure since NOLOGIN means it can never authenticate a connection),
+# then execs Spilo's real, UNMODIFIED post_init.sh with all original
+# arguments passed through — Zalando's script itself is never patched or
+# forked. This is UNRELATED to the standby_cluster change and still
+# required for the same reasons as before.
+# ─────────────────────────────────────────────────────────────────────────
+version: "3.6"
+
+services:
+
+  # Stand-in for the production single-instance postgres (remote primary
+  # for standby_cluster streaming)
+  legacy:
+    image: public.ecr.aws/docker/library/postgres:17
+    hostname: db
+    entrypoint: ["/bin/sh", "-c"]
+    command:
+      - |
+        mkdir -p /docker-entrypoint-initdb.d
+        cat > /docker-entrypoint-initdb.d/zz-enable-replication.sh <<'EOF'
+        #!/bin/sh
+        echo "host replication all all trust" >> "$$PGDATA/pg_hba.conf"
+        EOF
+        chmod +x /docker-entrypoint-initdb.d/zz-enable-replication.sh
+        cat > /docker-entrypoint-initdb.d/zz-create-standby-role.sql <<SQL
+        CREATE ROLE standby WITH REPLICATION LOGIN PASSWORD '$$(cat /run/secrets/postgresql_replication_password)';
+        SQL
+        exec docker-entrypoint.sh postgres
+    environment:
+      POSTGRES_USER: PGadmin
+      POSTGRES_PASSWORD_FILE: /run/secrets/postgresql_password
+    secrets:
+      - postgresql_password
+      - postgresql_replication_password
+    volumes:
+      - /volume1/docker/PostgreSQL/dryrun/legacy:/var/lib/postgresql/data
+    networks:
+      db-backend:
+        aliases:
+          - postgresql
+          - db
+    deploy:
+      placement:
+        constraints:
+          - node.hostname == docker-2
+
+  etcd-1:
+    image: quay.io/coreos/etcd:v3.5.9
+    hostname: etcd-1
+    command:
+      - etcd
+      - --name=etcd-1
+      - --data-dir=/etcd-data
+      - --initial-advertise-peer-urls=http://etcd-1:2380
+      - --listen-peer-urls=http://0.0.0.0:2380
+      - --listen-client-urls=http://0.0.0.0:2379
+      - --advertise-client-urls=http://etcd-1:2379
+      - --initial-cluster=etcd-1=http://etcd-1:2380,etcd-2=http://etcd-2:2380,etcd-3=http://etcd-3:2380
+      - --initial-cluster-state=new
+      - --initial-cluster-token=pgha-test-etcd
+    volumes:
+      - /volume1/docker/PostgreSQL/dryrun/etcd-1:/etcd-data
+    networks:
+      - db-backend
+    deploy:
+      placement:
+        constraints:
+          - node.hostname == docker-1
+
+  etcd-2:
+    image: quay.io/coreos/etcd:v3.5.9
+    hostname: etcd-2
+    command:
+      - etcd
+      - --name=etcd-2
+      - --data-dir=/etcd-data
+      - --initial-advertise-peer-urls=http://etcd-2:2380
+      - --listen-peer-urls=http://0.0.0.0:2380
+      - --listen-client-urls=http://0.0.0.0:2379
+      - --advertise-client-urls=http://etcd-2:2379
+      - --initial-cluster=etcd-1=http://etcd-1:2380,etcd-2=http://etcd-2:2380,etcd-3=http://etcd-3:2380
+      - --initial-cluster-state=new
+      - --initial-cluster-token=pgha-test-etcd
+    volumes:
+      - /volume1/docker/PostgreSQL/dryrun/etcd-2:/etcd-data
+    networks:
+      - db-backend
+    deploy:
+      placement:
+        constraints:
+          - node.hostname == docker-2
+
+  etcd-3:
+    image: quay.io/coreos/etcd:v3.5.9
+    hostname: etcd-3
+    command:
+      - etcd
+      - --name=etcd-3
+      - --data-dir=/etcd-data
+      - --initial-advertise-peer-urls=http://etcd-3:2380
+      - --listen-peer-urls=http://0.0.0.0:2380
+      - --listen-client-urls=http://0.0.0.0:2379
+      - --advertise-client-urls=http://etcd-3:2379
+      - --initial-cluster=etcd-1=http://etcd-1:2380,etcd-2=http://etcd-2:2380,etcd-3=http://etcd-3:2380
+      - --initial-cluster-state=new
+      - --initial-cluster-token=pgha-test-etcd
+    volumes:
+      - /volume1/docker/PostgreSQL/dryrun/etcd-3:/etcd-data
+    networks:
+      - db-backend
+    deploy:
+      placement:
+        constraints:
+          - node.hostname == docker-3
+
+  patroni-0:
+    image: ghcr.io/zalando/spilo-17:4.0-p3
+    hostname: patroni-0
+    command:
+      - /bin/sh
+      - -c
+      - |
+        export PGPASSWORD_SUPERUSER="$$(cat /run/secrets/postgresql_password)"
+        export PGPASSWORD_STANDBY="$$(cat /run/secrets/postgresql_replication_password)"
+        export PATRONI_RESTAPI_PASSWORD="$$(cat /run/secrets/postgresql_patroni_password)"
+        mkdir -p /scripts
+        cat > /scripts/post_init_wrapper.sh <<'WRAP'
+        #!/bin/bash
+        set -e
+        psql -d "$$2" -v ON_ERROR_STOP=1 -c 'DO $$do$$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = $$x$$postgres$$x$$) THEN CREATE ROLE postgres WITH SUPERUSER NOLOGIN; END IF; END $$do$$;'
+        exec /scripts/post_init.sh "$$@"
+        WRAP
+        chmod +x /scripts/post_init_wrapper.sh
+        exec /bin/sh /launch.sh init
+    environment:
+      SCOPE: pgha-test
+      PATRONI_NAME: patroni-0
+      ETCD3_HOSTS: '"etcd-1:2379","etcd-2:2379","etcd-3:2379"'
+      PGUSER_SUPERUSER: PGadmin
+      PGUSER_STANDBY: standby
+      PATRONI_RESTAPI_USERNAME: patroni
+      PGROOT: /home/postgres/pgdata/pgroot
+      # Standby-cluster mode: continuous streaming from "legacy" instead of
+      # a one-shot CLONE_WITH_BASEBACKUP snapshot. Written once into shared
+      # DCS state by whichever of patroni-0/patroni-1 wins the initial
+      # bootstrap race — both nodes carry the identical block for that
+      # reason. "postgresql" is legacy's network alias (mirrors prod
+      # CLONE_HOST naming). basebackup_fast_xlog is Spilo's own generated
+      # replica-method key (confirmed from configure_spilo.py source).
+      SPILO_CONFIGURATION: |
+        bootstrap:
+          post_init: /scripts/post_init_wrapper.sh "zalandos"
+          dcs:
+            standby_cluster:
+              host: postgresql
+              port: 5432
+              create_replica_methods:
+              - basebackup_fast_xlog
+    secrets:
+      - postgresql_password
+      - postgresql_replication_password
+      - postgresql_patroni_password
+    volumes:
+      - /volume1/docker/PostgreSQL/dryrun/patroni-0:/home/postgres/pgdata
+    networks:
+      - db-backend
+    deploy:
+      placement:
+        constraints:
+          - node.labels.pg-role == primary
+
+  patroni-1:
+    image: ghcr.io/zalando/spilo-17:4.0-p3
+    hostname: patroni-1
+    command:
+      - /bin/sh
+      - -c
+      - |
+        export PGPASSWORD_SUPERUSER="$$(cat /run/secrets/postgresql_password)"
+        export PGPASSWORD_STANDBY="$$(cat /run/secrets/postgresql_replication_password)"
+        export PATRONI_RESTAPI_PASSWORD="$$(cat /run/secrets/postgresql_patroni_password)"
+        mkdir -p /scripts
+        cat > /scripts/post_init_wrapper.sh <<'WRAP'
+        #!/bin/bash
+        set -e
+        psql -d "$$2" -v ON_ERROR_STOP=1 -c 'DO $$do$$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = $$x$$postgres$$x$$) THEN CREATE ROLE postgres WITH SUPERUSER NOLOGIN; END IF; END $$do$$;'
+        exec /scripts/post_init.sh "$$@"
+        WRAP
+        chmod +x /scripts/post_init_wrapper.sh
+        exec /bin/sh /launch.sh init
+    environment:
+      SCOPE: pgha-test
+      PATRONI_NAME: patroni-1
+      ETCD3_HOSTS: '"etcd-1:2379","etcd-2:2379","etcd-3:2379"'
+      PGUSER_SUPERUSER: PGadmin
+      PGUSER_STANDBY: standby
+      PATRONI_RESTAPI_USERNAME: patroni
+      PGROOT: /home/postgres/pgdata/pgroot
+      # Same standby_cluster block as patroni-0 — see comment there for why
+      # both nodes carry it identically.
+      SPILO_CONFIGURATION: |
+        bootstrap:
+          post_init: /scripts/post_init_wrapper.sh "zalandos"
+          dcs:
+            standby_cluster:
+              host: postgresql
+              port: 5432
+              create_replica_methods:
+              - basebackup_fast_xlog
+    secrets:
+      - postgresql_password
+      - postgresql_replication_password
+      - postgresql_patroni_password
+    volumes:
+      - /volume1/docker/PostgreSQL/dryrun/patroni-1:/home/postgres/pgdata
+    networks:
+      - db-backend
+    deploy:
+      placement:
+        constraints:
+          - node.labels.pg-role == replica
+
+  haproxy:
+    image: haproxy:2.9-alpine
+    hostname: haproxy
+    volumes:
+      - /volume1/docker/compose-files/postgresql/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
+    networks:
+      - db-backend
+    deploy:
+      mode: global
+
+networks:
+  db-backend:
+    attachable: true
+    driver: overlay
+
+secrets:
+  postgresql_password:
+    external: true
+  postgresql_replication_password:
+    external: true
+  postgresql_patroni_password:
+    external: true

postgresql/haproxy.cfg

@@ -0,0 +1,37 @@
+# ─────────────────────────────────────────────────────────────────────────
+# postgresql/haproxy.cfg — Patroni-aware TCP router (ADR-0001)
+# Routes to whichever Patroni node currently answers 200 on GET /primary.
+# No credentials needed here — Patroni's REST API leaves GET/health-check
+# endpoints (/primary, /replica, /health) open; only PATCH/POST config
+# endpoints require postgresql_patroni_password basic-auth.
+# ─────────────────────────────────────────────────────────────────────────
+
+global
+    maxconn 200
+    log stdout format raw local0
+
+defaults
+    log global
+    mode tcp
+    retries 2
+    timeout client 30m
+    timeout connect 4s
+    timeout server 30m
+    timeout check 5s
+
+# Internal-only stats page — reachable within postgresql_db-backend overlay
+# network only (no published port in postgresql.yaml), not exposed externally.
+listen stats
+    mode http
+    bind *:7000
+    stats enable
+    stats uri /
+    stats refresh 10s
+
+listen postgres
+    bind *:5432
+    option httpchk GET /primary
+    http-check expect status 200
+    default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions maxconn 100
+    server patroni-0 patroni-0:5432 check port 8008
+    server patroni-1 patroni-1:5432 check port 8008

postgresql/postgresql.yaml

@@ -0,0 +1,319 @@
+# ─────────────────────────────────────────────────────────────────────────
+# postgresql/postgresql.yaml — Patroni + etcd + HAProxy HA stack (ADR-0001)
+# BOOTSTRAP TIER — manual deploy only via:
+#   bash /volume1/docker/compose-files/deploy/stack-deploy.sh postgresql
+#
+# Env var names for Spilo/Patroni verified against upstream source
+# (zalando/spilo ENVIRONMENT.rst, configure_spilo.py, launch.sh) and the
+# Patroni ENVIRONMENT docs — NOT guessed. Spilo has no native Docker-secret
+# _FILE suffix support, so patroni-0/patroni-1 use a command wrapper to
+# read secrets from /run/secrets and export them as the plain env vars
+# Spilo's configure_spilo.py actually expects, before invoking the image's
+# real entrypoint chain (/launch.sh init).
+#
+# NOTE: command: blocks use $$(...) not $(...) — Compose's own variable
+# interpolation parses $( as an attempted ${VAR} reference and fails with
+# "invalid interpolation format" / "you may need to escape any $ with
+# another $". $$ escapes to a literal $ for the shell at runtime. This
+# applies uniformly to EVERY literal $ character anywhere in a command:
+# block, including inside heredocs and nested quoting — Compose scans the
+# raw text before any shell/heredoc logic ever runs, so quoting context
+# doesn't exempt anything from this escaping requirement. (Confirmed the
+# hard way in the pgha-test dry run — see "ADR-0001 Dry-Run Debugging Log"
+# note — this file originally had the unescaped $(...) bug.)
+#
+# NOTE: ETCD3_HOSTS not ETCD_HOSTS — confirmed against zalando/spilo
+# configure_spilo.py: PATRONI_DCS includes both "etcd" (legacy v2 API,
+# python-etcd client) and "etcd3" (v3 API, python-etcd3 client) as
+# distinct DCS backends selected by env var prefix. Our etcd containers
+# (v3.5.9) have the v2 API disabled by default, so ETCD_HOSTS causes
+# Patroni to hit /v2 endpoints that 404. ETCD3_HOSTS selects the correct
+# v3-API client. (Also originally wrong in this file — fixed after dry run.)
+#
+# NOTE: patroni-0/patroni-1 override bootstrap.post_init via
+# SPILO_CONFIGURATION (Spilo's documented, supported mechanism for
+# overriding any generated Patroni config — configure_spilo.py deep-merges
+# user-supplied SPILO_CONFIGURATION on top of its own generated config).
+# This is needed because Spilo's own /scripts/post_init.sh hardcodes
+# "ALTER VIEW ... OWNER TO postgres" with no way to parameterize that role
+# name via env vars. Since our superuser is PGUSER_SUPERUSER=PGadmin (to
+# match production), there is no role literally named "postgres" in the
+# cloned data, so Spilo's unmodified script fails with
+# 'ERROR: role "postgres" does not exist'. The override points
+# bootstrap.post_init at our own wrapper script instead, which creates a
+# harmless, idempotent "postgres" role (WITH SUPERUSER NOLOGIN — SUPERUSER
+# is required because Spilo's own _zmon_schema.dump does
+# "SET ROLE TO postgres; CREATE EXTENSION plpython3u", which needs real
+# superuser privileges, not just role existence; NOLOGIN means it can never
+# be used to establish a real connection, so this carries no security
+# exposure), then execs Spilo's real, UNMODIFIED post_init.sh with all
+# original arguments passed through — Zalando's script itself is never
+# patched or forked. Both fixes (role existence + SUPERUSER) were found
+# and validated end-to-end in the pgha-test dry run before being ported
+# here — see the dry-run debugging note for full detail.
+# ─────────────────────────────────────────────────────────────────────────
+version: "3.6"
+
+services:
+
+  # ── etcd (3-node Raft quorum — NOT data replicas, ~150MB RAM total) ──────
+  etcd-1:
+    image: quay.io/coreos/etcd:v3.5.9
+    hostname: etcd-1
+    command:
+      - etcd
+      - --name=etcd-1
+      - --data-dir=/etcd-data
+      - --initial-advertise-peer-urls=http://etcd-1:2380
+      - --listen-peer-urls=http://0.0.0.0:2380
+      - --listen-client-urls=http://0.0.0.0:2379
+      - --advertise-client-urls=http://etcd-1:2379
+      - --initial-cluster=etcd-1=http://etcd-1:2380,etcd-2=http://etcd-2:2380,etcd-3=http://etcd-3:2380
+      - --initial-cluster-state=new
+      - --initial-cluster-token=postgresql-ha-etcd
+    volumes:
+      - /volume1/docker/PostgreSQL/etcd-1-data:/etcd-data
+    networks:
+      - postgresql_db-backend
+    deploy:
+      placement:
+        constraints:
+          - node.hostname == docker-1
+
+  etcd-2:
+    image: quay.io/coreos/etcd:v3.5.9
+    hostname: etcd-2
+    command:
+      - etcd
+      - --name=etcd-2
+      - --data-dir=/etcd-data
+      - --initial-advertise-peer-urls=http://etcd-2:2380
+      - --listen-peer-urls=http://0.0.0.0:2380
+      - --listen-client-urls=http://0.0.0.0:2379
+      - --advertise-client-urls=http://etcd-2:2379
+      - --initial-cluster=etcd-1=http://etcd-1:2380,etcd-2=http://etcd-2:2380,etcd-3=http://etcd-3:2380
+      - --initial-cluster-state=new
+      - --initial-cluster-token=postgresql-ha-etcd
+    volumes:
+      - /volume1/docker/PostgreSQL/etcd-2-data:/etcd-data
+    networks:
+      - postgresql_db-backend
+    deploy:
+      placement:
+        constraints:
+          - node.hostname == docker-2
+
+  etcd-3:
+    image: quay.io/coreos/etcd:v3.5.9
+    hostname: etcd-3
+    command:
+      - etcd
+      - --name=etcd-3
+      - --data-dir=/etcd-data
+      - --initial-advertise-peer-urls=http://etcd-3:2380
+      - --listen-peer-urls=http://0.0.0.0:2380
+      - --listen-client-urls=http://0.0.0.0:2379
+      - --advertise-client-urls=http://etcd-3:2379
+      - --initial-cluster=etcd-1=http://etcd-1:2380,etcd-2=http://etcd-2:2380,etcd-3=http://etcd-3:2380
+      - --initial-cluster-state=new
+      - --initial-cluster-token=postgresql-ha-etcd
+    volumes:
+      - /volume1/docker/PostgreSQL/etcd-3-data:/etcd-data
+    networks:
+      - postgresql_db-backend
+    deploy:
+      placement:
+        constraints:
+          - node.hostname == docker-3
+
+  # ── Patroni / Spilo data replicas (2 total, per user constraint) ────────
+  # Image tag pinned to 4.0-p3 (Zalando postgres-operator's own referenced
+  # default at time of writing) — re-verify against current releases
+  # before actual deploy, as patch tags move. Decision (2026-07-30): staying
+  # on PostgreSQL 17 / this Spilo line for now; PG18 deferred as a separate
+  # future project pending app-compatibility checks. See ADR-0001 notes.
+  patroni-0:
+    image: ghcr.io/zalando/spilo-17:4.0-p3
+    hostname: patroni-0
+    command:
+      - /bin/sh
+      - -c
+      - |
+        export PGPASSWORD_SUPERUSER="$$(cat /run/secrets/postgresql_password)"
+        export PGPASSWORD_STANDBY="$$(cat /run/secrets/postgresql_replication_password)"
+        export PATRONI_RESTAPI_PASSWORD="$$(cat /run/secrets/postgresql_patroni_password)"
+        mkdir -p /scripts
+        cat > /scripts/post_init_wrapper.sh <<'WRAP'
+        #!/bin/bash
+        set -e
+        psql -d "$$2" -v ON_ERROR_STOP=1 -c 'DO $$do$$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = $$x$$postgres$$x$$) THEN CREATE ROLE postgres WITH SUPERUSER NOLOGIN; END IF; END $$do$$;'
+        exec /scripts/post_init.sh "$$@"
+        WRAP
+        chmod +x /scripts/post_init_wrapper.sh
+        exec /bin/sh /launch.sh init
+    environment:
+      SCOPE: postgres-ha
+      PATRONI_NAME: patroni-0
+      ETCD3_HOSTS: '"etcd-1:2379","etcd-2:2379","etcd-3:2379"'
+      PGUSER_STANDBY: standby
+      PATRONI_RESTAPI_USERNAME: patroni
+      PGROOT: /home/postgres/pgdata/pgroot
+      SPILO_CONFIGURATION: |
+        bootstrap:
+          post_init: /scripts/post_init_wrapper.sh "zalandos"
+      # standby_cluster is NOT configured in this steady-state bootstrap file —
+      # it only applies during Phase 3 cutover staging
+      # (postgresql/cutover/postgresql-ha-staging.yaml), which streams
+      # continuously from the live single instance until promotion. See
+      # ADR-0001 Dry-Run Debugging Log note for the standby_cluster design.
+    secrets:
+      - postgresql_password
+      - postgresql_replication_password
+      - postgresql_patroni_password
+    volumes:
+      - /volume1/docker/PostgreSQL/patroni-0-data:/home/postgres/pgdata
+    networks:
+      - postgresql_db-backend
+    deploy:
+      placement:
+        constraints:
+          - node.labels.pg-role == primary
+
+  patroni-1:
+    image: ghcr.io/zalando/spilo-17:4.0-p3
+    hostname: patroni-1
+    command:
+      - /bin/sh
+      - -c
+      - |
+        export PGPASSWORD_SUPERUSER="$$(cat /run/secrets/postgresql_password)"
+        export PGPASSWORD_STANDBY="$$(cat /run/secrets/postgresql_replication_password)"
+        export PATRONI_RESTAPI_PASSWORD="$$(cat /run/secrets/postgresql_patroni_password)"
+        mkdir -p /scripts
+        cat > /scripts/post_init_wrapper.sh <<'WRAP'
+        #!/bin/bash
+        set -e
+        psql -d "$$2" -v ON_ERROR_STOP=1 -c 'DO $$do$$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = $$x$$postgres$$x$$) THEN CREATE ROLE postgres WITH SUPERUSER NOLOGIN; END IF; END $$do$$;'
+        exec /scripts/post_init.sh "$$@"
+        WRAP
+        chmod +x /scripts/post_init_wrapper.sh
+        exec /bin/sh /launch.sh init
+    environment:
+      SCOPE: postgres-ha
+      PATRONI_NAME: patroni-1
+      ETCD3_HOSTS: '"etcd-1:2379","etcd-2:2379","etcd-3:2379"'
+      PGUSER_STANDBY: standby
+      PATRONI_RESTAPI_USERNAME: patroni
+      PGROOT: /home/postgres/pgdata/pgroot
+      SPILO_CONFIGURATION: |
+        bootstrap:
+          post_init: /scripts/post_init_wrapper.sh "zalandos"
+    secrets:
+      - postgresql_password
+      - postgresql_replication_password
+      - postgresql_patroni_password
+    volumes:
+      - /volume1/docker/PostgreSQL/patroni-1-data:/home/postgres/pgdata
+    networks:
+      - postgresql_db-backend
+    deploy:
+      placement:
+        constraints:
+          - node.labels.pg-role == replica
+
+  # ── HAProxy — TCP routing via Patroni REST /primary health check only ───
+  # No placement constraint: all 3 nodes are managers (node.role == worker
+  # matches nothing in this cluster). mode: global + no constraint = one
+  # replica per node (docker-1/2/3), matching ADR's "HAProxy global, 3x".
+  haproxy:
+    image: haproxy:2.9-alpine
+    hostname: haproxy
+    volumes:
+      - /volume1/docker/compose-files/postgresql/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
+    networks:
+      postgresql_db-backend:
+        aliases:
+          - postgresql
+          - db
+      authentik_backend: {}
+    ports:
+      - 5430:5432/tcp
+    deploy:
+      mode: global
+      labels:
+        - traefik.enable=true
+        - traefik.tcp.routers.postgres.entrypoints=postgresql
+        - traefik.tcp.routers.postgres.rule=HostSNI(`*`)
+        - traefik.tcp.services.postgres.loadbalancer.server.port=5432
+        - traefik.tcp.routers.postgres.service=postgres
+        - traefik.swarm.network=traefik_backend
+
+  # ── Unchanged from current flat postgresql.yaml ──────────────────────────
+  databasus:
+    hostname: databasus
+    image: databasus/databasus:latest
+    networks:
+      - traefik_backend
+      - postgresql_db-backend
+    volumes:
+      - /volume1/docker/databasus:/databasus-data
+    deploy:
+      labels:
+        - traefik.enable=true
+        - traefik.http.routers.databasus.rule=Host(`${DATABASUS_HOST}`)
+        - traefik.http.routers.databasus.tls=true
+        - traefik.http.routers.databasus.tls.certresolver=letsencrypt
+        - traefik.http.routers.databasus.entrypoints=websecure
+        - traefik.http.services.databasus.loadbalancer.server.port=4005
+        - traefik.swarm.network=traefik_backend
+
+  pgadmin:
+    image: dpage/pgadmin4:latest
+    environment:
+      PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL}
+      PGADMIN_DEFAULT_PASSWORD_FILE: /run/secrets/postgresql_pgadmin_password
+      PGADMIN_LISTEN_PORT: 80
+    secrets:
+      - postgresql_pgadmin_password
+    volumes:
+      - "/volume1/docker/PostgreSQL Admin:/var/lib/pgadmin"
+    ports:
+      - 3030:80
+    networks:
+      - postgresql_db-backend
+      - traefik_backend
+    deploy:
+      labels:
+        - traefik.enable=true
+        - traefik.http.routers.pgadmin.rule=Host(`${PGADMIN_HOST}`)
+        - traefik.http.routers.pgadmin.tls=true
+        - traefik.http.routers.pgadmin.tls.certresolver=letsencrypt
+        - traefik.http.routers.pgadmin.entrypoints=websecure
+        - traefik.http.services.pgadmin.loadbalancer.server.port=80
+        - traefik.swarm.network=traefik_backend
+      resources:
+        limits:
+          cpus: '1'
+          memory: 512M
+
+networks:
+  postgresql_db-backend:
+    name: postgresql_db-backend
+    attachable: true
+    driver: overlay
+    driver_opts:
+      com.docker.network.driver.mtu: "8950"  # preserve — MTU mismatch previously broke Vaultwarden/Immich
+  traefik_backend:
+    external: true
+  authentik_backend:
+    external: true
+
+secrets:
+  postgresql_password:
+    external: true
+  postgresql_pgadmin_password:
+    external: true
+  postgresql_replication_password:
+    external: true
+  postgresql_patroni_password:
+    external: true

renovate.json

@@ -0,0 +1,102 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": ["config:base"],
+
+  "timezone": "America/Los_Angeles",
+  "schedule": ["after 2am and before 6am every weekday"],
+
+  "labels": ["renovate"],
+  "assignees": ["admin"],
+  "reviewers": ["admin"],
+
+  "prConcurrentLimit": 5,
+  "prHourlyLimit": 2,
+  "branchConcurrentLimit": 10,
+
+  "commitMessagePrefix": "chore(deps):",
+  "semanticCommits": "enabled",
+
+  "packageRules": [
+    {
+      "description": "Auto-merge patch updates for low-risk app stacks",
+      "matchFileNames": [
+        "mealie/**",
+        "vaultwarden/**",
+        "maintenance/**",
+        "meshcentral/**"
+      ],
+      "matchUpdateTypes": ["patch"],
+      "automerge": true,
+      "automergeType": "pr",
+      "automergeStrategy": "merge"
+    },
+    {
+      "description": "Auto-merge minor+patch for media stack (Plex/Radarr/Sonarr update frequently)",
+      "matchFileNames": ["media/**"],
+      "matchUpdateTypes": ["minor", "patch"],
+      "automerge": true,
+      "automergeType": "pr",
+      "automergeStrategy": "merge"
+    },
+    {
+      "description": "NEVER auto-merge bootstrap/critical stacks - manual review required",
+      "matchFileNames": [
+        "traefik.yaml",
+        "traefik.env",
+        "auth.yaml",
+        "auth.env",
+        "postgresql.yaml",
+        "woodpecker.yaml",
+        "git.yaml"
+      ],
+      "automerge": false,
+      "labels": ["renovate", "high-risk"],
+      "reviewers": ["admin"]
+    },
+    {
+      "description": "Never auto-merge major version bumps for anything",
+      "matchUpdateTypes": ["major"],
+      "automerge": false,
+      "labels": ["renovate", "major-update"]
+    },
+    {
+      "description": "Group all immich updates together (server+microservices must stay in sync)",
+      "matchFileNames": ["immich/**"],
+      "groupName": "immich",
+      "automerge": false,
+      "labels": ["renovate", "immich"]
+    },
+    {
+      "description": "Group all AI stack updates together",
+      "matchFileNames": ["ai.yaml"],
+      "groupName": "ai-stack",
+      "automerge": false,
+      "labels": ["renovate", "ai"]
+    },
+    {
+      "description": "Pin digest updates silently (SHA pinning) - auto-merge",
+      "matchUpdateTypes": ["digest"],
+      "automerge": true,
+      "automergeType": "pr"
+    }
+  ],
+
+  "docker-compose": {
+    "fileMatch": [
+      "(^|/)docker-compose\\.ya?ml$",
+      "(^|/)[a-z0-9-]+\\.ya?ml$"
+    ]
+  },
+
+  "woodpecker": {
+    "enabled": false
+  },
+
+  "ignorePaths": [
+    "deploy/**",
+    "secrets/**",
+    ".woodpecker/**",
+    "*.env",
+    "*.env.example"
+  ]
+}

renovate/renovate.env.example

@@ -0,0 +1,7 @@
+# renovate/renovate.env
+# Copy to renovate.env on the host (not committed - contains token)
+# The actual token is provisioned as a Woodpecker secret: renovate_token
+# This file is only needed if running the stack manually.
+
+GITEA_HOST=git.bryanmail.net
+RENOVATE_TOKEN=<gitea-renovate-bot-token>

renovate/renovate.yaml

@@ -0,0 +1,22 @@
+services:
+  renovate:
+    image: renovate/renovate:latest
+    # Renovate runs as a one-shot job triggered by Woodpecker cron.
+    # This service definition exists for reference and manual runs only.
+    # Normal execution path: .woodpecker/renovate.yml cron pipeline.
+    environment:
+      RENOVATE_TOKEN: ${RENOVATE_TOKEN}
+      RENOVATE_PLATFORM: gitea
+      RENOVATE_ENDPOINT: https://${GITEA_HOST}
+      RENOVATE_GIT_AUTHOR: "Renovate Bot <renovate-bot@bryanmail.net>"
+      RENOVATE_REPOSITORIES: "homelab/compose-files"
+      LOG_LEVEL: info
+    deploy:
+      replicas: 0
+      placement:
+        constraints:
+          - node.hostname == docker-2
+      restart_policy:
+        condition: none
+
+networks: {}

secrets/postgresql.secrets.example

@@ -14,29 +14,44 @@
 #     take down multiple services simultaneously. Ensure you have a
 #     working backup before making any changes.
 
-# ── SECRETS (add to Woodpecker) ───────────────────────────────────────────────
+# ── SECRETS (add to Woodpecker) ──────────────────────────────────────────
 
 # Woodpecker secret name:  postgresql_password
-# Used for:               PostgreSQL master password (PGadmin superuser)
+# Used for:                PostgreSQL master password (PGadmin superuser)
-#                         ⚠️  Used by nearly every other stack
+#                          ⚠️  Used by nearly every other stack
-# Env var in .env:        POSTGRES_PASSWORD
+# Env var in .env:         POSTGRES_USER
 postgresql_password=
 
 # Woodpecker secret name:  postgresql_pgadmin_password
-# Used for:               PGAdmin web UI login password
+# Used for:                PGAdmin web UI login password
-# Env var in .env:        PGADMIN_DEFAULT_PASSWORD
+# Env var in .env:         PGADMIN_DEFAULT_PASSWORD
 postgresql_pgadmin_password=
 
-# ── NON-SECRETS (safe in compose file or .env) ────────────────────────────────
+# Woodpecker secret name:  postgresql_replication_password
+# Used for:                Patroni streaming-replication role password
+#                          (used by patroni-0/patroni-1 for pg_basebackup +
+#                          WAL streaming between primary and standby)
+# Added:                   ADR-0001 HA build-out (Phase 1)
+postgresql_replication_password=
+
+# Woodpecker secret name:  postgresql_patroni_password
+# Used for:                Patroni REST API basic-auth password (:8008)
+#                          (used by HAProxy health checks and patronictl)
+# Added:                   ADR-0001 HA build-out (Phase 1)
+postgresql_patroni_password=
+
+# ── NON-SECRETS (safe in compose file or .env) ───────────────────────────
 
 # POSTGRES_USER              Master PostgreSQL username (PGadmin)
 # PGADMIN_DEFAULT_EMAIL      PGAdmin login email
 # DATABASUS_HOST             Traefik hostname for pgAdmin
 # PGADMIN_HOST               Traefik hostname for pgAdmin UI
 
-# ── Woodpecker provision-secrets case entry ───────────────────────────────────
+# ── Woodpecker provision-secrets case entry ──────────────────────────────
 #
 #   postgresql)
-#     create_or_update_secret "postgresql_password"         "$POSTGRESQL_PASSWORD"
+#     create_or_update_secret "postgresql_password"             "$POSTGRESQL_PASSWORD"
-#     create_or_update_secret "postgresql_pgadmin_password" "$POSTGRESQL_PGADMIN_PASSWORD"
+#     create_or_update_secret "postgresql_pgadmin_password"     "$POSTGRESQL_PGADMIN_PASSWORD"
+#     create_or_update_secret "postgresql_replication_password" "$POSTGRESQL_REPLICATION_PASSWORD"
+#     create_or_update_secret "postgresql_patroni_password"     "$POSTGRESQL_PATRONI_PASSWORD"
 #     ;;

secrets/secrets-map.yaml

@@ -0,0 +1,38 @@
+# ─────────────────────────────────────────────────────────────────────────────
+# secrets-map.yaml — DATA-ONLY manifest for deploy/provision-stack.py
+#
+# RULES:
+#   - This file contains NO code, NO shell, NO secret values — only names.
+#   - Each stack entry declares:
+#       env_template:   repo path of the FULL env-file template (tracked).
+#                       The template is authoritative: the COMPLETE env file is
+#                       rendered from it on every provisioning run. Nothing is
+#                       line-edited in place, so keys can never silently go
+#                       missing.
+#       env_dest:       host path (relative to /volume1/docker/compose-files/)
+#                       the rendered env file is shipped to. Rendered file
+#                       exists ONLY on the host — never committed to git.
+#       docker_secrets: map of docker-swarm-secret-name -> CI ENV VAR NAME
+#                       (Pattern C). The env var must be declared via
+#                       from_secret: in .woodpecker/deploy.yml's
+#                       provision-secrets step (Woodpecker v3 requires explicit
+#                       per-secret declaration; there is no expose-all).
+#
+# ADDING A NEW SECRET (3 small steps, no shell edits):
+#   1. Add the secret value in Woodpecker UI (repo Settings -> Secrets).
+#   2. Declare it in .woodpecker/deploy.yml provision-secrets environment:
+#      block (from_secret) — mechanical two-line addition.
+#   3. Reference it here (docker_secrets:) and/or in the stack's
+#      .env.template as a dollar-brace placeholder.
+#
+# Stacks not listed here fall through to deploy.yml's legacy case-entries
+# untouched. Migration is deliberately one stack per PR.
+# ─────────────────────────────────────────────────────────────────────────────
+stacks:
+  ai:
+    env_template: ai/ai.env.template
+    env_dest: ai/ai.env
+    docker_secrets:
+      flowagent_azure_client_id: FLOWAGENT_AZURE_CLIENT_ID
+      flowagent_azure_tenant_id: FLOWAGENT_AZURE_TENANT_ID
+      flowagent_azure_client_secret: FLOWAGENT_AZURE_CLIENT_SECRET

security/security.env

@@ -0,0 +1,5 @@
+# Security (CrowdSec) - Pattern B env (folder-mode)
+CROWDSEC_PGDSN='postgres://CrowdSec:***REDACTED***@db:5432/crowdsec?sslmode=disable'
+BOUNCER_KEY_TRAEFIK=***REDACTED***
+CROWDSEC_API_KEY=***REDACTED***
+CROWDSEC_PASSWORD=***REDACTED***

security/security.yaml

@@ -32,7 +32,7 @@ services:
           memory: 512M
 
   crowdsec-web-ui:
-    image: ghcr.io/thebluffman85/crowdsec-web-ui:latest
+    image: ghcr.io/theduffman85/crowdsec-web-ui:latest
     environment:
       CROWDSEC_API_URL: http://crowdsec:8080
       CROWDSEC_API_KEY: ${CROWDSEC_API_KEY}

traefik/traefik.env

@@ -0,0 +1,2 @@
+KEEPALIVED_PASSWORD=***REDACTED***
+KEEPALIVED_VIRTUAL_IPS="#PYTHON2BASH:['192.168.4.30']"

traefik/traefik.yaml

@@ -5,8 +5,8 @@ services:
       KEEPALIVED_INTERFACE: eth0
       KEEPALIVED_PRIORITY: "150"
       KEEPALIVED_ROUTER_ID: "51"
-      KEEPALIVED_PASSWORD: ${KEEPALIVED_PASSWORD}
+      KEEPALIVED_PASSWORD: "${KEEPALIVED_PASSWORD}"
-      KEEPALIVED_VIRTUAL_IPS: ${KEEPALIVED_VIRTUAL_IPS}
+      KEEPALIVED_VIRTUAL_IPS: "${KEEPALIVED_VIRTUAL_IPS}"
       KEEPALIVED_UNICAST_PEERS: "#PYTHON2BASH:['192.168.4.31', '192.168.4.32']"
     command: --copy-service
     cap_add: [NET_ADMIN, NET_BROADCAST, NET_RAW]
@@ -20,6 +20,18 @@ services:
       labels: [traefik.enable=false]
       resources:
         limits: {cpus: '1', memory: 128M}
+      # stop-first: host network + VRRP — two concurrent keepalived speakers with the
+      # same router-id would fight over the VIP. Never run old+new simultaneously.
+      update_config:
+        parallelism: 1
+        delay: 30s
+        failure_action: rollback
+        monitor: 15s
+        order: stop-first
+      rollback_config:
+        parallelism: 1
+        failure_action: pause
+        order: stop-first
 
   keepalived-backup:
     environment:
@@ -27,8 +39,8 @@ services:
       KEEPALIVED_INTERFACE: eth0
       KEEPALIVED_PRIORITY: "100"
       KEEPALIVED_ROUTER_ID: "51"
-      KEEPALIVED_PASSWORD: ${KEEPALIVED_PASSWORD}
+      KEEPALIVED_PASSWORD: "${KEEPALIVED_PASSWORD}"
-      KEEPALIVED_VIRTUAL_IPS: ${KEEPALIVED_VIRTUAL_IPS}
+      KEEPALIVED_VIRTUAL_IPS: "${KEEPALIVED_VIRTUAL_IPS}"
       KEEPALIVED_UNICAST_PEERS: "#PYTHON2BASH:['192.168.4.32', '192.168.4.33']"
     cap_add: [NET_ADMIN, NET_BROADCAST, NET_RAW]
     networks: [host]
@@ -41,6 +53,17 @@ services:
       labels: [traefik.enable=false]
       resources:
         limits: {cpus: '1', memory: 128M}
+      # stop-first: host network + VRRP (see keepalived-master)
+      update_config:
+        parallelism: 1
+        delay: 30s
+        failure_action: rollback
+        monitor: 15s
+        order: stop-first
+      rollback_config:
+        parallelism: 1
+        failure_action: pause
+        order: stop-first
 
   reverse-proxy:
     image: public.ecr.aws/docker/library/traefik:latest
@@ -108,9 +131,23 @@ services:
         - traefik.http.routers.traefik.middlewares=forwardAuth-authentik@file, crowdsec@file
       resources:
         limits: {cpus: '4', memory: 512M}
+      # stop-first: host-mode published ports (80/443/1883/5201/5431) — a start-first
+      # replacement task can never bind while the old task holds the ports, so
+      # start-first would deadlock/rollback every update. Global mode + parallelism 1
+      # means only one node is briefly down; keepalived VIP + remaining nodes cover it.
+      update_config:
+        parallelism: 1
+        delay: 30s
+        failure_action: rollback
+        monitor: 15s
+        order: stop-first
+      rollback_config:
+        parallelism: 1
+        failure_action: pause
+        order: stop-first
 
   traefik-certs-dumper:
-    image: ghcr.io/ldez/traefik-certs-dumper:v2.10.0
+    image: ghcr.io/ldez/traefik-certs-dumper:v2.11.4
     entrypoint: sh -c 'apk add jq ; while ! [ -e /letsencrypt/acme.json ] || ! [ `jq ".[] | .Certificates | length" /letsencrypt/acme.json` != 0 ]; do sleep 1 ; done && traefik-certs-dumper file --version v3 --watch --source /letsencrypt/acme.json --dest /letsencrypt/certs'
     volumes:
       - /volume1/docker/letsencrypt:/letsencrypt
@@ -119,6 +156,16 @@ services:
       labels: [traefik.enable=false]
       resources:
         limits: {cpus: '1', memory: 64M}
+      update_config:
+        parallelism: 1
+        delay: 30s
+        failure_action: rollback
+        monitor: 15s
+        order: start-first
+      rollback_config:
+        parallelism: 1
+        failure_action: pause
+        order: start-first
 
   speedtest:
     image: openspeedtest/latest
@@ -142,6 +189,16 @@ services:
         - autoheal=true
       resources:
         limits: {cpus: '8', memory: 1G}
+      update_config:
+        parallelism: 1
+        delay: 30s
+        failure_action: rollback
+        monitor: 15s
+        order: start-first
+      rollback_config:
+        parallelism: 1
+        failure_action: pause
+        order: start-first
 
   iperf3-server:
     image: networkstatic/iperf3:latest
@@ -164,6 +221,16 @@ services:
         - autoheal=true
       resources:
         limits: {cpus: '4', memory: 128M}
+      update_config:
+        parallelism: 1
+        delay: 30s
+        failure_action: rollback
+        monitor: 15s
+        order: start-first
+      rollback_config:
+        parallelism: 1
+        failure_action: pause
+        order: start-first
 
   whoami:
     image: emilevauge/whoami
@@ -181,6 +248,16 @@ services:
         - traefik.http.routers.whoami.middlewares=crowdsec@file, forwardAuth-authentik@file
       resources:
         limits: {cpus: '1', memory: 16M}
+      update_config:
+        parallelism: 1
+        delay: 30s
+        failure_action: rollback
+        monitor: 15s
+        order: start-first
+      rollback_config:
+        parallelism: 1
+        failure_action: pause
+        order: start-first
 
 networks:
   traefik_frontend:

vaultwarden/vaultwarden.env

@@ -0,0 +1,8 @@
+# Vaultwarden - Pattern B env (folder-mode); ADMIN_TOKEN/DATABASE_URL are Docker secrets
+# 2026-07: DATABASE_URL secret rotated to point at 'postgresql' alias instead of 'db'
+# as part of PostgreSQL HA migration prep (ADR-0001).
+# Retry #8: retry #7's pipeline failed to compile ("missing closing brace") because
+# the new documentation header in deploy.yml itself contained a literal dollar-brace
+# example that Woodpecker's substitution engine tried to parse -- even in comments.
+# Header rewritten without literal sequences. This trigger should finally run clean:
+# provision vaultwarden_database_url_v2, deploy the stack, verify the task.

vaultwarden/vaultwarden.yaml

@@ -39,8 +39,10 @@ services:
     volumes:
       - "/volume1/docker/Vaultwarden/data:/data"
     secrets:
-      - vaultwarden_admin_token
+      - source: vaultwarden_admin_token_v2
-      - vaultwarden_database_url
+        target: vaultwarden_admin_token
+      - source: vaultwarden_database_url_v2
+        target: vaultwarden_database_url
     working_dir: "/"
     deploy:
       labels:
@@ -57,9 +59,9 @@ services:
           memory: 1G
 
 secrets:
-  vaultwarden_admin_token:
+  vaultwarden_admin_token_v2:
     external: true
-  vaultwarden_database_url:
+  vaultwarden_database_url_v2:
     external: true
 
 networks:

woodpecker.yaml

@@ -1,37 +1,22 @@
-version: '3.8'
-
 services:
-  # Woodpecker Server - Central orchestration
   woodpecker-server:
     image: woodpeckerci/woodpecker-server:latest
     ports:
-      - "8000:8000"  # HTTP
+      - "8000:8000"
-      - "9000:9000"  # gRPC (agents communicate here)
+      - "9000:9000"
     environment:
-      # Server configuration
       WOODPECKER_HOST: https://${WOODPECKER_HOST}
       WOODPECKER_PROTO: https
-      WOODPECKER_OPEN: "false"  # Private instance
+      WOODPECKER_OPEN: "false"
-      
-      # Gitea integration
       WOODPECKER_GITEA: "true"
       WOODPECKER_GITEA_URL: https://${GITEA_HOST}
       WOODPECKER_GITEA_CLIENT: ${WOODPECKER_GITEA_CLIENT}
       WOODPECKER_GITEA_SECRET: ${WOODPECKER_GITEA_SECRET}
-      
-      # Agent security
       WOODPECKER_AGENT_SECRET: ${WOODPECKER_AGENT_SECRET}
-      
-      # Allow organizations
       WOODPECKER_ORGS: homelab,admin
-      
-      # Logging
       WOODPECKER_LOG_LEVEL: info
-      
-      # Database (using existing PostgreSQL)
       WOODPECKER_DATABASE_DRIVER: postgres
       WOODPECKER_DATABASE_DATASOURCE: postgres://${DB_USER}:${DB_PASS}@${POSTGRES_HOST}:${POSTGRES_PORT}/woodpecker?sslmode=disable
-      
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       - /etc/timezone:/etc/timezone:ro
@@ -43,7 +28,7 @@ services:
       replicas: 1
       placement:
         constraints:
-          - node.hostname == docker-2  # Run on manager node only
+          - node.hostname == docker-2
       labels:
         - traefik.enable=true
         - traefik.http.routers.woodpecker.entrypoints=websecure
@@ -52,23 +37,26 @@ services:
         - traefik.http.routers.woodpecker.tls.certresolver=letsencrypt
         - traefik.docker.network=traefik_backend
         - traefik.http.services.woodpecker.loadbalancer.server.port=8000
+      update_config:
+        parallelism: 1
+        delay: 15s
+        failure_action: rollback
+        monitor: 30s
+        order: start-first
+      rollback_config:
+        parallelism: 1
+        failure_action: pause
+        order: start-first
 
-  # Woodpecker Agents - Pipeline execution
   woodpecker-agent:
     image: woodpeckerci/woodpecker-agent:latest
     command: agent
     environment:
-      # Server connection
       WOODPECKER_SERVER: woodpecker-server:9000
       WOODPECKER_AGENT_SECRET: ${WOODPECKER_AGENT_SECRET}
-      
+      WOODPECKER_MAX_PROCS: 4
-      # Agent configuration
-      WOODPECKER_MAX_PROCS: 4  # Max parallel builds per agent
       WOODPECKER_LOG_LEVEL: info
-      
-      # Labels for scheduling
       WOODPECKER_FILTER_LABELS: "true"
-      
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       - /etc/timezone:/etc/timezone:ro
@@ -76,9 +64,17 @@ services:
     networks:
       - traefik_backend
     deploy:
-      mode: global  # One agent per node
+      mode: global
-  depends_on:
+      update_config:
-    - woodpecker-server
+        parallelism: 1
+        delay: 30s
+        failure_action: rollback
+        monitor: 15s
+        order: start-first
+      rollback_config:
+        parallelism: 1
+        failure_action: pause
+        order: start-first
 
 networks:
   traefik_backend: