Merge pull request 'traefik: add log rotation for access.log / traefik.log' (#23 ) from fix/traefik-log-rotation into main

Reviewed-on: #23 https://ai.bryanmail.net/c/5d375026-71a6-444a-a406-55fd55ad1f73
traefik: fix corrupted README (was double base64-encoded)
2026-09-12 00:29:27 -07:00 · 2026-09-12 00:21:36 -07:00 · 2026-09-12 00:21:10 -07:00 · 2026-09-12 00:20:56 -07:00 · 2026-09-12 00:20:56 -07:00 · 2026-09-12 00:07:58 -07:00
23 changed files with 1907 additions and 566 deletions
@@ -2,8 +2,17 @@
 *.swp
 *.swo
 .DS_Store
+
+# Secrets / stack env files - never commit these.
+# deploy/global.env and immich.env-example etc. are the intended committed
+# templates/examples; actual secret-bearing .env files must stay untracked.
 .env
 .env.local
+*.env
+!*.env-example
+!deploy/global.env
+*.env.bak
+*.env.backup
 secrets/
 keys/
 *.key
@@ -15,3 +24,7 @@ keys/
 # Local overrides
 docker-compose.override.yml
 local/
+
+# Editor/adhoc backups that sometimes get left in the tree
+*.bak
+*.bak-*
@@ -26,6 +26,60 @@ when:
 #   This was the root cause of a long-running "SWARM_MANAGER_IP secret is
 #   empty" failure: braced references were blanked at compile time before the
 #   shell ever saw them.
+#
+#   2026-08-26 HOTFIX: a full-file rewrite (AI secrets migration PR) dropped
+#   one $ from every $${VAR} occurrence throughout this file, re-introducing
+#   exactly the bug described above for EVERY secret-backed var, not just the
+#   new AI ones. The if-empty guards caught it immediately (SWARM_MANAGER_IP
+#   came back blank) and aborted before any ssh/scp/rsync ran, so no live
+#   secret or service was touched — but no CI provisioning/deploy could run
+#   until this was restored. Lesson: grep for the literal string '$${' and
+#   diff the count against the previous version before ever committing a
+#   full-file rewrite of this pipeline.
+#
+#   2026-08-26 FIX: FOLDER_STACKS/FOLDERS detection (grep -E '^[^/.][^/]*/')
+#   matches ANY non-dot top-level folder in the changed-files list, including
+#   deploy/ — the shared tooling folder, not a stack. A PR touching only
+#   deploy/envparse.py caused stack-deploy.sh to be invoked with "deploy" as
+#   a stack name, which correctly errored ("No main compose file... in
+#   .../deploy") since deploy/ has no deploy.yaml. No live service was
+#   affected (the error occurs before any redeploy), but it produced a
+#   confusing pipeline failure on an otherwise-correct change. deploy/ is
+#   already unconditionally rsynced at the top of the deploy step regardless
+#   of which stacks changed, so it's safe to exclude it from the stack list
+#   everywhere folders are detected below.
+#
+#   2026-09-03/07 INCIDENT + REDESIGN: hand-maintained grep -v + printf
+#   line-surgery cases repeatedly drifted (missing '=', duplicated keys,
+#   mismatched env var names rendering EMPTY secrets) and broke live
+#   services. Root cause: no authoritative key list and shell heredocs
+#   hostile to hand-editing. Stacks migrate one at a time to a data-driven
+#   model: secrets/secrets-map.yaml (data only) + per-stack .env.template
+#   (authoritative FULL file) + deploy/provision-stack.py (whole-file
+#   render, hard failure naming any missing value). Migrated stacks call
+#   the script; unmigrated stacks keep legacy case entries until their own
+#   PR. See the "Secrets & Deployment Architecture — Global Direction" note.
+#
+#   2026-09-08 FIX: secrets/ is a tooling/docs folder (secrets-map.yaml +
+#   *.secrets.example), not a stack — but folder-detection treated it as one
+#   the first time a commit touched it (PR #16). deploy survived only because
+#   'secrets' sits in the bootstrap-tier skip list; verify had no guard and
+#   died on `docker stack ps secrets` failing under errexit (assignment from
+#   a failing command substitution aborts the step). Fixed by excluding
+#   secrets/ alongside deploy/ in ALL folder-detection sites, and by
+#   tolerating a failing stack-ps in verify (|| true) so a genuinely missing
+#   stack produces the designed WARNING instead of killing the step. This
+#   hazard was first flagged in July (PR #3, closed unmerged).
+#
+#   2026-09-12 MIGRATION: traefik moved from Pattern B (secrets hand-typed
+#   into the host-only traefik.env, no CI involvement at all) to the same
+#   manifest-driven provisioning as ai) below. Root cause of the migration:
+#   traefik.env had been accidentally committed to git with a literal
+#   "***REDACTED***" placeholder as KEEPALIVED_PASSWORD; every git-guard
+#   resync/checkout silently restored that broken value, which diverged from
+#   keepalived-backup's stale-but-correct in-memory value and produced a
+#   continuous VRRP auth failure ("received an invalid passwd!") and VIP
+#   instability. The manifest-driven path never commits the rendered file.
 # ─────────────────────────────────────────────────────────────────────────────

 steps:
@@ -43,8 +97,9 @@ steps:
        FLAT=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.ya?ml$' || true)

        # Folder: any file under a subfolder (e.g. immich/immich.yml).
-        # Exclude dotfolders (.woodpecker, .git, .gitea, etc.)
-        FOLDERS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true)
+        # Exclude dotfolders (.woodpecker, .git, .gitea, etc.) and the
+        # non-stack tooling folders deploy/ and secrets/ (see notes above).
+        FOLDERS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -vE '^(deploy|secrets)$' | sort -u || true)

        [ -z "$FLAT" ] && [ -z "$FOLDERS" ] && echo "No stacks changed" && exit 0

@@ -140,6 +195,11 @@ steps:
        from_secret: entertainment_sparky_encryption_key
      ENTERTAINMENT_BETTER_AUTH_SECRET:
        from_secret: entertainment_better_auth_secret
+      # ── ai stack (manifest-driven — secrets/secrets-map.yaml +
+      #    ai/ai.env.template + deploy/provision-stack.py). Env var names
+      #    below match the template placeholders EXACTLY; this block is the
+      #    only per-secret touchpoint left in this file for migrated stacks
+      #    (Woodpecker v3 requires explicit from_secret declarations). ──
      AI_AWS_ACCESS_KEY_ID:
        from_secret: ai_aws_access_key_id
      AI_AWS_SECRET_ACCESS_KEY:
@@ -148,130 +208,169 @@ steps:
        from_secret: ai_litellm_master_key
      AI_LITELLM_SALT_KEY:
        from_secret: ai_litellm_salt_key
-      AI_LITELLM_DB_PASSWORD:
-        from_secret: ai_litellm_db_password
-      AI_WEBUI_SECRET_KEY:
+      AI_LITELLM_DATABASE_URL:
+        from_secret: ai_litellm_database_url
+      AI_LITELLM_POSTGRES_PASSWORD:
+        from_secret: ai_litellm_postgres_password
+      AI_OPEN_WEBUI_SECRET_KEY:
        from_secret: ai_webui_secret_key
      AI_OPEN_WEBUI_DATABASE_URL:
        from_secret: ai_open_webui_database_url
-      AI_OAUTH_CLIENT_SECRET:
+      AI_OPEN_WEBUI_OAUTH_CLIENT_SECRET:
        from_secret: ai_oauth_client_secret
+      AI_MCPO_API_KEY:
+        from_secret: ai_mcpo_api_key
+      FLOWAGENT_AZURE_CLIENT_ID:
+        from_secret: flowagent_azure_client_id
+      FLOWAGENT_AZURE_TENANT_ID:
+        from_secret: flowagent_azure_tenant_id
+      FLOWAGENT_AZURE_CLIENT_SECRET:
+        from_secret: flowagent_azure_client_secret
+      # ── traefik stack (manifest-driven — secrets/secrets-map.yaml +
+      #    traefik/traefik.env.template + deploy/provision-stack.py).
+      #    Migrated 2026-09-12; see header note above for root cause. ──
+      TRAEFIK_KEEPALIVED_PASSWORD:
+        from_secret: traefik_keepalived_password
    commands:
-      - apk add --no-cache openssh-client
+      - apk add --no-cache openssh-client python3 py3-yaml
      - mkdir -p ~/.ssh
      - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa
      - chmod 600 ~/.ssh/id_rsa
      - |
-        if [ -z "${SWARM_MANAGER_IP}" ]; then
+        if [ -z "$${SWARM_MANAGER_IP}" ]; then
          echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets."
          exit 1
        fi
-      - ssh-keyscan -H ${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
+      - ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
      - |
        CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
        FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
-        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true)
+        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -vE '^(deploy|secrets)$' | sort -u || true)
        ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
        [ -z "$ALL_STACKS" ] && echo "No stacks changed, skipping" && exit 0
-      - scp -o StrictHostKeyChecking=no deploy/create-secrets.sh root@${SWARM_MANAGER_IP}:/tmp/cs.sh
+      - scp -o StrictHostKeyChecking=no deploy/create-secrets.sh root@$${SWARM_MANAGER_IP}:/tmp/cs.sh
      - |
        CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
        FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
-        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true)
+        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -vE '^(deploy|secrets)$' | sort -u || true)
        ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)

        for STACK in $ALL_STACKS; do
          echo "Provisioning: $STACK"
          case "$STACK" in
-            maintenance|media|unifi|guacamole|security|auth|traefik|meshcentral|ddm)
+            maintenance|media|unifi|guacamole|security|auth|meshcentral|ddm)
              echo "  No Docker secrets for $STACK — secrets in host .env";;
+            traefik)
+              # MIGRATED (2026-09-12) to manifest-driven provisioning after
+              # discovering traefik.env was committed to git with a literal
+              # "***REDACTED***" placeholder as KEEPALIVED_PASSWORD — every
+              # git-guard resync/checkout restored the broken value, causing
+              # VRRP auth mismatch between keepalived-master/-backup and VIP
+              # instability. All logic lives in deploy/provision-stack.py;
+              # the authoritative key list lives in
+              # traefik/traefik.env.template; the mapping lives in
+              # secrets/secrets-map.yaml. This case is intentionally one line.
+              python3 deploy/provision-stack.py traefik;;
            immich)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'immich_db_password'     '${IMMICH_DB_PASSWORD}'
-                create_or_update_secret 'immich_kiosk_basicauth' '${IMMICH_KIOSK_BASICAUTH}'";;
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'immich_db_password'     '$${IMMICH_DB_PASSWORD}'
+                create_or_update_secret 'immich_kiosk_basicauth' '$${IMMICH_KIOSK_BASICAUTH}'";;
            woodpecker)
              echo "  Manual only — skipping";;
            3dprint)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret '3dprint_db_password' '${PRINT3D_DB_PASSWORD}'";;
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret '3dprint_db_password' '$${PRINT3D_DB_PASSWORD}'";;
            gamma)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'gamma_auth_token' '${GAMMA_AUTH_TOKEN}'";;
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'gamma_auth_token' '$${GAMMA_AUTH_TOKEN}'";;
            git)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'git_db_password'       '${GIT_DB_PASSWORD}'
-                create_or_update_secret 'git_runner_token'      '${GIT_RUNNER_TOKEN}'
-                create_or_update_secret 'git_mcp_access_token'  '${GIT_MCP_ACCESS_TOKEN}'";;
+              # PATTERN B, DELIBERATE (see decision notes) — git hosts the source
+              # of truth for every other stack's compose files, so it must be
+              # restorable from a flat git.yaml + git.env backup alone, with zero
+              # dependency on a running Swarm's Docker secret store. Native Docker
+              # secrets (Pattern C) can't satisfy that: they only exist inside an
+              # already-running Swarm, which is exactly the circular dependency
+              # this stack can't have. Mirrors the retired ai) case's grep -v +
+              # printf rewrite-in-place approach, never sed (values may contain
+              # slash, dollar sign, ampersand).
+              #
+              # TEST PHASE: target is git.env.pipelinetest, NOT the real git.env.
+              # The real file is never opened for writing by this step. First run
+              # seeds the test file from the real git.env (carries over all
+              # non-secret lines untouched); every push after that only refreshes
+              # the 3 secret lines below. Also strips the legacy GITEA_ACCESS_TOKEN
+              # key name so the test file converges on the git.env.example-
+              # documented GITEA_MCP_ACCESS_TOKEN key. Cutover to the real file —
+              # and pointing git.yaml/stack-deploy at it — is a deliberate,
+              # separate follow-up after manually diffing this render.
+              # (Candidate for the secrets-map.yaml/provision-stack.py migration
+              # in its own PR; kept legacy for now.)
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "FILE=/volume1/docker/compose-files/git.env.pipelinetest
+                TMP=\$FILE.tmp.\$\$
+                [ -f \$FILE ] || cp /volume1/docker/compose-files/git.env \$FILE
+                grep -vE '^(GITEA__database__PASSWD|GITEA_RUNNER_REGISTRATION_TOKEN|GITEA_MCP_ACCESS_TOKEN|GITEA_ACCESS_TOKEN)=' \$FILE > \$TMP 2>/dev/null || touch \$TMP
+                { cat \$TMP
+                  printf 'GITEA__database__PASSWD=%s\n' '$${GIT_DB_PASSWORD}'
+                  printf 'GITEA_RUNNER_REGISTRATION_TOKEN=%s\n' '$${GIT_RUNNER_TOKEN}'
+                  printf 'GITEA_MCP_ACCESS_TOKEN=%s\n' '$${GIT_MCP_ACCESS_TOKEN}'
+                } > \$FILE
+                rm -f \$TMP
+                echo '  [OK] git.env.pipelinetest updated - real git.env untouched'";;
            homeassistant)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'homeassistant_frigate_rtsp_password' '${HOMEASSISTANT_FRIGATE_RTSP_PASSWORD}'
-                create_or_update_secret 'homeassistant_immich_api_key'        '${HOMEASSISTANT_IMMICH_API_KEY}'";;
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'homeassistant_frigate_rtsp_password' '$${HOMEASSISTANT_FRIGATE_RTSP_PASSWORD}'
+                create_or_update_secret 'homeassistant_immich_api_key'        '$${HOMEASSISTANT_IMMICH_API_KEY}'";;
            mealie)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'mealie_db_password'         '${MEALIE_DB_PASSWORD}'
-                create_or_update_secret 'mealie_ldap_query_password' '${MEALIE_LDAP_QUERY_PASSWORD}'";;
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'mealie_db_password'         '$${MEALIE_DB_PASSWORD}'
+                create_or_update_secret 'mealie_ldap_query_password' '$${MEALIE_LDAP_QUERY_PASSWORD}'";;
            n8n)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'n8n_db_password'     '${N8N_DB_PASSWORD}'
-                create_or_update_secret 'n8n_encryption_key'  '${N8N_ENCRYPTION_KEY}'";;
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'n8n_db_password'     '$${N8N_DB_PASSWORD}'
+                create_or_update_secret 'n8n_encryption_key'  '$${N8N_ENCRYPTION_KEY}'";;
            postgresql)
              # NOTE: bootstrap-tier stack (manual deploy only via stack-deploy.sh).
              # Secrets are still auto-provisioned here so they exist on the host
              # before the manual `stack-deploy.sh postgresql` run picks them up.
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'postgresql_password'             '${POSTGRESQL_PASSWORD}'
-                create_or_update_secret 'postgresql_pgadmin_password'     '${POSTGRESQL_PGADMIN_PASSWORD}'
-                create_or_update_secret 'postgresql_replication_password' '${POSTGRESQL_REPLICATION_PASSWORD}'
-                create_or_update_secret 'postgresql_patroni_password'     '${POSTGRESQL_PATRONI_PASSWORD}'";;
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'postgresql_password'             '$${POSTGRESQL_PASSWORD}'
+                create_or_update_secret 'postgresql_pgadmin_password'     '$${POSTGRESQL_PGADMIN_PASSWORD}'
+                create_or_update_secret 'postgresql_replication_password' '$${POSTGRESQL_REPLICATION_PASSWORD}'
+                create_or_update_secret 'postgresql_patroni_password'     '$${POSTGRESQL_PATRONI_PASSWORD}'";;
            productivity)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'productivity_paperless_secret_key' '${PRODUCTIVITY_PAPERLESS_SECRET_KEY}'
-                create_or_update_secret 'productivity_db_password'          '${PRODUCTIVITY_DB_PASSWORD}'
-                create_or_update_secret 'productivity_oidc_providers'       '${PRODUCTIVITY_OIDC_PROVIDERS}'";;
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'productivity_paperless_secret_key' '$${PRODUCTIVITY_PAPERLESS_SECRET_KEY}'
+                create_or_update_secret 'productivity_db_password'          '$${PRODUCTIVITY_DB_PASSWORD}'
+                create_or_update_secret 'productivity_oidc_providers'       '$${PRODUCTIVITY_OIDC_PROVIDERS}'";;
            vaultwarden)
              # NOTE: vaultwarden_database_url cannot be rotated in-place — Swarm refuses
              # to remove a secret referenced by a running service's spec. We provision
              # under a versioned name instead; vaultwarden.yaml maps it back to the same
              # in-container filename via target. The old secret is removed manually once
              # the compose file cutover is confirmed healthy.
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'vaultwarden_admin_token'     '${VAULTWARDEN_ADMIN_TOKEN}'
-                create_or_update_secret 'vaultwarden_database_url_v2' '${VAULTWARDEN_DATABASE_URL}'";;
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'vaultwarden_admin_token'     '$${VAULTWARDEN_ADMIN_TOKEN}'
+                create_or_update_secret 'vaultwarden_database_url_v2' '$${VAULTWARDEN_DATABASE_URL}'";;
            ai)
-              # NOTE: ai stack uses Pattern B (host .env, not native Docker secrets) —
-              # LiteLLM/boto3 and Open WebUI don't support the _FILE convention for these
-              # vars. Instead of Docker secrets, we regenerate ONLY the migrated lines in
-              # the remote ai/ai.env in place via grep -v + printf (never sed, since values
-              # may contain '/', '$', '&'). All other lines — including MCPO_API_KEY,
-              # OAUTH_CLIENT_ID, WEBUI_URL, and other non-secret config — are left
-              # completely untouched. MCPO_API_KEY migration is deferred to a follow-up;
-              # this step never reads or writes it.
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "FILE=/volume1/docker/compose-files/ai/ai.env
-                TMP=\$FILE.tmp.\$\$
-                grep -vE '^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|LITELLM_MASTER_KEY|LITELLM_SALT_KEY|POSTGRES_PASSWORD|DATABASE_URL|WEBUI_SECRET_KEY|OPEN_WEBUI_DATABASE_URL|OAUTH_CLIENT_SECRET)=' \$FILE > \$TMP 2>/dev/null || touch \$TMP
-                { cat \$TMP
-                  printf 'AWS_ACCESS_KEY_ID=%s\n' '${AI_AWS_ACCESS_KEY_ID}'
-                  printf 'AWS_SECRET_ACCESS_KEY=%s\n' '${AI_AWS_SECRET_ACCESS_KEY}'
-                  printf 'LITELLM_MASTER_KEY=%s\n' '${AI_LITELLM_MASTER_KEY}'
-                  printf 'LITELLM_SALT_KEY=%s\n' '${AI_LITELLM_SALT_KEY}'
-                  printf 'POSTGRES_PASSWORD=%s\n' '${AI_LITELLM_DB_PASSWORD}'
-                  printf 'DATABASE_URL=postgresql://LiteLLM:%s@postgresql:5432/litellm\n' '${AI_LITELLM_DB_PASSWORD}'
-                  printf 'WEBUI_SECRET_KEY=%s\n' '${AI_WEBUI_SECRET_KEY}'
-                  printf 'OPEN_WEBUI_DATABASE_URL=%s\n' '${AI_OPEN_WEBUI_DATABASE_URL}'
-                  printf 'OAUTH_CLIENT_SECRET=%s\n' '${AI_OAUTH_CLIENT_SECRET}'
-                } > \$FILE
-                rm -f \$TMP
-                echo '  [OK] ai/ai.env secrets updated'";;
+              # MIGRATED (2026-09-07) to manifest-driven provisioning after the
+              # line-surgery approach repeatedly drifted (dropped keys 2026-09-03;
+              # duplicate AI_LITELLM_POSTGRES_PASSWORD + stray legacy keys +
+              # WEB_UI/WEBUI env-name mismatch rendering an EMPTY OAuth client
+              # secret, found 2026-09-07). All logic lives in
+              # deploy/provision-stack.py; the authoritative key list lives in
+              # ai/ai.env.template; the mapping lives in secrets/secrets-map.yaml.
+              # This case is intentionally one line.
+              python3 deploy/provision-stack.py ai;;
            entertainment)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'entertainment_discord_token'          '${ENTERTAINMENT_DISCORD_TOKEN}'
-                create_or_update_secret 'entertainment_discord_client_secret'  '${ENTERTAINMENT_DISCORD_CLIENT_SECRET}'
-                create_or_update_secret 'entertainment_secret_key_base'        '${ENTERTAINMENT_SECRET_KEY_BASE}'
-                create_or_update_secret 'entertainment_basic_auth_password'    '${ENTERTAINMENT_BASIC_AUTH_PASSWORD}'
-                create_or_update_secret 'entertainment_sparky_db_password'     '${ENTERTAINMENT_SPARKY_DB_PASSWORD}'
-                create_or_update_secret 'entertainment_sparky_app_db_password' '${ENTERTAINMENT_SPARKY_APP_DB_PASSWORD}'
-                create_or_update_secret 'entertainment_sparky_encryption_key'  '${ENTERTAINMENT_SPARKY_ENCRYPTION_KEY}'
-                create_or_update_secret 'entertainment_better_auth_secret'     '${ENTERTAINMENT_BETTER_AUTH_SECRET}'";;
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'entertainment_discord_token'          '$${ENTERTAINMENT_DISCORD_TOKEN}'
+                create_or_update_secret 'entertainment_discord_client_secret'  '$${ENTERTAINMENT_DISCORD_CLIENT_SECRET}'
+                create_or_update_secret 'entertainment_secret_key_base'        '$${ENTERTAINMENT_SECRET_KEY_BASE}'
+                create_or_update_secret 'entertainment_basic_auth_password'    '$${ENTERTAINMENT_BASIC_AUTH_PASSWORD}'
+                create_or_update_secret 'entertainment_sparky_db_password'     '$${ENTERTAINMENT_SPARKY_DB_PASSWORD}'
+                create_or_update_secret 'entertainment_sparky_app_db_password' '$${ENTERTAINMENT_SPARKY_APP_DB_PASSWORD}'
+                create_or_update_secret 'entertainment_sparky_encryption_key'  '$${ENTERTAINMENT_SPARKY_ENCRYPTION_KEY}'
+                create_or_update_secret 'entertainment_better_auth_secret'     '$${ENTERTAINMENT_BETTER_AUTH_SECRET}'";;
            *)
              echo "  No secrets case for $STACK";;
          esac
@@ -290,43 +389,50 @@ steps:
      - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa
      - chmod 600 ~/.ssh/id_rsa
      - |
-        if [ -z "${SWARM_MANAGER_IP}" ]; then
+        if [ -z "$${SWARM_MANAGER_IP}" ]; then
          echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets."
          exit 1
        fi
-      - ssh-keyscan -H ${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
+      - ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
      - |
        CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
        FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
-        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true)
+        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -vE '^(deploy|secrets)$' | sort -u || true)
        ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
        [ -z "$ALL_STACKS" ] && echo "No stacks changed" && exit 0

        # Always sync deploy/ scripts first
        rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
-          deploy/ root@${SWARM_MANAGER_IP}:/volume1/docker/compose-files/deploy/
+          deploy/ root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/deploy/
+
+        # Sync secrets/ tooling (manifest + examples) alongside deploy/ —
+        # provision-stack.py reads secrets/secrets-map.yaml from the CI
+        # checkout, but the host mirror should stay complete for emergency
+        # manual provisioning runs.
+        rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
+          secrets/ root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/secrets/

        for STACK in $ALL_STACKS; do
          echo "--- Deploying: $STACK ---"
          # Sync files to host first (always, even for bootstrap stacks)
          if [ -d "$STACK" ]; then
            rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
-              "$STACK/" root@${SWARM_MANAGER_IP}:/volume1/docker/compose-files/$STACK/
+              "$STACK/" root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/$STACK/
          elif [ -f "$STACK.yaml" ]; then
            rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
-              "$STACK.yaml" root@${SWARM_MANAGER_IP}:/volume1/docker/compose-files/
+              "$STACK.yaml" root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/
          fi

          # Bootstrap-tier guard: file synced to host, deploy is MANUAL
          case "$STACK" in
-            traefik|woodpecker|postgresql|secrets)
+            traefik|woodpecker|postgresql|secrets|git)
              echo "  [BOOTSTRAP] $STACK: file synced. Deploy is MANUAL."
              echo "  Run: bash /volume1/docker/compose-files/deploy/stack-deploy.sh $STACK"
              continue ;;
          esac

          # Tier 2: auto-deploy
-          ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} \
+          ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} \
            "bash /volume1/docker/compose-files/deploy/stack-deploy.sh $STACK" \
            && echo "  OK $STACK" || { echo "  FAIL $STACK"; exit 1; }
        done
@@ -344,23 +450,51 @@ steps:
      - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa
      - chmod 600 ~/.ssh/id_rsa
      - |
-        if [ -z "${SWARM_MANAGER_IP}" ]; then
+        if [ -z "$${SWARM_MANAGER_IP}" ]; then
          echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets."
          exit 1
        fi
-      - ssh-keyscan -H ${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
+      - ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
      - |
        CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
        FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
-        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true)
+        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -vE '^(deploy|secrets)$' | sort -u || true)
        ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
        [ -z "$ALL_STACKS" ] && exit 0
-        sleep 5
+
+        # NOTE: `docker stack deploy` briefly tears down and recreates tasks in
+        # Swarm's internal bookkeeping, so `docker stack ps` can transiently
+        # return nothing right after deploy even when the service is healthy.
+        # A single `sleep 5` + one-shot check produced false-alarm-looking
+        # "nothing found in stack" output on ordinary deploys (e.g. vaultwarden,
+        # 2026-08-25). Retry with backoff instead of a single fixed sleep, and
+        # only warn (don't fail the pipeline) if tasks never show up.
+        # 2026-09-08: `|| true` inside the command substitution is REQUIRED —
+        # this step runs under errexit, and an assignment from a failing
+        # command substitution (e.g. `docker stack ps` on a stack that doesn't
+        # exist) kills the whole step before the WARNING path can run.
+        ATTEMPTS=6
+        DELAY=5
        for STACK in $ALL_STACKS; do
          echo "--- $STACK ---"
-          ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} \
+          i=1
+          while [ "$i" -le "$ATTEMPTS" ]; do
+            OUTPUT=$(ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} \
              "docker stack ps $STACK --filter desired-state=running \
-            --format '  {{.Name}} {{.CurrentState}}'"
+              --format '  {{.Name}} {{.CurrentState}}'" 2>/dev/null || true)
+            if [ -n "$OUTPUT" ]; then
+              echo "$OUTPUT"
+              break
+            fi
+            if [ "$i" -eq "$ATTEMPTS" ]; then
+              echo "  WARNING: no running tasks found for $STACK after $((ATTEMPTS * DELAY))s."
+              echo "  This may be transient Swarm settle time, or a real problem — check manually:"
+              echo "  ssh root@$${SWARM_MANAGER_IP} 'docker stack ps $STACK --no-trunc'"
+            else
+              sleep "$DELAY"
+            fi
+            i=$((i + 1))
+          done
        done

  notify-success:
@@ -0,0 +1,55 @@
+# ─────────────────────────────────────────────────────────────────────────────
+# ai.env.template — AUTHORITATIVE template for ai/ai.env (rendered by
+# deploy/provision-stack.py per secrets/secrets-map.yaml).
+#
+#   - This file IS the complete key list for ai.env. The whole file is
+#     rendered on every provisioning run — no line surgery, so a key can
+#     never silently go missing again (root cause of the 2026-09-03 outage).
+#   - Key names match EXACTLY what ai/ai.yaml references (AI_<SERVICE>_*
+#     naming adopted on main 2026-09-06).
+#   - Non-secret config lives here as LITERAL values (visible, reviewable).
+#   - Secret values are dollar-brace placeholders resolved from the CI env
+#     (Woodpecker from_secret vars) at provisioning time. provision-stack.py
+#     FAILS HARD if any placeholder is missing/empty.
+#   - The rendered ai/ai.env exists only on the host (gitignored).
+#   - Rendered by provision-stack.py, NOT Woodpecker's yaml preprocessor —
+#     single-dollar placeholders are safe here (deploy.yml's double-dollar
+#     rule does NOT apply to this file).
+#
+# Consumed by ai/ai.yaml. DOMAIN_NAME comes from deploy/global.env, not here.
+# ─────────────────────────────────────────────────────────────────────────────
+
+# ── LiteLLM (non-secret config) ──────────────────────────────────────────────
+AI_AWS_REGION_NAME=us-east-2
+AI_LITELLM_MODIFY_PARAMS=False
+AI_LITELLM_DATABASE_MIGRATIONS=True
+
+# ── LiteLLM (secrets) ────────────────────────────────────────────────────────
+AI_AWS_ACCESS_KEY_ID=${AI_AWS_ACCESS_KEY_ID}
+AI_AWS_SECRET_ACCESS_KEY=${AI_AWS_SECRET_ACCESS_KEY}
+AI_LITELLM_MASTER_KEY=${AI_LITELLM_MASTER_KEY}
+AI_LITELLM_SALT_KEY=${AI_LITELLM_SALT_KEY}
+# Full connection URL is itself a secret (ai_litellm_database_url) — the
+# URL structure never appears in git.
+AI_LITELLM_DATABASE_URL=${AI_LITELLM_DATABASE_URL}
+AI_LITELLM_POSTGRES_PASSWORD=${AI_LITELLM_POSTGRES_PASSWORD}
+
+# ── Open WebUI (non-secret config) ───────────────────────────────────────────
+AI_OPEN_WEBUI_URL=https://ai.bryanmail.net
+AI_OPEN_WEBUI_ENABLE_OAUTH_SIGNUP=true
+AI_OPEN_WEBUI_OAUTH_MERGE_ACCOUNTS_BY_EMAIL=true
+AI_OPEN_WEBUI_OAUTH_PROVIDER_NAME=Authentik
+AI_OPEN_WEBUI_OPENID_PROVIDER_URL=https://auth.bryanmail.net/application/o/open-web-ui/.well-known/openid-configuration
+# OAuth client ID is a public identifier by OAuth2 design (it is sent to the
+# browser); the client SECRET below is the protected credential.
+AI_OPEN_WEBUI_OAUTH_CLIENT_ID=hVmhi1dS3TnG2cUw5QwLOx5FDLSWtnQUdZyeB5zK
+AI_OPEN_WEBUI_OAUTH_SCOPES=openid email profile
+AI_OPEN_WEBUI_OPENID_REDIRECT_URI=https://ai.bryanmail.net/oauth/oidc/callback
+
+# ── Open WebUI (secrets) ─────────────────────────────────────────────────────
+AI_OPEN_WEBUI_SECRET_KEY=${AI_OPEN_WEBUI_SECRET_KEY}
+AI_OPEN_WEBUI_DATABASE_URL=${AI_OPEN_WEBUI_DATABASE_URL}
+AI_OPEN_WEBUI_OAUTH_CLIENT_SECRET=${AI_OPEN_WEBUI_OAUTH_CLIENT_SECRET}
+
+# ── mcpo / mcpo-critical (secrets) ───────────────────────────────────────────
+MCPO_API_KEY=${AI_MCPO_API_KEY}
@@ -1,6 +1,6 @@
 services:
  open-webui:
-    image: ghcr.io/open-webui/open-webui:0.11.1
+    image: ghcr.io/open-webui/open-webui:0.11.3 #
    entrypoint:
    - /bin/bash
    - /app/tools/startup.sh
@@ -16,20 +16,20 @@ services:
      start_period: 60s
    environment:
    - OLLAMA_BASE_URL=http://ollama-intel-arc:11434
-    - WEBUI_SECRET_KEY=${WEBUI_SECRET_KEY}
+    - WEBUI_SECRET_KEY=${AI_OPEN_WEBUI_SECRET_KEY}
    - WEBUI_DB_HOST=postgresql
-    - DATABASE_URL=${OPEN_WEBUI_DATABASE_URL}
+    - DATABASE_URL=${AI_OPEN_WEBUI_DATABASE_URL}
    - ENABLE_TOOL_SERVER_CALLING=true
    - TOOL_SERVER_CALLING=true
-    - WEBUI_URL=${WEBUI_URL}
-    - ENABLE_OAUTH_SIGNUP=${ENABLE_OAUTH_SIGNUP}
-    - OAUTH_MERGE_ACCOUNTS_BY_EMAIL=${OAUTH_MERGE_ACCOUNTS_BY_EMAIL}
-    - OAUTH_PROVIDER_NAME=${OAUTH_PROVIDER_NAME}
-    - OPENID_PROVIDER_URL=${OPENID_PROVIDER_URL}
-    - OAUTH_CLIENT_ID=${OAUTH_CLIENT_ID}
-    - OAUTH_CLIENT_SECRET=${OAUTH_CLIENT_SECRET}
-    - OAUTH_SCOPES=${OAUTH_SCOPES}
-    - OPENID_REDIRECT_URI=${OPENID_REDIRECT_URI}
+    - WEBUI_URL=${AI_OPEN_WEBUI_URL}
+    - ENABLE_OAUTH_SIGNUP=${AI_OPEN_WEBUI_ENABLE_OAUTH_SIGNUP}
+    - OAUTH_MERGE_ACCOUNTS_BY_EMAIL=${AI_OPEN_WEBUI_OAUTH_MERGE_ACCOUNTS_BY_EMAIL}
+    - OAUTH_PROVIDER_NAME=${AI_OPEN_WEBUI_OAUTH_PROVIDER_NAME}
+    - OPENID_PROVIDER_URL=${AI_OPEN_WEBUI_OPENID_PROVIDER_URL}
+    - OAUTH_CLIENT_ID=${AI_OPEN_WEBUI_OAUTH_CLIENT_ID}
+    - OAUTH_CLIENT_SECRET=${AI_OPEN_WEBUI_OAUTH_CLIENT_SECRET}
+    - OAUTH_SCOPES=${AI_OPEN_WEBUI_OAUTH_SCOPES}
+    - OPENID_REDIRECT_URI=${AI_OPEN_WEBUI_OPENID_REDIRECT_URI}
    networks:
    - traefik_backend
    - postgresql_db-backend
@@ -62,13 +62,13 @@ services:
    environment:
    - AWS_ACCESS_KEY_ID=${AI_AWS_ACCESS_KEY_ID}
    - AWS_SECRET_ACCESS_KEY=${AI_AWS_SECRET_ACCESS_KEY}
-    - AWS_REGION_NAME=${AWS_REGION_NAME}
-    - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY}
-    - LITELLM_SALT_KEY=${LITELLM_SALT_KEY}
-    - DATABASE_URL=${DATABASE_URL}
-    - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
-    - LITELLM_MODIFY_PARAMS=${LITELLM_MODIFY_PARAMS}
-    - LITELLM_DATABASE_MIGRATIONS=${LITELLM_DATABASE_MIGRATIONS}
+    - AWS_REGION_NAME=${AI_AWS_REGION_NAME}
+    - LITELLM_MASTER_KEY=${AI_LITELLM_MASTER_KEY}
+    - LITELLM_SALT_KEY=${AI_LITELLM_SALT_KEY}
+    - DATABASE_URL=${AI_LITELLM_DATABASE_URL}
+    - POSTGRES_PASSWORD=${AI_LITELLM_POSTGRES_PASSWORD}
+    - LITELLM_MODIFY_PARAMS=${AI_LITELLM_MODIFY_PARAMS}
+    - LITELLM_DATABASE_MIGRATIONS=${AI_LITELLM_DATABASE_MIGRATIONS}
    networks:
    - traefik_backend
    - postgresql_db-backend
@@ -127,7 +127,7 @@ services:
      labels:
      - traefik.enable=false
  mcpo:
-    image: ghcr.io/open-webui/mcpo:main
+    image: git.bryanmail.net/homelab/flowagent-mcpo:c5b22618404a
    command:
    - --config
    - /app/config/config.json
@@ -141,6 +141,14 @@ services:
    - /volume1/docker/mcpo/data:/mcpo_data
    - /volume1/docker/cronicle/ssh_keys:/app/ssh_keys:ro
    - /volume1/docker/mcpo/uv-cache:/app/uv-cache
+    - /volume1/docker/mcpo/flowagent-auth:/app/flowagent-auth
+    secrets:
+    - source: flowagent_azure_client_id
+      target: flowagent_azure_client_id
+    - source: flowagent_azure_tenant_id
+      target: flowagent_azure_tenant_id
+    - source: flowagent_azure_client_secret
+      target: flowagent_azure_client_secret
    networks:
    - traefik_backend
    deploy:
@@ -173,6 +181,13 @@ services:
      - traefik.http.middlewares.n8n.headers.STSPreload=true
      - traefik.http.routers.n8n.middlewares=forwardAuth-authentik@file, crowdsec@file
      - traefik.swarm.network=traefik_backend
+secrets:
+  flowagent_azure_client_id:
+    external: true
+  flowagent_azure_tenant_id:
+    external: true
+  flowagent_azure_client_secret:
+    external: true
 networks:
  traefik_backend:
    external: true
@@ -22,12 +22,229 @@ def merge_envs(base_path, override_path):
    merged = {**base, **override}
    return list(merged.items())

+# ──────────────────────────────────────────────────────────────────────────
+# 2026-08-26 FIX — double-interpolation truncation bug (Pattern B stacks):
+#
+#   stack-deploy.sh's single-file/no-extras path is:
+#     envsubst "$VARS" < ai.yaml | docker stack deploy -c - ai
+#
+#   envsubst substitutes ${VAR} placeholders in the compose YAML with the
+#   literal, raw value of each shell-exported variable. If that raw value
+#   itself contains a literal '$' followed by word characters (e.g. a
+#   randomly-generated secret like "...i*Edu$RyAVYTqr4yzSS##..."), the
+#   resulting YAML text now contains what LOOKS like a second variable
+#   reference. `docker stack deploy -c -` runs Compose's own interpolation
+#   pass on that YAML text before creating the service — and Compose sees
+#   that leftover "$RyAVYTqr4yzSS", finds no such env var, and silently
+#   substitutes empty string. The secret gets truncated in the running
+#   container with NO error or warning.
+#
+#   Confirmed impact (2026-08-26): LITELLM_MASTER_KEY and LITELLM_SALT_KEY
+#   in the `ai` stack were both truncated at their first literal '$' after
+#   a real deploy — 87-char secret arrived in the container as 73 chars.
+#
+#   This affects every stack using Pattern B (host .env + envsubst, not
+#   native Docker secrets): ai, maintenance, media, unifi, guacamole,
+#   security, auth, traefik, meshcentral, ddm — any of them could have a
+#   '$'-containing value silently truncating right now without detection,
+#   since the failure is silent and only visible by diffing the source
+#   value against the live container env.
+#
+#   Fix: escape every literal '$' in a value as '$$' at export time, BEFORE
+#   envsubst ever sees it. envsubst does not interpret '$' in the
+#   replacement text (only in the template), so the doubled dollar survives
+#   envsubst untouched. Compose's interpolation pass then consumes exactly
+#   one level of escaping ('$$' -> literal '$'), landing on the correct
+#   original single '$' with no leftover variable-reference lookalike.
+#
+#   ONLY applies to the single-file path (no `docker compose config` step
+#   downstream). See export_raw/export_raw_merged below for why the
+#   folder+extras path must NOT use this.
+# ──────────────────────────────────────────────────────────────────────────
+
+def escape_dollar(v):
+    return v.replace('$', '$$')
+
+# ──────────────────────────────────────────────────────────────────────────
+# 2026-08-26 FIX #2 — SECOND double-interpolation bug, folder+extras path
+# (discovered fixing IMMICH_KIOSK_BASICAUTH, a bcrypt hash full of '$'):
+#
+#   The folder+extras render path is:
+#     docker compose <files> config | strip | envsubst "$VARS"
+#     -> docker stack deploy -c -
+#
+#   `docker compose config` performs its OWN ${VAR} interpolation AND its
+#   own re-escaping of the output: any literal '$' character that ends up
+#   in the rendered YAML — escaped or not — gets doubled to '$$' by
+#   `docker compose config` itself, unconditionally, as part of producing
+#   spec-safe output. Confirmed by isolated test:
+#     raw MYVAR='a$b$c'   -> docker compose config -> "a$$b$$c"   (correct,
+#                                                        one level added)
+#     escaped MYVAR='a$$b$$c' (i.e. pre-doubled by escape_dollar) ->
+#       docker compose config -> "a$$$$b$$$$c"        (WRONG, doubled twice)
+#
+#   `docker stack deploy -c -` still only removes exactly ONE level of
+#   escaping on its way in (confirmed: "a$$b$$c" -> container label
+#   "a$b$c", correct). So across the whole folder+extras pipeline there is
+#   exactly ONE implicit escaping step (`docker compose config`) and ONE
+#   implicit un-escaping step (`docker stack deploy`) already built in —
+#   pre-escaping the exported value on top of that leaves one extra,
+#   uncollapsed level of '$$' in the final container label/env value.
+#
+#   Confirmed impact (2026-08-26): IMMICH_KIOSK_BASICAUTH
+#   ("BabyBryan:$2y$05$...") rendered as "BabyBryan:$$2y$$05$$..." in the
+#   final container label — Traefik basic auth would never match the real
+#   password hash, silently locking out the kiosk with no error.
+#
+#   Fix: use export_raw / export_raw_merged (NO escape_dollar) whenever the
+#   render path goes through `docker compose config` — i.e. any stack with
+#   extension files. Use export / export_merged (WITH escape_dollar) only
+#   for the single-file path, which has no `docker compose config` step and
+#   therefore only Swarm's own interpolation pass to protect against.
+# ──────────────────────────────────────────────────────────────────────────
+
+# ──────────────────────────────────────────────────────────────────────────
+# 2026-08-26 FIX — depends_on long-form vs Swarm short-form:
+#
+#   Folder-based stacks with extension files (e.g. immich + hwaccel.*.yml)
+#   go through the `docker compose <files> config | strip | docker stack
+#   deploy -c -` merge path. Modern `docker compose config` normalizes the
+#   short-form list syntax:
+#     depends_on: [database, redis]
+#   into the long-form condition mapping:
+#     depends_on:
+#       database:
+#         condition: service_started
+#         required: true
+#       redis:
+#         condition: service_started
+#         required: true
+#
+#   `docker stack deploy` (Swarm mode) does NOT understand the long-form
+#   mapping and rejects it with: "services.<svc>.depends_on must be a list".
+#   Single-file stacks (no extras) never hit this because they skip the
+#   `docker compose config` step entirely and go straight through envsubst.
+#
+#   Fix: collapse any long-form depends_on mapping back into the Swarm-
+#   compatible short-form list, purely as a text transform on the rendered
+#   YAML, right alongside the existing env_file stripping.
+# ──────────────────────────────────────────────────────────────────────────
+
+_DEP_RE = re.compile(r'^([ \t]*)depends_on:\s*$')
+_CHILD_KEY_RE = re.compile(r'^([ \t]*)([\w.\-]+):\s*$')
+_CHILD_LIST_RE = re.compile(r'^([ \t]*)-\s*(\S+)\s*$')
+
+def collapse_depends_on(text):
+    lines = text.split('\n')
+    out = []
+    i = 0
+    n = len(lines)
+    while i < n:
+        line = lines[i]
+        m = _DEP_RE.match(line)
+        if not m:
+            out.append(line)
+            i += 1
+            continue
+
+        indent = m.group(1)
+        base_indent = len(indent)
+        out.append(line)
+        i += 1
+
+        services = []
+        parse_ok = True
+        while i < n:
+            l = lines[i]
+            if l.strip() == '':
+                i += 1
+                continue
+            cur_indent = len(l) - len(l.lstrip(' \t'))
+            if cur_indent <= base_indent:
+                break  # dedent — end of this depends_on block
+
+            lm = _CHILD_LIST_RE.match(l)
+            if lm and cur_indent == base_indent + 2:
+                services.append(lm.group(2))
+                i += 1
+                continue
+
+            km = _CHILD_KEY_RE.match(l)
+            if km and cur_indent == base_indent + 2:
+                services.append(km.group(2))
+                i += 1
+                # skip nested condition/required/restart lines under this service
+                while i < n:
+                    nl = lines[i]
+                    if nl.strip() == '':
+                        i += 1
+                        continue
+                    nl_indent = len(nl) - len(nl.lstrip(' \t'))
+                    if nl_indent > base_indent + 2:
+                        i += 1
+                        continue
+                    break
+                continue
+
+            # Unexpected shape — bail without transforming to avoid corrupting output
+            parse_ok = False
+            break
+
+        if not parse_ok or not services:
+            # Re-emit whatever we consumed as-is (best effort: fall back to
+            # original slice) rather than risk mangling an unfamiliar shape.
+            pass
+
+        for svc in services:
+            out.append(indent + '  - ' + svc)
+
+    return '\n'.join(out)
+
+# ──────────────────────────────────────────────────────────────────────────
+# 2026-08-26 FIX — top-level `name:` property rejected by Swarm:
+#
+#   `docker compose config` (Compose Spec output) emits a top-level
+#   `name: <project>` key as the first line of the rendered document, e.g.:
+#     name: immich
+#     services:
+#       ...
+#
+#   This is valid Compose Spec but `docker stack deploy` (Swarm mode) uses
+#   a stricter schema that does NOT allow a top-level `name` property, and
+#   rejects the whole document with: "(root) Additional property name is
+#   not allowed". Single-file stacks (no extras) never hit this because
+#   they skip `docker compose config` and go straight through envsubst on
+#   the raw source YAML, which never had a top-level `name:` to begin with.
+#
+#   Fix: drop ONLY the top-level `name:` line (column 0, i.e. no leading
+#   whitespace) at the START of the document. Nested `name:` fields under
+#   networks/volumes/secrets (e.g. `name: postgresql_db-backend`, always
+#   indented) are legitimate and must NOT be touched.
+# ──────────────────────────────────────────────────────────────────────────
+
+_TOP_NAME_RE = re.compile(r'^name:\s*\S+\s*$')
+
+def strip_top_level_name(text):
+    lines = text.split('\n')
+    if lines and _TOP_NAME_RE.match(lines[0]):
+        lines = lines[1:]
+    return '\n'.join(lines)
+
 mode = sys.argv[1]
 if mode == 'export':
    for k, v in parse_env(sys.argv[2]):
-        print('export {}={}'.format(k, repr(v)))
+        print('export {}={}'.format(k, repr(escape_dollar(v))))
 elif mode == 'export_merged':
    # export_merged <global.env> <stack.env>
+    for k, v in merge_envs(sys.argv[2], sys.argv[3]):
+        print('export {}={}'.format(k, repr(escape_dollar(v))))
+elif mode == 'export_raw':
+    # export_raw <env-file> — NO escape_dollar. Use for the docker-compose-
+    # config render path (folder+extras), which does its own '$' escaping.
+    for k, v in parse_env(sys.argv[2]):
+        print('export {}={}'.format(k, repr(v)))
+elif mode == 'export_raw_merged':
+    # export_raw_merged <global.env> <stack.env> — NO escape_dollar.
    for k, v in merge_envs(sys.argv[2], sys.argv[3]):
        print('export {}={}'.format(k, repr(v)))
 elif mode == 'vars':
@@ -38,4 +255,6 @@ elif mode == 'vars_merged':
 elif mode == 'strip':
    t = sys.stdin.read()
    t = re.sub(r'[ \t]*env_file:[ \t]*\n([ \t]+-[^\n]*\n)+', '', t)
+    t = collapse_depends_on(t)
+    t = strip_top_level_name(t)
    sys.stdout.write(t)
@@ -0,0 +1,445 @@
+#!/usr/bin/env bash
+# git-guard.sh — Ensures the compose-files working tree is in sync with Gitea
+# before any deploy proceeds. Called automatically by stack-deploy.sh.
+#
+# Behavior:
+#   - Clean + up to date        -> pass silently
+#   - Clean + behind (ff-only)  -> auto `git pull --ff-only`, then pass
+#   - Clean + ahead only        -> interactive: offer to push; non-interactive: BLOCK
+#     (unpushed)
+#   - Unresolved merge conflict -> REFUSE immediately. Never auto-commits over
+#     markers present               conflict markers. Prints remediation options.
+#   - Dirty + local in sync     -> offer to commit + push right now
+#     with origin                 (auto in non-interactive/CI runs, after a
+#                                   secret-pattern scan of the staged diff)
+#   - Dirty + local STALE/      -> NEVER commit on top of a stale base. Stash
+#     diverged vs origin           the dirty changes first, resync main with
+#                                   origin using the same behind/ahead/diverged
+#                                   rules as the clean-tree case, then reapply
+#                                   the stash and re-run. On any failure the
+#                                   stash is preserved and remediation options
+#                                   (with exact commands) are printed.
+#   - Diverged (local AND       -> REFUSE. Never auto-resolves. Prints the
+#     remote both moved)           backup/stash/reset recovery steps and exits.
+#
+# Exit codes: 0 = safe to deploy, 1 = blocked, needs human intervention
+#
+# ── Flags (added 2026-09-12, circular-dependency bootstrap incident) ────────
+#
+#   -e, --emergency
+#     Before the normal fetch, tries each candidate Gitea endpoint in order
+#     (git.bryanmail.net -> 192.168.4.30 VIP -> .31 -> .32 -> .33 node IPs,
+#     each over plain http on port 3000, which is Gitea's direct ingress
+#     port — bypasses Traefik/VIP entirely) and switches `origin` to the
+#     first one that responds to `git ls-remote` within a short timeout.
+#     This solves the bootstrap circular dependency where Traefik is down,
+#     so HTTPS access to git.bryanmail.net is unreachable, so git-guard
+#     can't fetch, so stack-deploy.sh can't redeploy traefik to fix itself.
+#     Once a reachable endpoint is found, NORMAL sync logic still runs
+#     (behind/ahead/diverged handling is unchanged) — this flag only changes
+#     *which host* is used, never skips the safety checks themselves.
+#     Prints a loud reminder to restore the real origin URL afterward; never
+#     commits the swapped URL anywhere.
+#
+#   -f, --force
+#     Skips the sync check ENTIRELY — no fetch, no comparison, no commit/push
+#     offer. Deploys whatever is on disk right now, as-is. This is the last
+#     resort for a genuine emergency where NONE of the candidate hosts in
+#     --emergency are reachable (e.g. Gitea itself is down, not just
+#     routing). Prints a loud warning banner. Never use this for routine
+#     work — it defeats the entire purpose of this script and is the exact
+#     failure mode (deploying a stale/unreviewed tree) git-guard exists to
+#     prevent.
+#
+#   Both flags are passed through from stack-deploy.sh's own -e/-f flags;
+#   see that script's header for the calling convention.
+# ─────────────────────────────────────────────────────────────────────────
+
+set -euo pipefail
+
+DIR="/volume1/docker/compose-files"
+cd "$DIR"
+
+# ---- Flag parsing ----
+EMERGENCY=0
+FORCE=0
+for arg in "$@"; do
+  case "$arg" in
+    -e|--emergency) EMERGENCY=1 ;;
+    -f|--force)     FORCE=1 ;;
+  esac
+done
+
+# ---- Force mode: skip everything ----
+if [ "$FORCE" -eq 1 ]; then
+  echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+  echo "!! FORCE MODE (-f/--force): git-guard sync check SKIPPED ENTIRELY."
+  echo "!! Deploying whatever is on disk right now, as-is. No fetch, no"
+  echo "!! comparison with origin/main was performed. This is a LAST RESORT"
+  echo "!! for emergencies where origin is completely unreachable — verify"
+  echo "!! independently that the local tree is what you intend to deploy."
+  echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+  exit 0
+fi
+
+# Non-interactive detection (Woodpecker/cron have no TTY on stdin)
+INTERACTIVE=0
+[ -t 0 ] && INTERACTIVE=1
+
+echo "==> git-guard: checking repo sync state"
+
+# ---- Emergency mode: find a reachable Gitea endpoint before fetching ----
+# Candidate order: public hostname (normal path) -> VIP -> each node's direct
+# IP. Each is tried over plain http on port 3000 (Gitea's direct ingress
+# port, published outside Traefik — see traefik/traefik.yaml git service
+# port mapping), since the whole point is to bypass Traefik/VIP when THOSE
+# are what's broken. A short `git ls-remote` timeout keeps an unreachable
+# candidate from stalling the whole check for long.
+if [ "$EMERGENCY" -eq 1 ]; then
+  echo "!! EMERGENCY MODE (-e/--emergency): probing candidate Gitea endpoints"
+  echo "!! (bypassing the normal https://git.bryanmail.net path if needed)..."
+
+  ORIGINAL_URL="$(git remote get-url origin)"
+  CANDIDATES=(
+    "https://git.bryanmail.net/homelab/compose-files.git"
+    "http://192.168.4.30:3000/homelab/compose-files.git"
+    "http://192.168.4.31:3000/homelab/compose-files.git"
+    "http://192.168.4.32:3000/homelab/compose-files.git"
+    "http://192.168.4.33:3000/homelab/compose-files.git"
+  )
+
+  FOUND=""
+  for candidate in "${CANDIDATES[@]}"; do
+    echo -n "     trying $candidate ... "
+    if timeout 5 git ls-remote "$candidate" HEAD >/dev/null 2>&1; then
+      echo "OK"
+      FOUND="$candidate"
+      break
+    else
+      echo "unreachable"
+    fi
+  done
+
+  if [ -z "$FOUND" ]; then
+    echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+    echo "!! EMERGENCY MODE: none of the candidate endpoints responded."
+    echo "!! Gitea itself may be down (not just routing) — this is beyond what"
+    echo "!! an alternate host path can fix. Options:"
+    echo "!!   A) Diagnose Gitea directly: check the git_gitea-server service"
+    echo "!!      and container on docker-1."
+    echo "!!   B) If you are certain the on-disk tree is correct and Gitea is"
+    echo "!!      genuinely unreachable, re-run with -f/--force instead — but"
+    echo "!!      read that flag's warning carefully first."
+    echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+    exit 1
+  fi
+
+  if [ "$FOUND" != "$ORIGINAL_URL" ]; then
+    git remote set-url origin "$FOUND"
+    echo "==> origin temporarily switched to: $FOUND"
+    echo "!! REMINDER: once the normal path (git.bryanmail.net / Traefik) is"
+    echo "!! confirmed healthy again, restore the real origin URL:"
+    echo "!!   cd $DIR && git remote set-url origin \"$ORIGINAL_URL\""
+    echo "!! This swap is never committed anywhere and only affects this"
+    echo "!! local checkout's git config."
+  else
+    echo "==> Normal origin URL ($FOUND) is reachable — no swap needed."
+  fi
+  echo
+fi
+
+git fetch origin --quiet
+
+LOCAL="$(git rev-parse main)"
+REMOTE="$(git rev-parse origin/main)"
+BASE="$(git merge-base main origin/main)"
+
+DIRTY=0
+git status --porcelain | grep -q . && DIRTY=1
+
+SECRET_PATTERN='(-----BEGIN [A-Z]+ PRIVATE KEY-----|AKIA[0-9A-Z]{16}|xox[baprs]-[0-9a-zA-Z-]+|password[[:space:]]*[:=][[:space:]]*[^$ ]|api[_-]?key[[:space:]]*[:=][[:space:]]*[^$ ])'
+
+# resync_with_origin <local_sha> <remote_sha> <base_sha>
+#
+# Handles the behind/ahead/diverged cases against a CLEAN working tree.
+# Shared by both the "tree was already clean" path and the new
+# "dirty tree turned out to be stale, so we stashed first" path, so the
+# two paths can never drift out of sync with each other.
+#
+# Returns 0 if it's now safe to deploy, 1 if it could not safely resolve
+# (guidance already printed to stdout in that case).
+resync_with_origin() {
+  local local_sha="$1" remote_sha="$2" base_sha="$3"
+
+  # ---- Case: fully in sync ----
+  if [ "$local_sha" = "$remote_sha" ]; then
+    echo "==> In sync with origin/main ($local_sha). OK to deploy."
+    return 0
+  fi
+
+  # ---- Case: behind only (fast-forwardable) ----
+  if [ "$local_sha" = "$base_sha" ]; then
+    echo "!! Local main is behind origin/main."
+    if [ "$INTERACTIVE" -eq 1 ]; then
+      read -rp "Fast-forward pull now? [y/N] " ans
+    else
+      ans="y"
+      echo "(non-interactive session — auto fast-forwarding)"
+    fi
+    if [[ "$ans" =~ ^[Yy]$ ]]; then
+      git pull --ff-only origin main
+      echo "==> Fast-forwarded to $(git rev-parse --short main). OK to deploy."
+      return 0
+    else
+      echo "Aborting - pull manually, then retry:"
+      echo "  cd $DIR && git pull --ff-only origin main"
+      return 1
+    fi
+  fi
+
+  # ---- Case: ahead only (local commits not yet pushed) ----
+  if [ "$remote_sha" = "$base_sha" ]; then
+    echo "!! Local main is AHEAD of origin/main (unpushed commits):"
+    git log --oneline "origin/main..main"
+    echo
+    if [ "$INTERACTIVE" -eq 1 ]; then
+      read -rp "Push local commits to origin/main now? [y/N] " ans
+    else
+      ans="n"
+      echo "(non-interactive session — will NOT auto-push ahead commits; needs human review)"
+    fi
+    if [[ "$ans" =~ ^[Yy]$ ]]; then
+      git push origin main
+      echo "==> Pushed. OK to deploy."
+      return 0
+    else
+      echo "Aborting. Review with:"
+      echo "  cd $DIR && git log origin/main..main"
+      echo "Then push manually when ready:"
+      echo "  git push origin main"
+      return 1
+    fi
+  fi
+
+  # ---- Case: true divergence (both ahead and behind) — NEVER auto-fix ----
+  echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+  echo "!! DIVERGED: local main and origin/main have both moved independently."
+  echo "!!"
+  echo "!!   Local-only commits:"
+  git log --oneline "$base_sha..main" | sed 's/^/!!     /'
+  echo "!!"
+  echo "!!   Remote-only commits:"
+  git log --oneline "$base_sha..origin/main" | sed 's/^/!!     /'
+  echo "!!"
+  echo "!! This requires a human decision - git-guard will NOT auto-resolve this."
+  echo "!!"
+  echo "!! Before stashing anything new, check whether a git-guard safety stash"
+  echo "!! ALREADY exists from this same run (avoids confusing duplicate stashes):"
+  echo "!!   git stash list"
+  echo "!!"
+  echo "!! Recommended recovery:"
+  echo "!!   1. tar backup:      tar czf /volume1/docker/compose-files-backup-\$(date +%Y%m%d-%H%M%S).tar.gz -C /volume1/docker compose-files"
+  echo "!!   2. name the branch: git branch backup/pre-reset-\$(date +%Y%m%d)"
+  echo "!!   3. stash any NEW uncommitted state only if 'git stash list' above"
+  echo "!!      didn't already show one for this run:"
+  echo "!!        git stash push -u -m 'pre-reset-snapshot'"
+  echo "!!   4. reset to origin: git reset --hard origin/main"
+  echo "!!   5. selectively restore needed files from the stash/backup branch:"
+  echo "!!        git stash list"
+  echo "!!        git stash show -p stash@{N}"
+  echo "!!        git stash apply stash@{N}   # 'apply' keeps the stash as a backup; use 'pop' to also drop it"
+  echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+  return 1
+}
+
+# ---- Case: unresolved merge conflict already present ----
+# Can happen if a PRIOR git-guard run's `git stash pop` conflicted and the
+# resulting conflict markers were never resolved before the next deploy
+# attempt. Must be checked BEFORE the dirty-tree commit flow below, because
+# an unmerged path shows up as "dirty" too, and `git add -A` would silently
+# stage the literal <<<<<<< / ======= / >>>>>>> markers into a real commit.
+if git ls-files -u | grep -q .; then
+  echo "ERROR: unresolved merge conflict markers present in the working tree."
+  echo "Refusing to auto-commit over a conflict — this would push literal"
+  echo "<<<<<<< / ======= / >>>>>>> markers to origin/main."
+  echo
+  echo "Conflicted paths:"
+  git diff --name-only --diff-filter=U | sed 's/^/  /'
+  echo
+  echo "Remediation options:"
+  echo "  A) Resolve the conflict by hand, then commit and push:"
+  echo "       cd $DIR"
+  echo "       git status                       # see conflicted paths"
+  echo "       git diff                          # inspect the conflict markers"
+  echo "       \$EDITOR <conflicted-file>          # remove markers, keep correct content"
+  echo "       git add <conflicted-file>"
+  echo "       git commit -m 'resolve git-guard stash-pop conflict'"
+  echo "       git push origin main"
+  echo "  B) Discard the conflicted merge attempt entirely and start clean from"
+  echo "     origin/main, then decide separately whether to re-apply anything"
+  echo "     from a prior safety stash:"
+  echo "       cd $DIR"
+  echo "       git checkout -- ."
+  echo "       git reset --hard origin/main"
+  echo "       git stash list                    # look for a git-guard-safety-stash-* entry"
+  echo "       git stash show -p stash@{N}         # inspect before deciding"
+  echo "  C) Once resolved (via A or B) and no longer needed, clean up the stash:"
+  echo "       git stash drop stash@{N}"
+  exit 1
+fi
+
+# ---- Case: dirty tracked changes ----
+if [ "$DIRTY" -eq 1 ]; then
+  echo "!! WORKING TREE DIRTY — uncommitted changes detected:"
+  git status --short
+  echo
+
+  # If local is ALSO stale/diverged from origin, committing right now would
+  # create a doomed commit on top of a base that's about to be rejected on
+  # push (this is exactly what caused a real incident: a stray on-disk edit
+  # sat on a checkout that was 4 commits behind, git-guard auto-committed
+  # anyway, then the push bounced). Stash first, resync safely using the
+  # same rules as the clean-tree path, then reapply.
+  if [ "$LOCAL" != "$REMOTE" ]; then
+    echo "!! Local main is ALSO stale/diverged from origin/main."
+    echo "   Refusing to commit on top of a stale base — stashing the dirty"
+    echo "   changes safely first, then resyncing with origin."
+    echo
+
+    STASH_MSG="git-guard-safety-stash-$(date -u +%Y%m%dT%H%M%SZ)"
+    if ! git stash push -u -m "$STASH_MSG"; then
+      echo "ERROR: 'git stash push' itself failed (disk full, permissions, or"
+      echo "       some other git error). Your changes are still on disk,"
+      echo "       uncommitted — nothing has been lost, but git-guard cannot"
+      echo "       proceed safely until this is resolved."
+      echo
+      echo "Remediation options:"
+      echo "  A) Check disk space and permissions, then retry the deploy:"
+      echo "       df -h $DIR"
+      echo "       ls -la $DIR"
+      echo "  B) Identify and manually move aside whatever is blocking the stash,"
+      echo "     then retry:"
+      echo "       cd $DIR"
+      echo "       git status --short              # find the offending path(s)"
+      echo "       mv <path> <path>.bak-\$(date +%s)"
+      echo "  C) Inspect the raw git error above for specifics before proceeding."
+      exit 1
+    fi
+    echo "==> Stashed as: $STASH_MSG"
+
+    if resync_with_origin "$LOCAL" "$REMOTE" "$BASE"; then
+      echo "==> Resync succeeded. Reapplying stashed changes..."
+      if git stash pop; then
+        echo "==> Stash reapplied cleanly. Re-checking sync state..."
+        exec bash "$0" "$@"
+      else
+        echo "ERROR: 'git stash pop' did not complete successfully."
+        echo
+        if git ls-files -u | grep -q .; then
+          echo "This is a MERGE CONFLICT — your stashed changes were partially"
+          echo "applied and conflict markers (<<<<<<< / ======= / >>>>>>>) are now"
+          echo "in the working tree. The stash itself is still preserved as a backup."
+          echo
+          echo "Remediation options:"
+          echo "  A) Resolve the conflict by hand, then commit and push:"
+          echo "       cd $DIR"
+          echo "       git status                       # see conflicted paths"
+          echo "       git diff                          # inspect the markers"
+          echo "       \$EDITOR <conflicted-file>          # remove markers, keep correct content"
+          echo "       git add <conflicted-file>"
+          echo "       git commit -m 'resolve git-guard stash-pop conflict'"
+          echo "       git push origin main"
+          echo "       git stash list                    # confirm which entry is: $STASH_MSG"
+          echo "       git stash drop stash@{N}           # once confirmed no longer needed"
+          echo "  B) Abandon this merge attempt and fall back to a clean, resynced"
+          echo "     tree, then re-apply the change manually with full visibility:"
+          echo "       cd $DIR"
+          echo "       git checkout -- ."
+          echo "       git reset --hard origin/main      # now matches origin, no conflict"
+          echo "       git stash list                    # find: $STASH_MSG"
+          echo "       git stash show -p stash@{N}         # review the content"
+          echo "       git stash apply stash@{N}          # 'apply' keeps the backup; use 'pop' to also drop it"
+        else
+          echo "This looks like an UNTRACKED-FILE COLLISION, not a merge conflict"
+          echo "(a file added upstream shares a path with an untracked file in your"
+          echo "stash). No conflict markers were written; the stash was NOT applied"
+          echo "and remains fully intact."
+          echo
+          echo "Remediation options:"
+          echo "  A) Move the colliding upstream file aside, pop, then reconcile:"
+          echo "       cd $DIR"
+          echo "       git status --short                         # identify the colliding path"
+          echo "       mv <path> <path>.upstream-\$(date +%s)"
+          echo "       git stash pop"
+          echo "       diff <path> <path>.upstream-*               # reconcile manually, then remove the .upstream-* backup"
+          echo "  B) Inspect the stash without applying, and hand-merge the needed"
+          echo "     pieces instead:"
+          echo "       git stash list                              # find N"
+          echo "       git stash show -p stash@{N}"
+        fi
+        echo
+        echo "Your stash reference for this run: $STASH_MSG"
+        exit 1
+      fi
+    else
+      echo "ERROR: could not safely resync with origin/main."
+      echo "Your uncommitted changes are preserved in the stash: $STASH_MSG"
+      echo
+      echo "Remediation options:"
+      echo "  A) Follow the manual recovery steps printed above (from the"
+      echo "     behind/ahead/diverged case), THEN reapply your change:"
+      echo "       cd $DIR"
+      echo "       git stash list                 # find: $STASH_MSG"
+      echo "       git stash apply stash@{N}      # or 'pop' to also drop it once resynced"
+      echo "  B) If the stashed change is no longer needed (e.g. it's already"
+      echo "     represented in a since-merged PR), verify then drop it:"
+      echo "       git stash show -p stash@{N}"
+      echo "       git stash drop stash@{N}"
+      exit 1
+    fi
+  fi
+
+  if [ "$INTERACTIVE" -eq 1 ]; then
+    read -rp "Commit and push these changes to origin/main now? [y/N] " ans
+  else
+    ans="y"
+    echo "(non-interactive session — auto-committing and pushing)"
+  fi
+
+  if [[ "$ans" =~ ^[Yy]$ ]]; then
+    git add -A
+
+    if git diff --cached | grep -Eiq "$SECRET_PATTERN"; then
+      echo "ERROR: possible secret detected in staged changes. Refusing to auto-commit."
+      echo "Review manually:"
+      echo "  cd $DIR && git diff --cached"
+      git reset
+      exit 1
+    fi
+
+    git commit -m "chore(auto): git-guard autofix - commit local changes before deploy $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+
+    if git push origin main; then
+      echo "==> Pushed. Re-checking sync state..."
+      exec bash "$0" "$@"
+    else
+      echo "ERROR: push failed (likely diverged from origin). Aborting deploy."
+      echo "Run:"
+      echo "  cd $DIR && git status"
+      exit 1
+    fi
+  else
+    echo "Aborting deploy - commit or stash changes manually, then retry:"
+    echo "  cd $DIR"
+    echo "  git add -A && git commit -m 'your message' && git push origin main"
+    echo "  # or: git stash push -u -m 'manual-stash'"
+    exit 1
+  fi
+fi
+
+# ---- Clean tree: resync with origin using the shared logic above ----
+if resync_with_origin "$LOCAL" "$REMOTE" "$BASE"; then
+  exit 0
+else
+  exit 1
+fi
@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+"""
+mount-guard.py — Pre-flight check for bind mount paths before `docker stack deploy`.
+
+Usage: mount-guard.py <rendered-compose.yml>
+
+Reads the FINAL rendered compose YAML (after env substitution, right before
+it would be handed to `docker stack deploy -c -`) and checks:
+
+  1. MISSING PATHS — every bind-mount source path referenced by any service
+     actually exists on disk. Swarm rejects the task at schedule time if not
+     (see: "bind source path does not exist"), but catching it here is
+     faster and clearer, and gives the option to create it on the spot.
+
+  2. SUSPICIOUS POSTGRES DATA DIRS — for any bind mount whose *target*
+     looks like a Postgres data directory, warn if the *source* exists but
+     is empty or missing a PG_VERSION file. This is the class of near-miss
+     from the 2026-08-26 Immich incident: a wrong-but-existing empty path
+     would have let Postgres silently initialize a brand-new database while
+     the real data sat orphaned elsewhere, with no error at all.
+
+Exit codes: 0 = safe to deploy, 1 = blocked / aborted.
+
+Interactive sessions get a prompt with remediation options. Non-interactive
+sessions (Woodpecker, cron, CI) NEVER auto-proceed past a finding here —
+this is a data-safety check, not a convenience autofix.
+"""
+
+import sys
+import os
+import re
+import yaml
+
+
+def is_bind_like(path):
+    return isinstance(path, str) and (
+        path.startswith('/') or path.startswith('./') or path.startswith('../')
+    )
+
+
+def gather_bind_mounts(doc):
+    """Return list of (service_name, target, source) for host-path bind mounts.
+
+    Handles both compose-file syntaxes that can appear in a rendered stack:
+      - short form:  "hostpath:containerpath[:mode]"        (raw single-file path)
+      - long form:   {type: bind, source: ..., target: ...} (post `docker compose config`)
+    Named volumes (source has no leading '/', './', '../') are intentionally skipped.
+    """
+    results = []
+    services = doc.get('services') or {}
+    for svc_name, svc in services.items():
+        if not isinstance(svc, dict):
+            continue
+
+        for v in (svc.get('volumes') or []):
+            if isinstance(v, str):
+                parts = v.split(':')
+                if len(parts) >= 2 and is_bind_like(parts[0]):
+                    results.append((svc_name, parts[1], parts[0]))
+            elif isinstance(v, dict):
+                if v.get('type') == 'bind':
+                    src = v.get('source')
+                    tgt = v.get('target')
+                    if is_bind_like(src):
+                        results.append((svc_name, tgt, src))
+
+        # devices: short form "host:container[:mode]" (rarely used in this repo —
+        # devices are conventionally expressed as bind-mounted volumes instead,
+        # for Docker Swarm / DDM compatibility — but handle it if present).
+        for d in (svc.get('devices') or []):
+            if isinstance(d, str):
+                parts = d.split(':')
+                if parts and is_bind_like(parts[0]):
+                    tgt = parts[1] if len(parts) > 1 else parts[0]
+                    results.append((svc_name, tgt, parts[0]))
+
+    return results
+
+
+_PG_TARGET_RE = re.compile(r'postgres', re.IGNORECASE)
+
+
+def looks_like_postgres_target(target):
+    return bool(target) and bool(_PG_TARGET_RE.search(target))
+
+
+def check_postgres_data(path):
+    """Return a warning string if `path` looks like an empty/uninitialized
+    Postgres data directory. Returns None if it looks fine. Caller must
+    ensure `path` already exists before calling this."""
+    if not os.path.isdir(path):
+        # Exists but isn't a directory (e.g. a file) — a different, separate
+        # problem; the missing-path style check doesn't apply, but this is
+        # clearly wrong too.
+        return "exists but is not a directory"
+    try:
+        entries = os.listdir(path)
+    except PermissionError:
+        return "cannot list directory contents (permission denied) — unable to verify"
+    if not entries:
+        return "directory exists but is EMPTY — looks like an uninitialized/wrong Postgres data dir"
+    if 'PG_VERSION' not in entries:
+        return "directory exists and is non-empty but has no PG_VERSION file — does not look like a real Postgres data directory"
+    return None
+
+
+def prompt(question, choices):
+    """choices: dict of key -> description. Returns the chosen key (lowercase)."""
+    print("Options:")
+    for k, desc in choices.items():
+        print(f"  [{k}] {desc}")
+    while True:
+        ans = input(f"{question} [{'/'.join(choices.keys())}]: ").strip().lower()
+        if ans in choices:
+            return ans
+        print(f"Please enter one of: {', '.join(choices.keys())}")
+
+
+def main():
+    if len(sys.argv) != 2:
+        print("Usage: mount-guard.py <rendered-compose.yml>", file=sys.stderr)
+        return 1
+
+    rendered_path = sys.argv[1]
+    with open(rendered_path) as f:
+        doc = yaml.safe_load(f)
+
+    if not doc or 'services' not in doc:
+        print("!! mount-guard: rendered YAML has no 'services' key — refusing to guess, blocking.")
+        return 1
+
+    mounts = gather_bind_mounts(doc)
+
+    missing = []
+    pg_warnings = []
+
+    for svc_name, target, src in mounts:
+        if not os.path.exists(src):
+            missing.append((svc_name, target, src))
+            continue
+        if looks_like_postgres_target(target):
+            warn = check_postgres_data(src)
+            if warn:
+                pg_warnings.append((svc_name, target, src, warn))
+
+    if not missing and not pg_warnings:
+        print(f"==> mount-guard: {len(mounts)} bind mount path(s) checked, all present and sane. OK to deploy.")
+        return 0
+
+    print("!! mount-guard found potential issues with bind mount paths:\n")
+
+    if missing:
+        print("MISSING PATHS (Swarm will reject the task if these aren't created first):")
+        for svc_name, target, src in missing:
+            print(f"  - service '{svc_name}': {src}  (mounted at {target})")
+        print()
+
+    if pg_warnings:
+        print("SUSPICIOUS POSTGRES DATA DIRECTORIES:")
+        print("  Path exists, but contents don't look like a real Postgres data dir.")
+        print("  This is the exact shape of the 2026-08-26 Immich near-miss: a wrong")
+        print("  bind path that HAPPENED to exist (empty) would have let Postgres")
+        print("  silently init a new empty DB while the real data sat orphaned")
+        print("  elsewhere — with no error or warning from Docker at all.")
+        for svc_name, target, src, warn in pg_warnings:
+            print(f"  - service '{svc_name}': {src}  (mounted at {target})")
+            print(f"      {warn}")
+        print()
+
+    interactive = sys.stdin.isatty()
+
+    if not interactive:
+        print("(non-interactive session — mount-guard will NOT auto-proceed on any finding above)")
+        print("Re-run interactively to review and decide, or fix the paths and retry.")
+        return 1
+
+    if missing:
+        choices = {
+            'a': 'Abort deploy (recommended if any path above is unexpected)',
+            'm': 'mkdir -p the missing path(s) now, then continue',
+            'c': 'Continue anyway without creating them (will likely fail at Swarm scheduling)',
+        }
+        ans = prompt("Missing bind mount paths found. Choice", choices)
+        if ans == 'a':
+            print("Aborting deploy.")
+            return 1
+        if ans == 'm':
+            for _, _, src in missing:
+                print(f"  mkdir -p {src}")
+                os.makedirs(src, exist_ok=True)
+            print("==> Created missing path(s). Continuing.")
+        # 'c' falls through and continues without creating
+
+    if pg_warnings:
+        choices = {
+            'a': 'Abort deploy (recommended unless you intended a fresh Postgres init here)',
+            'c': 'Continue anyway (I have verified this is expected, e.g. legitimate first-time init)',
+        }
+        ans = prompt("Suspicious Postgres data directory found. Choice", choices)
+        if ans == 'a':
+            print("Aborting deploy.")
+            return 1
+        print("==> Continuing deploy despite Postgres data warning, per operator confirmation.")
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())
@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+"""provision-stack.py — manifest-driven env-file rendering + Docker secret
+provisioning for one stack.
+
+Usage (from the CI workspace root, inside the provision-secrets step):
+    python3 deploy/provision-stack.py <stack>
+
+Reads secrets/secrets-map.yaml (data only — no code, no values) and, for the
+named stack:
+
+  1. env_template -> renders the COMPLETE env file. Placeholders of the form
+     dollar-brace VARNAME are resolved from this process's environment (the
+     Woodpecker from_secret-backed vars). The whole file is rendered every
+     run; nothing is line-edited in place, so keys can never silently go
+     missing (root cause of the 2026-09-03 ai.env incident).
+  2. env_dest -> ships the rendered file to the Swarm manager over ssh stdin
+     (write-to-temp + atomic mv, mode 600). The rendered file never touches
+     the CI workspace disk under the repo (no chance of being committed) and
+     never appears on a command line.
+  3. docker_secrets -> for each swarm-secret-name -> ENV_VAR mapping, creates
+     or rotates the Docker secret. Values are passed via ssh stdin only.
+     Rotation uses the same sha256-checksum-label convention as
+     deploy/create-secrets.sh (unchanged secrets are skipped silently).
+
+Safety properties:
+  - FAILS HARD (non-zero) if any referenced env var is missing or empty, and
+    lists the missing NAMES. A partial/broken render can never ship.
+  - FAILS HARD if any unresolved placeholder remains after rendering.
+  - NEVER prints a secret value — names and counts only.
+  - Requires SWARM_MANAGER_IP in the environment and a usable ssh identity
+    (both already set up by the provision-secrets step).
+
+Stacks not present in the manifest exit 0 with a notice, so this script is
+safe to call unconditionally; legacy case-entries in deploy.yml keep handling
+unmigrated stacks.
+"""
+
+import hashlib
+import os
+import re
+import subprocess
+import sys
+
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+MANIFEST_PATH = os.path.join(REPO_ROOT, "secrets", "secrets-map.yaml")
+REMOTE_BASE = "/volume1/docker/compose-files"
+PLACEHOLDER_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
+
+SSH_OPTS = ["-o", "StrictHostKeyChecking=no"]
+
+
+def die(msg: str) -> None:
+    print(f"ERROR: {msg}", file=sys.stderr)
+    sys.exit(1)
+
+
+def load_manifest() -> dict:
+    try:
+        import yaml  # py3-yaml, installed by the provision-secrets step
+    except ImportError:
+        die("PyYAML not available — provision-secrets step must apk add py3-yaml")
+    if not os.path.isfile(MANIFEST_PATH):
+        die(f"manifest not found: {MANIFEST_PATH}")
+    with open(MANIFEST_PATH, "r", encoding="utf-8") as fh:
+        data = yaml.safe_load(fh) or {}
+    stacks = data.get("stacks")
+    if not isinstance(stacks, dict):
+        die("manifest has no 'stacks:' mapping")
+    return stacks
+
+
+def ssh_target() -> str:
+    ip = os.environ.get("SWARM_MANAGER_IP", "").strip()
+    if not ip:
+        die("SWARM_MANAGER_IP is empty — check Woodpecker repo secrets")
+    return f"root@{ip}"
+
+
+def ssh_run(target: str, remote_cmd: str, stdin_data: bytes | None = None,
+            check: bool = True) -> subprocess.CompletedProcess:
+    proc = subprocess.run(
+        ["ssh", *SSH_OPTS, target, remote_cmd],
+        input=stdin_data, capture_output=True,
+    )
+    if check and proc.returncode != 0:
+        # stderr may be verbose but must never contain our secret values —
+        # we only ever send values via stdin, never embed them in remote_cmd.
+        die(f"remote command failed (rc={proc.returncode}): {remote_cmd}\n"
+            f"{proc.stderr.decode(errors='replace').strip()}")
+    return proc
+
+
+def render_template(template_path: str) -> str:
+    if not os.path.isfile(template_path):
+        die(f"env_template not found: {template_path}")
+    with open(template_path, "r", encoding="utf-8") as fh:
+        raw = fh.read()
+
+    referenced = sorted(set(PLACEHOLDER_RE.findall(raw)))
+    missing = [v for v in referenced
+               if not os.environ.get(v, "").strip()]
+    if missing:
+        die("template references vars that are MISSING or EMPTY in the CI "
+            "environment (add them via from_secret in "
+            ".woodpecker/deploy.yml provision-secrets, and as Woodpecker "
+            "secrets):\n  " + "\n  ".join(missing))
+
+    rendered = PLACEHOLDER_RE.sub(lambda m: os.environ[m.group(1)], raw)
+
+    # Belt-and-braces: nothing placeholder-shaped may survive the render.
+    leftover = sorted(set(PLACEHOLDER_RE.findall(rendered)))
+    if leftover:
+        die("unresolved placeholders remain after rendering: "
+            + ", ".join(leftover))
+
+    print(f"  [render] {template_path}: {len(referenced)} secret placeholder(s) "
+          f"resolved: {', '.join(referenced)}")
+    return rendered
+
+
+def ship_env_file(target: str, rendered: str, dest_rel: str) -> None:
+    dest = f"{REMOTE_BASE}/{dest_rel}"
+    tmp = f"{dest}.provision-tmp"
+    # Value travels over ssh stdin; never on a command line; atomic mv.
+    ssh_run(target,
+            f"umask 077 && cat > {tmp} && chmod 600 {tmp} && mv {tmp} {dest}",
+            stdin_data=rendered.encode())
+    keys = [ln.split("=", 1)[0] for ln in rendered.splitlines()
+            if "=" in ln and not ln.lstrip().startswith("#") and ln.strip()]
+    print(f"  [env] shipped {dest} ({len(keys)} keys): {', '.join(keys)}")
+
+
+def provision_docker_secret(target: str, name: str, env_var: str) -> None:
+    value = os.environ.get(env_var, "")
+    if not value.strip():
+        die(f"docker secret '{name}': env var {env_var} is missing/empty")
+
+    new_hash = hashlib.sha256(value.encode()).hexdigest()
+    probe = ssh_run(
+        target,
+        f"docker secret inspect {name} "
+        "--format '{{index .Spec.Labels \"checksum\"}}' 2>/dev/null || true",
+        check=True)
+    old_hash = probe.stdout.decode().strip()
+
+    if old_hash == new_hash:
+        print(f"  [skip]   docker secret {name} (unchanged)")
+        return
+
+    if old_hash:
+        rm = ssh_run(target, f"docker secret rm {name}", check=False)
+        if rm.returncode != 0:
+            die(f"docker secret {name}: value changed but removal failed — "
+                "it is probably referenced by a running service. Provision "
+                "under a versioned name (see vaultwarden_database_url_v2 "
+                "precedent) or scale the service down first.")
+        action = "update"
+    else:
+        action = "create"
+
+    ssh_run(target,
+            f"docker secret create --label checksum={new_hash} "
+            f"--label managed-by=woodpecker {name} -",
+            stdin_data=value.encode())
+    print(f"  [{action}] docker secret {name} (value via stdin)")
+
+
+def main() -> None:
+    if len(sys.argv) != 2:
+        die("usage: provision-stack.py <stack>")
+    stack = sys.argv[1]
+
+    stacks = load_manifest()
+    cfg = stacks.get(stack)
+    if cfg is None:
+        print(f"  [info] stack '{stack}' not in secrets-map.yaml — "
+              "legacy provisioning (deploy.yml case-entry) applies. Nothing to do.")
+        return
+
+    target = ssh_target()
+    print(f"==> provision-stack: {stack}")
+
+    template_rel = cfg.get("env_template")
+    dest_rel = cfg.get("env_dest")
+    if template_rel and not dest_rel:
+        die("env_template set but env_dest missing in manifest")
+    if template_rel:
+        rendered = render_template(os.path.join(REPO_ROOT, template_rel))
+        ship_env_file(target, rendered, dest_rel)
+
+    for name, env_var in (cfg.get("docker_secrets") or {}).items():
+        provision_docker_secret(target, name, env_var)
+
+    print(f"==> provision-stack: {stack} done")
+
+
+if __name__ == "__main__":
+    main()
@@ -20,14 +20,64 @@
 # All files in the folder matching *.yml or *.yaml are included.
 # Main file (<stack>.yml or <stack>.yaml) is always passed FIRST.
 # Remaining files are sorted and appended.
+#
+# Render pipeline (unified for all modes as of 2026-08-26):
+#   1. Render full compose YAML (with extras merged + env substituted) to a
+#      temp file.
+#   2. Run mount-guard.py against that temp file — checks every bind mount
+#      source path exists, and flags suspicious-looking empty Postgres data
+#      dirs, before anything touches Swarm.
+#   3. docker stack deploy -c <tempfile> <stack>
+#
+# Dollar-escaping (2026-08-26, see envparse.py comments for full detail):
+#   - Single-file path (no extras): envsubst does no '$' escaping of its
+#     own, and only Swarm's `docker stack deploy` interpolation pass runs
+#     downstream -> use export/export_merged (escapes '$' -> '$$' once).
+#   - Folder+extras path: `docker compose config` ALSO does its own '$'
+#     escaping on top of Swarm's -> use export_raw/export_raw_merged (no
+#     pre-escaping) or values get doubled twice. Getting this wrong
+#     silently corrupts any secret/hash containing '$' (confirmed impact:
+#     LITELLM keys truncated, IMMICH_KIOSK_BASICAUTH bcrypt hash mismatched).
+#
+# Usage: stack-deploy.sh [-e|--emergency] [-f|--force] <stack-name>
+#   Flags may appear before or after the stack name, e.g. both
+#   `stack-deploy.sh traefik -e` and `stack-deploy.sh -e traefik` work.
+#
+# -e/--emergency and -f/--force (added 2026-09-12, circular-dependency
+# bootstrap incident — see deploy/git-guard.sh header for full detail):
+#   Both are passed straight through to git-guard.sh unchanged; this script
+#   does not interpret them itself beyond stripping them from the stack-name
+#   argument list. -e tries alternate Gitea endpoints (VIP, then each node's
+#   direct IP) before falling back to normal sync logic against whichever
+#   one responds. -f skips the sync check entirely — last resort only, read
+#   the warning banner it prints. Neither flag changes anything about the
+#   render/mount-guard/deploy steps below; they only affect whether and how
+#   git-guard.sh's pre-flight check runs.

 set -euo pipefail

-STACK="${1:?Usage: stack-deploy.sh <stack-name>}"
+# ── Flag parsing (stack name is whatever's left after flags are stripped) ──
+GUARD_FLAGS=()
+STACK=""
+for arg in "$@"; do
+  case "$arg" in
+    -e|--emergency) GUARD_FLAGS+=(-e) ;;
+    -f|--force)     GUARD_FLAGS+=(-f) ;;
+    *)              STACK="$arg" ;;
+  esac
+done
+: "${STACK:?Usage: stack-deploy.sh [-e|--emergency] [-f|--force] <stack-name>}"
+
 DIR="/volume1/docker/compose-files"
 PY="$DIR/deploy/envparse.py"
+MOUNT_GUARD="$DIR/deploy/mount-guard.py"
 GLOBAL_ENV="$DIR/deploy/global.env"

+# ── Pre-flight: ensure local checkout is in sync with Gitea ─────────────────
+# Prevents deploying from a stale/diverged local tree (see incident 2026-08-26).
+# Invoked via `bash` explicitly so the tracked file's exec bit doesn't matter.
+bash "$DIR/deploy/git-guard.sh" "${GUARD_FLAGS[@]:-}" || { echo "ERROR: git-guard check failed. Deploy aborted."; exit 1; }
+
 # ── Locate compose file(s) ──────────────────────────────────────────────────

 FOLDER="$DIR/$STACK"
@@ -80,6 +130,12 @@ for f in "${EXTRAS[@]:-}"; do
  [ -n "$f" ] && F_FLAGS+=(-f "$f")
 done

+# Whether the render will go through `docker compose config` (folder mode
+# with 1+ extras). This determines which escaping mode is correct — see
+# header comment and envparse.py for why these must differ.
+USES_COMPOSE_CONFIG=0
+[ "${#F_FLAGS[@]}" -gt 2 ] && USES_COMPOSE_CONFIG=1
+
 # ── Load env (global base + optional stack override) ─────────────────────────

 HINT="  Hint: 'secret not found' means Woodpecker hasn't provisioned secrets yet.\n  Trigger the pipeline: https://woodpecker.bryanmail.net\n"
@@ -89,19 +145,25 @@ STACK_EXISTS=0
 [ -f "$GLOBAL_ENV" ] && GLOBAL_EXISTS=1
 [ -f "$ENVFILE"    ] && STACK_EXISTS=1

+if [ "$USES_COMPOSE_CONFIG" -eq 1 ]; then
+  EXPORT_MODE="export_raw"; EXPORT_MERGED_MODE="export_raw_merged"
+else
+  EXPORT_MODE="export"; EXPORT_MERGED_MODE="export_merged"
+fi
+
 if [ "$GLOBAL_EXISTS" -eq 1 ] && [ "$STACK_EXISTS" -eq 1 ]; then
-  echo "    Env:     $GLOBAL_ENV + $ENVFILE (stack overrides global)"
-  eval "$(python3 "$PY" export_merged "$GLOBAL_ENV" "$ENVFILE")"
+  echo "    Env:     $GLOBAL_ENV + $ENVFILE (stack overrides global) [$EXPORT_MERGED_MODE]"
+  eval "$(python3 "$PY" "$EXPORT_MERGED_MODE" "$GLOBAL_ENV" "$ENVFILE")"
  VARS="$(python3 "$PY" vars_merged "$GLOBAL_ENV" "$ENVFILE")"

 elif [ "$GLOBAL_EXISTS" -eq 1 ]; then
-  echo "    Env:     $GLOBAL_ENV (no stack env)"
-  eval "$(python3 "$PY" export "$GLOBAL_ENV")"
+  echo "    Env:     $GLOBAL_ENV (no stack env) [$EXPORT_MODE]"
+  eval "$(python3 "$PY" "$EXPORT_MODE" "$GLOBAL_ENV")"
  VARS="$(python3 "$PY" vars "$GLOBAL_ENV")"

 elif [ "$STACK_EXISTS" -eq 1 ]; then
-  echo "    Env:     $ENVFILE (no global env)"
-  eval "$(python3 "$PY" export "$ENVFILE")"
+  echo "    Env:     $ENVFILE (no global env) [$EXPORT_MODE]"
+  eval "$(python3 "$PY" "$EXPORT_MODE" "$ENVFILE")"
  VARS="$(python3 "$PY" vars "$ENVFILE")"

 else
@@ -111,33 +173,44 @@ fi

 echo "    Vars:    $VARS"

-# ── Deploy ───────────────────────────────────────────────────────────────────
+# ── Render final compose YAML to a temp file ──────────────────────────────────
+
+RENDERED="$(mktemp /tmp/stack-deploy.XXXXXX.yml)"
+trap 'rm -f "$RENDERED"' EXIT

 if [ -n "$VARS" ]; then
-  if [ "${#F_FLAGS[@]}" -gt 2 ]; then
+  if [ "$USES_COMPOSE_CONFIG" -eq 1 ]; then
    # Folder mode with extras: merge via docker compose config
    docker compose "${F_FLAGS[@]}" config \
      | python3 "$PY" strip \
      | envsubst "$VARS" \
-      | docker stack deploy -c - "$STACK" \
-      || { printf "\n%b" "$HINT"; exit 1; }
+      > "$RENDERED"
  else
    # Single file
    envsubst "$VARS" < "$MAIN" \
      | python3 "$PY" strip \
-      | docker stack deploy -c - "$STACK" \
-      || { printf "\n%b" "$HINT"; exit 1; }
+      > "$RENDERED"
  fi
 else
-  if [ "${#F_FLAGS[@]}" -gt 2 ]; then
+  if [ "$USES_COMPOSE_CONFIG" -eq 1 ]; then
    docker compose "${F_FLAGS[@]}" config \
      | python3 "$PY" strip \
-      | docker stack deploy -c - "$STACK" \
-      || { printf "\n%b" "$HINT"; exit 1; }
+      > "$RENDERED"
  else
-    docker stack deploy -c "$MAIN" "$STACK" \
-      || { printf "\n%b" "$HINT"; exit 1; }
+    python3 "$PY" strip < "$MAIN" > "$RENDERED"
  fi
 fi

+# ── Pre-flight: bind mount paths exist + Postgres-data sanity ───────────────
+# See deploy/mount-guard.py for details. Blocks on missing paths or
+# suspicious-looking empty/uninitialized Postgres data directories (see
+# incident 2026-08-26: a wrong-but-existing empty bind path would have let
+# Postgres silently init a fresh DB while real data sat orphaned elsewhere).
+python3 "$MOUNT_GUARD" "$RENDERED" || { echo "ERROR: mount-guard check failed. Deploy aborted."; exit 1; }
+
+# ── Deploy ───────────────────────────────────────────────────────────────────
+
+docker stack deploy -c "$RENDERED" "$STACK" \
+  || { printf "\n%b" "$HINT"; exit 1; }
+
 echo "==> Done: $STACK"
@@ -63,6 +63,15 @@ services:
      FRIGATE_RTSP_PASSWORD: "${FRIGATE_RTSP_PASSWORD}"
    networks:
      - traefik_backend
+    ports:
+      - target: 8555
+        published: 8555
+        protocol: udp
+        mode: ingress
+      - target: 8555
+        published: 8555
+        protocol: tcp
+        mode: ingress
    deploy:
      labels:
        - traefik.enable=true
@@ -1,11 +1,25 @@
 # Immich stack environment variables - EXAMPLE
-# Copy to immich.env and fill in actual values
-# Pattern C (partial): DB_PASSWORD as Docker secret (_FILE), IMMICH_KIOSK_BASICAUTH in host .env (Traefik label)
+# Copy this to immich.env (untracked, see .gitignore) and fill in
+# IMMICH_KIOSK_BASICAUTH with a real value. Do NOT commit immich.env.
+#
+# Pattern C (partial): DB_PASSWORD as Docker secret (_FILE), IMMICH_KIOSK_BASICAUTH
+# in host .env (Traefik label).
+#
+# NOTE 2026-08-26: the paths below are the REAL, confirmed-correct values for
+# this homelab (verified via `docker service inspect immich_<svc> --format
+# '{{json .PreviousSpec.TaskTemplate.ContainerSpec.Mounts}}'` against actual
+# on-disk data). An earlier version of this file had plausible-looking but
+# WRONG generic paths (/volume1/docker/immich/db, /volume1/docker/immich/uploads),
+# which caused a real incident when immich.env was regenerated from this
+# template without checking against the live services first. If you ever
+# need to regenerate immich.env from this file, these paths should still be
+# correct — but if in doubt, re-verify with the PreviousSpec command above
+# before deploying.

 IMMICH_VERSION=release
-UPLOAD_LOCATION=/volume1/docker/immich/uploads
-BULK_UPLOAD_LOCATION=/volume1/docker/immich/bulk-uploads
-DB_DATA_LOCATION=/volume1/docker/immich/db
+UPLOAD_LOCATION=/volume1/Immich-Photos/
+BULK_UPLOAD_LOCATION=/volume1/Immich-Photos/bulk-upload
+DB_DATA_LOCATION=/volume1/docker/immich-postgresql
 REDIS_DATA_LOCATION=/volume1/docker/immich/redis

 DB_USERNAME=immich
@@ -20,4 +34,4 @@ REDIS_DBINDEX=0
 IMMICH_TRUSTED_PROXIES=172.16.0.0/12
 IMMICH_TRAEFIK_HOST=immich.bryanmail.net
 IMMICH_KIOSK_HOST=immich-kiosk.bryanmail.net
-IMMICH_KIOSK_BASICAUTH=user:password
+IMMICH_KIOSK_BASICAUTH=user:changeme
@@ -1,186 +0,0 @@
-{
-  "mcpServers": {
-    "filesystem": {
-      "command": "npx",
-      "args": [
-        "-y",
-        "@modelcontextprotocol/server-filesystem",
-        "/mcpo_data/filesystem"
-      ]
-    },
-    "memory": {
-      "command": "npx",
-      "args": [
-        "-y",
-        "@modelcontextprotocol/server-memory"
-      ]
-    },
-    "proxmox-nuck7-1": {
-      "command": "sh",
-      "args": [
-        "-c",
-        "LOG_LEVEL=silent npx -y --package @bldg-7/proxmox-mcp@1.2.1 --package pino-pretty proxmox-mcp 2>/dev/null"
-      ],
-      "env": {
-        "PROXMOX_HOST": "192.168.4.11",
-        "PROXMOX_PORT": "8006",
-        "PROXMOX_USER": "MCP@pve",
-        "PROXMOX_TOKEN_NAME": "MCP",
-        "PROXMOX_TOKEN_VALUE": "2052990e-749f-43f6-be7f-c9ad206281cc",
-        "PROXMOX_SSL_MODE": "insecure",
-        "PROXMOX_ALLOW_ELEVATED": "true",
-        "PROXMOX_SSH_ENABLED": "true",
-        "PROXMOX_SSH_HOST": "192.168.4.11",
-        "PROXMOX_SSH_PORT": "22",
-        "PROXMOX_SSH_USER": "root",
-        "PROXMOX_SSH_KEY_PATH": "/app/ssh_keys/nuc-cluster",
-        "PROXMOX_SSH_NODE": "nuck7-1",
-        "PROXMOX_ALLOW_UNSAFE_COMMANDS": "true"
-      }
-    },
-    "proxmox-nuck7-2": {
-      "command": "sh",
-      "args": [
-        "-c",
-        "LOG_LEVEL=silent npx -y --package @bldg-7/proxmox-mcp@1.2.1 --package pino-pretty proxmox-mcp 2>/dev/null"
-      ],
-      "env": {
-        "PROXMOX_HOST": "192.168.4.12",
-        "PROXMOX_PORT": "8006",
-        "PROXMOX_USER": "MCP@pve",
-        "PROXMOX_TOKEN_NAME": "MCP",
-        "PROXMOX_TOKEN_VALUE": "2052990e-749f-43f6-be7f-c9ad206281cc",
-        "PROXMOX_SSL_MODE": "insecure",
-        "PROXMOX_ALLOW_ELEVATED": "true",
-        "PROXMOX_SSH_ENABLED": "true",
-        "PROXMOX_SSH_HOST": "192.168.4.12",
-        "PROXMOX_SSH_PORT": "22",
-        "PROXMOX_SSH_USER": "root",
-        "PROXMOX_SSH_KEY_PATH": "/app/ssh_keys/nuc-cluster",
-        "PROXMOX_SSH_NODE": "nuck7-2",
-        "PROXMOX_ALLOW_UNSAFE_COMMANDS": "true"
-      }
-    },
-    "proxmox-nuck7-3": {
-      "command": "sh",
-      "args": [
-        "-c",
-        "LOG_LEVEL=silent npx -y --package @bldg-7/proxmox-mcp@1.2.1 --package pino-pretty proxmox-mcp 2>/dev/null"
-      ],
-      "env": {
-        "PROXMOX_HOST": "192.168.4.13",
-        "PROXMOX_PORT": "8006",
-        "PROXMOX_USER": "MCP@pve",
-        "PROXMOX_TOKEN_NAME": "MCP",
-        "PROXMOX_TOKEN_VALUE": "2052990e-749f-43f6-be7f-c9ad206281cc",
-        "PROXMOX_SSL_MODE": "insecure",
-        "PROXMOX_ALLOW_ELEVATED": "true",
-        "PROXMOX_SSH_ENABLED": "true",
-        "PROXMOX_SSH_HOST": "192.168.4.13",
-        "PROXMOX_SSH_PORT": "22",
-        "PROXMOX_SSH_USER": "root",
-        "PROXMOX_SSH_KEY_PATH": "/app/ssh_keys/nuc-cluster",
-        "PROXMOX_SSH_NODE": "nuck7-3",
-        "PROXMOX_ALLOW_UNSAFE_COMMANDS": "true"
-      }
-    },
-    "homeassistant": {
-      "command": "npx",
-      "args": [
-        "-y",
-        "mcp-remote",
-        "https://home.bryanmail.net/mcp_server/sse",
-        "--header",
-        "Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiIxNzhmYzI0NjA2Y2I0ZTg4ODI1N2VmMzdiZTNhY2E5YSIsImlhdCI6MTc3NDY4MzAxNiwiZXhwIjoyMDkwMDQzMDE2fQ.32kY2LVHzKZHWLc96T6z2P-8beNTnp2DHRUf2UEie2w"
-      ]
-    },
-    "teams": {
-      "command": "npx",
-      "args": [
-        "-y",
-        "@floriscornel/teams-mcp@latest"
-      ],
-      "env": {
-        "TEAMS_MCP_READ_ONLY": "true",
-        "HOME": "/app/teams-mcp-auth"
-      }
-    },
-    "ms365": {
-      "command": "npx",
-      "args": [
-        "-y",
-        "@softeria/ms-365-mcp-server@0.129.0",
-        "--preset",
-        "personal",
-        "--read-only",
-        "--discovery"
-      ],
-      "env": {
-        "HOME": "/app/teams-mcp-auth",
-        "MS365_MCP_CLIENT_ID": "14d82eec-204b-4c2f-b7e8-296a70dab67e",
-        "MS365_MCP_TOKEN_CACHE_PATH": "/app/teams-mcp-auth/.teams-mcp-token-cache.json",
-        "SILENT": "true"
-      }
-    },
-    "unifi-network": {
-      "command": "uvx",
-      "args": [
-        "unifi-network-mcp@latest"
-      ],
-      "env": {
-        "UNIFI_HOST": "192.168.4.1",
-        "UNIFI_USERNAME": "unifi-mcp",
-        "UNIFI_PASSWORD": "3dHOEOMygTYeX3",
-        "UNIFI_PORT": "443",
-        "UNIFI_VERIFY_SSL": "false",
-        "UV_CACHE_DIR": "/app/uv-cache"
-      }
-    },
-    "authentik": {
-      "command": "uvx",
-      "args": [
-        "authentik-diag-mcp",
-        "--base-url",
-        "https://auth.bryanmail.net",
-        "--token",
-        "LAm3lBTumOmsU8AiFQM2FmCZyoj8bTSR0FQnAcBy1QnTMEyU4oWozwdxTUap"
-      ]
-    },
-    "gitea": {
-      "command": "/mcpo_data/gitea-mcp",
-      "args": [
-        "-t",
-        "stdio"
-      ],
-      "env": {
-        "GITEA_HOST": "https://git.bryanmail.net",
-        "GITEA_ACCESS_TOKEN": "5aa3a554c001b1dbe5215cb8cb388112801930e9"
-      }
-    },
-    "gitea-admin": {
-      "command": "/mcpo_data/gitea-mcp",
-      "args": [
-        "-t",
-        "stdio"
-      ],
-      "env": {
-        "GITEA_HOST": "https://git.bryanmail.net",
-        "GITEA_ACCESS_TOKEN": "289225b0b8f1827242191874b2408db76af06321"
-      }
-    },
-    "powerautomate": {
-      "command": "npx",
-      "args": [
-        "-y",
-        "powerautomate-mcp@latest",
-        "--stdio"
-      ],
-      "env": {
-        "HOME": "/app/powerautomate-auth",
-        "PA_MCP_CLIENT_ID": "84b431ed-ef5d-48a0-b0a1-878cfdb71453",
-        "PA_MCP_TENANT_ID": "0f6cf991-c449-480a-a71b-83003ce6edc1",
-        "PA_CONFIG_PATH": "/app/powerautomate-auth/config.json"
-      }
-    }
-  }
-}
@@ -1,217 +0,0 @@
-#!/usr/bin/env node
-// Minimal MCP server (stdio, JSON-RPC 2.0) for Woodpecker CI.
-// Zero dependencies — plain Node 18+ (built-in fetch). Config via env vars.
-//
-// Configuration (env):
-//   WOODPECKER_URL    e.g. https://your-woodpecker.example.com
-//   WOODPECKER_TOKEN  Woodpecker Personal Access Token (JWT)
-//
-// Tools:
-//   woodpecker_list_repos
-//   woodpecker_list_pipelines  { repo_id, limit? }
-//   woodpecker_get_pipeline    { repo_id, number }   -> status + workflow/step tree
-//   woodpecker_pipeline_logs   { repo_id, number, step_id, tail? } -> decoded step logs
-
-import { Buffer } from "node:buffer";
-
-const BASE = (process.env.WOODPECKER_URL || "").replace(/\/+$/, "");
-const TOKEN = process.env.WOODPECKER_TOKEN || "";
-
-function log(...a) {
-  // Diagnostics go to stderr so they never corrupt the stdout JSON-RPC stream.
-  process.stderr.write("[woodpecker-mcp] " + a.join(" ") + "\n");
-}
-
-async function api(path) {
-  if (!BASE || !TOKEN) throw new Error("Missing WOODPECKER_URL or WOODPECKER_TOKEN in environment");
-  const res = await fetch(`${BASE}/api${path}`, {
-    headers: { Authorization: `Bearer ${TOKEN}`, Accept: "application/json" },
-  });
-  const body = await res.text();
-  if (!res.ok) throw new Error(`HTTP ${res.status} ${path}: ${body.slice(0, 300)}`);
-  return body ? JSON.parse(body) : null;
-}
-
-// --- tool response formatting ---
-
-function summarizePipeline(p) {
-  const lines = [];
-  lines.push(`pipeline #${p.number} (id=${p.id})  status=${p.status}  event=${p.event}  branch=${p.branch}`);
-  lines.push(`commit=${(p.commit || "").slice(0, 12)}  author=${p.author}`);
-  if (p.message) lines.push(`message: ${p.message.split("\n")[0]}`);
-  for (const wf of p.workflows || []) {
-    lines.push(`  WORKFLOW "${wf.name}" (pid=${wf.pid}) state=${wf.state}${wf.error ? ` error=${wf.error}` : ""}`);
-    for (const c of wf.children || []) {
-      lines.push(
-        `    step "${c.name}" (id=${c.id}, pid=${c.pid}) ${c.state} exit=${c.exit_code ?? "-"} type=${c.type}`
-      );
-    }
-  }
-  return lines.join("\n");
-}
-
-function decodeLogs(entries) {
-  if (!Array.isArray(entries)) return String(entries);
-  return entries
-    .map((e) => {
-      const d = e?.data;
-      if (d == null) return "";
-      try {
-        return Buffer.from(d, "base64").toString("utf-8");
-      } catch {
-        return String(d);
-      }
-    })
-    .join("");
-}
-
-const TOOLS = [
-  {
-    name: "woodpecker_list_repos",
-    description: "List repositories the token can access (id, full name, default branch).",
-    inputSchema: { type: "object", properties: {}, additionalProperties: false },
-    handler: async () => {
-      const repos = await api(`/user/repos`);
-      return (repos || [])
-        .map((r) => `id=${r.id}  ${r.full_name}  default_branch=${r.default_branch}`)
-        .join("\n") || "(no repositories)";
-    },
-  },
-  {
-    name: "woodpecker_list_pipelines",
-    description: "Recent pipelines for a repository. Params: repo_id (number), limit (number, default 20).",
-    inputSchema: {
-      type: "object",
-      properties: {
-        repo_id: { type: "number", description: "Woodpecker repository ID" },
-        limit: { type: "number", description: "How many pipelines to return (default 20)" },
-      },
-      required: ["repo_id"],
-      additionalProperties: false,
-    },
-    handler: async ({ repo_id, limit }) => {
-      const list = await api(`/repos/${repo_id}/pipelines?perPage=${limit || 20}`);
-      return (list || [])
-        .map(
-          (p) =>
-            `#${p.number}  ${p.status.padEnd(8)}  ${p.event.padEnd(12)}  ${p.branch}  ${(p.commit || "").slice(0, 8)}  ${(p.message || "").split("\n")[0]}`
-        )
-        .join("\n") || "(no pipelines)";
-    },
-  },
-  {
-    name: "woodpecker_get_pipeline",
-    description: "Pipeline details: status plus the workflow/step tree (with step ids for fetching logs). Params: repo_id, number.",
-    inputSchema: {
-      type: "object",
-      properties: {
-        repo_id: { type: "number" },
-        number: { type: "number", description: "Pipeline number (as shown in the UI)" },
-      },
-      required: ["repo_id", "number"],
-      additionalProperties: false,
-    },
-    handler: async ({ repo_id, number }) => {
-      const p = await api(`/repos/${repo_id}/pipelines/${number}`);
-      return summarizePipeline(p);
-    },
-  },
-  {
-    name: "woodpecker_pipeline_logs",
-    description: "Decoded logs for a single step. Params: repo_id, number (pipeline), step_id (from woodpecker_get_pipeline). Optional tail (last N lines).",
-    inputSchema: {
-      type: "object",
-      properties: {
-        repo_id: { type: "number" },
-        number: { type: "number" },
-        step_id: { type: "number", description: "Step id from woodpecker_get_pipeline" },
-        tail: { type: "number", description: "Return only the last N lines (optional)" },
-      },
-      required: ["repo_id", "number", "step_id"],
-      additionalProperties: false,
-    },
-    handler: async ({ repo_id, number, step_id, tail }) => {
-      const entries = await api(`/repos/${repo_id}/logs/${number}/${step_id}`);
-      let txt = decodeLogs(entries);
-      if (tail && tail > 0) {
-        txt = txt.split("\n").slice(-tail).join("\n");
-      }
-      return txt || "(no logs)";
-    },
-  },
-];
-
-// --- JSON-RPC over stdio loop ---
-
-function send(msg) {
-  process.stdout.write(JSON.stringify(msg) + "\n");
-}
-
-function reply(id, result) {
-  send({ jsonrpc: "2.0", id, result });
-}
-
-function replyError(id, code, message) {
-  send({ jsonrpc: "2.0", id, error: { code, message } });
-}
-
-async function handle(req) {
-  const { id, method, params } = req;
-  if (method === "initialize") {
-    reply(id, {
-      protocolVersion: params?.protocolVersion || "2024-11-05",
-      capabilities: { tools: {} },
-      serverInfo: { name: "woodpecker-mcp", version: "1.0.0" },
-    });
-    return;
-  }
-  if (method === "notifications/initialized" || method === "notifications/cancelled") {
-    return; // notifications carry no response
-  }
-  if (method === "ping") {
-    reply(id, {});
-    return;
-  }
-  if (method === "tools/list") {
-    reply(id, {
-      tools: TOOLS.map((t) => ({ name: t.name, description: t.description, inputSchema: t.inputSchema })),
-    });
-    return;
-  }
-  if (method === "tools/call") {
-    const tool = TOOLS.find((t) => t.name === params?.name);
-    if (!tool) {
-      replyError(id, -32602, `Unknown tool: ${params?.name}`);
-      return;
-    }
-    try {
-      const text = await tool.handler(params.arguments || {});
-      reply(id, { content: [{ type: "text", text }] });
-    } catch (e) {
-      reply(id, { content: [{ type: "text", text: `Error: ${e.message}` }], isError: true });
-    }
-    return;
-  }
-  if (id !== undefined) replyError(id, -32601, `Unsupported method: ${method}`);
-}
-
-let buf = "";
-process.stdin.setEncoding("utf-8");
-process.stdin.on("data", (chunk) => {
-  buf += chunk;
-  let nl;
-  while ((nl = buf.indexOf("\n")) >= 0) {
-    const line = buf.slice(0, nl).trim();
-    buf = buf.slice(nl + 1);
-    if (!line) continue;
-    let req;
-    try {
-      req = JSON.parse(line);
-    } catch {
-      continue;
-    }
-    handle(req).catch((e) => log("handler error:", e.message));
-  }
-});
-process.stdin.on("end", () => process.exit(0));
-log("ready", BASE ? `(${BASE})` : "(WOODPECKER_URL is not set!)");
@@ -6,9 +6,9 @@ This directory documents the secrets required for each Docker Swarm stack.

 ```
 Secret values live in Woodpecker (encrypted)
-    ↓ pipeline reads them at deploy time
+    → pipeline reads them at deploy time
 Docker Swarm secret store (encrypted Raft DB, replicated across all nodes)
-    ↓ mounted into containers at runtime
+    → mounted into containers at runtime
 /run/secrets/<secret_name>
 ```

@@ -38,7 +38,10 @@ Use the stack's `.secrets.example` file as your checklist.

 ### Step 2 — Add the stack's case to `.woodpecker.yml`
 In the `provision-secrets` step, add a case for the stack that calls
-`create_or_update_secret` for each secret.
+`create_or_update_secret` for each secret. (Or, preferred for new
+migrations: add an entry to `secrets/secrets-map.yaml` + a
+`<stack>.env.template` and call `deploy/provision-stack.py <stack>`
+instead — see the `ai` and `traefik` entries for the current pattern.)

 ### Step 3 — Test by pushing a trivial change to the stack's yaml file
 Watch the pipeline run: provision-secrets → validate → deploy → verify → notify.
@@ -71,10 +74,20 @@ Examples:
 - **PostgreSQL is highest risk.** Its master password is used by nearly every
  other stack. Migrate it last.

+- **Never commit a rendered env file, even by accident.** `traefik/traefik.env`
+  was committed to git for a period (discovered/fixed 2026-09-12) with a
+  literal "***REDACTED***" placeholder as KEEPALIVED_PASSWORD, which was
+  silently restored every time the file was deleted or the checkout resynced
+  from git — causing a real VRRP auth outage. `.gitignore` blanket-excludes
+  `*.env`, but that rule does NOT retroactively untrack a file already
+  committed before the rule existed. If you ever see a stack's `.env` file
+  show up in `git status` as tracked, stop and untrack it (`git rm --cached`)
+  before doing anything else.
+
 ## Migration Status

 | Stack | Secrets in Woodpecker | Pipeline Step Added | .env Removed |
-|-------|----------------------|---------------------|--------------|
+|-------|------------------------|----------------------|---------------|
 | 3dprint | ⏳ | ⏳ | ⏳ |
 | ai | ⏳ | ⏳ | ⏳ |
 | auth | ⏳ | ⏳ | ⏳ |
@@ -91,7 +104,7 @@ Examples:
 | postgresql | ⏳ | ⏳ | ⏳ |
 | productivity | ⏳ | ⏳ | ⏳ |
 | security | ⏳ | ⏳ | ⏳ |
-| traefik | ⏳ | ⏳ | ⏳ |
+| traefik | ✅ (manifest-driven, 2026-09-12) | ✅ (manifest-driven, 2026-09-12) | ⏳ |
 | unifi | ✅ N/A (no secrets) | ✅ N/A | ⏳ |
 | vaultwarden | ⏳ | ⏳ | ⏳ |
 | woodpecker | ⏳ Manual only | ⏳ N/A | ⏳ |
@@ -0,0 +1,41 @@
+# ─────────────────────────────────────────────────────────────────────────
+# secrets-map.yaml — DATA-ONLY manifest for deploy/provision-stack.py
+#
+# RULES:
+#   - This file contains NO code, NO shell, NO secret values — only names.
+#   - Each stack entry declares:
+#       env_template:   repo path of the FULL env-file template (tracked).
+#                        The template is authoritative: the COMPLETE env file is
+#                        rendered from it on every provisioning run. Nothing is
+#                        line-edited in place, so keys can never silently go
+#                        missing.
+#       env_dest:       host path (relative to /volume1/docker/compose-files/)
+#                        the rendered env file is shipped to. Rendered file
+#                        exists ONLY on the host — never committed to git.
+#       docker_secrets: map of docker-swarm-secret-name -> CI ENV VAR NAME
+#                        (Pattern C). The env var must be declared via
+#                        from_secret: in .woodpecker/deploy.yml's
+#                        provision-secrets step (Woodpecker v3 requires explicit
+#                        per-secret declaration; there is no expose-all).
+#
+# ADDING A NEW SECRET (3 small steps, no shell edits):
+#   1. Add the secret value in Woodpecker UI (repo Settings -> Secrets).
+#   2. Declare it in .woodpecker/deploy.yml provision-secrets environment:
+#      block (from_secret) — mechanical two-line addition.
+#   3. Reference it here (docker_secrets:) and/or in the stack's
+#      .env.template as a dollar-brace placeholder.
+#
+# Stacks not listed here fall through to deploy.yml's legacy case-entries
+# untouched. Migration is deliberately one stack per PR.
+# ─────────────────────────────────────────────────────────────────────────
+stacks:
+  ai:
+    env_template: ai/ai.env.template
+    env_dest: ai/ai.env
+    docker_secrets:
+      flowagent_azure_client_id: FLOWAGENT_AZURE_CLIENT_ID
+      flowagent_azure_tenant_id: FLOWAGENT_AZURE_TENANT_ID
+      flowagent_azure_client_secret: FLOWAGENT_AZURE_CLIENT_SECRET
+  traefik:
+    env_template: traefik/traefik.env.template
+    env_dest: traefik/traefik.env
@@ -1,5 +1,8 @@
 # traefik Stack — Secrets Reference
-# Source: traefik.env
+# Source: traefik.env.template (rendered by deploy/provision-stack.py per
+#         secrets/secrets-map.yaml — see secrets-map.yaml header for how
+#         this works). traefik/traefik.env is rendered fresh on every
+#         provisioning run and is NEVER committed to git.
 #
 # Add SECRET values to Woodpecker at:
 #   https://woodpecker.bryanmail.net
@@ -8,27 +11,44 @@
 # ⚠️  HIGH RISK: Traefik is the entry point for all homelab services.
 #     If this stack fails, nothing is reachable from outside.
 #     Migrate carefully. The Keepalived VIP (192.168.4.30) depends on this stack.
+#
+# ⚠️  2026-09-12 INCIDENT: traefik.env was previously committed to git with a
+#     literal "***REDACTED***" placeholder as KEEPALIVED_PASSWORD. Every
+#     git-guard resync/checkout restored that broken value onto disk,
+#     diverging from keepalived-backup's stale-but-correct in-memory value
+#     and causing continuous VRRP auth failures + VIP instability. This was
+#     the trigger for migrating this stack to Pattern C. If you ever see
+#     KEEPALIVED_PASSWORD as a literal placeholder-looking string on disk
+#     again, do NOT hand-edit it — check `git log traefik/` for a stray
+#     commit and fix the template in Gitea instead.

-# ── SECRETS (add to Woodpecker) ───────────────────────────────────────────────
+# ── SECRETS (add to Woodpecker) ─────────────────────────────────────────

 # Woodpecker secret name:  traefik_keepalived_password
 # Used for:                Keepalived VRRP authentication password
 #                          Must match across all 3 nodes (docker-1, docker-2, docker-3)
-# Env var in .env:        KEEPALIVED_PASSWORD
+#                          NOTE: classic VRRP simple-auth is silently
+#                          truncated to 8 chars by keepalived itself — keep
+#                          the value <= 8 characters, or be aware only the
+#                          first 8 are actually significant on the wire.
+# Env var in .env.template: KEEPALIVED_PASSWORD (via TRAEFIK_KEEPALIVED_PASSWORD)
 traefik_keepalived_password=

-# ── NON-SECRETS (safe in compose file or .env) ────────────────────────────────
+# ── NON-SECRETS (safe in compose file or .env.template) ─────────────────

-# KEEPALIVED_UNICAST_PEERS     Python2BASH list of peer IPs
-# KEEPALIVED_VIRTUAL_IPS       Python2BASH list of VIP addresses (192.168.4.30)
+# KEEPALIVED_VIRTUAL_IPS       Python2BASH list of VIP addresses (192.168.4.30) — literal in template
 # ACME_EMAIL                   Let's Encrypt certificate email
 # TRUSTED_IPS                  Trusted proxy CIDR ranges
 # TRAEFIK_HOST                 Traefik dashboard hostname
 # SPEEDTEST_HOST                Speedtest Traefik hostname
 # WHOAMI_HOST                  Whoami Traefik hostname

-# ── Woodpecker provision-secrets case entry ───────────────────────────────────
+# ── Provisioning (manifest-driven, deploy/provision-stack.py) ───────────
 #
-#   traefik)
-#     create_or_update_secret "traefik_keepalived_password" "$TRAEFIK_KEEPALIVED_PASSWORD"
-#     ;;
+# This stack is migrated — provisioning happens automatically via:
+#   secrets/secrets-map.yaml (traefik: entry)
+#   traefik/traefik.env.template (authoritative key list)
+#   deploy/provision-stack.py (renders + ships traefik/traefik.env)
+#
+# The .woodpecker/deploy.yml provision-secrets step calls this with a
+# single line: `python3 deploy/provision-stack.py traefik`
@@ -0,0 +1,120 @@
+# Traefik Log Rotation
+
+## Why this exists
+
+`traefik/traefik.yaml` runs Traefik with:
+- `--accesslog.filePath=/traefik/logs/access.log`
+- `--log.filePath=/traefik/logs/traefik.log`
+- `--log.level=DEBUG`
+
+Neither file has any built-in rotation -- Traefik has no native rotate-on-size
+nor a SIGUSR1/reopen handler. Docker's `json-file` log-driver rotation
+(`max-size`/`max-file`) only applies to stdout, not to files Traefik writes
+directly via `--accesslog.filePath`/`--log.filePath`. Result: `access.log` grew
+to **~15.5GB unrotated** before this was caught, on a CephFS volume
+(`/volume1/docker-root`) already at 82-84% used. Both the disk pressure and
+the ongoing write latency of appending to a 15GB file on a network filesystem
+on every request through Traefik were flagged as a real risk factor during
+troubleshooting (Uptime Kuma WebSocket flapping investigation, Sep 2026).
+
+**Note:** during that investigation, the actual root cause of the WebSocket
+flapping turned out to be Uptime Kuma monitor misconfigurations (a Postgres
+monitor throwing a null-reference error, and several monitors failing TLS
+validation against self-signed/internal-IP certs) -- not this log file. This
+rotation fix is still worth doing as general disk/IO hygiene, just not
+causally tied to that incident.
+
+## Architecture constraint this design accounts for
+
+`traefik_reverse-proxy` runs Swarm *`mode: global`* -- one instance on **each**
+of docker-1, docker-2, docker-3. All three write to the **same physical file**
+via the shared CephFS bind mount `/volume1/docker/traefik -> /traefik`
+(identical mount, visible identically from any node). That rules out:
+
+- **Signal-based rotation** (classic `create` + `postrotate` sending SIGUSR1):
+  Traefik doesn't implement a reopen signal, and even if it did, you'd need to
+  signal 3 separate per-node containers in lockstep.
+- **Running logrotate on just one node**: works until that node is down, then
+  rotation silently stops with no alert.
+
+So this setup uses:
+1. **`copytruncate`** (see `traefik-logs.conf`) -- all three Traefik processes
+   keep writing to the same inode, no signaling needed. Tradeoff: a few log
+   lines written in the exact copy/truncate instant can be lost -- fine for
+   diagnostic logs.
+2. **Cron on all three nodes**, coordinated via a shared `flock` + shared
+   logrotate state file, both also on the CephFS mount (see
+   `traefik-logrotate.sh`). Whichever node's cron fires first grabs the lock,
+   rotates if due, and updates the shared state so the other two nodes' cron
+   runs see it's already done. No single node is a rotation SPOF.
+3. **Size-triggered** (`size 250M`) rather than calendar-triggered (`daily`) --
+   this file can grow fast under bursts (see incident background above); a
+   purely daily interval would still let it balloon between runs. Cron checks
+   every 15 minutes, so it cannot grow much past the 250M threshold in practice.
+
+## Install (one-time, per node)
+
+Must run on **all three** docker LXCs -- this is host-level cron/logrotate
+config, not something `stack-deploy.sh` can reach (it only touches Swarm
+services, not host cron jobs).
+
+```bash
+# On each of docker-1 (192.168.4.31), docker-2 (192.168.4.32), docker-3 (192.168.4.33):
+ssh root@<192.168.4.31|.32|.33>
+cd /volume1/docker/compose-files
+bash deploy/git-guard.sh         # confirm sync first, as always
+sudo bash traefik/logrotate/install.sh
+```
+
+Verify on each node:
+```bash
+cat /etc/cron.d/traefik-logrotate
+logrotate -d /etc/logrotate.d/traefik-logs   # dry-run, confirms syntax
+```
+
+## Bootstrapping -- shrinking the *existing* oversized log file
+
+**Not done automatically by `install.sh`.** The new size-triggered config only
+prevents *future* unbounded growth -- it won't touch the current 15.5GB file
+until the next time it crosses 250M (i.e. never, since it's already well past
+that and logrotate only acts on crossing the threshold going forward from its
+recorded size at last check).
+
+`copytruncate` always copies the full current file before truncating it --
+that's inherent to how it works, not a bug. Forcing a rotation of the current
+15.5GB file would momentarily need roughly another 15-GB-sized chunk of free
+space -- and `/volume1/docker-root` only had **~14G free** at last check.
+Doing this blindly could tip the volume to 100% mid-operation.
+
+**Recommended manual step (operator-run, on any one node -- it's the same
+CephFS file from all three)**: these are diagnostic access/debug logs, not
+something worth preserving in full, so just truncate directly rather than
+compress-then-truncate:
+
+```bash
+# Optional: keep a small tail sample for reference before truncating
+tail -c 50000000 /volume1/docker/traefik/logs/access.log > \
+  /volume1/docker/traefik-access-log-archive-$(date +%Y%m%d).log
+
+# Then truncate in place (safe -- Traefik's existing file handles on all 3
+# nodes stay valid, same as copytruncate's own mechanism):
+: > /volume1/docker/traefik/logs/access.log
+: > /volume1/docker/traefik/logs/traefik.log
+
+# Confirm:
+df -h /volume1/docker-root
+ls -la /volume1/docker/traefik/logs/
+```
+
+After this one-time bootstrap, the cron+logrotate setup keeps it bounded
+(rotates at 250M, keeps 48 compressed generations, prunes anything over 14
+days old) going forward without needing any further manual intervention.
+
+## Related, NOT included in this change (follow-up to consider separately)
+
+`traefik.yaml` currently runs `--log.level=DEBUG` -- verbose debug logging in
+production, which is a meaningful contributor to how fast these files grow.
+Lowering to `INFO` would reduce volume significantly but requires a real
+modification + redeploy of the high-blast-radius `traefik` stack (all
+HTTP/HTTPS routing depends on it), so it's intentionally left out of this PR
+and should be its own reviewed change if wanted.
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# One-time installer for Traefik log rotation.
+#
+# MUST be run manually, once, on EACH of docker-1, docker-2, docker-3.
+# This is intentionally NOT part of stack-deploy.sh / the Woodpecker
+# pipeline: it installs a host-level cron.d entry and /etc/logrotate.d
+# config, and `docker stack deploy` has no mechanism to reach outside the
+# Swarm/container boundary onto host cron. See README.md for why this needs
+# to run on all three nodes.
+#
+# Usage (from a checkout of this repo, on each node):
+#   sudo bash traefik/logrotate/install.sh
+set -euo pipefail
+
+if [[ $EUID -ne 0 ]]; then
+  echo "Run as root (sudo)." >&2
+  exit 1
+fi
+
+REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+echo "Installing logrotate config..."
+install -m 0644 "$REPO_DIR/traefik-logs.conf" /etc/logrotate.d/traefik-logs
+
+echo "Installing rotation wrapper..."
+install -m 0755 "$REPO_DIR/traefik-logrotate.sh" /usr/local/sbin/traefik-logrotate.sh
+
+echo "Installing cron.d schedule (every 15 minutes)..."
+cat > /etc/cron.d/traefik-logrotate << 'EOF'
+# Managed by homelab/compose-files traefik/logrotate/install.sh -- do not
+# hand-edit; update traefik/logrotate/*.{conf,sh} in Gitea and re-run
+# install.sh instead.
+*/15 * * * * root /usr/local/sbin/traefik-logrotate.sh
+EOF
+chmod 0644 /etc/cron.d/traefik-logrotate
+
+echo "Verifying logrotate config syntax..."
+logrotate -d /etc/logrotate.d/traefik-logs
+
+echo "Done. First rotation check runs on the next cron tick (up to 15 min)."
+echo "This only rotates going forward once the file crosses the size threshold."
+echo "To shrink the EXISTING already-large log file, see README.md -- that is"
+echo "a separate, deliberate manual step (not done by this script) because it"
+echo "needs disk headroom awareness first."
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Wrapper invoked by cron on docker-1/docker-2/docker-3 to rotate the shared
+# Traefik access/error logs (see traefik-logs.conf for the "full why").
+#
+# Because /volume1/docker/traefik/logs is the SAME physical CephFS path on
+# all three nodes, and cron on all three nodes runs this independently, we
+# use a shared flock (also on the CephFS mount, so it's visible cluster-wide)
+# to guarantee only one node actually executes logrotate at a time, and a
+# SHARED state file so whichever node runs it knows the true last-rotated
+# time regardless of which node rotated it last. If a node is down, the
+# other two still cover the schedule -- none of this relies on a specific
+# node being up.
+set -euo pipefail
+
+LOCK_DIR="/volume1/docker/traefik/logrotate-state"
+LOCK_FILE="$LOCK_DIR/rotate.lock"
+STATE_FILE="$LOCK_DIR/status"
+CONF_FILE="/etc/logrotate.d/traefik-logs"
+
+mkdir -p "$LOCK_DIR"
+touch "$STATE_FILE"
+
+exec 200>"$LOCK_FILE"
+if ! flock -n 200; then
+  # Another node already holds the lock this cycle -- normal, not an error.
+  exit 0
+fi
+
+/usr/sbin/logrotate -s "$STATE_FILE" "$CONF_FILE"
@@ -0,0 +1,43 @@
+# Traefik access/error log rotation
+#
+# CONTEXT: traefik_reverse-proxy runs in Swarm `mode: global` (traefik.yaml),
+# meaning one instance runs on EACH of docker-1/docker-2/docker-3. All three
+# write to the SAME physical file via the shared CephFS bind mount
+# /volume1/docker/traefik/logs -> /traefik (identical path from any node --
+# see infra context: /volume1/docker is a shared CephFS mount).
+#
+# `copytruncate` is REQUIRED here (not the default create+signal approach):
+# Traefik has no SIGUSR1/SIGHUP "reopen log file" handling, and even if it
+# did, coordinating a reopen signal across 3 independent per-node containers
+# writing to one shared inode is unnecessary complexity. copytruncate keeps
+# every writer's existing file descriptor valid (truncates in place) so all
+# three Traefik processes keep appending to the same inode with zero
+# signaling. Tradeoff: a handful of log lines written in the exact
+# copy/truncate instant can be lost -- acceptable for diagnostic access/error
+# logs, not used for anything transactional.
+#
+# Size-triggered (not calendar-triggered) on purpose: this file can grow fast
+# under bursts (see incident that prompted this -- 15.5GB accumulated with
+# --log.level=DEBUG set). `size` is checked every time the wrapper script runs
+# (cron, every 15 minutes -- see install.sh), so it cannot balloon unbounded
+# between checks the way a plain `daily` interval would.
+#
+# Installed via install.sh on ALL THREE docker LXCs (docker-1, docker-2,
+# docker-3) -- see README.md. This is a HOST-level cron/logrotate config,
+# outside the Woodpecker/stack-deploy.sh pipeline (docker stack deploy has no
+# mechanism to touch host cron), so it must be applied manually once per node,
+# not via a stack redeploy.
+
+/volume1/docker/traefik/logs/*.log {
+    size 250M
+    rotate 48
+    maxage 14
+    compress
+    delaycompress
+    missingok
+    notifempty
+    copytruncate
+    dateext
+    dateformat -%Y%m%d-%H%M%S
+    su root root
+}
@@ -1,2 +0,0 @@
-KEEPALIVED_PASSWORD=***REDACTED***
-KEEPALIVED_VIRTUAL_IPS="#PYTHON2BASH:['192.168.4.30']"
@@ -0,0 +1,52 @@
+# ─────────────────────────────────────────────────────────────────────────
+# traefik.env.template — AUTHORITATIVE template for traefik/traefik.env
+# (rendered by deploy/provision-stack.py per secrets/secrets-map.yaml)
+#
+#   - This file IS the complete key list for traefik.env. The whole file is
+#     rendered on every provisioning run — no line surgery, so a key can
+#     never silently go missing.
+#   - Non-secret config lives here as LITERAL values (visible, reviewable).
+#   - Secret values are dollar-brace placeholders resolved from the CI env
+#     (Woodpecker from_secret vars) at provisioning time. provision-stack.py
+#     FAILS HARD if any placeholder is missing/empty.
+#   - The rendered traefik/traefik.env exists only on the host (gitignored).
+#   - Rendered by provision-stack.py, NOT Woodpecker's yaml preprocessor —
+#     single-dollar placeholders are safe here (deploy.yml's double-dollar
+#     rule does NOT apply to this file).
+#
+# Consumed by traefik/traefik.yaml (KEEPALIVED_PASSWORD, KEEPALIVED_VIRTUAL_IPS).
+# DOMAIN comes from deploy/global.env, not here.
+#
+# ⚠️ HIGH RISK: Traefik is the entry point for all homelab services, and the
+#    Keepalived VIP (192.168.4.30) depends on this stack. KEEPALIVED_PASSWORD
+#    MUST be identical across keepalived-master and keepalived-backup — both
+#    consume this same rendered value — or VRRP auth fails and the VIP
+#    becomes unstable (this is exactly what the 2026-09-12 incident was:
+#    a committed literal "***REDACTED***" placeholder in git, restored every
+#    time the file was deleted/re-synced, diverging from -backup's stale but
+#    correct in-memory value).
+#
+#    Classic VRRP simple-auth is silently truncated to 8 characters by
+#    keepalived itself. Set TRAEFIK_KEEPALIVED_PASSWORD in Woodpecker to a
+#    value 8 characters or fewer (or accept that only the first 8 chars are
+#    actually significant) so the effective negotiated value is unambiguous.
+#
+#   2026-09-12 RE-TRIGGER: traefik.env was found MISSING on disk after the
+#   Pattern C migration merged — the manifest-driven provisioning render only
+#   happens on a push that changes traefik/, and traefik is bootstrap-tier
+#   (stack-deploy.sh must be run manually; the pipeline never auto-deploys
+#   this stack). A manual `stack-deploy.sh traefik` had apparently been run
+#   while traefik.env was absent, causing Swarm's own interpolation to
+#   silently render KEEPALIVED_PASSWORD/KEEPALIVED_VIRTUAL_IPS as EMPTY
+#   STRINGS on keepalived-master (not missing — empty, which is worse,
+#   since it's a valid-looking VRRP auth string that mismatches everything).
+#   This comment-only change re-triggers provision-secrets to re-render and
+#   re-ship traefik.env; a manual `stack-deploy.sh traefik` must still be run
+#   afterward to actually apply it to the running services.
+# ─────────────────────────────────────────────────────────────────────────
+
+# ── Keepalived (non-secret config) ────────────────────────────────────────
+KEEPALIVED_VIRTUAL_IPS="#PYTHON2BASH:['192.168.4.30']"
+
+# ── Keepalived (secret) ────────────────────────────────────────────────────
+KEEPALIVED_PASSWORD=${TRAEFIK_KEEPALIVED_PASSWORD}
@@ -5,8 +5,8 @@ services:
      KEEPALIVED_INTERFACE: eth0
      KEEPALIVED_PRIORITY: "150"
      KEEPALIVED_ROUTER_ID: "51"
-      KEEPALIVED_PASSWORD: ${KEEPALIVED_PASSWORD}
-      KEEPALIVED_VIRTUAL_IPS: ${KEEPALIVED_VIRTUAL_IPS}
+      KEEPALIVED_PASSWORD: "${KEEPALIVED_PASSWORD}"
+      KEEPALIVED_VIRTUAL_IPS: "${KEEPALIVED_VIRTUAL_IPS}"
      KEEPALIVED_UNICAST_PEERS: "#PYTHON2BASH:['192.168.4.31', '192.168.4.32']"
    command: --copy-service
    cap_add: [NET_ADMIN, NET_BROADCAST, NET_RAW]
@@ -39,8 +39,8 @@ services:
      KEEPALIVED_INTERFACE: eth0
      KEEPALIVED_PRIORITY: "100"
      KEEPALIVED_ROUTER_ID: "51"
-      KEEPALIVED_PASSWORD: ${KEEPALIVED_PASSWORD}
-      KEEPALIVED_VIRTUAL_IPS: ${KEEPALIVED_VIRTUAL_IPS}
+      KEEPALIVED_PASSWORD: "${KEEPALIVED_PASSWORD}"
+      KEEPALIVED_VIRTUAL_IPS: "${KEEPALIVED_VIRTUAL_IPS}"
      KEEPALIVED_UNICAST_PEERS: "#PYTHON2BASH:['192.168.4.32', '192.168.4.33']"
    cap_add: [NET_ADMIN, NET_BROADCAST, NET_RAW]
    networks: [host]
@@ -147,7 +147,7 @@ services:
        order: stop-first

  traefik-certs-dumper:
-    image: ghcr.io/ldez/traefik-certs-dumper:v2.10.0
+    image: ghcr.io/ldez/traefik-certs-dumper:v2.11.4
    entrypoint: sh -c 'apk add jq ; while ! [ -e /letsencrypt/acme.json ] || ! [ `jq ".[] | .Certificates | length" /letsencrypt/acme.json` != 0 ]; do sleep 1 ; done && traefik-certs-dumper file --version v3 --watch --source /letsencrypt/acme.json --dest /letsencrypt/certs'
    volumes:
      - /volume1/docker/letsencrypt:/letsencrypt