Add flowagent secrets reference doc (placeholders only, no real values)

Documents the Pattern C secrets needed once mcpo/config.json + ai.yaml are
updated in a follow-up PR to actually wire FlowAgent into the mcpo service.

.gitignore

@@ -2,8 +2,17 @@
 *.swp
 *.swo
 .DS_Store
+
+# Secrets / stack env files - never commit these.
+# deploy/global.env and immich.env-example etc. are the intended committed
+# templates/examples; actual secret-bearing .env files must stay untracked.
 .env
 .env.local
+*.env
+!*.env-example
+!deploy/global.env
+*.env.bak
+*.env.backup
 secrets/
 keys/
 *.key
@@ -15,3 +24,7 @@ keys/
 # Local overrides
 docker-compose.override.yml
 local/
+
+# Editor/adhoc backups that sometimes get left in the tree
+*.bak
+*.bak-*

.woodpecker/deploy.yml

@@ -26,6 +26,28 @@ when:
 #   This was the root cause of a long-running "SWARM_MANAGER_IP secret is
 #   empty" failure: braced references were blanked at compile time before the
 #   shell ever saw them.
+#
+#   2026-08-26 HOTFIX: a full-file rewrite (AI secrets migration PR) dropped
+#   one $ from every $${VAR} occurrence throughout this file, re-introducing
+#   exactly the bug described above for EVERY secret-backed var, not just the
+#   new AI ones. The if-empty guards caught it immediately (SWARM_MANAGER_IP
+#   came back blank) and aborted before any ssh/scp/rsync ran, so no live
+#   secret or service was touched — but no CI provisioning/deploy could run
+#   until this was restored. Lesson: grep for the literal string '$${' and
+#   diff the count against the previous version before ever committing a
+#   full-file rewrite of this pipeline.
+#
+#   2026-08-26 FIX: FOLDER_STACKS/FOLDERS detection (grep -E '^[^/.][^/]*/')
+#   matches ANY non-dot top-level folder in the changed-files list, including
+#   deploy/ — the shared tooling folder, not a stack. A PR touching only
+#   deploy/envparse.py caused stack-deploy.sh to be invoked with "deploy" as
+#   a stack name, which correctly errored ("No main compose file... in
+#   .../deploy") since deploy/ has no deploy.yaml. No live service was
+#   affected (the error occurs before any redeploy), but it produced a
+#   confusing pipeline failure on an otherwise-correct change. deploy/ is
+#   already unconditionally rsynced at the top of the deploy step regardless
+#   of which stacks changed, so it's safe to exclude it from the stack list
+#   everywhere folders are detected below.
 # ─────────────────────────────────────────────────────────────────────────────
 
 steps:
@@ -43,8 +65,9 @@ steps:
         FLAT=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.ya?ml$' || true)
 
         # Folder: any file under a subfolder (e.g. immich/immich.yml).
-        # Exclude dotfolders (.woodpecker, .git, .gitea, etc.)
+        # Exclude dotfolders (.woodpecker, .git, .gitea, etc.) and deploy/
-        FOLDERS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true)
+        # (shared tooling, not a stack — see note above).
+        FOLDERS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -v '^deploy$' | sort -u || true)
 
         [ -z "$FLAT" ] && [ -z "$FOLDERS" ] && echo "No stacks changed" && exit 0
 
@@ -162,22 +185,22 @@ steps:
       - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa
       - chmod 600 ~/.ssh/id_rsa
       - |
-        if [ -z "${SWARM_MANAGER_IP}" ]; then
+        if [ -z "$${SWARM_MANAGER_IP}" ]; then
           echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets."
           exit 1
         fi
-      - ssh-keyscan -H ${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
+      - ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
       - |
         CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
         FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
-        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true)
+        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -v '^deploy$' | sort -u || true)
         ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
         [ -z "$ALL_STACKS" ] && echo "No stacks changed, skipping" && exit 0
-      - scp -o StrictHostKeyChecking=no deploy/create-secrets.sh root@${SWARM_MANAGER_IP}:/tmp/cs.sh
+      - scp -o StrictHostKeyChecking=no deploy/create-secrets.sh root@$${SWARM_MANAGER_IP}:/tmp/cs.sh
       - |
         CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
         FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
-        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true)
+        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -v '^deploy$' | sort -u || true)
         ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
 
         for STACK in $ALL_STACKS; do
@@ -186,92 +209,102 @@ steps:
             maintenance|media|unifi|guacamole|security|auth|traefik|meshcentral|ddm)
               echo "  No Docker secrets for $STACK — secrets in host .env";;
             immich)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'immich_db_password'     '${IMMICH_DB_PASSWORD}'
+                create_or_update_secret 'immich_db_password'     '$${IMMICH_DB_PASSWORD}'
-                create_or_update_secret 'immich_kiosk_basicauth' '${IMMICH_KIOSK_BASICAUTH}'";;
+                create_or_update_secret 'immich_kiosk_basicauth' '$${IMMICH_KIOSK_BASICAUTH}'";;
             woodpecker)
               echo "  Manual only — skipping";;
             3dprint)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret '3dprint_db_password' '${PRINT3D_DB_PASSWORD}'";;
+                create_or_update_secret '3dprint_db_password' '$${PRINT3D_DB_PASSWORD}'";;
             gamma)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'gamma_auth_token' '${GAMMA_AUTH_TOKEN}'";;
+                create_or_update_secret 'gamma_auth_token' '$${GAMMA_AUTH_TOKEN}'";;
             git)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'git_db_password'       '${GIT_DB_PASSWORD}'
+                create_or_update_secret 'git_db_password'       '$${GIT_DB_PASSWORD}'
-                create_or_update_secret 'git_runner_token'      '${GIT_RUNNER_TOKEN}'
+                create_or_update_secret 'git_runner_token'      '$${GIT_RUNNER_TOKEN}'
-                create_or_update_secret 'git_mcp_access_token'  '${GIT_MCP_ACCESS_TOKEN}'";;
+                create_or_update_secret 'git_mcp_access_token'  '$${GIT_MCP_ACCESS_TOKEN}'";;
             homeassistant)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'homeassistant_frigate_rtsp_password' '${HOMEASSISTANT_FRIGATE_RTSP_PASSWORD}'
+                create_or_update_secret 'homeassistant_frigate_rtsp_password' '$${HOMEASSISTANT_FRIGATE_RTSP_PASSWORD}'
-                create_or_update_secret 'homeassistant_immich_api_key'        '${HOMEASSISTANT_IMMICH_API_KEY}'";;
+                create_or_update_secret 'homeassistant_immich_api_key'        '$${HOMEASSISTANT_IMMICH_API_KEY}'";;
             mealie)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'mealie_db_password'         '${MEALIE_DB_PASSWORD}'
+                create_or_update_secret 'mealie_db_password'         '$${MEALIE_DB_PASSWORD}'
-                create_or_update_secret 'mealie_ldap_query_password' '${MEALIE_LDAP_QUERY_PASSWORD}'";;
+                create_or_update_secret 'mealie_ldap_query_password' '$${MEALIE_LDAP_QUERY_PASSWORD}'";;
             n8n)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'n8n_db_password'     '${N8N_DB_PASSWORD}'
+                create_or_update_secret 'n8n_db_password'     '$${N8N_DB_PASSWORD}'
-                create_or_update_secret 'n8n_encryption_key'  '${N8N_ENCRYPTION_KEY}'";;
+                create_or_update_secret 'n8n_encryption_key'  '$${N8N_ENCRYPTION_KEY}'";;
             postgresql)
               # NOTE: bootstrap-tier stack (manual deploy only via stack-deploy.sh).
               # Secrets are still auto-provisioned here so they exist on the host
               # before the manual `stack-deploy.sh postgresql` run picks them up.
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'postgresql_password'             '${POSTGRESQL_PASSWORD}'
+                create_or_update_secret 'postgresql_password'             '$${POSTGRESQL_PASSWORD}'
-                create_or_update_secret 'postgresql_pgadmin_password'     '${POSTGRESQL_PGADMIN_PASSWORD}'
+                create_or_update_secret 'postgresql_pgadmin_password'     '$${POSTGRESQL_PGADMIN_PASSWORD}'
-                create_or_update_secret 'postgresql_replication_password' '${POSTGRESQL_REPLICATION_PASSWORD}'
+                create_or_update_secret 'postgresql_replication_password' '$${POSTGRESQL_REPLICATION_PASSWORD}'
-                create_or_update_secret 'postgresql_patroni_password'     '${POSTGRESQL_PATRONI_PASSWORD}'";;
+                create_or_update_secret 'postgresql_patroni_password'     '$${POSTGRESQL_PATRONI_PASSWORD}'";;
             productivity)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'productivity_paperless_secret_key' '${PRODUCTIVITY_PAPERLESS_SECRET_KEY}'
+                create_or_update_secret 'productivity_paperless_secret_key' '$${PRODUCTIVITY_PAPERLESS_SECRET_KEY}'
-                create_or_update_secret 'productivity_db_password'          '${PRODUCTIVITY_DB_PASSWORD}'
+                create_or_update_secret 'productivity_db_password'          '$${PRODUCTIVITY_DB_PASSWORD}'
-                create_or_update_secret 'productivity_oidc_providers'       '${PRODUCTIVITY_OIDC_PROVIDERS}'";;
+                create_or_update_secret 'productivity_oidc_providers'       '$${PRODUCTIVITY_OIDC_PROVIDERS}'";;
             vaultwarden)
               # NOTE: vaultwarden_database_url cannot be rotated in-place — Swarm refuses
               # to remove a secret referenced by a running service's spec. We provision
               # under a versioned name instead; vaultwarden.yaml maps it back to the same
               # in-container filename via target. The old secret is removed manually once
               # the compose file cutover is confirmed healthy.
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'vaultwarden_admin_token'     '${VAULTWARDEN_ADMIN_TOKEN}'
+                create_or_update_secret 'vaultwarden_admin_token'     '$${VAULTWARDEN_ADMIN_TOKEN}'
-                create_or_update_secret 'vaultwarden_database_url_v2' '${VAULTWARDEN_DATABASE_URL}'";;
+                create_or_update_secret 'vaultwarden_database_url_v2' '$${VAULTWARDEN_DATABASE_URL}'";;
             ai)
               # NOTE: ai stack uses Pattern B (host .env, not native Docker secrets) —
               # LiteLLM/boto3 and Open WebUI don't support the _FILE convention for these
               # vars. Instead of Docker secrets, we regenerate ONLY the migrated lines in
               # the remote ai/ai.env in place via grep -v + printf (never sed, since values
-              # may contain '/', '$', '&'). All other lines — including MCPO_API_KEY,
+              # may contain slash, dollar sign, ampersand). All other lines — including
-              # OAUTH_CLIENT_ID, WEBUI_URL, and other non-secret config — are left
+              # MCPO_API_KEY, OAUTH_CLIENT_ID, WEBUI_URL, and other non-secret config — are
-              # completely untouched. MCPO_API_KEY migration is deferred to a follow-up;
+              # left completely untouched. MCPO_API_KEY migration is deferred to a
-              # this step never reads or writes it.
+              # follow-up; this step never reads or writes it.
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "FILE=/volume1/docker/compose-files/ai/ai.env
+              #
+              # IMPORTANT: ai.yaml's litellm service references ${AI_AWS_ACCESS_KEY_ID} /
+              # ${AI_AWS_SECRET_ACCESS_KEY} (AI_-prefixed) and renders them into the
+              # container as plain AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (see commit
+              # 37ed671a, "Change AWS keys to use Woodpecker Secrets"). So ai.env must be
+              # written with the AI_-prefixed key names, NOT the plain ones — writing
+              # plain AWS_ACCESS_KEY_ID here would leave ${AI_AWS_ACCESS_KEY_ID}
+              # unresolved at compose-render time (empty), silently breaking Bedrock auth.
+              # All other migrated vars in ai.yaml use plain (unprefixed) names, so only
+              # these two lines need the AI_ prefix.
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "FILE=/volume1/docker/compose-files/ai/ai.env
                 TMP=\$FILE.tmp.\$\$
-                grep -vE '^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|LITELLM_MASTER_KEY|LITELLM_SALT_KEY|POSTGRES_PASSWORD|DATABASE_URL|WEBUI_SECRET_KEY|OPEN_WEBUI_DATABASE_URL|OAUTH_CLIENT_SECRET)=' \$FILE > \$TMP 2>/dev/null || touch \$TMP
+                grep -vE '^(AI_AWS_ACCESS_KEY_ID|AI_AWS_SECRET_ACCESS_KEY|LITELLM_MASTER_KEY|LITELLM_SALT_KEY|POSTGRES_PASSWORD|DATABASE_URL|WEBUI_SECRET_KEY|OPEN_WEBUI_DATABASE_URL|OAUTH_CLIENT_SECRET)=' \$FILE > \$TMP 2>/dev/null || touch \$TMP
                 { cat \$TMP
-                  printf 'AWS_ACCESS_KEY_ID=%s\n' '${AI_AWS_ACCESS_KEY_ID}'
+                  printf 'AI_AWS_ACCESS_KEY_ID=%s\n' '$${AI_AWS_ACCESS_KEY_ID}'
-                  printf 'AWS_SECRET_ACCESS_KEY=%s\n' '${AI_AWS_SECRET_ACCESS_KEY}'
+                  printf 'AI_AWS_SECRET_ACCESS_KEY=%s\n' '$${AI_AWS_SECRET_ACCESS_KEY}'
-                  printf 'LITELLM_MASTER_KEY=%s\n' '${AI_LITELLM_MASTER_KEY}'
+                  printf 'LITELLM_MASTER_KEY=%s\n' '$${AI_LITELLM_MASTER_KEY}'
-                  printf 'LITELLM_SALT_KEY=%s\n' '${AI_LITELLM_SALT_KEY}'
+                  printf 'LITELLM_SALT_KEY=%s\n' '$${AI_LITELLM_SALT_KEY}'
-                  printf 'POSTGRES_PASSWORD=%s\n' '${AI_LITELLM_DB_PASSWORD}'
+                  printf 'POSTGRES_PASSWORD=%s\n' '$${AI_LITELLM_DB_PASSWORD}'
-                  printf 'DATABASE_URL=postgresql://LiteLLM:%s@postgresql:5432/litellm\n' '${AI_LITELLM_DB_PASSWORD}'
+                  printf 'DATABASE_URL=postgresql://LiteLLM:%s@postgresql:5432/litellm\n' '$${AI_LITELLM_DB_PASSWORD}'
-                  printf 'WEBUI_SECRET_KEY=%s\n' '${AI_WEBUI_SECRET_KEY}'
+                  printf 'WEBUI_SECRET_KEY=%s\n' '$${AI_WEBUI_SECRET_KEY}'
-                  printf 'OPEN_WEBUI_DATABASE_URL=%s\n' '${AI_OPEN_WEBUI_DATABASE_URL}'
+                  printf 'OPEN_WEBUI_DATABASE_URL=%s\n' '$${AI_OPEN_WEBUI_DATABASE_URL}'
-                  printf 'OAUTH_CLIENT_SECRET=%s\n' '${AI_OAUTH_CLIENT_SECRET}'
+                  printf 'OAUTH_CLIENT_SECRET=%s\n' '$${AI_OAUTH_CLIENT_SECRET}'
                 } > \$FILE
                 rm -f \$TMP
                 echo '  [OK] ai/ai.env secrets updated'";;
             entertainment)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'entertainment_discord_token'          '${ENTERTAINMENT_DISCORD_TOKEN}'
+                create_or_update_secret 'entertainment_discord_token'          '$${ENTERTAINMENT_DISCORD_TOKEN}'
-                create_or_update_secret 'entertainment_discord_client_secret'  '${ENTERTAINMENT_DISCORD_CLIENT_SECRET}'
+                create_or_update_secret 'entertainment_discord_client_secret'  '$${ENTERTAINMENT_DISCORD_CLIENT_SECRET}'
-                create_or_update_secret 'entertainment_secret_key_base'        '${ENTERTAINMENT_SECRET_KEY_BASE}'
+                create_or_update_secret 'entertainment_secret_key_base'        '$${ENTERTAINMENT_SECRET_KEY_BASE}'
-                create_or_update_secret 'entertainment_basic_auth_password'    '${ENTERTAINMENT_BASIC_AUTH_PASSWORD}'
+                create_or_update_secret 'entertainment_basic_auth_password'    '$${ENTERTAINMENT_BASIC_AUTH_PASSWORD}'
-                create_or_update_secret 'entertainment_sparky_db_password'     '${ENTERTAINMENT_SPARKY_DB_PASSWORD}'
+                create_or_update_secret 'entertainment_sparky_db_password'     '$${ENTERTAINMENT_SPARKY_DB_PASSWORD}'
-                create_or_update_secret 'entertainment_sparky_app_db_password' '${ENTERTAINMENT_SPARKY_APP_DB_PASSWORD}'
+                create_or_update_secret 'entertainment_sparky_app_db_password' '$${ENTERTAINMENT_SPARKY_APP_DB_PASSWORD}'
-                create_or_update_secret 'entertainment_sparky_encryption_key'  '${ENTERTAINMENT_SPARKY_ENCRYPTION_KEY}'
+                create_or_update_secret 'entertainment_sparky_encryption_key'  '$${ENTERTAINMENT_SPARKY_ENCRYPTION_KEY}'
-                create_or_update_secret 'entertainment_better_auth_secret'     '${ENTERTAINMENT_BETTER_AUTH_SECRET}'";;
+                create_or_update_secret 'entertainment_better_auth_secret'     '$${ENTERTAINMENT_BETTER_AUTH_SECRET}'";;
             *)
               echo "  No secrets case for $STACK";;
           esac
@@ -290,31 +323,31 @@ steps:
       - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa
       - chmod 600 ~/.ssh/id_rsa
       - |
-        if [ -z "${SWARM_MANAGER_IP}" ]; then
+        if [ -z "$${SWARM_MANAGER_IP}" ]; then
           echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets."
           exit 1
         fi
-      - ssh-keyscan -H ${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
+      - ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
       - |
         CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
         FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
-        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true)
+        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -v '^deploy$' | sort -u || true)
         ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
         [ -z "$ALL_STACKS" ] && echo "No stacks changed" && exit 0
 
         # Always sync deploy/ scripts first
         rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
-          deploy/ root@${SWARM_MANAGER_IP}:/volume1/docker/compose-files/deploy/
+          deploy/ root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/deploy/
 
         for STACK in $ALL_STACKS; do
           echo "--- Deploying: $STACK ---"
           # Sync files to host first (always, even for bootstrap stacks)
           if [ -d "$STACK" ]; then
             rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
-              "$STACK/" root@${SWARM_MANAGER_IP}:/volume1/docker/compose-files/$STACK/
+              "$STACK/" root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/$STACK/
           elif [ -f "$STACK.yaml" ]; then
             rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
-              "$STACK.yaml" root@${SWARM_MANAGER_IP}:/volume1/docker/compose-files/
+              "$STACK.yaml" root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/
           fi
 
           # Bootstrap-tier guard: file synced to host, deploy is MANUAL
@@ -326,7 +359,7 @@ steps:
           esac
 
           # Tier 2: auto-deploy
-          ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} \
+          ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} \
             "bash /volume1/docker/compose-files/deploy/stack-deploy.sh $STACK" \
             && echo "  OK $STACK" || { echo "  FAIL $STACK"; exit 1; }
         done
@@ -344,23 +377,47 @@ steps:
       - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa
       - chmod 600 ~/.ssh/id_rsa
       - |
-        if [ -z "${SWARM_MANAGER_IP}" ]; then
+        if [ -z "$${SWARM_MANAGER_IP}" ]; then
           echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets."
           exit 1
         fi
-      - ssh-keyscan -H ${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
+      - ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
       - |
         CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
         FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
-        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true)
+        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -v '^deploy$' | sort -u || true)
         ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
         [ -z "$ALL_STACKS" ] && exit 0
-        sleep 5
+
+        # NOTE: `docker stack deploy` briefly tears down and recreates tasks in
+        # Swarm's internal bookkeeping, so `docker stack ps` can transiently
+        # return nothing right after deploy even when the service is healthy.
+        # A single `sleep 5` + one-shot check produced false-alarm-looking
+        # "nothing found in stack" output on ordinary deploys (e.g. vaultwarden,
+        # 2026-08-25). Retry with backoff instead of a single fixed sleep, and
+        # only warn (don't fail the pipeline) if tasks never show up.
+        ATTEMPTS=6
+        DELAY=5
         for STACK in $ALL_STACKS; do
           echo "--- $STACK ---"
-          ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} \
+          i=1
+          while [ "$i" -le "$ATTEMPTS" ]; do
+            OUTPUT=$(ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} \
               "docker stack ps $STACK --filter desired-state=running \
-            --format '  {{.Name}} {{.CurrentState}}'"
+              --format '  {{.Name}} {{.CurrentState}}'" 2>/dev/null)
+            if [ -n "$OUTPUT" ]; then
+              echo "$OUTPUT"
+              break
+            fi
+            if [ "$i" -eq "$ATTEMPTS" ]; then
+              echo "  WARNING: no running tasks found for $STACK after $((ATTEMPTS * DELAY))s."
+              echo "  This may be transient Swarm settle time, or a real problem — check manually:"
+              echo "  ssh root@$${SWARM_MANAGER_IP} 'docker stack ps $STACK --no-trunc'"
+            else
+              sleep "$DELAY"
+            fi
+            i=$((i + 1))
+          done
         done
 
   notify-success:

deploy/envparse.py

@@ -22,12 +22,229 @@ def merge_envs(base_path, override_path):
     merged = {**base, **override}
     return list(merged.items())
 
+# ──────────────────────────────────────────────────────────────────────────
+# 2026-08-26 FIX — double-interpolation truncation bug (Pattern B stacks):
+#
+#   stack-deploy.sh's single-file/no-extras path is:
+#     envsubst "$VARS" < ai.yaml | docker stack deploy -c - ai
+#
+#   envsubst substitutes ${VAR} placeholders in the compose YAML with the
+#   literal, raw value of each shell-exported variable. If that raw value
+#   itself contains a literal '$' followed by word characters (e.g. a
+#   randomly-generated secret like "...i*Edu$RyAVYTqr4yzSS##..."), the
+#   resulting YAML text now contains what LOOKS like a second variable
+#   reference. `docker stack deploy -c -` runs Compose's own interpolation
+#   pass on that YAML text before creating the service — and Compose sees
+#   that leftover "$RyAVYTqr4yzSS", finds no such env var, and silently
+#   substitutes empty string. The secret gets truncated in the running
+#   container with NO error or warning.
+#
+#   Confirmed impact (2026-08-26): LITELLM_MASTER_KEY and LITELLM_SALT_KEY
+#   in the `ai` stack were both truncated at their first literal '$' after
+#   a real deploy — 87-char secret arrived in the container as 73 chars.
+#
+#   This affects every stack using Pattern B (host .env + envsubst, not
+#   native Docker secrets): ai, maintenance, media, unifi, guacamole,
+#   security, auth, traefik, meshcentral, ddm — any of them could have a
+#   '$'-containing value silently truncating right now without detection,
+#   since the failure is silent and only visible by diffing the source
+#   value against the live container env.
+#
+#   Fix: escape every literal '$' in a value as '$$' at export time, BEFORE
+#   envsubst ever sees it. envsubst does not interpret '$' in the
+#   replacement text (only in the template), so the doubled dollar survives
+#   envsubst untouched. Compose's interpolation pass then consumes exactly
+#   one level of escaping ('$$' -> literal '$'), landing on the correct
+#   original single '$' with no leftover variable-reference lookalike.
+#
+#   ONLY applies to the single-file path (no `docker compose config` step
+#   downstream). See export_raw/export_raw_merged below for why the
+#   folder+extras path must NOT use this.
+# ──────────────────────────────────────────────────────────────────────────
+
+def escape_dollar(v):
+    return v.replace('$', '$$')
+
+# ──────────────────────────────────────────────────────────────────────────
+# 2026-08-26 FIX #2 — SECOND double-interpolation bug, folder+extras path
+# (discovered fixing IMMICH_KIOSK_BASICAUTH, a bcrypt hash full of '$'):
+#
+#   The folder+extras render path is:
+#     docker compose <files> config | strip | envsubst "$VARS"
+#     -> docker stack deploy -c -
+#
+#   `docker compose config` performs its OWN ${VAR} interpolation AND its
+#   own re-escaping of the output: any literal '$' character that ends up
+#   in the rendered YAML — escaped or not — gets doubled to '$$' by
+#   `docker compose config` itself, unconditionally, as part of producing
+#   spec-safe output. Confirmed by isolated test:
+#     raw MYVAR='a$b$c'   -> docker compose config -> "a$$b$$c"   (correct,
+#                                                        one level added)
+#     escaped MYVAR='a$$b$$c' (i.e. pre-doubled by escape_dollar) ->
+#       docker compose config -> "a$$$$b$$$$c"        (WRONG, doubled twice)
+#
+#   `docker stack deploy -c -` still only removes exactly ONE level of
+#   escaping on its way in (confirmed: "a$$b$$c" -> container label
+#   "a$b$c", correct). So across the whole folder+extras pipeline there is
+#   exactly ONE implicit escaping step (`docker compose config`) and ONE
+#   implicit un-escaping step (`docker stack deploy`) already built in —
+#   pre-escaping the exported value on top of that leaves one extra,
+#   uncollapsed level of '$$' in the final container label/env value.
+#
+#   Confirmed impact (2026-08-26): IMMICH_KIOSK_BASICAUTH
+#   ("BabyBryan:$2y$05$...") rendered as "BabyBryan:$$2y$$05$$..." in the
+#   final container label — Traefik basic auth would never match the real
+#   password hash, silently locking out the kiosk with no error.
+#
+#   Fix: use export_raw / export_raw_merged (NO escape_dollar) whenever the
+#   render path goes through `docker compose config` — i.e. any stack with
+#   extension files. Use export / export_merged (WITH escape_dollar) only
+#   for the single-file path, which has no `docker compose config` step and
+#   therefore only Swarm's own interpolation pass to protect against.
+# ──────────────────────────────────────────────────────────────────────────
+
+# ──────────────────────────────────────────────────────────────────────────
+# 2026-08-26 FIX — depends_on long-form vs Swarm short-form:
+#
+#   Folder-based stacks with extension files (e.g. immich + hwaccel.*.yml)
+#   go through the `docker compose <files> config | strip | docker stack
+#   deploy -c -` merge path. Modern `docker compose config` normalizes the
+#   short-form list syntax:
+#     depends_on: [database, redis]
+#   into the long-form condition mapping:
+#     depends_on:
+#       database:
+#         condition: service_started
+#         required: true
+#       redis:
+#         condition: service_started
+#         required: true
+#
+#   `docker stack deploy` (Swarm mode) does NOT understand the long-form
+#   mapping and rejects it with: "services.<svc>.depends_on must be a list".
+#   Single-file stacks (no extras) never hit this because they skip the
+#   `docker compose config` step entirely and go straight through envsubst.
+#
+#   Fix: collapse any long-form depends_on mapping back into the Swarm-
+#   compatible short-form list, purely as a text transform on the rendered
+#   YAML, right alongside the existing env_file stripping.
+# ──────────────────────────────────────────────────────────────────────────
+
+_DEP_RE = re.compile(r'^([ \t]*)depends_on:\s*$')
+_CHILD_KEY_RE = re.compile(r'^([ \t]*)([\w.\-]+):\s*$')
+_CHILD_LIST_RE = re.compile(r'^([ \t]*)-\s*(\S+)\s*$')
+
+def collapse_depends_on(text):
+    lines = text.split('\n')
+    out = []
+    i = 0
+    n = len(lines)
+    while i < n:
+        line = lines[i]
+        m = _DEP_RE.match(line)
+        if not m:
+            out.append(line)
+            i += 1
+            continue
+
+        indent = m.group(1)
+        base_indent = len(indent)
+        out.append(line)
+        i += 1
+
+        services = []
+        parse_ok = True
+        while i < n:
+            l = lines[i]
+            if l.strip() == '':
+                i += 1
+                continue
+            cur_indent = len(l) - len(l.lstrip(' \t'))
+            if cur_indent <= base_indent:
+                break  # dedent — end of this depends_on block
+
+            lm = _CHILD_LIST_RE.match(l)
+            if lm and cur_indent == base_indent + 2:
+                services.append(lm.group(2))
+                i += 1
+                continue
+
+            km = _CHILD_KEY_RE.match(l)
+            if km and cur_indent == base_indent + 2:
+                services.append(km.group(2))
+                i += 1
+                # skip nested condition/required/restart lines under this service
+                while i < n:
+                    nl = lines[i]
+                    if nl.strip() == '':
+                        i += 1
+                        continue
+                    nl_indent = len(nl) - len(nl.lstrip(' \t'))
+                    if nl_indent > base_indent + 2:
+                        i += 1
+                        continue
+                    break
+                continue
+
+            # Unexpected shape — bail without transforming to avoid corrupting output
+            parse_ok = False
+            break
+
+        if not parse_ok or not services:
+            # Re-emit whatever we consumed as-is (best effort: fall back to
+            # original slice) rather than risk mangling an unfamiliar shape.
+            pass
+
+        for svc in services:
+            out.append(indent + '  - ' + svc)
+
+    return '\n'.join(out)
+
+# ──────────────────────────────────────────────────────────────────────────
+# 2026-08-26 FIX — top-level `name:` property rejected by Swarm:
+#
+#   `docker compose config` (Compose Spec output) emits a top-level
+#   `name: <project>` key as the first line of the rendered document, e.g.:
+#     name: immich
+#     services:
+#       ...
+#
+#   This is valid Compose Spec but `docker stack deploy` (Swarm mode) uses
+#   a stricter schema that does NOT allow a top-level `name` property, and
+#   rejects the whole document with: "(root) Additional property name is
+#   not allowed". Single-file stacks (no extras) never hit this because
+#   they skip `docker compose config` and go straight through envsubst on
+#   the raw source YAML, which never had a top-level `name:` to begin with.
+#
+#   Fix: drop ONLY the top-level `name:` line (column 0, i.e. no leading
+#   whitespace) at the START of the document. Nested `name:` fields under
+#   networks/volumes/secrets (e.g. `name: postgresql_db-backend`, always
+#   indented) are legitimate and must NOT be touched.
+# ──────────────────────────────────────────────────────────────────────────
+
+_TOP_NAME_RE = re.compile(r'^name:\s*\S+\s*$')
+
+def strip_top_level_name(text):
+    lines = text.split('\n')
+    if lines and _TOP_NAME_RE.match(lines[0]):
+        lines = lines[1:]
+    return '\n'.join(lines)
+
 mode = sys.argv[1]
 if mode == 'export':
     for k, v in parse_env(sys.argv[2]):
-        print('export {}={}'.format(k, repr(v)))
+        print('export {}={}'.format(k, repr(escape_dollar(v))))
 elif mode == 'export_merged':
     # export_merged <global.env> <stack.env>
+    for k, v in merge_envs(sys.argv[2], sys.argv[3]):
+        print('export {}={}'.format(k, repr(escape_dollar(v))))
+elif mode == 'export_raw':
+    # export_raw <env-file> — NO escape_dollar. Use for the docker-compose-
+    # config render path (folder+extras), which does its own '$' escaping.
+    for k, v in parse_env(sys.argv[2]):
+        print('export {}={}'.format(k, repr(v)))
+elif mode == 'export_raw_merged':
+    # export_raw_merged <global.env> <stack.env> — NO escape_dollar.
     for k, v in merge_envs(sys.argv[2], sys.argv[3]):
         print('export {}={}'.format(k, repr(v)))
 elif mode == 'vars':
@@ -38,4 +255,6 @@ elif mode == 'vars_merged':
 elif mode == 'strip':
     t = sys.stdin.read()
     t = re.sub(r'[ \t]*env_file:[ \t]*\n([ \t]+-[^\n]*\n)+', '', t)
+    t = collapse_depends_on(t)
+    t = strip_top_level_name(t)
     sys.stdout.write(t)

deploy/git-guard.sh

@@ -0,0 +1,142 @@
+#!/usr/bin/env bash
+# git-guard.sh — Ensures the compose-files working tree is in sync with Gitea
+# before any deploy proceeds. Called automatically by stack-deploy.sh.
+#
+# Behavior:
+#   - Clean + up to date        -> pass silently
+#   - Clean + behind (ff-only)  -> auto `git pull --ff-only`, then pass
+#   - Ahead only (unpushed)     -> interactive: offer to push; non-interactive: BLOCK
+#   - Dirty tracked changes     -> offer to commit + push right now
+#                                   (auto in non-interactive/CI runs, after a
+#                                   secret-pattern scan of the staged diff)
+#   - Diverged (local AND       -> REFUSE. Never auto-resolves. Prints the
+#     remote both moved)           backup/stash/reset recovery steps and exits.
+#
+# Exit codes: 0 = safe to deploy, 1 = blocked, needs human intervention
+
+set -euo pipefail
+
+DIR="/volume1/docker/compose-files"
+cd "$DIR"
+
+# Non-interactive detection (Woodpecker/cron have no TTY on stdin)
+INTERACTIVE=0
+[ -t 0 ] && INTERACTIVE=1
+
+echo "==> git-guard: checking repo sync state"
+
+git fetch origin --quiet
+
+LOCAL="$(git rev-parse main)"
+REMOTE="$(git rev-parse origin/main)"
+BASE="$(git merge-base main origin/main)"
+
+DIRTY=0
+git status --porcelain | grep -q . && DIRTY=1
+
+SECRET_PATTERN='(-----BEGIN [A-Z]+ PRIVATE KEY-----|AKIA[0-9A-Z]{16}|xox[baprs]-[0-9a-zA-Z-]+|password[[:space:]]*[:=][[:space:]]*[^$ ]|api[_-]?key[[:space:]]*[:=][[:space:]]*[^$ ])'
+
+# ---- Case: dirty tracked changes ----
+if [ "$DIRTY" -eq 1 ]; then
+  echo "!! WORKING TREE DIRTY — uncommitted changes detected:"
+  git status --short
+  echo
+
+  if [ "$INTERACTIVE" -eq 1 ]; then
+    read -rp "Commit and push these changes to origin/main now? [y/N] " ans
+  else
+    ans="y"
+    echo "(non-interactive session — auto-committing and pushing)"
+  fi
+
+  if [[ "$ans" =~ ^[Yy]$ ]]; then
+    git add -A
+
+    if git diff --cached | grep -Eiq "$SECRET_PATTERN"; then
+      echo "ERROR: possible secret detected in staged changes. Refusing to auto-commit."
+      echo "Review manually: git diff --cached"
+      git reset
+      exit 1
+    fi
+
+    git commit -m "chore(auto): git-guard autofix - commit local changes before deploy $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+
+    if git push origin main; then
+      echo "==> Pushed. Re-checking sync state..."
+      exec "$0" "$@"
+    else
+      echo "ERROR: push failed (likely diverged from origin). Aborting deploy."
+      echo "Run: cd $DIR && git status"
+      exit 1
+    fi
+  else
+    echo "Aborting deploy - commit or stash changes manually, then retry."
+    exit 1
+  fi
+fi
+
+# ---- Case: fully in sync ----
+if [ "$LOCAL" = "$REMOTE" ]; then
+  echo "==> In sync with origin/main ($LOCAL). OK to deploy."
+  exit 0
+fi
+
+# ---- Case: behind only (fast-forwardable) ----
+if [ "$LOCAL" = "$BASE" ]; then
+  echo "!! Local main is behind origin/main."
+  if [ "$INTERACTIVE" -eq 1 ]; then
+    read -rp "Fast-forward pull now? [y/N] " ans
+  else
+    ans="y"
+    echo "(non-interactive session — auto fast-forwarding)"
+  fi
+  if [[ "$ans" =~ ^[Yy]$ ]]; then
+    git pull --ff-only origin main
+    echo "==> Fast-forwarded to $(git rev-parse --short main). OK to deploy."
+    exit 0
+  else
+    echo "Aborting deploy - pull manually, then retry."
+    exit 1
+  fi
+fi
+
+# ---- Case: ahead only (local commits not yet pushed) ----
+if [ "$REMOTE" = "$BASE" ]; then
+  echo "!! Local main is AHEAD of origin/main (unpushed commits):"
+  git log --oneline "origin/main..main"
+  echo
+  if [ "$INTERACTIVE" -eq 1 ]; then
+    read -rp "Push local commits to origin/main now? [y/N] " ans
+  else
+    ans="n"
+    echo "(non-interactive session — will NOT auto-push ahead commits; needs human review)"
+  fi
+  if [[ "$ans" =~ ^[Yy]$ ]]; then
+    git push origin main
+    echo "==> Pushed. OK to deploy."
+    exit 0
+  else
+    echo "Aborting deploy. Review with: git log origin/main..main"
+    exit 1
+  fi
+fi
+
+# ---- Case: true divergence (both ahead and behind) — NEVER auto-fix ----
+echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+echo "!! DIVERGED: local main and origin/main have both moved independently."
+echo "!!"
+echo "!!   Local-only commits:"
+git log --oneline "$BASE..main" | sed 's/^/!!     /'
+echo "!!"
+echo "!!   Remote-only commits:"
+git log --oneline "$BASE..origin/main" | sed 's/^/!!     /'
+echo "!!"
+echo "!! This requires a human decision - git-guard will NOT auto-resolve this."
+echo "!! Recommended recovery:"
+echo "!!   1. tar backup:      tar czf /volume1/docker/compose-files-backup-\$(date +%Y%m%d-%H%M%S).tar.gz -C /volume1/docker compose-files"
+echo "!!   2. name the branch: git branch backup/pre-reset-\$(date +%Y%m%d)"
+echo "!!   3. stash all state: git stash push -u -m 'pre-reset-snapshot'"
+echo "!!   4. reset to origin: git reset --hard origin/main"
+echo "!!   5. selectively restore needed files from the stash/backup branch"
+echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+exit 1

deploy/mount-guard.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+"""
+mount-guard.py — Pre-flight check for bind mount paths before `docker stack deploy`.
+
+Usage: mount-guard.py <rendered-compose.yml>
+
+Reads the FINAL rendered compose YAML (after env substitution, right before
+it would be handed to `docker stack deploy -c -`) and checks:
+
+  1. MISSING PATHS — every bind-mount source path referenced by any service
+     actually exists on disk. Swarm rejects the task at schedule time if not
+     (see: "bind source path does not exist"), but catching it here is
+     faster and clearer, and gives the option to create it on the spot.
+
+  2. SUSPICIOUS POSTGRES DATA DIRS — for any bind mount whose *target*
+     looks like a Postgres data directory, warn if the *source* exists but
+     is empty or missing a PG_VERSION file. This is the class of near-miss
+     from the 2026-08-26 Immich incident: a wrong-but-existing empty path
+     would have let Postgres silently initialize a brand-new database while
+     the real data sat orphaned elsewhere, with no error at all.
+
+Exit codes: 0 = safe to deploy, 1 = blocked / aborted.
+
+Interactive sessions get a prompt with remediation options. Non-interactive
+sessions (Woodpecker, cron, CI) NEVER auto-proceed past a finding here —
+this is a data-safety check, not a convenience autofix.
+"""
+
+import sys
+import os
+import re
+import yaml
+
+
+def is_bind_like(path):
+    return isinstance(path, str) and (
+        path.startswith('/') or path.startswith('./') or path.startswith('../')
+    )
+
+
+def gather_bind_mounts(doc):
+    """Return list of (service_name, target, source) for host-path bind mounts.
+
+    Handles both compose-file syntaxes that can appear in a rendered stack:
+      - short form:  "hostpath:containerpath[:mode]"        (raw single-file path)
+      - long form:   {type: bind, source: ..., target: ...} (post `docker compose config`)
+    Named volumes (source has no leading '/', './', '../') are intentionally skipped.
+    """
+    results = []
+    services = doc.get('services') or {}
+    for svc_name, svc in services.items():
+        if not isinstance(svc, dict):
+            continue
+
+        for v in (svc.get('volumes') or []):
+            if isinstance(v, str):
+                parts = v.split(':')
+                if len(parts) >= 2 and is_bind_like(parts[0]):
+                    results.append((svc_name, parts[1], parts[0]))
+            elif isinstance(v, dict):
+                if v.get('type') == 'bind':
+                    src = v.get('source')
+                    tgt = v.get('target')
+                    if is_bind_like(src):
+                        results.append((svc_name, tgt, src))
+
+        # devices: short form "host:container[:mode]" (rarely used in this repo —
+        # devices are conventionally expressed as bind-mounted volumes instead,
+        # for Docker Swarm / DDM compatibility — but handle it if present).
+        for d in (svc.get('devices') or []):
+            if isinstance(d, str):
+                parts = d.split(':')
+                if parts and is_bind_like(parts[0]):
+                    tgt = parts[1] if len(parts) > 1 else parts[0]
+                    results.append((svc_name, tgt, parts[0]))
+
+    return results
+
+
+_PG_TARGET_RE = re.compile(r'postgres', re.IGNORECASE)
+
+
+def looks_like_postgres_target(target):
+    return bool(target) and bool(_PG_TARGET_RE.search(target))
+
+
+def check_postgres_data(path):
+    """Return a warning string if `path` looks like an empty/uninitialized
+    Postgres data directory. Returns None if it looks fine. Caller must
+    ensure `path` already exists before calling this."""
+    if not os.path.isdir(path):
+        # Exists but isn't a directory (e.g. a file) — a different, separate
+        # problem; the missing-path style check doesn't apply, but this is
+        # clearly wrong too.
+        return "exists but is not a directory"
+    try:
+        entries = os.listdir(path)
+    except PermissionError:
+        return "cannot list directory contents (permission denied) — unable to verify"
+    if not entries:
+        return "directory exists but is EMPTY — looks like an uninitialized/wrong Postgres data dir"
+    if 'PG_VERSION' not in entries:
+        return "directory exists and is non-empty but has no PG_VERSION file — does not look like a real Postgres data directory"
+    return None
+
+
+def prompt(question, choices):
+    """choices: dict of key -> description. Returns the chosen key (lowercase)."""
+    print("Options:")
+    for k, desc in choices.items():
+        print(f"  [{k}] {desc}")
+    while True:
+        ans = input(f"{question} [{'/'.join(choices.keys())}]: ").strip().lower()
+        if ans in choices:
+            return ans
+        print(f"Please enter one of: {', '.join(choices.keys())}")
+
+
+def main():
+    if len(sys.argv) != 2:
+        print("Usage: mount-guard.py <rendered-compose.yml>", file=sys.stderr)
+        return 1
+
+    rendered_path = sys.argv[1]
+    with open(rendered_path) as f:
+        doc = yaml.safe_load(f)
+
+    if not doc or 'services' not in doc:
+        print("!! mount-guard: rendered YAML has no 'services' key — refusing to guess, blocking.")
+        return 1
+
+    mounts = gather_bind_mounts(doc)
+
+    missing = []
+    pg_warnings = []
+
+    for svc_name, target, src in mounts:
+        if not os.path.exists(src):
+            missing.append((svc_name, target, src))
+            continue
+        if looks_like_postgres_target(target):
+            warn = check_postgres_data(src)
+            if warn:
+                pg_warnings.append((svc_name, target, src, warn))
+
+    if not missing and not pg_warnings:
+        print(f"==> mount-guard: {len(mounts)} bind mount path(s) checked, all present and sane. OK to deploy.")
+        return 0
+
+    print("!! mount-guard found potential issues with bind mount paths:\n")
+
+    if missing:
+        print("MISSING PATHS (Swarm will reject the task if these aren't created first):")
+        for svc_name, target, src in missing:
+            print(f"  - service '{svc_name}': {src}  (mounted at {target})")
+        print()
+
+    if pg_warnings:
+        print("SUSPICIOUS POSTGRES DATA DIRECTORIES:")
+        print("  Path exists, but contents don't look like a real Postgres data dir.")
+        print("  This is the exact shape of the 2026-08-26 Immich near-miss: a wrong")
+        print("  bind path that HAPPENED to exist (empty) would have let Postgres")
+        print("  silently init a new empty DB while the real data sat orphaned")
+        print("  elsewhere — with no error or warning from Docker at all.")
+        for svc_name, target, src, warn in pg_warnings:
+            print(f"  - service '{svc_name}': {src}  (mounted at {target})")
+            print(f"      {warn}")
+        print()
+
+    interactive = sys.stdin.isatty()
+
+    if not interactive:
+        print("(non-interactive session — mount-guard will NOT auto-proceed on any finding above)")
+        print("Re-run interactively to review and decide, or fix the paths and retry.")
+        return 1
+
+    if missing:
+        choices = {
+            'a': 'Abort deploy (recommended if any path above is unexpected)',
+            'm': 'mkdir -p the missing path(s) now, then continue',
+            'c': 'Continue anyway without creating them (will likely fail at Swarm scheduling)',
+        }
+        ans = prompt("Missing bind mount paths found. Choice", choices)
+        if ans == 'a':
+            print("Aborting deploy.")
+            return 1
+        if ans == 'm':
+            for _, _, src in missing:
+                print(f"  mkdir -p {src}")
+                os.makedirs(src, exist_ok=True)
+            print("==> Created missing path(s). Continuing.")
+        # 'c' falls through and continues without creating
+
+    if pg_warnings:
+        choices = {
+            'a': 'Abort deploy (recommended unless you intended a fresh Postgres init here)',
+            'c': 'Continue anyway (I have verified this is expected, e.g. legitimate first-time init)',
+        }
+        ans = prompt("Suspicious Postgres data directory found. Choice", choices)
+        if ans == 'a':
+            print("Aborting deploy.")
+            return 1
+        print("==> Continuing deploy despite Postgres data warning, per operator confirmation.")
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

deploy/stack-deploy.sh

@@ -20,14 +20,38 @@
 # All files in the folder matching *.yml or *.yaml are included.
 # Main file (<stack>.yml or <stack>.yaml) is always passed FIRST.
 # Remaining files are sorted and appended.
+#
+# Render pipeline (unified for all modes as of 2026-08-26):
+#   1. Render full compose YAML (with extras merged + env substituted) to a
+#      temp file.
+#   2. Run mount-guard.py against that temp file — checks every bind mount
+#      source path exists, and flags suspicious-looking empty Postgres data
+#      dirs, before anything touches Swarm.
+#   3. docker stack deploy -c <tempfile> <stack>
+#
+# Dollar-escaping (2026-08-26, see envparse.py comments for full detail):
+#   - Single-file path (no extras): envsubst does no '$' escaping of its
+#     own, and only Swarm's `docker stack deploy` interpolation pass runs
+#     downstream -> use export/export_merged (escapes '$' -> '$$' once).
+#   - Folder+extras path: `docker compose config` ALSO does its own '$'
+#     escaping on top of Swarm's -> use export_raw/export_raw_merged (no
+#     pre-escaping) or values get doubled twice. Getting this wrong
+#     silently corrupts any secret/hash containing '$' (confirmed impact:
+#     LITELLM keys truncated, IMMICH_KIOSK_BASICAUTH bcrypt hash mismatched).
 
 set -euo pipefail
 
 STACK="${1:?Usage: stack-deploy.sh <stack-name>}"
 DIR="/volume1/docker/compose-files"
 PY="$DIR/deploy/envparse.py"
+MOUNT_GUARD="$DIR/deploy/mount-guard.py"
 GLOBAL_ENV="$DIR/deploy/global.env"
 
+# ── Pre-flight: ensure local checkout is in sync with Gitea ─────────────────
+# Prevents deploying from a stale/diverged local tree (see incident 2026-08-26).
+# Invoked via `bash` explicitly so the tracked file's exec bit doesn't matter.
+bash "$DIR/deploy/git-guard.sh" || { echo "ERROR: git-guard check failed. Deploy aborted."; exit 1; }
+
 # ── Locate compose file(s) ──────────────────────────────────────────────────
 
 FOLDER="$DIR/$STACK"
@@ -80,6 +104,12 @@ for f in "${EXTRAS[@]:-}"; do
   [ -n "$f" ] && F_FLAGS+=(-f "$f")
 done
 
+# Whether the render will go through `docker compose config` (folder mode
+# with 1+ extras). This determines which escaping mode is correct — see
+# header comment and envparse.py for why these must differ.
+USES_COMPOSE_CONFIG=0
+[ "${#F_FLAGS[@]}" -gt 2 ] && USES_COMPOSE_CONFIG=1
+
 # ── Load env (global base + optional stack override) ─────────────────────────
 
 HINT="  Hint: 'secret not found' means Woodpecker hasn't provisioned secrets yet.\n  Trigger the pipeline: https://woodpecker.bryanmail.net\n"
@@ -89,19 +119,25 @@ STACK_EXISTS=0
 [ -f "$GLOBAL_ENV" ] && GLOBAL_EXISTS=1
 [ -f "$ENVFILE"    ] && STACK_EXISTS=1
 
+if [ "$USES_COMPOSE_CONFIG" -eq 1 ]; then
+  EXPORT_MODE="export_raw"; EXPORT_MERGED_MODE="export_raw_merged"
+else
+  EXPORT_MODE="export"; EXPORT_MERGED_MODE="export_merged"
+fi
+
 if [ "$GLOBAL_EXISTS" -eq 1 ] && [ "$STACK_EXISTS" -eq 1 ]; then
-  echo "    Env:     $GLOBAL_ENV + $ENVFILE (stack overrides global)"
+  echo "    Env:     $GLOBAL_ENV + $ENVFILE (stack overrides global) [$EXPORT_MERGED_MODE]"
-  eval "$(python3 "$PY" export_merged "$GLOBAL_ENV" "$ENVFILE")"
+  eval "$(python3 "$PY" "$EXPORT_MERGED_MODE" "$GLOBAL_ENV" "$ENVFILE")"
   VARS="$(python3 "$PY" vars_merged "$GLOBAL_ENV" "$ENVFILE")"
 
 elif [ "$GLOBAL_EXISTS" -eq 1 ]; then
-  echo "    Env:     $GLOBAL_ENV (no stack env)"
+  echo "    Env:     $GLOBAL_ENV (no stack env) [$EXPORT_MODE]"
-  eval "$(python3 "$PY" export "$GLOBAL_ENV")"
+  eval "$(python3 "$PY" "$EXPORT_MODE" "$GLOBAL_ENV")"
   VARS="$(python3 "$PY" vars "$GLOBAL_ENV")"
 
 elif [ "$STACK_EXISTS" -eq 1 ]; then
-  echo "    Env:     $ENVFILE (no global env)"
+  echo "    Env:     $ENVFILE (no global env) [$EXPORT_MODE]"
-  eval "$(python3 "$PY" export "$ENVFILE")"
+  eval "$(python3 "$PY" "$EXPORT_MODE" "$ENVFILE")"
   VARS="$(python3 "$PY" vars "$ENVFILE")"
 
 else
@@ -111,33 +147,44 @@ fi
 
 echo "    Vars:    $VARS"
 
-# ── Deploy ───────────────────────────────────────────────────────────────────
+# ── Render final compose YAML to a temp file ──────────────────────────────────
+
+RENDERED="$(mktemp /tmp/stack-deploy.XXXXXX.yml)"
+trap 'rm -f "$RENDERED"' EXIT
 
 if [ -n "$VARS" ]; then
-  if [ "${#F_FLAGS[@]}" -gt 2 ]; then
+  if [ "$USES_COMPOSE_CONFIG" -eq 1 ]; then
     # Folder mode with extras: merge via docker compose config
     docker compose "${F_FLAGS[@]}" config \
       | python3 "$PY" strip \
       | envsubst "$VARS" \
-      | docker stack deploy -c - "$STACK" \
+      > "$RENDERED"
-      || { printf "\n%b" "$HINT"; exit 1; }
   else
     # Single file
     envsubst "$VARS" < "$MAIN" \
       | python3 "$PY" strip \
-      | docker stack deploy -c - "$STACK" \
+      > "$RENDERED"
-      || { printf "\n%b" "$HINT"; exit 1; }
   fi
 else
-  if [ "${#F_FLAGS[@]}" -gt 2 ]; then
+  if [ "$USES_COMPOSE_CONFIG" -eq 1 ]; then
     docker compose "${F_FLAGS[@]}" config \
       | python3 "$PY" strip \
-      | docker stack deploy -c - "$STACK" \
+      > "$RENDERED"
-      || { printf "\n%b" "$HINT"; exit 1; }
   else
-    docker stack deploy -c "$MAIN" "$STACK" \
+    python3 "$PY" strip < "$MAIN" > "$RENDERED"
-      || { printf "\n%b" "$HINT"; exit 1; }
   fi
 fi
 
+# ── Pre-flight: bind mount paths exist + Postgres-data sanity ───────────────
+# See deploy/mount-guard.py for details. Blocks on missing paths or
+# suspicious-looking empty/uninitialized Postgres data directories (see
+# incident 2026-08-26: a wrong-but-existing empty bind path would have let
+# Postgres silently init a fresh DB while real data sat orphaned elsewhere).
+python3 "$MOUNT_GUARD" "$RENDERED" || { echo "ERROR: mount-guard check failed. Deploy aborted."; exit 1; }
+
+# ── Deploy ───────────────────────────────────────────────────────────────────
+
+docker stack deploy -c "$RENDERED" "$STACK" \
+  || { printf "\n%b" "$HINT"; exit 1; }
+
 echo "==> Done: $STACK"

immich/immich.env-example

@@ -1,11 +1,25 @@
 # Immich stack environment variables - EXAMPLE
-# Copy to immich.env and fill in actual values
+# Copy this to immich.env (untracked, see .gitignore) and fill in
-# Pattern C (partial): DB_PASSWORD as Docker secret (_FILE), IMMICH_KIOSK_BASICAUTH in host .env (Traefik label)
+# IMMICH_KIOSK_BASICAUTH with a real value. Do NOT commit immich.env.
+#
+# Pattern C (partial): DB_PASSWORD as Docker secret (_FILE), IMMICH_KIOSK_BASICAUTH
+# in host .env (Traefik label).
+#
+# NOTE 2026-08-26: the paths below are the REAL, confirmed-correct values for
+# this homelab (verified via `docker service inspect immich_<svc> --format
+# '{{json .PreviousSpec.TaskTemplate.ContainerSpec.Mounts}}'` against actual
+# on-disk data). An earlier version of this file had plausible-looking but
+# WRONG generic paths (/volume1/docker/immich/db, /volume1/docker/immich/uploads),
+# which caused a real incident when immich.env was regenerated from this
+# template without checking against the live services first. If you ever
+# need to regenerate immich.env from this file, these paths should still be
+# correct — but if in doubt, re-verify with the PreviousSpec command above
+# before deploying.
 
 IMMICH_VERSION=release
-UPLOAD_LOCATION=/volume1/docker/immich/uploads
+UPLOAD_LOCATION=/volume1/Immich-Photos/
-BULK_UPLOAD_LOCATION=/volume1/docker/immich/bulk-uploads
+BULK_UPLOAD_LOCATION=/volume1/Immich-Photos/bulk-upload
-DB_DATA_LOCATION=/volume1/docker/immich/db
+DB_DATA_LOCATION=/volume1/docker/immich-postgresql
 REDIS_DATA_LOCATION=/volume1/docker/immich/redis
 
 DB_USERNAME=immich
@@ -20,4 +34,4 @@ REDIS_DBINDEX=0
 IMMICH_TRUSTED_PROXIES=172.16.0.0/12
 IMMICH_TRAEFIK_HOST=immich.bryanmail.net
 IMMICH_KIOSK_HOST=immich-kiosk.bryanmail.net
-IMMICH_KIOSK_BASICAUTH=user:password
+IMMICH_KIOSK_BASICAUTH=user:changeme

mcpo/flowagent/Dockerfile

@@ -0,0 +1,21 @@
+FROM ghcr.io/open-webui/mcpo:main
+
+# Azure CLI — required for non-interactive service-principal auth
+# (`az login --service-principal`) performed by entrypoint.sh at container
+# start. FlowAgent's auth model is Azure CLI + MSAL; see
+# https://github.com/microsoft/power-platform-skills/blob/main/plugins/power-automate/references/connection-patterns.md
+RUN apk add --no-cache py3-pip curl \
+ && pip install --no-cache-dir --break-system-packages azure-cli
+
+# FlowAgent self-contained MCP bundle (stdio transport, all tools inlined,
+# Node 18+ only — no npm install / remote host needed at runtime).
+# Pinned to a ref (commit SHA or tag), NOT `main`, for reproducible builds.
+# Bump deliberately via PR when upstream ships updates:
+# https://github.com/microsoft/power-platform-skills/tree/main/plugins/power-automate/server
+ARG FLOWAGENT_REF=main
+RUN mkdir -p /app/flowagent \
+ && curl -fsSL "https://raw.githubusercontent.com/microsoft/power-platform-skills/${FLOWAGENT_REF}/plugins/power-automate/server/mcp.mjs" \
+      -o /app/flowagent/mcp.mjs
+
+COPY entrypoint.sh /app/flowagent/entrypoint.sh
+RUN chmod +x /app/flowagent/entrypoint.sh

mcpo/flowagent/entrypoint.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+# FlowAgent MCP entrypoint — non-interactive Azure service-principal login.
+#
+# Expects three env vars pointing at Docker secret files (Pattern C,
+# _FILE convention, provisioned by Woodpecker — never hand-typed):
+#   FLOWAGENT_AZURE_CLIENT_ID_FILE
+#   FLOWAGENT_AZURE_TENANT_ID_FILE
+#   FLOWAGENT_AZURE_CLIENT_SECRET_FILE
+#
+# On success, execs into the real command (node /app/flowagent/mcp.mjs),
+# replacing this shell so mcpo's stdio pipe talks directly to the MCP process.
+set -eu
+
+: "${FLOWAGENT_AZURE_CLIENT_ID_FILE:?FLOWAGENT_AZURE_CLIENT_ID_FILE not set}"
+: "${FLOWAGENT_AZURE_TENANT_ID_FILE:?FLOWAGENT_AZURE_TENANT_ID_FILE not set}"
+: "${FLOWAGENT_AZURE_CLIENT_SECRET_FILE:?FLOWAGENT_AZURE_CLIENT_SECRET_FILE not set}"
+
+CLIENT_ID="$(cat "$FLOWAGENT_AZURE_CLIENT_ID_FILE")"
+TENANT_ID="$(cat "$FLOWAGENT_AZURE_TENANT_ID_FILE")"
+CLIENT_SECRET="$(cat "$FLOWAGENT_AZURE_CLIENT_SECRET_FILE")"
+
+az login --service-principal \
+  -u "$CLIENT_ID" \
+  -p "$CLIENT_SECRET" \
+  --tenant "$TENANT_ID" \
+  --output none
+
+unset CLIENT_SECRET
+
+exec "$@"

secrets/flowagent.secrets.example

@@ -0,0 +1,53 @@
+# flowagent (Power Automate MCP, via mcpo service) — Secrets Reference
+# Source: mcpo/flowagent — built into a custom image, consumed by the
+# "mcpo" service in ai.yaml (NEVER mcpo-critical).
+#
+# Add SECRET values to Woodpecker at:
+#   https://woodpecker.bryanmail.net
+#   homelab/compose-files → Settings → Secrets
+#
+# Prerequisite (manual, outside GitOps — Azure Portal):
+#   1. Entra ID → App registrations → New registration
+#      Name: flowagent-mcp-homelab, single tenant
+#   2. API permissions → add Power Automate / Flow Service application
+#      permissions + Dynamics CRM user_impersonation (Dataverse access)
+#   3. Grant admin consent
+#   4. Certificates & secrets → new client secret → copy value immediately
+#   5. Power Platform Admin Center → target environment → S2S apps →
+#      register the application user for this app ID (required for
+#      Dataverse/environment access by a service principal)
+#
+# NOTE: the OLD "powerautomate" mcpo/config.json entry (npm package
+# powerautomate-mcp, client_id 84b431ed-..., tenant_id 0f6cf991-...) was
+# never onboarded to this secrets pipeline and is being replaced by this.
+# Do not reuse those IDs unless you've independently confirmed in Azure
+# Portal that the old App Registration still exists, still has valid
+# permissions/consent, and you intend to reuse it — otherwise register new.
+
+# ── SECRETS (add to Woodpecker) ──────────────────────────────────────────
+
+# Woodpecker secret name:  flowagent_azure_client_id
+# Used for:                Azure AD App Registration client ID
+# Env var in entrypoint:   FLOWAGENT_AZURE_CLIENT_ID_FILE (Docker secret _FILE)
+flowagent_azure_client_id=
+
+# Woodpecker secret name:  flowagent_azure_tenant_id
+# Used for:                Azure AD tenant ID
+# Env var in entrypoint:   FLOWAGENT_AZURE_TENANT_ID_FILE (Docker secret _FILE)
+flowagent_azure_tenant_id=
+
+# Woodpecker secret name:  flowagent_azure_client_secret
+# Used for:                Azure AD App Registration client secret (rotate if leaked)
+# Env var in entrypoint:   FLOWAGENT_AZURE_CLIENT_SECRET_FILE (Docker secret _FILE)
+flowagent_azure_client_secret=
+
+# ── Woodpecker provision-secrets case entry ──────────────────────────────
+# Add this to the provision-secrets step in .woodpecker/deploy.yml
+# (separate follow-up PR — this file only documents it):
+#
+#   ai)
+#     ... existing ai-stack secret provisioning ...
+#     create_or_update_secret "flowagent_azure_client_id"     "$FLOWAGENT_AZURE_CLIENT_ID"
+#     create_or_update_secret "flowagent_azure_tenant_id"     "$FLOWAGENT_AZURE_TENANT_ID"
+#     create_or_update_secret "flowagent_azure_client_secret" "$FLOWAGENT_AZURE_CLIENT_SECRET"
+#     ;;

traefik/traefik.yaml

@@ -5,8 +5,8 @@ services:
       KEEPALIVED_INTERFACE: eth0
       KEEPALIVED_PRIORITY: "150"
       KEEPALIVED_ROUTER_ID: "51"
-      KEEPALIVED_PASSWORD: ${KEEPALIVED_PASSWORD}
+      KEEPALIVED_PASSWORD: "${KEEPALIVED_PASSWORD}"
-      KEEPALIVED_VIRTUAL_IPS: ${KEEPALIVED_VIRTUAL_IPS}
+      KEEPALIVED_VIRTUAL_IPS: "${KEEPALIVED_VIRTUAL_IPS}"
       KEEPALIVED_UNICAST_PEERS: "#PYTHON2BASH:['192.168.4.31', '192.168.4.32']"
     command: --copy-service
     cap_add: [NET_ADMIN, NET_BROADCAST, NET_RAW]
@@ -39,8 +39,8 @@ services:
       KEEPALIVED_INTERFACE: eth0
       KEEPALIVED_PRIORITY: "100"
       KEEPALIVED_ROUTER_ID: "51"
-      KEEPALIVED_PASSWORD: ${KEEPALIVED_PASSWORD}
+      KEEPALIVED_PASSWORD: "${KEEPALIVED_PASSWORD}"
-      KEEPALIVED_VIRTUAL_IPS: ${KEEPALIVED_VIRTUAL_IPS}
+      KEEPALIVED_VIRTUAL_IPS: "${KEEPALIVED_VIRTUAL_IPS}"
       KEEPALIVED_UNICAST_PEERS: "#PYTHON2BASH:['192.168.4.32', '192.168.4.33']"
     cap_add: [NET_ADMIN, NET_BROADCAST, NET_RAW]
     networks: [host]
@@ -147,7 +147,7 @@ services:
         order: stop-first
 
   traefik-certs-dumper:
-    image: ghcr.io/ldez/traefik-certs-dumper:v2.10.0
+    image: ghcr.io/ldez/traefik-certs-dumper:v2.11.4
     entrypoint: sh -c 'apk add jq ; while ! [ -e /letsencrypt/acme.json ] || ! [ `jq ".[] | .Certificates | length" /letsencrypt/acme.json` != 0 ]; do sleep 1 ; done && traefik-certs-dumper file --version v3 --watch --source /letsencrypt/acme.json --dest /letsencrypt/certs'
     volumes:
       - /volume1/docker/letsencrypt:/letsencrypt