Merge pull request 'HOTFIX: restore $${VAR} escaping dropped by AI secrets migration rewrite' (#6) from hotfix-dollar-escaping into main

Reviewed-on: #6

.woodpecker/deploy.yml

@@ -26,6 +26,16 @@ when:
 #   This was the root cause of a long-running "SWARM_MANAGER_IP secret is
 #   empty" failure: braced references were blanked at compile time before the
 #   shell ever saw them.
+#
+#   2026-08-26 HOTFIX: a full-file rewrite (AI secrets migration PR) dropped
+#   one $ from every $${VAR} occurrence throughout this file, re-introducing
+#   exactly the bug described above for EVERY secret-backed var, not just the
+#   new AI ones. The if-empty guards caught it immediately (SWARM_MANAGER_IP
+#   came back blank) and aborted before any ssh/scp/rsync ran, so no live
+#   secret or service was touched — but no CI provisioning/deploy could run
+#   until this was restored. Lesson: grep for the literal string '$${' and
+#   diff the count against the previous version before ever committing a
+#   full-file rewrite of this pipeline.
 # ─────────────────────────────────────────────────────────────────────────────
 
 steps:
@@ -162,18 +172,18 @@ steps:
       - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa
       - chmod 600 ~/.ssh/id_rsa
       - |
-        if [ -z "${SWARM_MANAGER_IP}" ]; then
+        if [ -z "$${SWARM_MANAGER_IP}" ]; then
           echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets."
           exit 1
         fi
-      - ssh-keyscan -H ${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
+      - ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
       - |
         CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
         FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
         FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true)
         ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
         [ -z "$ALL_STACKS" ] && echo "No stacks changed, skipping" && exit 0
-      - scp -o StrictHostKeyChecking=no deploy/create-secrets.sh root@${SWARM_MANAGER_IP}:/tmp/cs.sh
+      - scp -o StrictHostKeyChecking=no deploy/create-secrets.sh root@$${SWARM_MANAGER_IP}:/tmp/cs.sh
       - |
         CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
         FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
@@ -186,92 +196,92 @@ steps:
             maintenance|media|unifi|guacamole|security|auth|traefik|meshcentral|ddm)
               echo "  No Docker secrets for $STACK — secrets in host .env";;
             immich)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'immich_db_password'     '${IMMICH_DB_PASSWORD}'
-                create_or_update_secret 'immich_kiosk_basicauth' '${IMMICH_KIOSK_BASICAUTH}'";;
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'immich_db_password'     '$${IMMICH_DB_PASSWORD}'
+                create_or_update_secret 'immich_kiosk_basicauth' '$${IMMICH_KIOSK_BASICAUTH}'";;
             woodpecker)
               echo "  Manual only — skipping";;
             3dprint)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret '3dprint_db_password' '${PRINT3D_DB_PASSWORD}'";;
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret '3dprint_db_password' '$${PRINT3D_DB_PASSWORD}'";;
             gamma)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'gamma_auth_token' '${GAMMA_AUTH_TOKEN}'";;
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'gamma_auth_token' '$${GAMMA_AUTH_TOKEN}'";;
             git)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'git_db_password'       '${GIT_DB_PASSWORD}'
-                create_or_update_secret 'git_runner_token'      '${GIT_RUNNER_TOKEN}'
-                create_or_update_secret 'git_mcp_access_token'  '${GIT_MCP_ACCESS_TOKEN}'";;
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'git_db_password'       '$${GIT_DB_PASSWORD}'
+                create_or_update_secret 'git_runner_token'      '$${GIT_RUNNER_TOKEN}'
+                create_or_update_secret 'git_mcp_access_token'  '$${GIT_MCP_ACCESS_TOKEN}'";;
             homeassistant)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'homeassistant_frigate_rtsp_password' '${HOMEASSISTANT_FRIGATE_RTSP_PASSWORD}'
-                create_or_update_secret 'homeassistant_immich_api_key'        '${HOMEASSISTANT_IMMICH_API_KEY}'";;
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'homeassistant_frigate_rtsp_password' '$${HOMEASSISTANT_FRIGATE_RTSP_PASSWORD}'
+                create_or_update_secret 'homeassistant_immich_api_key'        '$${HOMEASSISTANT_IMMICH_API_KEY}'";;
             mealie)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'mealie_db_password'         '${MEALIE_DB_PASSWORD}'
-                create_or_update_secret 'mealie_ldap_query_password' '${MEALIE_LDAP_QUERY_PASSWORD}'";;
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'mealie_db_password'         '$${MEALIE_DB_PASSWORD}'
+                create_or_update_secret 'mealie_ldap_query_password' '$${MEALIE_LDAP_QUERY_PASSWORD}'";;
             n8n)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'n8n_db_password'     '${N8N_DB_PASSWORD}'
-                create_or_update_secret 'n8n_encryption_key'  '${N8N_ENCRYPTION_KEY}'";;
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'n8n_db_password'     '$${N8N_DB_PASSWORD}'
+                create_or_update_secret 'n8n_encryption_key'  '$${N8N_ENCRYPTION_KEY}'";;
             postgresql)
               # NOTE: bootstrap-tier stack (manual deploy only via stack-deploy.sh).
               # Secrets are still auto-provisioned here so they exist on the host
               # before the manual `stack-deploy.sh postgresql` run picks them up.
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'postgresql_password'             '${POSTGRESQL_PASSWORD}'
-                create_or_update_secret 'postgresql_pgadmin_password'     '${POSTGRESQL_PGADMIN_PASSWORD}'
-                create_or_update_secret 'postgresql_replication_password' '${POSTGRESQL_REPLICATION_PASSWORD}'
-                create_or_update_secret 'postgresql_patroni_password'     '${POSTGRESQL_PATRONI_PASSWORD}'";;
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'postgresql_password'             '$${POSTGRESQL_PASSWORD}'
+                create_or_update_secret 'postgresql_pgadmin_password'     '$${POSTGRESQL_PGADMIN_PASSWORD}'
+                create_or_update_secret 'postgresql_replication_password' '$${POSTGRESQL_REPLICATION_PASSWORD}'
+                create_or_update_secret 'postgresql_patroni_password'     '$${POSTGRESQL_PATRONI_PASSWORD}'";;
             productivity)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'productivity_paperless_secret_key' '${PRODUCTIVITY_PAPERLESS_SECRET_KEY}'
-                create_or_update_secret 'productivity_db_password'          '${PRODUCTIVITY_DB_PASSWORD}'
-                create_or_update_secret 'productivity_oidc_providers'       '${PRODUCTIVITY_OIDC_PROVIDERS}'";;
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'productivity_paperless_secret_key' '$${PRODUCTIVITY_PAPERLESS_SECRET_KEY}'
+                create_or_update_secret 'productivity_db_password'          '$${PRODUCTIVITY_DB_PASSWORD}'
+                create_or_update_secret 'productivity_oidc_providers'       '$${PRODUCTIVITY_OIDC_PROVIDERS}'";;
             vaultwarden)
               # NOTE: vaultwarden_database_url cannot be rotated in-place — Swarm refuses
               # to remove a secret referenced by a running service's spec. We provision
               # under a versioned name instead; vaultwarden.yaml maps it back to the same
               # in-container filename via target. The old secret is removed manually once
               # the compose file cutover is confirmed healthy.
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'vaultwarden_admin_token'     '${VAULTWARDEN_ADMIN_TOKEN}'
-                create_or_update_secret 'vaultwarden_database_url_v2' '${VAULTWARDEN_DATABASE_URL}'";;
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'vaultwarden_admin_token'     '$${VAULTWARDEN_ADMIN_TOKEN}'
+                create_or_update_secret 'vaultwarden_database_url_v2' '$${VAULTWARDEN_DATABASE_URL}'";;
             ai)
               # NOTE: ai stack uses Pattern B (host .env, not native Docker secrets) —
               # LiteLLM/boto3 and Open WebUI don't support the _FILE convention for these
               # vars. Instead of Docker secrets, we regenerate ONLY the migrated lines in
               # the remote ai/ai.env in place via grep -v + printf (never sed, since values
-              # may contain '/', '$', '&'). All other lines — including MCPO_API_KEY,
-              # OAUTH_CLIENT_ID, WEBUI_URL, and other non-secret config — are left
-              # completely untouched. MCPO_API_KEY migration is deferred to a follow-up;
-              # this step never reads or writes it.
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "FILE=/volume1/docker/compose-files/ai/ai.env
+              # may contain slash, dollar sign, ampersand). All other lines — including
+              # MCPO_API_KEY, OAUTH_CLIENT_ID, WEBUI_URL, and other non-secret config — are
+              # left completely untouched. MCPO_API_KEY migration is deferred to a
+              # follow-up; this step never reads or writes it.
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "FILE=/volume1/docker/compose-files/ai/ai.env
                 TMP=\$FILE.tmp.\$\$
                 grep -vE '^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|LITELLM_MASTER_KEY|LITELLM_SALT_KEY|POSTGRES_PASSWORD|DATABASE_URL|WEBUI_SECRET_KEY|OPEN_WEBUI_DATABASE_URL|OAUTH_CLIENT_SECRET)=' \$FILE > \$TMP 2>/dev/null || touch \$TMP
                 { cat \$TMP
-                  printf 'AWS_ACCESS_KEY_ID=%s\n' '${AI_AWS_ACCESS_KEY_ID}'
-                  printf 'AWS_SECRET_ACCESS_KEY=%s\n' '${AI_AWS_SECRET_ACCESS_KEY}'
-                  printf 'LITELLM_MASTER_KEY=%s\n' '${AI_LITELLM_MASTER_KEY}'
-                  printf 'LITELLM_SALT_KEY=%s\n' '${AI_LITELLM_SALT_KEY}'
-                  printf 'POSTGRES_PASSWORD=%s\n' '${AI_LITELLM_DB_PASSWORD}'
-                  printf 'DATABASE_URL=postgresql://LiteLLM:%s@postgresql:5432/litellm\n' '${AI_LITELLM_DB_PASSWORD}'
-                  printf 'WEBUI_SECRET_KEY=%s\n' '${AI_WEBUI_SECRET_KEY}'
-                  printf 'OPEN_WEBUI_DATABASE_URL=%s\n' '${AI_OPEN_WEBUI_DATABASE_URL}'
-                  printf 'OAUTH_CLIENT_SECRET=%s\n' '${AI_OAUTH_CLIENT_SECRET}'
+                  printf 'AWS_ACCESS_KEY_ID=%s\n' '$${AI_AWS_ACCESS_KEY_ID}'
+                  printf 'AWS_SECRET_ACCESS_KEY=%s\n' '$${AI_AWS_SECRET_ACCESS_KEY}'
+                  printf 'LITELLM_MASTER_KEY=%s\n' '$${AI_LITELLM_MASTER_KEY}'
+                  printf 'LITELLM_SALT_KEY=%s\n' '$${AI_LITELLM_SALT_KEY}'
+                  printf 'POSTGRES_PASSWORD=%s\n' '$${AI_LITELLM_DB_PASSWORD}'
+                  printf 'DATABASE_URL=postgresql://LiteLLM:%s@postgresql:5432/litellm\n' '$${AI_LITELLM_DB_PASSWORD}'
+                  printf 'WEBUI_SECRET_KEY=%s\n' '$${AI_WEBUI_SECRET_KEY}'
+                  printf 'OPEN_WEBUI_DATABASE_URL=%s\n' '$${AI_OPEN_WEBUI_DATABASE_URL}'
+                  printf 'OAUTH_CLIENT_SECRET=%s\n' '$${AI_OAUTH_CLIENT_SECRET}'
                 } > \$FILE
                 rm -f \$TMP
                 echo '  [OK] ai/ai.env secrets updated'";;
             entertainment)
-              ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
-                create_or_update_secret 'entertainment_discord_token'          '${ENTERTAINMENT_DISCORD_TOKEN}'
-                create_or_update_secret 'entertainment_discord_client_secret'  '${ENTERTAINMENT_DISCORD_CLIENT_SECRET}'
-                create_or_update_secret 'entertainment_secret_key_base'        '${ENTERTAINMENT_SECRET_KEY_BASE}'
-                create_or_update_secret 'entertainment_basic_auth_password'    '${ENTERTAINMENT_BASIC_AUTH_PASSWORD}'
-                create_or_update_secret 'entertainment_sparky_db_password'     '${ENTERTAINMENT_SPARKY_DB_PASSWORD}'
-                create_or_update_secret 'entertainment_sparky_app_db_password' '${ENTERTAINMENT_SPARKY_APP_DB_PASSWORD}'
-                create_or_update_secret 'entertainment_sparky_encryption_key'  '${ENTERTAINMENT_SPARKY_ENCRYPTION_KEY}'
-                create_or_update_secret 'entertainment_better_auth_secret'     '${ENTERTAINMENT_BETTER_AUTH_SECRET}'";;
+              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
+                create_or_update_secret 'entertainment_discord_token'          '$${ENTERTAINMENT_DISCORD_TOKEN}'
+                create_or_update_secret 'entertainment_discord_client_secret'  '$${ENTERTAINMENT_DISCORD_CLIENT_SECRET}'
+                create_or_update_secret 'entertainment_secret_key_base'        '$${ENTERTAINMENT_SECRET_KEY_BASE}'
+                create_or_update_secret 'entertainment_basic_auth_password'    '$${ENTERTAINMENT_BASIC_AUTH_PASSWORD}'
+                create_or_update_secret 'entertainment_sparky_db_password'     '$${ENTERTAINMENT_SPARKY_DB_PASSWORD}'
+                create_or_update_secret 'entertainment_sparky_app_db_password' '$${ENTERTAINMENT_SPARKY_APP_DB_PASSWORD}'
+                create_or_update_secret 'entertainment_sparky_encryption_key'  '$${ENTERTAINMENT_SPARKY_ENCRYPTION_KEY}'
+                create_or_update_secret 'entertainment_better_auth_secret'     '$${ENTERTAINMENT_BETTER_AUTH_SECRET}'";;
             *)
               echo "  No secrets case for $STACK";;
           esac
@@ -290,11 +300,11 @@ steps:
       - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa
       - chmod 600 ~/.ssh/id_rsa
       - |
-        if [ -z "${SWARM_MANAGER_IP}" ]; then
+        if [ -z "$${SWARM_MANAGER_IP}" ]; then
           echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets."
           exit 1
         fi
-      - ssh-keyscan -H ${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
+      - ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
       - |
         CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
         FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
@@ -304,17 +314,17 @@ steps:
 
         # Always sync deploy/ scripts first
         rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
-          deploy/ root@${SWARM_MANAGER_IP}:/volume1/docker/compose-files/deploy/
+          deploy/ root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/deploy/
 
         for STACK in $ALL_STACKS; do
           echo "--- Deploying: $STACK ---"
           # Sync files to host first (always, even for bootstrap stacks)
           if [ -d "$STACK" ]; then
             rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
-              "$STACK/" root@${SWARM_MANAGER_IP}:/volume1/docker/compose-files/$STACK/
+              "$STACK/" root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/$STACK/
           elif [ -f "$STACK.yaml" ]; then
             rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
-              "$STACK.yaml" root@${SWARM_MANAGER_IP}:/volume1/docker/compose-files/
+              "$STACK.yaml" root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/
           fi
 
           # Bootstrap-tier guard: file synced to host, deploy is MANUAL
@@ -326,7 +336,7 @@ steps:
           esac
 
           # Tier 2: auto-deploy
-          ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} \
+          ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} \
             "bash /volume1/docker/compose-files/deploy/stack-deploy.sh $STACK" \
             && echo "  OK $STACK" || { echo "  FAIL $STACK"; exit 1; }
         done
@@ -344,23 +354,47 @@ steps:
       - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa
       - chmod 600 ~/.ssh/id_rsa
       - |
-        if [ -z "${SWARM_MANAGER_IP}" ]; then
+        if [ -z "$${SWARM_MANAGER_IP}" ]; then
           echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets."
           exit 1
         fi
-      - ssh-keyscan -H ${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
+      - ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
       - |
         CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
         FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
         FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true)
         ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
         [ -z "$ALL_STACKS" ] && exit 0
-        sleep 5
+
+        # NOTE: `docker stack deploy` briefly tears down and recreates tasks in
+        # Swarm's internal bookkeeping, so `docker stack ps` can transiently
+        # return nothing right after deploy even when the service is healthy.
+        # A single `sleep 5` + one-shot check produced false-alarm-looking
+        # "nothing found in stack" output on ordinary deploys (e.g. vaultwarden,
+        # 2026-08-25). Retry with backoff instead of a single fixed sleep, and
+        # only warn (don't fail the pipeline) if tasks never show up.
+        ATTEMPTS=6
+        DELAY=5
         for STACK in $ALL_STACKS; do
           echo "--- $STACK ---"
-          ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} \
+          i=1
+          while [ "$i" -le "$ATTEMPTS" ]; do
+            OUTPUT=$(ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} \
               "docker stack ps $STACK --filter desired-state=running \
-            --format '  {{.Name}} {{.CurrentState}}'"
+              --format '  {{.Name}} {{.CurrentState}}'" 2>/dev/null)
+            if [ -n "$OUTPUT" ]; then
+              echo "$OUTPUT"
+              break
+            fi
+            if [ "$i" -eq "$ATTEMPTS" ]; then
+              echo "  WARNING: no running tasks found for $STACK after $((ATTEMPTS * DELAY))s."
+              echo "  This may be transient Swarm settle time, or a real problem — check manually:"
+              echo "  ssh root@$${SWARM_MANAGER_IP} 'docker stack ps $STACK --no-trunc'"
+            else
+              sleep "$DELAY"
+            fi
+            i=$((i + 1))
+          done
         done
 
   notify-success: