From 6757b0dfe104ca0f2466b168386a56c7ea8bb968 Mon Sep 17 00:00:00 2001 From: admin Date: Mon, 27 Jul 2026 12:50:21 -0700 Subject: [PATCH] deploy.yml: THE ACTUAL ROOT CAUSE FIX. Woodpecker substitutes ${VAR} at pipeline-compile time from its CI metadata map, where secrets do not exist -- so every braced reference to a secret-backed env var (SWARM_MANAGER_IP, all provisioning values) was silently blanked before the shell ever ran. Proven by debug output showing eval "val=\$$v" arriving as val=\$v ($$ unescaped to $) and lengths matching var NAMES. Fix: escape all runtime secret refs as $${VAR}; keep ${CI_PIPELINE_FILES} braced (compile-time is correct for CI metadata). Removes flawed debug step; adds header documenting the substitution rules. --- .woodpecker/deploy.yml | 217 ++++++++++++++--------------------------- 1 file changed, 75 insertions(+), 142 deletions(-) diff --git a/.woodpecker/deploy.yml b/.woodpecker/deploy.yml index ebe4648..0077d71 100644 --- a/.woodpecker/deploy.yml +++ b/.woodpecker/deploy.yml @@ -2,93 +2,28 @@ when: - event: push branch: main -steps: - debug-secrets-all: - image: alpine:latest - environment: - SSH_KEY: - from_secret: ssh_key - SWARM_MANAGER_IP: - from_secret: swarm_manager_ip - PRINT3D_DB_PASSWORD: - from_secret: 3dprint_db_password - GAMMA_AUTH_TOKEN: - from_secret: gamma_auth_token - GIT_DB_PASSWORD: - from_secret: git_db_password - GIT_RUNNER_TOKEN: - from_secret: git_runner_token - GIT_MCP_ACCESS_TOKEN: - from_secret: git_mcp_access_token - HOMEASSISTANT_FRIGATE_RTSP_PASSWORD: - from_secret: homeassistant_frigate_rtsp_password - HOMEASSISTANT_IMMICH_API_KEY: - from_secret: homeassistant_immich_api_key - IMMICH_DB_PASSWORD: - from_secret: immich_db_password - IMMICH_KIOSK_BASICAUTH: - from_secret: immich_kiosk_basicauth - MEALIE_DB_PASSWORD: - from_secret: mealie_db_password - MEALIE_LDAP_QUERY_PASSWORD: - from_secret: mealie_ldap_query_password - MESHCENTRAL_BACKUP_PASSWORD: - from_secret: meshcentral_backup_password - N8N_DB_PASSWORD: - from_secret: n8n_db_password - N8N_ENCRYPTION_KEY: - from_secret: n8n_encryption_key - POSTGRESQL_PASSWORD: - from_secret: postgresql_password - POSTGRESQL_PGADMIN_PASSWORD: - from_secret: postgresql_pgadmin_password - PRODUCTIVITY_PAPERLESS_SECRET_KEY: - from_secret: productivity_paperless_secret_key - PRODUCTIVITY_DB_PASSWORD: - from_secret: productivity_db_password - PRODUCTIVITY_OIDC_PROVIDERS: - from_secret: productivity_oidc_providers - VAULTWARDEN_ADMIN_TOKEN: - from_secret: vaultwarden_admin_token - VAULTWARDEN_DATABASE_URL: - from_secret: vaultwarden_database_url - ENTERTAINMENT_DISCORD_TOKEN: - from_secret: entertainment_discord_token - ENTERTAINMENT_DISCORD_CLIENT_SECRET: - from_secret: entertainment_discord_client_secret - ENTERTAINMENT_SECRET_KEY_BASE: - from_secret: entertainment_secret_key_base - ENTERTAINMENT_BASIC_AUTH_PASSWORD: - from_secret: entertainment_basic_auth_password - ENTERTAINMENT_SPARKY_DB_PASSWORD: - from_secret: entertainment_sparky_db_password - ENTERTAINMENT_SPARKY_APP_DB_PASSWORD: - from_secret: entertainment_sparky_app_db_password - ENTERTAINMENT_SPARKY_ENCRYPTION_KEY: - from_secret: entertainment_sparky_encryption_key - ENTERTAINMENT_BETTER_AUTH_SECRET: - from_secret: entertainment_better_auth_secret - commands: - - | - echo "=== DIAGNOSTIC: byte lengths of ALL provision-secrets vars (temporary, never prints values) ===" - for v in SSH_KEY SWARM_MANAGER_IP PRINT3D_DB_PASSWORD GAMMA_AUTH_TOKEN \ - GIT_DB_PASSWORD GIT_RUNNER_TOKEN GIT_MCP_ACCESS_TOKEN \ - HOMEASSISTANT_FRIGATE_RTSP_PASSWORD HOMEASSISTANT_IMMICH_API_KEY \ - IMMICH_DB_PASSWORD IMMICH_KIOSK_BASICAUTH \ - MEALIE_DB_PASSWORD MEALIE_LDAP_QUERY_PASSWORD \ - MESHCENTRAL_BACKUP_PASSWORD N8N_DB_PASSWORD N8N_ENCRYPTION_KEY \ - POSTGRESQL_PASSWORD POSTGRESQL_PGADMIN_PASSWORD \ - PRODUCTIVITY_PAPERLESS_SECRET_KEY PRODUCTIVITY_DB_PASSWORD PRODUCTIVITY_OIDC_PROVIDERS \ - VAULTWARDEN_ADMIN_TOKEN VAULTWARDEN_DATABASE_URL \ - ENTERTAINMENT_DISCORD_TOKEN ENTERTAINMENT_DISCORD_CLIENT_SECRET \ - ENTERTAINMENT_SECRET_KEY_BASE ENTERTAINMENT_BASIC_AUTH_PASSWORD \ - ENTERTAINMENT_SPARKY_DB_PASSWORD ENTERTAINMENT_SPARKY_APP_DB_PASSWORD \ - ENTERTAINMENT_SPARKY_ENCRYPTION_KEY ENTERTAINMENT_BETTER_AUTH_SECRET; do - eval "val=\$$v" - echo "$v length: $(echo -n "$val" | wc -c)" - done - echo "=== END DIAGNOSTIC ===" +# ───────────────────────────────────────────────────────────────────────────── +# IMPORTANT — Woodpecker variable substitution rules (learned the hard way): +# +# Woodpecker pre-processes command strings BEFORE the shell runs them: +# - ${VAR} is substituted at pipeline-compile time from Woodpecker's own +# metadata (CI_* vars). Secrets DO NOT exist in that map, so any braced +# reference to a secret-backed env var silently becomes an EMPTY STRING. +# - $$ is unescaped to a single $ and passed to the shell. +# +# Therefore: +# - Use ${CI_...} (braced, unescaped) ONLY for Woodpecker metadata vars. +# - Use $${VAR} for anything that must be resolved by the shell at runtime +# (i.e., every from_secret-backed environment variable). +# - Bare $VAR (no braces) also works for runtime resolution, but $${VAR} +# is preferred for anything adjacent to other characters. +# +# This was the root cause of a long-running "SWARM_MANAGER_IP secret is +# empty" failure: the braced references were being blanked at compile time +# before the shell ever saw them. +# ───────────────────────────────────────────────────────────────────────────── +steps: validate: image: alpine:latest commands: @@ -96,16 +31,14 @@ steps: - | # Collect changed stacks — both flat files and folder-based # NOTE: Woodpecker 3.16 exposes changed files as CI_PIPELINE_FILES, a JSON - # array string (e.g. [".woodpecker/deploy.yml","vaultwarden/vaultwarden.yaml"]). - # The older CI_COMMIT_CHANGED_FILES (comma-separated) no longer exists. + # array string. CI_* metadata vars are correctly substituted at compile time. CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n') # Flat: any root-level *.yaml FLAT=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.ya?ml$' || true) # Folder: any file under a subfolder (e.g. immich/immich.yml). - # Exclude dotfolders (.woodpecker, .git, .gitea, etc.) -- these are never - # deployable stacks and were previously matching as bogus "stack names". + # Exclude dotfolders (.woodpecker, .git, .gitea, etc.) FOLDERS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true) [ -z "$FLAT" ] && [ -z "$FOLDERS" ] && echo "No stacks changed" && exit 0 @@ -204,18 +137,18 @@ steps: - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - | - if [ -z "${SWARM_MANAGER_IP}" ]; then + if [ -z "$${SWARM_MANAGER_IP}" ]; then echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets." exit 1 fi - - ssh-keyscan -H ${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true + - ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true - | CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n') FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true) FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true) ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u) [ -z "$ALL_STACKS" ] && echo "No stacks changed, skipping" && exit 0 - - scp -o StrictHostKeyChecking=no deploy/create-secrets.sh root@${SWARM_MANAGER_IP}:/tmp/cs.sh + - scp -o StrictHostKeyChecking=no deploy/create-secrets.sh root@$${SWARM_MANAGER_IP}:/tmp/cs.sh - | CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n') FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true) @@ -228,64 +161,64 @@ steps: maintenance|media|unifi|guacamole|security|auth|traefik|meshcentral|ddm) echo " No Docker secrets for $STACK — secrets in host .env";; immich) - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh - create_or_update_secret 'immich_db_password' '${IMMICH_DB_PASSWORD}' - create_or_update_secret 'immich_kiosk_basicauth' '${IMMICH_KIOSK_BASICAUTH}'";; + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh + create_or_update_secret 'immich_db_password' '$${IMMICH_DB_PASSWORD}' + create_or_update_secret 'immich_kiosk_basicauth' '$${IMMICH_KIOSK_BASICAUTH}'";; woodpecker) echo " Manual only — skipping";; 3dprint) - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh - create_or_update_secret '3dprint_db_password' '${PRINT3D_DB_PASSWORD}'";; + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh + create_or_update_secret '3dprint_db_password' '$${PRINT3D_DB_PASSWORD}'";; gamma) - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh - create_or_update_secret 'gamma_auth_token' '${GAMMA_AUTH_TOKEN}'";; + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh + create_or_update_secret 'gamma_auth_token' '$${GAMMA_AUTH_TOKEN}'";; git) - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh - create_or_update_secret 'git_db_password' '${GIT_DB_PASSWORD}' - create_or_update_secret 'git_runner_token' '${GIT_RUNNER_TOKEN}' - create_or_update_secret 'git_mcp_access_token' '${GIT_MCP_ACCESS_TOKEN}'";; + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh + create_or_update_secret 'git_db_password' '$${GIT_DB_PASSWORD}' + create_or_update_secret 'git_runner_token' '$${GIT_RUNNER_TOKEN}' + create_or_update_secret 'git_mcp_access_token' '$${GIT_MCP_ACCESS_TOKEN}'";; homeassistant) - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh - create_or_update_secret 'homeassistant_frigate_rtsp_password' '${HOMEASSISTANT_FRIGATE_RTSP_PASSWORD}' - create_or_update_secret 'homeassistant_immich_api_key' '${HOMEASSISTANT_IMMICH_API_KEY}'";; + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh + create_or_update_secret 'homeassistant_frigate_rtsp_password' '$${HOMEASSISTANT_FRIGATE_RTSP_PASSWORD}' + create_or_update_secret 'homeassistant_immich_api_key' '$${HOMEASSISTANT_IMMICH_API_KEY}'";; mealie) - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh - create_or_update_secret 'mealie_db_password' '${MEALIE_DB_PASSWORD}' - create_or_update_secret 'mealie_ldap_query_password' '${MEALIE_LDAP_QUERY_PASSWORD}'";; + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh + create_or_update_secret 'mealie_db_password' '$${MEALIE_DB_PASSWORD}' + create_or_update_secret 'mealie_ldap_query_password' '$${MEALIE_LDAP_QUERY_PASSWORD}'";; n8n) - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh - create_or_update_secret 'n8n_db_password' '${N8N_DB_PASSWORD}' - create_or_update_secret 'n8n_encryption_key' '${N8N_ENCRYPTION_KEY}'";; + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh + create_or_update_secret 'n8n_db_password' '$${N8N_DB_PASSWORD}' + create_or_update_secret 'n8n_encryption_key' '$${N8N_ENCRYPTION_KEY}'";; postgresql) - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh - create_or_update_secret 'postgresql_password' '${POSTGRESQL_PASSWORD}' - create_or_update_secret 'postgresql_pgadmin_password' '${POSTGRESQL_PGADMIN_PASSWORD}'";; + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh + create_or_update_secret 'postgresql_password' '$${POSTGRESQL_PASSWORD}' + create_or_update_secret 'postgresql_pgadmin_password' '$${POSTGRESQL_PGADMIN_PASSWORD}'";; productivity) - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh - create_or_update_secret 'productivity_paperless_secret_key' '${PRODUCTIVITY_PAPERLESS_SECRET_KEY}' - create_or_update_secret 'productivity_db_password' '${PRODUCTIVITY_DB_PASSWORD}' - create_or_update_secret 'productivity_oidc_providers' '${PRODUCTIVITY_OIDC_PROVIDERS}'";; + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh + create_or_update_secret 'productivity_paperless_secret_key' '$${PRODUCTIVITY_PAPERLESS_SECRET_KEY}' + create_or_update_secret 'productivity_db_password' '$${PRODUCTIVITY_DB_PASSWORD}' + create_or_update_secret 'productivity_oidc_providers' '$${PRODUCTIVITY_OIDC_PROVIDERS}'";; vaultwarden) # NOTE: vaultwarden_database_url cannot be rotated in-place — Swarm refuses # to `docker secret rm` a secret referenced by a running service's spec. # We provision under a versioned name (_v2) instead; vaultwarden.yaml maps # it back to the same in-container filename via `target:`. The old secret # is removed manually once the compose file cutover is confirmed healthy. - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh - create_or_update_secret 'vaultwarden_admin_token' '${VAULTWARDEN_ADMIN_TOKEN}' - create_or_update_secret 'vaultwarden_database_url_v2' '${VAULTWARDEN_DATABASE_URL}'";; + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh + create_or_update_secret 'vaultwarden_admin_token' '$${VAULTWARDEN_ADMIN_TOKEN}' + create_or_update_secret 'vaultwarden_database_url_v2' '$${VAULTWARDEN_DATABASE_URL}'";; ai) echo " No Docker secrets for ai -- secrets in host .env";; entertainment) - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh - create_or_update_secret 'entertainment_discord_token' '${ENTERTAINMENT_DISCORD_TOKEN}' - create_or_update_secret 'entertainment_discord_client_secret' '${ENTERTAINMENT_DISCORD_CLIENT_SECRET}' - create_or_update_secret 'entertainment_secret_key_base' '${ENTERTAINMENT_SECRET_KEY_BASE}' - create_or_update_secret 'entertainment_basic_auth_password' '${ENTERTAINMENT_BASIC_AUTH_PASSWORD}' - create_or_update_secret 'entertainment_sparky_db_password' '${ENTERTAINMENT_SPARKY_DB_PASSWORD}' - create_or_update_secret 'entertainment_sparky_app_db_password' '${ENTERTAINMENT_SPARKY_APP_DB_PASSWORD}' - create_or_update_secret 'entertainment_sparky_encryption_key' '${ENTERTAINMENT_SPARKY_ENCRYPTION_KEY}' - create_or_update_secret 'entertainment_better_auth_secret' '${ENTERTAINMENT_BETTER_AUTH_SECRET}'";; + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh + create_or_update_secret 'entertainment_discord_token' '$${ENTERTAINMENT_DISCORD_TOKEN}' + create_or_update_secret 'entertainment_discord_client_secret' '$${ENTERTAINMENT_DISCORD_CLIENT_SECRET}' + create_or_update_secret 'entertainment_secret_key_base' '$${ENTERTAINMENT_SECRET_KEY_BASE}' + create_or_update_secret 'entertainment_basic_auth_password' '$${ENTERTAINMENT_BASIC_AUTH_PASSWORD}' + create_or_update_secret 'entertainment_sparky_db_password' '$${ENTERTAINMENT_SPARKY_DB_PASSWORD}' + create_or_update_secret 'entertainment_sparky_app_db_password' '$${ENTERTAINMENT_SPARKY_APP_DB_PASSWORD}' + create_or_update_secret 'entertainment_sparky_encryption_key' '$${ENTERTAINMENT_SPARKY_ENCRYPTION_KEY}' + create_or_update_secret 'entertainment_better_auth_secret' '$${ENTERTAINMENT_BETTER_AUTH_SECRET}'";; *) echo " No secrets case for $STACK";; esac @@ -304,11 +237,11 @@ steps: - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - | - if [ -z "${SWARM_MANAGER_IP}" ]; then + if [ -z "$${SWARM_MANAGER_IP}" ]; then echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets." exit 1 fi - - ssh-keyscan -H ${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true + - ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true - | CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n') FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true) @@ -318,17 +251,17 @@ steps: # Always sync deploy/ scripts first rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \ - deploy/ root@${SWARM_MANAGER_IP}:/volume1/docker/compose-files/deploy/ + deploy/ root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/deploy/ for STACK in $ALL_STACKS; do echo "--- Deploying: $STACK ---" # Sync files to host first (always, even for bootstrap stacks) if [ -d "$STACK" ]; then rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \ - "$STACK/" root@${SWARM_MANAGER_IP}:/volume1/docker/compose-files/$STACK/ - elif [ -f "${STACK}.yaml" ]; then + "$STACK/" root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/$STACK/ + elif [ -f "$STACK.yaml" ]; then rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \ - "${STACK}.yaml" root@${SWARM_MANAGER_IP}:/volume1/docker/compose-files/ + "$STACK.yaml" root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/ fi # Bootstrap-tier guard: file synced to host, deploy is MANUAL @@ -340,7 +273,7 @@ steps: esac # Tier 2: auto-deploy - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} \ + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} \ "bash /volume1/docker/compose-files/deploy/stack-deploy.sh $STACK" \ && echo " OK $STACK" || { echo " FAIL $STACK"; exit 1; } done @@ -358,11 +291,11 @@ steps: - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - | - if [ -z "${SWARM_MANAGER_IP}" ]; then + if [ -z "$${SWARM_MANAGER_IP}" ]; then echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets." exit 1 fi - - ssh-keyscan -H ${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true + - ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true - | CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n') FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true) @@ -372,7 +305,7 @@ steps: sleep 5 for STACK in $ALL_STACKS; do echo "--- $STACK ---" - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} \ + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} \ "docker stack ps $STACK --filter desired-state=running \ --format ' {{.Name}} {{.CurrentState}}'" done