deploy.yml: THE ACTUAL ROOT CAUSE FIX. Woodpecker substitutes ${VAR} at pipeline-compile time from its CI metadata map, where secrets do not exist -- so every braced reference to a secret-backed env var (SWARM_MANAGER_IP, all provisioning values) was silently blanked before the shell ever ran. Proven by debug output showing eval "val=\$$v" arriving as val=\$v ($$ unescaped to $) and lengths matching var NAMES. Fix: escape all runtime secret refs as $${VAR}; keep ${CI_PIPELINE_FILES} braced (compile-time is correct for CI metadata). Removes flawed debug step; adds header documenting the substitution rules.

This commit is contained in:
2026-07-27 12:50:21 -07:00
parent 7a81bc73bd
commit 6757b0dfe1
+75 -142
View File
@@ -2,93 +2,28 @@ when:
- event: push
branch: main
steps:
debug-secrets-all:
image: alpine:latest
environment:
SSH_KEY:
from_secret: ssh_key
SWARM_MANAGER_IP:
from_secret: swarm_manager_ip
PRINT3D_DB_PASSWORD:
from_secret: 3dprint_db_password
GAMMA_AUTH_TOKEN:
from_secret: gamma_auth_token
GIT_DB_PASSWORD:
from_secret: git_db_password
GIT_RUNNER_TOKEN:
from_secret: git_runner_token
GIT_MCP_ACCESS_TOKEN:
from_secret: git_mcp_access_token
HOMEASSISTANT_FRIGATE_RTSP_PASSWORD:
from_secret: homeassistant_frigate_rtsp_password
HOMEASSISTANT_IMMICH_API_KEY:
from_secret: homeassistant_immich_api_key
IMMICH_DB_PASSWORD:
from_secret: immich_db_password
IMMICH_KIOSK_BASICAUTH:
from_secret: immich_kiosk_basicauth
MEALIE_DB_PASSWORD:
from_secret: mealie_db_password
MEALIE_LDAP_QUERY_PASSWORD:
from_secret: mealie_ldap_query_password
MESHCENTRAL_BACKUP_PASSWORD:
from_secret: meshcentral_backup_password
N8N_DB_PASSWORD:
from_secret: n8n_db_password
N8N_ENCRYPTION_KEY:
from_secret: n8n_encryption_key
POSTGRESQL_PASSWORD:
from_secret: postgresql_password
POSTGRESQL_PGADMIN_PASSWORD:
from_secret: postgresql_pgadmin_password
PRODUCTIVITY_PAPERLESS_SECRET_KEY:
from_secret: productivity_paperless_secret_key
PRODUCTIVITY_DB_PASSWORD:
from_secret: productivity_db_password
PRODUCTIVITY_OIDC_PROVIDERS:
from_secret: productivity_oidc_providers
VAULTWARDEN_ADMIN_TOKEN:
from_secret: vaultwarden_admin_token
VAULTWARDEN_DATABASE_URL:
from_secret: vaultwarden_database_url
ENTERTAINMENT_DISCORD_TOKEN:
from_secret: entertainment_discord_token
ENTERTAINMENT_DISCORD_CLIENT_SECRET:
from_secret: entertainment_discord_client_secret
ENTERTAINMENT_SECRET_KEY_BASE:
from_secret: entertainment_secret_key_base
ENTERTAINMENT_BASIC_AUTH_PASSWORD:
from_secret: entertainment_basic_auth_password
ENTERTAINMENT_SPARKY_DB_PASSWORD:
from_secret: entertainment_sparky_db_password
ENTERTAINMENT_SPARKY_APP_DB_PASSWORD:
from_secret: entertainment_sparky_app_db_password
ENTERTAINMENT_SPARKY_ENCRYPTION_KEY:
from_secret: entertainment_sparky_encryption_key
ENTERTAINMENT_BETTER_AUTH_SECRET:
from_secret: entertainment_better_auth_secret
commands:
- |
echo "=== DIAGNOSTIC: byte lengths of ALL provision-secrets vars (temporary, never prints values) ==="
for v in SSH_KEY SWARM_MANAGER_IP PRINT3D_DB_PASSWORD GAMMA_AUTH_TOKEN \
GIT_DB_PASSWORD GIT_RUNNER_TOKEN GIT_MCP_ACCESS_TOKEN \
HOMEASSISTANT_FRIGATE_RTSP_PASSWORD HOMEASSISTANT_IMMICH_API_KEY \
IMMICH_DB_PASSWORD IMMICH_KIOSK_BASICAUTH \
MEALIE_DB_PASSWORD MEALIE_LDAP_QUERY_PASSWORD \
MESHCENTRAL_BACKUP_PASSWORD N8N_DB_PASSWORD N8N_ENCRYPTION_KEY \
POSTGRESQL_PASSWORD POSTGRESQL_PGADMIN_PASSWORD \
PRODUCTIVITY_PAPERLESS_SECRET_KEY PRODUCTIVITY_DB_PASSWORD PRODUCTIVITY_OIDC_PROVIDERS \
VAULTWARDEN_ADMIN_TOKEN VAULTWARDEN_DATABASE_URL \
ENTERTAINMENT_DISCORD_TOKEN ENTERTAINMENT_DISCORD_CLIENT_SECRET \
ENTERTAINMENT_SECRET_KEY_BASE ENTERTAINMENT_BASIC_AUTH_PASSWORD \
ENTERTAINMENT_SPARKY_DB_PASSWORD ENTERTAINMENT_SPARKY_APP_DB_PASSWORD \
ENTERTAINMENT_SPARKY_ENCRYPTION_KEY ENTERTAINMENT_BETTER_AUTH_SECRET; do
eval "val=\$$v"
echo "$v length: $(echo -n "$val" | wc -c)"
done
echo "=== END DIAGNOSTIC ==="
# ─────────────────────────────────────────────────────────────────────────────
# IMPORTANT — Woodpecker variable substitution rules (learned the hard way):
#
# Woodpecker pre-processes command strings BEFORE the shell runs them:
# - ${VAR} is substituted at pipeline-compile time from Woodpecker's own
# metadata (CI_* vars). Secrets DO NOT exist in that map, so any braced
# reference to a secret-backed env var silently becomes an EMPTY STRING.
# - $$ is unescaped to a single $ and passed to the shell.
#
# Therefore:
# - Use ${CI_...} (braced, unescaped) ONLY for Woodpecker metadata vars.
# - Use $${VAR} for anything that must be resolved by the shell at runtime
# (i.e., every from_secret-backed environment variable).
# - Bare $VAR (no braces) also works for runtime resolution, but $${VAR}
# is preferred for anything adjacent to other characters.
#
# This was the root cause of a long-running "SWARM_MANAGER_IP secret is
# empty" failure: the braced references were being blanked at compile time
# before the shell ever saw them.
# ─────────────────────────────────────────────────────────────────────────────
steps:
validate:
image: alpine:latest
commands:
@@ -96,16 +31,14 @@ steps:
- |
# Collect changed stacks — both flat files and folder-based
# NOTE: Woodpecker 3.16 exposes changed files as CI_PIPELINE_FILES, a JSON
# array string (e.g. [".woodpecker/deploy.yml","vaultwarden/vaultwarden.yaml"]).
# The older CI_COMMIT_CHANGED_FILES (comma-separated) no longer exists.
# array string. CI_* metadata vars are correctly substituted at compile time.
CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
# Flat: any root-level *.yaml
FLAT=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.ya?ml$' || true)
# Folder: any file under a subfolder (e.g. immich/immich.yml).
# Exclude dotfolders (.woodpecker, .git, .gitea, etc.) -- these are never
# deployable stacks and were previously matching as bogus "stack names".
# Exclude dotfolders (.woodpecker, .git, .gitea, etc.)
FOLDERS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true)
[ -z "$FLAT" ] && [ -z "$FOLDERS" ] && echo "No stacks changed" && exit 0
@@ -204,18 +137,18 @@ steps:
- echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa
- |
if [ -z "${SWARM_MANAGER_IP}" ]; then
if [ -z "$${SWARM_MANAGER_IP}" ]; then
echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets."
exit 1
fi
- ssh-keyscan -H ${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
- ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
- |
CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true)
ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
[ -z "$ALL_STACKS" ] && echo "No stacks changed, skipping" && exit 0
- scp -o StrictHostKeyChecking=no deploy/create-secrets.sh root@${SWARM_MANAGER_IP}:/tmp/cs.sh
- scp -o StrictHostKeyChecking=no deploy/create-secrets.sh root@$${SWARM_MANAGER_IP}:/tmp/cs.sh
- |
CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
@@ -228,64 +161,64 @@ steps:
maintenance|media|unifi|guacamole|security|auth|traefik|meshcentral|ddm)
echo " No Docker secrets for $STACK — secrets in host .env";;
immich)
ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
create_or_update_secret 'immich_db_password' '${IMMICH_DB_PASSWORD}'
create_or_update_secret 'immich_kiosk_basicauth' '${IMMICH_KIOSK_BASICAUTH}'";;
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
create_or_update_secret 'immich_db_password' '$${IMMICH_DB_PASSWORD}'
create_or_update_secret 'immich_kiosk_basicauth' '$${IMMICH_KIOSK_BASICAUTH}'";;
woodpecker)
echo " Manual only — skipping";;
3dprint)
ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
create_or_update_secret '3dprint_db_password' '${PRINT3D_DB_PASSWORD}'";;
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
create_or_update_secret '3dprint_db_password' '$${PRINT3D_DB_PASSWORD}'";;
gamma)
ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
create_or_update_secret 'gamma_auth_token' '${GAMMA_AUTH_TOKEN}'";;
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
create_or_update_secret 'gamma_auth_token' '$${GAMMA_AUTH_TOKEN}'";;
git)
ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
create_or_update_secret 'git_db_password' '${GIT_DB_PASSWORD}'
create_or_update_secret 'git_runner_token' '${GIT_RUNNER_TOKEN}'
create_or_update_secret 'git_mcp_access_token' '${GIT_MCP_ACCESS_TOKEN}'";;
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
create_or_update_secret 'git_db_password' '$${GIT_DB_PASSWORD}'
create_or_update_secret 'git_runner_token' '$${GIT_RUNNER_TOKEN}'
create_or_update_secret 'git_mcp_access_token' '$${GIT_MCP_ACCESS_TOKEN}'";;
homeassistant)
ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
create_or_update_secret 'homeassistant_frigate_rtsp_password' '${HOMEASSISTANT_FRIGATE_RTSP_PASSWORD}'
create_or_update_secret 'homeassistant_immich_api_key' '${HOMEASSISTANT_IMMICH_API_KEY}'";;
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
create_or_update_secret 'homeassistant_frigate_rtsp_password' '$${HOMEASSISTANT_FRIGATE_RTSP_PASSWORD}'
create_or_update_secret 'homeassistant_immich_api_key' '$${HOMEASSISTANT_IMMICH_API_KEY}'";;
mealie)
ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
create_or_update_secret 'mealie_db_password' '${MEALIE_DB_PASSWORD}'
create_or_update_secret 'mealie_ldap_query_password' '${MEALIE_LDAP_QUERY_PASSWORD}'";;
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
create_or_update_secret 'mealie_db_password' '$${MEALIE_DB_PASSWORD}'
create_or_update_secret 'mealie_ldap_query_password' '$${MEALIE_LDAP_QUERY_PASSWORD}'";;
n8n)
ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
create_or_update_secret 'n8n_db_password' '${N8N_DB_PASSWORD}'
create_or_update_secret 'n8n_encryption_key' '${N8N_ENCRYPTION_KEY}'";;
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
create_or_update_secret 'n8n_db_password' '$${N8N_DB_PASSWORD}'
create_or_update_secret 'n8n_encryption_key' '$${N8N_ENCRYPTION_KEY}'";;
postgresql)
ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
create_or_update_secret 'postgresql_password' '${POSTGRESQL_PASSWORD}'
create_or_update_secret 'postgresql_pgadmin_password' '${POSTGRESQL_PGADMIN_PASSWORD}'";;
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
create_or_update_secret 'postgresql_password' '$${POSTGRESQL_PASSWORD}'
create_or_update_secret 'postgresql_pgadmin_password' '$${POSTGRESQL_PGADMIN_PASSWORD}'";;
productivity)
ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
create_or_update_secret 'productivity_paperless_secret_key' '${PRODUCTIVITY_PAPERLESS_SECRET_KEY}'
create_or_update_secret 'productivity_db_password' '${PRODUCTIVITY_DB_PASSWORD}'
create_or_update_secret 'productivity_oidc_providers' '${PRODUCTIVITY_OIDC_PROVIDERS}'";;
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
create_or_update_secret 'productivity_paperless_secret_key' '$${PRODUCTIVITY_PAPERLESS_SECRET_KEY}'
create_or_update_secret 'productivity_db_password' '$${PRODUCTIVITY_DB_PASSWORD}'
create_or_update_secret 'productivity_oidc_providers' '$${PRODUCTIVITY_OIDC_PROVIDERS}'";;
vaultwarden)
# NOTE: vaultwarden_database_url cannot be rotated in-place — Swarm refuses
# to `docker secret rm` a secret referenced by a running service's spec.
# We provision under a versioned name (_v2) instead; vaultwarden.yaml maps
# it back to the same in-container filename via `target:`. The old secret
# is removed manually once the compose file cutover is confirmed healthy.
ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
create_or_update_secret 'vaultwarden_admin_token' '${VAULTWARDEN_ADMIN_TOKEN}'
create_or_update_secret 'vaultwarden_database_url_v2' '${VAULTWARDEN_DATABASE_URL}'";;
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
create_or_update_secret 'vaultwarden_admin_token' '$${VAULTWARDEN_ADMIN_TOKEN}'
create_or_update_secret 'vaultwarden_database_url_v2' '$${VAULTWARDEN_DATABASE_URL}'";;
ai)
echo " No Docker secrets for ai -- secrets in host .env";;
entertainment)
ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh
create_or_update_secret 'entertainment_discord_token' '${ENTERTAINMENT_DISCORD_TOKEN}'
create_or_update_secret 'entertainment_discord_client_secret' '${ENTERTAINMENT_DISCORD_CLIENT_SECRET}'
create_or_update_secret 'entertainment_secret_key_base' '${ENTERTAINMENT_SECRET_KEY_BASE}'
create_or_update_secret 'entertainment_basic_auth_password' '${ENTERTAINMENT_BASIC_AUTH_PASSWORD}'
create_or_update_secret 'entertainment_sparky_db_password' '${ENTERTAINMENT_SPARKY_DB_PASSWORD}'
create_or_update_secret 'entertainment_sparky_app_db_password' '${ENTERTAINMENT_SPARKY_APP_DB_PASSWORD}'
create_or_update_secret 'entertainment_sparky_encryption_key' '${ENTERTAINMENT_SPARKY_ENCRYPTION_KEY}'
create_or_update_secret 'entertainment_better_auth_secret' '${ENTERTAINMENT_BETTER_AUTH_SECRET}'";;
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
create_or_update_secret 'entertainment_discord_token' '$${ENTERTAINMENT_DISCORD_TOKEN}'
create_or_update_secret 'entertainment_discord_client_secret' '$${ENTERTAINMENT_DISCORD_CLIENT_SECRET}'
create_or_update_secret 'entertainment_secret_key_base' '$${ENTERTAINMENT_SECRET_KEY_BASE}'
create_or_update_secret 'entertainment_basic_auth_password' '$${ENTERTAINMENT_BASIC_AUTH_PASSWORD}'
create_or_update_secret 'entertainment_sparky_db_password' '$${ENTERTAINMENT_SPARKY_DB_PASSWORD}'
create_or_update_secret 'entertainment_sparky_app_db_password' '$${ENTERTAINMENT_SPARKY_APP_DB_PASSWORD}'
create_or_update_secret 'entertainment_sparky_encryption_key' '$${ENTERTAINMENT_SPARKY_ENCRYPTION_KEY}'
create_or_update_secret 'entertainment_better_auth_secret' '$${ENTERTAINMENT_BETTER_AUTH_SECRET}'";;
*)
echo " No secrets case for $STACK";;
esac
@@ -304,11 +237,11 @@ steps:
- echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa
- |
if [ -z "${SWARM_MANAGER_IP}" ]; then
if [ -z "$${SWARM_MANAGER_IP}" ]; then
echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets."
exit 1
fi
- ssh-keyscan -H ${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
- ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
- |
CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
@@ -318,17 +251,17 @@ steps:
# Always sync deploy/ scripts first
rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
deploy/ root@${SWARM_MANAGER_IP}:/volume1/docker/compose-files/deploy/
deploy/ root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/deploy/
for STACK in $ALL_STACKS; do
echo "--- Deploying: $STACK ---"
# Sync files to host first (always, even for bootstrap stacks)
if [ -d "$STACK" ]; then
rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
"$STACK/" root@${SWARM_MANAGER_IP}:/volume1/docker/compose-files/$STACK/
elif [ -f "${STACK}.yaml" ]; then
"$STACK/" root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/$STACK/
elif [ -f "$STACK.yaml" ]; then
rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
"${STACK}.yaml" root@${SWARM_MANAGER_IP}:/volume1/docker/compose-files/
"$STACK.yaml" root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/
fi
# Bootstrap-tier guard: file synced to host, deploy is MANUAL
@@ -340,7 +273,7 @@ steps:
esac
# Tier 2: auto-deploy
ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} \
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} \
"bash /volume1/docker/compose-files/deploy/stack-deploy.sh $STACK" \
&& echo " OK $STACK" || { echo " FAIL $STACK"; exit 1; }
done
@@ -358,11 +291,11 @@ steps:
- echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa
- |
if [ -z "${SWARM_MANAGER_IP}" ]; then
if [ -z "$${SWARM_MANAGER_IP}" ]; then
echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets."
exit 1
fi
- ssh-keyscan -H ${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
- ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
- |
CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
@@ -372,7 +305,7 @@ steps:
sleep 5
for STACK in $ALL_STACKS; do
echo "--- $STACK ---"
ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} \
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} \
"docker stack ps $STACK --filter desired-state=running \
--format ' {{.Name}} {{.CurrentState}}'"
done