diff --git a/.woodpecker/deploy.yml b/.woodpecker/deploy.yml index 2b54170..5dc1abf 100644 --- a/.woodpecker/deploy.yml +++ b/.woodpecker/deploy.yml @@ -26,6 +26,16 @@ when: # This was the root cause of a long-running "SWARM_MANAGER_IP secret is # empty" failure: braced references were blanked at compile time before the # shell ever saw them. +# +# 2026-08-26 HOTFIX: a full-file rewrite (AI secrets migration PR) dropped +# one $ from every $${VAR} occurrence throughout this file, re-introducing +# exactly the bug described above for EVERY secret-backed var, not just the +# new AI ones. The if-empty guards caught it immediately (SWARM_MANAGER_IP +# came back blank) and aborted before any ssh/scp/rsync ran, so no live +# secret or service was touched — but no CI provisioning/deploy could run +# until this was restored. Lesson: grep for the literal string '$${' and +# diff the count against the previous version before ever committing a +# full-file rewrite of this pipeline. # ───────────────────────────────────────────────────────────────────────────── steps: @@ -162,18 +172,18 @@ steps: - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - | - if [ -z "${SWARM_MANAGER_IP}" ]; then + if [ -z "$${SWARM_MANAGER_IP}" ]; then echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets." exit 1 fi - - ssh-keyscan -H ${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true + - ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true - | CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n') FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true) FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true) ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u) [ -z "$ALL_STACKS" ] && echo "No stacks changed, skipping" && exit 0 - - scp -o StrictHostKeyChecking=no deploy/create-secrets.sh root@${SWARM_MANAGER_IP}:/tmp/cs.sh + - scp -o StrictHostKeyChecking=no deploy/create-secrets.sh root@$${SWARM_MANAGER_IP}:/tmp/cs.sh - | CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n') FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true) @@ -186,92 +196,92 @@ steps: maintenance|media|unifi|guacamole|security|auth|traefik|meshcentral|ddm) echo " No Docker secrets for $STACK — secrets in host .env";; immich) - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh - create_or_update_secret 'immich_db_password' '${IMMICH_DB_PASSWORD}' - create_or_update_secret 'immich_kiosk_basicauth' '${IMMICH_KIOSK_BASICAUTH}'";; + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh + create_or_update_secret 'immich_db_password' '$${IMMICH_DB_PASSWORD}' + create_or_update_secret 'immich_kiosk_basicauth' '$${IMMICH_KIOSK_BASICAUTH}'";; woodpecker) echo " Manual only — skipping";; 3dprint) - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh - create_or_update_secret '3dprint_db_password' '${PRINT3D_DB_PASSWORD}'";; + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh + create_or_update_secret '3dprint_db_password' '$${PRINT3D_DB_PASSWORD}'";; gamma) - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh - create_or_update_secret 'gamma_auth_token' '${GAMMA_AUTH_TOKEN}'";; + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh + create_or_update_secret 'gamma_auth_token' '$${GAMMA_AUTH_TOKEN}'";; git) - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh - create_or_update_secret 'git_db_password' '${GIT_DB_PASSWORD}' - create_or_update_secret 'git_runner_token' '${GIT_RUNNER_TOKEN}' - create_or_update_secret 'git_mcp_access_token' '${GIT_MCP_ACCESS_TOKEN}'";; + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh + create_or_update_secret 'git_db_password' '$${GIT_DB_PASSWORD}' + create_or_update_secret 'git_runner_token' '$${GIT_RUNNER_TOKEN}' + create_or_update_secret 'git_mcp_access_token' '$${GIT_MCP_ACCESS_TOKEN}'";; homeassistant) - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh - create_or_update_secret 'homeassistant_frigate_rtsp_password' '${HOMEASSISTANT_FRIGATE_RTSP_PASSWORD}' - create_or_update_secret 'homeassistant_immich_api_key' '${HOMEASSISTANT_IMMICH_API_KEY}'";; + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh + create_or_update_secret 'homeassistant_frigate_rtsp_password' '$${HOMEASSISTANT_FRIGATE_RTSP_PASSWORD}' + create_or_update_secret 'homeassistant_immich_api_key' '$${HOMEASSISTANT_IMMICH_API_KEY}'";; mealie) - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh - create_or_update_secret 'mealie_db_password' '${MEALIE_DB_PASSWORD}' - create_or_update_secret 'mealie_ldap_query_password' '${MEALIE_LDAP_QUERY_PASSWORD}'";; + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh + create_or_update_secret 'mealie_db_password' '$${MEALIE_DB_PASSWORD}' + create_or_update_secret 'mealie_ldap_query_password' '$${MEALIE_LDAP_QUERY_PASSWORD}'";; n8n) - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh - create_or_update_secret 'n8n_db_password' '${N8N_DB_PASSWORD}' - create_or_update_secret 'n8n_encryption_key' '${N8N_ENCRYPTION_KEY}'";; + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh + create_or_update_secret 'n8n_db_password' '$${N8N_DB_PASSWORD}' + create_or_update_secret 'n8n_encryption_key' '$${N8N_ENCRYPTION_KEY}'";; postgresql) # NOTE: bootstrap-tier stack (manual deploy only via stack-deploy.sh). # Secrets are still auto-provisioned here so they exist on the host # before the manual `stack-deploy.sh postgresql` run picks them up. - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh - create_or_update_secret 'postgresql_password' '${POSTGRESQL_PASSWORD}' - create_or_update_secret 'postgresql_pgadmin_password' '${POSTGRESQL_PGADMIN_PASSWORD}' - create_or_update_secret 'postgresql_replication_password' '${POSTGRESQL_REPLICATION_PASSWORD}' - create_or_update_secret 'postgresql_patroni_password' '${POSTGRESQL_PATRONI_PASSWORD}'";; + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh + create_or_update_secret 'postgresql_password' '$${POSTGRESQL_PASSWORD}' + create_or_update_secret 'postgresql_pgadmin_password' '$${POSTGRESQL_PGADMIN_PASSWORD}' + create_or_update_secret 'postgresql_replication_password' '$${POSTGRESQL_REPLICATION_PASSWORD}' + create_or_update_secret 'postgresql_patroni_password' '$${POSTGRESQL_PATRONI_PASSWORD}'";; productivity) - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh - create_or_update_secret 'productivity_paperless_secret_key' '${PRODUCTIVITY_PAPERLESS_SECRET_KEY}' - create_or_update_secret 'productivity_db_password' '${PRODUCTIVITY_DB_PASSWORD}' - create_or_update_secret 'productivity_oidc_providers' '${PRODUCTIVITY_OIDC_PROVIDERS}'";; + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh + create_or_update_secret 'productivity_paperless_secret_key' '$${PRODUCTIVITY_PAPERLESS_SECRET_KEY}' + create_or_update_secret 'productivity_db_password' '$${PRODUCTIVITY_DB_PASSWORD}' + create_or_update_secret 'productivity_oidc_providers' '$${PRODUCTIVITY_OIDC_PROVIDERS}'";; vaultwarden) # NOTE: vaultwarden_database_url cannot be rotated in-place — Swarm refuses # to remove a secret referenced by a running service's spec. We provision # under a versioned name instead; vaultwarden.yaml maps it back to the same # in-container filename via target. The old secret is removed manually once # the compose file cutover is confirmed healthy. - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh - create_or_update_secret 'vaultwarden_admin_token' '${VAULTWARDEN_ADMIN_TOKEN}' - create_or_update_secret 'vaultwarden_database_url_v2' '${VAULTWARDEN_DATABASE_URL}'";; + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh + create_or_update_secret 'vaultwarden_admin_token' '$${VAULTWARDEN_ADMIN_TOKEN}' + create_or_update_secret 'vaultwarden_database_url_v2' '$${VAULTWARDEN_DATABASE_URL}'";; ai) # NOTE: ai stack uses Pattern B (host .env, not native Docker secrets) — # LiteLLM/boto3 and Open WebUI don't support the _FILE convention for these # vars. Instead of Docker secrets, we regenerate ONLY the migrated lines in # the remote ai/ai.env in place via grep -v + printf (never sed, since values - # may contain '/', '$', '&'). All other lines — including MCPO_API_KEY, - # OAUTH_CLIENT_ID, WEBUI_URL, and other non-secret config — are left - # completely untouched. MCPO_API_KEY migration is deferred to a follow-up; - # this step never reads or writes it. - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "FILE=/volume1/docker/compose-files/ai/ai.env + # may contain slash, dollar sign, ampersand). All other lines — including + # MCPO_API_KEY, OAUTH_CLIENT_ID, WEBUI_URL, and other non-secret config — are + # left completely untouched. MCPO_API_KEY migration is deferred to a + # follow-up; this step never reads or writes it. + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "FILE=/volume1/docker/compose-files/ai/ai.env TMP=\$FILE.tmp.\$\$ grep -vE '^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|LITELLM_MASTER_KEY|LITELLM_SALT_KEY|POSTGRES_PASSWORD|DATABASE_URL|WEBUI_SECRET_KEY|OPEN_WEBUI_DATABASE_URL|OAUTH_CLIENT_SECRET)=' \$FILE > \$TMP 2>/dev/null || touch \$TMP { cat \$TMP - printf 'AWS_ACCESS_KEY_ID=%s\n' '${AI_AWS_ACCESS_KEY_ID}' - printf 'AWS_SECRET_ACCESS_KEY=%s\n' '${AI_AWS_SECRET_ACCESS_KEY}' - printf 'LITELLM_MASTER_KEY=%s\n' '${AI_LITELLM_MASTER_KEY}' - printf 'LITELLM_SALT_KEY=%s\n' '${AI_LITELLM_SALT_KEY}' - printf 'POSTGRES_PASSWORD=%s\n' '${AI_LITELLM_DB_PASSWORD}' - printf 'DATABASE_URL=postgresql://LiteLLM:%s@postgresql:5432/litellm\n' '${AI_LITELLM_DB_PASSWORD}' - printf 'WEBUI_SECRET_KEY=%s\n' '${AI_WEBUI_SECRET_KEY}' - printf 'OPEN_WEBUI_DATABASE_URL=%s\n' '${AI_OPEN_WEBUI_DATABASE_URL}' - printf 'OAUTH_CLIENT_SECRET=%s\n' '${AI_OAUTH_CLIENT_SECRET}' + printf 'AWS_ACCESS_KEY_ID=%s\n' '$${AI_AWS_ACCESS_KEY_ID}' + printf 'AWS_SECRET_ACCESS_KEY=%s\n' '$${AI_AWS_SECRET_ACCESS_KEY}' + printf 'LITELLM_MASTER_KEY=%s\n' '$${AI_LITELLM_MASTER_KEY}' + printf 'LITELLM_SALT_KEY=%s\n' '$${AI_LITELLM_SALT_KEY}' + printf 'POSTGRES_PASSWORD=%s\n' '$${AI_LITELLM_DB_PASSWORD}' + printf 'DATABASE_URL=postgresql://LiteLLM:%s@postgresql:5432/litellm\n' '$${AI_LITELLM_DB_PASSWORD}' + printf 'WEBUI_SECRET_KEY=%s\n' '$${AI_WEBUI_SECRET_KEY}' + printf 'OPEN_WEBUI_DATABASE_URL=%s\n' '$${AI_OPEN_WEBUI_DATABASE_URL}' + printf 'OAUTH_CLIENT_SECRET=%s\n' '$${AI_OAUTH_CLIENT_SECRET}' } > \$FILE rm -f \$TMP echo ' [OK] ai/ai.env secrets updated'";; entertainment) - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} "source /tmp/cs.sh - create_or_update_secret 'entertainment_discord_token' '${ENTERTAINMENT_DISCORD_TOKEN}' - create_or_update_secret 'entertainment_discord_client_secret' '${ENTERTAINMENT_DISCORD_CLIENT_SECRET}' - create_or_update_secret 'entertainment_secret_key_base' '${ENTERTAINMENT_SECRET_KEY_BASE}' - create_or_update_secret 'entertainment_basic_auth_password' '${ENTERTAINMENT_BASIC_AUTH_PASSWORD}' - create_or_update_secret 'entertainment_sparky_db_password' '${ENTERTAINMENT_SPARKY_DB_PASSWORD}' - create_or_update_secret 'entertainment_sparky_app_db_password' '${ENTERTAINMENT_SPARKY_APP_DB_PASSWORD}' - create_or_update_secret 'entertainment_sparky_encryption_key' '${ENTERTAINMENT_SPARKY_ENCRYPTION_KEY}' - create_or_update_secret 'entertainment_better_auth_secret' '${ENTERTAINMENT_BETTER_AUTH_SECRET}'";; + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh + create_or_update_secret 'entertainment_discord_token' '$${ENTERTAINMENT_DISCORD_TOKEN}' + create_or_update_secret 'entertainment_discord_client_secret' '$${ENTERTAINMENT_DISCORD_CLIENT_SECRET}' + create_or_update_secret 'entertainment_secret_key_base' '$${ENTERTAINMENT_SECRET_KEY_BASE}' + create_or_update_secret 'entertainment_basic_auth_password' '$${ENTERTAINMENT_BASIC_AUTH_PASSWORD}' + create_or_update_secret 'entertainment_sparky_db_password' '$${ENTERTAINMENT_SPARKY_DB_PASSWORD}' + create_or_update_secret 'entertainment_sparky_app_db_password' '$${ENTERTAINMENT_SPARKY_APP_DB_PASSWORD}' + create_or_update_secret 'entertainment_sparky_encryption_key' '$${ENTERTAINMENT_SPARKY_ENCRYPTION_KEY}' + create_or_update_secret 'entertainment_better_auth_secret' '$${ENTERTAINMENT_BETTER_AUTH_SECRET}'";; *) echo " No secrets case for $STACK";; esac @@ -290,11 +300,11 @@ steps: - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - | - if [ -z "${SWARM_MANAGER_IP}" ]; then + if [ -z "$${SWARM_MANAGER_IP}" ]; then echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets." exit 1 fi - - ssh-keyscan -H ${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true + - ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true - | CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n') FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true) @@ -304,17 +314,17 @@ steps: # Always sync deploy/ scripts first rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \ - deploy/ root@${SWARM_MANAGER_IP}:/volume1/docker/compose-files/deploy/ + deploy/ root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/deploy/ for STACK in $ALL_STACKS; do echo "--- Deploying: $STACK ---" # Sync files to host first (always, even for bootstrap stacks) if [ -d "$STACK" ]; then rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \ - "$STACK/" root@${SWARM_MANAGER_IP}:/volume1/docker/compose-files/$STACK/ + "$STACK/" root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/$STACK/ elif [ -f "$STACK.yaml" ]; then rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \ - "$STACK.yaml" root@${SWARM_MANAGER_IP}:/volume1/docker/compose-files/ + "$STACK.yaml" root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/ fi # Bootstrap-tier guard: file synced to host, deploy is MANUAL @@ -326,7 +336,7 @@ steps: esac # Tier 2: auto-deploy - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} \ + ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} \ "bash /volume1/docker/compose-files/deploy/stack-deploy.sh $STACK" \ && echo " OK $STACK" || { echo " FAIL $STACK"; exit 1; } done @@ -344,23 +354,47 @@ steps: - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - | - if [ -z "${SWARM_MANAGER_IP}" ]; then + if [ -z "$${SWARM_MANAGER_IP}" ]; then echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets." exit 1 fi - - ssh-keyscan -H ${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true + - ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true - | CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n') FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true) FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true) ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u) [ -z "$ALL_STACKS" ] && exit 0 - sleep 5 + + # NOTE: `docker stack deploy` briefly tears down and recreates tasks in + # Swarm's internal bookkeeping, so `docker stack ps` can transiently + # return nothing right after deploy even when the service is healthy. + # A single `sleep 5` + one-shot check produced false-alarm-looking + # "nothing found in stack" output on ordinary deploys (e.g. vaultwarden, + # 2026-08-25). Retry with backoff instead of a single fixed sleep, and + # only warn (don't fail the pipeline) if tasks never show up. + ATTEMPTS=6 + DELAY=5 for STACK in $ALL_STACKS; do echo "--- $STACK ---" - ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} \ - "docker stack ps $STACK --filter desired-state=running \ - --format ' {{.Name}} {{.CurrentState}}'" + i=1 + while [ "$i" -le "$ATTEMPTS" ]; do + OUTPUT=$(ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} \ + "docker stack ps $STACK --filter desired-state=running \ + --format ' {{.Name}} {{.CurrentState}}'" 2>/dev/null) + if [ -n "$OUTPUT" ]; then + echo "$OUTPUT" + break + fi + if [ "$i" -eq "$ATTEMPTS" ]; then + echo " WARNING: no running tasks found for $STACK after $((ATTEMPTS * DELAY))s." + echo " This may be transient Swarm settle time, or a real problem — check manually:" + echo " ssh root@$${SWARM_MANAGER_IP} 'docker stack ps $STACK --no-trunc'" + else + sleep "$DELAY" + fi + i=$((i + 1)) + done done notify-success: