when: - event: push branch: main # ───────────────────────────────────────────────────────────────────────────── # IMPORTANT — Woodpecker variable substitution rules (learned the hard way): # # Woodpecker pre-processes the ENTIRE yaml text (comments included!) before # the shell runs anything: # - A dollar sign followed by a braced variable name is substituted at # pipeline-compile time from Woodpecker's own metadata (the CI_* vars). # Secrets DO NOT exist in that map, so a braced reference to a # secret-backed env var silently becomes an EMPTY STRING. # - A double dollar sign is unescaped to a single dollar sign and passed # through to the shell untouched. # # Therefore: # - Braced, single-dollar form: ONLY for Woodpecker CI_* metadata vars. # - Double-dollar braced form: for everything that must be resolved by the # shell at runtime (i.e., every from_secret-backed environment variable). # - Bare single-dollar VAR (no braces) also passes through to the shell. # - NEVER write a literal dollar-brace sequence in comments either — the # substitution engine parses comments too and will fail the pipeline # with "missing closing brace" on anything it cannot parse. # # This was the root cause of a long-running "SWARM_MANAGER_IP secret is # empty" failure: braced references were blanked at compile time before the # shell ever saw them. # ───────────────────────────────────────────────────────────────────────────── steps: validate: image: alpine:latest commands: - apk add --no-cache docker-cli docker-cli-compose - | # Collect changed stacks — both flat files and folder-based # NOTE: Woodpecker 3.16 exposes changed files as CI_PIPELINE_FILES, a JSON # array string. CI_* metadata vars are correctly substituted at compile time. CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n') # Flat: any root-level *.yaml FLAT=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.ya?ml$' || true) # Folder: any file under a subfolder (e.g. immich/immich.yml). # Exclude dotfolders (.woodpecker, .git, .gitea, etc.) FOLDERS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true) [ -z "$FLAT" ] && [ -z "$FOLDERS" ] && echo "No stacks changed" && exit 0 for f in $FLAT; do [ -f "$f" ] || continue docker compose -f "$f" config --no-interpolate -q \ && echo " OK $f" || { echo " FAIL $f"; exit 1; } done for d in $FOLDERS; do [ -d "$d" ] || continue # Build -f flags for all compose files in folder, main first MAIN="" EXTRAS="" for ext in yml yaml; do [ -f "$d/$d.$ext" ] && MAIN="$d/$d.$ext" && break done [ -z "$MAIN" ] && echo " SKIP $d (no main compose file)" && continue for cf in $(find "$d" -maxdepth 1 \( -name '*.yml' -o -name '*.yaml' \) | sort); do [ "$cf" = "$MAIN" ] && continue EXTRAS="$EXTRAS -f $cf" done docker compose -f "$MAIN" $EXTRAS config --no-interpolate -q \ && echo " OK $d/" || { echo " FAIL $d/"; exit 1; } done provision-secrets: image: alpine:latest environment: SSH_KEY: from_secret: ssh_key SWARM_MANAGER_IP: from_secret: swarm_manager_ip PRINT3D_DB_PASSWORD: from_secret: 3dprint_db_password GAMMA_AUTH_TOKEN: from_secret: gamma_auth_token GIT_DB_PASSWORD: from_secret: git_db_password GIT_RUNNER_TOKEN: from_secret: git_runner_token GIT_MCP_ACCESS_TOKEN: from_secret: git_mcp_access_token HOMEASSISTANT_FRIGATE_RTSP_PASSWORD: from_secret: homeassistant_frigate_rtsp_password HOMEASSISTANT_IMMICH_API_KEY: from_secret: homeassistant_immich_api_key IMMICH_DB_PASSWORD: from_secret: immich_db_password IMMICH_KIOSK_BASICAUTH: from_secret: immich_kiosk_basicauth MEALIE_DB_PASSWORD: from_secret: mealie_db_password MEALIE_LDAP_QUERY_PASSWORD: from_secret: mealie_ldap_query_password MESHCENTRAL_BACKUP_PASSWORD: from_secret: meshcentral_backup_password N8N_DB_PASSWORD: from_secret: n8n_db_password N8N_ENCRYPTION_KEY: from_secret: n8n_encryption_key POSTGRESQL_PASSWORD: from_secret: postgresql_password POSTGRESQL_PGADMIN_PASSWORD: from_secret: postgresql_pgadmin_password POSTGRESQL_REPLICATION_PASSWORD: from_secret: postgresql_replication_password POSTGRESQL_PATRONI_PASSWORD: from_secret: postgresql_patroni_password PRODUCTIVITY_PAPERLESS_SECRET_KEY: from_secret: productivity_paperless_secret_key PRODUCTIVITY_DB_PASSWORD: from_secret: productivity_db_password PRODUCTIVITY_OIDC_PROVIDERS: from_secret: productivity_oidc_providers VAULTWARDEN_ADMIN_TOKEN: from_secret: vaultwarden_admin_token VAULTWARDEN_DATABASE_URL: from_secret: vaultwarden_database_url ENTERTAINMENT_DISCORD_TOKEN: from_secret: entertainment_discord_token ENTERTAINMENT_DISCORD_CLIENT_SECRET: from_secret: entertainment_discord_client_secret ENTERTAINMENT_SECRET_KEY_BASE: from_secret: entertainment_secret_key_base ENTERTAINMENT_BASIC_AUTH_PASSWORD: from_secret: entertainment_basic_auth_password ENTERTAINMENT_SPARKY_DB_PASSWORD: from_secret: entertainment_sparky_db_password ENTERTAINMENT_SPARKY_APP_DB_PASSWORD: from_secret: entertainment_sparky_app_db_password ENTERTAINMENT_SPARKY_ENCRYPTION_KEY: from_secret: entertainment_sparky_encryption_key ENTERTAINMENT_BETTER_AUTH_SECRET: from_secret: entertainment_better_auth_secret commands: - apk add --no-cache openssh-client - mkdir -p ~/.ssh - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - | if [ -z "$${SWARM_MANAGER_IP}" ]; then echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets." exit 1 fi - ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true - | CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n') FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true) FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true) ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u) [ -z "$ALL_STACKS" ] && echo "No stacks changed, skipping" && exit 0 - scp -o StrictHostKeyChecking=no deploy/create-secrets.sh root@$${SWARM_MANAGER_IP}:/tmp/cs.sh - | CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n') FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true) FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true) ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u) for STACK in $ALL_STACKS; do echo "Provisioning: $STACK" case "$STACK" in maintenance|media|unifi|guacamole|security|auth|traefik|meshcentral|ddm) echo " No Docker secrets for $STACK — secrets in host .env";; immich) ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh create_or_update_secret 'immich_db_password' '$${IMMICH_DB_PASSWORD}' create_or_update_secret 'immich_kiosk_basicauth' '$${IMMICH_KIOSK_BASICAUTH}'";; woodpecker) echo " Manual only — skipping";; 3dprint) ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh create_or_update_secret '3dprint_db_password' '$${PRINT3D_DB_PASSWORD}'";; gamma) ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh create_or_update_secret 'gamma_auth_token' '$${GAMMA_AUTH_TOKEN}'";; git) ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh create_or_update_secret 'git_db_password' '$${GIT_DB_PASSWORD}' create_or_update_secret 'git_runner_token' '$${GIT_RUNNER_TOKEN}' create_or_update_secret 'git_mcp_access_token' '$${GIT_MCP_ACCESS_TOKEN}'";; homeassistant) ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh create_or_update_secret 'homeassistant_frigate_rtsp_password' '$${HOMEASSISTANT_FRIGATE_RTSP_PASSWORD}' create_or_update_secret 'homeassistant_immich_api_key' '$${HOMEASSISTANT_IMMICH_API_KEY}'";; mealie) ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh create_or_update_secret 'mealie_db_password' '$${MEALIE_DB_PASSWORD}' create_or_update_secret 'mealie_ldap_query_password' '$${MEALIE_LDAP_QUERY_PASSWORD}'";; n8n) ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh create_or_update_secret 'n8n_db_password' '$${N8N_DB_PASSWORD}' create_or_update_secret 'n8n_encryption_key' '$${N8N_ENCRYPTION_KEY}'";; postgresql) # NOTE: bootstrap-tier stack (manual deploy only via stack-deploy.sh). # Secrets are still auto-provisioned here so they exist on the host # before the manual `stack-deploy.sh postgresql` run picks them up. ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh create_or_update_secret 'postgresql_password' '$${POSTGRESQL_PASSWORD}' create_or_update_secret 'postgresql_pgadmin_password' '$${POSTGRESQL_PGADMIN_PASSWORD}' create_or_update_secret 'postgresql_replication_password' '$${POSTGRESQL_REPLICATION_PASSWORD}' create_or_update_secret 'postgresql_patroni_password' '$${POSTGRESQL_PATRONI_PASSWORD}'";; productivity) ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh create_or_update_secret 'productivity_paperless_secret_key' '$${PRODUCTIVITY_PAPERLESS_SECRET_KEY}' create_or_update_secret 'productivity_db_password' '$${PRODUCTIVITY_DB_PASSWORD}' create_or_update_secret 'productivity_oidc_providers' '$${PRODUCTIVITY_OIDC_PROVIDERS}'";; vaultwarden) # NOTE: vaultwarden_database_url cannot be rotated in-place — Swarm refuses # to remove a secret referenced by a running service's spec. We provision # under a versioned name instead; vaultwarden.yaml maps it back to the same # in-container filename via target. The old secret is removed manually once # the compose file cutover is confirmed healthy. ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh create_or_update_secret 'vaultwarden_admin_token' '$${VAULTWARDEN_ADMIN_TOKEN}' create_or_update_secret 'vaultwarden_database_url_v2' '$${VAULTWARDEN_DATABASE_URL}'";; ai) echo " No Docker secrets for ai -- secrets in host .env";; entertainment) ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh create_or_update_secret 'entertainment_discord_token' '$${ENTERTAINMENT_DISCORD_TOKEN}' create_or_update_secret 'entertainment_discord_client_secret' '$${ENTERTAINMENT_DISCORD_CLIENT_SECRET}' create_or_update_secret 'entertainment_secret_key_base' '$${ENTERTAINMENT_SECRET_KEY_BASE}' create_or_update_secret 'entertainment_basic_auth_password' '$${ENTERTAINMENT_BASIC_AUTH_PASSWORD}' create_or_update_secret 'entertainment_sparky_db_password' '$${ENTERTAINMENT_SPARKY_DB_PASSWORD}' create_or_update_secret 'entertainment_sparky_app_db_password' '$${ENTERTAINMENT_SPARKY_APP_DB_PASSWORD}' create_or_update_secret 'entertainment_sparky_encryption_key' '$${ENTERTAINMENT_SPARKY_ENCRYPTION_KEY}' create_or_update_secret 'entertainment_better_auth_secret' '$${ENTERTAINMENT_BETTER_AUTH_SECRET}'";; *) echo " No secrets case for $STACK";; esac done deploy: image: alpine:latest environment: SSH_KEY: from_secret: ssh_key SWARM_MANAGER_IP: from_secret: swarm_manager_ip commands: - apk add --no-cache openssh-client rsync - mkdir -p ~/.ssh - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - | if [ -z "$${SWARM_MANAGER_IP}" ]; then echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets." exit 1 fi - ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true - | CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n') FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true) FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true) ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u) [ -z "$ALL_STACKS" ] && echo "No stacks changed" && exit 0 # Always sync deploy/ scripts first rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \ deploy/ root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/deploy/ for STACK in $ALL_STACKS; do echo "--- Deploying: $STACK ---" # Sync files to host first (always, even for bootstrap stacks) if [ -d "$STACK" ]; then rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \ "$STACK/" root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/$STACK/ elif [ -f "$STACK.yaml" ]; then rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \ "$STACK.yaml" root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/ fi # Bootstrap-tier guard: file synced to host, deploy is MANUAL case "$STACK" in traefik|woodpecker|postgresql) echo " [BOOTSTRAP] $STACK: file synced. Deploy is MANUAL." echo " Run: bash /volume1/docker/compose-files/deploy/stack-deploy.sh $STACK" continue ;; esac # Tier 2: auto-deploy ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} \ "bash /volume1/docker/compose-files/deploy/stack-deploy.sh $STACK" \ && echo " OK $STACK" || { echo " FAIL $STACK"; exit 1; } done verify: image: alpine:latest environment: SSH_KEY: from_secret: ssh_key SWARM_MANAGER_IP: from_secret: swarm_manager_ip commands: - apk add --no-cache openssh-client - mkdir -p ~/.ssh - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - | if [ -z "$${SWARM_MANAGER_IP}" ]; then echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets." exit 1 fi - ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true - | CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n') FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true) FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true) ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u) [ -z "$ALL_STACKS" ] && exit 0 sleep 5 for STACK in $ALL_STACKS; do echo "--- $STACK ---" ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} \ "docker stack ps $STACK --filter desired-state=running \ --format ' {{.Name}} {{.CurrentState}}'" done notify-success: image: alpine:latest environment: TEAMS_WEBHOOK: from_secret: teams_webhook commands: - apk add --no-cache curl - | CHANGED=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' ' ') PAYLOAD=$(printf '{"title":"Swarm Deploy OK","text":"Stacks: %s Commit: %s","themeColor":"00aa00"}' "$CHANGED" "$CI_COMMIT_MESSAGE") curl -sf -X POST -H 'Content-Type: application/json' -d "$PAYLOAD" "$TEAMS_WEBHOOK" || true when: - status: success notify-failure: image: alpine:latest environment: TEAMS_WEBHOOK: from_secret: teams_webhook WOODPECKER_HOST: from_secret: woodpecker_host commands: - apk add --no-cache curl - | PAYLOAD=$(printf '{"title":"Swarm Deploy FAILED","text":"Commit: %s See: https://%s","themeColor":"ff0000"}' "$CI_COMMIT_MESSAGE" "$WOODPECKER_HOST") curl -sf -X POST -H 'Content-Type: application/json' -d "$PAYLOAD" "$TEAMS_WEBHOOK" || true when: - status: failure