admin
6af2633936
ai.yaml's litellm service (as of commit 37ed671a, "Change AWS keys to
use Woodpecker Secrets") references ${AI_AWS_ACCESS_KEY_ID} /
${AI_AWS_SECRET_ACCESS_KEY} and renders them into the container as
plain AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY. The ai) provisioning
case added in the earlier secrets-migration PR wrote the plain
(unprefixed) names into ai.env instead, which would leave
${AI_AWS_ACCESS_KEY_ID} unresolved at compose-render time (renders
empty) -- silently breaking Bedrock auth in litellm on the next ai
stack deploy.
Fixed both the grep -vE exclusion pattern and the two printf lines to
use the AI_-prefixed names. All other migrated vars in ai.yaml use
plain names and are unaffected.
No other changes in this file.
438 lines
23 KiB
YAML
438 lines
23 KiB
YAML
when:
|
|
- event: push
|
|
branch: main
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# IMPORTANT — Woodpecker variable substitution rules (learned the hard way):
|
|
#
|
|
# Woodpecker pre-processes the ENTIRE yaml text (comments included!) before
|
|
# the shell runs anything:
|
|
# - A dollar sign followed by a braced variable name is substituted at
|
|
# pipeline-compile time from Woodpecker's own metadata (the CI_* vars).
|
|
# Secrets DO NOT exist in that map, so a braced reference to a
|
|
# secret-backed env var silently becomes an EMPTY STRING.
|
|
# - A double dollar sign is unescaped to a single dollar sign and passed
|
|
# through to the shell untouched.
|
|
#
|
|
# Therefore:
|
|
# - Braced, single-dollar form: ONLY for Woodpecker CI_* metadata vars.
|
|
# - Double-dollar braced form: for everything that must be resolved by the
|
|
# shell at runtime (i.e., every from_secret-backed environment variable).
|
|
# - Bare single-dollar VAR (no braces) also passes through to the shell.
|
|
# - NEVER write a literal dollar-brace sequence in comments either — the
|
|
# substitution engine parses comments too and will fail the pipeline
|
|
# with "missing closing brace" on anything it cannot parse.
|
|
#
|
|
# This was the root cause of a long-running "SWARM_MANAGER_IP secret is
|
|
# empty" failure: braced references were blanked at compile time before the
|
|
# shell ever saw them.
|
|
#
|
|
# 2026-08-26 HOTFIX: a full-file rewrite (AI secrets migration PR) dropped
|
|
# one $ from every $${VAR} occurrence throughout this file, re-introducing
|
|
# exactly the bug described above for EVERY secret-backed var, not just the
|
|
# new AI ones. The if-empty guards caught it immediately (SWARM_MANAGER_IP
|
|
# came back blank) and aborted before any ssh/scp/rsync ran, so no live
|
|
# secret or service was touched — but no CI provisioning/deploy could run
|
|
# until this was restored. Lesson: grep for the literal string '$${' and
|
|
# diff the count against the previous version before ever committing a
|
|
# full-file rewrite of this pipeline.
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
steps:
|
|
validate:
|
|
image: alpine:latest
|
|
commands:
|
|
- apk add --no-cache docker-cli docker-cli-compose
|
|
- |
|
|
# Collect changed stacks — both flat files and folder-based
|
|
# NOTE: Woodpecker 3.16 exposes changed files as CI_PIPELINE_FILES, a JSON
|
|
# array string. CI_* metadata vars are correctly substituted at compile time.
|
|
CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
|
|
|
|
# Flat: any root-level *.yaml
|
|
FLAT=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.ya?ml$' || true)
|
|
|
|
# Folder: any file under a subfolder (e.g. immich/immich.yml).
|
|
# Exclude dotfolders (.woodpecker, .git, .gitea, etc.)
|
|
FOLDERS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true)
|
|
|
|
[ -z "$FLAT" ] && [ -z "$FOLDERS" ] && echo "No stacks changed" && exit 0
|
|
|
|
for f in $FLAT; do
|
|
[ -f "$f" ] || continue
|
|
docker compose -f "$f" config --no-interpolate -q \
|
|
&& echo " OK $f" || { echo " FAIL $f"; exit 1; }
|
|
done
|
|
|
|
for d in $FOLDERS; do
|
|
[ -d "$d" ] || continue
|
|
# Build -f flags for all compose files in folder, main first
|
|
MAIN=""
|
|
EXTRAS=""
|
|
for ext in yml yaml; do
|
|
[ -f "$d/$d.$ext" ] && MAIN="$d/$d.$ext" && break
|
|
done
|
|
[ -z "$MAIN" ] && echo " SKIP $d (no main compose file)" && continue
|
|
for cf in $(find "$d" -maxdepth 1 \( -name '*.yml' -o -name '*.yaml' \) | sort); do
|
|
[ "$cf" = "$MAIN" ] && continue
|
|
EXTRAS="$EXTRAS -f $cf"
|
|
done
|
|
docker compose -f "$MAIN" $EXTRAS config --no-interpolate -q \
|
|
&& echo " OK $d/" || { echo " FAIL $d/"; exit 1; }
|
|
done
|
|
|
|
provision-secrets:
|
|
image: alpine:latest
|
|
environment:
|
|
SSH_KEY:
|
|
from_secret: ssh_key
|
|
SWARM_MANAGER_IP:
|
|
from_secret: swarm_manager_ip
|
|
PRINT3D_DB_PASSWORD:
|
|
from_secret: 3dprint_db_password
|
|
GAMMA_AUTH_TOKEN:
|
|
from_secret: gamma_auth_token
|
|
GIT_DB_PASSWORD:
|
|
from_secret: git_db_password
|
|
GIT_RUNNER_TOKEN:
|
|
from_secret: git_runner_token
|
|
GIT_MCP_ACCESS_TOKEN:
|
|
from_secret: git_mcp_access_token
|
|
HOMEASSISTANT_FRIGATE_RTSP_PASSWORD:
|
|
from_secret: homeassistant_frigate_rtsp_password
|
|
HOMEASSISTANT_IMMICH_API_KEY:
|
|
from_secret: homeassistant_immich_api_key
|
|
IMMICH_DB_PASSWORD:
|
|
from_secret: immich_db_password
|
|
IMMICH_KIOSK_BASICAUTH:
|
|
from_secret: immich_kiosk_basicauth
|
|
MEALIE_DB_PASSWORD:
|
|
from_secret: mealie_db_password
|
|
MEALIE_LDAP_QUERY_PASSWORD:
|
|
from_secret: mealie_ldap_query_password
|
|
MESHCENTRAL_BACKUP_PASSWORD:
|
|
from_secret: meshcentral_backup_password
|
|
N8N_DB_PASSWORD:
|
|
from_secret: n8n_db_password
|
|
N8N_ENCRYPTION_KEY:
|
|
from_secret: n8n_encryption_key
|
|
POSTGRESQL_PASSWORD:
|
|
from_secret: postgresql_password
|
|
POSTGRESQL_PGADMIN_PASSWORD:
|
|
from_secret: postgresql_pgadmin_password
|
|
POSTGRESQL_REPLICATION_PASSWORD:
|
|
from_secret: postgresql_replication_password
|
|
POSTGRESQL_PATRONI_PASSWORD:
|
|
from_secret: postgresql_patroni_password
|
|
PRODUCTIVITY_PAPERLESS_SECRET_KEY:
|
|
from_secret: productivity_paperless_secret_key
|
|
PRODUCTIVITY_DB_PASSWORD:
|
|
from_secret: productivity_db_password
|
|
PRODUCTIVITY_OIDC_PROVIDERS:
|
|
from_secret: productivity_oidc_providers
|
|
VAULTWARDEN_ADMIN_TOKEN:
|
|
from_secret: vaultwarden_admin_token
|
|
VAULTWARDEN_DATABASE_URL:
|
|
from_secret: vaultwarden_database_url
|
|
ENTERTAINMENT_DISCORD_TOKEN:
|
|
from_secret: entertainment_discord_token
|
|
ENTERTAINMENT_DISCORD_CLIENT_SECRET:
|
|
from_secret: entertainment_discord_client_secret
|
|
ENTERTAINMENT_SECRET_KEY_BASE:
|
|
from_secret: entertainment_secret_key_base
|
|
ENTERTAINMENT_BASIC_AUTH_PASSWORD:
|
|
from_secret: entertainment_basic_auth_password
|
|
ENTERTAINMENT_SPARKY_DB_PASSWORD:
|
|
from_secret: entertainment_sparky_db_password
|
|
ENTERTAINMENT_SPARKY_APP_DB_PASSWORD:
|
|
from_secret: entertainment_sparky_app_db_password
|
|
ENTERTAINMENT_SPARKY_ENCRYPTION_KEY:
|
|
from_secret: entertainment_sparky_encryption_key
|
|
ENTERTAINMENT_BETTER_AUTH_SECRET:
|
|
from_secret: entertainment_better_auth_secret
|
|
AI_AWS_ACCESS_KEY_ID:
|
|
from_secret: ai_aws_access_key_id
|
|
AI_AWS_SECRET_ACCESS_KEY:
|
|
from_secret: ai_aws_secret_access_key
|
|
AI_LITELLM_MASTER_KEY:
|
|
from_secret: ai_litellm_master_key
|
|
AI_LITELLM_SALT_KEY:
|
|
from_secret: ai_litellm_salt_key
|
|
AI_LITELLM_DB_PASSWORD:
|
|
from_secret: ai_litellm_db_password
|
|
AI_WEBUI_SECRET_KEY:
|
|
from_secret: ai_webui_secret_key
|
|
AI_OPEN_WEBUI_DATABASE_URL:
|
|
from_secret: ai_open_webui_database_url
|
|
AI_OAUTH_CLIENT_SECRET:
|
|
from_secret: ai_oauth_client_secret
|
|
commands:
|
|
- apk add --no-cache openssh-client
|
|
- mkdir -p ~/.ssh
|
|
- echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa
|
|
- chmod 600 ~/.ssh/id_rsa
|
|
- |
|
|
if [ -z "$${SWARM_MANAGER_IP}" ]; then
|
|
echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets."
|
|
exit 1
|
|
fi
|
|
- ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
|
|
- |
|
|
CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
|
|
FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
|
|
FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true)
|
|
ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
|
|
[ -z "$ALL_STACKS" ] && echo "No stacks changed, skipping" && exit 0
|
|
- scp -o StrictHostKeyChecking=no deploy/create-secrets.sh root@$${SWARM_MANAGER_IP}:/tmp/cs.sh
|
|
- |
|
|
CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
|
|
FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
|
|
FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true)
|
|
ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
|
|
|
|
for STACK in $ALL_STACKS; do
|
|
echo "Provisioning: $STACK"
|
|
case "$STACK" in
|
|
maintenance|media|unifi|guacamole|security|auth|traefik|meshcentral|ddm)
|
|
echo " No Docker secrets for $STACK — secrets in host .env";;
|
|
immich)
|
|
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
|
|
create_or_update_secret 'immich_db_password' '$${IMMICH_DB_PASSWORD}'
|
|
create_or_update_secret 'immich_kiosk_basicauth' '$${IMMICH_KIOSK_BASICAUTH}'";;
|
|
woodpecker)
|
|
echo " Manual only — skipping";;
|
|
3dprint)
|
|
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
|
|
create_or_update_secret '3dprint_db_password' '$${PRINT3D_DB_PASSWORD}'";;
|
|
gamma)
|
|
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
|
|
create_or_update_secret 'gamma_auth_token' '$${GAMMA_AUTH_TOKEN}'";;
|
|
git)
|
|
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
|
|
create_or_update_secret 'git_db_password' '$${GIT_DB_PASSWORD}'
|
|
create_or_update_secret 'git_runner_token' '$${GIT_RUNNER_TOKEN}'
|
|
create_or_update_secret 'git_mcp_access_token' '$${GIT_MCP_ACCESS_TOKEN}'";;
|
|
homeassistant)
|
|
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
|
|
create_or_update_secret 'homeassistant_frigate_rtsp_password' '$${HOMEASSISTANT_FRIGATE_RTSP_PASSWORD}'
|
|
create_or_update_secret 'homeassistant_immich_api_key' '$${HOMEASSISTANT_IMMICH_API_KEY}'";;
|
|
mealie)
|
|
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
|
|
create_or_update_secret 'mealie_db_password' '$${MEALIE_DB_PASSWORD}'
|
|
create_or_update_secret 'mealie_ldap_query_password' '$${MEALIE_LDAP_QUERY_PASSWORD}'";;
|
|
n8n)
|
|
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
|
|
create_or_update_secret 'n8n_db_password' '$${N8N_DB_PASSWORD}'
|
|
create_or_update_secret 'n8n_encryption_key' '$${N8N_ENCRYPTION_KEY}'";;
|
|
postgresql)
|
|
# NOTE: bootstrap-tier stack (manual deploy only via stack-deploy.sh).
|
|
# Secrets are still auto-provisioned here so they exist on the host
|
|
# before the manual `stack-deploy.sh postgresql` run picks them up.
|
|
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
|
|
create_or_update_secret 'postgresql_password' '$${POSTGRESQL_PASSWORD}'
|
|
create_or_update_secret 'postgresql_pgadmin_password' '$${POSTGRESQL_PGADMIN_PASSWORD}'
|
|
create_or_update_secret 'postgresql_replication_password' '$${POSTGRESQL_REPLICATION_PASSWORD}'
|
|
create_or_update_secret 'postgresql_patroni_password' '$${POSTGRESQL_PATRONI_PASSWORD}'";;
|
|
productivity)
|
|
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
|
|
create_or_update_secret 'productivity_paperless_secret_key' '$${PRODUCTIVITY_PAPERLESS_SECRET_KEY}'
|
|
create_or_update_secret 'productivity_db_password' '$${PRODUCTIVITY_DB_PASSWORD}'
|
|
create_or_update_secret 'productivity_oidc_providers' '$${PRODUCTIVITY_OIDC_PROVIDERS}'";;
|
|
vaultwarden)
|
|
# NOTE: vaultwarden_database_url cannot be rotated in-place — Swarm refuses
|
|
# to remove a secret referenced by a running service's spec. We provision
|
|
# under a versioned name instead; vaultwarden.yaml maps it back to the same
|
|
# in-container filename via target. The old secret is removed manually once
|
|
# the compose file cutover is confirmed healthy.
|
|
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
|
|
create_or_update_secret 'vaultwarden_admin_token' '$${VAULTWARDEN_ADMIN_TOKEN}'
|
|
create_or_update_secret 'vaultwarden_database_url_v2' '$${VAULTWARDEN_DATABASE_URL}'";;
|
|
ai)
|
|
# NOTE: ai stack uses Pattern B (host .env, not native Docker secrets) —
|
|
# LiteLLM/boto3 and Open WebUI don't support the _FILE convention for these
|
|
# vars. Instead of Docker secrets, we regenerate ONLY the migrated lines in
|
|
# the remote ai/ai.env in place via grep -v + printf (never sed, since values
|
|
# may contain slash, dollar sign, ampersand). All other lines — including
|
|
# MCPO_API_KEY, OAUTH_CLIENT_ID, WEBUI_URL, and other non-secret config — are
|
|
# left completely untouched. MCPO_API_KEY migration is deferred to a
|
|
# follow-up; this step never reads or writes it.
|
|
#
|
|
# IMPORTANT: ai.yaml's litellm service references ${AI_AWS_ACCESS_KEY_ID} /
|
|
# ${AI_AWS_SECRET_ACCESS_KEY} (AI_-prefixed) and renders them into the
|
|
# container as plain AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (see commit
|
|
# 37ed671a, "Change AWS keys to use Woodpecker Secrets"). So ai.env must be
|
|
# written with the AI_-prefixed key names, NOT the plain ones — writing
|
|
# plain AWS_ACCESS_KEY_ID here would leave ${AI_AWS_ACCESS_KEY_ID}
|
|
# unresolved at compose-render time (empty), silently breaking Bedrock auth.
|
|
# All other migrated vars in ai.yaml use plain (unprefixed) names, so only
|
|
# these two lines need the AI_ prefix.
|
|
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "FILE=/volume1/docker/compose-files/ai/ai.env
|
|
TMP=\$FILE.tmp.\$\$
|
|
grep -vE '^(AI_AWS_ACCESS_KEY_ID|AI_AWS_SECRET_ACCESS_KEY|LITELLM_MASTER_KEY|LITELLM_SALT_KEY|POSTGRES_PASSWORD|DATABASE_URL|WEBUI_SECRET_KEY|OPEN_WEBUI_DATABASE_URL|OAUTH_CLIENT_SECRET)=' \$FILE > \$TMP 2>/dev/null || touch \$TMP
|
|
{ cat \$TMP
|
|
printf 'AI_AWS_ACCESS_KEY_ID=%s\n' '$${AI_AWS_ACCESS_KEY_ID}'
|
|
printf 'AI_AWS_SECRET_ACCESS_KEY=%s\n' '$${AI_AWS_SECRET_ACCESS_KEY}'
|
|
printf 'LITELLM_MASTER_KEY=%s\n' '$${AI_LITELLM_MASTER_KEY}'
|
|
printf 'LITELLM_SALT_KEY=%s\n' '$${AI_LITELLM_SALT_KEY}'
|
|
printf 'POSTGRES_PASSWORD=%s\n' '$${AI_LITELLM_DB_PASSWORD}'
|
|
printf 'DATABASE_URL=postgresql://LiteLLM:%s@postgresql:5432/litellm\n' '$${AI_LITELLM_DB_PASSWORD}'
|
|
printf 'WEBUI_SECRET_KEY=%s\n' '$${AI_WEBUI_SECRET_KEY}'
|
|
printf 'OPEN_WEBUI_DATABASE_URL=%s\n' '$${AI_OPEN_WEBUI_DATABASE_URL}'
|
|
printf 'OAUTH_CLIENT_SECRET=%s\n' '$${AI_OAUTH_CLIENT_SECRET}'
|
|
} > \$FILE
|
|
rm -f \$TMP
|
|
echo ' [OK] ai/ai.env secrets updated'";;
|
|
entertainment)
|
|
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
|
|
create_or_update_secret 'entertainment_discord_token' '$${ENTERTAINMENT_DISCORD_TOKEN}'
|
|
create_or_update_secret 'entertainment_discord_client_secret' '$${ENTERTAINMENT_DISCORD_CLIENT_SECRET}'
|
|
create_or_update_secret 'entertainment_secret_key_base' '$${ENTERTAINMENT_SECRET_KEY_BASE}'
|
|
create_or_update_secret 'entertainment_basic_auth_password' '$${ENTERTAINMENT_BASIC_AUTH_PASSWORD}'
|
|
create_or_update_secret 'entertainment_sparky_db_password' '$${ENTERTAINMENT_SPARKY_DB_PASSWORD}'
|
|
create_or_update_secret 'entertainment_sparky_app_db_password' '$${ENTERTAINMENT_SPARKY_APP_DB_PASSWORD}'
|
|
create_or_update_secret 'entertainment_sparky_encryption_key' '$${ENTERTAINMENT_SPARKY_ENCRYPTION_KEY}'
|
|
create_or_update_secret 'entertainment_better_auth_secret' '$${ENTERTAINMENT_BETTER_AUTH_SECRET}'";;
|
|
*)
|
|
echo " No secrets case for $STACK";;
|
|
esac
|
|
done
|
|
|
|
deploy:
|
|
image: alpine:latest
|
|
environment:
|
|
SSH_KEY:
|
|
from_secret: ssh_key
|
|
SWARM_MANAGER_IP:
|
|
from_secret: swarm_manager_ip
|
|
commands:
|
|
- apk add --no-cache openssh-client rsync
|
|
- mkdir -p ~/.ssh
|
|
- echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa
|
|
- chmod 600 ~/.ssh/id_rsa
|
|
- |
|
|
if [ -z "$${SWARM_MANAGER_IP}" ]; then
|
|
echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets."
|
|
exit 1
|
|
fi
|
|
- ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
|
|
- |
|
|
CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
|
|
FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
|
|
FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true)
|
|
ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
|
|
[ -z "$ALL_STACKS" ] && echo "No stacks changed" && exit 0
|
|
|
|
# Always sync deploy/ scripts first
|
|
rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
|
|
deploy/ root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/deploy/
|
|
|
|
for STACK in $ALL_STACKS; do
|
|
echo "--- Deploying: $STACK ---"
|
|
# Sync files to host first (always, even for bootstrap stacks)
|
|
if [ -d "$STACK" ]; then
|
|
rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
|
|
"$STACK/" root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/$STACK/
|
|
elif [ -f "$STACK.yaml" ]; then
|
|
rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
|
|
"$STACK.yaml" root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/
|
|
fi
|
|
|
|
# Bootstrap-tier guard: file synced to host, deploy is MANUAL
|
|
case "$STACK" in
|
|
traefik|woodpecker|postgresql|secrets)
|
|
echo " [BOOTSTRAP] $STACK: file synced. Deploy is MANUAL."
|
|
echo " Run: bash /volume1/docker/compose-files/deploy/stack-deploy.sh $STACK"
|
|
continue ;;
|
|
esac
|
|
|
|
# Tier 2: auto-deploy
|
|
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} \
|
|
"bash /volume1/docker/compose-files/deploy/stack-deploy.sh $STACK" \
|
|
&& echo " OK $STACK" || { echo " FAIL $STACK"; exit 1; }
|
|
done
|
|
|
|
verify:
|
|
image: alpine:latest
|
|
environment:
|
|
SSH_KEY:
|
|
from_secret: ssh_key
|
|
SWARM_MANAGER_IP:
|
|
from_secret: swarm_manager_ip
|
|
commands:
|
|
- apk add --no-cache openssh-client
|
|
- mkdir -p ~/.ssh
|
|
- echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa
|
|
- chmod 600 ~/.ssh/id_rsa
|
|
- |
|
|
if [ -z "$${SWARM_MANAGER_IP}" ]; then
|
|
echo "ERROR: SWARM_MANAGER_IP secret is empty. Check Woodpecker repo secrets."
|
|
exit 1
|
|
fi
|
|
- ssh-keyscan -H $${SWARM_MANAGER_IP} >> ~/.ssh/known_hosts 2>/dev/null || true
|
|
- |
|
|
CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
|
|
FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
|
|
FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true)
|
|
ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
|
|
[ -z "$ALL_STACKS" ] && exit 0
|
|
|
|
# NOTE: `docker stack deploy` briefly tears down and recreates tasks in
|
|
# Swarm's internal bookkeeping, so `docker stack ps` can transiently
|
|
# return nothing right after deploy even when the service is healthy.
|
|
# A single `sleep 5` + one-shot check produced false-alarm-looking
|
|
# "nothing found in stack" output on ordinary deploys (e.g. vaultwarden,
|
|
# 2026-08-25). Retry with backoff instead of a single fixed sleep, and
|
|
# only warn (don't fail the pipeline) if tasks never show up.
|
|
ATTEMPTS=6
|
|
DELAY=5
|
|
for STACK in $ALL_STACKS; do
|
|
echo "--- $STACK ---"
|
|
i=1
|
|
while [ "$i" -le "$ATTEMPTS" ]; do
|
|
OUTPUT=$(ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} \
|
|
"docker stack ps $STACK --filter desired-state=running \
|
|
--format ' {{.Name}} {{.CurrentState}}'" 2>/dev/null)
|
|
if [ -n "$OUTPUT" ]; then
|
|
echo "$OUTPUT"
|
|
break
|
|
fi
|
|
if [ "$i" -eq "$ATTEMPTS" ]; then
|
|
echo " WARNING: no running tasks found for $STACK after $((ATTEMPTS * DELAY))s."
|
|
echo " This may be transient Swarm settle time, or a real problem — check manually:"
|
|
echo " ssh root@$${SWARM_MANAGER_IP} 'docker stack ps $STACK --no-trunc'"
|
|
else
|
|
sleep "$DELAY"
|
|
fi
|
|
i=$((i + 1))
|
|
done
|
|
done
|
|
|
|
notify-success:
|
|
image: alpine:latest
|
|
environment:
|
|
TEAMS_WEBHOOK:
|
|
from_secret: teams_webhook
|
|
commands:
|
|
- apk add --no-cache curl
|
|
- |
|
|
CHANGED=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' ' ')
|
|
PAYLOAD=$(printf '{"title":"Swarm Deploy OK","text":"Stacks: %s Commit: %s","themeColor":"00aa00"}' "$CHANGED" "$CI_COMMIT_MESSAGE")
|
|
curl -sf -X POST -H 'Content-Type: application/json' -d "$PAYLOAD" "$TEAMS_WEBHOOK" || true
|
|
when:
|
|
- status: success
|
|
|
|
notify-failure:
|
|
image: alpine:latest
|
|
environment:
|
|
TEAMS_WEBHOOK:
|
|
from_secret: teams_webhook
|
|
WOODPECKER_HOST:
|
|
from_secret: woodpecker_host
|
|
commands:
|
|
- apk add --no-cache curl
|
|
- |
|
|
PAYLOAD=$(printf '{"title":"Swarm Deploy FAILED","text":"Commit: %s See: https://%s","themeColor":"ff0000"}' "$CI_COMMIT_MESSAGE" "$WOODPECKER_HOST")
|
|
curl -sf -X POST -H 'Content-Type: application/json' -d "$PAYLOAD" "$TEAMS_WEBHOOK" || true
|
|
when:
|
|
- status: failure
|