Bot
9ec1b1811b
Flags may appear before or after the stack name (stack-deploy.sh traefik -e and stack-deploy.sh -e traefik both work) — the stack name is whatever argument isn't a recognized flag. No change to the render/mount-guard/deploy logic; this only changes how the git-guard.sh pre-flight is invoked. See deploy/git-guard.sh header for what each flag actually does.
217 lines
8.6 KiB
Bash
Executable File
217 lines
8.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# stack-deploy.sh — Safe Docker Swarm stack deployer
|
|
# Supports:
|
|
# - Flat: <stack>.yaml (legacy, all existing stacks)
|
|
# - Folder: <stack>/ (new, multi-compose stacks like immich)
|
|
#
|
|
# Folder layout:
|
|
# compose-files/
|
|
# └── immich/
|
|
# ├── immich.yml ← main compose (must match stack name)
|
|
# ├── hwaccel.transcoding.yml ← extension files, merged in glob order
|
|
# ├── hwaccel.ml.yml
|
|
# └── immich.env ← optional, same as flat pattern
|
|
#
|
|
# Env layering:
|
|
# deploy/global.env ← always loaded first (DOMAIN, DOMAIN_NAME, ALT_DOMAIN, etc.)
|
|
# <stack>.env ← loaded second; stack values override globals
|
|
#
|
|
# Merge strategy: docker compose ... config | docker stack deploy -c -
|
|
# All files in the folder matching *.yml or *.yaml are included.
|
|
# Main file (<stack>.yml or <stack>.yaml) is always passed FIRST.
|
|
# Remaining files are sorted and appended.
|
|
#
|
|
# Render pipeline (unified for all modes as of 2026-08-26):
|
|
# 1. Render full compose YAML (with extras merged + env substituted) to a
|
|
# temp file.
|
|
# 2. Run mount-guard.py against that temp file — checks every bind mount
|
|
# source path exists, and flags suspicious-looking empty Postgres data
|
|
# dirs, before anything touches Swarm.
|
|
# 3. docker stack deploy -c <tempfile> <stack>
|
|
#
|
|
# Dollar-escaping (2026-08-26, see envparse.py comments for full detail):
|
|
# - Single-file path (no extras): envsubst does no '$' escaping of its
|
|
# own, and only Swarm's `docker stack deploy` interpolation pass runs
|
|
# downstream -> use export/export_merged (escapes '$' -> '$$' once).
|
|
# - Folder+extras path: `docker compose config` ALSO does its own '$'
|
|
# escaping on top of Swarm's -> use export_raw/export_raw_merged (no
|
|
# pre-escaping) or values get doubled twice. Getting this wrong
|
|
# silently corrupts any secret/hash containing '$' (confirmed impact:
|
|
# LITELLM keys truncated, IMMICH_KIOSK_BASICAUTH bcrypt hash mismatched).
|
|
#
|
|
# Usage: stack-deploy.sh [-e|--emergency] [-f|--force] <stack-name>
|
|
# Flags may appear before or after the stack name, e.g. both
|
|
# `stack-deploy.sh traefik -e` and `stack-deploy.sh -e traefik` work.
|
|
#
|
|
# -e/--emergency and -f/--force (added 2026-09-12, circular-dependency
|
|
# bootstrap incident — see deploy/git-guard.sh header for full detail):
|
|
# Both are passed straight through to git-guard.sh unchanged; this script
|
|
# does not interpret them itself beyond stripping them from the stack-name
|
|
# argument list. -e tries alternate Gitea endpoints (VIP, then each node's
|
|
# direct IP) before falling back to normal sync logic against whichever
|
|
# one responds. -f skips the sync check entirely — last resort only, read
|
|
# the warning banner it prints. Neither flag changes anything about the
|
|
# render/mount-guard/deploy steps below; they only affect whether and how
|
|
# git-guard.sh's pre-flight check runs.
|
|
|
|
set -euo pipefail
|
|
|
|
# ── Flag parsing (stack name is whatever's left after flags are stripped) ──
|
|
GUARD_FLAGS=()
|
|
STACK=""
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
-e|--emergency) GUARD_FLAGS+=(-e) ;;
|
|
-f|--force) GUARD_FLAGS+=(-f) ;;
|
|
*) STACK="$arg" ;;
|
|
esac
|
|
done
|
|
: "${STACK:?Usage: stack-deploy.sh [-e|--emergency] [-f|--force] <stack-name>}"
|
|
|
|
DIR="/volume1/docker/compose-files"
|
|
PY="$DIR/deploy/envparse.py"
|
|
MOUNT_GUARD="$DIR/deploy/mount-guard.py"
|
|
GLOBAL_ENV="$DIR/deploy/global.env"
|
|
|
|
# ── Pre-flight: ensure local checkout is in sync with Gitea ─────────────────
|
|
# Prevents deploying from a stale/diverged local tree (see incident 2026-08-26).
|
|
# Invoked via `bash` explicitly so the tracked file's exec bit doesn't matter.
|
|
bash "$DIR/deploy/git-guard.sh" "${GUARD_FLAGS[@]:-}" || { echo "ERROR: git-guard check failed. Deploy aborted."; exit 1; }
|
|
|
|
# ── Locate compose file(s) ──────────────────────────────────────────────────
|
|
|
|
FOLDER="$DIR/$STACK"
|
|
FLAT_YAML="$DIR/${STACK}.yaml"
|
|
FLAT_YML="$DIR/${STACK}.yml"
|
|
|
|
if [ -d "$FOLDER" ]; then
|
|
MODE="folder"
|
|
|
|
# Main file must be named <stack>.yml or <stack>.yaml
|
|
if [ -f "$FOLDER/${STACK}.yml" ]; then MAIN="$FOLDER/${STACK}.yml"
|
|
elif [ -f "$FOLDER/${STACK}.yaml" ]; then MAIN="$FOLDER/${STACK}.yaml"
|
|
else echo "ERROR: No main compose file ($STACK.yml or $STACK.yaml) in $FOLDER"; exit 1
|
|
fi
|
|
|
|
# Collect extension files: everything else in the folder, sorted
|
|
EXTRAS=()
|
|
while IFS= read -r -d '' f; do
|
|
[ "$f" = "$MAIN" ] && continue
|
|
EXTRAS+=("$f")
|
|
done < <(find "$FOLDER" -maxdepth 1 \( -name '*.yml' -o -name '*.yaml' \) -print0 | sort -z)
|
|
|
|
ENVFILE="$FOLDER/${STACK}.env"
|
|
|
|
elif [ -f "$FLAT_YAML" ]; then
|
|
MODE="flat"
|
|
MAIN="$FLAT_YAML"
|
|
EXTRAS=()
|
|
ENVFILE="$DIR/${STACK}.env"
|
|
|
|
elif [ -f "$FLAT_YML" ]; then
|
|
MODE="flat"
|
|
MAIN="$FLAT_YML"
|
|
EXTRAS=()
|
|
ENVFILE="$DIR/${STACK}.env"
|
|
|
|
else
|
|
echo "ERROR: No compose file or folder found for stack '$STACK' in $DIR"
|
|
exit 1
|
|
fi
|
|
|
|
echo "==> Deploying stack: $STACK [$MODE]"
|
|
echo " Main: $MAIN"
|
|
for f in "${EXTRAS[@]:-}"; do [ -n "$f" ] && echo " Extra: $f"; done
|
|
|
|
# ── Build -f flag list ───────────────────────────────────────────────────────
|
|
|
|
F_FLAGS=(-f "$MAIN")
|
|
for f in "${EXTRAS[@]:-}"; do
|
|
[ -n "$f" ] && F_FLAGS+=(-f "$f")
|
|
done
|
|
|
|
# Whether the render will go through `docker compose config` (folder mode
|
|
# with 1+ extras). This determines which escaping mode is correct — see
|
|
# header comment and envparse.py for why these must differ.
|
|
USES_COMPOSE_CONFIG=0
|
|
[ "${#F_FLAGS[@]}" -gt 2 ] && USES_COMPOSE_CONFIG=1
|
|
|
|
# ── Load env (global base + optional stack override) ─────────────────────────
|
|
|
|
HINT=" Hint: 'secret not found' means Woodpecker hasn't provisioned secrets yet.\n Trigger the pipeline: https://woodpecker.bryanmail.net\n"
|
|
|
|
GLOBAL_EXISTS=0
|
|
STACK_EXISTS=0
|
|
[ -f "$GLOBAL_ENV" ] && GLOBAL_EXISTS=1
|
|
[ -f "$ENVFILE" ] && STACK_EXISTS=1
|
|
|
|
if [ "$USES_COMPOSE_CONFIG" -eq 1 ]; then
|
|
EXPORT_MODE="export_raw"; EXPORT_MERGED_MODE="export_raw_merged"
|
|
else
|
|
EXPORT_MODE="export"; EXPORT_MERGED_MODE="export_merged"
|
|
fi
|
|
|
|
if [ "$GLOBAL_EXISTS" -eq 1 ] && [ "$STACK_EXISTS" -eq 1 ]; then
|
|
echo " Env: $GLOBAL_ENV + $ENVFILE (stack overrides global) [$EXPORT_MERGED_MODE]"
|
|
eval "$(python3 "$PY" "$EXPORT_MERGED_MODE" "$GLOBAL_ENV" "$ENVFILE")"
|
|
VARS="$(python3 "$PY" vars_merged "$GLOBAL_ENV" "$ENVFILE")"
|
|
|
|
elif [ "$GLOBAL_EXISTS" -eq 1 ]; then
|
|
echo " Env: $GLOBAL_ENV (no stack env) [$EXPORT_MODE]"
|
|
eval "$(python3 "$PY" "$EXPORT_MODE" "$GLOBAL_ENV")"
|
|
VARS="$(python3 "$PY" vars "$GLOBAL_ENV")"
|
|
|
|
elif [ "$STACK_EXISTS" -eq 1 ]; then
|
|
echo " Env: $ENVFILE (no global env) [$EXPORT_MODE]"
|
|
eval "$(python3 "$PY" "$EXPORT_MODE" "$ENVFILE")"
|
|
VARS="$(python3 "$PY" vars "$ENVFILE")"
|
|
|
|
else
|
|
echo " Env: (none)"
|
|
VARS=""
|
|
fi
|
|
|
|
echo " Vars: $VARS"
|
|
|
|
# ── Render final compose YAML to a temp file ──────────────────────────────────
|
|
|
|
RENDERED="$(mktemp /tmp/stack-deploy.XXXXXX.yml)"
|
|
trap 'rm -f "$RENDERED"' EXIT
|
|
|
|
if [ -n "$VARS" ]; then
|
|
if [ "$USES_COMPOSE_CONFIG" -eq 1 ]; then
|
|
# Folder mode with extras: merge via docker compose config
|
|
docker compose "${F_FLAGS[@]}" config \
|
|
| python3 "$PY" strip \
|
|
| envsubst "$VARS" \
|
|
> "$RENDERED"
|
|
else
|
|
# Single file
|
|
envsubst "$VARS" < "$MAIN" \
|
|
| python3 "$PY" strip \
|
|
> "$RENDERED"
|
|
fi
|
|
else
|
|
if [ "$USES_COMPOSE_CONFIG" -eq 1 ]; then
|
|
docker compose "${F_FLAGS[@]}" config \
|
|
| python3 "$PY" strip \
|
|
> "$RENDERED"
|
|
else
|
|
python3 "$PY" strip < "$MAIN" > "$RENDERED"
|
|
fi
|
|
fi
|
|
|
|
# ── Pre-flight: bind mount paths exist + Postgres-data sanity ───────────────
|
|
# See deploy/mount-guard.py for details. Blocks on missing paths or
|
|
# suspicious-looking empty/uninitialized Postgres data directories (see
|
|
# incident 2026-08-26: a wrong-but-existing empty bind path would have let
|
|
# Postgres silently init a fresh DB while real data sat orphaned elsewhere).
|
|
python3 "$MOUNT_GUARD" "$RENDERED" || { echo "ERROR: mount-guard check failed. Deploy aborted."; exit 1; }
|
|
|
|
# ── Deploy ───────────────────────────────────────────────────────────────────
|
|
|
|
docker stack deploy -c "$RENDERED" "$STACK" \
|
|
|| { printf "\n%b" "$HINT"; exit 1; }
|
|
|
|
echo "==> Done: $STACK"
|