diff --git a/deploy/git-guard.sh b/deploy/git-guard.sh index 7b629c8..0dc4988 100644 --- a/deploy/git-guard.sh +++ b/deploy/git-guard.sh @@ -23,18 +23,131 @@ # remote both moved) backup/stash/reset recovery steps and exits. # # Exit codes: 0 = safe to deploy, 1 = blocked, needs human intervention +# +# ── Flags (added 2026-09-12, circular-dependency bootstrap incident) ──────── +# +# -e, --emergency +# Before the normal fetch, tries each candidate Gitea endpoint in order +# (git.bryanmail.net -> 192.168.4.30 VIP -> .31 -> .32 -> .33 node IPs, +# each over plain http on port 3000, which is Gitea's direct ingress +# port — bypasses Traefik/VIP entirely) and switches `origin` to the +# first one that responds to `git ls-remote` within a short timeout. +# This solves the bootstrap circular dependency where Traefik is down, +# so HTTPS access to git.bryanmail.net is unreachable, so git-guard +# can't fetch, so stack-deploy.sh can't redeploy traefik to fix itself. +# Once a reachable endpoint is found, NORMAL sync logic still runs +# (behind/ahead/diverged handling is unchanged) — this flag only changes +# *which host* is used, never skips the safety checks themselves. +# Prints a loud reminder to restore the real origin URL afterward; never +# commits the swapped URL anywhere. +# +# -f, --force +# Skips the sync check ENTIRELY — no fetch, no comparison, no commit/push +# offer. Deploys whatever is on disk right now, as-is. This is the last +# resort for a genuine emergency where NONE of the candidate hosts in +# --emergency are reachable (e.g. Gitea itself is down, not just +# routing). Prints a loud warning banner. Never use this for routine +# work — it defeats the entire purpose of this script and is the exact +# failure mode (deploying a stale/unreviewed tree) git-guard exists to +# prevent. +# +# Both flags are passed through from stack-deploy.sh's own -e/-f flags; +# see that script's header for the calling convention. +# ───────────────────────────────────────────────────────────────────────── set -euo pipefail DIR="/volume1/docker/compose-files" cd "$DIR" +# ---- Flag parsing ---- +EMERGENCY=0 +FORCE=0 +for arg in "$@"; do + case "$arg" in + -e|--emergency) EMERGENCY=1 ;; + -f|--force) FORCE=1 ;; + esac +done + +# ---- Force mode: skip everything ---- +if [ "$FORCE" -eq 1 ]; then + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo "!! FORCE MODE (-f/--force): git-guard sync check SKIPPED ENTIRELY." + echo "!! Deploying whatever is on disk right now, as-is. No fetch, no" + echo "!! comparison with origin/main was performed. This is a LAST RESORT" + echo "!! for emergencies where origin is completely unreachable — verify" + echo "!! independently that the local tree is what you intend to deploy." + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + exit 0 +fi + # Non-interactive detection (Woodpecker/cron have no TTY on stdin) INTERACTIVE=0 [ -t 0 ] && INTERACTIVE=1 echo "==> git-guard: checking repo sync state" +# ---- Emergency mode: find a reachable Gitea endpoint before fetching ---- +# Candidate order: public hostname (normal path) -> VIP -> each node's direct +# IP. Each is tried over plain http on port 3000 (Gitea's direct ingress +# port, published outside Traefik — see traefik/traefik.yaml git service +# port mapping), since the whole point is to bypass Traefik/VIP when THOSE +# are what's broken. A short `git ls-remote` timeout keeps an unreachable +# candidate from stalling the whole check for long. +if [ "$EMERGENCY" -eq 1 ]; then + echo "!! EMERGENCY MODE (-e/--emergency): probing candidate Gitea endpoints" + echo "!! (bypassing the normal https://git.bryanmail.net path if needed)..." + + ORIGINAL_URL="$(git remote get-url origin)" + CANDIDATES=( + "https://git.bryanmail.net/homelab/compose-files.git" + "http://192.168.4.30:3000/homelab/compose-files.git" + "http://192.168.4.31:3000/homelab/compose-files.git" + "http://192.168.4.32:3000/homelab/compose-files.git" + "http://192.168.4.33:3000/homelab/compose-files.git" + ) + + FOUND="" + for candidate in "${CANDIDATES[@]}"; do + echo -n " trying $candidate ... " + if timeout 5 git ls-remote "$candidate" HEAD >/dev/null 2>&1; then + echo "OK" + FOUND="$candidate" + break + else + echo "unreachable" + fi + done + + if [ -z "$FOUND" ]; then + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo "!! EMERGENCY MODE: none of the candidate endpoints responded." + echo "!! Gitea itself may be down (not just routing) — this is beyond what" + echo "!! an alternate host path can fix. Options:" + echo "!! A) Diagnose Gitea directly: check the git_gitea-server service" + echo "!! and container on docker-1." + echo "!! B) If you are certain the on-disk tree is correct and Gitea is" + echo "!! genuinely unreachable, re-run with -f/--force instead — but" + echo "!! read that flag's warning carefully first." + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + exit 1 + fi + + if [ "$FOUND" != "$ORIGINAL_URL" ]; then + git remote set-url origin "$FOUND" + echo "==> origin temporarily switched to: $FOUND" + echo "!! REMINDER: once the normal path (git.bryanmail.net / Traefik) is" + echo "!! confirmed healthy again, restore the real origin URL:" + echo "!! cd $DIR && git remote set-url origin \"$ORIGINAL_URL\"" + echo "!! This swap is never committed anywhere and only affects this" + echo "!! local checkout's git config." + else + echo "==> Normal origin URL ($FOUND) is reachable — no swap needed." + fi + echo +fi + git fetch origin --quiet LOCAL="$(git rev-parse main)" diff --git a/deploy/stack-deploy.sh b/deploy/stack-deploy.sh index 17ffda8..33807cf 100755 --- a/deploy/stack-deploy.sh +++ b/deploy/stack-deploy.sh @@ -38,10 +38,36 @@ # pre-escaping) or values get doubled twice. Getting this wrong # silently corrupts any secret/hash containing '$' (confirmed impact: # LITELLM keys truncated, IMMICH_KIOSK_BASICAUTH bcrypt hash mismatched). +# +# Usage: stack-deploy.sh [-e|--emergency] [-f|--force] +# Flags may appear before or after the stack name, e.g. both +# `stack-deploy.sh traefik -e` and `stack-deploy.sh -e traefik` work. +# +# -e/--emergency and -f/--force (added 2026-09-12, circular-dependency +# bootstrap incident — see deploy/git-guard.sh header for full detail): +# Both are passed straight through to git-guard.sh unchanged; this script +# does not interpret them itself beyond stripping them from the stack-name +# argument list. -e tries alternate Gitea endpoints (VIP, then each node's +# direct IP) before falling back to normal sync logic against whichever +# one responds. -f skips the sync check entirely — last resort only, read +# the warning banner it prints. Neither flag changes anything about the +# render/mount-guard/deploy steps below; they only affect whether and how +# git-guard.sh's pre-flight check runs. set -euo pipefail -STACK="${1:?Usage: stack-deploy.sh }" +# ── Flag parsing (stack name is whatever's left after flags are stripped) ── +GUARD_FLAGS=() +STACK="" +for arg in "$@"; do + case "$arg" in + -e|--emergency) GUARD_FLAGS+=(-e) ;; + -f|--force) GUARD_FLAGS+=(-f) ;; + *) STACK="$arg" ;; + esac +done +: "${STACK:?Usage: stack-deploy.sh [-e|--emergency] [-f|--force] }" + DIR="/volume1/docker/compose-files" PY="$DIR/deploy/envparse.py" MOUNT_GUARD="$DIR/deploy/mount-guard.py" @@ -50,7 +76,7 @@ GLOBAL_ENV="$DIR/deploy/global.env" # ── Pre-flight: ensure local checkout is in sync with Gitea ───────────────── # Prevents deploying from a stale/diverged local tree (see incident 2026-08-26). # Invoked via `bash` explicitly so the tracked file's exec bit doesn't matter. -bash "$DIR/deploy/git-guard.sh" || { echo "ERROR: git-guard check failed. Deploy aborted."; exit 1; } +bash "$DIR/deploy/git-guard.sh" "${GUARD_FLAGS[@]:-}" || { echo "ERROR: git-guard check failed. Deploy aborted."; exit 1; } # ── Locate compose file(s) ──────────────────────────────────────────────────