deploy/git-guard.sh

@@ -23,18 +23,131 @@
 #     remote both moved)           backup/stash/reset recovery steps and exits.
 #
 # Exit codes: 0 = safe to deploy, 1 = blocked, needs human intervention
+#
+# ── Flags (added 2026-09-12, circular-dependency bootstrap incident) ────────
+#
+#   -e, --emergency
+#     Before the normal fetch, tries each candidate Gitea endpoint in order
+#     (git.bryanmail.net -> 192.168.4.30 VIP -> .31 -> .32 -> .33 node IPs,
+#     each over plain http on port 3000, which is Gitea's direct ingress
+#     port — bypasses Traefik/VIP entirely) and switches `origin` to the
+#     first one that responds to `git ls-remote` within a short timeout.
+#     This solves the bootstrap circular dependency where Traefik is down,
+#     so HTTPS access to git.bryanmail.net is unreachable, so git-guard
+#     can't fetch, so stack-deploy.sh can't redeploy traefik to fix itself.
+#     Once a reachable endpoint is found, NORMAL sync logic still runs
+#     (behind/ahead/diverged handling is unchanged) — this flag only changes
+#     *which host* is used, never skips the safety checks themselves.
+#     Prints a loud reminder to restore the real origin URL afterward; never
+#     commits the swapped URL anywhere.
+#
+#   -f, --force
+#     Skips the sync check ENTIRELY — no fetch, no comparison, no commit/push
+#     offer. Deploys whatever is on disk right now, as-is. This is the last
+#     resort for a genuine emergency where NONE of the candidate hosts in
+#     --emergency are reachable (e.g. Gitea itself is down, not just
+#     routing). Prints a loud warning banner. Never use this for routine
+#     work — it defeats the entire purpose of this script and is the exact
+#     failure mode (deploying a stale/unreviewed tree) git-guard exists to
+#     prevent.
+#
+#   Both flags are passed through from stack-deploy.sh's own -e/-f flags;
+#   see that script's header for the calling convention.
+# ─────────────────────────────────────────────────────────────────────────
 
 set -euo pipefail
 
 DIR="/volume1/docker/compose-files"
 cd "$DIR"
 
+# ---- Flag parsing ----
+EMERGENCY=0
+FORCE=0
+for arg in "$@"; do
+  case "$arg" in
+    -e|--emergency) EMERGENCY=1 ;;
+    -f|--force)     FORCE=1 ;;
+  esac
+done
+
+# ---- Force mode: skip everything ----
+if [ "$FORCE" -eq 1 ]; then
+  echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+  echo "!! FORCE MODE (-f/--force): git-guard sync check SKIPPED ENTIRELY."
+  echo "!! Deploying whatever is on disk right now, as-is. No fetch, no"
+  echo "!! comparison with origin/main was performed. This is a LAST RESORT"
+  echo "!! for emergencies where origin is completely unreachable — verify"
+  echo "!! independently that the local tree is what you intend to deploy."
+  echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+  exit 0
+fi
+
 # Non-interactive detection (Woodpecker/cron have no TTY on stdin)
 INTERACTIVE=0
 [ -t 0 ] && INTERACTIVE=1
 
 echo "==> git-guard: checking repo sync state"
 
+# ---- Emergency mode: find a reachable Gitea endpoint before fetching ----
+# Candidate order: public hostname (normal path) -> VIP -> each node's direct
+# IP. Each is tried over plain http on port 3000 (Gitea's direct ingress
+# port, published outside Traefik — see traefik/traefik.yaml git service
+# port mapping), since the whole point is to bypass Traefik/VIP when THOSE
+# are what's broken. A short `git ls-remote` timeout keeps an unreachable
+# candidate from stalling the whole check for long.
+if [ "$EMERGENCY" -eq 1 ]; then
+  echo "!! EMERGENCY MODE (-e/--emergency): probing candidate Gitea endpoints"
+  echo "!! (bypassing the normal https://git.bryanmail.net path if needed)..."
+
+  ORIGINAL_URL="$(git remote get-url origin)"
+  CANDIDATES=(
+    "https://git.bryanmail.net/homelab/compose-files.git"
+    "http://192.168.4.30:3000/homelab/compose-files.git"
+    "http://192.168.4.31:3000/homelab/compose-files.git"
+    "http://192.168.4.32:3000/homelab/compose-files.git"
+    "http://192.168.4.33:3000/homelab/compose-files.git"
+  )
+
+  FOUND=""
+  for candidate in "${CANDIDATES[@]}"; do
+    echo -n "     trying $candidate ... "
+    if timeout 5 git ls-remote "$candidate" HEAD >/dev/null 2>&1; then
+      echo "OK"
+      FOUND="$candidate"
+      break
+    else
+      echo "unreachable"
+    fi
+  done
+
+  if [ -z "$FOUND" ]; then
+    echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+    echo "!! EMERGENCY MODE: none of the candidate endpoints responded."
+    echo "!! Gitea itself may be down (not just routing) — this is beyond what"
+    echo "!! an alternate host path can fix. Options:"
+    echo "!!   A) Diagnose Gitea directly: check the git_gitea-server service"
+    echo "!!      and container on docker-1."
+    echo "!!   B) If you are certain the on-disk tree is correct and Gitea is"
+    echo "!!      genuinely unreachable, re-run with -f/--force instead — but"
+    echo "!!      read that flag's warning carefully first."
+    echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+    exit 1
+  fi
+
+  if [ "$FOUND" != "$ORIGINAL_URL" ]; then
+    git remote set-url origin "$FOUND"
+    echo "==> origin temporarily switched to: $FOUND"
+    echo "!! REMINDER: once the normal path (git.bryanmail.net / Traefik) is"
+    echo "!! confirmed healthy again, restore the real origin URL:"
+    echo "!!   cd $DIR && git remote set-url origin \"$ORIGINAL_URL\""
+    echo "!! This swap is never committed anywhere and only affects this"
+    echo "!! local checkout's git config."
+  else
+    echo "==> Normal origin URL ($FOUND) is reachable — no swap needed."
+  fi
+  echo
+fi
+
 git fetch origin --quiet
 
 LOCAL="$(git rev-parse main)"

deploy/stack-deploy.sh

@@ -38,10 +38,36 @@
 #     pre-escaping) or values get doubled twice. Getting this wrong
 #     silently corrupts any secret/hash containing '$' (confirmed impact:
 #     LITELLM keys truncated, IMMICH_KIOSK_BASICAUTH bcrypt hash mismatched).
+#
+# Usage: stack-deploy.sh [-e|--emergency] [-f|--force] <stack-name>
+#   Flags may appear before or after the stack name, e.g. both
+#   `stack-deploy.sh traefik -e` and `stack-deploy.sh -e traefik` work.
+#
+# -e/--emergency and -f/--force (added 2026-09-12, circular-dependency
+# bootstrap incident — see deploy/git-guard.sh header for full detail):
+#   Both are passed straight through to git-guard.sh unchanged; this script
+#   does not interpret them itself beyond stripping them from the stack-name
+#   argument list. -e tries alternate Gitea endpoints (VIP, then each node's
+#   direct IP) before falling back to normal sync logic against whichever
+#   one responds. -f skips the sync check entirely — last resort only, read
+#   the warning banner it prints. Neither flag changes anything about the
+#   render/mount-guard/deploy steps below; they only affect whether and how
+#   git-guard.sh's pre-flight check runs.
 
 set -euo pipefail
 
-STACK="${1:?Usage: stack-deploy.sh <stack-name>}"
+# ── Flag parsing (stack name is whatever's left after flags are stripped) ──
+GUARD_FLAGS=()
+STACK=""
+for arg in "$@"; do
+  case "$arg" in
+    -e|--emergency) GUARD_FLAGS+=(-e) ;;
+    -f|--force)     GUARD_FLAGS+=(-f) ;;
+    *)              STACK="$arg" ;;
+  esac
+done
+: "${STACK:?Usage: stack-deploy.sh [-e|--emergency] [-f|--force] <stack-name>}"
+
 DIR="/volume1/docker/compose-files"
 PY="$DIR/deploy/envparse.py"
 MOUNT_GUARD="$DIR/deploy/mount-guard.py"
@@ -50,7 +76,7 @@ GLOBAL_ENV="$DIR/deploy/global.env"
 # ── Pre-flight: ensure local checkout is in sync with Gitea ─────────────────
 # Prevents deploying from a stale/diverged local tree (see incident 2026-08-26).
 # Invoked via `bash` explicitly so the tracked file's exec bit doesn't matter.
-bash "$DIR/deploy/git-guard.sh" || { echo "ERROR: git-guard check failed. Deploy aborted."; exit 1; }
+bash "$DIR/deploy/git-guard.sh" "${GUARD_FLAGS[@]:-}" || { echo "ERROR: git-guard check failed. Deploy aborted."; exit 1; }
 
 # ── Locate compose file(s) ──────────────────────────────────────────────────