feat(deploy): add -e/--emergency and -f/--force flags to break the Gitea/Traefik circular dependency #22
@@ -23,18 +23,131 @@
|
|||||||
# remote both moved) backup/stash/reset recovery steps and exits.
|
# remote both moved) backup/stash/reset recovery steps and exits.
|
||||||
#
|
#
|
||||||
# Exit codes: 0 = safe to deploy, 1 = blocked, needs human intervention
|
# Exit codes: 0 = safe to deploy, 1 = blocked, needs human intervention
|
||||||
|
#
|
||||||
|
# ── Flags (added 2026-09-12, circular-dependency bootstrap incident) ────────
|
||||||
|
#
|
||||||
|
# -e, --emergency
|
||||||
|
# Before the normal fetch, tries each candidate Gitea endpoint in order
|
||||||
|
# (git.bryanmail.net -> 192.168.4.30 VIP -> .31 -> .32 -> .33 node IPs,
|
||||||
|
# each over plain http on port 3000, which is Gitea's direct ingress
|
||||||
|
# port — bypasses Traefik/VIP entirely) and switches `origin` to the
|
||||||
|
# first one that responds to `git ls-remote` within a short timeout.
|
||||||
|
# This solves the bootstrap circular dependency where Traefik is down,
|
||||||
|
# so HTTPS access to git.bryanmail.net is unreachable, so git-guard
|
||||||
|
# can't fetch, so stack-deploy.sh can't redeploy traefik to fix itself.
|
||||||
|
# Once a reachable endpoint is found, NORMAL sync logic still runs
|
||||||
|
# (behind/ahead/diverged handling is unchanged) — this flag only changes
|
||||||
|
# *which host* is used, never skips the safety checks themselves.
|
||||||
|
# Prints a loud reminder to restore the real origin URL afterward; never
|
||||||
|
# commits the swapped URL anywhere.
|
||||||
|
#
|
||||||
|
# -f, --force
|
||||||
|
# Skips the sync check ENTIRELY — no fetch, no comparison, no commit/push
|
||||||
|
# offer. Deploys whatever is on disk right now, as-is. This is the last
|
||||||
|
# resort for a genuine emergency where NONE of the candidate hosts in
|
||||||
|
# --emergency are reachable (e.g. Gitea itself is down, not just
|
||||||
|
# routing). Prints a loud warning banner. Never use this for routine
|
||||||
|
# work — it defeats the entire purpose of this script and is the exact
|
||||||
|
# failure mode (deploying a stale/unreviewed tree) git-guard exists to
|
||||||
|
# prevent.
|
||||||
|
#
|
||||||
|
# Both flags are passed through from stack-deploy.sh's own -e/-f flags;
|
||||||
|
# see that script's header for the calling convention.
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
DIR="/volume1/docker/compose-files"
|
DIR="/volume1/docker/compose-files"
|
||||||
cd "$DIR"
|
cd "$DIR"
|
||||||
|
|
||||||
|
# ---- Flag parsing ----
|
||||||
|
EMERGENCY=0
|
||||||
|
FORCE=0
|
||||||
|
for arg in "$@"; do
|
||||||
|
case "$arg" in
|
||||||
|
-e|--emergency) EMERGENCY=1 ;;
|
||||||
|
-f|--force) FORCE=1 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
# ---- Force mode: skip everything ----
|
||||||
|
if [ "$FORCE" -eq 1 ]; then
|
||||||
|
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
|
||||||
|
echo "!! FORCE MODE (-f/--force): git-guard sync check SKIPPED ENTIRELY."
|
||||||
|
echo "!! Deploying whatever is on disk right now, as-is. No fetch, no"
|
||||||
|
echo "!! comparison with origin/main was performed. This is a LAST RESORT"
|
||||||
|
echo "!! for emergencies where origin is completely unreachable — verify"
|
||||||
|
echo "!! independently that the local tree is what you intend to deploy."
|
||||||
|
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
# Non-interactive detection (Woodpecker/cron have no TTY on stdin)
|
# Non-interactive detection (Woodpecker/cron have no TTY on stdin)
|
||||||
INTERACTIVE=0
|
INTERACTIVE=0
|
||||||
[ -t 0 ] && INTERACTIVE=1
|
[ -t 0 ] && INTERACTIVE=1
|
||||||
|
|
||||||
echo "==> git-guard: checking repo sync state"
|
echo "==> git-guard: checking repo sync state"
|
||||||
|
|
||||||
|
# ---- Emergency mode: find a reachable Gitea endpoint before fetching ----
|
||||||
|
# Candidate order: public hostname (normal path) -> VIP -> each node's direct
|
||||||
|
# IP. Each is tried over plain http on port 3000 (Gitea's direct ingress
|
||||||
|
# port, published outside Traefik — see traefik/traefik.yaml git service
|
||||||
|
# port mapping), since the whole point is to bypass Traefik/VIP when THOSE
|
||||||
|
# are what's broken. A short `git ls-remote` timeout keeps an unreachable
|
||||||
|
# candidate from stalling the whole check for long.
|
||||||
|
if [ "$EMERGENCY" -eq 1 ]; then
|
||||||
|
echo "!! EMERGENCY MODE (-e/--emergency): probing candidate Gitea endpoints"
|
||||||
|
echo "!! (bypassing the normal https://git.bryanmail.net path if needed)..."
|
||||||
|
|
||||||
|
ORIGINAL_URL="$(git remote get-url origin)"
|
||||||
|
CANDIDATES=(
|
||||||
|
"https://git.bryanmail.net/homelab/compose-files.git"
|
||||||
|
"http://192.168.4.30:3000/homelab/compose-files.git"
|
||||||
|
"http://192.168.4.31:3000/homelab/compose-files.git"
|
||||||
|
"http://192.168.4.32:3000/homelab/compose-files.git"
|
||||||
|
"http://192.168.4.33:3000/homelab/compose-files.git"
|
||||||
|
)
|
||||||
|
|
||||||
|
FOUND=""
|
||||||
|
for candidate in "${CANDIDATES[@]}"; do
|
||||||
|
echo -n " trying $candidate ... "
|
||||||
|
if timeout 5 git ls-remote "$candidate" HEAD >/dev/null 2>&1; then
|
||||||
|
echo "OK"
|
||||||
|
FOUND="$candidate"
|
||||||
|
break
|
||||||
|
else
|
||||||
|
echo "unreachable"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ -z "$FOUND" ]; then
|
||||||
|
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
|
||||||
|
echo "!! EMERGENCY MODE: none of the candidate endpoints responded."
|
||||||
|
echo "!! Gitea itself may be down (not just routing) — this is beyond what"
|
||||||
|
echo "!! an alternate host path can fix. Options:"
|
||||||
|
echo "!! A) Diagnose Gitea directly: check the git_gitea-server service"
|
||||||
|
echo "!! and container on docker-1."
|
||||||
|
echo "!! B) If you are certain the on-disk tree is correct and Gitea is"
|
||||||
|
echo "!! genuinely unreachable, re-run with -f/--force instead — but"
|
||||||
|
echo "!! read that flag's warning carefully first."
|
||||||
|
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$FOUND" != "$ORIGINAL_URL" ]; then
|
||||||
|
git remote set-url origin "$FOUND"
|
||||||
|
echo "==> origin temporarily switched to: $FOUND"
|
||||||
|
echo "!! REMINDER: once the normal path (git.bryanmail.net / Traefik) is"
|
||||||
|
echo "!! confirmed healthy again, restore the real origin URL:"
|
||||||
|
echo "!! cd $DIR && git remote set-url origin \"$ORIGINAL_URL\""
|
||||||
|
echo "!! This swap is never committed anywhere and only affects this"
|
||||||
|
echo "!! local checkout's git config."
|
||||||
|
else
|
||||||
|
echo "==> Normal origin URL ($FOUND) is reachable — no swap needed."
|
||||||
|
fi
|
||||||
|
echo
|
||||||
|
fi
|
||||||
|
|
||||||
git fetch origin --quiet
|
git fetch origin --quiet
|
||||||
|
|
||||||
LOCAL="$(git rev-parse main)"
|
LOCAL="$(git rev-parse main)"
|
||||||
|
|||||||
Reference in New Issue
Block a user