ADR-0001 cutover.sh: fix Phase 1 node-label check (broken Go template on hyphenated label) and accept any 2xx in consumer health checks

postgresql/cutover/cutover.sh

@@ -59,6 +59,26 @@
 # etcd-2 is guaranteed to be on docker-2 (node.hostname == docker-2
 # constraint in both staging/final yaml), so this exec always works
 # locally if needed.
+#
+# ── FIX LOG (post-first-real-run) ──────────────────────────────────────
+# 2026 run #1 aborted at Phase 1's node-label check with a false negative
+# despite labels genuinely being set (docker-2=primary, docker-3=replica,
+# confirmed via `docker node inspect --format '{{json .Spec.Labels}}'`).
+# Root cause: the original check used
+#   docker node inspect {} --format '{{.Spec.Labels.pg-role}}'
+# — Go's template engine cannot parse a bare `.pg-role` field reference
+# (the hyphen is invalid in that position), so the command errored
+# silently (stderr was redirected to /dev/null) and returned empty every
+# time, regardless of actual label state. FIXED below by switching to
+# `docker node ls --filter "node.label=pg-role=primary"`, which uses
+# Docker's own filter syntax instead of Go template field access and
+# sidesteps the hyphen problem entirely.
+# Separately, rollback.sh's automatic consumer re-verification (triggered
+# by this false-negative rollback) flagged woodpecker's healthz as
+# "failed" because it returned HTTP 204 — a legitimate, common
+# "healthy, no content" response for that kind of endpoint. The health
+# checks below (and the matching ones in rollback.sh) now accept any 2xx
+# status code, not just literal 200.
 
 set -uo pipefail   # NOTE: deliberately not -e — every phase below checks
                    # its own command's exit status explicitly so we can
@@ -96,6 +116,9 @@ CONSUMER_STABILITY_ONLY=(
 log()  { echo "[cutover $(date +%H:%M:%S)] $*"; }
 warn() { echo "[cutover WARN $(date +%H:%M:%S)] $*" >&2; }
 
+# Returns success (0) if $1 looks like a 2xx HTTP status code.
+is_2xx() { [[ "$1" =~ ^2[0-9][0-9]$ ]]; }
+
 # Any hard failure calls this — invokes rollback.sh, then exits nonzero.
 # No manual step should be required after this runs.
 trigger_rollback() {
@@ -194,11 +217,15 @@ log "PASS: Phase 0 preflight"
 # ── Phase 1: deploy postgresqlha staging stack ─────────────────────────
 log "--- Phase 1: deploy postgresqlha (staging) ---"
 
-PRIMARY_LABELED=$(docker node ls --format '{{.Hostname}}' | xargs -I{} docker node inspect {} --format '{{.Spec.Labels.pg_role}}{{.Spec.Labels.pg-role}}' 2>/dev/null | grep -c primary || true)
+# FIXED (see header "FIX LOG"): use docker's own --filter syntax instead
-REPLICA_LABELED=$(docker node ls --format '{{.Hostname}}' | xargs -I{} docker node inspect {} --format '{{.Spec.Labels.pg_role}}{{.Spec.Labels.pg-role}}' 2>/dev/null | grep -c replica || true)
+# of Go template field access on a hyphenated label name (which silently
+# fails to parse and always returns empty).
+PRIMARY_LABELED=$(docker node ls --filter "node.label=pg-role=primary" --format '{{.Hostname}}' | grep -c . || true)
+REPLICA_LABELED=$(docker node ls --filter "node.label=pg-role=replica" --format '{{.Hostname}}' | grep -c . || true)
 if [ "$PRIMARY_LABELED" -lt 1 ] || [ "$REPLICA_LABELED" -lt 1 ]; then
   trigger_rollback "No node(s) found with pg-role=primary/replica labels — patroni-0/1 placement constraints cannot schedule. Set via: docker node update --label-add pg-role=primary <node> (and =replica on another). Nothing deployed yet, rollback is a no-op safety call."
 fi
+log "Node labels confirmed: ${PRIMARY_LABELED} node(s) with pg-role=primary, ${REPLICA_LABELED} node(s) with pg-role=replica."
 
 docker stack deploy -c "${CUTOVER_DIR}/postgresql-ha-staging.yaml" "${HA_STACK}"
 if [ $? -ne 0 ]; then
@@ -285,7 +312,7 @@ log "PASS: Phase 4 — legacy is read-only, no further writes possible there"
 
 # ── Phase 5: promote Patroni ─────────────────────────────────────────
 log "--- Phase 5: promote patroni-1 (remove standby_cluster from DCS config) ---"
-log "⚠️  Highest-scrutiny phase — see script header for the documented deviation from the dry-run's raw-etcdctl method (using Patroni's official REST PATCH /config instead). Manual etcdctl fallback documented above and in RUNBOOK.md if needed."
+log "⚠️  Highest-scrutiny phase — see script header for the documented deviation from the dry-run's proven method (using Patroni's official REST PATCH /config instead). Manual etcdctl fallback documented above and in RUNBOOK.md if needed."
 
 PATRONI_PASS=$(docker exec "$LEGACY_CID" cat /run/secrets/postgresql_patroni_password 2>/dev/null)
 if [ -z "$PATRONI_PASS" ]; then
@@ -386,8 +413,8 @@ for svc in "${!CONSUMER_HEALTH_URLS[@]}"; do
   url="${CONSUMER_HEALTH_URLS[$svc]}"
   CODE=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 10 "$url")
   log "  ${svc} -> ${url} -> HTTP ${CODE}"
-  if [ "$CODE" != "200" ]; then
+  if ! is_2xx "$CODE"; then
-    warn "  ${svc} did not return 200 (got ${CODE})"
+    warn "  ${svc} did not return a 2xx status (got ${CODE})"
     CONSUMER_FAIL=1
   fi
 done
@@ -433,7 +460,7 @@ for svc in "${!CONSUMER_HEALTH_URLS[@]}"; do
   url="${CONSUMER_HEALTH_URLS[$svc]}"
   CODE=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 10 "$url")
   log "  ${svc} -> ${url} -> HTTP ${CODE}"
-  [ "$CODE" != "200" ] && CONSUMER_FAIL2=1
+  is_2xx "$CODE" || CONSUMER_FAIL2=1
 done
 if [ "$CONSUMER_FAIL2" -ne 0 ]; then
   trigger_rollback "Consumer check failed AFTER stopping legacy. Per RUNBOOK.md, rollback past this point is asymmetric (restores legacy as primary of record, not just a flag flip) — invoking rollback.sh now rather than leaving this in a broken state."