diff --git a/postgresql/cutover/cutover.sh b/postgresql/cutover/cutover.sh index 8daf1eb..5c478b0 100644 --- a/postgresql/cutover/cutover.sh +++ b/postgresql/cutover/cutover.sh @@ -106,6 +106,18 @@ # been observed to hang on CephFS under heavy concurrent write load) # logged on every iteration, so progress is directly observable # instead of a silent binary wait. +# +# 2026 run #3 follow-up (operator request, no new failure observed yet): +# Phase 2's cascade-replica bootstrap adds an EXTRA network/streaming +# hop on top of the same basebackup workload Phase 1 already does +# (patroni-0/1's loser bootstraps FROM the winner, not from legacy +# directly) — plausibly slower than Phase 1's copy, not just equal to +# it. Phase 2's window is extended further, from 1200s (20 min) to +# 2100s (35 min), specifically to give more margin on this second hop. +# Phase 1's window is intentionally left at 1200s (20 min) — it +# already matches the directly-observed 8-11 min precedent with +# comfortable headroom, and its bootstrap source (legacy directly) is +# the case we actually have real timing data for. set -uo pipefail # NOTE: deliberately not -e — every phase below checks # its own command's exit status explicitly so we can @@ -325,11 +337,15 @@ log "PASS: Phase 1 — ${LEADER_HOST} is standby_leader (bootstrap winner determ log "--- Phase 2: confirm streaming lag = 0 (cascade replica: ${REPLICA_HOST}) ---" # FIXED (see header "FIX LOG" item b): ${REPLICA_HOST} is itself # bootstrapping via a full basebackup from ${LEADER_HOST} — same -# 8-11-min-for-~42GB precedent as Phase 1. Window extended from 300s to -# 1200s (20 min), with per-poll data-dir size logging. -log "Note: ${REPLICA_HOST} is bootstrapping via a full basebackup from ${LEADER_HOST} — same basebackup precedent as Phase 1, so this window is generous." +# 8-11-min-for-~42GB precedent as Phase 1. +# FURTHER EXTENDED (see header "FIX LOG", 2026 run #3 follow-up, operator +# request): this cascade-replica bootstrap adds an EXTRA network/streaming +# hop on top of the same basebackup workload, so its window is now longer +# than Phase 1's rather than merely equal to it — extended from 1200s +# (20 min) to 2100s (35 min), with per-poll data-dir size logging. +log "Note: ${REPLICA_HOST} is bootstrapping via a full basebackup from ${LEADER_HOST} — an extra hop beyond Phase 1's basebackup, so this window is deliberately longer than Phase 1's." CONSECUTIVE_ZERO=0 -PHASE2_MAX_WAIT=1200 +PHASE2_MAX_WAIT=2100 PHASE2_POLL_INTERVAL=15 waited=0 REPLICA_ROLE=""