From e57f4b3592f816c27d7bea46a2f0bbe09ec8668d Mon Sep 17 00:00:00 2001 From: admin Date: Tue, 4 Aug 2026 00:48:32 -0700 Subject: [PATCH] cutover.sh: extend Phase 2 cascade-replica wait window to 35 min MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2's cascade-replica bootstrap does its own full basebackup FROM the new standby_leader (an extra hop beyond Phase 1's legacy-direct copy), so per operator request its window is extended further than Phase 1's — from 1200s (20 min, matched to observed 8-11 min legacy-direct timing) to 2100s (35 min), giving more margin for the additional hop. Phase 1's window is intentionally left unchanged at 1200s since it already has comfortable headroom against the timing we've actually observed for that specific bootstrap path. --- postgresql/cutover/cutover.sh | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/postgresql/cutover/cutover.sh b/postgresql/cutover/cutover.sh index 8daf1eb..5c478b0 100644 --- a/postgresql/cutover/cutover.sh +++ b/postgresql/cutover/cutover.sh @@ -106,6 +106,18 @@ # been observed to hang on CephFS under heavy concurrent write load) # logged on every iteration, so progress is directly observable # instead of a silent binary wait. +# +# 2026 run #3 follow-up (operator request, no new failure observed yet): +# Phase 2's cascade-replica bootstrap adds an EXTRA network/streaming +# hop on top of the same basebackup workload Phase 1 already does +# (patroni-0/1's loser bootstraps FROM the winner, not from legacy +# directly) — plausibly slower than Phase 1's copy, not just equal to +# it. Phase 2's window is extended further, from 1200s (20 min) to +# 2100s (35 min), specifically to give more margin on this second hop. +# Phase 1's window is intentionally left at 1200s (20 min) — it +# already matches the directly-observed 8-11 min precedent with +# comfortable headroom, and its bootstrap source (legacy directly) is +# the case we actually have real timing data for. set -uo pipefail # NOTE: deliberately not -e — every phase below checks # its own command's exit status explicitly so we can @@ -325,11 +337,15 @@ log "PASS: Phase 1 — ${LEADER_HOST} is standby_leader (bootstrap winner determ log "--- Phase 2: confirm streaming lag = 0 (cascade replica: ${REPLICA_HOST}) ---" # FIXED (see header "FIX LOG" item b): ${REPLICA_HOST} is itself # bootstrapping via a full basebackup from ${LEADER_HOST} — same -# 8-11-min-for-~42GB precedent as Phase 1. Window extended from 300s to -# 1200s (20 min), with per-poll data-dir size logging. -log "Note: ${REPLICA_HOST} is bootstrapping via a full basebackup from ${LEADER_HOST} — same basebackup precedent as Phase 1, so this window is generous." +# 8-11-min-for-~42GB precedent as Phase 1. +# FURTHER EXTENDED (see header "FIX LOG", 2026 run #3 follow-up, operator +# request): this cascade-replica bootstrap adds an EXTRA network/streaming +# hop on top of the same basebackup workload, so its window is now longer +# than Phase 1's rather than merely equal to it — extended from 1200s +# (20 min) to 2100s (35 min), with per-poll data-dir size logging. +log "Note: ${REPLICA_HOST} is bootstrapping via a full basebackup from ${LEADER_HOST} — an extra hop beyond Phase 1's basebackup, so this window is deliberately longer than Phase 1's." CONSECUTIVE_ZERO=0 -PHASE2_MAX_WAIT=1200 +PHASE2_MAX_WAIT=2100 PHASE2_POLL_INTERVAL=15 waited=0 REPLICA_ROLE=""