postgresql-ha-staging.yaml: switch from CLONE_WITH_BASEBACKUP to Patroni standby_cluster (continuous streaming, validated in pgha-test dry run) - closes pre-cutover write gap

postgresql/cutover/postgresql-ha-staging.yaml

@@ -14,16 +14,55 @@
 #   - Joins the EXISTING postgresql_db-backend overlay (external) — the
 #     live single-instance stack is untouched and keeps the "postgresql"
 #     and "db" aliases.
-#   - patroni-0/patroni-1 bootstrap a NEW cluster (scope: postgres-ha) by
-#     CLONING the live instance via pg_basebackup (CLONE_WITH_BASEBACKUP,
-#     CLONE_HOST=postgresql → resolves to the OLD instance during staging).
-#     CLONE_* is set on BOTH patroni nodes: whichever wins the initial
-#     leader race performs the clone; the other syncs from it.
+#   - patroni-0/patroni-1 bootstrap a NEW cluster (scope: postgres-ha) as a
+#     Patroni STANDBY CLUSTER continuously streaming from the live
+#     instance (standby_cluster, host: postgresql — resolves to the OLD
+#     instance during staging). This REPLACES the earlier
+#     CLONE_WITH_BASEBACKUP design (one-shot snapshot) — see "REVISION"
+#     note below for why.
 #   - HAProxy runs WITHOUT production aliases and WITHOUT published ports.
 #     It only gets those in postgresql-ha-final.yaml at promote time.
 #   - Superuser is PGadmin (matches the cloned data's existing superuser,
 #     same password secret), NOT Spilo's default "postgres".
 #
+# ─── REVISION (2026-08-01): switched from CLONE_WITH_BASEBACKUP to a
+# Patroni "standby cluster" (continuous streaming) ───
+#
+# WHY: CLONE_WITH_BASEBACKUP is a ONE-SHOT snapshot — once pg_basebackup
+# completes, the new cluster has ZERO further connection to the live
+# database. Any write landing on production between the snapshot and the
+# actual traffic cutover would be silently lost — a real risk given ~13
+# active consumers and a 42GB production database. Patroni's standby_cluster
+# mode (https://patroni.readthedocs.io/en/latest/standby_cluster.html)
+# instead makes the new cluster's leader ("standby leader") continuously
+# stream from the remote primary indefinitely, right up until an explicit
+# promotion (done by removing the standby_cluster key from DCS config —
+# there is no dedicated patronictl subcommand for this) — closing the gap
+# to near-zero. Fully validated end-to-end in the pgha-test dry run
+# (2026-08-01): bootstrap, continuous live-write propagation to the far
+# cascade replica in <3s, and a full promote-with-zero-data-loss cycle —
+# see "ADR-0001 Dry-Run Debugging Log" note for the complete play-by-play.
+#
+# ⚠️ NEW PRODUCTION PREREQUISITE THIS INTRODUCES — NOT YET DONE, must
+# happen before this file can actually be deployed for real: the dry run's
+# "legacy" stand-in creates a genuine replication role named "standby"
+# (matching PGUSER_STANDBY below) via a fresh-initdb hook script, since
+# standby_cluster streaming authenticates using the cluster's real
+# replication identity (PGUSER_STANDBY/PGPASSWORD_STANDBY), NOT the
+# PGadmin superuser that CLONE_WITH_BASEBACKUP used. The LIVE
+# `postgresql_postgresql` container has NO such hook mechanism available
+# (it's long since initialized, hooks only run on fresh initdb) and does
+# NOT currently have a "standby" role. This must be created manually
+# against the live database before real cutover, analogous to how the
+# pg_hba.conf replication rule was added manually — e.g.:
+#   CREATE ROLE standby WITH REPLICATION LOGIN PASSWORD '<same value as
+#   the postgresql_replication_password secret>';
+# This is NOT part of this compose file (can't be — it's a one-time SQL
+# statement against already-running production, not something a compose
+# file can express) but MUST be tracked and done as an explicit manual
+# step in the real Phase 3 procedure. See ADR-0001 Dry-Run Debugging Log
+# note, "Not yet done" section, for status.
+#
 # NOTE: command: blocks use $$(...) not $(...) — Compose's own variable
 # interpolation parses $( as an attempted ${VAR} reference and fails with
 # "invalid interpolation format" / "you may need to escape any $ with
@@ -32,9 +71,8 @@
 # block, including inside heredocs and nested quoting — Compose scans the
 # raw text before any shell/heredoc logic ever runs, so quoting context
 # doesn't exempt anything from this escaping requirement. (Confirmed the
-# hard way in the pgha-test dry run — see "ADR-0001 Dry-Run Debugging Log"
-# note — this file originally had the unescaped $(...) bug, including on
-# the CLONE_PASSWORD line which is unique to this staging file.)
+# hard way in the pgha-test dry run — this file originally had the
+# unescaped $(...) bug too, before this standby_cluster rewrite.)
 #
 # NOTE: ETCD3_HOSTS not ETCD_HOSTS — confirmed against zalando/spilo
 # configure_spilo.py: PATRONI_DCS includes both "etcd" (legacy v2 API,
@@ -42,7 +80,7 @@
 # distinct DCS backends selected by env var prefix. Our etcd containers
 # (v3.5.9) have the v2 API disabled by default, so ETCD_HOSTS causes
 # Patroni to hit /v2 endpoints that 404. ETCD3_HOSTS selects the correct
-# v3-API client. (Also originally wrong in this file — fixed after dry run.)
+# v3-API client.
 #
 # NOTE: patroni-0/patroni-1 override bootstrap.post_init via
 # SPILO_CONFIGURATION (Spilo's documented, supported mechanism for
@@ -65,7 +103,7 @@
 # existence + SUPERUSER) were found and validated end-to-end in the
 # pgha-test dry run before being ported here — see the dry-run debugging
 # note for full detail. This is CRITICAL for this file specifically since
-# it clones real production data with the real PGadmin superuser.
+# it streams real production data with the real PGadmin superuser.
 # ─────────────────────────────────────────────────────────────────────────
 version: "3.6"
 
@@ -150,7 +188,6 @@ services:
         export PGPASSWORD_SUPERUSER="$$(cat /run/secrets/postgresql_password)"
         export PGPASSWORD_STANDBY="$$(cat /run/secrets/postgresql_replication_password)"
         export PATRONI_RESTAPI_PASSWORD="$$(cat /run/secrets/postgresql_patroni_password)"
-        export CLONE_PASSWORD="$$(cat /run/secrets/postgresql_password)"
         mkdir -p /scripts
         cat > /scripts/post_init_wrapper.sh <<'WRAP'
         #!/bin/bash
@@ -171,13 +208,12 @@ services:
       SPILO_CONFIGURATION: |
         bootstrap:
           post_init: /scripts/post_init_wrapper.sh "zalandos"
-      # Clone-from-live: only runs when PGDATA is empty (first bootstrap).
-      # "postgresql" resolves to the OLD single instance during staging.
-      CLONE_METHOD: CLONE_WITH_BASEBACKUP
-      CLONE_SCOPE: legacy-single
-      CLONE_HOST: postgresql
-      CLONE_PORT: "5432"
-      CLONE_USER: PGadmin
+          dcs:
+            standby_cluster:
+              host: postgresql
+              port: 5432
+              create_replica_methods:
+              - basebackup_fast_xlog
     secrets:
       - postgresql_password
       - postgresql_replication_password
@@ -201,7 +237,6 @@ services:
         export PGPASSWORD_SUPERUSER="$$(cat /run/secrets/postgresql_password)"
         export PGPASSWORD_STANDBY="$$(cat /run/secrets/postgresql_replication_password)"
         export PATRONI_RESTAPI_PASSWORD="$$(cat /run/secrets/postgresql_patroni_password)"
-        export CLONE_PASSWORD="$$(cat /run/secrets/postgresql_password)"
         mkdir -p /scripts
         cat > /scripts/post_init_wrapper.sh <<'WRAP'
         #!/bin/bash
@@ -222,11 +257,12 @@ services:
       SPILO_CONFIGURATION: |
         bootstrap:
           post_init: /scripts/post_init_wrapper.sh "zalandos"
-      CLONE_METHOD: CLONE_WITH_BASEBACKUP
-      CLONE_SCOPE: legacy-single
-      CLONE_HOST: postgresql
-      CLONE_PORT: "5432"
-      CLONE_USER: PGadmin
+          dcs:
+            standby_cluster:
+              host: postgresql
+              port: 5432
+              create_replica_methods:
+              - basebackup_fast_xlog
     secrets:
       - postgresql_password
       - postgresql_replication_password