diff --git a/postgresql/cutover/postgresql-ha-staging.yaml b/postgresql/cutover/postgresql-ha-staging.yaml index 4e24f29..5a40faa 100644 --- a/postgresql/cutover/postgresql-ha-staging.yaml +++ b/postgresql/cutover/postgresql-ha-staging.yaml @@ -14,16 +14,55 @@ # - Joins the EXISTING postgresql_db-backend overlay (external) — the # live single-instance stack is untouched and keeps the "postgresql" # and "db" aliases. -# - patroni-0/patroni-1 bootstrap a NEW cluster (scope: postgres-ha) by -# CLONING the live instance via pg_basebackup (CLONE_WITH_BASEBACKUP, -# CLONE_HOST=postgresql → resolves to the OLD instance during staging). -# CLONE_* is set on BOTH patroni nodes: whichever wins the initial -# leader race performs the clone; the other syncs from it. +# - patroni-0/patroni-1 bootstrap a NEW cluster (scope: postgres-ha) as a +# Patroni STANDBY CLUSTER continuously streaming from the live +# instance (standby_cluster, host: postgresql — resolves to the OLD +# instance during staging). This REPLACES the earlier +# CLONE_WITH_BASEBACKUP design (one-shot snapshot) — see "REVISION" +# note below for why. # - HAProxy runs WITHOUT production aliases and WITHOUT published ports. # It only gets those in postgresql-ha-final.yaml at promote time. # - Superuser is PGadmin (matches the cloned data's existing superuser, # same password secret), NOT Spilo's default "postgres". # +# ─── REVISION (2026-08-01): switched from CLONE_WITH_BASEBACKUP to a +# Patroni "standby cluster" (continuous streaming) ─── +# +# WHY: CLONE_WITH_BASEBACKUP is a ONE-SHOT snapshot — once pg_basebackup +# completes, the new cluster has ZERO further connection to the live +# database. Any write landing on production between the snapshot and the +# actual traffic cutover would be silently lost — a real risk given ~13 +# active consumers and a 42GB production database. Patroni's standby_cluster +# mode (https://patroni.readthedocs.io/en/latest/standby_cluster.html) +# instead makes the new cluster's leader ("standby leader") continuously +# stream from the remote primary indefinitely, right up until an explicit +# promotion (done by removing the standby_cluster key from DCS config — +# there is no dedicated patronictl subcommand for this) — closing the gap +# to near-zero. Fully validated end-to-end in the pgha-test dry run +# (2026-08-01): bootstrap, continuous live-write propagation to the far +# cascade replica in <3s, and a full promote-with-zero-data-loss cycle — +# see "ADR-0001 Dry-Run Debugging Log" note for the complete play-by-play. +# +# ⚠️ NEW PRODUCTION PREREQUISITE THIS INTRODUCES — NOT YET DONE, must +# happen before this file can actually be deployed for real: the dry run's +# "legacy" stand-in creates a genuine replication role named "standby" +# (matching PGUSER_STANDBY below) via a fresh-initdb hook script, since +# standby_cluster streaming authenticates using the cluster's real +# replication identity (PGUSER_STANDBY/PGPASSWORD_STANDBY), NOT the +# PGadmin superuser that CLONE_WITH_BASEBACKUP used. The LIVE +# `postgresql_postgresql` container has NO such hook mechanism available +# (it's long since initialized, hooks only run on fresh initdb) and does +# NOT currently have a "standby" role. This must be created manually +# against the live database before real cutover, analogous to how the +# pg_hba.conf replication rule was added manually — e.g.: +# CREATE ROLE standby WITH REPLICATION LOGIN PASSWORD ''; +# This is NOT part of this compose file (can't be — it's a one-time SQL +# statement against already-running production, not something a compose +# file can express) but MUST be tracked and done as an explicit manual +# step in the real Phase 3 procedure. See ADR-0001 Dry-Run Debugging Log +# note, "Not yet done" section, for status. +# # NOTE: command: blocks use $$(...) not $(...) — Compose's own variable # interpolation parses $( as an attempted ${VAR} reference and fails with # "invalid interpolation format" / "you may need to escape any $ with @@ -32,9 +71,8 @@ # block, including inside heredocs and nested quoting — Compose scans the # raw text before any shell/heredoc logic ever runs, so quoting context # doesn't exempt anything from this escaping requirement. (Confirmed the -# hard way in the pgha-test dry run — see "ADR-0001 Dry-Run Debugging Log" -# note — this file originally had the unescaped $(...) bug, including on -# the CLONE_PASSWORD line which is unique to this staging file.) +# hard way in the pgha-test dry run — this file originally had the +# unescaped $(...) bug too, before this standby_cluster rewrite.) # # NOTE: ETCD3_HOSTS not ETCD_HOSTS — confirmed against zalando/spilo # configure_spilo.py: PATRONI_DCS includes both "etcd" (legacy v2 API, @@ -42,7 +80,7 @@ # distinct DCS backends selected by env var prefix. Our etcd containers # (v3.5.9) have the v2 API disabled by default, so ETCD_HOSTS causes # Patroni to hit /v2 endpoints that 404. ETCD3_HOSTS selects the correct -# v3-API client. (Also originally wrong in this file — fixed after dry run.) +# v3-API client. # # NOTE: patroni-0/patroni-1 override bootstrap.post_init via # SPILO_CONFIGURATION (Spilo's documented, supported mechanism for @@ -65,7 +103,7 @@ # existence + SUPERUSER) were found and validated end-to-end in the # pgha-test dry run before being ported here — see the dry-run debugging # note for full detail. This is CRITICAL for this file specifically since -# it clones real production data with the real PGadmin superuser. +# it streams real production data with the real PGadmin superuser. # ───────────────────────────────────────────────────────────────────────── version: "3.6" @@ -150,7 +188,6 @@ services: export PGPASSWORD_SUPERUSER="$$(cat /run/secrets/postgresql_password)" export PGPASSWORD_STANDBY="$$(cat /run/secrets/postgresql_replication_password)" export PATRONI_RESTAPI_PASSWORD="$$(cat /run/secrets/postgresql_patroni_password)" - export CLONE_PASSWORD="$$(cat /run/secrets/postgresql_password)" mkdir -p /scripts cat > /scripts/post_init_wrapper.sh <<'WRAP' #!/bin/bash @@ -171,13 +208,12 @@ services: SPILO_CONFIGURATION: | bootstrap: post_init: /scripts/post_init_wrapper.sh "zalandos" - # Clone-from-live: only runs when PGDATA is empty (first bootstrap). - # "postgresql" resolves to the OLD single instance during staging. - CLONE_METHOD: CLONE_WITH_BASEBACKUP - CLONE_SCOPE: legacy-single - CLONE_HOST: postgresql - CLONE_PORT: "5432" - CLONE_USER: PGadmin + dcs: + standby_cluster: + host: postgresql + port: 5432 + create_replica_methods: + - basebackup_fast_xlog secrets: - postgresql_password - postgresql_replication_password @@ -201,7 +237,6 @@ services: export PGPASSWORD_SUPERUSER="$$(cat /run/secrets/postgresql_password)" export PGPASSWORD_STANDBY="$$(cat /run/secrets/postgresql_replication_password)" export PATRONI_RESTAPI_PASSWORD="$$(cat /run/secrets/postgresql_patroni_password)" - export CLONE_PASSWORD="$$(cat /run/secrets/postgresql_password)" mkdir -p /scripts cat > /scripts/post_init_wrapper.sh <<'WRAP' #!/bin/bash @@ -222,11 +257,12 @@ services: SPILO_CONFIGURATION: | bootstrap: post_init: /scripts/post_init_wrapper.sh "zalandos" - CLONE_METHOD: CLONE_WITH_BASEBACKUP - CLONE_SCOPE: legacy-single - CLONE_HOST: postgresql - CLONE_PORT: "5432" - CLONE_USER: PGadmin + dcs: + standby_cluster: + host: postgresql + port: 5432 + create_replica_methods: + - basebackup_fast_xlog secrets: - postgresql_password - postgresql_replication_password