389 lines
17 KiB
YAML
389 lines
17 KiB
YAML
# ─────────────────────────────────────────────────────────────────────────
|
|
# postgresql/cutover/test/pgha-dryrun.yaml — DISPOSABLE dry-run stack
|
|
#
|
|
# TRIGGER NOTE (2026-08-01): touched to force Woodpecker to re-run secret
|
|
# provisioning for the `postgresql` stack case after regenerating
|
|
# postgresql_replication_password to exclude &<>" (see ADR-0001 Dry-Run
|
|
# Debugging Log note for context — Spilo's pystache config templating
|
|
# HTML-escapes double-brace {{PGPASSWORD_STANDBY}} substitutions, corrupting
|
|
# any password containing those characters).
|
|
#
|
|
# Validates the ADR-0001 cutover mechanics WITHOUT touching production:
|
|
# - own overlay network (pgha-test_db-backend), own stack name
|
|
# - throwaway "legacy" postgres:17 seeded with marker data, carrying the
|
|
# in-network alias "postgresql" (mirrors prod CLONE_HOST/standby_cluster
|
|
# host)
|
|
# - same etcd/patroni/haproxy topology and images as the real staging file
|
|
# - reuses the real Docker secrets (read-only mounts; harmless)
|
|
# - disposable data dirs under /volume1/docker/PostgreSQL/dryrun/
|
|
#
|
|
# Deploy: docker stack deploy -c <this file> pgha-test
|
|
# Teardown: docker stack rm pgha-test && rm -rf /volume1/docker/PostgreSQL/dryrun
|
|
#
|
|
# ─── REVISION (2026-08-01): switched from CLONE_WITH_BASEBACKUP to a
|
|
# Patroni "standby cluster" (continuous streaming) — see rationale below ───
|
|
#
|
|
# WHY THIS CHANGED: the original CLONE_WITH_BASEBACKUP design (still used by
|
|
# postgresql/postgresql.yaml and postgresql/cutover/postgresql-ha-staging.yaml
|
|
# as of this revision — NOT YET ported here) is a ONE-SHOT snapshot: once
|
|
# pg_basebackup completes, the new cluster has ZERO further connection to
|
|
# the old database. Any write landing on the old DB between the snapshot
|
|
# and the actual traffic cutover is silently lost — a real gap given ~13
|
|
# active consumers and a 42GB production database. Patroni's "standby
|
|
# cluster" mode (https://patroni.readthedocs.io/en/latest/standby_cluster.html)
|
|
# instead makes the new cluster's leader ("standby leader") continuously
|
|
# stream from the remote primary indefinitely, right up until an explicit
|
|
# promotion — closing that gap to near-zero. This file is the dry-run
|
|
# validation of that mode, BEFORE porting it to the real staging/production
|
|
# files. Do not port until this dry run passes end-to-end.
|
|
#
|
|
# KEY DIFFERENCES FROM THE CLONE-BASED DESIGN:
|
|
# - No CLONE_METHOD/CLONE_SCOPE/CLONE_HOST/CLONE_PORT/CLONE_USER/
|
|
# CLONE_PASSWORD anywhere. Replaced by a `bootstrap.dcs.standby_cluster`
|
|
# block inside SPILO_CONFIGURATION on BOTH patroni-0 and patroni-1 —
|
|
# both get it, not just one, because standby_cluster config is written
|
|
# ONCE into shared DCS state by whichever node wins the initial
|
|
# bootstrap race (same non-determinism previously handled the same way
|
|
# for CLONE_*). Per Patroni docs: "these options will be applied only
|
|
# once during cluster bootstrap, and the only way to change them
|
|
# afterwards is through DCS."
|
|
# - `legacy` now needs a REAL replication role named "standby" (matching
|
|
# PGUSER_STANDBY), not just the PGadmin superuser. CLONE_WITH_BASEBACKUP
|
|
# ran pg_basebackup as CLONE_USER=PGadmin (a superuser, so no dedicated
|
|
# role was needed); standby_cluster streaming instead authenticates
|
|
# using the cluster's own replication identity (PGUSER_STANDBY/
|
|
# PGPASSWORD_STANDBY), which never existed on "legacy" before — added
|
|
# via a second initdb.d hook, reading the password from the SAME
|
|
# postgresql_replication_password secret already used between
|
|
# patroni-0/patroni-1 for their own internal replication.
|
|
# - patroni-1 no longer references CLONE_* either (it never talked to
|
|
# legacy directly in the old design either — it bootstrapped from
|
|
# whichever node held the leader lock, exactly as a Patroni "cascade
|
|
# replica" does in standby-cluster mode too — this is actually simpler
|
|
# now, not different).
|
|
# - create_replica_methods references "basebackup_fast_xlog" — this is
|
|
# Spilo's OWN generated method key (confirmed from configure_spilo.py's
|
|
# TEMPLATE constant, fetched directly from zalando/spilo source this
|
|
# project), not the generic "basebackup" name shown in Patroni's own
|
|
# docs example (that name assumes a plain Patroni method map without
|
|
# Spilo's wrapper naming).
|
|
#
|
|
# BUG FOUND DURING THIS DRY RUN (2026-08-01, unrelated to standby_cluster
|
|
# itself, but discovered while testing it): Spilo's pystache templating
|
|
# engine renders `password: '{{PGPASSWORD_STANDBY}}'` using DOUBLE braces
|
|
# in its TEMPLATE constant (confirmed from configure_spilo.py source) —
|
|
# pystache HTML-escapes double-brace substitutions by default (only
|
|
# triple-brace {{{...}}} skips escaping, which Spilo deliberately uses for
|
|
# archive_command elsewhere in the same template for exactly this reason).
|
|
# This means any password containing &, <, >, or " gets corrupted into
|
|
# HTML entities (e.g. & becomes &) inside Patroni's own rendered
|
|
# /run/postgres.yml AND /run/postgresql/pgpass — breaking real
|
|
# password-based Patroni-to-Patroni replication auth (confirmed via direct
|
|
# file inspection: the raw secret was verified clean via `cat | xxd`, but
|
|
# the rendered config had "&" baked in). This surfaced here specifically
|
|
# because standby_cluster's cascade-replica bootstrap (patroni-0 replicating
|
|
# from patroni-1) uses genuine hostssl+md5 auth, unlike legacy's disposable
|
|
# "trust" rule which never actually checks the password. Fix: regenerate
|
|
# postgresql_replication_password to exclude &<>" entirely — done
|
|
# 2026-08-01, this file re-tests with the corrected secret.
|
|
#
|
|
# OPEN QUESTIONS THIS DRY RUN IS DESIGNED TO ANSWER (do not assume the
|
|
# answer — observe actual logs):
|
|
# - Does Patroni require a pre-created replication slot on "legacy" for
|
|
# the standby_cluster link (primary_slot_name), or does it work
|
|
# without one since we haven't set that key? ANSWERED (first attempt,
|
|
# pre-password-fix): works fine without one — patroni-1 successfully
|
|
# bootstrapped as standby leader and began streaming from legacy with
|
|
# no slot configured.
|
|
# - Does "legacy" (vanilla, unmanaged postgres:17) actually need
|
|
# postgresql.conf discoverable in PGDATA for Patroni's remote checks?
|
|
# ANSWERED: yes, works fine — vanilla image keeps it there by default,
|
|
# no complaint logged.
|
|
# - Does promotion off the standby cluster (detaching from "legacy" and
|
|
# becoming a normal read-write cluster) work cleanly and pick up the
|
|
# very latest streamed WAL, closing the gap as intended? STILL TO TEST
|
|
# — this is the actual feature under test, blocked behind the password
|
|
# bug above on the first attempt, retesting now.
|
|
#
|
|
# ─── Fixes still carried forward from the CLONE_WITH_BASEBACKUP dry run ───
|
|
#
|
|
# NOTE: command: blocks use $$(...) not $(...) — Compose's own variable
|
|
# interpolation parses $( as an attempted ${VAR} reference and fails with
|
|
# "invalid interpolation format" / "you may need to escape any $ with
|
|
# another $". $$ escapes to a literal $ for the shell at runtime. This
|
|
# applies uniformly to EVERY literal $ character anywhere in a command:
|
|
# block, including inside heredocs and nested quoting — Compose scans the
|
|
# raw text before any shell/heredoc logic ever runs, so quoting context
|
|
# doesn't exempt anything from this escaping requirement.
|
|
#
|
|
# NOTE: ETCD3_HOSTS not ETCD_HOSTS — confirmed against zalando/spilo
|
|
# configure_spilo.py: PATRONI_DCS includes both "etcd" (legacy v2 API,
|
|
# python-etcd client) and "etcd3" (v3 API, python-etcd3 client) as
|
|
# distinct DCS backends selected by env var prefix. Our etcd containers
|
|
# (v3.5.9) have the v2 API disabled by default, so ETCD_HOSTS causes
|
|
# Patroni to hit /v2 endpoints that 404. ETCD3_HOSTS selects the correct
|
|
# v3-API client.
|
|
#
|
|
# NOTE: "legacy" needs a pg_hba.conf rule permitting REPLICATION-type
|
|
# connections (distinct from normal client connections) — this is STILL
|
|
# required under standby_cluster mode, since it's still a genuine
|
|
# replication connection under the hood, just a continuous one instead of
|
|
# one-shot. Fixed via a /docker-entrypoint-initdb.d/ hook script (official
|
|
# extension point). "trust" is acceptable ONLY because this container is
|
|
# fully disposable.
|
|
#
|
|
# NOTE: patroni-0/patroni-1 override bootstrap.post_init via
|
|
# SPILO_CONFIGURATION (Spilo's documented, supported mechanism for
|
|
# overriding any generated Patroni config — configure_spilo.py deep-merges
|
|
# user-supplied SPILO_CONFIGURATION on top of its own generated config).
|
|
# This is needed because Spilo's own /scripts/post_init.sh hardcodes
|
|
# "ALTER VIEW ... OWNER TO postgres" with no way to parameterize that role
|
|
# name via env vars, AND its embedded _zmon_schema.dump does
|
|
# "SET ROLE TO postgres; CREATE EXTENSION plpython3u" which requires
|
|
# genuine superuser. Our wrapper creates a harmless, idempotent
|
|
# NOLOGIN+SUPERUSER "postgres" role first (SUPERUSER carries no real
|
|
# exposure since NOLOGIN means it can never authenticate a connection),
|
|
# then execs Spilo's real, UNMODIFIED post_init.sh with all original
|
|
# arguments passed through — Zalando's script itself is never patched or
|
|
# forked. This is UNRELATED to the standby_cluster change and still
|
|
# required for the same reasons as before.
|
|
# ─────────────────────────────────────────────────────────────────────────
|
|
version: "3.6"
|
|
|
|
services:
|
|
|
|
# Stand-in for the production single-instance postgres (remote primary
|
|
# for standby_cluster streaming)
|
|
legacy:
|
|
image: public.ecr.aws/docker/library/postgres:17
|
|
hostname: db
|
|
entrypoint: ["/bin/sh", "-c"]
|
|
command:
|
|
- |
|
|
mkdir -p /docker-entrypoint-initdb.d
|
|
cat > /docker-entrypoint-initdb.d/zz-enable-replication.sh <<'EOF'
|
|
#!/bin/sh
|
|
echo "host replication all all trust" >> "$$PGDATA/pg_hba.conf"
|
|
EOF
|
|
chmod +x /docker-entrypoint-initdb.d/zz-enable-replication.sh
|
|
cat > /docker-entrypoint-initdb.d/zz-create-standby-role.sql <<SQL
|
|
CREATE ROLE standby WITH REPLICATION LOGIN PASSWORD '$$(cat /run/secrets/postgresql_replication_password)';
|
|
SQL
|
|
exec docker-entrypoint.sh postgres
|
|
environment:
|
|
POSTGRES_USER: PGadmin
|
|
POSTGRES_PASSWORD_FILE: /run/secrets/postgresql_password
|
|
secrets:
|
|
- postgresql_password
|
|
- postgresql_replication_password
|
|
volumes:
|
|
- /volume1/docker/PostgreSQL/dryrun/legacy:/var/lib/postgresql/data
|
|
networks:
|
|
db-backend:
|
|
aliases:
|
|
- postgresql
|
|
- db
|
|
deploy:
|
|
placement:
|
|
constraints:
|
|
- node.hostname == docker-2
|
|
|
|
etcd-1:
|
|
image: quay.io/coreos/etcd:v3.5.9
|
|
hostname: etcd-1
|
|
command:
|
|
- etcd
|
|
- --name=etcd-1
|
|
- --data-dir=/etcd-data
|
|
- --initial-advertise-peer-urls=http://etcd-1:2380
|
|
- --listen-peer-urls=http://0.0.0.0:2380
|
|
- --listen-client-urls=http://0.0.0.0:2379
|
|
- --advertise-client-urls=http://etcd-1:2379
|
|
- --initial-cluster=etcd-1=http://etcd-1:2380,etcd-2=http://etcd-2:2380,etcd-3=http://etcd-3:2380
|
|
- --initial-cluster-state=new
|
|
- --initial-cluster-token=pgha-test-etcd
|
|
volumes:
|
|
- /volume1/docker/PostgreSQL/dryrun/etcd-1:/etcd-data
|
|
networks:
|
|
- db-backend
|
|
deploy:
|
|
placement:
|
|
constraints:
|
|
- node.hostname == docker-1
|
|
|
|
etcd-2:
|
|
image: quay.io/coreos/etcd:v3.5.9
|
|
hostname: etcd-2
|
|
command:
|
|
- etcd
|
|
- --name=etcd-2
|
|
- --data-dir=/etcd-data
|
|
- --initial-advertise-peer-urls=http://etcd-2:2380
|
|
- --listen-peer-urls=http://0.0.0.0:2380
|
|
- --listen-client-urls=http://0.0.0.0:2379
|
|
- --advertise-client-urls=http://etcd-2:2379
|
|
- --initial-cluster=etcd-1=http://etcd-1:2380,etcd-2=http://etcd-2:2380,etcd-3=http://etcd-3:2380
|
|
- --initial-cluster-state=new
|
|
- --initial-cluster-token=pgha-test-etcd
|
|
volumes:
|
|
- /volume1/docker/PostgreSQL/dryrun/etcd-2:/etcd-data
|
|
networks:
|
|
- db-backend
|
|
deploy:
|
|
placement:
|
|
constraints:
|
|
- node.hostname == docker-2
|
|
|
|
etcd-3:
|
|
image: quay.io/coreos/etcd:v3.5.9
|
|
hostname: etcd-3
|
|
command:
|
|
- etcd
|
|
- --name=etcd-3
|
|
- --data-dir=/etcd-data
|
|
- --initial-advertise-peer-urls=http://etcd-3:2380
|
|
- --listen-peer-urls=http://0.0.0.0:2380
|
|
- --listen-client-urls=http://0.0.0.0:2379
|
|
- --advertise-client-urls=http://etcd-3:2379
|
|
- --initial-cluster=etcd-1=http://etcd-1:2380,etcd-2=http://etcd-2:2380,etcd-3=http://etcd-3:2380
|
|
- --initial-cluster-state=new
|
|
- --initial-cluster-token=pgha-test-etcd
|
|
volumes:
|
|
- /volume1/docker/PostgreSQL/dryrun/etcd-3:/etcd-data
|
|
networks:
|
|
- db-backend
|
|
deploy:
|
|
placement:
|
|
constraints:
|
|
- node.hostname == docker-3
|
|
|
|
patroni-0:
|
|
image: ghcr.io/zalando/spilo-17:4.0-p3
|
|
hostname: patroni-0
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- |
|
|
export PGPASSWORD_SUPERUSER="$$(cat /run/secrets/postgresql_password)"
|
|
export PGPASSWORD_STANDBY="$$(cat /run/secrets/postgresql_replication_password)"
|
|
export PATRONI_RESTAPI_PASSWORD="$$(cat /run/secrets/postgresql_patroni_password)"
|
|
mkdir -p /scripts
|
|
cat > /scripts/post_init_wrapper.sh <<'WRAP'
|
|
#!/bin/bash
|
|
set -e
|
|
psql -d "$$2" -v ON_ERROR_STOP=1 -c 'DO $$do$$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = $$x$$postgres$$x$$) THEN CREATE ROLE postgres WITH SUPERUSER NOLOGIN; END IF; END $$do$$;'
|
|
exec /scripts/post_init.sh "$$@"
|
|
WRAP
|
|
chmod +x /scripts/post_init_wrapper.sh
|
|
exec /bin/sh /launch.sh init
|
|
environment:
|
|
SCOPE: pgha-test
|
|
PATRONI_NAME: patroni-0
|
|
ETCD3_HOSTS: '"etcd-1:2379","etcd-2:2379","etcd-3:2379"'
|
|
PGUSER_SUPERUSER: PGadmin
|
|
PGUSER_STANDBY: standby
|
|
PATRONI_RESTAPI_USERNAME: patroni
|
|
PGROOT: /home/postgres/pgdata/pgroot
|
|
# Standby-cluster mode: continuous streaming from "legacy" instead of
|
|
# a one-shot CLONE_WITH_BASEBACKUP snapshot. Written once into shared
|
|
# DCS state by whichever of patroni-0/patroni-1 wins the initial
|
|
# bootstrap race — both nodes carry the identical block for that
|
|
# reason. "postgresql" is legacy's network alias (mirrors prod
|
|
# CLONE_HOST naming). basebackup_fast_xlog is Spilo's own generated
|
|
# replica-method key (confirmed from configure_spilo.py source).
|
|
SPILO_CONFIGURATION: |
|
|
bootstrap:
|
|
post_init: /scripts/post_init_wrapper.sh "zalandos"
|
|
dcs:
|
|
standby_cluster:
|
|
host: postgresql
|
|
port: 5432
|
|
create_replica_methods:
|
|
- basebackup_fast_xlog
|
|
secrets:
|
|
- postgresql_password
|
|
- postgresql_replication_password
|
|
- postgresql_patroni_password
|
|
volumes:
|
|
- /volume1/docker/PostgreSQL/dryrun/patroni-0:/home/postgres/pgdata
|
|
networks:
|
|
- db-backend
|
|
deploy:
|
|
placement:
|
|
constraints:
|
|
- node.labels.pg-role == primary
|
|
|
|
patroni-1:
|
|
image: ghcr.io/zalando/spilo-17:4.0-p3
|
|
hostname: patroni-1
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- |
|
|
export PGPASSWORD_SUPERUSER="$$(cat /run/secrets/postgresql_password)"
|
|
export PGPASSWORD_STANDBY="$$(cat /run/secrets/postgresql_replication_password)"
|
|
export PATRONI_RESTAPI_PASSWORD="$$(cat /run/secrets/postgresql_patroni_password)"
|
|
mkdir -p /scripts
|
|
cat > /scripts/post_init_wrapper.sh <<'WRAP'
|
|
#!/bin/bash
|
|
set -e
|
|
psql -d "$$2" -v ON_ERROR_STOP=1 -c 'DO $$do$$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = $$x$$postgres$$x$$) THEN CREATE ROLE postgres WITH SUPERUSER NOLOGIN; END IF; END $$do$$;'
|
|
exec /scripts/post_init.sh "$$@"
|
|
WRAP
|
|
chmod +x /scripts/post_init_wrapper.sh
|
|
exec /bin/sh /launch.sh init
|
|
environment:
|
|
SCOPE: pgha-test
|
|
PATRONI_NAME: patroni-1
|
|
ETCD3_HOSTS: '"etcd-1:2379","etcd-2:2379","etcd-3:2379"'
|
|
PGUSER_SUPERUSER: PGadmin
|
|
PGUSER_STANDBY: standby
|
|
PATRONI_RESTAPI_USERNAME: patroni
|
|
PGROOT: /home/postgres/pgdata/pgroot
|
|
# Same standby_cluster block as patroni-0 — see comment there for why
|
|
# both nodes carry it identically.
|
|
SPILO_CONFIGURATION: |
|
|
bootstrap:
|
|
post_init: /scripts/post_init_wrapper.sh "zalandos"
|
|
dcs:
|
|
standby_cluster:
|
|
host: postgresql
|
|
port: 5432
|
|
create_replica_methods:
|
|
- basebackup_fast_xlog
|
|
secrets:
|
|
- postgresql_password
|
|
- postgresql_replication_password
|
|
- postgresql_patroni_password
|
|
volumes:
|
|
- /volume1/docker/PostgreSQL/dryrun/patroni-1:/home/postgres/pgdata
|
|
networks:
|
|
- db-backend
|
|
deploy:
|
|
placement:
|
|
constraints:
|
|
- node.labels.pg-role == replica
|
|
|
|
haproxy:
|
|
image: haproxy:2.9-alpine
|
|
hostname: haproxy
|
|
volumes:
|
|
- /volume1/docker/compose-files/postgresql/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
|
networks:
|
|
- db-backend
|
|
deploy:
|
|
mode: global
|
|
|
|
networks:
|
|
db-backend:
|
|
attachable: true
|
|
driver: overlay
|
|
|
|
secrets:
|
|
postgresql_password:
|
|
external: true
|
|
postgresql_replication_password:
|
|
external: true
|
|
postgresql_patroni_password:
|
|
external: true
|