admin
8f860ab14d
ci/woodpecker/push/deploy Pipeline was successful
Mirrors the fix in postgresql-ha-staging.yaml: added primary_slot_name: standby_leader_slot to both patroni-0 and patroni-1's standby_cluster blocks so this file's specs stay byte-identical to staging's, per this file's own "do not recreate the live promoted primary/replica" design requirement. See postgresql-ha-staging.yaml header and the ADR-0001 note for full incident detail.
262 lines
11 KiB
YAML
262 lines
11 KiB
YAML
# ─────────────────────────────────────────────────────────────────────────
|
|
# postgresql/cutover/postgresql-ha-final.yaml — ADR-0001 Phase 3, Stage 3
|
|
#
|
|
# Deployed by cutover.sh (Phase 7) as an UPDATE to the SAME "postgresqlha"
|
|
# stack that postgresql-ha-staging.yaml created in Stage 2:
|
|
# docker stack deploy -c postgresql/cutover/postgresql-ha-final.yaml postgresqlha
|
|
#
|
|
# ⚠️ Do NOT deploy this before Patroni has been genuinely promoted
|
|
# (cutover.sh Phase 5) and confirmed healthy (Phase 6). Deploying it
|
|
# earlier adds the postgresql/db aliases to a backend HAProxy's own
|
|
# healthcheck (GET /primary) will correctly report as unhealthy (a
|
|
# standby_leader is read-only, not a real primary) — any client that
|
|
# Swarm's round-robin DNS routes to HAProxy during that window would
|
|
# hit zero healthy servers and fail outright. See RUNBOOK.md for the
|
|
# full rationale on phase ordering.
|
|
#
|
|
# What changed vs. postgresql-ha-staging.yaml (Stage 2):
|
|
# - etcd-1/2/3 and patroni-0/patroni-1 service specs are BYTE-IDENTICAL
|
|
# to the staging file, deliberately. `docker stack deploy` diffs each
|
|
# service's spec independently — since these five specs are unchanged,
|
|
# Swarm will NOT recreate/restart them. This matters enormously here:
|
|
# patroni-1 is the live, newly-promoted PRIMARY carrying real
|
|
# production traffic by the time this file is deployed, and
|
|
# patroni-0 is its live replica. Neither may be disrupted by this
|
|
# step. Only the haproxy service (below) has a spec change, so only
|
|
# it gets updated.
|
|
# - haproxy service: adds `postgresql` and `db` as network aliases on
|
|
# postgresql_db-backend — this is the actual DNS cutover mechanism.
|
|
# Still deliberately WITHOUT the external port publish (5430) or
|
|
# traefik_backend network/labels that the STEADY-STATE production
|
|
# postgresql.yaml's haproxy block eventually needs — taking over
|
|
# that external/Traefik-SNI path is a documented MANUAL, POST-cutover
|
|
# step (see RUNBOOK.md "Post-cutover follow-up"), not part of this
|
|
# automated procedure. Confirmed with the operator: no external
|
|
# consumer depends on port 5430 during the cutover window, so this
|
|
# is deferred deliberately, not an oversight.
|
|
# - Uses the same postgresql/haproxy.cfg already committed to git
|
|
# (Patroni-aware TCP router, GET /primary healthcheck, no changes
|
|
# needed for this stage).
|
|
#
|
|
# Once this is live and all consumers have verified (cutover.sh Phase 9),
|
|
# legacy is scaled to 0 (Phase 10) and this becomes the sole answer for
|
|
# the `postgresql`/`db` names on postgresql_db-backend.
|
|
#
|
|
# ── primary_slot_name (2026-08-04) ──────────────────────────────────────
|
|
# patroni-0/patroni-1's standby_cluster blocks below include
|
|
# `primary_slot_name: standby_leader_slot`, added to fix a real bootstrap
|
|
# failure (basebackup succeeded but the resulting Postgres process got
|
|
# stuck in "starting" forever because legacy had no replication slot
|
|
# protecting the WAL it needed to resume streaming — recycled by normal
|
|
# checkpoint activity during the 8-11 min basebackup window). Full
|
|
# incident detail and the matching production-side fix (a physical slot
|
|
# named standby_leader_slot created on legacy, plus wal_keep_size bumped
|
|
# to 4GB as defense-in-depth) are documented in postgresql-ha-staging.yaml's
|
|
# header and in the "ADR-0001 Dry-Run Debugging Log" note. This value MUST
|
|
# stay identical to the staging file's per the byte-identical requirement
|
|
# noted above — if you ever change one, change both.
|
|
# ─────────────────────────────────────────────────────────────────────────
|
|
version: "3.6"
|
|
|
|
services:
|
|
|
|
etcd-1:
|
|
image: quay.io/coreos/etcd:v3.5.9
|
|
hostname: etcd-1
|
|
command:
|
|
- etcd
|
|
- --name=etcd-1
|
|
- --data-dir=/etcd-data
|
|
- --initial-advertise-peer-urls=http://etcd-1:2380
|
|
- --listen-peer-urls=http://0.0.0.0:2380
|
|
- --listen-client-urls=http://0.0.0.0:2379
|
|
- --advertise-client-urls=http://etcd-1:2379
|
|
- --initial-cluster=etcd-1=http://etcd-1:2380,etcd-2=http://etcd-2:2380,etcd-3=http://etcd-3:2380
|
|
- --initial-cluster-state=new
|
|
- --initial-cluster-token=postgresql-ha-etcd
|
|
volumes:
|
|
- /volume1/docker/PostgreSQL/etcd-1-data:/etcd-data
|
|
networks:
|
|
- postgresql_db-backend
|
|
deploy:
|
|
placement:
|
|
constraints:
|
|
- node.hostname == docker-1
|
|
|
|
etcd-2:
|
|
image: quay.io/coreos/etcd:v3.5.9
|
|
hostname: etcd-2
|
|
command:
|
|
- etcd
|
|
- --name=etcd-2
|
|
- --data-dir=/etcd-data
|
|
- --initial-advertise-peer-urls=http://etcd-2:2380
|
|
- --listen-peer-urls=http://0.0.0.0:2380
|
|
- --listen-client-urls=http://0.0.0.0:2379
|
|
- --advertise-client-urls=http://etcd-2:2379
|
|
- --initial-cluster=etcd-1=http://etcd-1:2380,etcd-2=http://etcd-2:2380,etcd-3=http://etcd-3:2380
|
|
- --initial-cluster-state=new
|
|
- --initial-cluster-token=postgresql-ha-etcd
|
|
volumes:
|
|
- /volume1/docker/PostgreSQL/etcd-2-data:/etcd-data
|
|
networks:
|
|
- postgresql_db-backend
|
|
deploy:
|
|
placement:
|
|
constraints:
|
|
- node.hostname == docker-2
|
|
|
|
etcd-3:
|
|
image: quay.io/coreos/etcd:v3.5.9
|
|
hostname: etcd-3
|
|
command:
|
|
- etcd
|
|
- --name=etcd-3
|
|
- --data-dir=/etcd-data
|
|
- --initial-advertise-peer-urls=http://etcd-3:2380
|
|
- --listen-peer-urls=http://0.0.0.0:2380
|
|
- --listen-client-urls=http://0.0.0.0:2379
|
|
- --advertise-client-urls=http://etcd-3:2379
|
|
- --initial-cluster=etcd-1=http://etcd-1:2380,etcd-2=http://etcd-2:2380,etcd-3=http://etcd-3:2380
|
|
- --initial-cluster-state=new
|
|
- --initial-cluster-token=postgresql-ha-etcd
|
|
volumes:
|
|
- /volume1/docker/PostgreSQL/etcd-3-data:/etcd-data
|
|
networks:
|
|
- postgresql_db-backend
|
|
deploy:
|
|
placement:
|
|
constraints:
|
|
- node.hostname == docker-3
|
|
|
|
patroni-0:
|
|
image: ghcr.io/zalando/spilo-17:4.0-p3
|
|
hostname: patroni-0
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- |
|
|
export PGPASSWORD_SUPERUSER="$$(cat /run/secrets/postgresql_password)"
|
|
export PGPASSWORD_STANDBY="$$(cat /run/secrets/postgresql_replication_password)"
|
|
export PATRONI_RESTAPI_PASSWORD="$$(cat /run/secrets/postgresql_patroni_password)"
|
|
mkdir -p /scripts
|
|
cat > /scripts/post_init_wrapper.sh <<'WRAP'
|
|
#!/bin/bash
|
|
set -e
|
|
psql -d "$$2" -v ON_ERROR_STOP=1 -c 'DO $$do$$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = $$x$$postgres$$x$$) THEN CREATE ROLE postgres WITH SUPERUSER NOLOGIN; END IF; END $$do$$;'
|
|
exec /scripts/post_init.sh "$$@"
|
|
WRAP
|
|
chmod +x /scripts/post_init_wrapper.sh
|
|
exec /bin/sh /launch.sh init
|
|
environment:
|
|
SCOPE: postgres-ha
|
|
PATRONI_NAME: patroni-0
|
|
ETCD3_HOSTS: '"etcd-1:2379","etcd-2:2379","etcd-3:2379"'
|
|
PGUSER_SUPERUSER: PGadmin
|
|
PGUSER_STANDBY: standby
|
|
PATRONI_RESTAPI_USERNAME: patroni
|
|
PGROOT: /home/postgres/pgdata/pgroot
|
|
SPILO_CONFIGURATION: |
|
|
bootstrap:
|
|
post_init: /scripts/post_init_wrapper.sh "zalandos"
|
|
dcs:
|
|
standby_cluster:
|
|
host: postgresql
|
|
port: 5432
|
|
primary_slot_name: standby_leader_slot
|
|
create_replica_methods:
|
|
- basebackup_fast_xlog
|
|
secrets:
|
|
- postgresql_password
|
|
- postgresql_replication_password
|
|
- postgresql_patroni_password
|
|
volumes:
|
|
- /volume1/docker/PostgreSQL/patroni-0-data:/home/postgres/pgdata
|
|
networks:
|
|
- postgresql_db-backend
|
|
deploy:
|
|
placement:
|
|
constraints:
|
|
- node.labels.pg-role == primary
|
|
|
|
patroni-1:
|
|
image: ghcr.io/zalando/spilo-17:4.0-p3
|
|
hostname: patroni-1
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- |
|
|
export PGPASSWORD_SUPERUSER="$$(cat /run/secrets/postgresql_password)"
|
|
export PGPASSWORD_STANDBY="$$(cat /run/secrets/postgresql_replication_password)"
|
|
export PATRONI_RESTAPI_PASSWORD="$$(cat /run/secrets/postgresql_patroni_password)"
|
|
mkdir -p /scripts
|
|
cat > /scripts/post_init_wrapper.sh <<'WRAP'
|
|
#!/bin/bash
|
|
set -e
|
|
psql -d "$$2" -v ON_ERROR_STOP=1 -c 'DO $$do$$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = $$x$$postgres$$x$$) THEN CREATE ROLE postgres WITH SUPERUSER NOLOGIN; END IF; END $$do$$;'
|
|
exec /scripts/post_init.sh "$$@"
|
|
WRAP
|
|
chmod +x /scripts/post_init_wrapper.sh
|
|
exec /bin/sh /launch.sh init
|
|
environment:
|
|
SCOPE: postgres-ha
|
|
PATRONI_NAME: patroni-1
|
|
ETCD3_HOSTS: '"etcd-1:2379","etcd-2:2379","etcd-3:2379"'
|
|
PGUSER_SUPERUSER: PGadmin
|
|
PGUSER_STANDBY: standby
|
|
PATRONI_RESTAPI_USERNAME: patroni
|
|
PGROOT: /home/postgres/pgdata/pgroot
|
|
SPILO_CONFIGURATION: |
|
|
bootstrap:
|
|
post_init: /scripts/post_init_wrapper.sh "zalandos"
|
|
dcs:
|
|
standby_cluster:
|
|
host: postgresql
|
|
port: 5432
|
|
primary_slot_name: standby_leader_slot
|
|
create_replica_methods:
|
|
- basebackup_fast_xlog
|
|
secrets:
|
|
- postgresql_password
|
|
- postgresql_replication_password
|
|
- postgresql_patroni_password
|
|
volumes:
|
|
- /volume1/docker/PostgreSQL/patroni-1-data:/home/postgres/pgdata
|
|
networks:
|
|
- postgresql_db-backend
|
|
deploy:
|
|
placement:
|
|
constraints:
|
|
- node.labels.pg-role == replica
|
|
|
|
# ── Final-stage HAProxy: takes over the postgresql/db aliases ──────────
|
|
# Internal Swarm-DNS only at this stage — no published port, no
|
|
# traefik_backend network/labels. External access (port 5430 /
|
|
# Traefik SNI) is a deliberate, documented MANUAL post-cutover step,
|
|
# not part of this automated procedure — confirmed no external
|
|
# consumer depends on it during the cutover window.
|
|
haproxy:
|
|
image: haproxy:2.9-alpine
|
|
hostname: haproxy
|
|
volumes:
|
|
- /volume1/docker/compose-files/postgresql/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
|
networks:
|
|
postgresql_db-backend:
|
|
aliases:
|
|
- postgresql
|
|
- db
|
|
deploy:
|
|
mode: global
|
|
|
|
networks:
|
|
postgresql_db-backend:
|
|
external: true
|
|
|
|
secrets:
|
|
postgresql_password:
|
|
external: true
|
|
postgresql_replication_password:
|
|
external: true
|
|
postgresql_patroni_password:
|
|
external: true
|