compose-files/postgresql/cutover/postgresql-ha-staging.yaml

# ─────────────────────────────────────────────────────────────────────────
# postgresql/cutover/postgresql-ha-staging.yaml — ADR-0001 Phase 3, Stage 2
#
# Deployed by postgres-ha-cutover.sh as a SEPARATE stack:
#   docker stack deploy -c postgresql/cutover/postgresql-ha-staging.yaml postgresqlha
#
# ⚠️  NEVER deploy this via stack-deploy.sh or under the "postgresql" stack
#     name. It lives in cutover/ (a subdirectory) deliberately:
#     stack-deploy.sh folder mode merges all *.yml/yaml at -maxdepth 1 of
#     postgresql/, and merging this into the production stack would cause
#     an outage. Subdirectory files are excluded from that merge.
#
# What this stage does:
#   - Joins the EXISTING postgresql_db-backend overlay (external) — the
#     live single-instance stack is untouched and keeps the "postgresql"
#     and "db" aliases.
#   - patroni-0/patroni-1 bootstrap a NEW cluster (scope: postgres-ha) by
#     CLONING the live instance via pg_basebackup (CLONE_WITH_BASEBACKUP,
#     CLONE_HOST=postgresql → resolves to the OLD instance during staging).
#     CLONE_* is set on BOTH patroni nodes: whichever wins the initial
#     leader race performs the clone; the other syncs from it.
#   - HAProxy runs WITHOUT production aliases and WITHOUT published ports.
#     It only gets those in postgresql-ha-final.yaml at promote time.
#   - Superuser is PGadmin (matches the cloned data's existing superuser,
#     same password secret), NOT Spilo's default "postgres".
#
# NOTE: command: blocks use $$(...) not $(...) — Compose's own variable
# interpolation parses $( as an attempted ${VAR} reference and fails with
# "invalid interpolation format" / "you may need to escape any $ with
# another $". $$ escapes to a literal $ for the shell at runtime. This
# applies uniformly to EVERY literal $ character anywhere in a command:
# block, including inside heredocs and nested quoting — Compose scans the
# raw text before any shell/heredoc logic ever runs, so quoting context
# doesn't exempt anything from this escaping requirement. (Confirmed the
# hard way in the pgha-test dry run — see "ADR-0001 Dry-Run Debugging Log"
# note — this file originally had the unescaped $(...) bug, including on
# the CLONE_PASSWORD line which is unique to this staging file.)
#
# NOTE: ETCD3_HOSTS not ETCD_HOSTS — confirmed against zalando/spilo
# configure_spilo.py: PATRONI_DCS includes both "etcd" (legacy v2 API,
# python-etcd client) and "etcd3" (v3 API, python-etcd3 client) as
# distinct DCS backends selected by env var prefix. Our etcd containers
# (v3.5.9) have the v2 API disabled by default, so ETCD_HOSTS causes
# Patroni to hit /v2 endpoints that 404. ETCD3_HOSTS selects the correct
# v3-API client. (Also originally wrong in this file — fixed after dry run.)
#
# NOTE: patroni-0/patroni-1 override bootstrap.post_init via
# SPILO_CONFIGURATION (Spilo's documented, supported mechanism for
# overriding any generated Patroni config — configure_spilo.py deep-merges
# user-supplied SPILO_CONFIGURATION on top of its own generated config).
# This is needed because Spilo's own /scripts/post_init.sh hardcodes
# "ALTER VIEW ... OWNER TO postgres" with no way to parameterize that role
# name via env vars. Since our superuser is PGUSER_SUPERUSER=PGadmin, there
# is no role literally named "postgres" in the cloned data, so Spilo's
# unmodified script fails with 'ERROR: role "postgres" does not exist'.
# The override points bootstrap.post_init at our own wrapper script
# instead, which creates a harmless, idempotent "postgres" role (WITH
# SUPERUSER NOLOGIN — SUPERUSER is required because Spilo's own
# _zmon_schema.dump does "SET ROLE TO postgres; CREATE EXTENSION
# plpython3u", which needs real superuser privileges, not just role
# existence; NOLOGIN means it can never be used to establish a real
# connection, so this carries no security exposure), then execs Spilo's
# real, UNMODIFIED post_init.sh with all original arguments passed through
# — Zalando's script itself is never patched or forked. Both fixes (role
# existence + SUPERUSER) were found and validated end-to-end in the
# pgha-test dry run before being ported here — see the dry-run debugging
# note for full detail. This is CRITICAL for this file specifically since
# it clones real production data with the real PGadmin superuser.
# ─────────────────────────────────────────────────────────────────────────
version: "3.6"

services:

  etcd-1:
    image: quay.io/coreos/etcd:v3.5.9
    hostname: etcd-1
    command:
      - etcd
      - --name=etcd-1
      - --data-dir=/etcd-data
      - --initial-advertise-peer-urls=http://etcd-1:2380
      - --listen-peer-urls=http://0.0.0.0:2380
      - --listen-client-urls=http://0.0.0.0:2379
      - --advertise-client-urls=http://etcd-1:2379
      - --initial-cluster=etcd-1=http://etcd-1:2380,etcd-2=http://etcd-2:2380,etcd-3=http://etcd-3:2380
      - --initial-cluster-state=new
      - --initial-cluster-token=postgresql-ha-etcd
    volumes:
      - /volume1/docker/PostgreSQL/etcd-1-data:/etcd-data
    networks:
      - postgresql_db-backend
    deploy:
      placement:
        constraints:
          - node.hostname == docker-1

  etcd-2:
    image: quay.io/coreos/etcd:v3.5.9
    hostname: etcd-2
    command:
      - etcd
      - --name=etcd-2
      - --data-dir=/etcd-data
      - --initial-advertise-peer-urls=http://etcd-2:2380
      - --listen-peer-urls=http://0.0.0.0:2380
      - --listen-client-urls=http://0.0.0.0:2379
      - --advertise-client-urls=http://etcd-2:2379
      - --initial-cluster=etcd-1=http://etcd-1:2380,etcd-2=http://etcd-2:2380,etcd-3=http://etcd-3:2380
      - --initial-cluster-state=new
      - --initial-cluster-token=postgresql-ha-etcd
    volumes:
      - /volume1/docker/PostgreSQL/etcd-2-data:/etcd-data
    networks:
      - postgresql_db-backend
    deploy:
      placement:
        constraints:
          - node.hostname == docker-2

  etcd-3:
    image: quay.io/coreos/etcd:v3.5.9
    hostname: etcd-3
    command:
      - etcd
      - --name=etcd-3
      - --data-dir=/etcd-data
      - --initial-advertise-peer-urls=http://etcd-3:2380
      - --listen-peer-urls=http://0.0.0.0:2380
      - --listen-client-urls=http://0.0.0.0:2379
      - --advertise-client-urls=http://etcd-3:2379
      - --initial-cluster=etcd-1=http://etcd-1:2380,etcd-2=http://etcd-2:2380,etcd-3=http://etcd-3:2380
      - --initial-cluster-state=new
      - --initial-cluster-token=postgresql-ha-etcd
    volumes:
      - /volume1/docker/PostgreSQL/etcd-3-data:/etcd-data
    networks:
      - postgresql_db-backend
    deploy:
      placement:
        constraints:
          - node.hostname == docker-3

  patroni-0:
    image: ghcr.io/zalando/spilo-17:4.0-p3
    hostname: patroni-0
    command:
      - /bin/sh
      - -c
      - |
        export PGPASSWORD_SUPERUSER="$$(cat /run/secrets/postgresql_password)"
        export PGPASSWORD_STANDBY="$$(cat /run/secrets/postgresql_replication_password)"
        export PATRONI_RESTAPI_PASSWORD="$$(cat /run/secrets/postgresql_patroni_password)"
        export CLONE_PASSWORD="$$(cat /run/secrets/postgresql_password)"
        mkdir -p /scripts
        cat > /scripts/post_init_wrapper.sh <<'WRAP'
        #!/bin/bash
        set -e
        psql -d "$$2" -v ON_ERROR_STOP=1 -c 'DO $$do$$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = $$x$$postgres$$x$$) THEN CREATE ROLE postgres WITH SUPERUSER NOLOGIN; END IF; END $$do$$;'
        exec /scripts/post_init.sh "$$@"
        WRAP
        chmod +x /scripts/post_init_wrapper.sh
        exec /bin/sh /launch.sh init
    environment:
      SCOPE: postgres-ha
      PATRONI_NAME: patroni-0
      ETCD3_HOSTS: '"etcd-1:2379","etcd-2:2379","etcd-3:2379"'
      PGUSER_SUPERUSER: PGadmin
      PGUSER_STANDBY: standby
      PATRONI_RESTAPI_USERNAME: patroni
      PGROOT: /home/postgres/pgdata/pgroot
      SPILO_CONFIGURATION: |
        bootstrap:
          post_init: /scripts/post_init_wrapper.sh "zalandos"
      # Clone-from-live: only runs when PGDATA is empty (first bootstrap).
      # "postgresql" resolves to the OLD single instance during staging.
      CLONE_METHOD: CLONE_WITH_BASEBACKUP
      CLONE_SCOPE: legacy-single
      CLONE_HOST: postgresql
      CLONE_PORT: "5432"
      CLONE_USER: PGadmin
    secrets:
      - postgresql_password
      - postgresql_replication_password
      - postgresql_patroni_password
    volumes:
      - /volume1/docker/PostgreSQL/patroni-0-data:/home/postgres/pgdata
    networks:
      - postgresql_db-backend
    deploy:
      placement:
        constraints:
          - node.labels.pg-role == primary

  patroni-1:
    image: ghcr.io/zalando/spilo-17:4.0-p3
    hostname: patroni-1
    command:
      - /bin/sh
      - -c
      - |
        export PGPASSWORD_SUPERUSER="$$(cat /run/secrets/postgresql_password)"
        export PGPASSWORD_STANDBY="$$(cat /run/secrets/postgresql_replication_password)"
        export PATRONI_RESTAPI_PASSWORD="$$(cat /run/secrets/postgresql_patroni_password)"
        export CLONE_PASSWORD="$$(cat /run/secrets/postgresql_password)"
        mkdir -p /scripts
        cat > /scripts/post_init_wrapper.sh <<'WRAP'
        #!/bin/bash
        set -e
        psql -d "$$2" -v ON_ERROR_STOP=1 -c 'DO $$do$$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = $$x$$postgres$$x$$) THEN CREATE ROLE postgres WITH SUPERUSER NOLOGIN; END IF; END $$do$$;'
        exec /scripts/post_init.sh "$$@"
        WRAP
        chmod +x /scripts/post_init_wrapper.sh
        exec /bin/sh /launch.sh init
    environment:
      SCOPE: postgres-ha
      PATRONI_NAME: patroni-1
      ETCD3_HOSTS: '"etcd-1:2379","etcd-2:2379","etcd-3:2379"'
      PGUSER_SUPERUSER: PGadmin
      PGUSER_STANDBY: standby
      PATRONI_RESTAPI_USERNAME: patroni
      PGROOT: /home/postgres/pgdata/pgroot
      SPILO_CONFIGURATION: |
        bootstrap:
          post_init: /scripts/post_init_wrapper.sh "zalandos"
      CLONE_METHOD: CLONE_WITH_BASEBACKUP
      CLONE_SCOPE: legacy-single
      CLONE_HOST: postgresql
      CLONE_PORT: "5432"
      CLONE_USER: PGadmin
    secrets:
      - postgresql_password
      - postgresql_replication_password
      - postgresql_patroni_password
    volumes:
      - /volume1/docker/PostgreSQL/patroni-1-data:/home/postgres/pgdata
    networks:
      - postgresql_db-backend
    deploy:
      placement:
        constraints:
          - node.labels.pg-role == replica

  # Staging HAProxy: NO production aliases, NO published ports. Reachable
  # in-network as "haproxy" for pre-promote testing only.
  haproxy:
    image: haproxy:2.9-alpine
    hostname: haproxy
    volumes:
      - /volume1/docker/compose-files/postgresql/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
    networks:
      - postgresql_db-backend
    deploy:
      mode: global

networks:
  postgresql_db-backend:
    external: true

secrets:
  postgresql_password:
    external: true
  postgresql_replication_password:
    external: true
  postgresql_patroni_password:
    external: true