From 499402cd0d3554a55fcbf8fd293c4fd5a6301499 Mon Sep 17 00:00:00 2001 From: admin Date: Thu, 30 Jul 2026 15:56:43 -0700 Subject: [PATCH] postgresql-ha-staging.yaml: port dry-run fixes (bugs 1,2,4,5) - $$(...) escaping (incl. CLONE_PASSWORD), ETCD3_HOSTS, post_init_wrapper.sh SUPERUSER role fix --- postgresql/cutover/postgresql-ha-staging.yaml | 85 ++++++++++++++++--- 1 file changed, 75 insertions(+), 10 deletions(-) diff --git a/postgresql/cutover/postgresql-ha-staging.yaml b/postgresql/cutover/postgresql-ha-staging.yaml index c2b3ce1..4e24f29 100644 --- a/postgresql/cutover/postgresql-ha-staging.yaml +++ b/postgresql/cutover/postgresql-ha-staging.yaml @@ -23,6 +23,49 @@ # It only gets those in postgresql-ha-final.yaml at promote time. # - Superuser is PGadmin (matches the cloned data's existing superuser, # same password secret), NOT Spilo's default "postgres". +# +# NOTE: command: blocks use $$(...) not $(...) — Compose's own variable +# interpolation parses $( as an attempted ${VAR} reference and fails with +# "invalid interpolation format" / "you may need to escape any $ with +# another $". $$ escapes to a literal $ for the shell at runtime. This +# applies uniformly to EVERY literal $ character anywhere in a command: +# block, including inside heredocs and nested quoting — Compose scans the +# raw text before any shell/heredoc logic ever runs, so quoting context +# doesn't exempt anything from this escaping requirement. (Confirmed the +# hard way in the pgha-test dry run — see "ADR-0001 Dry-Run Debugging Log" +# note — this file originally had the unescaped $(...) bug, including on +# the CLONE_PASSWORD line which is unique to this staging file.) +# +# NOTE: ETCD3_HOSTS not ETCD_HOSTS — confirmed against zalando/spilo +# configure_spilo.py: PATRONI_DCS includes both "etcd" (legacy v2 API, +# python-etcd client) and "etcd3" (v3 API, python-etcd3 client) as +# distinct DCS backends selected by env var prefix. Our etcd containers +# (v3.5.9) have the v2 API disabled by default, so ETCD_HOSTS causes +# Patroni to hit /v2 endpoints that 404. ETCD3_HOSTS selects the correct +# v3-API client. (Also originally wrong in this file — fixed after dry run.) +# +# NOTE: patroni-0/patroni-1 override bootstrap.post_init via +# SPILO_CONFIGURATION (Spilo's documented, supported mechanism for +# overriding any generated Patroni config — configure_spilo.py deep-merges +# user-supplied SPILO_CONFIGURATION on top of its own generated config). +# This is needed because Spilo's own /scripts/post_init.sh hardcodes +# "ALTER VIEW ... OWNER TO postgres" with no way to parameterize that role +# name via env vars. Since our superuser is PGUSER_SUPERUSER=PGadmin, there +# is no role literally named "postgres" in the cloned data, so Spilo's +# unmodified script fails with 'ERROR: role "postgres" does not exist'. +# The override points bootstrap.post_init at our own wrapper script +# instead, which creates a harmless, idempotent "postgres" role (WITH +# SUPERUSER NOLOGIN — SUPERUSER is required because Spilo's own +# _zmon_schema.dump does "SET ROLE TO postgres; CREATE EXTENSION +# plpython3u", which needs real superuser privileges, not just role +# existence; NOLOGIN means it can never be used to establish a real +# connection, so this carries no security exposure), then execs Spilo's +# real, UNMODIFIED post_init.sh with all original arguments passed through +# — Zalando's script itself is never patched or forked. Both fixes (role +# existence + SUPERUSER) were found and validated end-to-end in the +# pgha-test dry run before being ported here — see the dry-run debugging +# note for full detail. This is CRITICAL for this file specifically since +# it clones real production data with the real PGadmin superuser. # ───────────────────────────────────────────────────────────────────────── version: "3.6" @@ -104,19 +147,30 @@ services: - /bin/sh - -c - | - export PGPASSWORD_SUPERUSER="$(cat /run/secrets/postgresql_password)" - export PGPASSWORD_STANDBY="$(cat /run/secrets/postgresql_replication_password)" - export PATRONI_RESTAPI_PASSWORD="$(cat /run/secrets/postgresql_patroni_password)" - export CLONE_PASSWORD="$(cat /run/secrets/postgresql_password)" + export PGPASSWORD_SUPERUSER="$$(cat /run/secrets/postgresql_password)" + export PGPASSWORD_STANDBY="$$(cat /run/secrets/postgresql_replication_password)" + export PATRONI_RESTAPI_PASSWORD="$$(cat /run/secrets/postgresql_patroni_password)" + export CLONE_PASSWORD="$$(cat /run/secrets/postgresql_password)" + mkdir -p /scripts + cat > /scripts/post_init_wrapper.sh <<'WRAP' + #!/bin/bash + set -e + psql -d "$$2" -v ON_ERROR_STOP=1 -c 'DO $$do$$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = $$x$$postgres$$x$$) THEN CREATE ROLE postgres WITH SUPERUSER NOLOGIN; END IF; END $$do$$;' + exec /scripts/post_init.sh "$$@" + WRAP + chmod +x /scripts/post_init_wrapper.sh exec /bin/sh /launch.sh init environment: SCOPE: postgres-ha PATRONI_NAME: patroni-0 - ETCD_HOSTS: '"etcd-1:2379","etcd-2:2379","etcd-3:2379"' + ETCD3_HOSTS: '"etcd-1:2379","etcd-2:2379","etcd-3:2379"' PGUSER_SUPERUSER: PGadmin PGUSER_STANDBY: standby PATRONI_RESTAPI_USERNAME: patroni PGROOT: /home/postgres/pgdata/pgroot + SPILO_CONFIGURATION: | + bootstrap: + post_init: /scripts/post_init_wrapper.sh "zalandos" # Clone-from-live: only runs when PGDATA is empty (first bootstrap). # "postgresql" resolves to the OLD single instance during staging. CLONE_METHOD: CLONE_WITH_BASEBACKUP @@ -144,19 +198,30 @@ services: - /bin/sh - -c - | - export PGPASSWORD_SUPERUSER="$(cat /run/secrets/postgresql_password)" - export PGPASSWORD_STANDBY="$(cat /run/secrets/postgresql_replication_password)" - export PATRONI_RESTAPI_PASSWORD="$(cat /run/secrets/postgresql_patroni_password)" - export CLONE_PASSWORD="$(cat /run/secrets/postgresql_password)" + export PGPASSWORD_SUPERUSER="$$(cat /run/secrets/postgresql_password)" + export PGPASSWORD_STANDBY="$$(cat /run/secrets/postgresql_replication_password)" + export PATRONI_RESTAPI_PASSWORD="$$(cat /run/secrets/postgresql_patroni_password)" + export CLONE_PASSWORD="$$(cat /run/secrets/postgresql_password)" + mkdir -p /scripts + cat > /scripts/post_init_wrapper.sh <<'WRAP' + #!/bin/bash + set -e + psql -d "$$2" -v ON_ERROR_STOP=1 -c 'DO $$do$$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = $$x$$postgres$$x$$) THEN CREATE ROLE postgres WITH SUPERUSER NOLOGIN; END IF; END $$do$$;' + exec /scripts/post_init.sh "$$@" + WRAP + chmod +x /scripts/post_init_wrapper.sh exec /bin/sh /launch.sh init environment: SCOPE: postgres-ha PATRONI_NAME: patroni-1 - ETCD_HOSTS: '"etcd-1:2379","etcd-2:2379","etcd-3:2379"' + ETCD3_HOSTS: '"etcd-1:2379","etcd-2:2379","etcd-3:2379"' PGUSER_SUPERUSER: PGadmin PGUSER_STANDBY: standby PATRONI_RESTAPI_USERNAME: patroni PGROOT: /home/postgres/pgdata/pgroot + SPILO_CONFIGURATION: | + bootstrap: + post_init: /scripts/post_init_wrapper.sh "zalandos" CLONE_METHOD: CLONE_WITH_BASEBACKUP CLONE_SCOPE: legacy-single CLONE_HOST: postgresql