postgresql-ha-staging.yaml: port dry-run fixes (bugs 1,2,4,5) - $$(...) escaping (incl. CLONE_PASSWORD), ETCD3_HOSTS, post_init_wrapper.sh SUPERUSER role fix

postgresql/cutover/postgresql-ha-staging.yaml

@@ -23,6 +23,49 @@
 #     It only gets those in postgresql-ha-final.yaml at promote time.
 #   - Superuser is PGadmin (matches the cloned data's existing superuser,
 #     same password secret), NOT Spilo's default "postgres".
+#
+# NOTE: command: blocks use $$(...) not $(...) — Compose's own variable
+# interpolation parses $( as an attempted ${VAR} reference and fails with
+# "invalid interpolation format" / "you may need to escape any $ with
+# another $". $$ escapes to a literal $ for the shell at runtime. This
+# applies uniformly to EVERY literal $ character anywhere in a command:
+# block, including inside heredocs and nested quoting — Compose scans the
+# raw text before any shell/heredoc logic ever runs, so quoting context
+# doesn't exempt anything from this escaping requirement. (Confirmed the
+# hard way in the pgha-test dry run — see "ADR-0001 Dry-Run Debugging Log"
+# note — this file originally had the unescaped $(...) bug, including on
+# the CLONE_PASSWORD line which is unique to this staging file.)
+#
+# NOTE: ETCD3_HOSTS not ETCD_HOSTS — confirmed against zalando/spilo
+# configure_spilo.py: PATRONI_DCS includes both "etcd" (legacy v2 API,
+# python-etcd client) and "etcd3" (v3 API, python-etcd3 client) as
+# distinct DCS backends selected by env var prefix. Our etcd containers
+# (v3.5.9) have the v2 API disabled by default, so ETCD_HOSTS causes
+# Patroni to hit /v2 endpoints that 404. ETCD3_HOSTS selects the correct
+# v3-API client. (Also originally wrong in this file — fixed after dry run.)
+#
+# NOTE: patroni-0/patroni-1 override bootstrap.post_init via
+# SPILO_CONFIGURATION (Spilo's documented, supported mechanism for
+# overriding any generated Patroni config — configure_spilo.py deep-merges
+# user-supplied SPILO_CONFIGURATION on top of its own generated config).
+# This is needed because Spilo's own /scripts/post_init.sh hardcodes
+# "ALTER VIEW ... OWNER TO postgres" with no way to parameterize that role
+# name via env vars. Since our superuser is PGUSER_SUPERUSER=PGadmin, there
+# is no role literally named "postgres" in the cloned data, so Spilo's
+# unmodified script fails with 'ERROR: role "postgres" does not exist'.
+# The override points bootstrap.post_init at our own wrapper script
+# instead, which creates a harmless, idempotent "postgres" role (WITH
+# SUPERUSER NOLOGIN — SUPERUSER is required because Spilo's own
+# _zmon_schema.dump does "SET ROLE TO postgres; CREATE EXTENSION
+# plpython3u", which needs real superuser privileges, not just role
+# existence; NOLOGIN means it can never be used to establish a real
+# connection, so this carries no security exposure), then execs Spilo's
+# real, UNMODIFIED post_init.sh with all original arguments passed through
+# — Zalando's script itself is never patched or forked. Both fixes (role
+# existence + SUPERUSER) were found and validated end-to-end in the
+# pgha-test dry run before being ported here — see the dry-run debugging
+# note for full detail. This is CRITICAL for this file specifically since
+# it clones real production data with the real PGadmin superuser.
 # ─────────────────────────────────────────────────────────────────────────
 version: "3.6"
 
@@ -104,19 +147,30 @@ services:
       - /bin/sh
       - -c
       - |
-        export PGPASSWORD_SUPERUSER="$(cat /run/secrets/postgresql_password)"
+        export PGPASSWORD_SUPERUSER="$$(cat /run/secrets/postgresql_password)"
-        export PGPASSWORD_STANDBY="$(cat /run/secrets/postgresql_replication_password)"
+        export PGPASSWORD_STANDBY="$$(cat /run/secrets/postgresql_replication_password)"
-        export PATRONI_RESTAPI_PASSWORD="$(cat /run/secrets/postgresql_patroni_password)"
+        export PATRONI_RESTAPI_PASSWORD="$$(cat /run/secrets/postgresql_patroni_password)"
-        export CLONE_PASSWORD="$(cat /run/secrets/postgresql_password)"
+        export CLONE_PASSWORD="$$(cat /run/secrets/postgresql_password)"
+        mkdir -p /scripts
+        cat > /scripts/post_init_wrapper.sh <<'WRAP'
+        #!/bin/bash
+        set -e
+        psql -d "$$2" -v ON_ERROR_STOP=1 -c 'DO $$do$$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = $$x$$postgres$$x$$) THEN CREATE ROLE postgres WITH SUPERUSER NOLOGIN; END IF; END $$do$$;'
+        exec /scripts/post_init.sh "$$@"
+        WRAP
+        chmod +x /scripts/post_init_wrapper.sh
         exec /bin/sh /launch.sh init
     environment:
       SCOPE: postgres-ha
       PATRONI_NAME: patroni-0
-      ETCD_HOSTS: '"etcd-1:2379","etcd-2:2379","etcd-3:2379"'
+      ETCD3_HOSTS: '"etcd-1:2379","etcd-2:2379","etcd-3:2379"'
       PGUSER_SUPERUSER: PGadmin
       PGUSER_STANDBY: standby
       PATRONI_RESTAPI_USERNAME: patroni
       PGROOT: /home/postgres/pgdata/pgroot
+      SPILO_CONFIGURATION: |
+        bootstrap:
+          post_init: /scripts/post_init_wrapper.sh "zalandos"
       # Clone-from-live: only runs when PGDATA is empty (first bootstrap).
       # "postgresql" resolves to the OLD single instance during staging.
       CLONE_METHOD: CLONE_WITH_BASEBACKUP
@@ -144,19 +198,30 @@ services:
       - /bin/sh
       - -c
       - |
-        export PGPASSWORD_SUPERUSER="$(cat /run/secrets/postgresql_password)"
+        export PGPASSWORD_SUPERUSER="$$(cat /run/secrets/postgresql_password)"
-        export PGPASSWORD_STANDBY="$(cat /run/secrets/postgresql_replication_password)"
+        export PGPASSWORD_STANDBY="$$(cat /run/secrets/postgresql_replication_password)"
-        export PATRONI_RESTAPI_PASSWORD="$(cat /run/secrets/postgresql_patroni_password)"
+        export PATRONI_RESTAPI_PASSWORD="$$(cat /run/secrets/postgresql_patroni_password)"
-        export CLONE_PASSWORD="$(cat /run/secrets/postgresql_password)"
+        export CLONE_PASSWORD="$$(cat /run/secrets/postgresql_password)"
+        mkdir -p /scripts
+        cat > /scripts/post_init_wrapper.sh <<'WRAP'
+        #!/bin/bash
+        set -e
+        psql -d "$$2" -v ON_ERROR_STOP=1 -c 'DO $$do$$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = $$x$$postgres$$x$$) THEN CREATE ROLE postgres WITH SUPERUSER NOLOGIN; END IF; END $$do$$;'
+        exec /scripts/post_init.sh "$$@"
+        WRAP
+        chmod +x /scripts/post_init_wrapper.sh
         exec /bin/sh /launch.sh init
     environment:
       SCOPE: postgres-ha
       PATRONI_NAME: patroni-1
-      ETCD_HOSTS: '"etcd-1:2379","etcd-2:2379","etcd-3:2379"'
+      ETCD3_HOSTS: '"etcd-1:2379","etcd-2:2379","etcd-3:2379"'
       PGUSER_SUPERUSER: PGadmin
       PGUSER_STANDBY: standby
       PATRONI_RESTAPI_USERNAME: patroni
       PGROOT: /home/postgres/pgdata/pgroot
+      SPILO_CONFIGURATION: |
+        bootstrap:
+          post_init: /scripts/post_init_wrapper.sh "zalandos"
       CLONE_METHOD: CLONE_WITH_BASEBACKUP
       CLONE_SCOPE: legacy-single
       CLONE_HOST: postgresql