provision-secrets: ai) case -> provision-stack.py + normalized env block (branch rebuilt on current main)

Same two scoped changes as PR #15's rebase, now applied on a branch cut
from TODAY's main so the merge base is current (PR #15 became
unmergeable — its 09-03 merge base predated main's heavy hand-edits):

1. ai) case -> `python3 deploy/provision-stack.py ai` (one line). Retires
   main's heredoc with its live drift: AI_LITELLM_POSTGRES_PASSWORD
   printed twice (one copy from retired AI_LITELLM_DB_PASSWORD), stray
   legacy AI_WEBUI_SECRET_KEY, and the WEB_UI/WEBUI shell-var mismatch
   that renders an EMPTY OAuth client secret into ai.env.
2. env block ai section: normalized to exact template placeholder names,
   added AI_MCPO_API_KEY (NEW Woodpecker secret required pre-merge),
   pruned now-template-literal vars.

apk gains python3 py3-yaml. Everything else byte-matches current main.
Dollar-escape audit done per header lesson.

.woodpecker/deploy.yml

@@ -48,6 +48,17 @@ when:
 #   already unconditionally rsynced at the top of the deploy step regardless
 #   of which stacks changed, so it's safe to exclude it from the stack list
 #   everywhere folders are detected below.
+#
+#   2026-09-03/07 INCIDENT + REDESIGN: hand-maintained grep -v + printf
+#   line-surgery cases repeatedly drifted (missing '=', duplicated keys,
+#   mismatched env var names rendering EMPTY secrets) and broke live
+#   services. Root cause: no authoritative key list and shell heredocs
+#   hostile to hand-editing. Stacks migrate one at a time to a data-driven
+#   model: secrets/secrets-map.yaml (data only) + per-stack .env.template
+#   (authoritative FULL file) + deploy/provision-stack.py (whole-file
+#   render, hard failure naming any missing value). Migrated stacks call
+#   the script; unmigrated stacks keep legacy case entries until their own
+#   PR. See the "Secrets & Deployment Architecture — Global Direction" note.
 # ─────────────────────────────────────────────────────────────────────────────
 
 steps:
@@ -163,6 +174,11 @@ steps:
         from_secret: entertainment_sparky_encryption_key
       ENTERTAINMENT_BETTER_AUTH_SECRET:
         from_secret: entertainment_better_auth_secret
+      # ── ai stack (manifest-driven — secrets/secrets-map.yaml +
+      #    ai/ai.env.template + deploy/provision-stack.py). Env var names
+      #    below match the template placeholders EXACTLY; this block is the
+      #    only per-secret touchpoint left in this file for migrated stacks
+      #    (Woodpecker v3 requires explicit from_secret declarations). ──
       AI_AWS_ACCESS_KEY_ID:
         from_secret: ai_aws_access_key_id
       AI_AWS_SECRET_ACCESS_KEY:
@@ -171,38 +187,18 @@ steps:
         from_secret: ai_litellm_master_key
       AI_LITELLM_SALT_KEY:
         from_secret: ai_litellm_salt_key
-      AI_LITELLM_DB_PASSWORD:
-        from_secret: ai_litellm_db_password
-      AI_LITELLM_DATABASE_MIGRATIONS:
-        from_secret: litellm_database_migrations
-      AI_LITELLM_MODIFY_PARAMS:
-        from_secret: litellm_modify_params
-      AI_OPEN_WEBUI_SECRET_KEY:
-        from_secret: ai_webui_secret_key
-      AI_OPEN_WEBUI_DATABASE_URL:
-        from_secret: ai_open_webui_database_url
-      AI_OPEN_WEB_UI_OAUTH_CLIENT_SECRET:
-        from_secret: ai_oauth_client_secret
-      AI_OPEN_WEB_UI_ENABLE_OAUTH_SIGNUP:
-        from_secret: ai_open_web_ui_enable_oauth_signup
-      AI_OPEN_WEB_UI_OAUTH_MERGE_ACCOUNTS_BY_EMAIL:
-        from_secret: ai_open_web_ui_oauth_merge_accounts_by_email
-      AI_OPEN_WEB_UI_OAUTH_PROVIDER_NAME:
-        from_secret: ai_open_web_ui_oauth_provider_name
-      AI_OPEN_WEB_UI_OPENID_PROVIDER_URL:
-        from_secret: ai_open_web_ui_openid_provider_url
-      AI_OPEN_WEB_UI_OAUTH_CLIENT_ID:
-        from_secret: ai_open_web_ui_oauth_client_id
-      AI_OPEN_WEB_UI_OAUTH_SCOPES:
-        from_secret: ai_open_web_ui_oauth_scopes
-      AI_OPEN_WEB_UI_OPENID_REDIRECT_URI:
-        from_secret: ai_open_web_ui_openid_redirect_uri
       AI_LITELLM_DATABASE_URL:
         from_secret: ai_litellm_database_url
       AI_LITELLM_POSTGRES_PASSWORD:
         from_secret: ai_litellm_postgres_password
-      AI_AWS_REGION_NAME:
+      AI_OPEN_WEBUI_SECRET_KEY:
-        from_secret: ai_aws_region_name
+        from_secret: ai_webui_secret_key
+      AI_OPEN_WEBUI_DATABASE_URL:
+        from_secret: ai_open_webui_database_url
+      AI_OPEN_WEBUI_OAUTH_CLIENT_SECRET:
+        from_secret: ai_oauth_client_secret
+      AI_MCPO_API_KEY:
+        from_secret: ai_mcpo_api_key
       FLOWAGENT_AZURE_CLIENT_ID:
         from_secret: flowagent_azure_client_id
       FLOWAGENT_AZURE_TENANT_ID:
@@ -210,7 +206,7 @@ steps:
       FLOWAGENT_AZURE_CLIENT_SECRET:
         from_secret: flowagent_azure_client_secret
     commands:
-      - apk add --no-cache openssh-client
+      - apk add --no-cache openssh-client python3 py3-yaml
       - mkdir -p ~/.ssh
       - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa
       - chmod 600 ~/.ssh/id_rsa
@@ -257,9 +253,9 @@ steps:
               # dependency on a running Swarm's Docker secret store. Native Docker
               # secrets (Pattern C) can't satisfy that: they only exist inside an
               # already-running Swarm, which is exactly the circular dependency
-              # this stack can't have. Mirrors the ai) case's grep -v + printf
+              # this stack can't have. Mirrors the retired ai) case's grep -v +
-              # rewrite-in-place approach, never sed (values may contain slash,
+              # printf rewrite-in-place approach, never sed (values may contain
-              # dollar sign, ampersand).
+              # slash, dollar sign, ampersand).
               #
               # TEST PHASE: target is git.env.pipelinetest, NOT the real git.env.
               # The real file is never opened for writing by this step. First run
@@ -270,6 +266,8 @@ steps:
               # documented GITEA_MCP_ACCESS_TOKEN key. Cutover to the real file —
               # and pointing git.yaml/stack-deploy at it — is a deliberate,
               # separate follow-up after manually diffing this render.
+              # (Candidate for the secrets-map.yaml/provision-stack.py migration
+              # in its own PR; kept legacy for now.)
               ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "FILE=/volume1/docker/compose-files/git.env.pipelinetest
                 TMP=\$FILE.tmp.\$\$
                 [ -f \$FILE ] || cp /volume1/docker/compose-files/git.env \$FILE
@@ -317,56 +315,15 @@ steps:
                 create_or_update_secret 'vaultwarden_admin_token'     '$${VAULTWARDEN_ADMIN_TOKEN}'
                 create_or_update_secret 'vaultwarden_database_url_v2' '$${VAULTWARDEN_DATABASE_URL}'";;
             ai)
-              # NOTE: ai stack uses Pattern B (host .env, not native Docker secrets) —
+              # MIGRATED (2026-09-07) to manifest-driven provisioning after the
-              # LiteLLM/boto3 and Open WebUI don't support the _FILE convention for these
+              # line-surgery approach repeatedly drifted (dropped keys 2026-09-03;
-              # vars. Instead of Docker secrets, we regenerate ONLY the migrated lines in
+              # duplicate AI_LITELLM_POSTGRES_PASSWORD + stray legacy keys +
-              # the remote ai/ai.env in place via grep -v + printf (never sed, since values
+              # WEB_UI/WEBUI env-name mismatch rendering an EMPTY OAuth client
-              # may contain slash, dollar sign, ampersand). All other lines — including
+              # secret, found 2026-09-07). All logic lives in
-              # MCPO_API_KEY, OAUTH_CLIENT_ID, WEBUI_URL, and other non-secret config — are
+              # deploy/provision-stack.py; the authoritative key list lives in
-              # left completely untouched. MCPO_API_KEY migration is deferred to a
+              # ai/ai.env.template; the mapping lives in secrets/secrets-map.yaml.
-              # follow-up; this step never reads or writes it.
+              # This case is intentionally one line.
-              #
+              python3 deploy/provision-stack.py ai;;
-              # IMPORTANT: ai.yaml's litellm service references ${AI_AWS_ACCESS_KEY_ID} /
-              # ${AI_AWS_SECRET_ACCESS_KEY} (AI_-prefixed) and renders them into the
-              # container as plain AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (see commit
-              # 37ed671a, "Change AWS keys to use Woodpecker Secrets"). So ai.env must be
-              # written with the AI_-prefixed key names, NOT the plain ones — writing
-              # plain AWS_ACCESS_KEY_ID here would leave ${AI_AWS_ACCESS_KEY_ID}
-              # unresolved at compose-render time (empty), silently breaking Bedrock auth.
-              # All other migrated vars in ai.yaml use plain (unprefixed) names, so only
-              # these two lines need the AI_ prefix.
-              #
-              # FLOWAGENT NOTE (added alongside the ai.yaml mcpo image/secrets cutover):
-              # the 3 flowagent_azure_* values are provisioned as native Docker secrets
-              # below (Pattern C, matches every other _FILE-convention stack), NOT written
-              # into ai/ai.env — mcpo's flowagent entry reads them via
-              # /run/secrets/flowagent_azure_* (see mcp-config/flowagent/entrypoint.sh),
-              # not via env var, so they don't belong in this stack's Pattern B .env block.
-              ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "FILE=/volume1/docker/compose-files/ai/ai.env
-                TMP=\$FILE.tmp.\$\$
-                grep -vE '^(AI_AWS_ACCESS_KEY_ID|AI_AWS_SECRET_ACCESS_KEY|AI_LITELLM_MASTER_KEY|AI_LITELLM_SALT_KEY|AI_LITELLM_POSTGRES_PASSWORD|AI_LITELLM_DATABASE_URL|AI_OPEN_WEBUI_SECRET_KEY|AI_OPEN_WEBUI_DATABASE_URL|AI_OPEN_WEB_UI_OAUTH_CLIENT_SECRET)=' \$FILE > \$TMP 2>/dev/null || touch \$TMP
-                { cat \$TMP
-                  printf 'AI_AWS_ACCESS_KEY_ID=%s\n' '$${AI_AWS_ACCESS_KEY_ID}'
-                  printf 'AI_AWS_SECRET_ACCESS_KEY=%s\n' '$${AI_AWS_SECRET_ACCESS_KEY}'
-                  printf 'AI_AWS_REGION_NAME=%s\n' '$${AI_AWS_REGION_NAME}'
-                  printf 'AI_LITELLM_MASTER_KEY=%s\n' '$${AI_LITELLM_MASTER_KEY}'
-                  printf 'AI_LITELLM_SALT_KEY=%s\n' '$${AI_LITELLM_SALT_KEY}'
-                  printf 'AI_LITELLM_POSTGRES_PASSWORD=%s\n' '$${AI_LITELLM_DB_PASSWORD}'
-                  printf 'AI_LITELLM_DATABASE_URL=%s\n' '$${AI_LITELLM_DATABASE_URL}'
-                  printf 'AI_LITELLM_POSTGRES_PASSWORD=%s\n' '$${AI_LITELLM_POSTGRES_PASSWORD}'
-                  printf 'AI_LITELLM_MODIFY_PARAMS=%s\n' '$${AI_LITELLM_MODIFY_PARAMS}'
-                  printf 'AI_LITELLM_DATABASE_MIGRATIONS=%s\n' '$${AI_LITELLM_DATABASE_MIGRATIONS}'
-                  printf 'AI_OPEN_WEBUI_SECRET_KEY=%s\n' '$${AI_OPEN_WEBUI_SECRET_KEY}'
-                  printf 'AI_OPEN_WEBUI_DATABASE_URL=%s\n' '$${AI_OPEN_WEBUI_DATABASE_URL}'
-                  printf 'AI_WEBUI_SECRET_KEY=%s\n' '$${AI_OPEN_WEBUI_SECRET_KEY}'
-                  printf 'AI_OPEN_WEBUI_OAUTH_CLIENT_SECRET=%s\n' '$${AI_OPEN_WEBUI_OAUTH_CLIENT_SECRET}'
-                } > \$FILE
-                rm -f \$TMP
-                echo '  [OK] ai/ai.env secrets updated'
-                source /tmp/cs.sh
-                create_or_update_secret 'flowagent_azure_client_id'     '$${FLOWAGENT_AZURE_CLIENT_ID}'
-                create_or_update_secret 'flowagent_azure_tenant_id'     '$${FLOWAGENT_AZURE_TENANT_ID}'
-                create_or_update_secret 'flowagent_azure_client_secret' '$${FLOWAGENT_AZURE_CLIENT_SECRET}'";;
             entertainment)
               ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
                 create_or_update_secret 'entertainment_discord_token'          '$${ENTERTAINMENT_DISCORD_TOKEN}'