From ef7e3d015e1f6602daf78fa2a6b176c5c988d7d2 Mon Sep 17 00:00:00 2001 From: Bot Date: Mon, 7 Sep 2026 21:47:04 -0700 Subject: [PATCH] provision-secrets: ai) case -> provision-stack.py + normalized env block (branch rebuilt on current main) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same two scoped changes as PR #15's rebase, now applied on a branch cut from TODAY's main so the merge base is current (PR #15 became unmergeable — its 09-03 merge base predated main's heavy hand-edits): 1. ai) case -> `python3 deploy/provision-stack.py ai` (one line). Retires main's heredoc with its live drift: AI_LITELLM_POSTGRES_PASSWORD printed twice (one copy from retired AI_LITELLM_DB_PASSWORD), stray legacy AI_WEBUI_SECRET_KEY, and the WEB_UI/WEBUI shell-var mismatch that renders an EMPTY OAuth client secret into ai.env. 2. env block ai section: normalized to exact template placeholder names, added AI_MCPO_API_KEY (NEW Woodpecker secret required pre-merge), pruned now-template-literal vars. apk gains python3 py3-yaml. Everything else byte-matches current main. Dollar-escape audit done per header lesson. --- .woodpecker/deploy.yml | 121 +++++++++++++---------------------------- 1 file changed, 39 insertions(+), 82 deletions(-) diff --git a/.woodpecker/deploy.yml b/.woodpecker/deploy.yml index ef9ea35..b7910ef 100644 --- a/.woodpecker/deploy.yml +++ b/.woodpecker/deploy.yml @@ -48,6 +48,17 @@ when: # already unconditionally rsynced at the top of the deploy step regardless # of which stacks changed, so it's safe to exclude it from the stack list # everywhere folders are detected below. +# +# 2026-09-03/07 INCIDENT + REDESIGN: hand-maintained grep -v + printf +# line-surgery cases repeatedly drifted (missing '=', duplicated keys, +# mismatched env var names rendering EMPTY secrets) and broke live +# services. Root cause: no authoritative key list and shell heredocs +# hostile to hand-editing. Stacks migrate one at a time to a data-driven +# model: secrets/secrets-map.yaml (data only) + per-stack .env.template +# (authoritative FULL file) + deploy/provision-stack.py (whole-file +# render, hard failure naming any missing value). Migrated stacks call +# the script; unmigrated stacks keep legacy case entries until their own +# PR. See the "Secrets & Deployment Architecture — Global Direction" note. # ───────────────────────────────────────────────────────────────────────────── steps: @@ -163,6 +174,11 @@ steps: from_secret: entertainment_sparky_encryption_key ENTERTAINMENT_BETTER_AUTH_SECRET: from_secret: entertainment_better_auth_secret + # ── ai stack (manifest-driven — secrets/secrets-map.yaml + + # ai/ai.env.template + deploy/provision-stack.py). Env var names + # below match the template placeholders EXACTLY; this block is the + # only per-secret touchpoint left in this file for migrated stacks + # (Woodpecker v3 requires explicit from_secret declarations). ── AI_AWS_ACCESS_KEY_ID: from_secret: ai_aws_access_key_id AI_AWS_SECRET_ACCESS_KEY: @@ -171,38 +187,18 @@ steps: from_secret: ai_litellm_master_key AI_LITELLM_SALT_KEY: from_secret: ai_litellm_salt_key - AI_LITELLM_DB_PASSWORD: - from_secret: ai_litellm_db_password - AI_LITELLM_DATABASE_MIGRATIONS: - from_secret: litellm_database_migrations - AI_LITELLM_MODIFY_PARAMS: - from_secret: litellm_modify_params - AI_OPEN_WEBUI_SECRET_KEY: - from_secret: ai_webui_secret_key - AI_OPEN_WEBUI_DATABASE_URL: - from_secret: ai_open_webui_database_url - AI_OPEN_WEB_UI_OAUTH_CLIENT_SECRET: - from_secret: ai_oauth_client_secret - AI_OPEN_WEB_UI_ENABLE_OAUTH_SIGNUP: - from_secret: ai_open_web_ui_enable_oauth_signup - AI_OPEN_WEB_UI_OAUTH_MERGE_ACCOUNTS_BY_EMAIL: - from_secret: ai_open_web_ui_oauth_merge_accounts_by_email - AI_OPEN_WEB_UI_OAUTH_PROVIDER_NAME: - from_secret: ai_open_web_ui_oauth_provider_name - AI_OPEN_WEB_UI_OPENID_PROVIDER_URL: - from_secret: ai_open_web_ui_openid_provider_url - AI_OPEN_WEB_UI_OAUTH_CLIENT_ID: - from_secret: ai_open_web_ui_oauth_client_id - AI_OPEN_WEB_UI_OAUTH_SCOPES: - from_secret: ai_open_web_ui_oauth_scopes - AI_OPEN_WEB_UI_OPENID_REDIRECT_URI: - from_secret: ai_open_web_ui_openid_redirect_uri AI_LITELLM_DATABASE_URL: from_secret: ai_litellm_database_url AI_LITELLM_POSTGRES_PASSWORD: from_secret: ai_litellm_postgres_password - AI_AWS_REGION_NAME: - from_secret: ai_aws_region_name + AI_OPEN_WEBUI_SECRET_KEY: + from_secret: ai_webui_secret_key + AI_OPEN_WEBUI_DATABASE_URL: + from_secret: ai_open_webui_database_url + AI_OPEN_WEBUI_OAUTH_CLIENT_SECRET: + from_secret: ai_oauth_client_secret + AI_MCPO_API_KEY: + from_secret: ai_mcpo_api_key FLOWAGENT_AZURE_CLIENT_ID: from_secret: flowagent_azure_client_id FLOWAGENT_AZURE_TENANT_ID: @@ -210,7 +206,7 @@ steps: FLOWAGENT_AZURE_CLIENT_SECRET: from_secret: flowagent_azure_client_secret commands: - - apk add --no-cache openssh-client + - apk add --no-cache openssh-client python3 py3-yaml - mkdir -p ~/.ssh - echo "$SSH_KEY" | base64 -d > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa @@ -257,9 +253,9 @@ steps: # dependency on a running Swarm's Docker secret store. Native Docker # secrets (Pattern C) can't satisfy that: they only exist inside an # already-running Swarm, which is exactly the circular dependency - # this stack can't have. Mirrors the ai) case's grep -v + printf - # rewrite-in-place approach, never sed (values may contain slash, - # dollar sign, ampersand). + # this stack can't have. Mirrors the retired ai) case's grep -v + + # printf rewrite-in-place approach, never sed (values may contain + # slash, dollar sign, ampersand). # # TEST PHASE: target is git.env.pipelinetest, NOT the real git.env. # The real file is never opened for writing by this step. First run @@ -270,6 +266,8 @@ steps: # documented GITEA_MCP_ACCESS_TOKEN key. Cutover to the real file — # and pointing git.yaml/stack-deploy at it — is a deliberate, # separate follow-up after manually diffing this render. + # (Candidate for the secrets-map.yaml/provision-stack.py migration + # in its own PR; kept legacy for now.) ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "FILE=/volume1/docker/compose-files/git.env.pipelinetest TMP=\$FILE.tmp.\$\$ [ -f \$FILE ] || cp /volume1/docker/compose-files/git.env \$FILE @@ -317,56 +315,15 @@ steps: create_or_update_secret 'vaultwarden_admin_token' '$${VAULTWARDEN_ADMIN_TOKEN}' create_or_update_secret 'vaultwarden_database_url_v2' '$${VAULTWARDEN_DATABASE_URL}'";; ai) - # NOTE: ai stack uses Pattern B (host .env, not native Docker secrets) — - # LiteLLM/boto3 and Open WebUI don't support the _FILE convention for these - # vars. Instead of Docker secrets, we regenerate ONLY the migrated lines in - # the remote ai/ai.env in place via grep -v + printf (never sed, since values - # may contain slash, dollar sign, ampersand). All other lines — including - # MCPO_API_KEY, OAUTH_CLIENT_ID, WEBUI_URL, and other non-secret config — are - # left completely untouched. MCPO_API_KEY migration is deferred to a - # follow-up; this step never reads or writes it. - # - # IMPORTANT: ai.yaml's litellm service references ${AI_AWS_ACCESS_KEY_ID} / - # ${AI_AWS_SECRET_ACCESS_KEY} (AI_-prefixed) and renders them into the - # container as plain AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (see commit - # 37ed671a, "Change AWS keys to use Woodpecker Secrets"). So ai.env must be - # written with the AI_-prefixed key names, NOT the plain ones — writing - # plain AWS_ACCESS_KEY_ID here would leave ${AI_AWS_ACCESS_KEY_ID} - # unresolved at compose-render time (empty), silently breaking Bedrock auth. - # All other migrated vars in ai.yaml use plain (unprefixed) names, so only - # these two lines need the AI_ prefix. - # - # FLOWAGENT NOTE (added alongside the ai.yaml mcpo image/secrets cutover): - # the 3 flowagent_azure_* values are provisioned as native Docker secrets - # below (Pattern C, matches every other _FILE-convention stack), NOT written - # into ai/ai.env — mcpo's flowagent entry reads them via - # /run/secrets/flowagent_azure_* (see mcp-config/flowagent/entrypoint.sh), - # not via env var, so they don't belong in this stack's Pattern B .env block. - ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "FILE=/volume1/docker/compose-files/ai/ai.env - TMP=\$FILE.tmp.\$\$ - grep -vE '^(AI_AWS_ACCESS_KEY_ID|AI_AWS_SECRET_ACCESS_KEY|AI_LITELLM_MASTER_KEY|AI_LITELLM_SALT_KEY|AI_LITELLM_POSTGRES_PASSWORD|AI_LITELLM_DATABASE_URL|AI_OPEN_WEBUI_SECRET_KEY|AI_OPEN_WEBUI_DATABASE_URL|AI_OPEN_WEB_UI_OAUTH_CLIENT_SECRET)=' \$FILE > \$TMP 2>/dev/null || touch \$TMP - { cat \$TMP - printf 'AI_AWS_ACCESS_KEY_ID=%s\n' '$${AI_AWS_ACCESS_KEY_ID}' - printf 'AI_AWS_SECRET_ACCESS_KEY=%s\n' '$${AI_AWS_SECRET_ACCESS_KEY}' - printf 'AI_AWS_REGION_NAME=%s\n' '$${AI_AWS_REGION_NAME}' - printf 'AI_LITELLM_MASTER_KEY=%s\n' '$${AI_LITELLM_MASTER_KEY}' - printf 'AI_LITELLM_SALT_KEY=%s\n' '$${AI_LITELLM_SALT_KEY}' - printf 'AI_LITELLM_POSTGRES_PASSWORD=%s\n' '$${AI_LITELLM_DB_PASSWORD}' - printf 'AI_LITELLM_DATABASE_URL=%s\n' '$${AI_LITELLM_DATABASE_URL}' - printf 'AI_LITELLM_POSTGRES_PASSWORD=%s\n' '$${AI_LITELLM_POSTGRES_PASSWORD}' - printf 'AI_LITELLM_MODIFY_PARAMS=%s\n' '$${AI_LITELLM_MODIFY_PARAMS}' - printf 'AI_LITELLM_DATABASE_MIGRATIONS=%s\n' '$${AI_LITELLM_DATABASE_MIGRATIONS}' - printf 'AI_OPEN_WEBUI_SECRET_KEY=%s\n' '$${AI_OPEN_WEBUI_SECRET_KEY}' - printf 'AI_OPEN_WEBUI_DATABASE_URL=%s\n' '$${AI_OPEN_WEBUI_DATABASE_URL}' - printf 'AI_WEBUI_SECRET_KEY=%s\n' '$${AI_OPEN_WEBUI_SECRET_KEY}' - printf 'AI_OPEN_WEBUI_OAUTH_CLIENT_SECRET=%s\n' '$${AI_OPEN_WEBUI_OAUTH_CLIENT_SECRET}' - } > \$FILE - rm -f \$TMP - echo ' [OK] ai/ai.env secrets updated' - source /tmp/cs.sh - create_or_update_secret 'flowagent_azure_client_id' '$${FLOWAGENT_AZURE_CLIENT_ID}' - create_or_update_secret 'flowagent_azure_tenant_id' '$${FLOWAGENT_AZURE_TENANT_ID}' - create_or_update_secret 'flowagent_azure_client_secret' '$${FLOWAGENT_AZURE_CLIENT_SECRET}'";; + # MIGRATED (2026-09-07) to manifest-driven provisioning after the + # line-surgery approach repeatedly drifted (dropped keys 2026-09-03; + # duplicate AI_LITELLM_POSTGRES_PASSWORD + stray legacy keys + + # WEB_UI/WEBUI env-name mismatch rendering an EMPTY OAuth client + # secret, found 2026-09-07). All logic lives in + # deploy/provision-stack.py; the authoritative key list lives in + # ai/ai.env.template; the mapping lives in secrets/secrets-map.yaml. + # This case is intentionally one line. + python3 deploy/provision-stack.py ai;; entertainment) ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh create_or_update_secret 'entertainment_discord_token' '$${ENTERTAINMENT_DISCORD_TOKEN}'