Provisioning v2: data-only secrets manifest + full-file env templates + generic provisioner (ai stack first) #15

Closed
Bot wants to merge 6 commits from secrets-provisioning-v2 into main
Showing only changes of commit cb10c2e22d - Show all commits
+24 -26
View File
@@ -49,17 +49,16 @@ when:
# of which stacks changed, so it's safe to exclude it from the stack list # of which stacks changed, so it's safe to exclude it from the stack list
# everywhere folders are detected below. # everywhere folders are detected below.
# #
# 2026-09-03 INCIDENT + REDESIGN: the hand-maintained ai) case (grep -v + # 2026-09-03/07 INCIDENT + REDESIGN: hand-maintained grep -v + printf
# printf line surgery on ai/ai.env) silently dropped env keys during manual # line-surgery cases repeatedly drifted (missing '=', duplicated keys,
# edits (missing '=', duplicated keys), breaking every ai-stack service and # mismatched env var names rendering EMPTY secrets) and broke live
# requiring manual host-side recovery. Root cause: no authoritative list of # services. Root cause: no authoritative key list and shell heredocs
# what a complete env file contains, and shell heredocs that are hostile to # hostile to hand-editing. Stacks migrate one at a time to a data-driven
# hand-editing. Stacks are being migrated one at a time to a data-driven
# model: secrets/secrets-map.yaml (data only) + per-stack .env.template # model: secrets/secrets-map.yaml (data only) + per-stack .env.template
# (authoritative full file) + deploy/provision-stack.py (generic renderer; # (authoritative FULL file) + deploy/provision-stack.py (whole-file
# whole file rendered every run, hard failure on missing values). Migrated # render, hard failure naming any missing value). Migrated stacks call
# stacks call the script below; unmigrated stacks keep their legacy case # the script; unmigrated stacks keep legacy case entries until their own
# entries until their own migration PR. # PR. See the "Secrets & Deployment Architecture — Global Direction" note.
# ───────────────────────────────────────────────────────────────────────────── # ─────────────────────────────────────────────────────────────────────────────
steps: steps:
@@ -175,10 +174,11 @@ steps:
from_secret: entertainment_sparky_encryption_key from_secret: entertainment_sparky_encryption_key
ENTERTAINMENT_BETTER_AUTH_SECRET: ENTERTAINMENT_BETTER_AUTH_SECRET:
from_secret: entertainment_better_auth_secret from_secret: entertainment_better_auth_secret
# ── ai stack (manifest-driven; see secrets/secrets-map.yaml and # ── ai stack (manifest-driven secrets/secrets-map.yaml +
# ai/ai.env.template — this env block is the ONLY per-secret # ai/ai.env.template + deploy/provision-stack.py). Env var names
# touchpoint left in this file for migrated stacks, because # below match the template placeholders EXACTLY; this block is the
# Woodpecker v3 requires explicit from_secret declarations) ── # only per-secret touchpoint left in this file for migrated stacks
# (Woodpecker v3 requires explicit from_secret declarations). ──
AI_AWS_ACCESS_KEY_ID: AI_AWS_ACCESS_KEY_ID:
from_secret: ai_aws_access_key_id from_secret: ai_aws_access_key_id
AI_AWS_SECRET_ACCESS_KEY: AI_AWS_SECRET_ACCESS_KEY:
@@ -191,11 +191,11 @@ steps:
from_secret: ai_litellm_database_url from_secret: ai_litellm_database_url
AI_LITELLM_POSTGRES_PASSWORD: AI_LITELLM_POSTGRES_PASSWORD:
from_secret: ai_litellm_postgres_password from_secret: ai_litellm_postgres_password
AI_WEBUI_SECRET_KEY: AI_OPEN_WEBUI_SECRET_KEY:
from_secret: ai_webui_secret_key from_secret: ai_webui_secret_key
AI_OPEN_WEBUI_DATABASE_URL: AI_OPEN_WEBUI_DATABASE_URL:
from_secret: ai_open_webui_database_url from_secret: ai_open_webui_database_url
AI_OPEN_WEB_UI_OAUTH_CLIENT_SECRET: AI_OPEN_WEBUI_OAUTH_CLIENT_SECRET:
from_secret: ai_oauth_client_secret from_secret: ai_oauth_client_secret
AI_MCPO_API_KEY: AI_MCPO_API_KEY:
from_secret: ai_mcpo_api_key from_secret: ai_mcpo_api_key
@@ -315,16 +315,14 @@ steps:
create_or_update_secret 'vaultwarden_admin_token' '$${VAULTWARDEN_ADMIN_TOKEN}' create_or_update_secret 'vaultwarden_admin_token' '$${VAULTWARDEN_ADMIN_TOKEN}'
create_or_update_secret 'vaultwarden_database_url_v2' '$${VAULTWARDEN_DATABASE_URL}'";; create_or_update_secret 'vaultwarden_database_url_v2' '$${VAULTWARDEN_DATABASE_URL}'";;
ai) ai)
# MIGRATED (2026-09-03) to manifest-driven provisioning after the # MIGRATED (2026-09-07) to manifest-driven provisioning after the
# line-surgery approach dropped env keys and broke the stack. # line-surgery approach repeatedly drifted (dropped keys 2026-09-03;
# All logic lives in deploy/provision-stack.py; the key list lives # duplicate AI_LITELLM_POSTGRES_PASSWORD + stray legacy keys +
# in ai/ai.env.template; the mapping lives in # WEB_UI/WEBUI env-name mismatch rendering an EMPTY OAuth client
# secrets/secrets-map.yaml. This case is intentionally one line. # secret, found 2026-09-07). All logic lives in
# The script renders the COMPLETE ai/ai.env from the template # deploy/provision-stack.py; the authoritative key list lives in
# (hard-failing on any missing value, listing names only) and # ai/ai.env.template; the mapping lives in secrets/secrets-map.yaml.
# provisions the flowagent_azure_* Docker secrets, values via # This case is intentionally one line.
# ssh stdin — no secret value or value structure appears in this
# file anymore.
python3 deploy/provision-stack.py ai;; python3 deploy/provision-stack.py ai;;
entertainment) entertainment)
ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh