.woodpecker/deploy.yml

@@ -70,6 +70,16 @@ when:
 #   tolerating a failing stack-ps in verify (|| true) so a genuinely missing
 #   stack produces the designed WARNING instead of killing the step. This
 #   hazard was first flagged in July (PR #3, closed unmerged).
+#
+#   2026-09-12 MIGRATION: traefik moved from Pattern B (secrets hand-typed
+#   into the host-only traefik.env, no CI involvement at all) to the same
+#   manifest-driven provisioning as ai) below. Root cause of the migration:
+#   traefik.env had been accidentally committed to git with a literal
+#   "***REDACTED***" placeholder as KEEPALIVED_PASSWORD; every git-guard
+#   resync/checkout silently restored that broken value, which diverged from
+#   keepalived-backup's stale-but-correct in-memory value and produced a
+#   continuous VRRP auth failure ("received an invalid passwd!") and VIP
+#   instability. The manifest-driven path never commits the rendered file.
 # ─────────────────────────────────────────────────────────────────────────────
 
 steps:
@@ -216,6 +226,11 @@ steps:
         from_secret: flowagent_azure_tenant_id
       FLOWAGENT_AZURE_CLIENT_SECRET:
         from_secret: flowagent_azure_client_secret
+      # ── traefik stack (manifest-driven — secrets/secrets-map.yaml +
+      #    traefik/traefik.env.template + deploy/provision-stack.py).
+      #    Migrated 2026-09-12; see header note above for root cause. ──
+      TRAEFIK_KEEPALIVED_PASSWORD:
+        from_secret: traefik_keepalived_password
     commands:
       - apk add --no-cache openssh-client python3 py3-yaml
       - mkdir -p ~/.ssh
@@ -243,8 +258,19 @@ steps:
         for STACK in $ALL_STACKS; do
           echo "Provisioning: $STACK"
           case "$STACK" in
-            maintenance|media|unifi|guacamole|security|auth|traefik|meshcentral|ddm)
+            maintenance|media|unifi|guacamole|security|auth|meshcentral|ddm)
               echo "  No Docker secrets for $STACK — secrets in host .env";;
+            traefik)
+              # MIGRATED (2026-09-12) to manifest-driven provisioning after
+              # discovering traefik.env was committed to git with a literal
+              # "***REDACTED***" placeholder as KEEPALIVED_PASSWORD — every
+              # git-guard resync/checkout restored the broken value, causing
+              # VRRP auth mismatch between keepalived-master/-backup and VIP
+              # instability. All logic lives in deploy/provision-stack.py;
+              # the authoritative key list lives in
+              # traefik/traefik.env.template; the mapping lives in
+              # secrets/secrets-map.yaml. This case is intentionally one line.
+              python3 deploy/provision-stack.py traefik;;
             immich)
               ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
                 create_or_update_secret 'immich_db_password'     '$${IMMICH_DB_PASSWORD}'

secrets/README.md

@@ -6,9 +6,9 @@ This directory documents the secrets required for each Docker Swarm stack.
 
 ```
 Secret values live in Woodpecker (encrypted)
-    ↓ pipeline reads them at deploy time
+    → pipeline reads them at deploy time
 Docker Swarm secret store (encrypted Raft DB, replicated across all nodes)
-    ↓ mounted into containers at runtime
+    → mounted into containers at runtime
 /run/secrets/<secret_name>
 ```
 
@@ -38,7 +38,10 @@ Use the stack's `.secrets.example` file as your checklist.
 
 ### Step 2 — Add the stack's case to `.woodpecker.yml`
 In the `provision-secrets` step, add a case for the stack that calls
-`create_or_update_secret` for each secret.
+`create_or_update_secret` for each secret. (Or, preferred for new
+migrations: add an entry to `secrets/secrets-map.yaml` + a
+`<stack>.env.template` and call `deploy/provision-stack.py <stack>`
+instead — see the `ai` and `traefik` entries for the current pattern.)
 
 ### Step 3 — Test by pushing a trivial change to the stack's yaml file
 Watch the pipeline run: provision-secrets → validate → deploy → verify → notify.
@@ -71,10 +74,20 @@ Examples:
 - **PostgreSQL is highest risk.** Its master password is used by nearly every
   other stack. Migrate it last.
 
+- **Never commit a rendered env file, even by accident.** `traefik/traefik.env`
+  was committed to git for a period (discovered/fixed 2026-09-12) with a
+  literal "***REDACTED***" placeholder as KEEPALIVED_PASSWORD, which was
+  silently restored every time the file was deleted or the checkout resynced
+  from git — causing a real VRRP auth outage. `.gitignore` blanket-excludes
+  `*.env`, but that rule does NOT retroactively untrack a file already
+  committed before the rule existed. If you ever see a stack's `.env` file
+  show up in `git status` as tracked, stop and untrack it (`git rm --cached`)
+  before doing anything else.
+
 ## Migration Status
 
 | Stack | Secrets in Woodpecker | Pipeline Step Added | .env Removed |
-|-------|----------------------|---------------------|--------------|
+|-------|------------------------|----------------------|---------------|
 | 3dprint | ⏳ | ⏳ | ⏳ |
 | ai | ⏳ | ⏳ | ⏳ |
 | auth | ⏳ | ⏳ | ⏳ |
@@ -91,7 +104,7 @@ Examples:
 | postgresql | ⏳ | ⏳ | ⏳ |
 | productivity | ⏳ | ⏳ | ⏳ |
 | security | ⏳ | ⏳ | ⏳ |
-| traefik | ⏳ | ⏳ | ⏳ |
+| traefik | ✅ (manifest-driven, 2026-09-12) | ✅ (manifest-driven, 2026-09-12) | ⏳ |
 | unifi | ✅ N/A (no secrets) | ✅ N/A | ⏳ |
 | vaultwarden | ⏳ | ⏳ | ⏳ |
 | woodpecker | ⏳ Manual only | ⏳ N/A | ⏳ |

secrets/secrets-map.yaml

@@ -1,4 +1,4 @@
-# ─────────────────────────────────────────────────────────────────────────────
+# ─────────────────────────────────────────────────────────────────────────
 # secrets-map.yaml — DATA-ONLY manifest for deploy/provision-stack.py
 #
 # RULES:
@@ -27,7 +27,7 @@
 #
 # Stacks not listed here fall through to deploy.yml's legacy case-entries
 # untouched. Migration is deliberately one stack per PR.
-# ─────────────────────────────────────────────────────────────────────────────
+# ─────────────────────────────────────────────────────────────────────────
 stacks:
   ai:
     env_template: ai/ai.env.template
@@ -36,3 +36,6 @@ stacks:
       flowagent_azure_client_id: FLOWAGENT_AZURE_CLIENT_ID
       flowagent_azure_tenant_id: FLOWAGENT_AZURE_TENANT_ID
       flowagent_azure_client_secret: FLOWAGENT_AZURE_CLIENT_SECRET
+  traefik:
+    env_template: traefik/traefik.env.template
+    env_dest: traefik/traefik.env

secrets/traefik.secrets.example

@@ -1,5 +1,8 @@
 # traefik Stack — Secrets Reference
-# Source: traefik.env
+# Source: traefik.env.template (rendered by deploy/provision-stack.py per
+#         secrets/secrets-map.yaml — see secrets-map.yaml header for how
+#         this works). traefik/traefik.env is rendered fresh on every
+#         provisioning run and is NEVER committed to git.
 #
 # Add SECRET values to Woodpecker at:
 #   https://woodpecker.bryanmail.net
@@ -8,27 +11,44 @@
 # ⚠️  HIGH RISK: Traefik is the entry point for all homelab services.
 #     If this stack fails, nothing is reachable from outside.
 #     Migrate carefully. The Keepalived VIP (192.168.4.30) depends on this stack.
+#
+# ⚠️  2026-09-12 INCIDENT: traefik.env was previously committed to git with a
+#     literal "***REDACTED***" placeholder as KEEPALIVED_PASSWORD. Every
+#     git-guard resync/checkout restored that broken value onto disk,
+#     diverging from keepalived-backup's stale-but-correct in-memory value
+#     and causing continuous VRRP auth failures + VIP instability. This was
+#     the trigger for migrating this stack to Pattern C. If you ever see
+#     KEEPALIVED_PASSWORD as a literal placeholder-looking string on disk
+#     again, do NOT hand-edit it — check `git log traefik/` for a stray
+#     commit and fix the template in Gitea instead.
 
-# ── SECRETS (add to Woodpecker) ───────────────────────────────────────────────
+# ── SECRETS (add to Woodpecker) ─────────────────────────────────────────
 
 # Woodpecker secret name:  traefik_keepalived_password
 # Used for:                Keepalived VRRP authentication password
 #                          Must match across all 3 nodes (docker-1, docker-2, docker-3)
-# Env var in .env:        KEEPALIVED_PASSWORD
+#                          NOTE: classic VRRP simple-auth is silently
+#                          truncated to 8 chars by keepalived itself — keep
+#                          the value <= 8 characters, or be aware only the
+#                          first 8 are actually significant on the wire.
+# Env var in .env.template: KEEPALIVED_PASSWORD (via TRAEFIK_KEEPALIVED_PASSWORD)
 traefik_keepalived_password=
 
-# ── NON-SECRETS (safe in compose file or .env) ────────────────────────────────
+# ── NON-SECRETS (safe in compose file or .env.template) ─────────────────
 
-# KEEPALIVED_UNICAST_PEERS     Python2BASH list of peer IPs
+# KEEPALIVED_VIRTUAL_IPS       Python2BASH list of VIP addresses (192.168.4.30) — literal in template
-# KEEPALIVED_VIRTUAL_IPS       Python2BASH list of VIP addresses (192.168.4.30)
 # ACME_EMAIL                   Let's Encrypt certificate email
 # TRUSTED_IPS                  Trusted proxy CIDR ranges
 # TRAEFIK_HOST                 Traefik dashboard hostname
 # SPEEDTEST_HOST                Speedtest Traefik hostname
 # WHOAMI_HOST                  Whoami Traefik hostname
 
-# ── Woodpecker provision-secrets case entry ───────────────────────────────────
+# ── Provisioning (manifest-driven, deploy/provision-stack.py) ───────────
 #
-#   traefik)
+# This stack is migrated — provisioning happens automatically via:
-#     create_or_update_secret "traefik_keepalived_password" "$TRAEFIK_KEEPALIVED_PASSWORD"
+#   secrets/secrets-map.yaml (traefik: entry)
-#     ;;
+#   traefik/traefik.env.template (authoritative key list)
+#   deploy/provision-stack.py (renders + ships traefik/traefik.env)
+#
+# The .woodpecker/deploy.yml provision-secrets step calls this with a
+# single line: `python3 deploy/provision-stack.py traefik`

traefik/traefik.env

@@ -1,2 +0,0 @@
-KEEPALIVED_PASSWORD=***REDACTED***
-KEEPALIVED_VIRTUAL_IPS="#PYTHON2BASH:['192.168.4.30']"

traefik/traefik.env.template

@@ -0,0 +1,39 @@
+# ─────────────────────────────────────────────────────────────────────────
+# traefik.env.template — AUTHORITATIVE template for traefik/traefik.env
+# (rendered by deploy/provision-stack.py per secrets/secrets-map.yaml)
+#
+#   - This file IS the complete key list for traefik.env. The whole file is
+#     rendered on every provisioning run — no line surgery, so a key can
+#     never silently go missing.
+#   - Non-secret config lives here as LITERAL values (visible, reviewable).
+#   - Secret values are dollar-brace placeholders resolved from the CI env
+#     (Woodpecker from_secret vars) at provisioning time. provision-stack.py
+#     FAILS HARD if any placeholder is missing/empty.
+#   - The rendered traefik/traefik.env exists only on the host (gitignored).
+#   - Rendered by provision-stack.py, NOT Woodpecker's yaml preprocessor —
+#     single-dollar placeholders are safe here (deploy.yml's double-dollar
+#     rule does NOT apply to this file).
+#
+# Consumed by traefik/traefik.yaml (KEEPALIVED_PASSWORD, KEEPALIVED_VIRTUAL_IPS).
+# DOMAIN comes from deploy/global.env, not here.
+#
+# ⚠️ HIGH RISK: Traefik is the entry point for all homelab services, and the
+#    Keepalived VIP (192.168.4.30) depends on this stack. KEEPALIVED_PASSWORD
+#    MUST be identical across keepalived-master and keepalived-backup — both
+#    consume this same rendered value — or VRRP auth fails and the VIP
+#    becomes unstable (this is exactly what the 2026-09-12 incident was:
+#    a committed literal "***REDACTED***" placeholder in git, restored every
+#    time the file was deleted/re-synced, diverging from -backup's stale but
+#    correct in-memory value).
+#
+#    Classic VRRP simple-auth is silently truncated to 8 characters by
+#    keepalived itself. Set TRAEFIK_KEEPALIVED_PASSWORD in Woodpecker to a
+#    value 8 characters or fewer (or accept that only the first 8 chars are
+#    actually significant) so the effective negotiated value is unambiguous.
+# ─────────────────────────────────────────────────────────────────────────
+
+# ── Keepalived (non-secret config) ────────────────────────────────────────
+KEEPALIVED_VIRTUAL_IPS="#PYTHON2BASH:['192.168.4.30']"
+
+# ── Keepalived (secret) ────────────────────────────────────────────────────
+KEEPALIVED_PASSWORD=${TRAEFIK_KEEPALIVED_PASSWORD}