Merge pull request 'fix(traefik): migrate KEEPALIVED_PASSWORD to Pattern C (manifest-driven), stop committing traefik.env' (#21) from migrate-traefik-keepalived-secret into main

Reviewed-on: #21
https://ai.bryanmail.net/c/7440e9d5-7e39-40f9-a35a-33016b6ae8d1

.woodpecker/deploy.yml

@@ -59,6 +59,27 @@ when:
 #   render, hard failure naming any missing value). Migrated stacks call
 #   the script; unmigrated stacks keep legacy case entries until their own
 #   PR. See the "Secrets & Deployment Architecture — Global Direction" note.
+#
+#   2026-09-08 FIX: secrets/ is a tooling/docs folder (secrets-map.yaml +
+#   *.secrets.example), not a stack — but folder-detection treated it as one
+#   the first time a commit touched it (PR #16). deploy survived only because
+#   'secrets' sits in the bootstrap-tier skip list; verify had no guard and
+#   died on `docker stack ps secrets` failing under errexit (assignment from
+#   a failing command substitution aborts the step). Fixed by excluding
+#   secrets/ alongside deploy/ in ALL folder-detection sites, and by
+#   tolerating a failing stack-ps in verify (|| true) so a genuinely missing
+#   stack produces the designed WARNING instead of killing the step. This
+#   hazard was first flagged in July (PR #3, closed unmerged).
+#
+#   2026-09-12 MIGRATION: traefik moved from Pattern B (secrets hand-typed
+#   into the host-only traefik.env, no CI involvement at all) to the same
+#   manifest-driven provisioning as ai) below. Root cause of the migration:
+#   traefik.env had been accidentally committed to git with a literal
+#   "***REDACTED***" placeholder as KEEPALIVED_PASSWORD; every git-guard
+#   resync/checkout silently restored that broken value, which diverged from
+#   keepalived-backup's stale-but-correct in-memory value and produced a
+#   continuous VRRP auth failure ("received an invalid passwd!") and VIP
+#   instability. The manifest-driven path never commits the rendered file.
 # ─────────────────────────────────────────────────────────────────────────────
 
 steps:
@@ -76,9 +97,9 @@ steps:
         FLAT=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.ya?ml$' || true)
 
         # Folder: any file under a subfolder (e.g. immich/immich.yml).
-        # Exclude dotfolders (.woodpecker, .git, .gitea, etc.) and deploy/
-        # (shared tooling, not a stack — see note above).
-        FOLDERS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -v '^deploy$' | sort -u || true)
+        # Exclude dotfolders (.woodpecker, .git, .gitea, etc.) and the
+        # non-stack tooling folders deploy/ and secrets/ (see notes above).
+        FOLDERS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -vE '^(deploy|secrets)$' | sort -u || true)
 
         [ -z "$FLAT" ] && [ -z "$FOLDERS" ] && echo "No stacks changed" && exit 0
 
@@ -205,6 +226,11 @@ steps:
         from_secret: flowagent_azure_tenant_id
       FLOWAGENT_AZURE_CLIENT_SECRET:
         from_secret: flowagent_azure_client_secret
+      # ── traefik stack (manifest-driven — secrets/secrets-map.yaml +
+      #    traefik/traefik.env.template + deploy/provision-stack.py).
+      #    Migrated 2026-09-12; see header note above for root cause. ──
+      TRAEFIK_KEEPALIVED_PASSWORD:
+        from_secret: traefik_keepalived_password
     commands:
       - apk add --no-cache openssh-client python3 py3-yaml
       - mkdir -p ~/.ssh
@@ -219,21 +245,32 @@ steps:
       - |
         CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
         FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
-        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -v '^deploy$' | sort -u || true)
+        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -vE '^(deploy|secrets)$' | sort -u || true)
         ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
         [ -z "$ALL_STACKS" ] && echo "No stacks changed, skipping" && exit 0
       - scp -o StrictHostKeyChecking=no deploy/create-secrets.sh root@$${SWARM_MANAGER_IP}:/tmp/cs.sh
       - |
         CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
         FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
-        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -v '^deploy$' | sort -u || true)
+        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -vE '^(deploy|secrets)$' | sort -u || true)
         ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
 
         for STACK in $ALL_STACKS; do
           echo "Provisioning: $STACK"
           case "$STACK" in
-            maintenance|media|unifi|guacamole|security|auth|traefik|meshcentral|ddm)
+            maintenance|media|unifi|guacamole|security|auth|meshcentral|ddm)
               echo "  No Docker secrets for $STACK — secrets in host .env";;
+            traefik)
+              # MIGRATED (2026-09-12) to manifest-driven provisioning after
+              # discovering traefik.env was committed to git with a literal
+              # "***REDACTED***" placeholder as KEEPALIVED_PASSWORD — every
+              # git-guard resync/checkout restored the broken value, causing
+              # VRRP auth mismatch between keepalived-master/-backup and VIP
+              # instability. All logic lives in deploy/provision-stack.py;
+              # the authoritative key list lives in
+              # traefik/traefik.env.template; the mapping lives in
+              # secrets/secrets-map.yaml. This case is intentionally one line.
+              python3 deploy/provision-stack.py traefik;;
             immich)
               ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} "source /tmp/cs.sh
                 create_or_update_secret 'immich_db_password'     '$${IMMICH_DB_PASSWORD}'
@@ -360,7 +397,7 @@ steps:
       - |
         CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
         FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
-        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -v '^deploy$' | sort -u || true)
+        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -vE '^(deploy|secrets)$' | sort -u || true)
         ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
         [ -z "$ALL_STACKS" ] && echo "No stacks changed" && exit 0
 
@@ -368,6 +405,13 @@ steps:
         rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
           deploy/ root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/deploy/
 
+        # Sync secrets/ tooling (manifest + examples) alongside deploy/ —
+        # provision-stack.py reads secrets/secrets-map.yaml from the CI
+        # checkout, but the host mirror should stay complete for emergency
+        # manual provisioning runs.
+        rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \
+          secrets/ root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/secrets/
+
         for STACK in $ALL_STACKS; do
           echo "--- Deploying: $STACK ---"
           # Sync files to host first (always, even for bootstrap stacks)
@@ -414,7 +458,7 @@ steps:
       - |
         CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n')
         FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true)
-        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -v '^deploy$' | sort -u || true)
+        FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -vE '^(deploy|secrets)$' | sort -u || true)
         ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
         [ -z "$ALL_STACKS" ] && exit 0
 
@@ -425,6 +469,10 @@ steps:
         # "nothing found in stack" output on ordinary deploys (e.g. vaultwarden,
         # 2026-08-25). Retry with backoff instead of a single fixed sleep, and
         # only warn (don't fail the pipeline) if tasks never show up.
+        # 2026-09-08: `|| true` inside the command substitution is REQUIRED —
+        # this step runs under errexit, and an assignment from a failing
+        # command substitution (e.g. `docker stack ps` on a stack that doesn't
+        # exist) kills the whole step before the WARNING path can run.
         ATTEMPTS=6
         DELAY=5
         for STACK in $ALL_STACKS; do
@@ -433,7 +481,7 @@ steps:
           while [ "$i" -le "$ATTEMPTS" ]; do
             OUTPUT=$(ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} \
               "docker stack ps $STACK --filter desired-state=running \
-              --format '  {{.Name}} {{.CurrentState}}'" 2>/dev/null)
+              --format '  {{.Name}} {{.CurrentState}}'" 2>/dev/null || true)
             if [ -n "$OUTPUT" ]; then
               echo "$OUTPUT"
               break

ai/ai.yaml

@@ -16,20 +16,20 @@ services:
       start_period: 60s
     environment:
     - OLLAMA_BASE_URL=http://ollama-intel-arc:11434
-    - WEBUI_SECRET_KEY=${WEBUI_SECRET_KEY}
+    - WEBUI_SECRET_KEY=${AI_OPEN_WEBUI_SECRET_KEY}
     - WEBUI_DB_HOST=postgresql
-    - DATABASE_URL=${OPEN_WEBUI_DATABASE_URL}
+    - DATABASE_URL=${AI_OPEN_WEBUI_DATABASE_URL}
     - ENABLE_TOOL_SERVER_CALLING=true
     - TOOL_SERVER_CALLING=true
-    - WEBUI_URL=${WEBUI_URL}
-    - ENABLE_OAUTH_SIGNUP=${ENABLE_OAUTH_SIGNUP}
-    - OAUTH_MERGE_ACCOUNTS_BY_EMAIL=${OAUTH_MERGE_ACCOUNTS_BY_EMAIL}
-    - OAUTH_PROVIDER_NAME=${OAUTH_PROVIDER_NAME}
-    - OPENID_PROVIDER_URL=${OPENID_PROVIDER_URL}
-    - OAUTH_CLIENT_ID=${OAUTH_CLIENT_ID}
-    - OAUTH_CLIENT_SECRET=${OAUTH_CLIENT_SECRET}
-    - OAUTH_SCOPES=${OAUTH_SCOPES}
-    - OPENID_REDIRECT_URI=${OPENID_REDIRECT_URI}
+    - WEBUI_URL=${AI_OPEN_WEBUI_URL}
+    - ENABLE_OAUTH_SIGNUP=${AI_OPEN_WEBUI_ENABLE_OAUTH_SIGNUP}
+    - OAUTH_MERGE_ACCOUNTS_BY_EMAIL=${AI_OPEN_WEBUI_OAUTH_MERGE_ACCOUNTS_BY_EMAIL}
+    - OAUTH_PROVIDER_NAME=${AI_OPEN_WEBUI_OAUTH_PROVIDER_NAME}
+    - OPENID_PROVIDER_URL=${AI_OPEN_WEBUI_OPENID_PROVIDER_URL}
+    - OAUTH_CLIENT_ID=${AI_OPEN_WEBUI_OAUTH_CLIENT_ID}
+    - OAUTH_CLIENT_SECRET=${AI_OPEN_WEBUI_OAUTH_CLIENT_SECRET}
+    - OAUTH_SCOPES=${AI_OPEN_WEBUI_OAUTH_SCOPES}
+    - OPENID_REDIRECT_URI=${AI_OPEN_WEBUI_OPENID_REDIRECT_URI}
     networks:
     - traefik_backend
     - postgresql_db-backend
@@ -62,13 +62,13 @@ services:
     environment:
     - AWS_ACCESS_KEY_ID=${AI_AWS_ACCESS_KEY_ID}
     - AWS_SECRET_ACCESS_KEY=${AI_AWS_SECRET_ACCESS_KEY}
-    - AWS_REGION_NAME=${AWS_REGION_NAME}
-    - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY}
-    - LITELLM_SALT_KEY=${LITELLM_SALT_KEY}
-    - DATABASE_URL=${DATABASE_URL}
-    - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
-    - LITELLM_MODIFY_PARAMS=${LITELLM_MODIFY_PARAMS}
-    - LITELLM_DATABASE_MIGRATIONS=${LITELLM_DATABASE_MIGRATIONS}
+    - AWS_REGION_NAME=${AI_AWS_REGION_NAME}
+    - LITELLM_MASTER_KEY=${AI_LITELLM_MASTER_KEY}
+    - LITELLM_SALT_KEY=${AI_LITELLM_SALT_KEY}
+    - DATABASE_URL=${AI_LITELLM_DATABASE_URL}
+    - POSTGRES_PASSWORD=${AI_LITELLM_POSTGRES_PASSWORD}
+    - LITELLM_MODIFY_PARAMS=${AI_LITELLM_MODIFY_PARAMS}
+    - LITELLM_DATABASE_MIGRATIONS=${AI_LITELLM_DATABASE_MIGRATIONS}
     networks:
     - traefik_backend
     - postgresql_db-backend
@@ -127,7 +127,7 @@ services:
       labels:
       - traefik.enable=false
   mcpo:
-    image: git.bryanmail.net/homelab/flowagent-mcpo:916164714429
+    image: git.bryanmail.net/homelab/flowagent-mcpo:c5b22618404a
     command:
     - --config
     - /app/config/config.json
@@ -141,6 +141,7 @@ services:
     - /volume1/docker/mcpo/data:/mcpo_data
     - /volume1/docker/cronicle/ssh_keys:/app/ssh_keys:ro
     - /volume1/docker/mcpo/uv-cache:/app/uv-cache
+    - /volume1/docker/mcpo/flowagent-auth:/app/flowagent-auth
     secrets:
     - source: flowagent_azure_client_id
       target: flowagent_azure_client_id

deploy/git-guard.sh

@@ -5,10 +5,20 @@
 # Behavior:
 #   - Clean + up to date        -> pass silently
 #   - Clean + behind (ff-only)  -> auto `git pull --ff-only`, then pass
-#   - Ahead only (unpushed)     -> interactive: offer to push; non-interactive: BLOCK
-#   - Dirty tracked changes     -> offer to commit + push right now
-#                                   (auto in non-interactive/CI runs, after a
+#   - Clean + ahead only        -> interactive: offer to push; non-interactive: BLOCK
+#     (unpushed)
+#   - Unresolved merge conflict -> REFUSE immediately. Never auto-commits over
+#     markers present               conflict markers. Prints remediation options.
+#   - Dirty + local in sync     -> offer to commit + push right now
+#     with origin                 (auto in non-interactive/CI runs, after a
 #                                   secret-pattern scan of the staged diff)
+#   - Dirty + local STALE/      -> NEVER commit on top of a stale base. Stash
+#     diverged vs origin           the dirty changes first, resync main with
+#                                   origin using the same behind/ahead/diverged
+#                                   rules as the clean-tree case, then reapply
+#                                   the stash and re-run. On any failure the
+#                                   stash is preserved and remediation options
+#                                   (with exact commands) are printed.
 #   - Diverged (local AND       -> REFUSE. Never auto-resolves. Prints the
 #     remote both moved)           backup/stash/reset recovery steps and exits.
 #
@@ -36,12 +46,246 @@ git status --porcelain | grep -q . && DIRTY=1
 
 SECRET_PATTERN='(-----BEGIN [A-Z]+ PRIVATE KEY-----|AKIA[0-9A-Z]{16}|xox[baprs]-[0-9a-zA-Z-]+|password[[:space:]]*[:=][[:space:]]*[^$ ]|api[_-]?key[[:space:]]*[:=][[:space:]]*[^$ ])'
 
+# resync_with_origin <local_sha> <remote_sha> <base_sha>
+#
+# Handles the behind/ahead/diverged cases against a CLEAN working tree.
+# Shared by both the "tree was already clean" path and the new
+# "dirty tree turned out to be stale, so we stashed first" path, so the
+# two paths can never drift out of sync with each other.
+#
+# Returns 0 if it's now safe to deploy, 1 if it could not safely resolve
+# (guidance already printed to stdout in that case).
+resync_with_origin() {
+  local local_sha="$1" remote_sha="$2" base_sha="$3"
+
+  # ---- Case: fully in sync ----
+  if [ "$local_sha" = "$remote_sha" ]; then
+    echo "==> In sync with origin/main ($local_sha). OK to deploy."
+    return 0
+  fi
+
+  # ---- Case: behind only (fast-forwardable) ----
+  if [ "$local_sha" = "$base_sha" ]; then
+    echo "!! Local main is behind origin/main."
+    if [ "$INTERACTIVE" -eq 1 ]; then
+      read -rp "Fast-forward pull now? [y/N] " ans
+    else
+      ans="y"
+      echo "(non-interactive session — auto fast-forwarding)"
+    fi
+    if [[ "$ans" =~ ^[Yy]$ ]]; then
+      git pull --ff-only origin main
+      echo "==> Fast-forwarded to $(git rev-parse --short main). OK to deploy."
+      return 0
+    else
+      echo "Aborting - pull manually, then retry:"
+      echo "  cd $DIR && git pull --ff-only origin main"
+      return 1
+    fi
+  fi
+
+  # ---- Case: ahead only (local commits not yet pushed) ----
+  if [ "$remote_sha" = "$base_sha" ]; then
+    echo "!! Local main is AHEAD of origin/main (unpushed commits):"
+    git log --oneline "origin/main..main"
+    echo
+    if [ "$INTERACTIVE" -eq 1 ]; then
+      read -rp "Push local commits to origin/main now? [y/N] " ans
+    else
+      ans="n"
+      echo "(non-interactive session — will NOT auto-push ahead commits; needs human review)"
+    fi
+    if [[ "$ans" =~ ^[Yy]$ ]]; then
+      git push origin main
+      echo "==> Pushed. OK to deploy."
+      return 0
+    else
+      echo "Aborting. Review with:"
+      echo "  cd $DIR && git log origin/main..main"
+      echo "Then push manually when ready:"
+      echo "  git push origin main"
+      return 1
+    fi
+  fi
+
+  # ---- Case: true divergence (both ahead and behind) — NEVER auto-fix ----
+  echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+  echo "!! DIVERGED: local main and origin/main have both moved independently."
+  echo "!!"
+  echo "!!   Local-only commits:"
+  git log --oneline "$base_sha..main" | sed 's/^/!!     /'
+  echo "!!"
+  echo "!!   Remote-only commits:"
+  git log --oneline "$base_sha..origin/main" | sed 's/^/!!     /'
+  echo "!!"
+  echo "!! This requires a human decision - git-guard will NOT auto-resolve this."
+  echo "!!"
+  echo "!! Before stashing anything new, check whether a git-guard safety stash"
+  echo "!! ALREADY exists from this same run (avoids confusing duplicate stashes):"
+  echo "!!   git stash list"
+  echo "!!"
+  echo "!! Recommended recovery:"
+  echo "!!   1. tar backup:      tar czf /volume1/docker/compose-files-backup-\$(date +%Y%m%d-%H%M%S).tar.gz -C /volume1/docker compose-files"
+  echo "!!   2. name the branch: git branch backup/pre-reset-\$(date +%Y%m%d)"
+  echo "!!   3. stash any NEW uncommitted state only if 'git stash list' above"
+  echo "!!      didn't already show one for this run:"
+  echo "!!        git stash push -u -m 'pre-reset-snapshot'"
+  echo "!!   4. reset to origin: git reset --hard origin/main"
+  echo "!!   5. selectively restore needed files from the stash/backup branch:"
+  echo "!!        git stash list"
+  echo "!!        git stash show -p stash@{N}"
+  echo "!!        git stash apply stash@{N}   # 'apply' keeps the stash as a backup; use 'pop' to also drop it"
+  echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+  return 1
+}
+
+# ---- Case: unresolved merge conflict already present ----
+# Can happen if a PRIOR git-guard run's `git stash pop` conflicted and the
+# resulting conflict markers were never resolved before the next deploy
+# attempt. Must be checked BEFORE the dirty-tree commit flow below, because
+# an unmerged path shows up as "dirty" too, and `git add -A` would silently
+# stage the literal <<<<<<< / ======= / >>>>>>> markers into a real commit.
+if git ls-files -u | grep -q .; then
+  echo "ERROR: unresolved merge conflict markers present in the working tree."
+  echo "Refusing to auto-commit over a conflict — this would push literal"
+  echo "<<<<<<< / ======= / >>>>>>> markers to origin/main."
+  echo
+  echo "Conflicted paths:"
+  git diff --name-only --diff-filter=U | sed 's/^/  /'
+  echo
+  echo "Remediation options:"
+  echo "  A) Resolve the conflict by hand, then commit and push:"
+  echo "       cd $DIR"
+  echo "       git status                       # see conflicted paths"
+  echo "       git diff                          # inspect the conflict markers"
+  echo "       \$EDITOR <conflicted-file>          # remove markers, keep correct content"
+  echo "       git add <conflicted-file>"
+  echo "       git commit -m 'resolve git-guard stash-pop conflict'"
+  echo "       git push origin main"
+  echo "  B) Discard the conflicted merge attempt entirely and start clean from"
+  echo "     origin/main, then decide separately whether to re-apply anything"
+  echo "     from a prior safety stash:"
+  echo "       cd $DIR"
+  echo "       git checkout -- ."
+  echo "       git reset --hard origin/main"
+  echo "       git stash list                    # look for a git-guard-safety-stash-* entry"
+  echo "       git stash show -p stash@{N}         # inspect before deciding"
+  echo "  C) Once resolved (via A or B) and no longer needed, clean up the stash:"
+  echo "       git stash drop stash@{N}"
+  exit 1
+fi
+
 # ---- Case: dirty tracked changes ----
 if [ "$DIRTY" -eq 1 ]; then
   echo "!! WORKING TREE DIRTY — uncommitted changes detected:"
   git status --short
   echo
 
+  # If local is ALSO stale/diverged from origin, committing right now would
+  # create a doomed commit on top of a base that's about to be rejected on
+  # push (this is exactly what caused a real incident: a stray on-disk edit
+  # sat on a checkout that was 4 commits behind, git-guard auto-committed
+  # anyway, then the push bounced). Stash first, resync safely using the
+  # same rules as the clean-tree path, then reapply.
+  if [ "$LOCAL" != "$REMOTE" ]; then
+    echo "!! Local main is ALSO stale/diverged from origin/main."
+    echo "   Refusing to commit on top of a stale base — stashing the dirty"
+    echo "   changes safely first, then resyncing with origin."
+    echo
+
+    STASH_MSG="git-guard-safety-stash-$(date -u +%Y%m%dT%H%M%SZ)"
+    if ! git stash push -u -m "$STASH_MSG"; then
+      echo "ERROR: 'git stash push' itself failed (disk full, permissions, or"
+      echo "       some other git error). Your changes are still on disk,"
+      echo "       uncommitted — nothing has been lost, but git-guard cannot"
+      echo "       proceed safely until this is resolved."
+      echo
+      echo "Remediation options:"
+      echo "  A) Check disk space and permissions, then retry the deploy:"
+      echo "       df -h $DIR"
+      echo "       ls -la $DIR"
+      echo "  B) Identify and manually move aside whatever is blocking the stash,"
+      echo "     then retry:"
+      echo "       cd $DIR"
+      echo "       git status --short              # find the offending path(s)"
+      echo "       mv <path> <path>.bak-\$(date +%s)"
+      echo "  C) Inspect the raw git error above for specifics before proceeding."
+      exit 1
+    fi
+    echo "==> Stashed as: $STASH_MSG"
+
+    if resync_with_origin "$LOCAL" "$REMOTE" "$BASE"; then
+      echo "==> Resync succeeded. Reapplying stashed changes..."
+      if git stash pop; then
+        echo "==> Stash reapplied cleanly. Re-checking sync state..."
+        exec bash "$0" "$@"
+      else
+        echo "ERROR: 'git stash pop' did not complete successfully."
+        echo
+        if git ls-files -u | grep -q .; then
+          echo "This is a MERGE CONFLICT — your stashed changes were partially"
+          echo "applied and conflict markers (<<<<<<< / ======= / >>>>>>>) are now"
+          echo "in the working tree. The stash itself is still preserved as a backup."
+          echo
+          echo "Remediation options:"
+          echo "  A) Resolve the conflict by hand, then commit and push:"
+          echo "       cd $DIR"
+          echo "       git status                       # see conflicted paths"
+          echo "       git diff                          # inspect the markers"
+          echo "       \$EDITOR <conflicted-file>          # remove markers, keep correct content"
+          echo "       git add <conflicted-file>"
+          echo "       git commit -m 'resolve git-guard stash-pop conflict'"
+          echo "       git push origin main"
+          echo "       git stash list                    # confirm which entry is: $STASH_MSG"
+          echo "       git stash drop stash@{N}           # once confirmed no longer needed"
+          echo "  B) Abandon this merge attempt and fall back to a clean, resynced"
+          echo "     tree, then re-apply the change manually with full visibility:"
+          echo "       cd $DIR"
+          echo "       git checkout -- ."
+          echo "       git reset --hard origin/main      # now matches origin, no conflict"
+          echo "       git stash list                    # find: $STASH_MSG"
+          echo "       git stash show -p stash@{N}         # review the content"
+          echo "       git stash apply stash@{N}          # 'apply' keeps the backup; use 'pop' to also drop it"
+        else
+          echo "This looks like an UNTRACKED-FILE COLLISION, not a merge conflict"
+          echo "(a file added upstream shares a path with an untracked file in your"
+          echo "stash). No conflict markers were written; the stash was NOT applied"
+          echo "and remains fully intact."
+          echo
+          echo "Remediation options:"
+          echo "  A) Move the colliding upstream file aside, pop, then reconcile:"
+          echo "       cd $DIR"
+          echo "       git status --short                         # identify the colliding path"
+          echo "       mv <path> <path>.upstream-\$(date +%s)"
+          echo "       git stash pop"
+          echo "       diff <path> <path>.upstream-*               # reconcile manually, then remove the .upstream-* backup"
+          echo "  B) Inspect the stash without applying, and hand-merge the needed"
+          echo "     pieces instead:"
+          echo "       git stash list                              # find N"
+          echo "       git stash show -p stash@{N}"
+        fi
+        echo
+        echo "Your stash reference for this run: $STASH_MSG"
+        exit 1
+      fi
+    else
+      echo "ERROR: could not safely resync with origin/main."
+      echo "Your uncommitted changes are preserved in the stash: $STASH_MSG"
+      echo
+      echo "Remediation options:"
+      echo "  A) Follow the manual recovery steps printed above (from the"
+      echo "     behind/ahead/diverged case), THEN reapply your change:"
+      echo "       cd $DIR"
+      echo "       git stash list                 # find: $STASH_MSG"
+      echo "       git stash apply stash@{N}      # or 'pop' to also drop it once resynced"
+      echo "  B) If the stashed change is no longer needed (e.g. it's already"
+      echo "     represented in a since-merged PR), verify then drop it:"
+      echo "       git stash show -p stash@{N}"
+      echo "       git stash drop stash@{N}"
+      exit 1
+    fi
+  fi
+
   if [ "$INTERACTIVE" -eq 1 ]; then
     read -rp "Commit and push these changes to origin/main now? [y/N] " ans
   else
@@ -54,7 +298,8 @@ if [ "$DIRTY" -eq 1 ]; then
 
     if git diff --cached | grep -Eiq "$SECRET_PATTERN"; then
       echo "ERROR: possible secret detected in staged changes. Refusing to auto-commit."
-      echo "Review manually: git diff --cached"
+      echo "Review manually:"
+      echo "  cd $DIR && git diff --cached"
       git reset
       exit 1
     fi
@@ -63,80 +308,25 @@ if [ "$DIRTY" -eq 1 ]; then
 
     if git push origin main; then
       echo "==> Pushed. Re-checking sync state..."
-      exec "$0" "$@"
+      exec bash "$0" "$@"
     else
       echo "ERROR: push failed (likely diverged from origin). Aborting deploy."
-      echo "Run: cd $DIR && git status"
+      echo "Run:"
+      echo "  cd $DIR && git status"
       exit 1
     fi
   else
-    echo "Aborting deploy - commit or stash changes manually, then retry."
+    echo "Aborting deploy - commit or stash changes manually, then retry:"
+    echo "  cd $DIR"
+    echo "  git add -A && git commit -m 'your message' && git push origin main"
+    echo "  # or: git stash push -u -m 'manual-stash'"
     exit 1
   fi
 fi
 
-# ---- Case: fully in sync ----
-if [ "$LOCAL" = "$REMOTE" ]; then
-  echo "==> In sync with origin/main ($LOCAL). OK to deploy."
+# ---- Clean tree: resync with origin using the shared logic above ----
+if resync_with_origin "$LOCAL" "$REMOTE" "$BASE"; then
   exit 0
+else
+  exit 1
 fi
-
-# ---- Case: behind only (fast-forwardable) ----
-if [ "$LOCAL" = "$BASE" ]; then
-  echo "!! Local main is behind origin/main."
-  if [ "$INTERACTIVE" -eq 1 ]; then
-    read -rp "Fast-forward pull now? [y/N] " ans
-  else
-    ans="y"
-    echo "(non-interactive session — auto fast-forwarding)"
-  fi
-  if [[ "$ans" =~ ^[Yy]$ ]]; then
-    git pull --ff-only origin main
-    echo "==> Fast-forwarded to $(git rev-parse --short main). OK to deploy."
-    exit 0
-  else
-    echo "Aborting deploy - pull manually, then retry."
-    exit 1
-  fi
-fi
-
-# ---- Case: ahead only (local commits not yet pushed) ----
-if [ "$REMOTE" = "$BASE" ]; then
-  echo "!! Local main is AHEAD of origin/main (unpushed commits):"
-  git log --oneline "origin/main..main"
-  echo
-  if [ "$INTERACTIVE" -eq 1 ]; then
-    read -rp "Push local commits to origin/main now? [y/N] " ans
-  else
-    ans="n"
-    echo "(non-interactive session — will NOT auto-push ahead commits; needs human review)"
-  fi
-  if [[ "$ans" =~ ^[Yy]$ ]]; then
-    git push origin main
-    echo "==> Pushed. OK to deploy."
-    exit 0
-  else
-    echo "Aborting deploy. Review with: git log origin/main..main"
-    exit 1
-  fi
-fi
-
-# ---- Case: true divergence (both ahead and behind) — NEVER auto-fix ----
-echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
-echo "!! DIVERGED: local main and origin/main have both moved independently."
-echo "!!"
-echo "!!   Local-only commits:"
-git log --oneline "$BASE..main" | sed 's/^/!!     /'
-echo "!!"
-echo "!!   Remote-only commits:"
-git log --oneline "$BASE..origin/main" | sed 's/^/!!     /'
-echo "!!"
-echo "!! This requires a human decision - git-guard will NOT auto-resolve this."
-echo "!! Recommended recovery:"
-echo "!!   1. tar backup:      tar czf /volume1/docker/compose-files-backup-\$(date +%Y%m%d-%H%M%S).tar.gz -C /volume1/docker compose-files"
-echo "!!   2. name the branch: git branch backup/pre-reset-\$(date +%Y%m%d)"
-echo "!!   3. stash all state: git stash push -u -m 'pre-reset-snapshot'"
-echo "!!   4. reset to origin: git reset --hard origin/main"
-echo "!!   5. selectively restore needed files from the stash/backup branch"
-echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
-exit 1

homeassistant/homeassistant.yaml

@@ -63,6 +63,15 @@ services:
       FRIGATE_RTSP_PASSWORD: "${FRIGATE_RTSP_PASSWORD}"
     networks:
       - traefik_backend
+    ports:
+      - target: 8555
+        published: 8555
+        protocol: udp
+        mode: ingress
+      - target: 8555
+        published: 8555
+        protocol: tcp
+        mode: ingress
     deploy:
       labels:
         - traefik.enable=true

secrets/README.md

@@ -6,9 +6,9 @@ This directory documents the secrets required for each Docker Swarm stack.
 
 ```
 Secret values live in Woodpecker (encrypted)
-    ↓ pipeline reads them at deploy time
+    → pipeline reads them at deploy time
 Docker Swarm secret store (encrypted Raft DB, replicated across all nodes)
-    ↓ mounted into containers at runtime
+    → mounted into containers at runtime
 /run/secrets/<secret_name>
 ```
 
@@ -38,7 +38,10 @@ Use the stack's `.secrets.example` file as your checklist.
 
 ### Step 2 — Add the stack's case to `.woodpecker.yml`
 In the `provision-secrets` step, add a case for the stack that calls
-`create_or_update_secret` for each secret.
+`create_or_update_secret` for each secret. (Or, preferred for new
+migrations: add an entry to `secrets/secrets-map.yaml` + a
+`<stack>.env.template` and call `deploy/provision-stack.py <stack>`
+instead — see the `ai` and `traefik` entries for the current pattern.)
 
 ### Step 3 — Test by pushing a trivial change to the stack's yaml file
 Watch the pipeline run: provision-secrets → validate → deploy → verify → notify.
@@ -71,10 +74,20 @@ Examples:
 - **PostgreSQL is highest risk.** Its master password is used by nearly every
   other stack. Migrate it last.
 
+- **Never commit a rendered env file, even by accident.** `traefik/traefik.env`
+  was committed to git for a period (discovered/fixed 2026-09-12) with a
+  literal "***REDACTED***" placeholder as KEEPALIVED_PASSWORD, which was
+  silently restored every time the file was deleted or the checkout resynced
+  from git — causing a real VRRP auth outage. `.gitignore` blanket-excludes
+  `*.env`, but that rule does NOT retroactively untrack a file already
+  committed before the rule existed. If you ever see a stack's `.env` file
+  show up in `git status` as tracked, stop and untrack it (`git rm --cached`)
+  before doing anything else.
+
 ## Migration Status
 
 | Stack | Secrets in Woodpecker | Pipeline Step Added | .env Removed |
-|-------|----------------------|---------------------|--------------|
+|-------|------------------------|----------------------|---------------|
 | 3dprint | ⏳ | ⏳ | ⏳ |
 | ai | ⏳ | ⏳ | ⏳ |
 | auth | ⏳ | ⏳ | ⏳ |
@@ -91,7 +104,7 @@ Examples:
 | postgresql | ⏳ | ⏳ | ⏳ |
 | productivity | ⏳ | ⏳ | ⏳ |
 | security | ⏳ | ⏳ | ⏳ |
-| traefik | ⏳ | ⏳ | ⏳ |
+| traefik | ✅ (manifest-driven, 2026-09-12) | ✅ (manifest-driven, 2026-09-12) | ⏳ |
 | unifi | ✅ N/A (no secrets) | ✅ N/A | ⏳ |
 | vaultwarden | ⏳ | ⏳ | ⏳ |
 | woodpecker | ⏳ Manual only | ⏳ N/A | ⏳ |

secrets/secrets-map.yaml

@@ -1,22 +1,22 @@
-# ─────────────────────────────────────────────────────────────────────────────
+# ─────────────────────────────────────────────────────────────────────────
 # secrets-map.yaml — DATA-ONLY manifest for deploy/provision-stack.py
 #
 # RULES:
 #   - This file contains NO code, NO shell, NO secret values — only names.
 #   - Each stack entry declares:
 #       env_template:   repo path of the FULL env-file template (tracked).
-#                       The template is authoritative: the COMPLETE env file is
-#                       rendered from it on every provisioning run. Nothing is
-#                       line-edited in place, so keys can never silently go
-#                       missing.
+#                        The template is authoritative: the COMPLETE env file is
+#                        rendered from it on every provisioning run. Nothing is
+#                        line-edited in place, so keys can never silently go
+#                        missing.
 #       env_dest:       host path (relative to /volume1/docker/compose-files/)
-#                       the rendered env file is shipped to. Rendered file
-#                       exists ONLY on the host — never committed to git.
+#                        the rendered env file is shipped to. Rendered file
+#                        exists ONLY on the host — never committed to git.
 #       docker_secrets: map of docker-swarm-secret-name -> CI ENV VAR NAME
-#                       (Pattern C). The env var must be declared via
-#                       from_secret: in .woodpecker/deploy.yml's
-#                       provision-secrets step (Woodpecker v3 requires explicit
-#                       per-secret declaration; there is no expose-all).
+#                        (Pattern C). The env var must be declared via
+#                        from_secret: in .woodpecker/deploy.yml's
+#                        provision-secrets step (Woodpecker v3 requires explicit
+#                        per-secret declaration; there is no expose-all).
 #
 # ADDING A NEW SECRET (3 small steps, no shell edits):
 #   1. Add the secret value in Woodpecker UI (repo Settings -> Secrets).
@@ -27,7 +27,7 @@
 #
 # Stacks not listed here fall through to deploy.yml's legacy case-entries
 # untouched. Migration is deliberately one stack per PR.
-# ─────────────────────────────────────────────────────────────────────────────
+# ─────────────────────────────────────────────────────────────────────────
 stacks:
   ai:
     env_template: ai/ai.env.template
@@ -36,3 +36,6 @@ stacks:
       flowagent_azure_client_id: FLOWAGENT_AZURE_CLIENT_ID
       flowagent_azure_tenant_id: FLOWAGENT_AZURE_TENANT_ID
       flowagent_azure_client_secret: FLOWAGENT_AZURE_CLIENT_SECRET
+  traefik:
+    env_template: traefik/traefik.env.template
+    env_dest: traefik/traefik.env

secrets/traefik.secrets.example

@@ -1,5 +1,8 @@
 # traefik Stack — Secrets Reference
-# Source: traefik.env
+# Source: traefik.env.template (rendered by deploy/provision-stack.py per
+#         secrets/secrets-map.yaml — see secrets-map.yaml header for how
+#         this works). traefik/traefik.env is rendered fresh on every
+#         provisioning run and is NEVER committed to git.
 #
 # Add SECRET values to Woodpecker at:
 #   https://woodpecker.bryanmail.net
@@ -8,27 +11,44 @@
 # ⚠️  HIGH RISK: Traefik is the entry point for all homelab services.
 #     If this stack fails, nothing is reachable from outside.
 #     Migrate carefully. The Keepalived VIP (192.168.4.30) depends on this stack.
+#
+# ⚠️  2026-09-12 INCIDENT: traefik.env was previously committed to git with a
+#     literal "***REDACTED***" placeholder as KEEPALIVED_PASSWORD. Every
+#     git-guard resync/checkout restored that broken value onto disk,
+#     diverging from keepalived-backup's stale-but-correct in-memory value
+#     and causing continuous VRRP auth failures + VIP instability. This was
+#     the trigger for migrating this stack to Pattern C. If you ever see
+#     KEEPALIVED_PASSWORD as a literal placeholder-looking string on disk
+#     again, do NOT hand-edit it — check `git log traefik/` for a stray
+#     commit and fix the template in Gitea instead.
 
-# ── SECRETS (add to Woodpecker) ───────────────────────────────────────────────
+# ── SECRETS (add to Woodpecker) ─────────────────────────────────────────
 
 # Woodpecker secret name:  traefik_keepalived_password
-# Used for:               Keepalived VRRP authentication password
-#                         Must match across all 3 nodes (docker-1, docker-2, docker-3)
-# Env var in .env:        KEEPALIVED_PASSWORD
+# Used for:                Keepalived VRRP authentication password
+#                          Must match across all 3 nodes (docker-1, docker-2, docker-3)
+#                          NOTE: classic VRRP simple-auth is silently
+#                          truncated to 8 chars by keepalived itself — keep
+#                          the value <= 8 characters, or be aware only the
+#                          first 8 are actually significant on the wire.
+# Env var in .env.template: KEEPALIVED_PASSWORD (via TRAEFIK_KEEPALIVED_PASSWORD)
 traefik_keepalived_password=
 
-# ── NON-SECRETS (safe in compose file or .env) ────────────────────────────────
+# ── NON-SECRETS (safe in compose file or .env.template) ─────────────────
 
-# KEEPALIVED_UNICAST_PEERS     Python2BASH list of peer IPs
-# KEEPALIVED_VIRTUAL_IPS       Python2BASH list of VIP addresses (192.168.4.30)
+# KEEPALIVED_VIRTUAL_IPS       Python2BASH list of VIP addresses (192.168.4.30) — literal in template
 # ACME_EMAIL                   Let's Encrypt certificate email
 # TRUSTED_IPS                  Trusted proxy CIDR ranges
 # TRAEFIK_HOST                 Traefik dashboard hostname
-# SPEEDTEST_HOST               Speedtest Traefik hostname
+# SPEEDTEST_HOST                Speedtest Traefik hostname
 # WHOAMI_HOST                  Whoami Traefik hostname
 
-# ── Woodpecker provision-secrets case entry ───────────────────────────────────
+# ── Provisioning (manifest-driven, deploy/provision-stack.py) ───────────
 #
-#   traefik)
-#     create_or_update_secret "traefik_keepalived_password" "$TRAEFIK_KEEPALIVED_PASSWORD"
-#     ;;
+# This stack is migrated — provisioning happens automatically via:
+#   secrets/secrets-map.yaml (traefik: entry)
+#   traefik/traefik.env.template (authoritative key list)
+#   deploy/provision-stack.py (renders + ships traefik/traefik.env)
+#
+# The .woodpecker/deploy.yml provision-secrets step calls this with a
+# single line: `python3 deploy/provision-stack.py traefik`

traefik/traefik.env

@@ -1,2 +0,0 @@
-KEEPALIVED_PASSWORD=***REDACTED***
-KEEPALIVED_VIRTUAL_IPS="#PYTHON2BASH:['192.168.4.30']"

traefik/traefik.env.template

@@ -0,0 +1,39 @@
+# ─────────────────────────────────────────────────────────────────────────
+# traefik.env.template — AUTHORITATIVE template for traefik/traefik.env
+# (rendered by deploy/provision-stack.py per secrets/secrets-map.yaml)
+#
+#   - This file IS the complete key list for traefik.env. The whole file is
+#     rendered on every provisioning run — no line surgery, so a key can
+#     never silently go missing.
+#   - Non-secret config lives here as LITERAL values (visible, reviewable).
+#   - Secret values are dollar-brace placeholders resolved from the CI env
+#     (Woodpecker from_secret vars) at provisioning time. provision-stack.py
+#     FAILS HARD if any placeholder is missing/empty.
+#   - The rendered traefik/traefik.env exists only on the host (gitignored).
+#   - Rendered by provision-stack.py, NOT Woodpecker's yaml preprocessor —
+#     single-dollar placeholders are safe here (deploy.yml's double-dollar
+#     rule does NOT apply to this file).
+#
+# Consumed by traefik/traefik.yaml (KEEPALIVED_PASSWORD, KEEPALIVED_VIRTUAL_IPS).
+# DOMAIN comes from deploy/global.env, not here.
+#
+# ⚠️ HIGH RISK: Traefik is the entry point for all homelab services, and the
+#    Keepalived VIP (192.168.4.30) depends on this stack. KEEPALIVED_PASSWORD
+#    MUST be identical across keepalived-master and keepalived-backup — both
+#    consume this same rendered value — or VRRP auth fails and the VIP
+#    becomes unstable (this is exactly what the 2026-09-12 incident was:
+#    a committed literal "***REDACTED***" placeholder in git, restored every
+#    time the file was deleted/re-synced, diverging from -backup's stale but
+#    correct in-memory value).
+#
+#    Classic VRRP simple-auth is silently truncated to 8 characters by
+#    keepalived itself. Set TRAEFIK_KEEPALIVED_PASSWORD in Woodpecker to a
+#    value 8 characters or fewer (or accept that only the first 8 chars are
+#    actually significant) so the effective negotiated value is unambiguous.
+# ─────────────────────────────────────────────────────────────────────────
+
+# ── Keepalived (non-secret config) ────────────────────────────────────────
+KEEPALIVED_VIRTUAL_IPS="#PYTHON2BASH:['192.168.4.30']"
+
+# ── Keepalived (secret) ────────────────────────────────────────────────────
+KEEPALIVED_PASSWORD=${TRAEFIK_KEEPALIVED_PASSWORD}