From eee6e543b162953e1c286d9bc02adeb5e9715ca7 Mon Sep 17 00:00:00 2001 From: Bot Date: Mon, 7 Sep 2026 22:26:43 -0700 Subject: [PATCH] Fix phantom verify failure: exclude secrets/ from stack detection + errexit-proof verify's stack-ps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of PR #16's pipeline failure (#398-#401): secrets/ is a tooling folder (secrets-map.yaml, *.secrets.example), not a stack, but folder-detection (cut -d/ -f1, excluding only deploy/) computed ALL_STACKS="ai secrets" the first time a commit touched secrets/. The deploy step survived only because 'secrets' coincidentally sits in the bootstrap-tier skip list; verify had no guard — `docker stack ps secrets` exited non-zero and, under errexit, the OUTPUT=$(...) assignment killed the step before the designed WARNING path could run. The ai deploy itself was fully successful (complete 21-key render, all 6 services healthy) — the failure was pure phantom. Hazard first flagged July 2026 in PR #3 (closed unmerged). Changes: - grep -v '^deploy$' -> grep -vE '^(deploy|secrets)$' in ALL five folder-detection sites (validate, provision-secrets x2, deploy, verify) - verify: `|| true` inside the stack-ps command substitution so a genuinely missing stack warns instead of aborting (comment added) - deploy step: also rsync secrets/ tooling to the host mirror unconditionally, same treatment as deploy/ - header: 2026-09-08 incident note Everything else byte-identical to main. Dollar-escape audit done. --- .woodpecker/deploy.yml | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/.woodpecker/deploy.yml b/.woodpecker/deploy.yml index b7910ef..b4bb40f 100644 --- a/.woodpecker/deploy.yml +++ b/.woodpecker/deploy.yml @@ -59,6 +59,17 @@ when: # render, hard failure naming any missing value). Migrated stacks call # the script; unmigrated stacks keep legacy case entries until their own # PR. See the "Secrets & Deployment Architecture — Global Direction" note. +# +# 2026-09-08 FIX: secrets/ is a tooling/docs folder (secrets-map.yaml + +# *.secrets.example), not a stack — but folder-detection treated it as one +# the first time a commit touched it (PR #16). deploy survived only because +# 'secrets' sits in the bootstrap-tier skip list; verify had no guard and +# died on `docker stack ps secrets` failing under errexit (assignment from +# a failing command substitution aborts the step). Fixed by excluding +# secrets/ alongside deploy/ in ALL folder-detection sites, and by +# tolerating a failing stack-ps in verify (|| true) so a genuinely missing +# stack produces the designed WARNING instead of killing the step. This +# hazard was first flagged in July (PR #3, closed unmerged). # ───────────────────────────────────────────────────────────────────────────── steps: @@ -76,9 +87,9 @@ steps: FLAT=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.ya?ml$' || true) # Folder: any file under a subfolder (e.g. immich/immich.yml). - # Exclude dotfolders (.woodpecker, .git, .gitea, etc.) and deploy/ - # (shared tooling, not a stack — see note above). - FOLDERS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -v '^deploy$' | sort -u || true) + # Exclude dotfolders (.woodpecker, .git, .gitea, etc.) and the + # non-stack tooling folders deploy/ and secrets/ (see notes above). + FOLDERS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -vE '^(deploy|secrets)$' | sort -u || true) [ -z "$FLAT" ] && [ -z "$FOLDERS" ] && echo "No stacks changed" && exit 0 @@ -219,14 +230,14 @@ steps: - | CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n') FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true) - FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -v '^deploy$' | sort -u || true) + FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -vE '^(deploy|secrets)$' | sort -u || true) ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u) [ -z "$ALL_STACKS" ] && echo "No stacks changed, skipping" && exit 0 - scp -o StrictHostKeyChecking=no deploy/create-secrets.sh root@$${SWARM_MANAGER_IP}:/tmp/cs.sh - | CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n') FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true) - FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -v '^deploy$' | sort -u || true) + FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -vE '^(deploy|secrets)$' | sort -u || true) ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u) for STACK in $ALL_STACKS; do @@ -360,7 +371,7 @@ steps: - | CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n') FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true) - FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -v '^deploy$' | sort -u || true) + FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -vE '^(deploy|secrets)$' | sort -u || true) ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u) [ -z "$ALL_STACKS" ] && echo "No stacks changed" && exit 0 @@ -368,6 +379,13 @@ steps: rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \ deploy/ root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/deploy/ + # Sync secrets/ tooling (manifest + examples) alongside deploy/ — + # provision-stack.py reads secrets/secrets-map.yaml from the CI + # checkout, but the host mirror should stay complete for emergency + # manual provisioning runs. + rsync -av -e "ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa" \ + secrets/ root@$${SWARM_MANAGER_IP}:/volume1/docker/compose-files/secrets/ + for STACK in $ALL_STACKS; do echo "--- Deploying: $STACK ---" # Sync files to host first (always, even for bootstrap stacks) @@ -414,7 +432,7 @@ steps: - | CHANGED_FILES=$(echo "${CI_PIPELINE_FILES}" | tr -d '[]"' | tr ',' '\n') FLAT_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/]+\.yaml$' | sed 's/\.yaml$//' || true) - FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -v '^deploy$' | sort -u || true) + FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | grep -vE '^(deploy|secrets)$' | sort -u || true) ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u) [ -z "$ALL_STACKS" ] && exit 0 @@ -425,6 +443,10 @@ steps: # "nothing found in stack" output on ordinary deploys (e.g. vaultwarden, # 2026-08-25). Retry with backoff instead of a single fixed sleep, and # only warn (don't fail the pipeline) if tasks never show up. + # 2026-09-08: `|| true` inside the command substitution is REQUIRED — + # this step runs under errexit, and an assignment from a failing + # command substitution (e.g. `docker stack ps` on a stack that doesn't + # exist) kills the whole step before the WARNING path can run. ATTEMPTS=6 DELAY=5 for STACK in $ALL_STACKS; do @@ -433,7 +455,7 @@ steps: while [ "$i" -le "$ATTEMPTS" ]; do OUTPUT=$(ssh -o StrictHostKeyChecking=no root@$${SWARM_MANAGER_IP} \ "docker stack ps $STACK --filter desired-state=running \ - --format ' {{.Name}} {{.CurrentState}}'" 2>/dev/null) + --format ' {{.Name}} {{.CurrentState}}'" 2>/dev/null || true) if [ -n "$OUTPUT" ]; then echo "$OUTPUT" break