From ca94b551d72e2944e16935424cc3aca2253425d6 Mon Sep 17 00:00:00 2001
From: admin <alec@bryanmail.net>
Date: Tue, 25 Aug 2026 23:06:56 -0700
Subject: [PATCH] verify: replace fixed sleep 5 + one-shot check with
 retry-with-backoff

`docker stack deploy` briefly tears down and recreates tasks in Swarm's
internal bookkeeping, so `docker stack ps` can transiently return nothing
right after a deploy even when the service is healthy. A single `sleep 5`
followed by one check produced a false-alarm-looking "nothing found in
stack: vaultwarden" on an otherwise-successful deploy (2026-08-25).

Now retries up to 6 times, 5s apart (~30s total) before printing a WARNING
with a manual-check command. Does not fail the pipeline on its own --
verify was already informational, not a hard gate.

No changes to provision-secrets or deploy steps.
---
 .woodpecker/deploy.yml | 32 ++++++++++++++++++++++++++++----
 1 file changed, 28 insertions(+), 4 deletions(-)

diff --git a/.woodpecker/deploy.yml b/.woodpecker/deploy.yml
index 2b54170..00bfd31 100644
--- a/.woodpecker/deploy.yml
+++ b/.woodpecker/deploy.yml
@@ -355,12 +355,36 @@ steps:
         FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true)
         ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
         [ -z "$ALL_STACKS" ] && exit 0
-        sleep 5
+
+        # NOTE: `docker stack deploy` briefly tears down and recreates tasks in
+        # Swarm's internal bookkeeping, so `docker stack ps` can transiently
+        # return nothing right after deploy even when the service is healthy.
+        # A single `sleep 5` + one-shot check produced false-alarm-looking
+        # "nothing found in stack" output on ordinary deploys (e.g. vaultwarden,
+        # 2026-08-25). Retry with backoff instead of a single fixed sleep, and
+        # only warn (don't fail the pipeline) if tasks never show up.
+        ATTEMPTS=6
+        DELAY=5
         for STACK in $ALL_STACKS; do
           echo "--- $STACK ---"
-          ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} \
-            "docker stack ps $STACK --filter desired-state=running \
-            --format '  {{.Name}} {{.CurrentState}}'"
+          i=1
+          while [ "$i" -le "$ATTEMPTS" ]; do
+            OUTPUT=$(ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} \
+              "docker stack ps $STACK --filter desired-state=running \
+              --format '  {{.Name}} {{.CurrentState}}'" 2>/dev/null)
+            if [ -n "$OUTPUT" ]; then
+              echo "$OUTPUT"
+              break
+            fi
+            if [ "$i" -eq "$ATTEMPTS" ]; then
+              echo "  WARNING: no running tasks found for $STACK after $((ATTEMPTS * DELAY))s."
+              echo "  This may be transient Swarm settle time, or a real problem — check manually:"
+              echo "  ssh root@${SWARM_MANAGER_IP} 'docker stack ps $STACK --no-trunc'"
+            else
+              sleep "$DELAY"
+            fi
+            i=$((i + 1))
+          done
         done
 
   notify-success:
-- 
2.54.0