verify: fix race condition causing false-alarm "nothing found in stack" #5

Closed
admin wants to merge 1 commits from fix-verify-race-condition into main
Showing only changes of commit ca94b551d7 - Show all commits
+28 -4
View File
@@ -355,12 +355,36 @@ steps:
FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true) FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true)
ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u) ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
[ -z "$ALL_STACKS" ] && exit 0 [ -z "$ALL_STACKS" ] && exit 0
sleep 5
# NOTE: `docker stack deploy` briefly tears down and recreates tasks in
# Swarm's internal bookkeeping, so `docker stack ps` can transiently
# return nothing right after deploy even when the service is healthy.
# A single `sleep 5` + one-shot check produced false-alarm-looking
# "nothing found in stack" output on ordinary deploys (e.g. vaultwarden,
# 2026-08-25). Retry with backoff instead of a single fixed sleep, and
# only warn (don't fail the pipeline) if tasks never show up.
ATTEMPTS=6
DELAY=5
for STACK in $ALL_STACKS; do for STACK in $ALL_STACKS; do
echo "--- $STACK ---" echo "--- $STACK ---"
ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} \ i=1
"docker stack ps $STACK --filter desired-state=running \ while [ "$i" -le "$ATTEMPTS" ]; do
--format ' {{.Name}} {{.CurrentState}}'" OUTPUT=$(ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} \
"docker stack ps $STACK --filter desired-state=running \
--format ' {{.Name}} {{.CurrentState}}'" 2>/dev/null)
if [ -n "$OUTPUT" ]; then
echo "$OUTPUT"
break
fi
if [ "$i" -eq "$ATTEMPTS" ]; then
echo " WARNING: no running tasks found for $STACK after $((ATTEMPTS * DELAY))s."
echo " This may be transient Swarm settle time, or a real problem — check manually:"
echo " ssh root@${SWARM_MANAGER_IP} 'docker stack ps $STACK --no-trunc'"
else
sleep "$DELAY"
fi
i=$((i + 1))
done
done done
notify-success: notify-success: