verify: fix race condition causing false-alarm "nothing found in stack" #5
+28
-4
@@ -355,12 +355,36 @@ steps:
|
|||||||
FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true)
|
FOLDER_STACKS=$(echo "$CHANGED_FILES" | grep -E '^[^/.][^/]*/' | cut -d/ -f1 | sort -u || true)
|
||||||
ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
|
ALL_STACKS=$(printf '%s\n%s' "$FLAT_STACKS" "$FOLDER_STACKS" | grep -v '^$' | sort -u)
|
||||||
[ -z "$ALL_STACKS" ] && exit 0
|
[ -z "$ALL_STACKS" ] && exit 0
|
||||||
sleep 5
|
|
||||||
|
# NOTE: `docker stack deploy` briefly tears down and recreates tasks in
|
||||||
|
# Swarm's internal bookkeeping, so `docker stack ps` can transiently
|
||||||
|
# return nothing right after deploy even when the service is healthy.
|
||||||
|
# A single `sleep 5` + one-shot check produced false-alarm-looking
|
||||||
|
# "nothing found in stack" output on ordinary deploys (e.g. vaultwarden,
|
||||||
|
# 2026-08-25). Retry with backoff instead of a single fixed sleep, and
|
||||||
|
# only warn (don't fail the pipeline) if tasks never show up.
|
||||||
|
ATTEMPTS=6
|
||||||
|
DELAY=5
|
||||||
for STACK in $ALL_STACKS; do
|
for STACK in $ALL_STACKS; do
|
||||||
echo "--- $STACK ---"
|
echo "--- $STACK ---"
|
||||||
ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} \
|
i=1
|
||||||
"docker stack ps $STACK --filter desired-state=running \
|
while [ "$i" -le "$ATTEMPTS" ]; do
|
||||||
--format ' {{.Name}} {{.CurrentState}}'"
|
OUTPUT=$(ssh -o StrictHostKeyChecking=no root@${SWARM_MANAGER_IP} \
|
||||||
|
"docker stack ps $STACK --filter desired-state=running \
|
||||||
|
--format ' {{.Name}} {{.CurrentState}}'" 2>/dev/null)
|
||||||
|
if [ -n "$OUTPUT" ]; then
|
||||||
|
echo "$OUTPUT"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
if [ "$i" -eq "$ATTEMPTS" ]; then
|
||||||
|
echo " WARNING: no running tasks found for $STACK after $((ATTEMPTS * DELAY))s."
|
||||||
|
echo " This may be transient Swarm settle time, or a real problem — check manually:"
|
||||||
|
echo " ssh root@${SWARM_MANAGER_IP} 'docker stack ps $STACK --no-trunc'"
|
||||||
|
else
|
||||||
|
sleep "$DELAY"
|
||||||
|
fi
|
||||||
|
i=$((i + 1))
|
||||||
|
done
|
||||||
done
|
done
|
||||||
|
|
||||||
notify-success:
|
notify-success:
|
||||||
|
|||||||
Reference in New Issue
Block a user