#!/bin/sh # Exercise a real retry on an explicitly owned disposable stack. set -eu : "${LOCAL_TEST_WORK:?}" "${LOCAL_TEST_S3:?}" "${BASE_URL:?}" "${TEMPO_URL:?}" export PROBE_WORK="$LOCAL_TEST_WORK/retry-probe" scripts/prepare-probe.sh worker=$(cat "$LOCAL_TEST_WORK/worker.pid") paused=false stopped=false cleanup() { if $paused; then docker unpause "$LOCAL_TEST_S3" >/dev/null; fi if $stopped; then kill -CONT "$worker"; fi } trap cleanup EXIT kill -STOP "$worker" stopped=true trace=$(openssl rand -hex 16) parent=$(openssl rand -hex 8) chat=$(cat "$PROBE_WORK/chat-id") curl -fsS --config "$PROBE_WORK/auth.conf" "$BASE_URL/api/v1/chats/$chat/attachments?filename=retry.png" -H 'Content-Type: image/png' -H "traceparent: 00-$trace-$parent-01" --data-binary @tests/fixtures/sample.png > "$PROBE_WORK/attachment.json" id=$(jq -r .id "$PROBE_WORK/attachment.json") docker pause "$LOCAL_TEST_S3" >/dev/null paused=true kill -CONT "$worker" stopped=false # S3 reads have a 15-second deadline; leave one attempt enough time to fail. sleep 20 docker unpause "$LOCAL_TEST_S3" >/dev/null paused=false ready=false for attempt in $(seq 1 90); do curl -fsS --config "$PROBE_WORK/auth.conf" "$BASE_URL/api/v1/attachments/$id" > "$PROBE_WORK/status.json" if jq -e '.status=="ready"' "$PROBE_WORK/status.json" >/dev/null; then ready=true; break; fi sleep 1 done $ready found=false for attempt in $(seq 1 30); do curl -fsS -H 'Accept: application/json' "$TEMPO_URL/api/traces/$trace" > evidence/current/retry-trace.json || true if jq -e '[..|objects|select(.name?=="attachment.process")]|length>=2' evidence/current/retry-trace.json >/dev/null 2>&1; then found=true; break; fi sleep 1 done $found jq -e '[..|objects|select(.name?=="attachment.process" and (.status.code?==2 or .status.code?=="STATUS_CODE_ERROR"))]|length>0' evidence/current/retry-trace.json >/dev/null jq -n --arg trace "$trace" '{trace_id:$trace,ready:true,failed_attempt_and_retry:true}' > evidence/current/retry-result.json printf 'PASS: S3 outage caused a failed consumer attempt; retry reached ready in the same trace\n'