Files
mtusi2026lab6/tests/observability_retry_trace.py

179 lines
5.4 KiB
Python

#!/usr/bin/env python3
"""Verify that a retried attachment job remains in the original upload trace."""
import argparse
import json
import subprocess
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from attachment_memory import cleanup
from client import Client
from observability_grade4 import flatten_trace, trace_id_for_request, wait_for
def compose(*args, check=True):
return subprocess.run(
["docker", "compose", *args],
check=check,
text=True,
capture_output=True,
)
def start_existing_container(service):
container_id = compose("ps", "-aq", service).stdout.strip()
if not container_id:
raise AssertionError(f"missing container for {service}")
subprocess.run(
["docker", "start", container_id],
check=True,
text=True,
capture_output=True,
)
def sql_scalar(statement):
result = compose(
"exec",
"-T",
"postgres",
"psql",
"-At",
"-U",
"messenger",
"-d",
"messenger",
"-c",
statement,
)
return result.stdout.strip()
def json_get(base_url, path, timeout=10):
with urllib.request.urlopen(base_url.rstrip("/") + path, timeout=timeout) as response:
return json.load(response)
def attempt_from_span(span):
for item in span.get("attributes", []):
if item.get("key") == "job.attempt":
return int(item.get("value", {}).get("intValue", 0))
return None
def complete_retry_trace(tempo_url, trace_id):
payload = json_get(tempo_url, f"/api/traces/{trace_id}")
services, spans = flatten_trace(payload)
consumers = [
span for _, span in spans if span.get("kind") == "SPAN_KIND_CONSUMER"
]
producers = [
span for _, span in spans if span.get("kind") == "SPAN_KIND_PRODUCER"
]
attempts = sorted(
attempt
for attempt in (attempt_from_span(span) for span in consumers)
if attempt is not None
)
if len(consumers) < 2 or len(producers) < 2 or len(set(attempts)) < 2:
return None
return services, spans, attempts, consumers, producers
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--loki-url", default="http://localhost:3100")
parser.add_argument("--tempo-url", default="http://localhost:3200")
parser.add_argument("--timeout", type=int, default=120)
args = parser.parse_args()
client = Client(args.base_url, timeout=20)
alice, bob, eve, chat = client.bootstrap()
users = [alice["id"], bob["id"], eve["id"]]
minio_stopped = False
worker_stopped = False
request_id = str(uuid.uuid4())
try:
compose("stop", "worker")
worker_stopped = True
content = open("tests/fixtures/sample.png", "rb").read()
upload = client.call(
"POST",
f'/api/v1/chats/{chat["id"]}/attachments?filename=retry-trace.png',
202,
actor=alice,
raw=content,
headers={"Content-Type": "image/png", "X-Request-Id": request_id},
schema="Attachment",
)
compose("stop", "minio")
minio_stopped = True
start_existing_container("worker")
worker_stopped = False
attachment_id = upload["json"]["id"]
first_retry = wait_for(
"first failed attempt",
lambda: int(
sql_scalar(
"SELECT attempt FROM attachment_jobs "
f"WHERE attachment_id='{attachment_id}'"
)
or "0"
)
>= 2,
30,
interval=0.05,
)
assert first_retry
compose("up", "-d", "--wait", "minio")
minio_stopped = False
ready = client.wait_ready(alice, upload["json"], args.timeout)
trace_id = wait_for(
"upload trace id",
lambda: trace_id_for_request(args.loki_url, request_id),
args.timeout,
)
services, spans, attempts, consumers, producers = wait_for(
"retry spans in the upload trace",
lambda: complete_retry_trace(args.tempo_url, trace_id),
args.timeout,
)
assert any(span.get("name", "").startswith("s3.") for _, span in spans)
assert any(
span.get("name", "").startswith("postgresql.") for _, span in spans
)
print(
json.dumps(
{
"status": "PASS",
"attachment_status": ready["status"],
"trace_id": trace_id,
"services": sorted(services),
"attempts": attempts,
"consumer_spans": len(consumers),
"producer_spans": len(producers),
"span_count": len(spans),
},
ensure_ascii=False,
indent=2,
)
)
finally:
if minio_stopped:
compose("up", "-d", "--wait", "minio", check=False)
if worker_stopped:
try:
start_existing_container("worker")
except (AssertionError, subprocess.CalledProcessError):
compose("up", "-d", "worker", check=False)
cleanup(chat["id"], users)
if __name__ == "__main__":
main()