forked from mtusi2026/mtusi2026lab6
179 lines
5.4 KiB
Python
179 lines
5.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Verify that a retried attachment job remains in the original upload trace."""
|
|
|
|
import argparse
|
|
import json
|
|
import subprocess
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
|
|
from attachment_memory import cleanup
|
|
from client import Client
|
|
from observability_grade4 import flatten_trace, trace_id_for_request, wait_for
|
|
|
|
|
|
def compose(*args, check=True):
|
|
return subprocess.run(
|
|
["docker", "compose", *args],
|
|
check=check,
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
|
|
|
|
def start_existing_container(service):
|
|
container_id = compose("ps", "-aq", service).stdout.strip()
|
|
if not container_id:
|
|
raise AssertionError(f"missing container for {service}")
|
|
subprocess.run(
|
|
["docker", "start", container_id],
|
|
check=True,
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
|
|
|
|
def sql_scalar(statement):
|
|
result = compose(
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"psql",
|
|
"-At",
|
|
"-U",
|
|
"messenger",
|
|
"-d",
|
|
"messenger",
|
|
"-c",
|
|
statement,
|
|
)
|
|
return result.stdout.strip()
|
|
|
|
|
|
def json_get(base_url, path, timeout=10):
|
|
with urllib.request.urlopen(base_url.rstrip("/") + path, timeout=timeout) as response:
|
|
return json.load(response)
|
|
|
|
|
|
def attempt_from_span(span):
|
|
for item in span.get("attributes", []):
|
|
if item.get("key") == "job.attempt":
|
|
return int(item.get("value", {}).get("intValue", 0))
|
|
return None
|
|
|
|
|
|
def complete_retry_trace(tempo_url, trace_id):
|
|
payload = json_get(tempo_url, f"/api/traces/{trace_id}")
|
|
services, spans = flatten_trace(payload)
|
|
consumers = [
|
|
span for _, span in spans if span.get("kind") == "SPAN_KIND_CONSUMER"
|
|
]
|
|
producers = [
|
|
span for _, span in spans if span.get("kind") == "SPAN_KIND_PRODUCER"
|
|
]
|
|
attempts = sorted(
|
|
attempt
|
|
for attempt in (attempt_from_span(span) for span in consumers)
|
|
if attempt is not None
|
|
)
|
|
if len(consumers) < 2 or len(producers) < 2 or len(set(attempts)) < 2:
|
|
return None
|
|
return services, spans, attempts, consumers, producers
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--base-url", default="http://localhost:8080")
|
|
parser.add_argument("--loki-url", default="http://localhost:3100")
|
|
parser.add_argument("--tempo-url", default="http://localhost:3200")
|
|
parser.add_argument("--timeout", type=int, default=120)
|
|
args = parser.parse_args()
|
|
|
|
client = Client(args.base_url, timeout=20)
|
|
alice, bob, eve, chat = client.bootstrap()
|
|
users = [alice["id"], bob["id"], eve["id"]]
|
|
minio_stopped = False
|
|
worker_stopped = False
|
|
request_id = str(uuid.uuid4())
|
|
try:
|
|
compose("stop", "worker")
|
|
worker_stopped = True
|
|
content = open("tests/fixtures/sample.png", "rb").read()
|
|
upload = client.call(
|
|
"POST",
|
|
f'/api/v1/chats/{chat["id"]}/attachments?filename=retry-trace.png',
|
|
202,
|
|
actor=alice,
|
|
raw=content,
|
|
headers={"Content-Type": "image/png", "X-Request-Id": request_id},
|
|
schema="Attachment",
|
|
)
|
|
compose("stop", "minio")
|
|
minio_stopped = True
|
|
start_existing_container("worker")
|
|
worker_stopped = False
|
|
attachment_id = upload["json"]["id"]
|
|
first_retry = wait_for(
|
|
"first failed attempt",
|
|
lambda: int(
|
|
sql_scalar(
|
|
"SELECT attempt FROM attachment_jobs "
|
|
f"WHERE attachment_id='{attachment_id}'"
|
|
)
|
|
or "0"
|
|
)
|
|
>= 2,
|
|
30,
|
|
interval=0.05,
|
|
)
|
|
assert first_retry
|
|
compose("up", "-d", "--wait", "minio")
|
|
minio_stopped = False
|
|
ready = client.wait_ready(alice, upload["json"], args.timeout)
|
|
trace_id = wait_for(
|
|
"upload trace id",
|
|
lambda: trace_id_for_request(args.loki_url, request_id),
|
|
args.timeout,
|
|
)
|
|
services, spans, attempts, consumers, producers = wait_for(
|
|
"retry spans in the upload trace",
|
|
lambda: complete_retry_trace(args.tempo_url, trace_id),
|
|
args.timeout,
|
|
)
|
|
assert any(span.get("name", "").startswith("s3.") for _, span in spans)
|
|
assert any(
|
|
span.get("name", "").startswith("postgresql.") for _, span in spans
|
|
)
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"status": "PASS",
|
|
"attachment_status": ready["status"],
|
|
"trace_id": trace_id,
|
|
"services": sorted(services),
|
|
"attempts": attempts,
|
|
"consumer_spans": len(consumers),
|
|
"producer_spans": len(producers),
|
|
"span_count": len(spans),
|
|
},
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
)
|
|
)
|
|
finally:
|
|
if minio_stopped:
|
|
compose("up", "-d", "--wait", "minio", check=False)
|
|
if worker_stopped:
|
|
try:
|
|
start_existing_container("worker")
|
|
except (AssertionError, subprocess.CalledProcessError):
|
|
compose("up", "-d", "worker", check=False)
|
|
cleanup(chat["id"], users)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|