Files
mtusi2026lab6/tests/retry_trace.py

145 lines
5.2 KiB
Python

#!/usr/bin/env python3
"""Создаёт временную S3-ошибку и доказывает retry в одной Tempo-трассе."""
import argparse
from datetime import datetime, timezone
import json
import os
import subprocess
import time
import urllib.parse
from client import Client
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
def compose(*args: str, capture: bool = False):
"""Запускает Docker Compose и при необходимости возвращает его вывод."""
return subprocess.run(
["docker", "compose", *args], cwd=ROOT, check=True, text=True,
capture_output=capture,
)
def internal_json(url: str) -> dict:
"""Читает Loki/Tempo JSON через внутреннюю сеть Compose."""
result = compose(
"exec", "-T", "grafana", "wget", "-qO-", url, capture=True
)
return json.loads(result.stdout)
def wait_retry_log(since: str, timeout: int = 45) -> None:
"""Ждёт запись worker о планировании повторной попытки."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
result = compose("logs", "--since", since, "worker", capture=True)
if "job.retry_scheduled" in result.stdout + result.stderr:
return
time.sleep(1)
raise RuntimeError("worker did not schedule retry")
def trace_id_for_attachment(attachment_id: str) -> str:
"""Находит trace_id лога приёма конкретного синтетического вложения."""
deadline = time.monotonic() + 45
while time.monotonic() < deadline:
now_ns = time.time_ns()
query = urllib.parse.urlencode(
{
"query": (
'{service_name="messenger-api"} | json '
f'| event="attachment.accepted" | attachment_id="{attachment_id}"'
),
"start": str(now_ns - 600_000_000_000),
"end": str(now_ns),
"direction": "backward",
"limit": "5",
}
)
payload = internal_json(
"http://loki:3100/loki/api/v1/query_range?" + query
)
streams = payload.get("data", {}).get("result", [])
if streams:
entry = streams[0]["values"][0]
metadata = entry[2] if len(entry) > 2 else {}
trace_id = metadata.get("trace_id")
if not trace_id:
trace_id = json.loads(entry[1]).get("trace_id")
if trace_id:
return trace_id
time.sleep(2)
raise RuntimeError("attachment trace_id is absent in Loki")
def trace_span_names(trace_id: str) -> list[str]:
"""Ждёт полную трассу и возвращает имена всех её spans."""
deadline = time.monotonic() + 60
last: list[str] = []
while time.monotonic() < deadline:
payload = internal_json(f"http://tempo:3200/api/traces/{trace_id}")
names: list[str] = []
for batch in payload.get("batches", []):
scopes = batch.get(
"scopeSpans", batch.get("instrumentationLibrarySpans", [])
)
for scope in scopes:
names.extend(span.get("name", "") for span in scope.get("spans", []))
last = names
if names.count("attachments publish") >= 2 and names.count("attachments process") >= 2:
return names
time.sleep(2)
raise RuntimeError(f"retry spans are incomplete: {last}")
def main():
"""Внедряет временный отказ S3, восстанавливает его и проверяет retry trace."""
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://localhost:8080")
args = parser.parse_args()
client = Client(args.base_url, timeout=10)
alice, bob, _, chat = client.bootstrap()
compose("stop", "-t", "1", "worker")
attachment = None
retry_seen = False
try:
attachment, _ = client.attachment(alice, chat)
since = datetime.now(timezone.utc).isoformat()
compose("stop", "s3")
compose("up", "-d", "--no-deps", "worker")
wait_retry_log(since)
retry_seen = True
finally:
compose("start", "s3")
compose("up", "-d", "--no-deps", "worker")
if attachment is None or not retry_seen:
raise RuntimeError("retry scenario was not created")
time.sleep(5)
ready = client.wait_ready(bob, attachment, wait_seconds=120)
trace_id = trace_id_for_attachment(ready["id"])
names = trace_span_names(trace_id)
if not any(name.startswith("s3 ") for name in names):
raise RuntimeError("S3 spans are absent from retry trace")
if not any(name.startswith(("SELECT ", "INSERT ", "UPDATE ", "connect")) for name in names):
raise RuntimeError("SQL spans are absent from retry trace")
print(json.dumps({
"scenario": "retry_trace",
"attachment_id": ready["id"],
"trace_id": trace_id,
"producer_spans": names.count("attachments publish"),
"consumer_spans": names.count("attachments process"),
"result": "PASS",
}))
if __name__ == "__main__":
main()