forked from mtusi2026/mtusi2026lab6
145 lines
5.2 KiB
Python
145 lines
5.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Создаёт временную S3-ошибку и доказывает retry в одной Tempo-трассе."""
|
|
import argparse
|
|
from datetime import datetime, timezone
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import time
|
|
import urllib.parse
|
|
|
|
from client import Client
|
|
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
|
|
def compose(*args: str, capture: bool = False):
|
|
"""Запускает Docker Compose и при необходимости возвращает его вывод."""
|
|
|
|
return subprocess.run(
|
|
["docker", "compose", *args], cwd=ROOT, check=True, text=True,
|
|
capture_output=capture,
|
|
)
|
|
|
|
|
|
def internal_json(url: str) -> dict:
|
|
"""Читает Loki/Tempo JSON через внутреннюю сеть Compose."""
|
|
|
|
result = compose(
|
|
"exec", "-T", "grafana", "wget", "-qO-", url, capture=True
|
|
)
|
|
return json.loads(result.stdout)
|
|
|
|
|
|
def wait_retry_log(since: str, timeout: int = 45) -> None:
|
|
"""Ждёт запись worker о планировании повторной попытки."""
|
|
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
result = compose("logs", "--since", since, "worker", capture=True)
|
|
if "job.retry_scheduled" in result.stdout + result.stderr:
|
|
return
|
|
time.sleep(1)
|
|
raise RuntimeError("worker did not schedule retry")
|
|
|
|
|
|
def trace_id_for_attachment(attachment_id: str) -> str:
|
|
"""Находит trace_id лога приёма конкретного синтетического вложения."""
|
|
|
|
deadline = time.monotonic() + 45
|
|
while time.monotonic() < deadline:
|
|
now_ns = time.time_ns()
|
|
query = urllib.parse.urlencode(
|
|
{
|
|
"query": (
|
|
'{service_name="messenger-api"} | json '
|
|
f'| event="attachment.accepted" | attachment_id="{attachment_id}"'
|
|
),
|
|
"start": str(now_ns - 600_000_000_000),
|
|
"end": str(now_ns),
|
|
"direction": "backward",
|
|
"limit": "5",
|
|
}
|
|
)
|
|
payload = internal_json(
|
|
"http://loki:3100/loki/api/v1/query_range?" + query
|
|
)
|
|
streams = payload.get("data", {}).get("result", [])
|
|
if streams:
|
|
entry = streams[0]["values"][0]
|
|
metadata = entry[2] if len(entry) > 2 else {}
|
|
trace_id = metadata.get("trace_id")
|
|
if not trace_id:
|
|
trace_id = json.loads(entry[1]).get("trace_id")
|
|
if trace_id:
|
|
return trace_id
|
|
time.sleep(2)
|
|
raise RuntimeError("attachment trace_id is absent in Loki")
|
|
|
|
|
|
def trace_span_names(trace_id: str) -> list[str]:
|
|
"""Ждёт полную трассу и возвращает имена всех её spans."""
|
|
|
|
deadline = time.monotonic() + 60
|
|
last: list[str] = []
|
|
while time.monotonic() < deadline:
|
|
payload = internal_json(f"http://tempo:3200/api/traces/{trace_id}")
|
|
names: list[str] = []
|
|
for batch in payload.get("batches", []):
|
|
scopes = batch.get(
|
|
"scopeSpans", batch.get("instrumentationLibrarySpans", [])
|
|
)
|
|
for scope in scopes:
|
|
names.extend(span.get("name", "") for span in scope.get("spans", []))
|
|
last = names
|
|
if names.count("attachments publish") >= 2 and names.count("attachments process") >= 2:
|
|
return names
|
|
time.sleep(2)
|
|
raise RuntimeError(f"retry spans are incomplete: {last}")
|
|
|
|
|
|
def main():
|
|
"""Внедряет временный отказ S3, восстанавливает его и проверяет retry trace."""
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--base-url", default="http://localhost:8080")
|
|
args = parser.parse_args()
|
|
client = Client(args.base_url, timeout=10)
|
|
alice, bob, _, chat = client.bootstrap()
|
|
compose("stop", "-t", "1", "worker")
|
|
attachment = None
|
|
retry_seen = False
|
|
try:
|
|
attachment, _ = client.attachment(alice, chat)
|
|
since = datetime.now(timezone.utc).isoformat()
|
|
compose("stop", "s3")
|
|
compose("up", "-d", "--no-deps", "worker")
|
|
wait_retry_log(since)
|
|
retry_seen = True
|
|
finally:
|
|
compose("start", "s3")
|
|
compose("up", "-d", "--no-deps", "worker")
|
|
if attachment is None or not retry_seen:
|
|
raise RuntimeError("retry scenario was not created")
|
|
time.sleep(5)
|
|
ready = client.wait_ready(bob, attachment, wait_seconds=120)
|
|
trace_id = trace_id_for_attachment(ready["id"])
|
|
names = trace_span_names(trace_id)
|
|
if not any(name.startswith("s3 ") for name in names):
|
|
raise RuntimeError("S3 spans are absent from retry trace")
|
|
if not any(name.startswith(("SELECT ", "INSERT ", "UPDATE ", "connect")) for name in names):
|
|
raise RuntimeError("SQL spans are absent from retry trace")
|
|
print(json.dumps({
|
|
"scenario": "retry_trace",
|
|
"attachment_id": ready["id"],
|
|
"trace_id": trace_id,
|
|
"producer_spans": names.count("attachments publish"),
|
|
"consumer_spans": names.count("attachments process"),
|
|
"result": "PASS",
|
|
}))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|