"""Track 2 measurements. Writes evidence/slo-track2.json without tokens or message text.""" import json import sys import time import urllib.parse import urllib.request from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client # noqa: E402 FAULTS = ROOT / "faults" OUT = ROOT / "evidence" / "slo-track2.json" def get(url): with urllib.request.urlopen(url, timeout=10) as response: return json.load(response) def prom(expr): return get("http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(expr))["data"]["result"] def alert_states(): found = {} for item in prom('ALERTS{alertname=~"Api.*|QueueLagHigh|WorkerConsumersZero|AttachmentReadyLatencyHigh|DatabaseUnready"}'): name = item["metric"].get("alertname") found[name] = item["metric"].get("alertstate") return found def write_fault(name, value): path = FAULTS / name if value is None: path.unlink(missing_ok=True) else: path.write_text(str(value), encoding="utf-8") def clear_faults(): for name in ("http_error_ratio", "db_slow_ms", "db_fail", "handler_slow_ms", "process_fail_remaining"): write_fault(name, None) def drive(client, actor, seconds, rps): interval = 1.0 / rps deadline = time.monotonic() + seconds counts = {} while time.monotonic() < deadline: started = time.monotonic() try: result = client.call("GET", "/api/v1/chats", [200, 500], actor=actor) status = result["status"] except Exception: status = "error" counts[str(status)] = counts.get(str(status), 0) + 1 sleep = interval - (time.monotonic() - started) if sleep > 0: time.sleep(sleep) return counts def wait_state(name, wanted, timeout): started = time.monotonic() last = {} while time.monotonic() - started < timeout: last = alert_states() if last.get(name) == wanted or (wanted is None and name not in last): return round(time.monotonic() - started, 1), last time.sleep(5) return None, last def span_summary(trace_id): payload = get(f"http://127.0.0.1:3200/api/traces/{trace_id}") spans = [] def walk(node): if isinstance(node, dict): if "name" in node and "spanId" in node: start = node.get("startTimeUnixNano") end = node.get("endTimeUnixNano") duration = None try: duration = round((int(end) - int(start)) / 1e6, 1) except (TypeError, ValueError): duration = None spans.append( { "name": node.get("name"), "span": node.get("spanId"), "parent": node.get("parentSpanId") or "", "ms": duration, } ) for value in node.values(): walk(value) elif isinstance(node, list): for item in node: walk(item) walk(payload) return spans def latest_worker_trace(event): query = '{service_name="messenger-worker"} |= "' + event + '"' payload = get( "http://127.0.0.1:3100/loki/api/v1/query_range?query=" + urllib.parse.quote(query) + "&limit=5" ) lines = [] for stream in payload["data"]["result"]: for _, line in stream["values"]: try: body = json.loads(line) except ValueError: continue if body.get("event") == event: lines.append(body) if not lines: return None body = lines[-1] return { "event": body.get("event"), "attempt": body.get("attempt"), "trace_id": body.get("trace_id"), "span_id": body.get("span_id"), "service": body.get("service.name"), } def main(): clear_faults() report = {"started": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())} client = Client("http://127.0.0.1:8080", timeout=8) alice, _bob, _eve, chat = client.bootstrap() print("bootstrap ok", flush=True) write_fault("process_fail_remaining", "1") attachment, _content = client.attachment(alice, chat) ready = client.wait_ready(alice, attachment, wait_seconds=40) time.sleep(3) log = latest_worker_trace("job.ready") spans = [] if log and log.get("trace_id"): try: spans = span_summary(log["trace_id"]) except Exception as exc: report["retry_trace_error"] = type(exc).__name__ report["retry_upload"] = { "final_status": ready["status"], "log": log, "consume_spans": sum(1 for span in spans if span["name"] == "attachment.process consume"), "span_names": [span["name"] for span in spans], "parents_linked": any(span["parent"] for span in spans), } print("retry", report["retry_upload"]["consume_spans"], report["retry_upload"]["log"], flush=True) print("warmup", flush=True) report["warmup"] = drive(client, alice, 90, 2) report["alerts_after_warmup"] = alert_states() write_fault("http_error_ratio", "1") print("short spike", flush=True) report["short_spike"] = drive(client, alice, 10, 2) time.sleep(35) report["alerts_after_short_spike"] = alert_states() clear_faults() report["recovery_after_spike"] = drive(client, alice, 20, 2) write_fault("http_error_ratio", "1") print("sustained", flush=True) sustained_started = time.monotonic() seen = None counts = {} deadline = time.monotonic() + 100 interval = 0.5 while time.monotonic() < deadline: started = time.monotonic() try: status = client.call("GET", "/api/v1/chats", [200, 500], actor=alice)["status"] except Exception: status = "error" counts[str(status)] = counts.get(str(status), 0) + 1 states = alert_states() if seen is None and states.get("ApiErrorBudgetFastBurn") == "firing": seen = round(time.monotonic() - sustained_started, 1) report["fast_burn_detected_after_seconds"] = seen report["alerts_when_fast_burn_fired"] = states break sleep = interval - (time.monotonic() - started) if sleep > 0: time.sleep(sleep) report["sustained_counts_until_detection_or_timeout"] = counts if seen is None: report["fast_burn_detected_after_seconds"] = None report["alerts_after_sustained"] = alert_states() clear_faults() print("resolve", flush=True) resolve_started = time.monotonic() resolved = None resolve_deadline = time.monotonic() + 180 while time.monotonic() < resolve_deadline: client.call("GET", "/api/v1/chats", [200, 500], actor=alice) if "ApiErrorBudgetFastBurn" not in alert_states(): resolved = round(time.monotonic() - resolve_started, 1) break time.sleep(5) report["fast_burn_resolved_after_seconds"] = resolved report["alerts_after_resolve"] = alert_states() print("worker stop", flush=True) import subprocess subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "5", "worker"], cwd=ROOT) client.attachment(alice, chat) consumer_wait, consumer_state = wait_state("WorkerConsumersZero", "firing", 150) queue_wait, queue_state = wait_state("QueueLagHigh", "firing", 150) report["worker_stopped"] = { "consumers_firing_after_seconds": consumer_wait, "queue_lag_firing_after_seconds": queue_wait, "alerts": queue_state or consumer_state, } subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "worker"], cwd=ROOT) restored, restored_state = wait_state("WorkerConsumersZero", None, 120) report["worker_restored"] = {"consumers_cleared_after_seconds": restored, "alerts": restored_state} print("latency split", flush=True) write_fault("db_slow_ms", "800") client.call("GET", "/api/v1/chats", [200, 500, 503], actor=alice) time.sleep(3) write_fault("db_slow_ms", None) write_fault("handler_slow_ms", "800") client.call("GET", "/api/v1/chats", [200, 500], actor=alice) time.sleep(3) write_fault("handler_slow_ms", None) report["latency_note"] = "db_slow_ms sleeps inside db.transaction; handler_slow_ms sleeps in the HTTP span and outside that span" print("tempo down", flush=True) before = prom("otelcol_exporter_send_failed_spans") subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "2", "tempo"], cwd=ROOT) tempo_counts = drive(client, alice, 8, 2) time.sleep(6) after = prom("otelcol_exporter_send_failed_spans") stats = subprocess.check_output( ["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", "mtusi-lab6-otel-collector-1"], text=True, ).strip() subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "tempo"], cwd=ROOT) report["tempo_stopped"] = { "api_results": tempo_counts, "failed_spans_before": [item["value"][1] for item in before], "failed_spans_after": [item["value"][1] for item in after], "collector_memory": stats, } clear_faults() report["finished"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") print("wrote", OUT, flush=True) if __name__ == "__main__": try: main() finally: clear_faults()