271 lines
9.5 KiB
Python
271 lines
9.5 KiB
Python
"""Track 2 measurements. Writes evidence/slo-track2.json without tokens or message text."""
|
|
|
|
import json
|
|
import sys
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client # noqa: E402
|
|
|
|
FAULTS = ROOT / "faults"
|
|
OUT = ROOT / "evidence" / "slo-track2.json"
|
|
|
|
|
|
def get(url):
|
|
with urllib.request.urlopen(url, timeout=10) as response:
|
|
return json.load(response)
|
|
|
|
|
|
def prom(expr):
|
|
return get("http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(expr))["data"]["result"]
|
|
|
|
|
|
def alert_states():
|
|
found = {}
|
|
for item in prom('ALERTS{alertname=~"Api.*|QueueLagHigh|WorkerConsumersZero|AttachmentReadyLatencyHigh|DatabaseUnready"}'):
|
|
name = item["metric"].get("alertname")
|
|
found[name] = item["metric"].get("alertstate")
|
|
return found
|
|
|
|
|
|
def write_fault(name, value):
|
|
path = FAULTS / name
|
|
if value is None:
|
|
path.unlink(missing_ok=True)
|
|
else:
|
|
path.write_text(str(value), encoding="utf-8")
|
|
|
|
|
|
def clear_faults():
|
|
for name in ("http_error_ratio", "db_slow_ms", "db_fail", "handler_slow_ms", "process_fail_remaining"):
|
|
write_fault(name, None)
|
|
|
|
|
|
def drive(client, actor, seconds, rps):
|
|
interval = 1.0 / rps
|
|
deadline = time.monotonic() + seconds
|
|
counts = {}
|
|
while time.monotonic() < deadline:
|
|
started = time.monotonic()
|
|
try:
|
|
result = client.call("GET", "/api/v1/chats", [200, 500], actor=actor)
|
|
status = result["status"]
|
|
except Exception:
|
|
status = "error"
|
|
counts[str(status)] = counts.get(str(status), 0) + 1
|
|
sleep = interval - (time.monotonic() - started)
|
|
if sleep > 0:
|
|
time.sleep(sleep)
|
|
return counts
|
|
|
|
|
|
def wait_state(name, wanted, timeout):
|
|
started = time.monotonic()
|
|
last = {}
|
|
while time.monotonic() - started < timeout:
|
|
last = alert_states()
|
|
if last.get(name) == wanted or (wanted is None and name not in last):
|
|
return round(time.monotonic() - started, 1), last
|
|
time.sleep(5)
|
|
return None, last
|
|
|
|
|
|
def span_summary(trace_id):
|
|
payload = get(f"http://127.0.0.1:3200/api/traces/{trace_id}")
|
|
spans = []
|
|
|
|
def walk(node):
|
|
if isinstance(node, dict):
|
|
if "name" in node and "spanId" in node:
|
|
start = node.get("startTimeUnixNano")
|
|
end = node.get("endTimeUnixNano")
|
|
duration = None
|
|
try:
|
|
duration = round((int(end) - int(start)) / 1e6, 1)
|
|
except (TypeError, ValueError):
|
|
duration = None
|
|
spans.append(
|
|
{
|
|
"name": node.get("name"),
|
|
"span": node.get("spanId"),
|
|
"parent": node.get("parentSpanId") or "",
|
|
"ms": duration,
|
|
}
|
|
)
|
|
for value in node.values():
|
|
walk(value)
|
|
elif isinstance(node, list):
|
|
for item in node:
|
|
walk(item)
|
|
|
|
walk(payload)
|
|
return spans
|
|
|
|
|
|
def latest_worker_trace(event):
|
|
query = '{service_name="messenger-worker"} |= "' + event + '"'
|
|
payload = get(
|
|
"http://127.0.0.1:3100/loki/api/v1/query_range?query="
|
|
+ urllib.parse.quote(query)
|
|
+ "&limit=5"
|
|
)
|
|
lines = []
|
|
for stream in payload["data"]["result"]:
|
|
for _, line in stream["values"]:
|
|
try:
|
|
body = json.loads(line)
|
|
except ValueError:
|
|
continue
|
|
if body.get("event") == event:
|
|
lines.append(body)
|
|
if not lines:
|
|
return None
|
|
body = lines[-1]
|
|
return {
|
|
"event": body.get("event"),
|
|
"attempt": body.get("attempt"),
|
|
"trace_id": body.get("trace_id"),
|
|
"span_id": body.get("span_id"),
|
|
"service": body.get("service.name"),
|
|
}
|
|
|
|
|
|
def main():
|
|
clear_faults()
|
|
report = {"started": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}
|
|
client = Client("http://127.0.0.1:8080", timeout=8)
|
|
alice, _bob, _eve, chat = client.bootstrap()
|
|
print("bootstrap ok", flush=True)
|
|
|
|
write_fault("process_fail_remaining", "1")
|
|
attachment, _content = client.attachment(alice, chat)
|
|
ready = client.wait_ready(alice, attachment, wait_seconds=40)
|
|
time.sleep(3)
|
|
log = latest_worker_trace("job.ready")
|
|
spans = []
|
|
if log and log.get("trace_id"):
|
|
try:
|
|
spans = span_summary(log["trace_id"])
|
|
except Exception as exc:
|
|
report["retry_trace_error"] = type(exc).__name__
|
|
report["retry_upload"] = {
|
|
"final_status": ready["status"],
|
|
"log": log,
|
|
"consume_spans": sum(1 for span in spans if span["name"] == "attachment.process consume"),
|
|
"span_names": [span["name"] for span in spans],
|
|
"parents_linked": any(span["parent"] for span in spans),
|
|
}
|
|
print("retry", report["retry_upload"]["consume_spans"], report["retry_upload"]["log"], flush=True)
|
|
|
|
print("warmup", flush=True)
|
|
report["warmup"] = drive(client, alice, 90, 2)
|
|
report["alerts_after_warmup"] = alert_states()
|
|
write_fault("http_error_ratio", "1")
|
|
print("short spike", flush=True)
|
|
report["short_spike"] = drive(client, alice, 10, 2)
|
|
time.sleep(35)
|
|
report["alerts_after_short_spike"] = alert_states()
|
|
clear_faults()
|
|
report["recovery_after_spike"] = drive(client, alice, 20, 2)
|
|
|
|
write_fault("http_error_ratio", "1")
|
|
print("sustained", flush=True)
|
|
sustained_started = time.monotonic()
|
|
seen = None
|
|
counts = {}
|
|
deadline = time.monotonic() + 100
|
|
interval = 0.5
|
|
while time.monotonic() < deadline:
|
|
started = time.monotonic()
|
|
try:
|
|
status = client.call("GET", "/api/v1/chats", [200, 500], actor=alice)["status"]
|
|
except Exception:
|
|
status = "error"
|
|
counts[str(status)] = counts.get(str(status), 0) + 1
|
|
states = alert_states()
|
|
if seen is None and states.get("ApiErrorBudgetFastBurn") == "firing":
|
|
seen = round(time.monotonic() - sustained_started, 1)
|
|
report["fast_burn_detected_after_seconds"] = seen
|
|
report["alerts_when_fast_burn_fired"] = states
|
|
break
|
|
sleep = interval - (time.monotonic() - started)
|
|
if sleep > 0:
|
|
time.sleep(sleep)
|
|
report["sustained_counts_until_detection_or_timeout"] = counts
|
|
if seen is None:
|
|
report["fast_burn_detected_after_seconds"] = None
|
|
report["alerts_after_sustained"] = alert_states()
|
|
clear_faults()
|
|
print("resolve", flush=True)
|
|
resolve_started = time.monotonic()
|
|
resolved = None
|
|
resolve_deadline = time.monotonic() + 180
|
|
while time.monotonic() < resolve_deadline:
|
|
client.call("GET", "/api/v1/chats", [200, 500], actor=alice)
|
|
if "ApiErrorBudgetFastBurn" not in alert_states():
|
|
resolved = round(time.monotonic() - resolve_started, 1)
|
|
break
|
|
time.sleep(5)
|
|
report["fast_burn_resolved_after_seconds"] = resolved
|
|
report["alerts_after_resolve"] = alert_states()
|
|
|
|
print("worker stop", flush=True)
|
|
import subprocess
|
|
|
|
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "5", "worker"], cwd=ROOT)
|
|
client.attachment(alice, chat)
|
|
consumer_wait, consumer_state = wait_state("WorkerConsumersZero", "firing", 150)
|
|
queue_wait, queue_state = wait_state("QueueLagHigh", "firing", 150)
|
|
report["worker_stopped"] = {
|
|
"consumers_firing_after_seconds": consumer_wait,
|
|
"queue_lag_firing_after_seconds": queue_wait,
|
|
"alerts": queue_state or consumer_state,
|
|
}
|
|
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "worker"], cwd=ROOT)
|
|
restored, restored_state = wait_state("WorkerConsumersZero", None, 120)
|
|
report["worker_restored"] = {"consumers_cleared_after_seconds": restored, "alerts": restored_state}
|
|
|
|
print("latency split", flush=True)
|
|
write_fault("db_slow_ms", "800")
|
|
client.call("GET", "/api/v1/chats", [200, 500, 503], actor=alice)
|
|
time.sleep(3)
|
|
write_fault("db_slow_ms", None)
|
|
write_fault("handler_slow_ms", "800")
|
|
client.call("GET", "/api/v1/chats", [200, 500], actor=alice)
|
|
time.sleep(3)
|
|
write_fault("handler_slow_ms", None)
|
|
report["latency_note"] = "db_slow_ms sleeps inside db.transaction; handler_slow_ms sleeps in the HTTP span and outside that span"
|
|
|
|
print("tempo down", flush=True)
|
|
before = prom("otelcol_exporter_send_failed_spans")
|
|
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "2", "tempo"], cwd=ROOT)
|
|
tempo_counts = drive(client, alice, 8, 2)
|
|
time.sleep(6)
|
|
after = prom("otelcol_exporter_send_failed_spans")
|
|
stats = subprocess.check_output(
|
|
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", "mtusi-lab6-otel-collector-1"],
|
|
text=True,
|
|
).strip()
|
|
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "tempo"], cwd=ROOT)
|
|
report["tempo_stopped"] = {
|
|
"api_results": tempo_counts,
|
|
"failed_spans_before": [item["value"][1] for item in before],
|
|
"failed_spans_after": [item["value"][1] for item in after],
|
|
"collector_memory": stats,
|
|
}
|
|
clear_faults()
|
|
report["finished"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
print("wrote", OUT, flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
finally:
|
|
clear_faults()
|