forked from mtusi2026/mtusi2026lab6
Собрать наблюдаемый стенд, чтобы причину деградации API и worker было видно по метрикам, логам и трассам.
This commit is contained in:
@@ -0,0 +1,270 @@
|
||||
"""Track 2 measurements. Writes evidence/slo-track2.json without tokens or message text."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client # noqa: E402
|
||||
|
||||
FAULTS = ROOT / "faults"
|
||||
OUT = ROOT / "evidence" / "slo-track2.json"
|
||||
|
||||
|
||||
def get(url):
|
||||
with urllib.request.urlopen(url, timeout=10) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def prom(expr):
|
||||
return get("http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(expr))["data"]["result"]
|
||||
|
||||
|
||||
def alert_states():
|
||||
found = {}
|
||||
for item in prom('ALERTS{alertname=~"Api.*|QueueLagHigh|WorkerConsumersZero|AttachmentReadyLatencyHigh|DatabaseUnready"}'):
|
||||
name = item["metric"].get("alertname")
|
||||
found[name] = item["metric"].get("alertstate")
|
||||
return found
|
||||
|
||||
|
||||
def write_fault(name, value):
|
||||
path = FAULTS / name
|
||||
if value is None:
|
||||
path.unlink(missing_ok=True)
|
||||
else:
|
||||
path.write_text(str(value), encoding="utf-8")
|
||||
|
||||
|
||||
def clear_faults():
|
||||
for name in ("http_error_ratio", "db_slow_ms", "db_fail", "handler_slow_ms", "process_fail_remaining"):
|
||||
write_fault(name, None)
|
||||
|
||||
|
||||
def drive(client, actor, seconds, rps):
|
||||
interval = 1.0 / rps
|
||||
deadline = time.monotonic() + seconds
|
||||
counts = {}
|
||||
while time.monotonic() < deadline:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
result = client.call("GET", "/api/v1/chats", [200, 500], actor=actor)
|
||||
status = result["status"]
|
||||
except Exception:
|
||||
status = "error"
|
||||
counts[str(status)] = counts.get(str(status), 0) + 1
|
||||
sleep = interval - (time.monotonic() - started)
|
||||
if sleep > 0:
|
||||
time.sleep(sleep)
|
||||
return counts
|
||||
|
||||
|
||||
def wait_state(name, wanted, timeout):
|
||||
started = time.monotonic()
|
||||
last = {}
|
||||
while time.monotonic() - started < timeout:
|
||||
last = alert_states()
|
||||
if last.get(name) == wanted or (wanted is None and name not in last):
|
||||
return round(time.monotonic() - started, 1), last
|
||||
time.sleep(5)
|
||||
return None, last
|
||||
|
||||
|
||||
def span_summary(trace_id):
|
||||
payload = get(f"http://127.0.0.1:3200/api/traces/{trace_id}")
|
||||
spans = []
|
||||
|
||||
def walk(node):
|
||||
if isinstance(node, dict):
|
||||
if "name" in node and "spanId" in node:
|
||||
start = node.get("startTimeUnixNano")
|
||||
end = node.get("endTimeUnixNano")
|
||||
duration = None
|
||||
try:
|
||||
duration = round((int(end) - int(start)) / 1e6, 1)
|
||||
except (TypeError, ValueError):
|
||||
duration = None
|
||||
spans.append(
|
||||
{
|
||||
"name": node.get("name"),
|
||||
"span": node.get("spanId"),
|
||||
"parent": node.get("parentSpanId") or "",
|
||||
"ms": duration,
|
||||
}
|
||||
)
|
||||
for value in node.values():
|
||||
walk(value)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
walk(item)
|
||||
|
||||
walk(payload)
|
||||
return spans
|
||||
|
||||
|
||||
def latest_worker_trace(event):
|
||||
query = '{service_name="messenger-worker"} |= "' + event + '"'
|
||||
payload = get(
|
||||
"http://127.0.0.1:3100/loki/api/v1/query_range?query="
|
||||
+ urllib.parse.quote(query)
|
||||
+ "&limit=5"
|
||||
)
|
||||
lines = []
|
||||
for stream in payload["data"]["result"]:
|
||||
for _, line in stream["values"]:
|
||||
try:
|
||||
body = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if body.get("event") == event:
|
||||
lines.append(body)
|
||||
if not lines:
|
||||
return None
|
||||
body = lines[-1]
|
||||
return {
|
||||
"event": body.get("event"),
|
||||
"attempt": body.get("attempt"),
|
||||
"trace_id": body.get("trace_id"),
|
||||
"span_id": body.get("span_id"),
|
||||
"service": body.get("service.name"),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
clear_faults()
|
||||
report = {"started": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}
|
||||
client = Client("http://127.0.0.1:8080", timeout=8)
|
||||
alice, _bob, _eve, chat = client.bootstrap()
|
||||
print("bootstrap ok", flush=True)
|
||||
|
||||
write_fault("process_fail_remaining", "1")
|
||||
attachment, _content = client.attachment(alice, chat)
|
||||
ready = client.wait_ready(alice, attachment, wait_seconds=40)
|
||||
time.sleep(3)
|
||||
log = latest_worker_trace("job.ready")
|
||||
spans = []
|
||||
if log and log.get("trace_id"):
|
||||
try:
|
||||
spans = span_summary(log["trace_id"])
|
||||
except Exception as exc:
|
||||
report["retry_trace_error"] = type(exc).__name__
|
||||
report["retry_upload"] = {
|
||||
"final_status": ready["status"],
|
||||
"log": log,
|
||||
"consume_spans": sum(1 for span in spans if span["name"] == "attachment.process consume"),
|
||||
"span_names": [span["name"] for span in spans],
|
||||
"parents_linked": any(span["parent"] for span in spans),
|
||||
}
|
||||
print("retry", report["retry_upload"]["consume_spans"], report["retry_upload"]["log"], flush=True)
|
||||
|
||||
print("warmup", flush=True)
|
||||
report["warmup"] = drive(client, alice, 90, 2)
|
||||
report["alerts_after_warmup"] = alert_states()
|
||||
write_fault("http_error_ratio", "1")
|
||||
print("short spike", flush=True)
|
||||
report["short_spike"] = drive(client, alice, 10, 2)
|
||||
time.sleep(35)
|
||||
report["alerts_after_short_spike"] = alert_states()
|
||||
clear_faults()
|
||||
report["recovery_after_spike"] = drive(client, alice, 20, 2)
|
||||
|
||||
write_fault("http_error_ratio", "1")
|
||||
print("sustained", flush=True)
|
||||
sustained_started = time.monotonic()
|
||||
seen = None
|
||||
counts = {}
|
||||
deadline = time.monotonic() + 100
|
||||
interval = 0.5
|
||||
while time.monotonic() < deadline:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
status = client.call("GET", "/api/v1/chats", [200, 500], actor=alice)["status"]
|
||||
except Exception:
|
||||
status = "error"
|
||||
counts[str(status)] = counts.get(str(status), 0) + 1
|
||||
states = alert_states()
|
||||
if seen is None and states.get("ApiErrorBudgetFastBurn") == "firing":
|
||||
seen = round(time.monotonic() - sustained_started, 1)
|
||||
report["fast_burn_detected_after_seconds"] = seen
|
||||
report["alerts_when_fast_burn_fired"] = states
|
||||
break
|
||||
sleep = interval - (time.monotonic() - started)
|
||||
if sleep > 0:
|
||||
time.sleep(sleep)
|
||||
report["sustained_counts_until_detection_or_timeout"] = counts
|
||||
if seen is None:
|
||||
report["fast_burn_detected_after_seconds"] = None
|
||||
report["alerts_after_sustained"] = alert_states()
|
||||
clear_faults()
|
||||
print("resolve", flush=True)
|
||||
resolve_started = time.monotonic()
|
||||
resolved = None
|
||||
resolve_deadline = time.monotonic() + 180
|
||||
while time.monotonic() < resolve_deadline:
|
||||
client.call("GET", "/api/v1/chats", [200, 500], actor=alice)
|
||||
if "ApiErrorBudgetFastBurn" not in alert_states():
|
||||
resolved = round(time.monotonic() - resolve_started, 1)
|
||||
break
|
||||
time.sleep(5)
|
||||
report["fast_burn_resolved_after_seconds"] = resolved
|
||||
report["alerts_after_resolve"] = alert_states()
|
||||
|
||||
print("worker stop", flush=True)
|
||||
import subprocess
|
||||
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "5", "worker"], cwd=ROOT)
|
||||
client.attachment(alice, chat)
|
||||
consumer_wait, consumer_state = wait_state("WorkerConsumersZero", "firing", 150)
|
||||
queue_wait, queue_state = wait_state("QueueLagHigh", "firing", 150)
|
||||
report["worker_stopped"] = {
|
||||
"consumers_firing_after_seconds": consumer_wait,
|
||||
"queue_lag_firing_after_seconds": queue_wait,
|
||||
"alerts": queue_state or consumer_state,
|
||||
}
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "worker"], cwd=ROOT)
|
||||
restored, restored_state = wait_state("WorkerConsumersZero", None, 120)
|
||||
report["worker_restored"] = {"consumers_cleared_after_seconds": restored, "alerts": restored_state}
|
||||
|
||||
print("latency split", flush=True)
|
||||
write_fault("db_slow_ms", "800")
|
||||
client.call("GET", "/api/v1/chats", [200, 500, 503], actor=alice)
|
||||
time.sleep(3)
|
||||
write_fault("db_slow_ms", None)
|
||||
write_fault("handler_slow_ms", "800")
|
||||
client.call("GET", "/api/v1/chats", [200, 500], actor=alice)
|
||||
time.sleep(3)
|
||||
write_fault("handler_slow_ms", None)
|
||||
report["latency_note"] = "db_slow_ms sleeps inside db.transaction; handler_slow_ms sleeps in the HTTP span and outside that span"
|
||||
|
||||
print("tempo down", flush=True)
|
||||
before = prom("otelcol_exporter_send_failed_spans")
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "2", "tempo"], cwd=ROOT)
|
||||
tempo_counts = drive(client, alice, 8, 2)
|
||||
time.sleep(6)
|
||||
after = prom("otelcol_exporter_send_failed_spans")
|
||||
stats = subprocess.check_output(
|
||||
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", "mtusi-lab6-otel-collector-1"],
|
||||
text=True,
|
||||
).strip()
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "tempo"], cwd=ROOT)
|
||||
report["tempo_stopped"] = {
|
||||
"api_results": tempo_counts,
|
||||
"failed_spans_before": [item["value"][1] for item in before],
|
||||
"failed_spans_after": [item["value"][1] for item in after],
|
||||
"collector_memory": stats,
|
||||
}
|
||||
clear_faults()
|
||||
report["finished"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print("wrote", OUT, flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
finally:
|
||||
clear_faults()
|
||||
Reference in New Issue
Block a user