Собрать наблюдаемый стенд, чтобы причину деградации API и worker было видно по метрикам, логам и трассам.

This commit is contained in:
2026-11-07 01:36:19 +03:00
parent 470d9d6f1e
commit 7d4e1237a3
56 changed files with 4419 additions and 53 deletions
+270
View File
@@ -0,0 +1,270 @@
"""Track 2 measurements. Writes evidence/slo-track2.json without tokens or message text."""
import json
import sys
import time
import urllib.parse
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client # noqa: E402
FAULTS = ROOT / "faults"
OUT = ROOT / "evidence" / "slo-track2.json"
def get(url):
with urllib.request.urlopen(url, timeout=10) as response:
return json.load(response)
def prom(expr):
return get("http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(expr))["data"]["result"]
def alert_states():
found = {}
for item in prom('ALERTS{alertname=~"Api.*|QueueLagHigh|WorkerConsumersZero|AttachmentReadyLatencyHigh|DatabaseUnready"}'):
name = item["metric"].get("alertname")
found[name] = item["metric"].get("alertstate")
return found
def write_fault(name, value):
path = FAULTS / name
if value is None:
path.unlink(missing_ok=True)
else:
path.write_text(str(value), encoding="utf-8")
def clear_faults():
for name in ("http_error_ratio", "db_slow_ms", "db_fail", "handler_slow_ms", "process_fail_remaining"):
write_fault(name, None)
def drive(client, actor, seconds, rps):
interval = 1.0 / rps
deadline = time.monotonic() + seconds
counts = {}
while time.monotonic() < deadline:
started = time.monotonic()
try:
result = client.call("GET", "/api/v1/chats", [200, 500], actor=actor)
status = result["status"]
except Exception:
status = "error"
counts[str(status)] = counts.get(str(status), 0) + 1
sleep = interval - (time.monotonic() - started)
if sleep > 0:
time.sleep(sleep)
return counts
def wait_state(name, wanted, timeout):
started = time.monotonic()
last = {}
while time.monotonic() - started < timeout:
last = alert_states()
if last.get(name) == wanted or (wanted is None and name not in last):
return round(time.monotonic() - started, 1), last
time.sleep(5)
return None, last
def span_summary(trace_id):
payload = get(f"http://127.0.0.1:3200/api/traces/{trace_id}")
spans = []
def walk(node):
if isinstance(node, dict):
if "name" in node and "spanId" in node:
start = node.get("startTimeUnixNano")
end = node.get("endTimeUnixNano")
duration = None
try:
duration = round((int(end) - int(start)) / 1e6, 1)
except (TypeError, ValueError):
duration = None
spans.append(
{
"name": node.get("name"),
"span": node.get("spanId"),
"parent": node.get("parentSpanId") or "",
"ms": duration,
}
)
for value in node.values():
walk(value)
elif isinstance(node, list):
for item in node:
walk(item)
walk(payload)
return spans
def latest_worker_trace(event):
query = '{service_name="messenger-worker"} |= "' + event + '"'
payload = get(
"http://127.0.0.1:3100/loki/api/v1/query_range?query="
+ urllib.parse.quote(query)
+ "&limit=5"
)
lines = []
for stream in payload["data"]["result"]:
for _, line in stream["values"]:
try:
body = json.loads(line)
except ValueError:
continue
if body.get("event") == event:
lines.append(body)
if not lines:
return None
body = lines[-1]
return {
"event": body.get("event"),
"attempt": body.get("attempt"),
"trace_id": body.get("trace_id"),
"span_id": body.get("span_id"),
"service": body.get("service.name"),
}
def main():
clear_faults()
report = {"started": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}
client = Client("http://127.0.0.1:8080", timeout=8)
alice, _bob, _eve, chat = client.bootstrap()
print("bootstrap ok", flush=True)
write_fault("process_fail_remaining", "1")
attachment, _content = client.attachment(alice, chat)
ready = client.wait_ready(alice, attachment, wait_seconds=40)
time.sleep(3)
log = latest_worker_trace("job.ready")
spans = []
if log and log.get("trace_id"):
try:
spans = span_summary(log["trace_id"])
except Exception as exc:
report["retry_trace_error"] = type(exc).__name__
report["retry_upload"] = {
"final_status": ready["status"],
"log": log,
"consume_spans": sum(1 for span in spans if span["name"] == "attachment.process consume"),
"span_names": [span["name"] for span in spans],
"parents_linked": any(span["parent"] for span in spans),
}
print("retry", report["retry_upload"]["consume_spans"], report["retry_upload"]["log"], flush=True)
print("warmup", flush=True)
report["warmup"] = drive(client, alice, 90, 2)
report["alerts_after_warmup"] = alert_states()
write_fault("http_error_ratio", "1")
print("short spike", flush=True)
report["short_spike"] = drive(client, alice, 10, 2)
time.sleep(35)
report["alerts_after_short_spike"] = alert_states()
clear_faults()
report["recovery_after_spike"] = drive(client, alice, 20, 2)
write_fault("http_error_ratio", "1")
print("sustained", flush=True)
sustained_started = time.monotonic()
seen = None
counts = {}
deadline = time.monotonic() + 100
interval = 0.5
while time.monotonic() < deadline:
started = time.monotonic()
try:
status = client.call("GET", "/api/v1/chats", [200, 500], actor=alice)["status"]
except Exception:
status = "error"
counts[str(status)] = counts.get(str(status), 0) + 1
states = alert_states()
if seen is None and states.get("ApiErrorBudgetFastBurn") == "firing":
seen = round(time.monotonic() - sustained_started, 1)
report["fast_burn_detected_after_seconds"] = seen
report["alerts_when_fast_burn_fired"] = states
break
sleep = interval - (time.monotonic() - started)
if sleep > 0:
time.sleep(sleep)
report["sustained_counts_until_detection_or_timeout"] = counts
if seen is None:
report["fast_burn_detected_after_seconds"] = None
report["alerts_after_sustained"] = alert_states()
clear_faults()
print("resolve", flush=True)
resolve_started = time.monotonic()
resolved = None
resolve_deadline = time.monotonic() + 180
while time.monotonic() < resolve_deadline:
client.call("GET", "/api/v1/chats", [200, 500], actor=alice)
if "ApiErrorBudgetFastBurn" not in alert_states():
resolved = round(time.monotonic() - resolve_started, 1)
break
time.sleep(5)
report["fast_burn_resolved_after_seconds"] = resolved
report["alerts_after_resolve"] = alert_states()
print("worker stop", flush=True)
import subprocess
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "5", "worker"], cwd=ROOT)
client.attachment(alice, chat)
consumer_wait, consumer_state = wait_state("WorkerConsumersZero", "firing", 150)
queue_wait, queue_state = wait_state("QueueLagHigh", "firing", 150)
report["worker_stopped"] = {
"consumers_firing_after_seconds": consumer_wait,
"queue_lag_firing_after_seconds": queue_wait,
"alerts": queue_state or consumer_state,
}
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "worker"], cwd=ROOT)
restored, restored_state = wait_state("WorkerConsumersZero", None, 120)
report["worker_restored"] = {"consumers_cleared_after_seconds": restored, "alerts": restored_state}
print("latency split", flush=True)
write_fault("db_slow_ms", "800")
client.call("GET", "/api/v1/chats", [200, 500, 503], actor=alice)
time.sleep(3)
write_fault("db_slow_ms", None)
write_fault("handler_slow_ms", "800")
client.call("GET", "/api/v1/chats", [200, 500], actor=alice)
time.sleep(3)
write_fault("handler_slow_ms", None)
report["latency_note"] = "db_slow_ms sleeps inside db.transaction; handler_slow_ms sleeps in the HTTP span and outside that span"
print("tempo down", flush=True)
before = prom("otelcol_exporter_send_failed_spans")
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "2", "tempo"], cwd=ROOT)
tempo_counts = drive(client, alice, 8, 2)
time.sleep(6)
after = prom("otelcol_exporter_send_failed_spans")
stats = subprocess.check_output(
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", "mtusi-lab6-otel-collector-1"],
text=True,
).strip()
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "tempo"], cwd=ROOT)
report["tempo_stopped"] = {
"api_results": tempo_counts,
"failed_spans_before": [item["value"][1] for item in before],
"failed_spans_after": [item["value"][1] for item in after],
"collector_memory": stats,
}
clear_faults()
report["finished"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
print("wrote", OUT, flush=True)
if __name__ == "__main__":
try:
main()
finally:
clear_faults()