Files
mtusi2026lab7/scripts/research_slo.py

637 lines
24 KiB
Python

"""Thirty-minute controlled SLO study with real DB/worker/backend failures."""
import argparse
import csv
import io
import json
import math
import os
import threading
import time
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime
from uuid import uuid4
import httpx
from PIL import Image
from scripts.tls_client import FreshTrustTransport
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
from scripts.observe import fetch, logs, queries, spans, wait
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
ERRORS = 'haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}'
RESPONSES = 'haproxy_backend_http_responses_total{proxy="messenger"}'
SIGNALS = {
"api_error_ratio_1m": f"sum(rate({ERRORS}[1m])) / clamp_min(sum(rate({RESPONSES}[1m])),0.001)",
"queue_depth": "max(messenger_queue_depth)",
"oldest_job_age": "max(messenger_oldest_job_age_seconds)",
"worker_busy": "sum(messenger_worker_busy)",
"worker_attempts": "sum(messenger_job_attempts_total)",
"pool_used": 'sum(messenger_db_pool_used{service_name="messenger-api"})',
"pool_waiting": 'sum(messenger_db_pool_waiting{service_name="messenger-api"})',
"http_p95": "histogram_quantile(0.95,sum by(le)(rate(messenger_http_duration_seconds_bucket[1m])))",
"slo_good": 'sum(messenger_slo_decisions_total{outcome="good"})',
"slo_bad": 'sum(messenger_slo_decisions_total{outcome="bad"})',
"collector_received_spans": "sum(otelcol_receiver_accepted_spans)",
"collector_sent_spans": "sum(otelcol_exporter_sent_spans)",
"collector_received_logs": "sum(otelcol_receiver_accepted_log_records)",
"collector_sent_logs": "sum(otelcol_exporter_sent_log_records)",
"collector_failed_spans": "sum(otelcol_exporter_send_failed_spans)",
"collector_queue": "sum(otelcol_exporter_queue_size)",
"collector_queue_capacity": "sum(otelcol_exporter_queue_capacity)",
"collector_memory": 'otelcol_process_memory_rss{job="collector"}',
"app_memory": "sum(messenger_process_memory_bytes)",
}
BURN_ALERTS = {"ApiFastBurn", "ApiSlowBurn", "ReadyFastBurn", "ReadySlowBurn"}
def snapshot(phase):
observed = queries(SIGNALS)
values = {}
for key, rows in observed.items():
value = float(rows[0]["value"][1]) if rows else None
values[key] = (
value if value is not None and math.isfinite(value) else None
)
alerts = fetch("prometheus", "/api/v1/alerts")["data"]["alerts"]
return {
"at": time.time(),
"phase": phase,
"signals": values,
"alerts": [
{
"name": a["labels"]["alertname"],
"state": a["state"],
"active_at": a.get("activeAt"),
}
for a in alerts
],
}
def firing(samples):
return {
a["name"]
for sample in samples
for a in sample["alerts"]
if a["state"] == "firing"
}
class Workload:
def __init__(self, client, headers, chat):
self.client, self.headers, self.chat = client, headers, chat
self.stop = threading.Event()
self.phase = "warmup"
self.requests = []
self.uploads = []
self.lock = threading.Lock()
self.slots = threading.BoundedSemaphore(16)
self.pool = ThreadPoolExecutor(max_workers=16)
image = io.BytesIO()
Image.new("RGB", (320, 200), (39, 102, 157)).save(image, "PNG")
self.raw = image.getvalue()
self.threads = [
threading.Thread(target=self.schedule_messages),
threading.Thread(target=self.schedule_uploads),
]
def write(self):
started, phase, trace_id = time.time(), self.phase, uuid4().hex
status, identity = 0, ""
try:
response = self.client.post(
f"/api/v1/chats/{self.chat}/messages",
headers={
**self.headers,
"Idempotency-Key": uuid4().hex,
"traceparent": "00-"
+ trace_id
+ "-"
+ uuid4().hex[:16]
+ "-01",
},
json={"text": "Synthetic SLO message " + uuid4().hex},
)
status = response.status_code
if status == 201:
identity = response.json()["id"]
except httpx.HTTPError:
pass
finally:
with self.lock:
self.requests.append(
[
started,
phase,
status,
time.time() - started,
trace_id,
identity,
]
)
self.slots.release()
def schedule_messages(self):
next_at = time.monotonic()
while not self.stop.is_set():
if self.slots.acquire(blocking=False):
self.pool.submit(self.write)
else:
with self.lock:
self.requests.append(
[time.time(), self.phase, -1, 0, "", ""]
)
next_at += 0.25
self.stop.wait(max(0, next_at - time.monotonic()))
def schedule_uploads(self):
while not self.stop.is_set():
started, phase, trace_id = time.time(), self.phase, uuid4().hex
status, identity = 0, ""
try:
response = self.client.post(
f"/api/v1/chats/{self.chat}/attachments",
params={"filename": "synthetic-slo.png"},
content=self.raw,
headers={
**self.headers,
"Content-Type": "image/png",
"traceparent": "00-"
+ trace_id
+ "-"
+ uuid4().hex[:16]
+ "-01",
},
)
status = response.status_code
if status == 202:
identity = response.json()["id"]
except httpx.HTTPError:
pass
with self.lock:
self.uploads.append(
[started, phase, status, identity, trace_id]
)
self.stop.wait(max(0, 5 - (time.time() - started)))
def start(self):
for thread in self.threads:
thread.start()
def close(self):
self.stop.set()
for thread in self.threads:
thread.join(20)
self.pool.shutdown(wait=True)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/slo-research-first.json")
parser.add_argument("--minimum-seconds", type=int, default=1800)
args = parser.parse_args()
output = ROOT / args.output
assert not output.exists(), "Choose a new output path"
report = {
"date": datetime.now(UTC).isoformat(),
"source_commit": command(["git", "rev-parse", "HEAD"]),
"scope": "full30m"
if args.minimum_seconds >= 1800
else "short-development",
"profile": {
"messages_per_second": 4,
"max_inflight": 16,
"upload_every_seconds": 5,
"image_pixels": [320, 200],
},
"complete": False,
"samples": [],
"phases": [],
}
study_started = time.monotonic()
output.parent.mkdir(parents=True, exist_ok=True)
load = None
trigger = "lab6_slo_delay_" + uuid4().hex[:8]
with httpx.Client(
transport=FreshTrustTransport(),
base_url=BASE,
timeout=8,
trust_env=False,
limits=httpx.Limits(max_connections=24, max_keepalive_connections=24),
) as client:
ready(timeout=90)
config = json.loads(compose("config", "--format", "json"))
report["container_limits"] = {
name: {
k: int(service[k])
if k in {"mem_limit", "memswap_limit", "scale"}
else service[k]
for k in ["cpus", "mem_limit", "memswap_limit", "scale"]
if k in service
}
for name, service in config["services"].items()
}
report["docker_vm"] = json.loads(
command(
[
"docker",
"info",
"--format",
'{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}',
]
)
)
report["runtime_versions"] = sorted(
{
row["metric"].get("service_version", "unknown")
for row in queries(
{"versions": "messenger_process_memory_bytes"}
)["versions"]
}
)
user, _, session = actor(client)
headers = auth(session)
response = client.post(
"/api/v1/chats",
headers=headers,
json={
"title": "Synthetic SLO experiment",
"member_ids": [user["id"]],
},
)
assert response.status_code == 201
chat = response.json()["id"]
load = Workload(client, headers, chat)
workload = load
def save():
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
def hold(name, seconds, *, until_clear=None, max_seconds=None):
load.phase = name
phase = {
"name": name,
"start": time.time(),
"planned_seconds": seconds,
"maximum_seconds": max_seconds or seconds,
}
report["phases"].append(phase)
minimum_end = time.monotonic() + seconds
deadline = time.monotonic() + (max_seconds or seconds)
start_index = len(report["samples"])
while time.monotonic() < deadline:
sample = snapshot(name)
report["samples"].append(sample)
save()
if (
until_clear is not None
and time.monotonic() >= minimum_end
and not until_clear.intersection(firing([sample]))
):
break
time.sleep(min(5, max(0, deadline - time.monotonic())))
phase["end"] = time.time()
phase["firing"] = sorted(firing(report["samples"][start_index:]))
return phase
def db_delay(on):
if on:
sql(
f"CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.chat_id='{chat}'::uuid THEN PERFORM pg_sleep(3); END IF; RETURN NEW; END $$; CREATE TRIGGER {trigger} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {trigger}()"
)
else:
sql(
f"DROP TRIGGER IF EXISTS {trigger} ON messages; DROP FUNCTION IF EXISTS {trigger}()"
)
try:
load.start()
warm = hold("warmup", 300)
settled = [
s for s in report["samples"] if s["at"] >= warm["end"] - 60
]
assert not BURN_ALERTS.intersection(firing(settled)), (
"Baseline is not clean"
)
assert (
report["samples"][-1]["signals"]["collector_memory"]
is not None
)
report["baseline"] = snapshot("baseline")
for signal in ["api", "ready"]:
if signal == "api":
activate, restore = (
lambda: db_delay(True),
lambda: db_delay(False),
)
else:
def activate():
return compose("stop", "--timeout", "1", "worker")
def restore():
return compose(
"up", "-d", "--no-deps", "--no-recreate", "worker"
)
brief_started = time.time()
activate()
brief = hold(signal + "_brief", 5)
restore()
settle = hold(
signal + "_brief_recovery", 120 if signal == "api" else 60
)
assert not BURN_ALERTS.intersection(
brief["firing"] + settle["firing"]
), "Brief incident triggered paging"
report.setdefault("short_spikes", []).append(
{
"signal": signal,
"action_started": brief_started,
"paging_events": 0,
}
)
action_started = time.time()
activate()
action_finished = time.time()
degraded = hold(
signal + "_sustained", 180 if signal == "api" else 150
)
restore_started = time.time()
restore()
restore_finished = time.time()
expected = (
{"ApiFastBurn", "ApiSlowBurn", "ApiHighErrorRate"}
if signal == "api"
else {"ReadyFastBurn", "ReadySlowBurn", "QueueLagGrowing"}
)
recovered = hold(
signal + "_recovery",
150 if signal == "api" else 240,
until_clear=expected,
max_seconds=600,
)
assert expected <= set(degraded["firing"]), (
signal,
degraded["firing"],
)
last = report["samples"][-1]
assert not expected.intersection(firing([last])), (
"Incident alerts did not resolve"
)
points = [
s
for s in report["samples"]
if action_finished <= s["at"] <= recovered["end"]
]
timings = {}
for name in sorted(expected):
first = next(
s["at"] for s in points if name in firing([s])
)
resolved = next(
s["at"]
for s in points
if s["at"] >= restore_finished
and name not in firing([s])
)
timings[name] = {
"detect_seconds": round(first - action_finished, 3),
"resolve_seconds": round(
resolved - restore_finished, 3
),
}
report.setdefault("incidents", []).append(
{
"signal": signal,
"action_started": action_started,
"action_finished": action_finished,
"restore_started": restore_started,
"restore_finished": restore_finished,
"timings": timings,
}
)
if signal == "api":
candidates = [
r
for r in load.requests
if action_finished <= r[0] <= restore_started
and r[2] >= 500
]
assert candidates
items = wait(
lambda identity=candidates[0][4]: spans(identity)
)
assert any(
s["name"] == "db.postgresql"
and s["duration_seconds"] >= 2
and s["status"].get("code") in {2, "STATUS_CODE_ERROR"}
for s in items
)
report["database_diagnostic_trace"] = items
else:
assert (
max(
s["signals"]["oldest_job_age"] or 0 for s in points
)
> 20
)
assert (
max(s["signals"]["queue_depth"] or 0 for s in points)
> 0
)
accepted = [
r
for r in load.uploads
if action_finished <= r[0] <= restore_started
and r[2] == 202
]
assert accepted
report["worker_recovered_trace"] = wait(
lambda identity=accepted[0][4]: spans(identity)
)
report["worker_recovered_attachment"] = accepted[0][3]
outage_before = snapshot("backend_before")
outage_started = time.time()
compose("stop", "--timeout", "1", "loki", "tempo")
outage = hold("backend_unavailable", 90)
points = [
s
for s in report["samples"]
if s["phase"] == "backend_unavailable"
]
assert max(
s["signals"]["collector_failed_spans"] or 0 for s in points
) > (outage_before["signals"]["collector_failed_spans"] or 0)
assert all(
s["signals"]["collector_queue"] is not None
and s["signals"]["collector_queue_capacity"] is not None
and s["signals"]["collector_memory"] is not None
for s in points
)
assert all(
(s["signals"]["collector_queue"] or 0)
<= (s["signals"]["collector_queue_capacity"] or 0)
for s in points
)
assert (
max(s["signals"]["collector_memory"] for s in points)
< report["container_limits"]["otel"]["mem_limit"]
)
assert not BURN_ALERTS.intersection(outage["firing"])
during = [
r for r in load.requests if r[1] == "backend_unavailable"
]
assert during and all(r[2] == 201 for r in during)
before_ready = sql(
"SELECT count(*) FROM attachments WHERE state='ready'"
)
compose("up", "-d", "--no-deps", "--no-recreate", "loki", "tempo")
wait(lambda: fetch("loki", "/ready", json_result=False))
wait(lambda: fetch("tempo", "/ready", json_result=False))
hold("backend_recovery", 60)
restored = next(
r
for r in reversed(load.uploads)
if r[1] == "backend_recovery" and r[2] == 202
)
wait(
lambda: any(
s["name"] == "attachment.process"
for s in spans(restored[4])
)
)
wait(lambda: logs(restored[4]).get("data", {}).get("result"))
report["backend_outage"] = {
"started": outage_started,
"finished": outage["end"],
"before": outage_before,
"after": snapshot("backend_after"),
"ready_count_before_backend_recovery": int(before_ready),
"successful_writes_during_outage": len(during),
"restored_trace_id": restored[4],
"queue_bounded": True,
"collector_rss_below_limit": True,
"send_failure_counter_counts_attempts_not_unique_loss": True,
}
remaining = args.minimum_seconds - (
time.monotonic() - study_started
)
if remaining > 0:
hold("steady_after_recovery", remaining)
load.close()
load = None
# Read through the public API: a successful SQL insert alone is not
# proof that every acknowledged message remains accessible.
seen, cursor = set(), None
while True:
page = client.get(
f"/api/v1/chats/{chat}/messages",
headers=headers,
params={
"limit": 100,
**({"cursor": cursor} if cursor else {}),
},
)
page.raise_for_status()
data = page.json()
ids = [row["id"] for row in data["items"]]
assert len(ids) == len(set(ids)) and not seen.intersection(ids)
seen.update(ids)
cursor = data.get("next_cursor")
if not cursor:
break
expected_messages = {
r[5] for r in workload.requests if r[2] == 201
}
assert seen == expected_messages
accepted_uploads = [r for r in workload.uploads if r[2] == 202]
for row in accepted_uploads:
def completed(identity=row[3]):
response = client.get(
f"/api/v1/attachments/{identity}", headers=headers
)
return (
response.status_code == 200
and response.json()["status"] == "ready"
)
wait(completed, timeout=120)
for row in accepted_uploads[
:: max(1, len(accepted_uploads) // 10)
]:
response = client.get(
f"/api/v1/attachments/{row[3]}/content", headers=headers
)
assert (
response.status_code == 200
and response.content == workload.raw
)
report["preservation"] = {
"acknowledged_messages": len(expected_messages),
"accessible_messages": len(seen),
"accepted_uploads": len(accepted_uploads),
"all_accepted_uploads_ready": True,
"sampled_originals_match": True,
}
report["rules"] = fetch("prometheus", "/api/v1/rules")
report["final_30m_sli"] = queries(
{
"api_good_ratio": f"1 - sum(increase({ERRORS}[30m])) / clamp_min(sum(increase({RESPONSES}[30m])),1)",
"ready_good_ratio": 'sum(increase(messenger_slo_decisions_total{outcome="good"}[30m])) / clamp_min(sum(increase(messenger_slo_decisions_total[30m])),1)',
}
)
report["complete"] = True
finally:
try:
db_delay(False)
finally:
compose(
"up",
"-d",
"--no-deps",
"--no-recreate",
"worker",
"loki",
"tempo",
)
if load is not None:
load.close()
report["elapsed_seconds"] = round(
time.monotonic() - study_started, 3
)
request_path = output.with_name(output.stem + "-requests.csv")
with request_path.open("w") as stream:
writer = csv.writer(stream)
writer.writerow(
[
"timestamp",
"phase",
"status",
"seconds",
"trace_id",
"message_id",
]
)
writer.writerows(workload.requests)
report["requests_csv"] = request_path.name
report["request_statuses"] = dict(
Counter(str(r[2]) for r in workload.requests)
)
report["uploads"] = workload.uploads
save()
print(
"PASS: short/sustained SLO incidents, detection/resolution and controlled false paging"
)
if __name__ == "__main__":
main()