637 lines
24 KiB
Python
637 lines
24 KiB
Python
"""Thirty-minute controlled SLO study with real DB/worker/backend failures."""
|
|
|
|
import argparse
|
|
import csv
|
|
import io
|
|
import json
|
|
import math
|
|
import os
|
|
import threading
|
|
import time
|
|
from collections import Counter
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import UTC, datetime
|
|
from uuid import uuid4
|
|
|
|
import httpx
|
|
from PIL import Image
|
|
|
|
from scripts.tls_client import FreshTrustTransport
|
|
|
|
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
|
|
from scripts.observe import fetch, logs, queries, spans, wait
|
|
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
|
|
|
|
ERRORS = 'haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}'
|
|
RESPONSES = 'haproxy_backend_http_responses_total{proxy="messenger"}'
|
|
SIGNALS = {
|
|
"api_error_ratio_1m": f"sum(rate({ERRORS}[1m])) / clamp_min(sum(rate({RESPONSES}[1m])),0.001)",
|
|
"queue_depth": "max(messenger_queue_depth)",
|
|
"oldest_job_age": "max(messenger_oldest_job_age_seconds)",
|
|
"worker_busy": "sum(messenger_worker_busy)",
|
|
"worker_attempts": "sum(messenger_job_attempts_total)",
|
|
"pool_used": 'sum(messenger_db_pool_used{service_name="messenger-api"})',
|
|
"pool_waiting": 'sum(messenger_db_pool_waiting{service_name="messenger-api"})',
|
|
"http_p95": "histogram_quantile(0.95,sum by(le)(rate(messenger_http_duration_seconds_bucket[1m])))",
|
|
"slo_good": 'sum(messenger_slo_decisions_total{outcome="good"})',
|
|
"slo_bad": 'sum(messenger_slo_decisions_total{outcome="bad"})',
|
|
"collector_received_spans": "sum(otelcol_receiver_accepted_spans)",
|
|
"collector_sent_spans": "sum(otelcol_exporter_sent_spans)",
|
|
"collector_received_logs": "sum(otelcol_receiver_accepted_log_records)",
|
|
"collector_sent_logs": "sum(otelcol_exporter_sent_log_records)",
|
|
"collector_failed_spans": "sum(otelcol_exporter_send_failed_spans)",
|
|
"collector_queue": "sum(otelcol_exporter_queue_size)",
|
|
"collector_queue_capacity": "sum(otelcol_exporter_queue_capacity)",
|
|
"collector_memory": 'otelcol_process_memory_rss{job="collector"}',
|
|
"app_memory": "sum(messenger_process_memory_bytes)",
|
|
}
|
|
BURN_ALERTS = {"ApiFastBurn", "ApiSlowBurn", "ReadyFastBurn", "ReadySlowBurn"}
|
|
|
|
|
|
def snapshot(phase):
|
|
observed = queries(SIGNALS)
|
|
values = {}
|
|
for key, rows in observed.items():
|
|
value = float(rows[0]["value"][1]) if rows else None
|
|
values[key] = (
|
|
value if value is not None and math.isfinite(value) else None
|
|
)
|
|
alerts = fetch("prometheus", "/api/v1/alerts")["data"]["alerts"]
|
|
return {
|
|
"at": time.time(),
|
|
"phase": phase,
|
|
"signals": values,
|
|
"alerts": [
|
|
{
|
|
"name": a["labels"]["alertname"],
|
|
"state": a["state"],
|
|
"active_at": a.get("activeAt"),
|
|
}
|
|
for a in alerts
|
|
],
|
|
}
|
|
|
|
|
|
def firing(samples):
|
|
return {
|
|
a["name"]
|
|
for sample in samples
|
|
for a in sample["alerts"]
|
|
if a["state"] == "firing"
|
|
}
|
|
|
|
|
|
class Workload:
|
|
def __init__(self, client, headers, chat):
|
|
self.client, self.headers, self.chat = client, headers, chat
|
|
self.stop = threading.Event()
|
|
self.phase = "warmup"
|
|
self.requests = []
|
|
self.uploads = []
|
|
self.lock = threading.Lock()
|
|
self.slots = threading.BoundedSemaphore(16)
|
|
self.pool = ThreadPoolExecutor(max_workers=16)
|
|
image = io.BytesIO()
|
|
Image.new("RGB", (320, 200), (39, 102, 157)).save(image, "PNG")
|
|
self.raw = image.getvalue()
|
|
self.threads = [
|
|
threading.Thread(target=self.schedule_messages),
|
|
threading.Thread(target=self.schedule_uploads),
|
|
]
|
|
|
|
def write(self):
|
|
started, phase, trace_id = time.time(), self.phase, uuid4().hex
|
|
status, identity = 0, ""
|
|
try:
|
|
response = self.client.post(
|
|
f"/api/v1/chats/{self.chat}/messages",
|
|
headers={
|
|
**self.headers,
|
|
"Idempotency-Key": uuid4().hex,
|
|
"traceparent": "00-"
|
|
+ trace_id
|
|
+ "-"
|
|
+ uuid4().hex[:16]
|
|
+ "-01",
|
|
},
|
|
json={"text": "Synthetic SLO message " + uuid4().hex},
|
|
)
|
|
status = response.status_code
|
|
if status == 201:
|
|
identity = response.json()["id"]
|
|
except httpx.HTTPError:
|
|
pass
|
|
finally:
|
|
with self.lock:
|
|
self.requests.append(
|
|
[
|
|
started,
|
|
phase,
|
|
status,
|
|
time.time() - started,
|
|
trace_id,
|
|
identity,
|
|
]
|
|
)
|
|
self.slots.release()
|
|
|
|
def schedule_messages(self):
|
|
next_at = time.monotonic()
|
|
while not self.stop.is_set():
|
|
if self.slots.acquire(blocking=False):
|
|
self.pool.submit(self.write)
|
|
else:
|
|
with self.lock:
|
|
self.requests.append(
|
|
[time.time(), self.phase, -1, 0, "", ""]
|
|
)
|
|
next_at += 0.25
|
|
self.stop.wait(max(0, next_at - time.monotonic()))
|
|
|
|
def schedule_uploads(self):
|
|
while not self.stop.is_set():
|
|
started, phase, trace_id = time.time(), self.phase, uuid4().hex
|
|
status, identity = 0, ""
|
|
try:
|
|
response = self.client.post(
|
|
f"/api/v1/chats/{self.chat}/attachments",
|
|
params={"filename": "synthetic-slo.png"},
|
|
content=self.raw,
|
|
headers={
|
|
**self.headers,
|
|
"Content-Type": "image/png",
|
|
"traceparent": "00-"
|
|
+ trace_id
|
|
+ "-"
|
|
+ uuid4().hex[:16]
|
|
+ "-01",
|
|
},
|
|
)
|
|
status = response.status_code
|
|
if status == 202:
|
|
identity = response.json()["id"]
|
|
except httpx.HTTPError:
|
|
pass
|
|
with self.lock:
|
|
self.uploads.append(
|
|
[started, phase, status, identity, trace_id]
|
|
)
|
|
self.stop.wait(max(0, 5 - (time.time() - started)))
|
|
|
|
def start(self):
|
|
for thread in self.threads:
|
|
thread.start()
|
|
|
|
def close(self):
|
|
self.stop.set()
|
|
for thread in self.threads:
|
|
thread.join(20)
|
|
self.pool.shutdown(wait=True)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--output", default="evidence/slo-research-first.json")
|
|
parser.add_argument("--minimum-seconds", type=int, default=1800)
|
|
args = parser.parse_args()
|
|
output = ROOT / args.output
|
|
assert not output.exists(), "Choose a new output path"
|
|
report = {
|
|
"date": datetime.now(UTC).isoformat(),
|
|
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
|
"scope": "full30m"
|
|
if args.minimum_seconds >= 1800
|
|
else "short-development",
|
|
"profile": {
|
|
"messages_per_second": 4,
|
|
"max_inflight": 16,
|
|
"upload_every_seconds": 5,
|
|
"image_pixels": [320, 200],
|
|
},
|
|
"complete": False,
|
|
"samples": [],
|
|
"phases": [],
|
|
}
|
|
study_started = time.monotonic()
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
load = None
|
|
trigger = "lab6_slo_delay_" + uuid4().hex[:8]
|
|
with httpx.Client(
|
|
transport=FreshTrustTransport(),
|
|
base_url=BASE,
|
|
timeout=8,
|
|
trust_env=False,
|
|
limits=httpx.Limits(max_connections=24, max_keepalive_connections=24),
|
|
) as client:
|
|
ready(timeout=90)
|
|
config = json.loads(compose("config", "--format", "json"))
|
|
report["container_limits"] = {
|
|
name: {
|
|
k: int(service[k])
|
|
if k in {"mem_limit", "memswap_limit", "scale"}
|
|
else service[k]
|
|
for k in ["cpus", "mem_limit", "memswap_limit", "scale"]
|
|
if k in service
|
|
}
|
|
for name, service in config["services"].items()
|
|
}
|
|
report["docker_vm"] = json.loads(
|
|
command(
|
|
[
|
|
"docker",
|
|
"info",
|
|
"--format",
|
|
'{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}',
|
|
]
|
|
)
|
|
)
|
|
report["runtime_versions"] = sorted(
|
|
{
|
|
row["metric"].get("service_version", "unknown")
|
|
for row in queries(
|
|
{"versions": "messenger_process_memory_bytes"}
|
|
)["versions"]
|
|
}
|
|
)
|
|
user, _, session = actor(client)
|
|
headers = auth(session)
|
|
response = client.post(
|
|
"/api/v1/chats",
|
|
headers=headers,
|
|
json={
|
|
"title": "Synthetic SLO experiment",
|
|
"member_ids": [user["id"]],
|
|
},
|
|
)
|
|
assert response.status_code == 201
|
|
chat = response.json()["id"]
|
|
load = Workload(client, headers, chat)
|
|
workload = load
|
|
|
|
def save():
|
|
output.write_text(
|
|
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
|
)
|
|
|
|
def hold(name, seconds, *, until_clear=None, max_seconds=None):
|
|
load.phase = name
|
|
phase = {
|
|
"name": name,
|
|
"start": time.time(),
|
|
"planned_seconds": seconds,
|
|
"maximum_seconds": max_seconds or seconds,
|
|
}
|
|
report["phases"].append(phase)
|
|
minimum_end = time.monotonic() + seconds
|
|
deadline = time.monotonic() + (max_seconds or seconds)
|
|
start_index = len(report["samples"])
|
|
while time.monotonic() < deadline:
|
|
sample = snapshot(name)
|
|
report["samples"].append(sample)
|
|
save()
|
|
if (
|
|
until_clear is not None
|
|
and time.monotonic() >= minimum_end
|
|
and not until_clear.intersection(firing([sample]))
|
|
):
|
|
break
|
|
time.sleep(min(5, max(0, deadline - time.monotonic())))
|
|
phase["end"] = time.time()
|
|
phase["firing"] = sorted(firing(report["samples"][start_index:]))
|
|
return phase
|
|
|
|
def db_delay(on):
|
|
if on:
|
|
sql(
|
|
f"CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.chat_id='{chat}'::uuid THEN PERFORM pg_sleep(3); END IF; RETURN NEW; END $$; CREATE TRIGGER {trigger} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {trigger}()"
|
|
)
|
|
else:
|
|
sql(
|
|
f"DROP TRIGGER IF EXISTS {trigger} ON messages; DROP FUNCTION IF EXISTS {trigger}()"
|
|
)
|
|
|
|
try:
|
|
load.start()
|
|
warm = hold("warmup", 300)
|
|
settled = [
|
|
s for s in report["samples"] if s["at"] >= warm["end"] - 60
|
|
]
|
|
assert not BURN_ALERTS.intersection(firing(settled)), (
|
|
"Baseline is not clean"
|
|
)
|
|
assert (
|
|
report["samples"][-1]["signals"]["collector_memory"]
|
|
is not None
|
|
)
|
|
report["baseline"] = snapshot("baseline")
|
|
for signal in ["api", "ready"]:
|
|
if signal == "api":
|
|
activate, restore = (
|
|
lambda: db_delay(True),
|
|
lambda: db_delay(False),
|
|
)
|
|
else:
|
|
|
|
def activate():
|
|
return compose("stop", "--timeout", "1", "worker")
|
|
|
|
def restore():
|
|
return compose(
|
|
"up", "-d", "--no-deps", "--no-recreate", "worker"
|
|
)
|
|
|
|
brief_started = time.time()
|
|
activate()
|
|
brief = hold(signal + "_brief", 5)
|
|
restore()
|
|
settle = hold(
|
|
signal + "_brief_recovery", 120 if signal == "api" else 60
|
|
)
|
|
assert not BURN_ALERTS.intersection(
|
|
brief["firing"] + settle["firing"]
|
|
), "Brief incident triggered paging"
|
|
report.setdefault("short_spikes", []).append(
|
|
{
|
|
"signal": signal,
|
|
"action_started": brief_started,
|
|
"paging_events": 0,
|
|
}
|
|
)
|
|
action_started = time.time()
|
|
activate()
|
|
action_finished = time.time()
|
|
degraded = hold(
|
|
signal + "_sustained", 180 if signal == "api" else 150
|
|
)
|
|
restore_started = time.time()
|
|
restore()
|
|
restore_finished = time.time()
|
|
expected = (
|
|
{"ApiFastBurn", "ApiSlowBurn", "ApiHighErrorRate"}
|
|
if signal == "api"
|
|
else {"ReadyFastBurn", "ReadySlowBurn", "QueueLagGrowing"}
|
|
)
|
|
recovered = hold(
|
|
signal + "_recovery",
|
|
150 if signal == "api" else 240,
|
|
until_clear=expected,
|
|
max_seconds=600,
|
|
)
|
|
assert expected <= set(degraded["firing"]), (
|
|
signal,
|
|
degraded["firing"],
|
|
)
|
|
last = report["samples"][-1]
|
|
assert not expected.intersection(firing([last])), (
|
|
"Incident alerts did not resolve"
|
|
)
|
|
points = [
|
|
s
|
|
for s in report["samples"]
|
|
if action_finished <= s["at"] <= recovered["end"]
|
|
]
|
|
timings = {}
|
|
for name in sorted(expected):
|
|
first = next(
|
|
s["at"] for s in points if name in firing([s])
|
|
)
|
|
resolved = next(
|
|
s["at"]
|
|
for s in points
|
|
if s["at"] >= restore_finished
|
|
and name not in firing([s])
|
|
)
|
|
timings[name] = {
|
|
"detect_seconds": round(first - action_finished, 3),
|
|
"resolve_seconds": round(
|
|
resolved - restore_finished, 3
|
|
),
|
|
}
|
|
report.setdefault("incidents", []).append(
|
|
{
|
|
"signal": signal,
|
|
"action_started": action_started,
|
|
"action_finished": action_finished,
|
|
"restore_started": restore_started,
|
|
"restore_finished": restore_finished,
|
|
"timings": timings,
|
|
}
|
|
)
|
|
if signal == "api":
|
|
candidates = [
|
|
r
|
|
for r in load.requests
|
|
if action_finished <= r[0] <= restore_started
|
|
and r[2] >= 500
|
|
]
|
|
assert candidates
|
|
items = wait(
|
|
lambda identity=candidates[0][4]: spans(identity)
|
|
)
|
|
assert any(
|
|
s["name"] == "db.postgresql"
|
|
and s["duration_seconds"] >= 2
|
|
and s["status"].get("code") in {2, "STATUS_CODE_ERROR"}
|
|
for s in items
|
|
)
|
|
report["database_diagnostic_trace"] = items
|
|
else:
|
|
assert (
|
|
max(
|
|
s["signals"]["oldest_job_age"] or 0 for s in points
|
|
)
|
|
> 20
|
|
)
|
|
assert (
|
|
max(s["signals"]["queue_depth"] or 0 for s in points)
|
|
> 0
|
|
)
|
|
accepted = [
|
|
r
|
|
for r in load.uploads
|
|
if action_finished <= r[0] <= restore_started
|
|
and r[2] == 202
|
|
]
|
|
assert accepted
|
|
report["worker_recovered_trace"] = wait(
|
|
lambda identity=accepted[0][4]: spans(identity)
|
|
)
|
|
report["worker_recovered_attachment"] = accepted[0][3]
|
|
outage_before = snapshot("backend_before")
|
|
outage_started = time.time()
|
|
compose("stop", "--timeout", "1", "loki", "tempo")
|
|
outage = hold("backend_unavailable", 90)
|
|
points = [
|
|
s
|
|
for s in report["samples"]
|
|
if s["phase"] == "backend_unavailable"
|
|
]
|
|
assert max(
|
|
s["signals"]["collector_failed_spans"] or 0 for s in points
|
|
) > (outage_before["signals"]["collector_failed_spans"] or 0)
|
|
assert all(
|
|
s["signals"]["collector_queue"] is not None
|
|
and s["signals"]["collector_queue_capacity"] is not None
|
|
and s["signals"]["collector_memory"] is not None
|
|
for s in points
|
|
)
|
|
assert all(
|
|
(s["signals"]["collector_queue"] or 0)
|
|
<= (s["signals"]["collector_queue_capacity"] or 0)
|
|
for s in points
|
|
)
|
|
assert (
|
|
max(s["signals"]["collector_memory"] for s in points)
|
|
< report["container_limits"]["otel"]["mem_limit"]
|
|
)
|
|
assert not BURN_ALERTS.intersection(outage["firing"])
|
|
during = [
|
|
r for r in load.requests if r[1] == "backend_unavailable"
|
|
]
|
|
assert during and all(r[2] == 201 for r in during)
|
|
before_ready = sql(
|
|
"SELECT count(*) FROM attachments WHERE state='ready'"
|
|
)
|
|
compose("up", "-d", "--no-deps", "--no-recreate", "loki", "tempo")
|
|
wait(lambda: fetch("loki", "/ready", json_result=False))
|
|
wait(lambda: fetch("tempo", "/ready", json_result=False))
|
|
hold("backend_recovery", 60)
|
|
restored = next(
|
|
r
|
|
for r in reversed(load.uploads)
|
|
if r[1] == "backend_recovery" and r[2] == 202
|
|
)
|
|
wait(
|
|
lambda: any(
|
|
s["name"] == "attachment.process"
|
|
for s in spans(restored[4])
|
|
)
|
|
)
|
|
wait(lambda: logs(restored[4]).get("data", {}).get("result"))
|
|
report["backend_outage"] = {
|
|
"started": outage_started,
|
|
"finished": outage["end"],
|
|
"before": outage_before,
|
|
"after": snapshot("backend_after"),
|
|
"ready_count_before_backend_recovery": int(before_ready),
|
|
"successful_writes_during_outage": len(during),
|
|
"restored_trace_id": restored[4],
|
|
"queue_bounded": True,
|
|
"collector_rss_below_limit": True,
|
|
"send_failure_counter_counts_attempts_not_unique_loss": True,
|
|
}
|
|
remaining = args.minimum_seconds - (
|
|
time.monotonic() - study_started
|
|
)
|
|
if remaining > 0:
|
|
hold("steady_after_recovery", remaining)
|
|
load.close()
|
|
load = None
|
|
# Read through the public API: a successful SQL insert alone is not
|
|
# proof that every acknowledged message remains accessible.
|
|
seen, cursor = set(), None
|
|
while True:
|
|
page = client.get(
|
|
f"/api/v1/chats/{chat}/messages",
|
|
headers=headers,
|
|
params={
|
|
"limit": 100,
|
|
**({"cursor": cursor} if cursor else {}),
|
|
},
|
|
)
|
|
page.raise_for_status()
|
|
data = page.json()
|
|
ids = [row["id"] for row in data["items"]]
|
|
assert len(ids) == len(set(ids)) and not seen.intersection(ids)
|
|
seen.update(ids)
|
|
cursor = data.get("next_cursor")
|
|
if not cursor:
|
|
break
|
|
expected_messages = {
|
|
r[5] for r in workload.requests if r[2] == 201
|
|
}
|
|
assert seen == expected_messages
|
|
accepted_uploads = [r for r in workload.uploads if r[2] == 202]
|
|
for row in accepted_uploads:
|
|
|
|
def completed(identity=row[3]):
|
|
response = client.get(
|
|
f"/api/v1/attachments/{identity}", headers=headers
|
|
)
|
|
return (
|
|
response.status_code == 200
|
|
and response.json()["status"] == "ready"
|
|
)
|
|
|
|
wait(completed, timeout=120)
|
|
for row in accepted_uploads[
|
|
:: max(1, len(accepted_uploads) // 10)
|
|
]:
|
|
response = client.get(
|
|
f"/api/v1/attachments/{row[3]}/content", headers=headers
|
|
)
|
|
assert (
|
|
response.status_code == 200
|
|
and response.content == workload.raw
|
|
)
|
|
report["preservation"] = {
|
|
"acknowledged_messages": len(expected_messages),
|
|
"accessible_messages": len(seen),
|
|
"accepted_uploads": len(accepted_uploads),
|
|
"all_accepted_uploads_ready": True,
|
|
"sampled_originals_match": True,
|
|
}
|
|
report["rules"] = fetch("prometheus", "/api/v1/rules")
|
|
report["final_30m_sli"] = queries(
|
|
{
|
|
"api_good_ratio": f"1 - sum(increase({ERRORS}[30m])) / clamp_min(sum(increase({RESPONSES}[30m])),1)",
|
|
"ready_good_ratio": 'sum(increase(messenger_slo_decisions_total{outcome="good"}[30m])) / clamp_min(sum(increase(messenger_slo_decisions_total[30m])),1)',
|
|
}
|
|
)
|
|
report["complete"] = True
|
|
finally:
|
|
try:
|
|
db_delay(False)
|
|
finally:
|
|
compose(
|
|
"up",
|
|
"-d",
|
|
"--no-deps",
|
|
"--no-recreate",
|
|
"worker",
|
|
"loki",
|
|
"tempo",
|
|
)
|
|
if load is not None:
|
|
load.close()
|
|
report["elapsed_seconds"] = round(
|
|
time.monotonic() - study_started, 3
|
|
)
|
|
request_path = output.with_name(output.stem + "-requests.csv")
|
|
with request_path.open("w") as stream:
|
|
writer = csv.writer(stream)
|
|
writer.writerow(
|
|
[
|
|
"timestamp",
|
|
"phase",
|
|
"status",
|
|
"seconds",
|
|
"trace_id",
|
|
"message_id",
|
|
]
|
|
)
|
|
writer.writerows(workload.requests)
|
|
report["requests_csv"] = request_path.name
|
|
report["request_statuses"] = dict(
|
|
Counter(str(r[2]) for r in workload.requests)
|
|
)
|
|
report["uploads"] = workload.uploads
|
|
save()
|
|
print(
|
|
"PASS: short/sustained SLO incidents, detection/resolution and controlled false paging"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|