Files
mtusi2026lab7/student-tests/incidents.py
T

500 lines
30 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Lab6, уровень 4: контролируемые инциденты, алерты firing -> resolved и разбор причины по телеметрии.
python student-tests/incidents.py worker-stop # (1) остановлен worker: растёт возраст задач
python student-tests/incidents.py db-slow # (2) PostgreSQL медленный (блокировка таблицы)
python student-tests/incidents.py telemetry-down # (3) недоступны Tempo и Loki
python student-tests/incidents.py collector-down # (3') недоступен сам Collector
python student-tests/incidents.py # все по очереди (~20 минут)
Каждый сценарий: нагрузка/загрузки -> отказ -> ожидание firing в Prometheus -> «диагностика» только по
сохранённым наблюдениям (метрики, логи, трассы) -> восстановление -> ожидание resolved. Хронология
печатается, итог пишется в evidence/incident-<имя>.json (без токенов и текстов). Тест сам возвращает
стенд в рабочее состояние (start/unlock), даже если упал по проверке.
"""
import json
import os
import subprocess
import sys
import threading
import time
import uuid
import obs
from common import ROOT, call, check, compose, new_chat, new_user, try_call, wait_attachment
from images import png
EVIDENCE = ROOT / os.environ.get("EVIDENCE_DIR", "evidence")
FIRE_TIMEOUT = 300
RESOLVE_TIMEOUT = 300
T0 = time.monotonic()
def say(text):
print(f"[{time.monotonic() - T0:6.1f}s] {text}", flush=True)
def save(name, data):
EVIDENCE.mkdir(exist_ok=True)
path = EVIDENCE / f"incident-{name}.json"
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
say(f"saved {path}")
def q(expr, default=None):
try:
value = obs.prom_value(expr, default)
except Exception: # noqa: BLE001 — Prometheus может быть недоступен в сценарии
return default
return default if value is None or value != value else value # NaN (нет данных) -> default
def rnd(value, digits=3):
return None if value is None else round(value, digits)
class Load:
"""Closed-loop POST сообщений в N потоков. Хранит (время, код или None, задержка) — без тела ответов."""
def __init__(self, threads=4, think=0.05, timeout=12):
self.user = new_user("incident")
self.chat = new_chat(self.user, title="Incident load")
self.threads, self.think, self.timeout = threads, think, timeout
self.samples = []
self.lock = threading.Lock()
self.stop_event = threading.Event()
self.workers = []
def _run(self):
while not self.stop_event.is_set():
started = time.monotonic()
reply = try_call("POST", f"/api/v1/chats/{self.chat['id']}/messages", body={"text": "load"},
token=self.user["token"], headers={"Idempotency-Key": uuid.uuid4().hex},
timeout=self.timeout)
with self.lock:
self.samples.append((started, reply.status if reply else None, time.monotonic() - started))
self.stop_event.wait(self.think)
def start(self):
for _ in range(self.threads):
thread = threading.Thread(target=self._run, daemon=True)
thread.start()
self.workers.append(thread)
return self
def stop(self):
self.stop_event.set()
for thread in self.workers:
thread.join(timeout=self.timeout + 2)
def window(self, since, until=None):
until = until or time.monotonic()
with self.lock:
rows = [s for s in self.samples if since <= s[0] < until]
codes = {}
for _, status, _ in rows:
key = str(status) if status else "timeout/conn"
codes[key] = codes.get(key, 0) + 1
ok = [lat for _, status, lat in rows if status and status < 500]
ok.sort()
p95 = ok[int(len(ok) * 0.95) - 1] if ok else None
seconds = max(until - since, 0.001)
return {"requests": len(rows), "codes": codes, "success_ratio": rnd(len(ok) / len(rows), 4) if rows else None,
"success_rps": rnd(len(ok) / seconds, 1), "p95_ms_successful": rnd(p95 * 1000 if p95 else None, 1)}
def container_mem_mib(service):
"""Память контейнера (docker stats) — работает и когда Prometheus/Collector недоступны."""
ids = subprocess.run(["docker", "compose", "ps", "-q", service], cwd=ROOT, capture_output=True,
text=True).stdout.split()
total = 0.0
for cid in ids:
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{json .}}", cid], cwd=ROOT,
capture_output=True, text=True).stdout.strip()
if not out:
continue
data = json.loads(out.splitlines()[0])
if "MemUsageBytes" in data:
total += data["MemUsageBytes"] / 2**20
continue
used = data.get("MemUsage", "0B").split("/")[0].strip()
for unit, factor in (("GiB", 1024), ("MiB", 1), ("KiB", 1 / 1024), ("kB", 1 / 1024), ("MB", 1), ("GB", 1024),
("B", 1 / 2**20)):
if used.endswith(unit):
total += float(used[:-len(unit)]) * factor
break
return round(total, 1)
def traced_upload(user, chat_id, size=(64, 48)):
trace_id, traceparent = obs.new_traceparent()
reply = call("POST", f"/api/v1/chats/{chat_id}/attachments?filename=incident.png", 202, raw=png(*size),
token=user["token"], headers={"Content-Type": "image/png", "traceparent": traceparent})
return reply.json["id"], trace_id
def probe(user, chat_id):
"""Запрос с известным trace_id: после восстановления проверим, дошла ли его трасса до Tempo."""
trace_id, traceparent = obs.new_traceparent()
reply = try_call("POST", f"/api/v1/chats/{chat_id}/messages", body={"text": "probe"}, token=user["token"],
headers={"Idempotency-Key": uuid.uuid4().hex, "traceparent": traceparent}, timeout=10)
return trace_id, reply.status if reply else None
def counter(expr):
return q(f"sum({expr})", 0.0) or 0.0
# ---------------------------------------------------------------------------------------------------
def worker_stop():
"""(1) Остановлен worker. Ожидаем: MessengerWorkerNoConsumers и MessengerQueueLagGrowing -> firing;
причина находится по метрикам очереди, логам и обрыву трасс; после start backlog -> 0, resolved."""
result = {"scenario": "worker-stop", "date": time.strftime("%Y-%m-%d %H:%M:%S")}
user = new_user("lag")
chat = new_chat(user, title="Lag chat")
check(obs.alerts().get("MessengerQueueLagGrowing") is None, "MessengerQueueLagGrowing is already active")
say("baseline: consumers=%s, oldest job age=%s s" % (
q('max(messenger_queue_consumers{messaging_destination_name="attachments.process.v1"})'),
q('max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"})')))
stopped = time.monotonic()
compose("stop", "worker")
say("docker compose stop worker")
try:
uploads = [traced_upload(user, chat["id"]) for _ in range(3)]
say(f"uploaded {len(uploads)} attachments: 202 queued (API accepts without worker)")
fired = {}
deadline = time.monotonic() + FIRE_TIMEOUT
while len(fired) < 2:
states = obs.alerts()
for name in ("MessengerWorkerNoConsumers", "MessengerQueueLagGrowing"):
if states.get(name) == "firing" and name not in fired:
fired[name] = round(time.monotonic() - stopped, 1)
say(f"ALERT {name} firing ({fired[name]} s after stop)")
if len(fired) < 2:
say(" consumers=%s backlog queued=%s oldest age=%s s, alerts: %s" % (
q('max(messenger_queue_consumers{messaging_destination_name="attachments.process.v1"})'),
q('max(messenger_jobs_backlog{status="queued"})'),
rnd(q('max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"})'), 1),
{k: v for k, v in states.items() if k.startswith("Messenger")}))
check(time.monotonic() < deadline, f"alerts not firing after {FIRE_TIMEOUT} s: {fired}")
time.sleep(5)
result["firing_after_stop_s"] = fired
# Диагностика только по сохранённым наблюдениям:
diag = {
"queue_consumers": q('max(messenger_queue_consumers{messaging_destination_name="attachments.process.v1"})'),
"broker_queue_depth": q('max(messenger_queue_depth{messaging_destination_name="attachments.process.v1"})'),
"jobs_backlog_queued": q('max(messenger_jobs_backlog{status="queued"})'),
"oldest_job_age_s": rnd(q('max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"})'), 1),
"outbox_oldest_age_s": rnd(q("max(messenger_outbox_oldest_age_seconds)"), 1),
"published_ok_last_5m": q('sum(increase(messenger_messages_published_total{outcome="ok"}[5m]))'),
"worker_metrics_last_seen_s": rnd(q('time() - max(timestamp(messenger_worker_capacity))'), 0),
}
last_worker_log = obs.loki_query('{service_name="messenger-worker"}', minutes=15, limit=1)
diag["last_worker_log"] = ({"event": last_worker_log[0][0].get("event"), "message": last_worker_log[0][1]}
if last_worker_log else None)
spans = obs.tempo_trace(uploads[0][1])
names = sorted({s["name"] for s in spans})
diag["upload_trace_span_names"] = names
diag["upload_trace_has_consumer_span"] = any(s["name"].startswith("process ") for s in spans)
result["diagnosis"] = diag
say("diagnosis: " + json.dumps(diag, ensure_ascii=False))
check(diag["queue_consumers"] == 0, "consumers must be 0 while worker is stopped")
check("publish attachments" in names and not diag["upload_trace_has_consumer_span"],
"trace must end at publish: message is in the broker, nobody consumes it")
finally:
started = time.monotonic()
compose("start", "worker")
say("docker compose start worker")
ready_at = []
for attachment_id, _ in uploads:
att = wait_attachment(user, attachment_id, seconds=120)
check(att["status"] == "ready", f"{attachment_id} is {att['status']}")
ready_at.append(round(time.monotonic() - started, 1))
result["ready_after_start_s"] = ready_at
say(f"all attachments ready {max(ready_at)} s after worker start; backlog now %s" %
q('max(messenger_jobs_backlog{status="queued"})'))
resolved = {}
for name in ("MessengerWorkerNoConsumers", "MessengerQueueLagGrowing"):
obs.wait_alert(name, "resolved", RESOLVE_TIMEOUT, log=say)
resolved[name] = round(time.monotonic() - started, 1)
result["resolved_after_start_s"] = resolved
result["after_recovery"] = {
"jobs_backlog_queued": q('max(messenger_jobs_backlog{status="queued"})'),
"oldest_job_age_s": rnd(q('max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"})'), 1),
"queue_consumers": q('max(messenger_queue_consumers{messaging_destination_name="attachments.process.v1"})')}
say(f"alerts resolved: {result['resolved_after_start_s']}; now {result['after_recovery']}")
result["time_to_ready_p95_5m_s"] = rnd(q('histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[5m])))'), 1)
save("worker-stop", result)
print("PASS: worker stop -> no consumers + growing queue lag -> firing; cause from metrics/logs/trace; "
"start -> backlog drained -> resolved")
# ---------------------------------------------------------------------------------------------------
LOCK_SECONDS = 210 # > окно rate (1m) + for (1m) + задержка экспорта метрик с запасом
def db_slow():
"""(2) PostgreSQL «медленный»: таблица messages заблокирована (ACCESS EXCLUSIVE) на LOCK_SECONDS.
Ожидаем MessengerHighErrorRate firing; трассы показывают, что время уходит в INSERT (PostgreSQL),
а не в обработчик; после снятия блокировки — resolved."""
result = {"scenario": "db-slow", "date": time.strftime("%Y-%m-%d %H:%M:%S"), "lock_seconds": LOCK_SECONDS}
load = Load(threads=8, think=0.05).start()
say("load: 8 threads POST /messages")
time.sleep(25)
base_from = time.monotonic() - 20
result["baseline"] = load.window(base_from)
result["baseline"]["server_p95_s"] = rnd(q('histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route="/api/v1/chats/{chat_id}/messages"}[1m])))'), 4)
say(f"baseline: {result['baseline']}")
check(obs.alerts().get("MessengerHighErrorRate") is None, "MessengerHighErrorRate is already active")
lock_out = {}
def hold_lock():
command = ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-X",
"-q", "-c", f"BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep({LOCK_SECONDS}); COMMIT;"]
lock_out["code"] = subprocess.run(command, cwd=ROOT, capture_output=True).returncode
lock_out["released"] = time.monotonic() # COMMIT выполнен: блокировка снята
locker = threading.Thread(target=hold_lock, daemon=True)
locked = time.monotonic()
locker.start()
say(f"LOCK TABLE messages IN ACCESS EXCLUSIVE MODE for {LOCK_SECONDS} s (PostgreSQL alive, writes wait)")
try:
fired_at = None
while fired_at is None:
if obs.alerts().get("MessengerHighErrorRate") == "firing":
fired_at = round(time.monotonic() - locked, 1)
break
say(" 5xx ratio=%s p95=%s s pool used=%s waiting=%s" % (
rnd(q("messenger:http_error_ratio:rate1m"), 3),
rnd(q('histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[1m])))'), 3),
q('sum(db_client_connection_count{db_client_connection_state="used"})'),
q("sum(db_client_connection_pending_requests)")))
check(time.monotonic() - locked < LOCK_SECONDS + 30, "MessengerHighErrorRate did not fire")
time.sleep(5)
say(f"ALERT MessengerHighErrorRate firing ({fired_at} s after lock)")
result["firing_after_lock_s"] = fired_at
# Диагностика: метрики
diag = {
"http_5xx_ratio_1m": rnd(q("messenger:http_error_ratio:rate1m"), 3),
"http_p95_messages_s": rnd(q('histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route="/api/v1/chats/{chat_id}/messages"}[1m])))'), 3),
"pool_used": q('sum(db_client_connection_count{db_client_connection_state="used"})'),
"pool_wait_p95_s": rnd(q("histogram_quantile(0.95, sum by (le) (rate(db_client_connection_wait_time_seconds_bucket[1m])))"), 3),
"status_codes_1m": {row[0].get("http_response_status_code"): rnd(row[1], 2) for row in obs.prom_query(
'sum by (http_response_status_code) (rate(http_server_request_duration_seconds_count{job="messenger-api", http_route="/api/v1/chats/{chat_id}/messages"}[1m]))')},
}
# Чтения других таблиц во время блокировки messages: быстрые -> «медленно» не всё, а одна операция
reads = []
for _ in range(5):
began = time.monotonic()
call("GET", "/api/v1/users/me", 200, token=load.user["token"])
call("GET", "/api/v1/chats", 200, token=load.user["token"])
reads.append((time.monotonic() - began) * 1000 / 2)
diag["reads_during_lock_ms"] = {"GET /users/me + GET /chats, avg per request": rnd(sum(reads) / len(reads), 1),
"max": rnd(max(reads), 1)}
# Логи: какие ошибки БД видит API
logs = obs.loki_query('{service_name="messenger-api"} | event="db.unavailable"', minutes=5, limit=500)
by_type = {}
for labels, _ in logs:
by_type[labels.get("error_type", "?")] = by_type.get(labels.get("error_type", "?"), 0) + 1
diag["api_log_db_unavailable_by_error_type"] = by_type
# Трассы: медленный запрос с ошибкой — где время
found = obs.tempo_search('{ resource.service.name = "messenger-api" && name = "POST /api/v1/chats/{chat_id}/messages" && status = error }',
minutes=5, limit=5)
check(found, "no error traces of POST messages in Tempo")
spans = obs.wait_trace(found[0]["traceID"].rjust(32, "0"), seconds=30)
server = next(s for s in spans if s["kind"] == "SERVER")
db_spans = [s for s in spans if s["kind"] == "CLIENT" and s["name"] in ("INSERT", "SELECT", "UPDATE")]
slowest = max(db_spans, key=lambda s: s["duration_ms"])
diag["error_trace"] = {"trace_id": found[0]["traceID"], "server_ms": rnd(server["duration_ms"], 1),
"slowest_db_span": slowest["name"], "slowest_db_ms": rnd(slowest["duration_ms"], 1),
"db_span_status": slowest["status"], "db_share": rnd(slowest["duration_ms"] / server["duration_ms"], 3),
"tree": obs.span_tree(spans).splitlines()}
# Контраст: медленный ОБРАБОТЧИК (CPU Argon2id при регистрации) — время в своём спане, БД быстрая
trace_id, traceparent = obs.new_traceparent()
call("POST", "/api/v1/users", 201, body={"username": f"cpu_{uuid.uuid4().hex[:10]}", "display_name": "Cpu",
"password": "Test-password-123"}, headers={"traceparent": traceparent})
cpu = obs.wait_trace(trace_id, lambda s: any(x["name"] == "argon2id hash" for x in s), 30)
cpu_server = next(s for s in cpu if s["kind"] == "SERVER")
argon = next(s for s in cpu if s["name"] == "argon2id hash")
cpu_db = [s for s in cpu if s["kind"] == "CLIENT"]
diag["handler_cpu_contrast"] = {"trace_id": trace_id, "server_ms": rnd(cpu_server["duration_ms"], 1),
"argon2id_ms": rnd(argon["duration_ms"], 1),
"db_ms_total": rnd(sum(s["duration_ms"] for s in cpu_db), 1),
"tree": obs.span_tree(cpu).splitlines()}
result["diagnosis"] = diag
say("diagnosis: " + json.dumps({k: v for k, v in diag.items() if k not in ("error_trace", "handler_cpu_contrast")},
ensure_ascii=False))
say("error trace (DB): " + json.dumps({k: v for k, v in diag["error_trace"].items() if k != "tree"}))
print("\n".join(" " + line for line in diag["error_trace"]["tree"]))
say("contrast, slow handler (CPU): " + json.dumps({k: v for k, v in diag["handler_cpu_contrast"].items() if k != "tree"}))
print("\n".join(" " + line for line in diag["handler_cpu_contrast"]["tree"]))
check(diag["error_trace"]["db_share"] > 0.8, "in the DB incident most of the request time must be in a DB span")
check(diag["handler_cpu_contrast"]["argon2id_ms"] > diag["handler_cpu_contrast"]["db_ms_total"],
"in the CPU contrast the handler span must dominate the DB spans")
finally:
locker.join(timeout=LOCK_SECONDS + 30)
unlocked = lock_out.get("released", time.monotonic())
say(f"lock released (psql exit {lock_out.get('code')}), {round(unlocked - locked, 1)} s after LOCK")
result["during_lock"] = load.window(locked + 5, unlocked)
obs.wait_alert("MessengerHighErrorRate", "resolved", RESOLVE_TIMEOUT, log=say)
result["resolved_after_unlock_s"] = round(time.monotonic() - unlocked, 1)
say(f"ALERT MessengerHighErrorRate resolved {result['resolved_after_unlock_s']} s after unlock")
result["after"] = load.window(unlocked + 5)
load.stop()
say(f"load: during lock {result['during_lock']}; after {result['after']}")
check((result["after"]["success_ratio"] or 0) > 0.99, "writes must recover after the lock")
save("db-slow", result)
print("PASS: slow DB -> 503 -> MessengerHighErrorRate firing; traces show the time in the INSERT span "
"(not the handler); unlock -> resolved")
# ---------------------------------------------------------------------------------------------------
def telemetry_down(stores=("tempo", "loki"), outage=150):
"""(3) Хранилища трасс и логов недоступны дольше окна повторов Collector (120 с).
Ожидаем: API работает без ошибок, RSS Collector ограничен, TelemetryExportFailing firing,
потери считаются (send_failed) и видны на пробных трассах; после start — данные снова идут."""
result = {"scenario": "telemetry-down", "date": time.strftime("%Y-%m-%d %H:%M:%S"), "stopped": list(stores),
"outage_s": outage}
load = Load(threads=4, think=0.05).start()
time.sleep(20)
result["baseline"] = load.window(time.monotonic() - 15)
failed_before = {sig: counter(f"otelcol_exporter_send_failed_{sig}") + counter(f"otelcol_exporter_enqueue_failed_{sig}")
for sig in ("spans", "log_records")}
heap_before = q("max(otelcol_process_runtime_heap_alloc_bytes)")
mem_before = container_mem_mib("otel-collector")
result["collector_heap_before_mib"] = rnd((heap_before or 0) / 2**20, 1)
result["collector_container_before_mib"] = mem_before
say(f"baseline: {result['baseline']}, collector heap {result['collector_heap_before_mib']} MiB, "
f"container {mem_before} MiB")
compose("stop", *stores)
stopped = time.monotonic() # отсчёт после остановки (Loki при stop сбрасывает данные на диск ~десятки секунд)
say(f"docker compose stop {' '.join(stores)}: stopped")
samples, probes = [], []
try:
while time.monotonic() - stopped < outage:
probes.append((round(time.monotonic() - stopped, 1), *probe(load.user, load.chat["id"])))
# memory_limiter ограничивает heap Go-процесса (RSS ещё включает отображённый в память бинарник ~150 MiB)
sample = {"t": round(time.monotonic() - stopped, 1),
"collector_heap_mib": rnd((q("max(otelcol_process_runtime_heap_alloc_bytes)") or 0) / 2**20, 1),
"collector_rss_mib": rnd((q("max(otelcol_process_memory_rss)") or 0) / 2**20, 1),
"collector_container_mib": container_mem_mib("otel-collector"),
"queue": {r[0].get("exporter"): r[1] for r in obs.prom_query("otelcol_exporter_queue_size")},
"alert": obs.alerts().get("TelemetryExportFailing", "inactive")}
samples.append(sample)
say(f" {sample}")
time.sleep(10)
finally:
compose("start", *stores)
started = time.monotonic()
say(f"docker compose start {' '.join(stores)}")
result["during"] = load.window(stopped, started)
result["samples"] = samples
result["collector_heap_max_mib"] = max(s["collector_heap_mib"] or 0 for s in samples)
result["collector_rss_max_mib"] = max(s["collector_rss_mib"] or 0 for s in samples)
result["collector_container_max_mib"] = max(s["collector_container_mib"] or 0 for s in samples)
fired = [s["t"] for s in samples if s["alert"] == "firing"]
result["TelemetryExportFailing_firing_at_s"] = fired[0] if fired else None
# после восстановления: Collector досылает очередь; затем проверяем, какие пробные трассы дошли
obs.wait_metric("sum(otelcol_exporter_queue_size)", lambda v: v == 0, 180)
result["queue_drained_after_start_s"] = round(time.monotonic() - started, 1)
say(f"exporter queues drained {result['queue_drained_after_start_s']} s after start")
time.sleep(15) # Tempo/Loki индексируют принятые пакеты
found = []
for t, trace_id, status in probes:
spans = []
for _ in range(5):
spans = obs.tempo_trace(trace_id)
if spans:
break
time.sleep(2)
found.append({"sent_at_s": t, "http": status, "trace_in_tempo": bool(spans)})
result["probe_traces"] = found
failed_after = {sig: counter(f"otelcol_exporter_send_failed_{sig}") + counter(f"otelcol_exporter_enqueue_failed_{sig}")
for sig in ("spans", "log_records")}
result["lost_in_collector"] = {sig: round(failed_after[sig] - failed_before[sig]) for sig in failed_after}
obs.wait_alert("TelemetryExportFailing", "resolved", RESOLVE_TIMEOUT, log=say)
result["resolved_after_start_s"] = round(time.monotonic() - started, 1)
result["after"] = load.window(started + 5)
load.stop()
say(f"API during outage: {result['during']}")
say(f"probe traces: {sum(p['trace_in_tempo'] for p in found)}/{len(found)} reached Tempo "
f"(lost: sent at {[p['sent_at_s'] for p in found if not p['trace_in_tempo']]} s)")
say(f"lost in Collector (send_failed+enqueue_failed): {result['lost_in_collector']}; collector heap "
f"{result['collector_heap_before_mib']} -> max {result['collector_heap_max_mib']} MiB (memory_limiter 200 MiB), "
f"container max {result['collector_container_max_mib']} MiB")
check(result["during"]["success_ratio"] == 1.0, "API must keep working while telemetry stores are down")
check(result["collector_heap_max_mib"] < 200, "collector heap must stay below memory_limiter limit_mib")
save("telemetry-down", result)
print("PASS: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted; recovery -> resolved")
def collector_down(outage=90):
"""(3') Сам Collector недоступен: SDK в процессах приложения ограничивает очередь и отбрасывает лишнее."""
result = {"scenario": "collector-down", "date": time.strftime("%Y-%m-%d %H:%M:%S"), "outage_s": outage}
load = Load(threads=4, think=0.05).start()
time.sleep(20)
result["baseline"] = load.window(time.monotonic() - 15)
# Потери SDK: отброшено процессором (очередь полна) + не принято exporter-ом (ошибка соединения).
sdk_lost = ('otel_sdk_processor_span_processed_total{job="messenger-api", error_type!=""}',
'otel_sdk_exporter_span_exported_total{job="messenger-api", error_type!=""}')
lost_before = [counter(expr) for expr in sdk_lost]
api_before = container_mem_mib("api")
say(f"baseline: {result['baseline']}, api containers {api_before} MiB")
stopped = time.monotonic()
compose("stop", "otel-collector")
say("docker compose stop otel-collector")
samples = []
try:
while time.monotonic() - stopped < outage:
samples.append({"t": round(time.monotonic() - stopped, 1), "api_mib": container_mem_mib("api")})
say(f" {samples[-1]}")
time.sleep(10)
finally:
compose("start", "otel-collector")
started = time.monotonic()
say("docker compose start otel-collector")
result["during"] = load.window(stopped, started)
result["api_mib_before"] = api_before
result["api_mib_max"] = max(s["api_mib"] for s in samples)
time.sleep(30) # SDK досылает cumulative-метрики, включая счётчик отброшенных спанов
lost_after = [counter(expr) for expr in sdk_lost]
result["sdk_lost_spans_api"] = {"processor_queue_full": round(lost_after[0] - lost_before[0]),
"exporter_failed": round(lost_after[1] - lost_before[1])}
result["sdk_exported_ok_after_restart"] = round(q(
'sum(increase(otel_sdk_exporter_span_exported_total{job="messenger-api", error_type=""}[1m]))', 0) or 0)
result["after"] = load.window(started + 5)
load.stop()
say(f"API during collector outage: {result['during']}; api memory {api_before} -> max {result['api_mib_max']} MiB; "
f"SDK lost spans (api): {result['sdk_lost_spans_api']}; exported OK in the last minute after restart: "
f"{result['sdk_exported_ok_after_restart']}")
check(sum(result["sdk_lost_spans_api"].values()) > 0, "losses during the outage must be counted by the SDK")
check(result["sdk_exported_ok_after_restart"] > 0, "spans must flow again after the Collector is back")
check(result["during"]["success_ratio"] == 1.0, "API must keep working without the Collector")
check(result["api_mib_max"] - api_before < 100, "API memory must not grow without bound")
save("collector-down", result)
print("PASS: Collector down -> API unaffected, SDK memory bounded, lost spans counted, export resumes after start")
SCENARIOS = {"worker-stop": worker_stop, "db-slow": db_slow, "telemetry-down": telemetry_down,
"collector-down": collector_down}
if __name__ == "__main__":
names = sys.argv[1:] or list(SCENARIOS)
unknown = [n for n in names if n not in SCENARIOS]
if unknown:
raise SystemExit(f"unknown scenarios {unknown}; available: {', '.join(SCENARIOS)}")
failed = 0
for name in names:
print(f"=== {name}", flush=True)
try:
SCENARIOS[name]()
except Exception as exc: # noqa: BLE001
failed += 1
print(f"FAIL: {name}: {exc}", flush=True)
for service in ("worker", "otel-collector", "tempo", "loki"):
compose("start", service)
print(f"{len(names) - failed} passed, {failed} failed")
raise SystemExit(1 if failed else 0)