forked from mtusi2026/mtusi2026lab7
500 lines
30 KiB
Python
500 lines
30 KiB
Python
"""Lab6, уровень 4: контролируемые инциденты, алерты firing -> resolved и разбор причины по телеметрии.
|
||
|
||
python student-tests/incidents.py worker-stop # (1) остановлен worker: растёт возраст задач
|
||
python student-tests/incidents.py db-slow # (2) PostgreSQL медленный (блокировка таблицы)
|
||
python student-tests/incidents.py telemetry-down # (3) недоступны Tempo и Loki
|
||
python student-tests/incidents.py collector-down # (3') недоступен сам Collector
|
||
python student-tests/incidents.py # все по очереди (~20 минут)
|
||
|
||
Каждый сценарий: нагрузка/загрузки -> отказ -> ожидание firing в Prometheus -> «диагностика» только по
|
||
сохранённым наблюдениям (метрики, логи, трассы) -> восстановление -> ожидание resolved. Хронология
|
||
печатается, итог пишется в evidence/incident-<имя>.json (без токенов и текстов). Тест сам возвращает
|
||
стенд в рабочее состояние (start/unlock), даже если упал по проверке.
|
||
"""
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import uuid
|
||
|
||
import obs
|
||
from common import ROOT, call, check, compose, new_chat, new_user, try_call, wait_attachment
|
||
from images import png
|
||
|
||
EVIDENCE = ROOT / os.environ.get("EVIDENCE_DIR", "evidence")
|
||
FIRE_TIMEOUT = 300
|
||
RESOLVE_TIMEOUT = 300
|
||
T0 = time.monotonic()
|
||
|
||
|
||
def say(text):
|
||
print(f"[{time.monotonic() - T0:6.1f}s] {text}", flush=True)
|
||
|
||
|
||
def save(name, data):
|
||
EVIDENCE.mkdir(exist_ok=True)
|
||
path = EVIDENCE / f"incident-{name}.json"
|
||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
say(f"saved {path}")
|
||
|
||
|
||
def q(expr, default=None):
|
||
try:
|
||
value = obs.prom_value(expr, default)
|
||
except Exception: # noqa: BLE001 — Prometheus может быть недоступен в сценарии
|
||
return default
|
||
return default if value is None or value != value else value # NaN (нет данных) -> default
|
||
|
||
|
||
def rnd(value, digits=3):
|
||
return None if value is None else round(value, digits)
|
||
|
||
|
||
class Load:
|
||
"""Closed-loop POST сообщений в N потоков. Хранит (время, код или None, задержка) — без тела ответов."""
|
||
|
||
def __init__(self, threads=4, think=0.05, timeout=12):
|
||
self.user = new_user("incident")
|
||
self.chat = new_chat(self.user, title="Incident load")
|
||
self.threads, self.think, self.timeout = threads, think, timeout
|
||
self.samples = []
|
||
self.lock = threading.Lock()
|
||
self.stop_event = threading.Event()
|
||
self.workers = []
|
||
|
||
def _run(self):
|
||
while not self.stop_event.is_set():
|
||
started = time.monotonic()
|
||
reply = try_call("POST", f"/api/v1/chats/{self.chat['id']}/messages", body={"text": "load"},
|
||
token=self.user["token"], headers={"Idempotency-Key": uuid.uuid4().hex},
|
||
timeout=self.timeout)
|
||
with self.lock:
|
||
self.samples.append((started, reply.status if reply else None, time.monotonic() - started))
|
||
self.stop_event.wait(self.think)
|
||
|
||
def start(self):
|
||
for _ in range(self.threads):
|
||
thread = threading.Thread(target=self._run, daemon=True)
|
||
thread.start()
|
||
self.workers.append(thread)
|
||
return self
|
||
|
||
def stop(self):
|
||
self.stop_event.set()
|
||
for thread in self.workers:
|
||
thread.join(timeout=self.timeout + 2)
|
||
|
||
def window(self, since, until=None):
|
||
until = until or time.monotonic()
|
||
with self.lock:
|
||
rows = [s for s in self.samples if since <= s[0] < until]
|
||
codes = {}
|
||
for _, status, _ in rows:
|
||
key = str(status) if status else "timeout/conn"
|
||
codes[key] = codes.get(key, 0) + 1
|
||
ok = [lat for _, status, lat in rows if status and status < 500]
|
||
ok.sort()
|
||
p95 = ok[int(len(ok) * 0.95) - 1] if ok else None
|
||
seconds = max(until - since, 0.001)
|
||
return {"requests": len(rows), "codes": codes, "success_ratio": rnd(len(ok) / len(rows), 4) if rows else None,
|
||
"success_rps": rnd(len(ok) / seconds, 1), "p95_ms_successful": rnd(p95 * 1000 if p95 else None, 1)}
|
||
|
||
|
||
def container_mem_mib(service):
|
||
"""Память контейнера (docker stats) — работает и когда Prometheus/Collector недоступны."""
|
||
ids = subprocess.run(["docker", "compose", "ps", "-q", service], cwd=ROOT, capture_output=True,
|
||
text=True).stdout.split()
|
||
total = 0.0
|
||
for cid in ids:
|
||
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{json .}}", cid], cwd=ROOT,
|
||
capture_output=True, text=True).stdout.strip()
|
||
if not out:
|
||
continue
|
||
data = json.loads(out.splitlines()[0])
|
||
if "MemUsageBytes" in data:
|
||
total += data["MemUsageBytes"] / 2**20
|
||
continue
|
||
used = data.get("MemUsage", "0B").split("/")[0].strip()
|
||
for unit, factor in (("GiB", 1024), ("MiB", 1), ("KiB", 1 / 1024), ("kB", 1 / 1024), ("MB", 1), ("GB", 1024),
|
||
("B", 1 / 2**20)):
|
||
if used.endswith(unit):
|
||
total += float(used[:-len(unit)]) * factor
|
||
break
|
||
return round(total, 1)
|
||
|
||
|
||
def traced_upload(user, chat_id, size=(64, 48)):
|
||
trace_id, traceparent = obs.new_traceparent()
|
||
reply = call("POST", f"/api/v1/chats/{chat_id}/attachments?filename=incident.png", 202, raw=png(*size),
|
||
token=user["token"], headers={"Content-Type": "image/png", "traceparent": traceparent})
|
||
return reply.json["id"], trace_id
|
||
|
||
|
||
def probe(user, chat_id):
|
||
"""Запрос с известным trace_id: после восстановления проверим, дошла ли его трасса до Tempo."""
|
||
trace_id, traceparent = obs.new_traceparent()
|
||
reply = try_call("POST", f"/api/v1/chats/{chat_id}/messages", body={"text": "probe"}, token=user["token"],
|
||
headers={"Idempotency-Key": uuid.uuid4().hex, "traceparent": traceparent}, timeout=10)
|
||
return trace_id, reply.status if reply else None
|
||
|
||
|
||
def counter(expr):
|
||
return q(f"sum({expr})", 0.0) or 0.0
|
||
|
||
|
||
# ---------------------------------------------------------------------------------------------------
|
||
def worker_stop():
|
||
"""(1) Остановлен worker. Ожидаем: MessengerWorkerNoConsumers и MessengerQueueLagGrowing -> firing;
|
||
причина находится по метрикам очереди, логам и обрыву трасс; после start backlog -> 0, resolved."""
|
||
result = {"scenario": "worker-stop", "date": time.strftime("%Y-%m-%d %H:%M:%S")}
|
||
user = new_user("lag")
|
||
chat = new_chat(user, title="Lag chat")
|
||
check(obs.alerts().get("MessengerQueueLagGrowing") is None, "MessengerQueueLagGrowing is already active")
|
||
say("baseline: consumers=%s, oldest job age=%s s" % (
|
||
q('max(messenger_queue_consumers{messaging_destination_name="attachments.process.v1"})'),
|
||
q('max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"})')))
|
||
stopped = time.monotonic()
|
||
compose("stop", "worker")
|
||
say("docker compose stop worker")
|
||
try:
|
||
uploads = [traced_upload(user, chat["id"]) for _ in range(3)]
|
||
say(f"uploaded {len(uploads)} attachments: 202 queued (API accepts without worker)")
|
||
fired = {}
|
||
deadline = time.monotonic() + FIRE_TIMEOUT
|
||
while len(fired) < 2:
|
||
states = obs.alerts()
|
||
for name in ("MessengerWorkerNoConsumers", "MessengerQueueLagGrowing"):
|
||
if states.get(name) == "firing" and name not in fired:
|
||
fired[name] = round(time.monotonic() - stopped, 1)
|
||
say(f"ALERT {name} firing ({fired[name]} s after stop)")
|
||
if len(fired) < 2:
|
||
say(" consumers=%s backlog queued=%s oldest age=%s s, alerts: %s" % (
|
||
q('max(messenger_queue_consumers{messaging_destination_name="attachments.process.v1"})'),
|
||
q('max(messenger_jobs_backlog{status="queued"})'),
|
||
rnd(q('max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"})'), 1),
|
||
{k: v for k, v in states.items() if k.startswith("Messenger")}))
|
||
check(time.monotonic() < deadline, f"alerts not firing after {FIRE_TIMEOUT} s: {fired}")
|
||
time.sleep(5)
|
||
result["firing_after_stop_s"] = fired
|
||
|
||
# Диагностика только по сохранённым наблюдениям:
|
||
diag = {
|
||
"queue_consumers": q('max(messenger_queue_consumers{messaging_destination_name="attachments.process.v1"})'),
|
||
"broker_queue_depth": q('max(messenger_queue_depth{messaging_destination_name="attachments.process.v1"})'),
|
||
"jobs_backlog_queued": q('max(messenger_jobs_backlog{status="queued"})'),
|
||
"oldest_job_age_s": rnd(q('max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"})'), 1),
|
||
"outbox_oldest_age_s": rnd(q("max(messenger_outbox_oldest_age_seconds)"), 1),
|
||
"published_ok_last_5m": q('sum(increase(messenger_messages_published_total{outcome="ok"}[5m]))'),
|
||
"worker_metrics_last_seen_s": rnd(q('time() - max(timestamp(messenger_worker_capacity))'), 0),
|
||
}
|
||
last_worker_log = obs.loki_query('{service_name="messenger-worker"}', minutes=15, limit=1)
|
||
diag["last_worker_log"] = ({"event": last_worker_log[0][0].get("event"), "message": last_worker_log[0][1]}
|
||
if last_worker_log else None)
|
||
spans = obs.tempo_trace(uploads[0][1])
|
||
names = sorted({s["name"] for s in spans})
|
||
diag["upload_trace_span_names"] = names
|
||
diag["upload_trace_has_consumer_span"] = any(s["name"].startswith("process ") for s in spans)
|
||
result["diagnosis"] = diag
|
||
say("diagnosis: " + json.dumps(diag, ensure_ascii=False))
|
||
check(diag["queue_consumers"] == 0, "consumers must be 0 while worker is stopped")
|
||
check("publish attachments" in names and not diag["upload_trace_has_consumer_span"],
|
||
"trace must end at publish: message is in the broker, nobody consumes it")
|
||
finally:
|
||
started = time.monotonic()
|
||
compose("start", "worker")
|
||
say("docker compose start worker")
|
||
ready_at = []
|
||
for attachment_id, _ in uploads:
|
||
att = wait_attachment(user, attachment_id, seconds=120)
|
||
check(att["status"] == "ready", f"{attachment_id} is {att['status']}")
|
||
ready_at.append(round(time.monotonic() - started, 1))
|
||
result["ready_after_start_s"] = ready_at
|
||
say(f"all attachments ready {max(ready_at)} s after worker start; backlog now %s" %
|
||
q('max(messenger_jobs_backlog{status="queued"})'))
|
||
resolved = {}
|
||
for name in ("MessengerWorkerNoConsumers", "MessengerQueueLagGrowing"):
|
||
obs.wait_alert(name, "resolved", RESOLVE_TIMEOUT, log=say)
|
||
resolved[name] = round(time.monotonic() - started, 1)
|
||
result["resolved_after_start_s"] = resolved
|
||
result["after_recovery"] = {
|
||
"jobs_backlog_queued": q('max(messenger_jobs_backlog{status="queued"})'),
|
||
"oldest_job_age_s": rnd(q('max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"})'), 1),
|
||
"queue_consumers": q('max(messenger_queue_consumers{messaging_destination_name="attachments.process.v1"})')}
|
||
say(f"alerts resolved: {result['resolved_after_start_s']}; now {result['after_recovery']}")
|
||
result["time_to_ready_p95_5m_s"] = rnd(q('histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[5m])))'), 1)
|
||
save("worker-stop", result)
|
||
print("PASS: worker stop -> no consumers + growing queue lag -> firing; cause from metrics/logs/trace; "
|
||
"start -> backlog drained -> resolved")
|
||
|
||
|
||
# ---------------------------------------------------------------------------------------------------
|
||
LOCK_SECONDS = 210 # > окно rate (1m) + for (1m) + задержка экспорта метрик с запасом
|
||
|
||
|
||
def db_slow():
|
||
"""(2) PostgreSQL «медленный»: таблица messages заблокирована (ACCESS EXCLUSIVE) на LOCK_SECONDS.
|
||
Ожидаем MessengerHighErrorRate firing; трассы показывают, что время уходит в INSERT (PostgreSQL),
|
||
а не в обработчик; после снятия блокировки — resolved."""
|
||
result = {"scenario": "db-slow", "date": time.strftime("%Y-%m-%d %H:%M:%S"), "lock_seconds": LOCK_SECONDS}
|
||
load = Load(threads=8, think=0.05).start()
|
||
say("load: 8 threads POST /messages")
|
||
time.sleep(25)
|
||
base_from = time.monotonic() - 20
|
||
result["baseline"] = load.window(base_from)
|
||
result["baseline"]["server_p95_s"] = rnd(q('histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route="/api/v1/chats/{chat_id}/messages"}[1m])))'), 4)
|
||
say(f"baseline: {result['baseline']}")
|
||
check(obs.alerts().get("MessengerHighErrorRate") is None, "MessengerHighErrorRate is already active")
|
||
|
||
lock_out = {}
|
||
|
||
def hold_lock():
|
||
command = ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-X",
|
||
"-q", "-c", f"BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep({LOCK_SECONDS}); COMMIT;"]
|
||
lock_out["code"] = subprocess.run(command, cwd=ROOT, capture_output=True).returncode
|
||
lock_out["released"] = time.monotonic() # COMMIT выполнен: блокировка снята
|
||
|
||
locker = threading.Thread(target=hold_lock, daemon=True)
|
||
locked = time.monotonic()
|
||
locker.start()
|
||
say(f"LOCK TABLE messages IN ACCESS EXCLUSIVE MODE for {LOCK_SECONDS} s (PostgreSQL alive, writes wait)")
|
||
try:
|
||
fired_at = None
|
||
while fired_at is None:
|
||
if obs.alerts().get("MessengerHighErrorRate") == "firing":
|
||
fired_at = round(time.monotonic() - locked, 1)
|
||
break
|
||
say(" 5xx ratio=%s p95=%s s pool used=%s waiting=%s" % (
|
||
rnd(q("messenger:http_error_ratio:rate1m"), 3),
|
||
rnd(q('histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[1m])))'), 3),
|
||
q('sum(db_client_connection_count{db_client_connection_state="used"})'),
|
||
q("sum(db_client_connection_pending_requests)")))
|
||
check(time.monotonic() - locked < LOCK_SECONDS + 30, "MessengerHighErrorRate did not fire")
|
||
time.sleep(5)
|
||
say(f"ALERT MessengerHighErrorRate firing ({fired_at} s after lock)")
|
||
result["firing_after_lock_s"] = fired_at
|
||
|
||
# Диагностика: метрики
|
||
diag = {
|
||
"http_5xx_ratio_1m": rnd(q("messenger:http_error_ratio:rate1m"), 3),
|
||
"http_p95_messages_s": rnd(q('histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route="/api/v1/chats/{chat_id}/messages"}[1m])))'), 3),
|
||
"pool_used": q('sum(db_client_connection_count{db_client_connection_state="used"})'),
|
||
"pool_wait_p95_s": rnd(q("histogram_quantile(0.95, sum by (le) (rate(db_client_connection_wait_time_seconds_bucket[1m])))"), 3),
|
||
"status_codes_1m": {row[0].get("http_response_status_code"): rnd(row[1], 2) for row in obs.prom_query(
|
||
'sum by (http_response_status_code) (rate(http_server_request_duration_seconds_count{job="messenger-api", http_route="/api/v1/chats/{chat_id}/messages"}[1m]))')},
|
||
}
|
||
# Чтения других таблиц во время блокировки messages: быстрые -> «медленно» не всё, а одна операция
|
||
reads = []
|
||
for _ in range(5):
|
||
began = time.monotonic()
|
||
call("GET", "/api/v1/users/me", 200, token=load.user["token"])
|
||
call("GET", "/api/v1/chats", 200, token=load.user["token"])
|
||
reads.append((time.monotonic() - began) * 1000 / 2)
|
||
diag["reads_during_lock_ms"] = {"GET /users/me + GET /chats, avg per request": rnd(sum(reads) / len(reads), 1),
|
||
"max": rnd(max(reads), 1)}
|
||
# Логи: какие ошибки БД видит API
|
||
logs = obs.loki_query('{service_name="messenger-api"} | event="db.unavailable"', minutes=5, limit=500)
|
||
by_type = {}
|
||
for labels, _ in logs:
|
||
by_type[labels.get("error_type", "?")] = by_type.get(labels.get("error_type", "?"), 0) + 1
|
||
diag["api_log_db_unavailable_by_error_type"] = by_type
|
||
# Трассы: медленный запрос с ошибкой — где время
|
||
found = obs.tempo_search('{ resource.service.name = "messenger-api" && name = "POST /api/v1/chats/{chat_id}/messages" && status = error }',
|
||
minutes=5, limit=5)
|
||
check(found, "no error traces of POST messages in Tempo")
|
||
spans = obs.wait_trace(found[0]["traceID"].rjust(32, "0"), seconds=30)
|
||
server = next(s for s in spans if s["kind"] == "SERVER")
|
||
db_spans = [s for s in spans if s["kind"] == "CLIENT" and s["name"] in ("INSERT", "SELECT", "UPDATE")]
|
||
slowest = max(db_spans, key=lambda s: s["duration_ms"])
|
||
diag["error_trace"] = {"trace_id": found[0]["traceID"], "server_ms": rnd(server["duration_ms"], 1),
|
||
"slowest_db_span": slowest["name"], "slowest_db_ms": rnd(slowest["duration_ms"], 1),
|
||
"db_span_status": slowest["status"], "db_share": rnd(slowest["duration_ms"] / server["duration_ms"], 3),
|
||
"tree": obs.span_tree(spans).splitlines()}
|
||
# Контраст: медленный ОБРАБОТЧИК (CPU Argon2id при регистрации) — время в своём спане, БД быстрая
|
||
trace_id, traceparent = obs.new_traceparent()
|
||
call("POST", "/api/v1/users", 201, body={"username": f"cpu_{uuid.uuid4().hex[:10]}", "display_name": "Cpu",
|
||
"password": "Test-password-123"}, headers={"traceparent": traceparent})
|
||
cpu = obs.wait_trace(trace_id, lambda s: any(x["name"] == "argon2id hash" for x in s), 30)
|
||
cpu_server = next(s for s in cpu if s["kind"] == "SERVER")
|
||
argon = next(s for s in cpu if s["name"] == "argon2id hash")
|
||
cpu_db = [s for s in cpu if s["kind"] == "CLIENT"]
|
||
diag["handler_cpu_contrast"] = {"trace_id": trace_id, "server_ms": rnd(cpu_server["duration_ms"], 1),
|
||
"argon2id_ms": rnd(argon["duration_ms"], 1),
|
||
"db_ms_total": rnd(sum(s["duration_ms"] for s in cpu_db), 1),
|
||
"tree": obs.span_tree(cpu).splitlines()}
|
||
result["diagnosis"] = diag
|
||
say("diagnosis: " + json.dumps({k: v for k, v in diag.items() if k not in ("error_trace", "handler_cpu_contrast")},
|
||
ensure_ascii=False))
|
||
say("error trace (DB): " + json.dumps({k: v for k, v in diag["error_trace"].items() if k != "tree"}))
|
||
print("\n".join(" " + line for line in diag["error_trace"]["tree"]))
|
||
say("contrast, slow handler (CPU): " + json.dumps({k: v for k, v in diag["handler_cpu_contrast"].items() if k != "tree"}))
|
||
print("\n".join(" " + line for line in diag["handler_cpu_contrast"]["tree"]))
|
||
check(diag["error_trace"]["db_share"] > 0.8, "in the DB incident most of the request time must be in a DB span")
|
||
check(diag["handler_cpu_contrast"]["argon2id_ms"] > diag["handler_cpu_contrast"]["db_ms_total"],
|
||
"in the CPU contrast the handler span must dominate the DB spans")
|
||
finally:
|
||
locker.join(timeout=LOCK_SECONDS + 30)
|
||
unlocked = lock_out.get("released", time.monotonic())
|
||
say(f"lock released (psql exit {lock_out.get('code')}), {round(unlocked - locked, 1)} s after LOCK")
|
||
result["during_lock"] = load.window(locked + 5, unlocked)
|
||
obs.wait_alert("MessengerHighErrorRate", "resolved", RESOLVE_TIMEOUT, log=say)
|
||
result["resolved_after_unlock_s"] = round(time.monotonic() - unlocked, 1)
|
||
say(f"ALERT MessengerHighErrorRate resolved {result['resolved_after_unlock_s']} s after unlock")
|
||
result["after"] = load.window(unlocked + 5)
|
||
load.stop()
|
||
say(f"load: during lock {result['during_lock']}; after {result['after']}")
|
||
check((result["after"]["success_ratio"] or 0) > 0.99, "writes must recover after the lock")
|
||
save("db-slow", result)
|
||
print("PASS: slow DB -> 503 -> MessengerHighErrorRate firing; traces show the time in the INSERT span "
|
||
"(not the handler); unlock -> resolved")
|
||
|
||
|
||
# ---------------------------------------------------------------------------------------------------
|
||
def telemetry_down(stores=("tempo", "loki"), outage=150):
|
||
"""(3) Хранилища трасс и логов недоступны дольше окна повторов Collector (120 с).
|
||
Ожидаем: API работает без ошибок, RSS Collector ограничен, TelemetryExportFailing firing,
|
||
потери считаются (send_failed) и видны на пробных трассах; после start — данные снова идут."""
|
||
result = {"scenario": "telemetry-down", "date": time.strftime("%Y-%m-%d %H:%M:%S"), "stopped": list(stores),
|
||
"outage_s": outage}
|
||
load = Load(threads=4, think=0.05).start()
|
||
time.sleep(20)
|
||
result["baseline"] = load.window(time.monotonic() - 15)
|
||
failed_before = {sig: counter(f"otelcol_exporter_send_failed_{sig}") + counter(f"otelcol_exporter_enqueue_failed_{sig}")
|
||
for sig in ("spans", "log_records")}
|
||
heap_before = q("max(otelcol_process_runtime_heap_alloc_bytes)")
|
||
mem_before = container_mem_mib("otel-collector")
|
||
result["collector_heap_before_mib"] = rnd((heap_before or 0) / 2**20, 1)
|
||
result["collector_container_before_mib"] = mem_before
|
||
say(f"baseline: {result['baseline']}, collector heap {result['collector_heap_before_mib']} MiB, "
|
||
f"container {mem_before} MiB")
|
||
compose("stop", *stores)
|
||
stopped = time.monotonic() # отсчёт после остановки (Loki при stop сбрасывает данные на диск ~десятки секунд)
|
||
say(f"docker compose stop {' '.join(stores)}: stopped")
|
||
samples, probes = [], []
|
||
try:
|
||
while time.monotonic() - stopped < outage:
|
||
probes.append((round(time.monotonic() - stopped, 1), *probe(load.user, load.chat["id"])))
|
||
# memory_limiter ограничивает heap Go-процесса (RSS ещё включает отображённый в память бинарник ~150 MiB)
|
||
sample = {"t": round(time.monotonic() - stopped, 1),
|
||
"collector_heap_mib": rnd((q("max(otelcol_process_runtime_heap_alloc_bytes)") or 0) / 2**20, 1),
|
||
"collector_rss_mib": rnd((q("max(otelcol_process_memory_rss)") or 0) / 2**20, 1),
|
||
"collector_container_mib": container_mem_mib("otel-collector"),
|
||
"queue": {r[0].get("exporter"): r[1] for r in obs.prom_query("otelcol_exporter_queue_size")},
|
||
"alert": obs.alerts().get("TelemetryExportFailing", "inactive")}
|
||
samples.append(sample)
|
||
say(f" {sample}")
|
||
time.sleep(10)
|
||
finally:
|
||
compose("start", *stores)
|
||
started = time.monotonic()
|
||
say(f"docker compose start {' '.join(stores)}")
|
||
result["during"] = load.window(stopped, started)
|
||
result["samples"] = samples
|
||
result["collector_heap_max_mib"] = max(s["collector_heap_mib"] or 0 for s in samples)
|
||
result["collector_rss_max_mib"] = max(s["collector_rss_mib"] or 0 for s in samples)
|
||
result["collector_container_max_mib"] = max(s["collector_container_mib"] or 0 for s in samples)
|
||
fired = [s["t"] for s in samples if s["alert"] == "firing"]
|
||
result["TelemetryExportFailing_firing_at_s"] = fired[0] if fired else None
|
||
# после восстановления: Collector досылает очередь; затем проверяем, какие пробные трассы дошли
|
||
obs.wait_metric("sum(otelcol_exporter_queue_size)", lambda v: v == 0, 180)
|
||
result["queue_drained_after_start_s"] = round(time.monotonic() - started, 1)
|
||
say(f"exporter queues drained {result['queue_drained_after_start_s']} s after start")
|
||
time.sleep(15) # Tempo/Loki индексируют принятые пакеты
|
||
found = []
|
||
for t, trace_id, status in probes:
|
||
spans = []
|
||
for _ in range(5):
|
||
spans = obs.tempo_trace(trace_id)
|
||
if spans:
|
||
break
|
||
time.sleep(2)
|
||
found.append({"sent_at_s": t, "http": status, "trace_in_tempo": bool(spans)})
|
||
result["probe_traces"] = found
|
||
failed_after = {sig: counter(f"otelcol_exporter_send_failed_{sig}") + counter(f"otelcol_exporter_enqueue_failed_{sig}")
|
||
for sig in ("spans", "log_records")}
|
||
result["lost_in_collector"] = {sig: round(failed_after[sig] - failed_before[sig]) for sig in failed_after}
|
||
obs.wait_alert("TelemetryExportFailing", "resolved", RESOLVE_TIMEOUT, log=say)
|
||
result["resolved_after_start_s"] = round(time.monotonic() - started, 1)
|
||
result["after"] = load.window(started + 5)
|
||
load.stop()
|
||
say(f"API during outage: {result['during']}")
|
||
say(f"probe traces: {sum(p['trace_in_tempo'] for p in found)}/{len(found)} reached Tempo "
|
||
f"(lost: sent at {[p['sent_at_s'] for p in found if not p['trace_in_tempo']]} s)")
|
||
say(f"lost in Collector (send_failed+enqueue_failed): {result['lost_in_collector']}; collector heap "
|
||
f"{result['collector_heap_before_mib']} -> max {result['collector_heap_max_mib']} MiB (memory_limiter 200 MiB), "
|
||
f"container max {result['collector_container_max_mib']} MiB")
|
||
check(result["during"]["success_ratio"] == 1.0, "API must keep working while telemetry stores are down")
|
||
check(result["collector_heap_max_mib"] < 200, "collector heap must stay below memory_limiter limit_mib")
|
||
save("telemetry-down", result)
|
||
print("PASS: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted; recovery -> resolved")
|
||
|
||
|
||
def collector_down(outage=90):
|
||
"""(3') Сам Collector недоступен: SDK в процессах приложения ограничивает очередь и отбрасывает лишнее."""
|
||
result = {"scenario": "collector-down", "date": time.strftime("%Y-%m-%d %H:%M:%S"), "outage_s": outage}
|
||
load = Load(threads=4, think=0.05).start()
|
||
time.sleep(20)
|
||
result["baseline"] = load.window(time.monotonic() - 15)
|
||
# Потери SDK: отброшено процессором (очередь полна) + не принято exporter-ом (ошибка соединения).
|
||
sdk_lost = ('otel_sdk_processor_span_processed_total{job="messenger-api", error_type!=""}',
|
||
'otel_sdk_exporter_span_exported_total{job="messenger-api", error_type!=""}')
|
||
lost_before = [counter(expr) for expr in sdk_lost]
|
||
api_before = container_mem_mib("api")
|
||
say(f"baseline: {result['baseline']}, api containers {api_before} MiB")
|
||
stopped = time.monotonic()
|
||
compose("stop", "otel-collector")
|
||
say("docker compose stop otel-collector")
|
||
samples = []
|
||
try:
|
||
while time.monotonic() - stopped < outage:
|
||
samples.append({"t": round(time.monotonic() - stopped, 1), "api_mib": container_mem_mib("api")})
|
||
say(f" {samples[-1]}")
|
||
time.sleep(10)
|
||
finally:
|
||
compose("start", "otel-collector")
|
||
started = time.monotonic()
|
||
say("docker compose start otel-collector")
|
||
result["during"] = load.window(stopped, started)
|
||
result["api_mib_before"] = api_before
|
||
result["api_mib_max"] = max(s["api_mib"] for s in samples)
|
||
time.sleep(30) # SDK досылает cumulative-метрики, включая счётчик отброшенных спанов
|
||
lost_after = [counter(expr) for expr in sdk_lost]
|
||
result["sdk_lost_spans_api"] = {"processor_queue_full": round(lost_after[0] - lost_before[0]),
|
||
"exporter_failed": round(lost_after[1] - lost_before[1])}
|
||
result["sdk_exported_ok_after_restart"] = round(q(
|
||
'sum(increase(otel_sdk_exporter_span_exported_total{job="messenger-api", error_type=""}[1m]))', 0) or 0)
|
||
result["after"] = load.window(started + 5)
|
||
load.stop()
|
||
say(f"API during collector outage: {result['during']}; api memory {api_before} -> max {result['api_mib_max']} MiB; "
|
||
f"SDK lost spans (api): {result['sdk_lost_spans_api']}; exported OK in the last minute after restart: "
|
||
f"{result['sdk_exported_ok_after_restart']}")
|
||
check(sum(result["sdk_lost_spans_api"].values()) > 0, "losses during the outage must be counted by the SDK")
|
||
check(result["sdk_exported_ok_after_restart"] > 0, "spans must flow again after the Collector is back")
|
||
check(result["during"]["success_ratio"] == 1.0, "API must keep working without the Collector")
|
||
check(result["api_mib_max"] - api_before < 100, "API memory must not grow without bound")
|
||
save("collector-down", result)
|
||
print("PASS: Collector down -> API unaffected, SDK memory bounded, lost spans counted, export resumes after start")
|
||
|
||
|
||
SCENARIOS = {"worker-stop": worker_stop, "db-slow": db_slow, "telemetry-down": telemetry_down,
|
||
"collector-down": collector_down}
|
||
|
||
if __name__ == "__main__":
|
||
names = sys.argv[1:] or list(SCENARIOS)
|
||
unknown = [n for n in names if n not in SCENARIOS]
|
||
if unknown:
|
||
raise SystemExit(f"unknown scenarios {unknown}; available: {', '.join(SCENARIOS)}")
|
||
failed = 0
|
||
for name in names:
|
||
print(f"=== {name}", flush=True)
|
||
try:
|
||
SCENARIOS[name]()
|
||
except Exception as exc: # noqa: BLE001
|
||
failed += 1
|
||
print(f"FAIL: {name}: {exc}", flush=True)
|
||
for service in ("worker", "otel-collector", "tempo", "loki"):
|
||
compose("start", service)
|
||
print(f"{len(names) - failed} passed, {failed} failed")
|
||
raise SystemExit(1 if failed else 0)
|