475 lines
18 KiB
Python
475 lines
18 KiB
Python
#!/usr/bin/env python3
|
||
"""Учебные проверки наблюдаемости и инцидентов. Пароли не печатает."""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
||
OUT = ROOT / "evidence" / "observability.json"
|
||
|
||
|
||
def http_json(method, path, body=None, headers=None, timeout=30):
|
||
data = None if body is None else json.dumps(body).encode("utf-8")
|
||
req = urllib.request.Request(
|
||
BASE + path,
|
||
data=data,
|
||
method=method,
|
||
headers=headers or {},
|
||
)
|
||
if data is not None:
|
||
req.add_header("Content-Type", "application/json")
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||
raw = resp.read()
|
||
payload = json.loads(raw.decode("utf-8")) if raw else None
|
||
return resp.status, dict(resp.headers), payload
|
||
except urllib.error.HTTPError as exc:
|
||
raw = exc.read()
|
||
payload = None
|
||
if raw:
|
||
try:
|
||
payload = json.loads(raw.decode("utf-8"))
|
||
except ValueError:
|
||
payload = {"raw": raw.decode("utf-8", errors="replace")[:200]}
|
||
return exc.code, dict(exc.headers), payload
|
||
|
||
|
||
def compose(*args):
|
||
cmd = ["docker", "compose", "--env-file", ".env", *args]
|
||
return subprocess.run(cmd, cwd=str(ROOT), capture_output=True, text=True, check=False)
|
||
|
||
|
||
def prom_query(expr):
|
||
q = urllib.parse.urlencode({"query": expr})
|
||
url = "http://127.0.0.1:9090/api/v1/query?" + q
|
||
# Prometheus не опубликован наружу. Читаем через docker exec.
|
||
result = compose("exec", "-T", "prometheus", "wget", "-qO-", "http://localhost:9090/api/v1/query?" + q)
|
||
if result.returncode != 0:
|
||
return None
|
||
try:
|
||
payload = json.loads(result.stdout)
|
||
except ValueError:
|
||
return None
|
||
rows = payload.get("data", {}).get("result", [])
|
||
if not rows:
|
||
return 0.0
|
||
return float(rows[0]["value"][1])
|
||
|
||
|
||
def prom_alerts():
|
||
result = compose("exec", "-T", "prometheus", "wget", "-qO-", "http://localhost:9090/api/v1/alerts")
|
||
if result.returncode != 0:
|
||
return []
|
||
try:
|
||
payload = json.loads(result.stdout)
|
||
except ValueError:
|
||
return []
|
||
return payload.get("data", {}).get("alerts", [])
|
||
|
||
|
||
def tempo_search(service, name=None, limit=10):
|
||
# Tag search + TraceQL fallback.
|
||
tags = "service.name=" + service
|
||
q = urllib.parse.urlencode({"tags": tags, "limit": str(limit)})
|
||
result = compose("exec", "-T", "tempo", "wget", "-qO-", "http://127.0.0.1:3200/api/search?" + q)
|
||
traces = []
|
||
if result.returncode == 0:
|
||
try:
|
||
traces = json.loads(result.stdout).get("traces", [])
|
||
except ValueError:
|
||
traces = []
|
||
if name:
|
||
traces = [t for t in traces if name in (t.get("rootTraceName") or "")]
|
||
if traces:
|
||
return traces
|
||
query = f'{{resource.service.name="{service}"}}'
|
||
if name:
|
||
query = f'{{resource.service.name="{service}" && name="{name}"}}'
|
||
q = urllib.parse.urlencode({"q": query, "limit": str(limit)})
|
||
result = compose("exec", "-T", "tempo", "wget", "-qO-", "http://127.0.0.1:3200/api/search?" + q)
|
||
if result.returncode != 0:
|
||
return []
|
||
try:
|
||
return json.loads(result.stdout).get("traces", [])
|
||
except ValueError:
|
||
return []
|
||
|
||
|
||
def register_and_chat():
|
||
stamp = datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S%f")
|
||
users = []
|
||
for idx in range(2):
|
||
username = f"obs_{stamp}_{idx}"
|
||
status, _, user = http_json(
|
||
"POST",
|
||
"/api/v1/users",
|
||
{"username": username, "display_name": username, "password": "Password123!"},
|
||
)
|
||
assert status == 201, status
|
||
status, _, session = http_json(
|
||
"POST",
|
||
"/api/v1/auth/sessions",
|
||
headers={"Authorization": "Basic " + __import__("base64").b64encode(f"{username}:Password123!".encode()).decode()},
|
||
)
|
||
assert status == 201, status
|
||
users.append({"user": user, "token": session["token"]})
|
||
status, _, chat = http_json(
|
||
"POST",
|
||
"/api/v1/chats",
|
||
{"title": "obs", "member_ids": [users[0]["user"]["id"], users[1]["user"]["id"]]},
|
||
headers={"Authorization": "Bearer " + users[0]["token"]},
|
||
)
|
||
assert status == 201, status
|
||
return users, chat
|
||
|
||
|
||
def upload_png(token, chat_id, name="probe.png"):
|
||
data = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
||
req = urllib.request.Request(
|
||
f"{BASE}/api/v1/chats/{chat_id}/attachments?filename={urllib.parse.quote(name)}",
|
||
data=data,
|
||
method="POST",
|
||
headers={
|
||
"Authorization": "Bearer " + token,
|
||
"Content-Type": "image/png",
|
||
"Content-Length": str(len(data)),
|
||
},
|
||
)
|
||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||
payload = json.loads(resp.read().decode("utf-8"))
|
||
return resp.status, dict(resp.headers), payload
|
||
|
||
|
||
def api_direct(method, path, token=None, timeout=3):
|
||
"""Запрос внутрь контейнера API, минуя HAProxy. Нужен, когда ready уже красный."""
|
||
headers = ["-H", "Accept: application/json"]
|
||
if token:
|
||
headers.extend(["-H", f"Authorization: Bearer {token}"])
|
||
cmd = [
|
||
"exec",
|
||
"-T",
|
||
"api",
|
||
"python",
|
||
"-c",
|
||
(
|
||
"import os,urllib.request,sys\n"
|
||
f"req=urllib.request.Request('http://127.0.0.1:8080{path}', method='{method}')\n"
|
||
),
|
||
]
|
||
# Проще через wget/python one-liner in api container.
|
||
py = (
|
||
"import urllib.request,urllib.error\n"
|
||
f"req=urllib.request.Request('http://127.0.0.1:8080{path}', method='{method}')\n"
|
||
)
|
||
if token:
|
||
py += f"req.add_header('Authorization', 'Bearer {token}')\n"
|
||
py += (
|
||
"try:\n"
|
||
f" r=urllib.request.urlopen(req, timeout={timeout})\n"
|
||
" print(r.status)\n"
|
||
"except urllib.error.HTTPError as e:\n"
|
||
" print(e.code)\n"
|
||
"except Exception as e:\n"
|
||
" print(type(e).__name__)\n"
|
||
)
|
||
result = compose("exec", "-T", "api", "python", "-c", py)
|
||
text = (result.stdout or "").strip().splitlines()
|
||
return text[-1] if text else "empty"
|
||
|
||
|
||
def wait_ready_attachment(token, attachment_id, timeout=60):
|
||
deadline = time.time() + timeout
|
||
while time.time() < deadline:
|
||
status, _, payload = http_json(
|
||
"GET",
|
||
f"/api/v1/attachments/{attachment_id}",
|
||
headers={"Authorization": "Bearer " + token},
|
||
)
|
||
if status == 200 and payload.get("status") == "ready":
|
||
return payload
|
||
time.sleep(0.5)
|
||
raise TimeoutError(attachment_id)
|
||
|
||
|
||
def scenario_trace_link():
|
||
users, chat = register_and_chat()
|
||
token = users[0]["token"]
|
||
status, headers, attachment = upload_png(token, chat["id"])
|
||
assert status == 202, status
|
||
ready = wait_ready_attachment(token, attachment["id"])
|
||
time.sleep(3)
|
||
api_traces = tempo_search("messenger-api", "POST /api/v1/chats/{chat_id}/attachments")
|
||
worker_traces = tempo_search("messenger-worker", "attachments.process consume")
|
||
return {
|
||
"ok": True,
|
||
"attachment_id": attachment["id"],
|
||
"attachment_status": ready["status"],
|
||
"request_id": headers.get("X-Request-Id") or headers.get("x-request-id"),
|
||
"api_upload_traces": len(api_traces),
|
||
"worker_consume_traces": len(worker_traces),
|
||
"sample_api_trace": api_traces[0]["traceID"] if api_traces else None,
|
||
"sample_worker_trace": worker_traces[0]["traceID"] if worker_traces else None,
|
||
}
|
||
|
||
|
||
def scenario_worker_stop():
|
||
users, chat = register_and_chat()
|
||
token = users[0]["token"]
|
||
compose("stop", "worker")
|
||
time.sleep(2)
|
||
status, _, attachment = upload_png(token, chat["id"], "queued.png")
|
||
assert status == 202, status
|
||
started = time.time()
|
||
fired_at = None
|
||
age_samples = []
|
||
while time.time() - started < 90:
|
||
age = prom_query('max(messenger_queue_oldest_job_age_seconds{messaging_destination_name="attachments.process"})')
|
||
age_samples.append({"t": round(time.time() - started, 1), "age": age})
|
||
alerts = [a for a in prom_alerts() if a.get("labels", {}).get("alertname") == "GrowingQueueLag" and a.get("state") == "firing"]
|
||
if alerts and fired_at is None:
|
||
fired_at = round(time.time() - started, 1)
|
||
break
|
||
time.sleep(5)
|
||
compose("start", "worker")
|
||
ready = wait_ready_attachment(token, attachment["id"], timeout=120)
|
||
resolved_at = None
|
||
recover_started = time.time()
|
||
while time.time() - recover_started < 120:
|
||
alerts = [a for a in prom_alerts() if a.get("labels", {}).get("alertname") == "GrowingQueueLag" and a.get("state") == "firing"]
|
||
if not alerts and fired_at is not None:
|
||
resolved_at = round(time.time() - recover_started, 1)
|
||
break
|
||
time.sleep(5)
|
||
return {
|
||
"ok": ready["status"] == "ready",
|
||
"attachment_id": attachment["id"],
|
||
"fired_after_seconds": fired_at,
|
||
"resolved_after_seconds": resolved_at,
|
||
"age_samples": age_samples[-6:],
|
||
"final_age": prom_query('messenger_queue_oldest_job_age_seconds{messaging_destination_name="attachments.process"}'),
|
||
}
|
||
|
||
|
||
def scenario_db_stop():
|
||
# Отказ Postgres. Через HAProxy ready красный, поэтому бизнес-5xx снимаю прямым
|
||
# запросом в контейнер API, чтобы метрики и traces остались от приложения.
|
||
users, chat = register_and_chat()
|
||
token = users[0]["token"]
|
||
before_5xx = prom_query('sum(messenger_http_requests_total{http_response_status_class="5xx"}) or vector(0)')
|
||
compose("stop", "postgres")
|
||
started = time.time()
|
||
business_codes = []
|
||
ready_via_lb = []
|
||
while time.time() - started < 90:
|
||
business_codes.append(api_direct("GET", "/api/v1/users/me", token=token))
|
||
try:
|
||
ready_status, _, _ = http_json("GET", "/health/ready", timeout=3)
|
||
except Exception as exc:
|
||
ready_status = type(exc).__name__
|
||
ready_via_lb.append(ready_status)
|
||
time.sleep(1)
|
||
high_fired = None
|
||
for sample_t in range(0, 90, 5):
|
||
pass
|
||
compose("start", "postgres")
|
||
deadline = time.time() + 90
|
||
while time.time() < deadline:
|
||
try:
|
||
status, _, payload = http_json("GET", "/health/ready", timeout=3)
|
||
if status == 200 and payload and payload.get("status") == "ok":
|
||
break
|
||
except Exception:
|
||
pass
|
||
time.sleep(2)
|
||
# ждём for: 1m у HighHttpErrorRate и запись метрик
|
||
watch_started = time.time()
|
||
alerts_timeline = []
|
||
while time.time() - watch_started < 150:
|
||
alerts = [
|
||
a
|
||
for a in prom_alerts()
|
||
if a.get("labels", {}).get("alertname")
|
||
in ("HighHttpErrorRate", "ErrorBudgetBurnFast", "ErrorBudgetBurnSlow")
|
||
]
|
||
alerts_timeline.append(
|
||
{
|
||
"t": round(time.time() - watch_started, 1),
|
||
"alerts": [
|
||
{"alertname": a.get("labels", {}).get("alertname"), "state": a.get("state")}
|
||
for a in alerts
|
||
],
|
||
}
|
||
)
|
||
if any(a.get("state") == "firing" for a in alerts) and high_fired is None:
|
||
high_fired = round(time.time() - watch_started, 1)
|
||
if high_fired is not None and not any(a.get("state") == "firing" for a in alerts):
|
||
break
|
||
time.sleep(5)
|
||
after_5xx = prom_query('sum(messenger_http_requests_total{http_response_status_class="5xx"}) or vector(0)')
|
||
return {
|
||
"ok": True,
|
||
"business_statuses_direct_api": business_codes[:8] + ["..."] + business_codes[-4:],
|
||
"ready_via_lb": ready_via_lb[:4] + ["..."] + ready_via_lb[-3:],
|
||
"http_5xx_before": before_5xx,
|
||
"http_5xx_after": after_5xx,
|
||
"high_error_seen_after_seconds": high_fired,
|
||
"alerts_timeline_tail": alerts_timeline[-6:],
|
||
"note": "Прямые запросы в api-контейнер, потому что HAProxy при красном ready отдаёт свою 503-страницу",
|
||
}
|
||
|
||
|
||
def scenario_telemetry_backend_down():
|
||
users, chat = register_and_chat()
|
||
token = users[0]["token"]
|
||
compose("stop", "tempo")
|
||
compose("stop", "loki")
|
||
compose("stop", "prometheus")
|
||
time.sleep(2)
|
||
mem_before = compose("stats", "--no-stream", "--format", "{{.MemUsage}}", "otel-collector").stdout.strip()
|
||
status, _, attachment = upload_png(token, chat["id"], "tel.png")
|
||
assert status == 202, status
|
||
# API должен жить без telemetry backends
|
||
ready_status, _, ready_body = http_json("GET", "/health/ready")
|
||
text_status, _, message = http_json(
|
||
"POST",
|
||
f"/api/v1/chats/{chat['id']}/messages",
|
||
{"text": "telemetry backends down"},
|
||
headers={
|
||
"Authorization": "Bearer " + token,
|
||
"Idempotency-Key": "obs-tel-1",
|
||
},
|
||
)
|
||
time.sleep(8)
|
||
mem_after = compose("stats", "--no-stream", "--format", "{{.MemUsage}}", "otel-collector").stdout.strip()
|
||
compose("start", "prometheus")
|
||
compose("start", "loki")
|
||
compose("start", "tempo")
|
||
time.sleep(5)
|
||
compose("restart", "otel-collector")
|
||
wait_ready_attachment(token, attachment["id"], timeout=120)
|
||
return {
|
||
"ok": ready_status == 200 and text_status == 201,
|
||
"upload_status": status,
|
||
"ready_status": ready_status,
|
||
"message_status": text_status,
|
||
"message_id": None if not message else message.get("id"),
|
||
"otel_mem_before": mem_before,
|
||
"otel_mem_after": mem_after,
|
||
}
|
||
|
||
|
||
def scenario_burn_rate_spike():
|
||
# Короткий всплеск и затем более длинная деградация для multi-window burn-rate.
|
||
users, _chat = register_and_chat()
|
||
token = users[0]["token"]
|
||
|
||
def outage(seconds):
|
||
compose("stop", "postgres")
|
||
end = time.time() + seconds
|
||
while time.time() < end:
|
||
api_direct("GET", "/api/v1/users/me", token=token)
|
||
time.sleep(0.5)
|
||
compose("start", "postgres")
|
||
deadline = time.time() + 90
|
||
while time.time() < deadline:
|
||
try:
|
||
status, _, payload = http_json("GET", "/health/ready", timeout=3)
|
||
if status == 200 and payload and payload.get("status") == "ok":
|
||
return
|
||
except Exception:
|
||
pass
|
||
time.sleep(2)
|
||
|
||
# краткий всплеск ~25 с
|
||
brief_started = time.time()
|
||
outage(25)
|
||
brief_high = None
|
||
brief_burn = None
|
||
watch = time.time()
|
||
while time.time() - watch < 120:
|
||
alerts = prom_alerts()
|
||
if brief_high is None and any(
|
||
a.get("labels", {}).get("alertname") == "HighHttpErrorRate" and a.get("state") == "firing" for a in alerts
|
||
):
|
||
brief_high = round(time.time() - brief_started, 1)
|
||
if brief_burn is None and any(
|
||
a.get("labels", {}).get("alertname") == "ErrorBudgetBurnFast" and a.get("state") == "firing" for a in alerts
|
||
):
|
||
brief_burn = round(time.time() - brief_started, 1)
|
||
if brief_high is not None:
|
||
break
|
||
time.sleep(5)
|
||
|
||
# длительная деградация ~6 минут
|
||
long_started = time.time()
|
||
outage(360)
|
||
long_slow = None
|
||
long_fast = None
|
||
watch = time.time()
|
||
while time.time() - watch < 240:
|
||
alerts = prom_alerts()
|
||
if long_slow is None and any(
|
||
a.get("labels", {}).get("alertname") == "ErrorBudgetBurnSlow" and a.get("state") == "firing" for a in alerts
|
||
):
|
||
long_slow = round(time.time() - long_started, 1)
|
||
if long_fast is None and any(
|
||
a.get("labels", {}).get("alertname") == "ErrorBudgetBurnFast" and a.get("state") == "firing" for a in alerts
|
||
):
|
||
long_fast = round(time.time() - long_started, 1)
|
||
if long_slow is not None:
|
||
break
|
||
time.sleep(5)
|
||
return {
|
||
"ok": True,
|
||
"brief_spike_seconds": 25,
|
||
"brief_high_error_fired_after_seconds": brief_high,
|
||
"brief_burn_fast_fired_after_seconds": brief_burn,
|
||
"long_degradation_seconds": 360,
|
||
"long_burn_slow_fired_after_seconds": long_slow,
|
||
"long_burn_fast_fired_after_seconds": long_fast,
|
||
"error_ratio_5m": prom_query("messenger:http_error_ratio:5m"),
|
||
"error_ratio_30m": prom_query("messenger:http_error_ratio:30m"),
|
||
"note": "Короткий всплеск может не зажечь BurnFast (нет ложного page). Длинная деградация должна зажечь BurnSlow.",
|
||
}
|
||
|
||
|
||
def main():
|
||
report = {
|
||
"started_at": datetime.now(timezone.utc).isoformat(),
|
||
"base_url": BASE,
|
||
"scenarios": {},
|
||
}
|
||
for name, func in [
|
||
("trace_link", scenario_trace_link),
|
||
("worker_stop", scenario_worker_stop),
|
||
("db_stop", scenario_db_stop),
|
||
("telemetry_backend_down", scenario_telemetry_backend_down),
|
||
("burn_rate_spike", scenario_burn_rate_spike),
|
||
]:
|
||
print("RUN", name, flush=True)
|
||
try:
|
||
report["scenarios"][name] = func()
|
||
print("OK", name, flush=True)
|
||
except Exception as exc:
|
||
report["scenarios"][name] = {"ok": False, "error": type(exc).__name__, "detail": str(exc)[:300]}
|
||
print("FAIL", name, type(exc).__name__, flush=True)
|
||
report["finished_at"] = datetime.now(timezone.utc).isoformat()
|
||
OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
print("WROTE", OUT)
|
||
failed = [k for k, v in report["scenarios"].items() if not v.get("ok")]
|
||
return 1 if failed else 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|