Files
mtusi2026lab7/student-tests/run_observability.py
T
Wladislava1 54dcf70ea6
Skeleton contracts / contracts (pull_request) Canceled after 0s
Lab7: HTTPS, mTLS, OpenBao, трек 2 PKI на оценку 5
2026-11-17 00:25:48 +03:00

475 lines
18 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Учебные проверки наблюдаемости и инцидентов. Пароли не печатает."""
from __future__ import annotations
import json
import os
import subprocess
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
OUT = ROOT / "evidence" / "observability.json"
def http_json(method, path, body=None, headers=None, timeout=30):
data = None if body is None else json.dumps(body).encode("utf-8")
req = urllib.request.Request(
BASE + path,
data=data,
method=method,
headers=headers or {},
)
if data is not None:
req.add_header("Content-Type", "application/json")
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
raw = resp.read()
payload = json.loads(raw.decode("utf-8")) if raw else None
return resp.status, dict(resp.headers), payload
except urllib.error.HTTPError as exc:
raw = exc.read()
payload = None
if raw:
try:
payload = json.loads(raw.decode("utf-8"))
except ValueError:
payload = {"raw": raw.decode("utf-8", errors="replace")[:200]}
return exc.code, dict(exc.headers), payload
def compose(*args):
cmd = ["docker", "compose", "--env-file", ".env", *args]
return subprocess.run(cmd, cwd=str(ROOT), capture_output=True, text=True, check=False)
def prom_query(expr):
q = urllib.parse.urlencode({"query": expr})
url = "http://127.0.0.1:9090/api/v1/query?" + q
# Prometheus не опубликован наружу. Читаем через docker exec.
result = compose("exec", "-T", "prometheus", "wget", "-qO-", "http://localhost:9090/api/v1/query?" + q)
if result.returncode != 0:
return None
try:
payload = json.loads(result.stdout)
except ValueError:
return None
rows = payload.get("data", {}).get("result", [])
if not rows:
return 0.0
return float(rows[0]["value"][1])
def prom_alerts():
result = compose("exec", "-T", "prometheus", "wget", "-qO-", "http://localhost:9090/api/v1/alerts")
if result.returncode != 0:
return []
try:
payload = json.loads(result.stdout)
except ValueError:
return []
return payload.get("data", {}).get("alerts", [])
def tempo_search(service, name=None, limit=10):
# Tag search + TraceQL fallback.
tags = "service.name=" + service
q = urllib.parse.urlencode({"tags": tags, "limit": str(limit)})
result = compose("exec", "-T", "tempo", "wget", "-qO-", "http://127.0.0.1:3200/api/search?" + q)
traces = []
if result.returncode == 0:
try:
traces = json.loads(result.stdout).get("traces", [])
except ValueError:
traces = []
if name:
traces = [t for t in traces if name in (t.get("rootTraceName") or "")]
if traces:
return traces
query = f'{{resource.service.name="{service}"}}'
if name:
query = f'{{resource.service.name="{service}" && name="{name}"}}'
q = urllib.parse.urlencode({"q": query, "limit": str(limit)})
result = compose("exec", "-T", "tempo", "wget", "-qO-", "http://127.0.0.1:3200/api/search?" + q)
if result.returncode != 0:
return []
try:
return json.loads(result.stdout).get("traces", [])
except ValueError:
return []
def register_and_chat():
stamp = datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S%f")
users = []
for idx in range(2):
username = f"obs_{stamp}_{idx}"
status, _, user = http_json(
"POST",
"/api/v1/users",
{"username": username, "display_name": username, "password": "Password123!"},
)
assert status == 201, status
status, _, session = http_json(
"POST",
"/api/v1/auth/sessions",
headers={"Authorization": "Basic " + __import__("base64").b64encode(f"{username}:Password123!".encode()).decode()},
)
assert status == 201, status
users.append({"user": user, "token": session["token"]})
status, _, chat = http_json(
"POST",
"/api/v1/chats",
{"title": "obs", "member_ids": [users[0]["user"]["id"], users[1]["user"]["id"]]},
headers={"Authorization": "Bearer " + users[0]["token"]},
)
assert status == 201, status
return users, chat
def upload_png(token, chat_id, name="probe.png"):
data = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
req = urllib.request.Request(
f"{BASE}/api/v1/chats/{chat_id}/attachments?filename={urllib.parse.quote(name)}",
data=data,
method="POST",
headers={
"Authorization": "Bearer " + token,
"Content-Type": "image/png",
"Content-Length": str(len(data)),
},
)
with urllib.request.urlopen(req, timeout=30) as resp:
payload = json.loads(resp.read().decode("utf-8"))
return resp.status, dict(resp.headers), payload
def api_direct(method, path, token=None, timeout=3):
"""Запрос внутрь контейнера API, минуя HAProxy. Нужен, когда ready уже красный."""
headers = ["-H", "Accept: application/json"]
if token:
headers.extend(["-H", f"Authorization: Bearer {token}"])
cmd = [
"exec",
"-T",
"api",
"python",
"-c",
(
"import os,urllib.request,sys\n"
f"req=urllib.request.Request('http://127.0.0.1:8080{path}', method='{method}')\n"
),
]
# Проще через wget/python one-liner in api container.
py = (
"import urllib.request,urllib.error\n"
f"req=urllib.request.Request('http://127.0.0.1:8080{path}', method='{method}')\n"
)
if token:
py += f"req.add_header('Authorization', 'Bearer {token}')\n"
py += (
"try:\n"
f" r=urllib.request.urlopen(req, timeout={timeout})\n"
" print(r.status)\n"
"except urllib.error.HTTPError as e:\n"
" print(e.code)\n"
"except Exception as e:\n"
" print(type(e).__name__)\n"
)
result = compose("exec", "-T", "api", "python", "-c", py)
text = (result.stdout or "").strip().splitlines()
return text[-1] if text else "empty"
def wait_ready_attachment(token, attachment_id, timeout=60):
deadline = time.time() + timeout
while time.time() < deadline:
status, _, payload = http_json(
"GET",
f"/api/v1/attachments/{attachment_id}",
headers={"Authorization": "Bearer " + token},
)
if status == 200 and payload.get("status") == "ready":
return payload
time.sleep(0.5)
raise TimeoutError(attachment_id)
def scenario_trace_link():
users, chat = register_and_chat()
token = users[0]["token"]
status, headers, attachment = upload_png(token, chat["id"])
assert status == 202, status
ready = wait_ready_attachment(token, attachment["id"])
time.sleep(3)
api_traces = tempo_search("messenger-api", "POST /api/v1/chats/{chat_id}/attachments")
worker_traces = tempo_search("messenger-worker", "attachments.process consume")
return {
"ok": True,
"attachment_id": attachment["id"],
"attachment_status": ready["status"],
"request_id": headers.get("X-Request-Id") or headers.get("x-request-id"),
"api_upload_traces": len(api_traces),
"worker_consume_traces": len(worker_traces),
"sample_api_trace": api_traces[0]["traceID"] if api_traces else None,
"sample_worker_trace": worker_traces[0]["traceID"] if worker_traces else None,
}
def scenario_worker_stop():
users, chat = register_and_chat()
token = users[0]["token"]
compose("stop", "worker")
time.sleep(2)
status, _, attachment = upload_png(token, chat["id"], "queued.png")
assert status == 202, status
started = time.time()
fired_at = None
age_samples = []
while time.time() - started < 90:
age = prom_query('max(messenger_queue_oldest_job_age_seconds{messaging_destination_name="attachments.process"})')
age_samples.append({"t": round(time.time() - started, 1), "age": age})
alerts = [a for a in prom_alerts() if a.get("labels", {}).get("alertname") == "GrowingQueueLag" and a.get("state") == "firing"]
if alerts and fired_at is None:
fired_at = round(time.time() - started, 1)
break
time.sleep(5)
compose("start", "worker")
ready = wait_ready_attachment(token, attachment["id"], timeout=120)
resolved_at = None
recover_started = time.time()
while time.time() - recover_started < 120:
alerts = [a for a in prom_alerts() if a.get("labels", {}).get("alertname") == "GrowingQueueLag" and a.get("state") == "firing"]
if not alerts and fired_at is not None:
resolved_at = round(time.time() - recover_started, 1)
break
time.sleep(5)
return {
"ok": ready["status"] == "ready",
"attachment_id": attachment["id"],
"fired_after_seconds": fired_at,
"resolved_after_seconds": resolved_at,
"age_samples": age_samples[-6:],
"final_age": prom_query('messenger_queue_oldest_job_age_seconds{messaging_destination_name="attachments.process"}'),
}
def scenario_db_stop():
# Отказ Postgres. Через HAProxy ready красный, поэтому бизнес-5xx снимаю прямым
# запросом в контейнер API, чтобы метрики и traces остались от приложения.
users, chat = register_and_chat()
token = users[0]["token"]
before_5xx = prom_query('sum(messenger_http_requests_total{http_response_status_class="5xx"}) or vector(0)')
compose("stop", "postgres")
started = time.time()
business_codes = []
ready_via_lb = []
while time.time() - started < 90:
business_codes.append(api_direct("GET", "/api/v1/users/me", token=token))
try:
ready_status, _, _ = http_json("GET", "/health/ready", timeout=3)
except Exception as exc:
ready_status = type(exc).__name__
ready_via_lb.append(ready_status)
time.sleep(1)
high_fired = None
for sample_t in range(0, 90, 5):
pass
compose("start", "postgres")
deadline = time.time() + 90
while time.time() < deadline:
try:
status, _, payload = http_json("GET", "/health/ready", timeout=3)
if status == 200 and payload and payload.get("status") == "ok":
break
except Exception:
pass
time.sleep(2)
# ждём for: 1m у HighHttpErrorRate и запись метрик
watch_started = time.time()
alerts_timeline = []
while time.time() - watch_started < 150:
alerts = [
a
for a in prom_alerts()
if a.get("labels", {}).get("alertname")
in ("HighHttpErrorRate", "ErrorBudgetBurnFast", "ErrorBudgetBurnSlow")
]
alerts_timeline.append(
{
"t": round(time.time() - watch_started, 1),
"alerts": [
{"alertname": a.get("labels", {}).get("alertname"), "state": a.get("state")}
for a in alerts
],
}
)
if any(a.get("state") == "firing" for a in alerts) and high_fired is None:
high_fired = round(time.time() - watch_started, 1)
if high_fired is not None and not any(a.get("state") == "firing" for a in alerts):
break
time.sleep(5)
after_5xx = prom_query('sum(messenger_http_requests_total{http_response_status_class="5xx"}) or vector(0)')
return {
"ok": True,
"business_statuses_direct_api": business_codes[:8] + ["..."] + business_codes[-4:],
"ready_via_lb": ready_via_lb[:4] + ["..."] + ready_via_lb[-3:],
"http_5xx_before": before_5xx,
"http_5xx_after": after_5xx,
"high_error_seen_after_seconds": high_fired,
"alerts_timeline_tail": alerts_timeline[-6:],
"note": "Прямые запросы в api-контейнер, потому что HAProxy при красном ready отдаёт свою 503-страницу",
}
def scenario_telemetry_backend_down():
users, chat = register_and_chat()
token = users[0]["token"]
compose("stop", "tempo")
compose("stop", "loki")
compose("stop", "prometheus")
time.sleep(2)
mem_before = compose("stats", "--no-stream", "--format", "{{.MemUsage}}", "otel-collector").stdout.strip()
status, _, attachment = upload_png(token, chat["id"], "tel.png")
assert status == 202, status
# API должен жить без telemetry backends
ready_status, _, ready_body = http_json("GET", "/health/ready")
text_status, _, message = http_json(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
{"text": "telemetry backends down"},
headers={
"Authorization": "Bearer " + token,
"Idempotency-Key": "obs-tel-1",
},
)
time.sleep(8)
mem_after = compose("stats", "--no-stream", "--format", "{{.MemUsage}}", "otel-collector").stdout.strip()
compose("start", "prometheus")
compose("start", "loki")
compose("start", "tempo")
time.sleep(5)
compose("restart", "otel-collector")
wait_ready_attachment(token, attachment["id"], timeout=120)
return {
"ok": ready_status == 200 and text_status == 201,
"upload_status": status,
"ready_status": ready_status,
"message_status": text_status,
"message_id": None if not message else message.get("id"),
"otel_mem_before": mem_before,
"otel_mem_after": mem_after,
}
def scenario_burn_rate_spike():
# Короткий всплеск и затем более длинная деградация для multi-window burn-rate.
users, _chat = register_and_chat()
token = users[0]["token"]
def outage(seconds):
compose("stop", "postgres")
end = time.time() + seconds
while time.time() < end:
api_direct("GET", "/api/v1/users/me", token=token)
time.sleep(0.5)
compose("start", "postgres")
deadline = time.time() + 90
while time.time() < deadline:
try:
status, _, payload = http_json("GET", "/health/ready", timeout=3)
if status == 200 and payload and payload.get("status") == "ok":
return
except Exception:
pass
time.sleep(2)
# краткий всплеск ~25 с
brief_started = time.time()
outage(25)
brief_high = None
brief_burn = None
watch = time.time()
while time.time() - watch < 120:
alerts = prom_alerts()
if brief_high is None and any(
a.get("labels", {}).get("alertname") == "HighHttpErrorRate" and a.get("state") == "firing" for a in alerts
):
brief_high = round(time.time() - brief_started, 1)
if brief_burn is None and any(
a.get("labels", {}).get("alertname") == "ErrorBudgetBurnFast" and a.get("state") == "firing" for a in alerts
):
brief_burn = round(time.time() - brief_started, 1)
if brief_high is not None:
break
time.sleep(5)
# длительная деградация ~6 минут
long_started = time.time()
outage(360)
long_slow = None
long_fast = None
watch = time.time()
while time.time() - watch < 240:
alerts = prom_alerts()
if long_slow is None and any(
a.get("labels", {}).get("alertname") == "ErrorBudgetBurnSlow" and a.get("state") == "firing" for a in alerts
):
long_slow = round(time.time() - long_started, 1)
if long_fast is None and any(
a.get("labels", {}).get("alertname") == "ErrorBudgetBurnFast" and a.get("state") == "firing" for a in alerts
):
long_fast = round(time.time() - long_started, 1)
if long_slow is not None:
break
time.sleep(5)
return {
"ok": True,
"brief_spike_seconds": 25,
"brief_high_error_fired_after_seconds": brief_high,
"brief_burn_fast_fired_after_seconds": brief_burn,
"long_degradation_seconds": 360,
"long_burn_slow_fired_after_seconds": long_slow,
"long_burn_fast_fired_after_seconds": long_fast,
"error_ratio_5m": prom_query("messenger:http_error_ratio:5m"),
"error_ratio_30m": prom_query("messenger:http_error_ratio:30m"),
"note": "Короткий всплеск может не зажечь BurnFast (нет ложного page). Длинная деградация должна зажечь BurnSlow.",
}
def main():
report = {
"started_at": datetime.now(timezone.utc).isoformat(),
"base_url": BASE,
"scenarios": {},
}
for name, func in [
("trace_link", scenario_trace_link),
("worker_stop", scenario_worker_stop),
("db_stop", scenario_db_stop),
("telemetry_backend_down", scenario_telemetry_backend_down),
("burn_rate_spike", scenario_burn_rate_spike),
]:
print("RUN", name, flush=True)
try:
report["scenarios"][name] = func()
print("OK", name, flush=True)
except Exception as exc:
report["scenarios"][name] = {"ok": False, "error": type(exc).__name__, "detail": str(exc)[:300]}
print("FAIL", name, type(exc).__name__, flush=True)
report["finished_at"] = datetime.now(timezone.utc).isoformat()
OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print("WROTE", OUT)
failed = [k for k, v in report["scenarios"].items() if not v.get("ok")]
return 1 if failed else 0
if __name__ == "__main__":
raise SystemExit(main())