Files
mtusi2026lab6-fork/scripts/incident.py
T

227 lines
9.5 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""lab6: контролируемый запрос и ограниченные отказы для проверки наблюдаемости.
Только стандартная библиотека; Docker управляется через scripts/ops.py / docker compose.
Печатает JSON-строки таймлайна (ID, статусы, счётчики) — без токенов и паролей.
python scripts/incident.py probe [--filename photo.png]
Один upload: печатает X-Request-Id, attachment_id и время до ready.
Затем в Grafana -> Explore -> Tempo (TraceQL):
{ span.messenger.request_id = "<X-Request-Id>" }
и в Loki: {service_name=~"messenger-.+"} | request_id="<X-Request-Id>"
python scripts/incident.py worker-stop [--duration 150]
Инцидент 1: worker остановлен, upload'ы продолжаются -> растут queue depth и
возраст старейшего задания -> QueueLagGrowing firing; worker запущен ->
очередь разбирается -> resolved.
python scripts/incident.py db-lock [--duration 180]
Инцидент 2: «медленная БД». В PostgreSQL держится ACCESS EXCLUSIVE lock на
таблице messages: запросы к сообщениям ждут до statement_timeout (5 с) и
получают 503 -> ApiHighErrorRate firing. В trace — SQL span ~5 с с ошибкой
QueryCanceled (а не долгий handler). Остальные маршруты работают.
python scripts/incident.py telemetry-down [--duration 90] [--services otel-collector]
Инцидент 3: telemetry backend недоступен. Под нагрузкой останавливаются
указанные сервисы; проверяется, что API отвечает, и RSS API не растёт.
"""
import argparse
import json
import shlex
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
HERE = Path(__file__).resolve().parent
ROOT = HERE.parent
sys.path.insert(0, str(ROOT / "student-tests"))
sys.path.insert(0, str(ROOT / "tests"))
import common # noqa: E402
from imagegen import sample_png # noqa: E402
from test_attachments import upload # noqa: E402
PY = sys.executable
T0 = time.monotonic()
def note(step, **kw):
kw = dict(step=step, t=round(time.monotonic() - T0, 1), **kw)
print(json.dumps(kw, ensure_ascii=False), flush=True)
def sh(cmd, check=True):
note("exec", cmd=cmd)
return subprocess.run(shlex.split(cmd), check=check, stdout=subprocess.DEVNULL if check else None)
def setup_chat():
owner = common.actor("obs")
chat = common.create_chat(owner)
return owner, chat
def one_upload(owner, chat, filename="photo.png", data=None):
rid = "probe-" + uuid.uuid4().hex[:12]
r = upload(owner["_token"], chat["id"], data or sample_png(), filename=filename,
headers={"X-Request-Id": rid})
return r, rid
def cmd_probe(args):
owner, chat = setup_chat()
started = time.monotonic()
r, rid = one_upload(owner, chat, args.filename)
note("upload", status=r.status, request_id=rid, attachment_id=(r.json or {}).get("id"),
instance=r.headers.get("X-Instance-Id"))
if r.status != 202:
return 1
ready = common.wait_ready(owner["_token"], r.json["id"], timeout=args.timeout, allow_failed=True)
note("final", status=(ready or {}).get("status"), seconds_to_final=round(time.monotonic() - started, 2))
print("\nTempo (TraceQL): { span.messenger.request_id = \"%s\" }" % rid)
print("Loki: {service_name=~\"messenger-.+\"} | request_id=\"%s\"" % rid)
return 0
def cmd_worker_stop(args):
owner, chat = setup_chat()
sh(args.stop_worker)
ids, end = [], time.monotonic() + args.duration
while time.monotonic() < end:
r, rid = one_upload(owner, chat)
ids.append(r.json["id"] if r.status == 202 else None)
note("upload_while_worker_down", status=r.status, request_id=rid)
time.sleep(args.interval)
sh(args.start_worker)
started = time.monotonic()
pending = [i for i in ids if i]
while pending and time.monotonic() - started < 300:
pending = [i for i in pending if common.request("GET", "/api/v1/attachments/" + i,
token=owner["_token"]).json.get("status") != "ready"]
time.sleep(2)
note("drained", uploads=len(ids), not_ready=len(pending), drain_seconds=round(time.monotonic() - started, 1))
return 0 if not pending else 1
class Load:
"""Постоянная нагрузка: POST сообщения + GET истории (closed loop, N потоков)."""
def __init__(self, owner, chat, threads):
self.owner, self.chat, self.stop = owner, chat, threading.Event()
self.counts, self.lock = {}, threading.Lock()
self.threads = [threading.Thread(target=self.run, daemon=True) for _ in range(threads)]
def run(self):
path = "/api/v1/chats/%s/messages" % self.chat["id"]
while not self.stop.is_set():
for method, payload in (("POST", {"text": "load"}), ("GET", None)):
try:
st = common.request(method, path, token=self.owner["_token"], payload=payload, timeout=15).status
except Exception:
st = "conn_error"
key = "%s %s" % (method, st)
with self.lock:
self.counts[key] = self.counts.get(key, 0) + 1
time.sleep(0.05)
def snapshot(self):
with self.lock:
snap, self.counts = self.counts, {}
return snap
def __enter__(self):
for t in self.threads:
t.start()
return self
def __exit__(self, *exc):
self.stop.set()
def cmd_db_lock(args):
owner, chat = setup_chat()
lock_sql = "BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(%d); COMMIT;" % args.duration
cmd = args.compose + ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
"-v", "ON_ERROR_STOP=1", "-c", lock_sql]
with Load(owner, chat, args.threads) as load:
time.sleep(args.warmup)
note("baseline", **load.snapshot())
note("db_lock_start", duration_s=args.duration)
proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL)
while proc.poll() is None:
time.sleep(10)
note("during_lock", **load.snapshot())
note("db_lock_end", rc=proc.returncode)
time.sleep(args.warmup)
note("after", **load.snapshot())
return 0
def rss_of(service):
out = subprocess.run(["docker", "compose", "-p", args_project(), "ps", "-q", service],
capture_output=True, text=True).stdout.split()
if not out:
return {}
stats = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"] + out,
capture_output=True, text=True).stdout.strip().splitlines()
return dict(line.split(" ", 1) for line in stats)
def args_project():
import os
return os.environ.get("COMPOSE_PROJECT_NAME", "messenger-lab6")
def cmd_telemetry_down(args):
owner, chat = setup_chat()
services = args.services.split(",")
with Load(owner, chat, args.threads) as load:
time.sleep(args.warmup)
note("baseline", **load.snapshot(), memory=rss_of("api"))
subprocess.run(args.compose + ["stop"] + services, check=True, stdout=subprocess.DEVNULL)
note("telemetry_stopped", services=services)
end = time.monotonic() + args.duration
while time.monotonic() < end:
time.sleep(15)
note("telemetry_down", **load.snapshot(), memory=rss_of("api"))
subprocess.run(args.compose + ["start"] + services, check=True, stdout=subprocess.DEVNULL)
note("telemetry_started", services=services)
time.sleep(args.warmup)
note("after", **load.snapshot(), memory=rss_of("api"))
return 0
def main():
p = argparse.ArgumentParser()
p.add_argument("--base-url", default="http://127.0.0.1:8080")
sub = p.add_subparsers(dest="cmd", required=True)
pr = sub.add_parser("probe")
pr.add_argument("--filename", default="photo.png")
pr.add_argument("--timeout", type=float, default=60)
ws = sub.add_parser("worker-stop")
ws.add_argument("--duration", type=float, default=150)
ws.add_argument("--interval", type=float, default=5)
ws.add_argument("--stop-worker", default="%s scripts/ops.py worker-stop" % shlex.quote(PY))
ws.add_argument("--start-worker", default="%s scripts/ops.py worker-start" % shlex.quote(PY))
for name in ("db-lock", "telemetry-down"):
sp = sub.add_parser(name)
sp.add_argument("--duration", type=int, default=180 if name == "db-lock" else 90)
sp.add_argument("--threads", type=int, default=4)
sp.add_argument("--warmup", type=float, default=20)
sp.add_argument("--compose", type=shlex.split, default=shlex.split("docker compose --env-file .env"))
if name == "telemetry-down":
sp.add_argument("--services", default="otel-collector")
args = p.parse_args()
common.BASE_URL = args.base_url.rstrip("/")
import test_attachments
test_attachments.BASE_URL = common.BASE_URL
fn = {"probe": cmd_probe, "worker-stop": cmd_worker_stop, "db-lock": cmd_db_lock,
"telemetry-down": cmd_telemetry_down}[args.cmd]
sys.exit(fn(args))
if __name__ == "__main__":
main()