#!/usr/bin/env python3 """lab6: контролируемый запрос и ограниченные отказы для проверки наблюдаемости. Только стандартная библиотека; Docker управляется через scripts/ops.py / docker compose. Печатает JSON-строки таймлайна (ID, статусы, счётчики) — без токенов и паролей. python scripts/incident.py probe [--filename photo.png] Один upload: печатает X-Request-Id, attachment_id и время до ready. Затем в Grafana -> Explore -> Tempo (TraceQL): { span.messenger.request_id = "" } и в Loki: {service_name=~"messenger-.+"} | request_id="" python scripts/incident.py worker-stop [--duration 150] Инцидент 1: worker остановлен, upload'ы продолжаются -> растут queue depth и возраст старейшего задания -> QueueLagGrowing firing; worker запущен -> очередь разбирается -> resolved. python scripts/incident.py db-lock [--duration 180] Инцидент 2: «медленная БД». В PostgreSQL держится ACCESS EXCLUSIVE lock на таблице messages: запросы к сообщениям ждут до statement_timeout (5 с) и получают 503 -> ApiHighErrorRate firing. В trace — SQL span ~5 с с ошибкой QueryCanceled (а не долгий handler). Остальные маршруты работают. python scripts/incident.py telemetry-down [--duration 90] [--services otel-collector] Инцидент 3: telemetry backend недоступен. Под нагрузкой останавливаются указанные сервисы; проверяется, что API отвечает, и RSS API не растёт. """ import argparse import json import shlex import subprocess import sys import threading import time import uuid from pathlib import Path HERE = Path(__file__).resolve().parent ROOT = HERE.parent sys.path.insert(0, str(ROOT / "student-tests")) sys.path.insert(0, str(ROOT / "tests")) import common # noqa: E402 from imagegen import sample_png # noqa: E402 from test_attachments import upload # noqa: E402 PY = sys.executable T0 = time.monotonic() def note(step, **kw): kw = dict(step=step, t=round(time.monotonic() - T0, 1), **kw) print(json.dumps(kw, ensure_ascii=False), flush=True) def sh(cmd, check=True): note("exec", cmd=cmd) return subprocess.run(shlex.split(cmd), check=check, stdout=subprocess.DEVNULL if check else None) def setup_chat(): owner = common.actor("obs") chat = common.create_chat(owner) return owner, chat def one_upload(owner, chat, filename="photo.png", data=None): rid = "probe-" + uuid.uuid4().hex[:12] r = upload(owner["_token"], chat["id"], data or sample_png(), filename=filename, headers={"X-Request-Id": rid}) return r, rid def cmd_probe(args): owner, chat = setup_chat() started = time.monotonic() r, rid = one_upload(owner, chat, args.filename) note("upload", status=r.status, request_id=rid, attachment_id=(r.json or {}).get("id"), instance=r.headers.get("X-Instance-Id")) if r.status != 202: return 1 ready = common.wait_ready(owner["_token"], r.json["id"], timeout=args.timeout, allow_failed=True) note("final", status=(ready or {}).get("status"), seconds_to_final=round(time.monotonic() - started, 2)) print("\nTempo (TraceQL): { span.messenger.request_id = \"%s\" }" % rid) print("Loki: {service_name=~\"messenger-.+\"} | request_id=\"%s\"" % rid) return 0 def cmd_worker_stop(args): owner, chat = setup_chat() sh(args.stop_worker) ids, end = [], time.monotonic() + args.duration while time.monotonic() < end: r, rid = one_upload(owner, chat) ids.append(r.json["id"] if r.status == 202 else None) note("upload_while_worker_down", status=r.status, request_id=rid) time.sleep(args.interval) sh(args.start_worker) started = time.monotonic() pending = [i for i in ids if i] while pending and time.monotonic() - started < 300: pending = [i for i in pending if common.request("GET", "/api/v1/attachments/" + i, token=owner["_token"]).json.get("status") != "ready"] time.sleep(2) note("drained", uploads=len(ids), not_ready=len(pending), drain_seconds=round(time.monotonic() - started, 1)) return 0 if not pending else 1 class Load: """Постоянная нагрузка: POST сообщения + GET истории (closed loop, N потоков).""" def __init__(self, owner, chat, threads): self.owner, self.chat, self.stop = owner, chat, threading.Event() self.counts, self.lock = {}, threading.Lock() self.threads = [threading.Thread(target=self.run, daemon=True) for _ in range(threads)] def run(self): path = "/api/v1/chats/%s/messages" % self.chat["id"] while not self.stop.is_set(): for method, payload in (("POST", {"text": "load"}), ("GET", None)): try: st = common.request(method, path, token=self.owner["_token"], payload=payload, timeout=15).status except Exception: st = "conn_error" key = "%s %s" % (method, st) with self.lock: self.counts[key] = self.counts.get(key, 0) + 1 time.sleep(0.05) def snapshot(self): with self.lock: snap, self.counts = self.counts, {} return snap def __enter__(self): for t in self.threads: t.start() return self def __exit__(self, *exc): self.stop.set() def cmd_db_lock(args): owner, chat = setup_chat() lock_sql = "BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(%d); COMMIT;" % args.duration cmd = args.compose + ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-v", "ON_ERROR_STOP=1", "-c", lock_sql] with Load(owner, chat, args.threads) as load: time.sleep(args.warmup) note("baseline", **load.snapshot()) note("db_lock_start", duration_s=args.duration) proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL) while proc.poll() is None: time.sleep(10) note("during_lock", **load.snapshot()) note("db_lock_end", rc=proc.returncode) time.sleep(args.warmup) note("after", **load.snapshot()) return 0 def rss_of(service): out = subprocess.run(["docker", "compose", "-p", args_project(), "ps", "-q", service], capture_output=True, text=True).stdout.split() if not out: return {} stats = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"] + out, capture_output=True, text=True).stdout.strip().splitlines() return dict(line.split(" ", 1) for line in stats) def args_project(): import os return os.environ.get("COMPOSE_PROJECT_NAME", "messenger-lab6") def cmd_telemetry_down(args): owner, chat = setup_chat() services = args.services.split(",") with Load(owner, chat, args.threads) as load: time.sleep(args.warmup) note("baseline", **load.snapshot(), memory=rss_of("api")) subprocess.run(args.compose + ["stop"] + services, check=True, stdout=subprocess.DEVNULL) note("telemetry_stopped", services=services) end = time.monotonic() + args.duration while time.monotonic() < end: time.sleep(15) note("telemetry_down", **load.snapshot(), memory=rss_of("api")) subprocess.run(args.compose + ["start"] + services, check=True, stdout=subprocess.DEVNULL) note("telemetry_started", services=services) time.sleep(args.warmup) note("after", **load.snapshot(), memory=rss_of("api")) return 0 def main(): p = argparse.ArgumentParser() p.add_argument("--base-url", default="http://127.0.0.1:8080") sub = p.add_subparsers(dest="cmd", required=True) pr = sub.add_parser("probe") pr.add_argument("--filename", default="photo.png") pr.add_argument("--timeout", type=float, default=60) ws = sub.add_parser("worker-stop") ws.add_argument("--duration", type=float, default=150) ws.add_argument("--interval", type=float, default=5) ws.add_argument("--stop-worker", default="%s scripts/ops.py worker-stop" % shlex.quote(PY)) ws.add_argument("--start-worker", default="%s scripts/ops.py worker-start" % shlex.quote(PY)) for name in ("db-lock", "telemetry-down"): sp = sub.add_parser(name) sp.add_argument("--duration", type=int, default=180 if name == "db-lock" else 90) sp.add_argument("--threads", type=int, default=4) sp.add_argument("--warmup", type=float, default=20) sp.add_argument("--compose", type=shlex.split, default=shlex.split("docker compose --env-file .env")) if name == "telemetry-down": sp.add_argument("--services", default="otel-collector") args = p.parse_args() common.BASE_URL = args.base_url.rstrip("/") import test_attachments test_attachments.BASE_URL = common.BASE_URL fn = {"probe": cmd_probe, "worker-stop": cmd_worker_stop, "db-lock": cmd_db_lock, "telemetry-down": cmd_telemetry_down}[args.cmd] sys.exit(fn(args)) if __name__ == "__main__": main()