Лабораторная работа 6

This commit is contained in:
2026-11-13 01:27:57 +03:00
parent 470d9d6f1e
commit 2dbdcf898c
218 changed files with 22057 additions and 37 deletions
+226
View File
@@ -0,0 +1,226 @@
#!/usr/bin/env python3
"""lab6: контролируемый запрос и ограниченные отказы для проверки наблюдаемости.
Только стандартная библиотека; Docker управляется через scripts/ops.py / docker compose.
Печатает JSON-строки таймлайна (ID, статусы, счётчики) — без токенов и паролей.
python scripts/incident.py probe [--filename photo.png]
Один upload: печатает X-Request-Id, attachment_id и время до ready.
Затем в Grafana -> Explore -> Tempo (TraceQL):
{ span.messenger.request_id = "<X-Request-Id>" }
и в Loki: {service_name=~"messenger-.+"} | request_id="<X-Request-Id>"
python scripts/incident.py worker-stop [--duration 150]
Инцидент 1: worker остановлен, upload'ы продолжаются -> растут queue depth и
возраст старейшего задания -> QueueLagGrowing firing; worker запущен ->
очередь разбирается -> resolved.
python scripts/incident.py db-lock [--duration 180]
Инцидент 2: «медленная БД». В PostgreSQL держится ACCESS EXCLUSIVE lock на
таблице messages: запросы к сообщениям ждут до statement_timeout (5 с) и
получают 503 -> ApiHighErrorRate firing. В trace — SQL span ~5 с с ошибкой
QueryCanceled (а не долгий handler). Остальные маршруты работают.
python scripts/incident.py telemetry-down [--duration 90] [--services otel-collector]
Инцидент 3: telemetry backend недоступен. Под нагрузкой останавливаются
указанные сервисы; проверяется, что API отвечает, и RSS API не растёт.
"""
import argparse
import json
import shlex
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
HERE = Path(__file__).resolve().parent
ROOT = HERE.parent
sys.path.insert(0, str(ROOT / "student-tests"))
sys.path.insert(0, str(ROOT / "tests"))
import common # noqa: E402
from imagegen import sample_png # noqa: E402
from test_attachments import upload # noqa: E402
PY = sys.executable
T0 = time.monotonic()
def note(step, **kw):
kw = dict(step=step, t=round(time.monotonic() - T0, 1), **kw)
print(json.dumps(kw, ensure_ascii=False), flush=True)
def sh(cmd, check=True):
note("exec", cmd=cmd)
return subprocess.run(shlex.split(cmd), check=check, stdout=subprocess.DEVNULL if check else None)
def setup_chat():
owner = common.actor("obs")
chat = common.create_chat(owner)
return owner, chat
def one_upload(owner, chat, filename="photo.png", data=None):
rid = "probe-" + uuid.uuid4().hex[:12]
r = upload(owner["_token"], chat["id"], data or sample_png(), filename=filename,
headers={"X-Request-Id": rid})
return r, rid
def cmd_probe(args):
owner, chat = setup_chat()
started = time.monotonic()
r, rid = one_upload(owner, chat, args.filename)
note("upload", status=r.status, request_id=rid, attachment_id=(r.json or {}).get("id"),
instance=r.headers.get("X-Instance-Id"))
if r.status != 202:
return 1
ready = common.wait_ready(owner["_token"], r.json["id"], timeout=args.timeout, allow_failed=True)
note("final", status=(ready or {}).get("status"), seconds_to_final=round(time.monotonic() - started, 2))
print("\nTempo (TraceQL): { span.messenger.request_id = \"%s\" }" % rid)
print("Loki: {service_name=~\"messenger-.+\"} | request_id=\"%s\"" % rid)
return 0
def cmd_worker_stop(args):
owner, chat = setup_chat()
sh(args.stop_worker)
ids, end = [], time.monotonic() + args.duration
while time.monotonic() < end:
r, rid = one_upload(owner, chat)
ids.append(r.json["id"] if r.status == 202 else None)
note("upload_while_worker_down", status=r.status, request_id=rid)
time.sleep(args.interval)
sh(args.start_worker)
started = time.monotonic()
pending = [i for i in ids if i]
while pending and time.monotonic() - started < 300:
pending = [i for i in pending if common.request("GET", "/api/v1/attachments/" + i,
token=owner["_token"]).json.get("status") != "ready"]
time.sleep(2)
note("drained", uploads=len(ids), not_ready=len(pending), drain_seconds=round(time.monotonic() - started, 1))
return 0 if not pending else 1
class Load:
"""Постоянная нагрузка: POST сообщения + GET истории (closed loop, N потоков)."""
def __init__(self, owner, chat, threads):
self.owner, self.chat, self.stop = owner, chat, threading.Event()
self.counts, self.lock = {}, threading.Lock()
self.threads = [threading.Thread(target=self.run, daemon=True) for _ in range(threads)]
def run(self):
path = "/api/v1/chats/%s/messages" % self.chat["id"]
while not self.stop.is_set():
for method, payload in (("POST", {"text": "load"}), ("GET", None)):
try:
st = common.request(method, path, token=self.owner["_token"], payload=payload, timeout=15).status
except Exception:
st = "conn_error"
key = "%s %s" % (method, st)
with self.lock:
self.counts[key] = self.counts.get(key, 0) + 1
time.sleep(0.05)
def snapshot(self):
with self.lock:
snap, self.counts = self.counts, {}
return snap
def __enter__(self):
for t in self.threads:
t.start()
return self
def __exit__(self, *exc):
self.stop.set()
def cmd_db_lock(args):
owner, chat = setup_chat()
lock_sql = "BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(%d); COMMIT;" % args.duration
cmd = args.compose + ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
"-v", "ON_ERROR_STOP=1", "-c", lock_sql]
with Load(owner, chat, args.threads) as load:
time.sleep(args.warmup)
note("baseline", **load.snapshot())
note("db_lock_start", duration_s=args.duration)
proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL)
while proc.poll() is None:
time.sleep(10)
note("during_lock", **load.snapshot())
note("db_lock_end", rc=proc.returncode)
time.sleep(args.warmup)
note("after", **load.snapshot())
return 0
def rss_of(service):
out = subprocess.run(["docker", "compose", "-p", args_project(), "ps", "-q", service],
capture_output=True, text=True).stdout.split()
if not out:
return {}
stats = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"] + out,
capture_output=True, text=True).stdout.strip().splitlines()
return dict(line.split(" ", 1) for line in stats)
def args_project():
import os
return os.environ.get("COMPOSE_PROJECT_NAME", "messenger-lab6")
def cmd_telemetry_down(args):
owner, chat = setup_chat()
services = args.services.split(",")
with Load(owner, chat, args.threads) as load:
time.sleep(args.warmup)
note("baseline", **load.snapshot(), memory=rss_of("api"))
subprocess.run(args.compose + ["stop"] + services, check=True, stdout=subprocess.DEVNULL)
note("telemetry_stopped", services=services)
end = time.monotonic() + args.duration
while time.monotonic() < end:
time.sleep(15)
note("telemetry_down", **load.snapshot(), memory=rss_of("api"))
subprocess.run(args.compose + ["start"] + services, check=True, stdout=subprocess.DEVNULL)
note("telemetry_started", services=services)
time.sleep(args.warmup)
note("after", **load.snapshot(), memory=rss_of("api"))
return 0
def main():
p = argparse.ArgumentParser()
p.add_argument("--base-url", default="http://127.0.0.1:8080")
sub = p.add_subparsers(dest="cmd", required=True)
pr = sub.add_parser("probe")
pr.add_argument("--filename", default="photo.png")
pr.add_argument("--timeout", type=float, default=60)
ws = sub.add_parser("worker-stop")
ws.add_argument("--duration", type=float, default=150)
ws.add_argument("--interval", type=float, default=5)
ws.add_argument("--stop-worker", default="%s scripts/ops.py worker-stop" % shlex.quote(PY))
ws.add_argument("--start-worker", default="%s scripts/ops.py worker-start" % shlex.quote(PY))
for name in ("db-lock", "telemetry-down"):
sp = sub.add_parser(name)
sp.add_argument("--duration", type=int, default=180 if name == "db-lock" else 90)
sp.add_argument("--threads", type=int, default=4)
sp.add_argument("--warmup", type=float, default=20)
sp.add_argument("--compose", type=shlex.split, default=shlex.split("docker compose --env-file .env"))
if name == "telemetry-down":
sp.add_argument("--services", default="otel-collector")
args = p.parse_args()
common.BASE_URL = args.base_url.rstrip("/")
import test_attachments
test_attachments.BASE_URL = common.BASE_URL
fn = {"probe": cmd_probe, "worker-stop": cmd_worker_stop, "db-lock": cmd_db_lock,
"telemetry-down": cmd_telemetry_down}[args.cmd]
sys.exit(fn(args))
if __name__ == "__main__":
main()