forked from mtusi2026/mtusi2026lab6
Лабораторная работа 6
This commit is contained in:
@@ -0,0 +1,226 @@
|
||||
#!/usr/bin/env python3
|
||||
"""lab6: контролируемый запрос и ограниченные отказы для проверки наблюдаемости.
|
||||
|
||||
Только стандартная библиотека; Docker управляется через scripts/ops.py / docker compose.
|
||||
Печатает JSON-строки таймлайна (ID, статусы, счётчики) — без токенов и паролей.
|
||||
|
||||
python scripts/incident.py probe [--filename photo.png]
|
||||
Один upload: печатает X-Request-Id, attachment_id и время до ready.
|
||||
Затем в Grafana -> Explore -> Tempo (TraceQL):
|
||||
{ span.messenger.request_id = "<X-Request-Id>" }
|
||||
и в Loki: {service_name=~"messenger-.+"} | request_id="<X-Request-Id>"
|
||||
|
||||
python scripts/incident.py worker-stop [--duration 150]
|
||||
Инцидент 1: worker остановлен, upload'ы продолжаются -> растут queue depth и
|
||||
возраст старейшего задания -> QueueLagGrowing firing; worker запущен ->
|
||||
очередь разбирается -> resolved.
|
||||
|
||||
python scripts/incident.py db-lock [--duration 180]
|
||||
Инцидент 2: «медленная БД». В PostgreSQL держится ACCESS EXCLUSIVE lock на
|
||||
таблице messages: запросы к сообщениям ждут до statement_timeout (5 с) и
|
||||
получают 503 -> ApiHighErrorRate firing. В trace — SQL span ~5 с с ошибкой
|
||||
QueryCanceled (а не долгий handler). Остальные маршруты работают.
|
||||
|
||||
python scripts/incident.py telemetry-down [--duration 90] [--services otel-collector]
|
||||
Инцидент 3: telemetry backend недоступен. Под нагрузкой останавливаются
|
||||
указанные сервисы; проверяется, что API отвечает, и RSS API не растёт.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ROOT = HERE.parent
|
||||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
import common # noqa: E402
|
||||
from imagegen import sample_png # noqa: E402
|
||||
from test_attachments import upload # noqa: E402
|
||||
|
||||
PY = sys.executable
|
||||
T0 = time.monotonic()
|
||||
|
||||
|
||||
def note(step, **kw):
|
||||
kw = dict(step=step, t=round(time.monotonic() - T0, 1), **kw)
|
||||
print(json.dumps(kw, ensure_ascii=False), flush=True)
|
||||
|
||||
|
||||
def sh(cmd, check=True):
|
||||
note("exec", cmd=cmd)
|
||||
return subprocess.run(shlex.split(cmd), check=check, stdout=subprocess.DEVNULL if check else None)
|
||||
|
||||
|
||||
def setup_chat():
|
||||
owner = common.actor("obs")
|
||||
chat = common.create_chat(owner)
|
||||
return owner, chat
|
||||
|
||||
|
||||
def one_upload(owner, chat, filename="photo.png", data=None):
|
||||
rid = "probe-" + uuid.uuid4().hex[:12]
|
||||
r = upload(owner["_token"], chat["id"], data or sample_png(), filename=filename,
|
||||
headers={"X-Request-Id": rid})
|
||||
return r, rid
|
||||
|
||||
|
||||
def cmd_probe(args):
|
||||
owner, chat = setup_chat()
|
||||
started = time.monotonic()
|
||||
r, rid = one_upload(owner, chat, args.filename)
|
||||
note("upload", status=r.status, request_id=rid, attachment_id=(r.json or {}).get("id"),
|
||||
instance=r.headers.get("X-Instance-Id"))
|
||||
if r.status != 202:
|
||||
return 1
|
||||
ready = common.wait_ready(owner["_token"], r.json["id"], timeout=args.timeout, allow_failed=True)
|
||||
note("final", status=(ready or {}).get("status"), seconds_to_final=round(time.monotonic() - started, 2))
|
||||
print("\nTempo (TraceQL): { span.messenger.request_id = \"%s\" }" % rid)
|
||||
print("Loki: {service_name=~\"messenger-.+\"} | request_id=\"%s\"" % rid)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_worker_stop(args):
|
||||
owner, chat = setup_chat()
|
||||
sh(args.stop_worker)
|
||||
ids, end = [], time.monotonic() + args.duration
|
||||
while time.monotonic() < end:
|
||||
r, rid = one_upload(owner, chat)
|
||||
ids.append(r.json["id"] if r.status == 202 else None)
|
||||
note("upload_while_worker_down", status=r.status, request_id=rid)
|
||||
time.sleep(args.interval)
|
||||
sh(args.start_worker)
|
||||
started = time.monotonic()
|
||||
pending = [i for i in ids if i]
|
||||
while pending and time.monotonic() - started < 300:
|
||||
pending = [i for i in pending if common.request("GET", "/api/v1/attachments/" + i,
|
||||
token=owner["_token"]).json.get("status") != "ready"]
|
||||
time.sleep(2)
|
||||
note("drained", uploads=len(ids), not_ready=len(pending), drain_seconds=round(time.monotonic() - started, 1))
|
||||
return 0 if not pending else 1
|
||||
|
||||
|
||||
class Load:
|
||||
"""Постоянная нагрузка: POST сообщения + GET истории (closed loop, N потоков)."""
|
||||
|
||||
def __init__(self, owner, chat, threads):
|
||||
self.owner, self.chat, self.stop = owner, chat, threading.Event()
|
||||
self.counts, self.lock = {}, threading.Lock()
|
||||
self.threads = [threading.Thread(target=self.run, daemon=True) for _ in range(threads)]
|
||||
|
||||
def run(self):
|
||||
path = "/api/v1/chats/%s/messages" % self.chat["id"]
|
||||
while not self.stop.is_set():
|
||||
for method, payload in (("POST", {"text": "load"}), ("GET", None)):
|
||||
try:
|
||||
st = common.request(method, path, token=self.owner["_token"], payload=payload, timeout=15).status
|
||||
except Exception:
|
||||
st = "conn_error"
|
||||
key = "%s %s" % (method, st)
|
||||
with self.lock:
|
||||
self.counts[key] = self.counts.get(key, 0) + 1
|
||||
time.sleep(0.05)
|
||||
|
||||
def snapshot(self):
|
||||
with self.lock:
|
||||
snap, self.counts = self.counts, {}
|
||||
return snap
|
||||
|
||||
def __enter__(self):
|
||||
for t in self.threads:
|
||||
t.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
self.stop.set()
|
||||
|
||||
|
||||
def cmd_db_lock(args):
|
||||
owner, chat = setup_chat()
|
||||
lock_sql = "BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(%d); COMMIT;" % args.duration
|
||||
cmd = args.compose + ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
|
||||
"-v", "ON_ERROR_STOP=1", "-c", lock_sql]
|
||||
with Load(owner, chat, args.threads) as load:
|
||||
time.sleep(args.warmup)
|
||||
note("baseline", **load.snapshot())
|
||||
note("db_lock_start", duration_s=args.duration)
|
||||
proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL)
|
||||
while proc.poll() is None:
|
||||
time.sleep(10)
|
||||
note("during_lock", **load.snapshot())
|
||||
note("db_lock_end", rc=proc.returncode)
|
||||
time.sleep(args.warmup)
|
||||
note("after", **load.snapshot())
|
||||
return 0
|
||||
|
||||
|
||||
def rss_of(service):
|
||||
out = subprocess.run(["docker", "compose", "-p", args_project(), "ps", "-q", service],
|
||||
capture_output=True, text=True).stdout.split()
|
||||
if not out:
|
||||
return {}
|
||||
stats = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"] + out,
|
||||
capture_output=True, text=True).stdout.strip().splitlines()
|
||||
return dict(line.split(" ", 1) for line in stats)
|
||||
|
||||
|
||||
def args_project():
|
||||
import os
|
||||
return os.environ.get("COMPOSE_PROJECT_NAME", "messenger-lab6")
|
||||
|
||||
|
||||
def cmd_telemetry_down(args):
|
||||
owner, chat = setup_chat()
|
||||
services = args.services.split(",")
|
||||
with Load(owner, chat, args.threads) as load:
|
||||
time.sleep(args.warmup)
|
||||
note("baseline", **load.snapshot(), memory=rss_of("api"))
|
||||
subprocess.run(args.compose + ["stop"] + services, check=True, stdout=subprocess.DEVNULL)
|
||||
note("telemetry_stopped", services=services)
|
||||
end = time.monotonic() + args.duration
|
||||
while time.monotonic() < end:
|
||||
time.sleep(15)
|
||||
note("telemetry_down", **load.snapshot(), memory=rss_of("api"))
|
||||
subprocess.run(args.compose + ["start"] + services, check=True, stdout=subprocess.DEVNULL)
|
||||
note("telemetry_started", services=services)
|
||||
time.sleep(args.warmup)
|
||||
note("after", **load.snapshot(), memory=rss_of("api"))
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
sub = p.add_subparsers(dest="cmd", required=True)
|
||||
pr = sub.add_parser("probe")
|
||||
pr.add_argument("--filename", default="photo.png")
|
||||
pr.add_argument("--timeout", type=float, default=60)
|
||||
ws = sub.add_parser("worker-stop")
|
||||
ws.add_argument("--duration", type=float, default=150)
|
||||
ws.add_argument("--interval", type=float, default=5)
|
||||
ws.add_argument("--stop-worker", default="%s scripts/ops.py worker-stop" % shlex.quote(PY))
|
||||
ws.add_argument("--start-worker", default="%s scripts/ops.py worker-start" % shlex.quote(PY))
|
||||
for name in ("db-lock", "telemetry-down"):
|
||||
sp = sub.add_parser(name)
|
||||
sp.add_argument("--duration", type=int, default=180 if name == "db-lock" else 90)
|
||||
sp.add_argument("--threads", type=int, default=4)
|
||||
sp.add_argument("--warmup", type=float, default=20)
|
||||
sp.add_argument("--compose", type=shlex.split, default=shlex.split("docker compose --env-file .env"))
|
||||
if name == "telemetry-down":
|
||||
sp.add_argument("--services", default="otel-collector")
|
||||
args = p.parse_args()
|
||||
common.BASE_URL = args.base_url.rstrip("/")
|
||||
import test_attachments
|
||||
test_attachments.BASE_URL = common.BASE_URL
|
||||
fn = {"probe": cmd_probe, "worker-stop": cmd_worker_stop, "db-lock": cmd_db_lock,
|
||||
"telemetry-down": cmd_telemetry_down}[args.cmd]
|
||||
sys.exit(fn(args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user