227 lines
9.5 KiB
Python
227 lines
9.5 KiB
Python
#!/usr/bin/env python3
|
||
"""lab6: контролируемый запрос и ограниченные отказы для проверки наблюдаемости.
|
||
|
||
Только стандартная библиотека; Docker управляется через scripts/ops.py / docker compose.
|
||
Печатает JSON-строки таймлайна (ID, статусы, счётчики) — без токенов и паролей.
|
||
|
||
python scripts/incident.py probe [--filename photo.png]
|
||
Один upload: печатает X-Request-Id, attachment_id и время до ready.
|
||
Затем в Grafana -> Explore -> Tempo (TraceQL):
|
||
{ span.messenger.request_id = "<X-Request-Id>" }
|
||
и в Loki: {service_name=~"messenger-.+"} | request_id="<X-Request-Id>"
|
||
|
||
python scripts/incident.py worker-stop [--duration 150]
|
||
Инцидент 1: worker остановлен, upload'ы продолжаются -> растут queue depth и
|
||
возраст старейшего задания -> QueueLagGrowing firing; worker запущен ->
|
||
очередь разбирается -> resolved.
|
||
|
||
python scripts/incident.py db-lock [--duration 180]
|
||
Инцидент 2: «медленная БД». В PostgreSQL держится ACCESS EXCLUSIVE lock на
|
||
таблице messages: запросы к сообщениям ждут до statement_timeout (5 с) и
|
||
получают 503 -> ApiHighErrorRate firing. В trace — SQL span ~5 с с ошибкой
|
||
QueryCanceled (а не долгий handler). Остальные маршруты работают.
|
||
|
||
python scripts/incident.py telemetry-down [--duration 90] [--services otel-collector]
|
||
Инцидент 3: telemetry backend недоступен. Под нагрузкой останавливаются
|
||
указанные сервисы; проверяется, что API отвечает, и RSS API не растёт.
|
||
"""
|
||
import argparse
|
||
import json
|
||
import shlex
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import uuid
|
||
from pathlib import Path
|
||
|
||
HERE = Path(__file__).resolve().parent
|
||
ROOT = HERE.parent
|
||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||
sys.path.insert(0, str(ROOT / "tests"))
|
||
import common # noqa: E402
|
||
from imagegen import sample_png # noqa: E402
|
||
from test_attachments import upload # noqa: E402
|
||
|
||
PY = sys.executable
|
||
T0 = time.monotonic()
|
||
|
||
|
||
def note(step, **kw):
|
||
kw = dict(step=step, t=round(time.monotonic() - T0, 1), **kw)
|
||
print(json.dumps(kw, ensure_ascii=False), flush=True)
|
||
|
||
|
||
def sh(cmd, check=True):
|
||
note("exec", cmd=cmd)
|
||
return subprocess.run(shlex.split(cmd), check=check, stdout=subprocess.DEVNULL if check else None)
|
||
|
||
|
||
def setup_chat():
|
||
owner = common.actor("obs")
|
||
chat = common.create_chat(owner)
|
||
return owner, chat
|
||
|
||
|
||
def one_upload(owner, chat, filename="photo.png", data=None):
|
||
rid = "probe-" + uuid.uuid4().hex[:12]
|
||
r = upload(owner["_token"], chat["id"], data or sample_png(), filename=filename,
|
||
headers={"X-Request-Id": rid})
|
||
return r, rid
|
||
|
||
|
||
def cmd_probe(args):
|
||
owner, chat = setup_chat()
|
||
started = time.monotonic()
|
||
r, rid = one_upload(owner, chat, args.filename)
|
||
note("upload", status=r.status, request_id=rid, attachment_id=(r.json or {}).get("id"),
|
||
instance=r.headers.get("X-Instance-Id"))
|
||
if r.status != 202:
|
||
return 1
|
||
ready = common.wait_ready(owner["_token"], r.json["id"], timeout=args.timeout, allow_failed=True)
|
||
note("final", status=(ready or {}).get("status"), seconds_to_final=round(time.monotonic() - started, 2))
|
||
print("\nTempo (TraceQL): { span.messenger.request_id = \"%s\" }" % rid)
|
||
print("Loki: {service_name=~\"messenger-.+\"} | request_id=\"%s\"" % rid)
|
||
return 0
|
||
|
||
|
||
def cmd_worker_stop(args):
|
||
owner, chat = setup_chat()
|
||
sh(args.stop_worker)
|
||
ids, end = [], time.monotonic() + args.duration
|
||
while time.monotonic() < end:
|
||
r, rid = one_upload(owner, chat)
|
||
ids.append(r.json["id"] if r.status == 202 else None)
|
||
note("upload_while_worker_down", status=r.status, request_id=rid)
|
||
time.sleep(args.interval)
|
||
sh(args.start_worker)
|
||
started = time.monotonic()
|
||
pending = [i for i in ids if i]
|
||
while pending and time.monotonic() - started < 300:
|
||
pending = [i for i in pending if common.request("GET", "/api/v1/attachments/" + i,
|
||
token=owner["_token"]).json.get("status") != "ready"]
|
||
time.sleep(2)
|
||
note("drained", uploads=len(ids), not_ready=len(pending), drain_seconds=round(time.monotonic() - started, 1))
|
||
return 0 if not pending else 1
|
||
|
||
|
||
class Load:
|
||
"""Постоянная нагрузка: POST сообщения + GET истории (closed loop, N потоков)."""
|
||
|
||
def __init__(self, owner, chat, threads):
|
||
self.owner, self.chat, self.stop = owner, chat, threading.Event()
|
||
self.counts, self.lock = {}, threading.Lock()
|
||
self.threads = [threading.Thread(target=self.run, daemon=True) for _ in range(threads)]
|
||
|
||
def run(self):
|
||
path = "/api/v1/chats/%s/messages" % self.chat["id"]
|
||
while not self.stop.is_set():
|
||
for method, payload in (("POST", {"text": "load"}), ("GET", None)):
|
||
try:
|
||
st = common.request(method, path, token=self.owner["_token"], payload=payload, timeout=15).status
|
||
except Exception:
|
||
st = "conn_error"
|
||
key = "%s %s" % (method, st)
|
||
with self.lock:
|
||
self.counts[key] = self.counts.get(key, 0) + 1
|
||
time.sleep(0.05)
|
||
|
||
def snapshot(self):
|
||
with self.lock:
|
||
snap, self.counts = self.counts, {}
|
||
return snap
|
||
|
||
def __enter__(self):
|
||
for t in self.threads:
|
||
t.start()
|
||
return self
|
||
|
||
def __exit__(self, *exc):
|
||
self.stop.set()
|
||
|
||
|
||
def cmd_db_lock(args):
|
||
owner, chat = setup_chat()
|
||
lock_sql = "BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(%d); COMMIT;" % args.duration
|
||
cmd = args.compose + ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
|
||
"-v", "ON_ERROR_STOP=1", "-c", lock_sql]
|
||
with Load(owner, chat, args.threads) as load:
|
||
time.sleep(args.warmup)
|
||
note("baseline", **load.snapshot())
|
||
note("db_lock_start", duration_s=args.duration)
|
||
proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL)
|
||
while proc.poll() is None:
|
||
time.sleep(10)
|
||
note("during_lock", **load.snapshot())
|
||
note("db_lock_end", rc=proc.returncode)
|
||
time.sleep(args.warmup)
|
||
note("after", **load.snapshot())
|
||
return 0
|
||
|
||
|
||
def rss_of(service):
|
||
out = subprocess.run(["docker", "compose", "-p", args_project(), "ps", "-q", service],
|
||
capture_output=True, text=True).stdout.split()
|
||
if not out:
|
||
return {}
|
||
stats = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"] + out,
|
||
capture_output=True, text=True).stdout.strip().splitlines()
|
||
return dict(line.split(" ", 1) for line in stats)
|
||
|
||
|
||
def args_project():
|
||
import os
|
||
return os.environ.get("COMPOSE_PROJECT_NAME", "messenger-lab6")
|
||
|
||
|
||
def cmd_telemetry_down(args):
|
||
owner, chat = setup_chat()
|
||
services = args.services.split(",")
|
||
with Load(owner, chat, args.threads) as load:
|
||
time.sleep(args.warmup)
|
||
note("baseline", **load.snapshot(), memory=rss_of("api"))
|
||
subprocess.run(args.compose + ["stop"] + services, check=True, stdout=subprocess.DEVNULL)
|
||
note("telemetry_stopped", services=services)
|
||
end = time.monotonic() + args.duration
|
||
while time.monotonic() < end:
|
||
time.sleep(15)
|
||
note("telemetry_down", **load.snapshot(), memory=rss_of("api"))
|
||
subprocess.run(args.compose + ["start"] + services, check=True, stdout=subprocess.DEVNULL)
|
||
note("telemetry_started", services=services)
|
||
time.sleep(args.warmup)
|
||
note("after", **load.snapshot(), memory=rss_of("api"))
|
||
return 0
|
||
|
||
|
||
def main():
|
||
p = argparse.ArgumentParser()
|
||
p.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||
sub = p.add_subparsers(dest="cmd", required=True)
|
||
pr = sub.add_parser("probe")
|
||
pr.add_argument("--filename", default="photo.png")
|
||
pr.add_argument("--timeout", type=float, default=60)
|
||
ws = sub.add_parser("worker-stop")
|
||
ws.add_argument("--duration", type=float, default=150)
|
||
ws.add_argument("--interval", type=float, default=5)
|
||
ws.add_argument("--stop-worker", default="%s scripts/ops.py worker-stop" % shlex.quote(PY))
|
||
ws.add_argument("--start-worker", default="%s scripts/ops.py worker-start" % shlex.quote(PY))
|
||
for name in ("db-lock", "telemetry-down"):
|
||
sp = sub.add_parser(name)
|
||
sp.add_argument("--duration", type=int, default=180 if name == "db-lock" else 90)
|
||
sp.add_argument("--threads", type=int, default=4)
|
||
sp.add_argument("--warmup", type=float, default=20)
|
||
sp.add_argument("--compose", type=shlex.split, default=shlex.split("docker compose --env-file .env"))
|
||
if name == "telemetry-down":
|
||
sp.add_argument("--services", default="otel-collector")
|
||
args = p.parse_args()
|
||
common.BASE_URL = args.base_url.rstrip("/")
|
||
import test_attachments
|
||
test_attachments.BASE_URL = common.BASE_URL
|
||
fn = {"probe": cmd_probe, "worker-stop": cmd_worker_stop, "db-lock": cmd_db_lock,
|
||
"telemetry-down": cmd_telemetry_down}[args.cmd]
|
||
sys.exit(fn(args))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|