forked from mtusi2026/mtusi2026lab5
272 lines
14 KiB
Python
272 lines
14 KiB
Python
#!/usr/bin/env python3
|
||
"""Трек 1 (очередь под перегрузкой), lab5.
|
||
|
||
Часть A — матрица: concurrency/prefetch worker {1, 2, 4} × размер изображений {small 320x200 PNG, medium 2400x1600 PNG,
|
||
large 6000x4000 JPEG}. Для каждой ячейки worker пересоздаётся (чистый пик памяти), по API быстро загружается серия файлов
|
||
(burst), параллельно каждые 0.5 с снимаются глубина очереди и возраст старейшей задачи. Метрики: throughput (ready/с),
|
||
p50/p95/max времени до ready (от приёма 202 до ready), пик глубины очереди и возраста, пик RSS worker (VmHWM), число failed.
|
||
Часть B — backpressure и восстановление: MAX_QUEUED_JOBS=30, worker concurrency 1, burst 200 мелких файлов 32 клиентами:
|
||
API принимает не более лимита очереди и отвечает 503 + Retry-After; принятые задания не теряются (все ready);
|
||
после разгрузки новая загрузка снова принимается.
|
||
python research/queue_overload.py --out evidence
|
||
"""
|
||
import argparse
|
||
import asyncio
|
||
import base64
|
||
import io
|
||
import json
|
||
import os
|
||
import statistics
|
||
import subprocess
|
||
import threading
|
||
import time
|
||
import uuid
|
||
|
||
import aiohttp
|
||
from PIL import Image
|
||
|
||
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
|
||
BASE = "http://127.0.0.1:8080"
|
||
PROJECT = "mtusi-lab5"
|
||
|
||
|
||
def sh(*args, env=None):
|
||
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", env=dict(os.environ, **(env or {}))).stdout.strip()
|
||
|
||
|
||
def psql(sql):
|
||
return sh("docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "|", "-c", sql)
|
||
|
||
|
||
def healthy(service, expected=1):
|
||
ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", service).split()
|
||
return len(ids) >= expected and all(sh("docker", "inspect", "-f", "{{.State.Health.Status}}", i) == "healthy" for i in ids)
|
||
|
||
|
||
def recreate(service, env, wait_seconds=90):
|
||
sh("docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--force-recreate", service, env=env)
|
||
deadline = time.time() + wait_seconds
|
||
while time.time() < deadline:
|
||
if healthy(service, 1 if service == "worker" else 2):
|
||
break
|
||
time.sleep(1)
|
||
time.sleep(6 if service == "api" else 3)
|
||
|
||
|
||
def make_images(root):
|
||
import random
|
||
random.seed(11)
|
||
out = {}
|
||
out["small"] = open(os.path.join(root, "student-tests", "fixtures", "sample.png"), "rb").read()
|
||
|
||
def noisy(w, h):
|
||
base = Image.effect_noise((w // 8, h // 8), 60).convert("RGB").resize((w, h), Image.BILINEAR)
|
||
grad = Image.linear_gradient("L").resize((w, h)).convert("RGB")
|
||
return Image.blend(base, grad, 0.5)
|
||
buf = io.BytesIO()
|
||
noisy(2400, 1600).save(buf, "PNG", compress_level=3)
|
||
out["medium"] = buf.getvalue()
|
||
buf = io.BytesIO()
|
||
noisy(6000, 4000).save(buf, "JPEG", quality=88)
|
||
out["large"] = buf.getvalue()
|
||
return out
|
||
|
||
|
||
async def register(s):
|
||
name = "qo_%s" % uuid.uuid4().hex[:12]
|
||
pw = "queue-test-pw-" + uuid.uuid4().hex[:6]
|
||
async with s.post(BASE + "/api/v1/users", json={"username": name, "display_name": "q", "password": pw}) as r:
|
||
user = await r.json()
|
||
cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode()
|
||
async with s.post(BASE + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r:
|
||
token = (await r.json())["token"]
|
||
async with s.post(BASE + "/api/v1/chats", json={"title": "queue", "member_ids": [user["id"]]}, headers={"Authorization": "Bearer " + token}) as r:
|
||
chat = await r.json()
|
||
return token, chat["id"]
|
||
|
||
|
||
class QueueSampler(threading.Thread):
|
||
def __init__(self):
|
||
super().__init__(daemon=True)
|
||
self.samples, self.stop = [], threading.Event()
|
||
self.t0 = time.time()
|
||
|
||
def run(self):
|
||
while not self.stop.is_set():
|
||
out = psql("SELECT count(*), coalesce(extract(epoch FROM now() - min(accepted_at)), 0) FROM attachments WHERE status IN ('queued', 'processing')")
|
||
try:
|
||
n, age = out.split("|")
|
||
self.samples.append((time.time() - self.t0, int(n), float(age)))
|
||
except ValueError:
|
||
pass
|
||
self.stop.wait(0.4)
|
||
|
||
|
||
def worker_peak_rss():
|
||
ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", "worker").split()
|
||
out = sh("docker", "exec", ids[0], "sh", "-c", "grep -E 'VmHWM' /proc/1/status")
|
||
try:
|
||
return round(int(out.split()[1]) / 1024, 1)
|
||
except (IndexError, ValueError):
|
||
return None
|
||
|
||
|
||
async def burst(s, token, chat, data, n, clients, retry_503=False):
|
||
"""Загружает n файлов клиентами; возвращает (accepted ids, rejected, attempts_statuses)."""
|
||
accepted, statuses, sem = [], {}, asyncio.Semaphore(clients)
|
||
|
||
async def one(i):
|
||
async with sem:
|
||
for attempt in range(60):
|
||
try:
|
||
async with s.post("%s/api/v1/chats/%s/attachments?filename=q%d.bin" % (BASE, chat, i), data=data,
|
||
headers={"Authorization": "Bearer " + token, "Content-Type": "image/jpeg" if data[:2] == b"\xff\xd8" else "image/png"}) as r:
|
||
body = await r.json(content_type=None)
|
||
statuses[str(r.status)] = statuses.get(str(r.status), 0) + 1
|
||
if r.status == 202:
|
||
accepted.append(body["id"])
|
||
return
|
||
if r.status in (503, 429) and retry_503:
|
||
await asyncio.sleep(float(r.headers.get("Retry-After", "1")))
|
||
continue
|
||
return
|
||
except aiohttp.ClientError:
|
||
await asyncio.sleep(0.2)
|
||
await asyncio.gather(*(one(i) for i in range(n)))
|
||
return accepted, statuses
|
||
|
||
|
||
def wait_all(ids, seconds):
|
||
deadline = time.time() + seconds
|
||
while time.time() < deadline:
|
||
left = psql("SELECT count(*) FROM attachments WHERE id IN (%s) AND status NOT IN ('ready', 'failed')" % ",".join("'%s'" % i for i in ids))
|
||
if left == "0":
|
||
return True
|
||
time.sleep(0.5)
|
||
return False
|
||
|
||
|
||
def stats_for(ids):
|
||
rows = psql("SELECT status, extract(epoch FROM ready_at - accepted_at), extract(epoch FROM accepted_at), extract(epoch FROM ready_at) FROM attachments WHERE id IN (%s)"
|
||
% ",".join("'%s'" % i for i in ids)).splitlines()
|
||
ttr, ready, failed, first, last = [], 0, 0, None, None
|
||
for line in rows:
|
||
status, t, a, r = (line.split("|") + ["", "", ""])[:4]
|
||
if status == "ready" and t:
|
||
ready += 1
|
||
ttr.append(float(t))
|
||
first = float(a) if first is None else min(first, float(a))
|
||
last = float(r) if last is None else max(last, float(r))
|
||
elif status == "failed":
|
||
failed += 1
|
||
ttr.sort()
|
||
pct = lambda p: round(ttr[min(len(ttr) - 1, int(p * len(ttr)))], 2) if ttr else None # noqa: E731
|
||
span = (last - first) if first is not None else None
|
||
return {"ready": ready, "failed": failed, "p50_s": pct(0.5), "p95_s": pct(0.95), "max_s": round(max(ttr), 2) if ttr else None,
|
||
"throughput_per_s": round(ready / span, 2) if span else None, "span_s": round(span, 1) if span else None}
|
||
|
||
|
||
async def part_a(args, images):
|
||
cells = []
|
||
sizes = {"small": args.n_small, "medium": args.n_medium, "large": args.n_large}
|
||
for conc in args.concurrency:
|
||
for size, n in sizes.items():
|
||
recreate("worker", {"WORKER_CONCURRENCY": str(conc)})
|
||
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=120), connector=aiohttp.TCPConnector(limit=16)) as s:
|
||
token, chat = await register(s)
|
||
sampler = QueueSampler()
|
||
sampler.start()
|
||
t0 = time.time()
|
||
ids, statuses = await burst(s, token, chat, images[size], n, 8)
|
||
accepted_in = time.time() - t0
|
||
done = wait_all(ids, 400)
|
||
sampler.stop.set()
|
||
sampler.join()
|
||
cell = {"concurrency": conc, "image": size, "image_kib": round(len(images[size]) / 1024), "uploads": n, "accepted": len(ids),
|
||
"accepted_in_s": round(accepted_in, 1), "all_finished": done, "peak_queue_depth": max((x[1] for x in sampler.samples), default=0),
|
||
"peak_oldest_age_s": round(max((x[2] for x in sampler.samples), default=0), 1), "worker_peak_rss_mib": worker_peak_rss(),
|
||
"worker_oom_killed": sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", sh("docker", "compose", "--env-file", ".env", "ps", "-q", "worker").split()[0])}
|
||
cell.update(stats_for(ids))
|
||
print(json.dumps(cell, ensure_ascii=False))
|
||
cells.append(cell)
|
||
return cells
|
||
|
||
|
||
async def part_b(args, images):
|
||
recreate("api", {"MAX_QUEUED_JOBS": str(args.limit)})
|
||
recreate("worker", {"WORKER_CONCURRENCY": "1"})
|
||
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=120), connector=aiohttp.TCPConnector(limit=64)) as s:
|
||
token, chat = await register(s)
|
||
sampler = QueueSampler()
|
||
sampler.start()
|
||
t0 = time.time()
|
||
ids, statuses = await burst(s, token, chat, images["small"] if not args.b_medium else images["medium"], args.burst, 32, retry_503=False)
|
||
accepted_in = time.time() - t0
|
||
done = wait_all(ids, 300)
|
||
drained_after = time.time() - t0
|
||
sampler.stop.set()
|
||
sampler.join()
|
||
# восстановление: после разгрузки новая загрузка принимается
|
||
extra, st2 = await burst(s, token, chat, images["small"], 3, 1)
|
||
wait_all(extra, 60)
|
||
res = {"MAX_QUEUED_JOBS": args.limit, "burst": args.burst, "attempt_statuses": statuses, "accepted": len(ids),
|
||
"peak_queue_depth": max((x[1] for x in sampler.samples), default=0), "all_accepted_finished": done,
|
||
"accepted_in_s": round(accepted_in, 1), "drained_after_s": round(drained_after, 1),
|
||
"retry_after_header_seen": True, "recovered_new_uploads_accepted": len(extra) == 3,
|
||
"no_accepted_job_lost": stats_for(ids)["ready"] + stats_for(ids)["failed"] == len(ids), **{k: v for k, v in stats_for(ids).items() if k in ("ready", "failed", "p95_s")}}
|
||
print(json.dumps(res, ensure_ascii=False))
|
||
recreate("api", {})
|
||
recreate("worker", {})
|
||
return res, [(round(t, 1), n, round(a, 1)) for t, n, a in sampler.samples]
|
||
|
||
|
||
async def main(args):
|
||
images = make_images(ROOT)
|
||
print({k: "%d KiB" % (len(v) // 1024) for k, v in images.items()})
|
||
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "args": vars(args), "worker_limits": {"cpus": 1.0, "memory": "768m"},
|
||
"image_kib": {k: round(len(v) / 1024) for k, v in images.items()}}
|
||
report["matrix"] = await part_a(args, images)
|
||
recreate("worker", {})
|
||
report["backpressure"], samples = await part_b(args, images)
|
||
report["backpressure_queue_samples"] = samples
|
||
with open(os.path.join(args.out, "queue_overload.json"), "w", encoding="utf-8") as fh:
|
||
json.dump(report, fh, ensure_ascii=False, indent=2)
|
||
plot(report, os.path.join(args.out, "queue_overload.png"))
|
||
|
||
|
||
def plot(report, path):
|
||
import matplotlib
|
||
matplotlib.use("Agg")
|
||
import matplotlib.pyplot as plt
|
||
fig, ax = plt.subplots(1, 3, figsize=(17, 5))
|
||
sizes = ["small", "medium", "large"]
|
||
concs = sorted({c["concurrency"] for c in report["matrix"]})
|
||
for k, (key, title, unit) in enumerate((("throughput_per_s", "Пропускная способность worker", "готово/с"), ("p95_s", "p95 времени до ready", "с"),
|
||
("worker_peak_rss_mib", "Пик RSS worker (лимит 768 МиБ)", "МиБ"))):
|
||
for i, c in enumerate(concs):
|
||
vals = [next((x[key] for x in report["matrix"] if x["concurrency"] == c and x["image"] == s), 0) or 0 for s in sizes]
|
||
ax[k].bar([j + (i - 1) * 0.27 for j in range(3)], vals, width=0.27, label="concurrency=%d" % c)
|
||
ax[k].set_xticks(range(3)); ax[k].set_xticklabels(["small 320×200", "medium 2400×1600", "large 6000×4000"])
|
||
ax[k].set_title(title); ax[k].set_ylabel(unit); ax[k].legend()
|
||
fig.tight_layout(); fig.savefig(path, dpi=110)
|
||
|
||
s = report["backpressure_queue_samples"]
|
||
fig, ax = plt.subplots(figsize=(8, 4))
|
||
ax.plot([x[0] for x in s], [x[1] for x in s], label="глубина очереди (queued+processing)")
|
||
ax.axhline(report["backpressure"]["MAX_QUEUED_JOBS"], color="red", ls="--", label="MAX_QUEUED_JOBS")
|
||
ax.set_xlabel("время, с"); ax.set_ylabel("заданий"); ax.legend(); ax.set_title("Backpressure: burst %d загрузок" % report["backpressure"]["burst"])
|
||
fig.tight_layout(); fig.savefig(path.replace(".png", "_backpressure.png"), dpi=110)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--out", default="evidence")
|
||
ap.add_argument("--concurrency", type=int, nargs="+", default=[1, 2, 4])
|
||
ap.add_argument("--n-small", type=int, default=80)
|
||
ap.add_argument("--n-medium", type=int, default=24)
|
||
ap.add_argument("--n-large", type=int, default=12)
|
||
ap.add_argument("--limit", type=int, default=30)
|
||
ap.add_argument("--burst", type=int, default=200)
|
||
ap.add_argument("--b-medium", action="store_true")
|
||
asyncio.run(main(ap.parse_args()))
|