forked from mtusi2026/mtusi2026lab5
ЛР5: брокер RabbitMQ, outbox, worker, варианты изображений, отказоустойчивость, исследование очереди
This commit is contained in:
@@ -0,0 +1,271 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Трек 1 (очередь под перегрузкой), lab5.
|
||||
|
||||
Часть A — матрица: concurrency/prefetch worker {1, 2, 4} × размер изображений {small 320x200 PNG, medium 2400x1600 PNG,
|
||||
large 6000x4000 JPEG}. Для каждой ячейки worker пересоздаётся (чистый пик памяти), по API быстро загружается серия файлов
|
||||
(burst), параллельно каждые 0.5 с снимаются глубина очереди и возраст старейшей задачи. Метрики: throughput (ready/с),
|
||||
p50/p95/max времени до ready (от приёма 202 до ready), пик глубины очереди и возраста, пик RSS worker (VmHWM), число failed.
|
||||
Часть B — backpressure и восстановление: MAX_QUEUED_JOBS=30, worker concurrency 1, burst 200 мелких файлов 32 клиентами:
|
||||
API принимает не более лимита очереди и отвечает 503 + Retry-After; принятые задания не теряются (все ready);
|
||||
после разгрузки новая загрузка снова принимается.
|
||||
python research/queue_overload.py --out evidence
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import aiohttp
|
||||
from PIL import Image
|
||||
|
||||
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
|
||||
BASE = "http://127.0.0.1:8080"
|
||||
PROJECT = "mtusi-lab5"
|
||||
|
||||
|
||||
def sh(*args, env=None):
|
||||
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", env=dict(os.environ, **(env or {}))).stdout.strip()
|
||||
|
||||
|
||||
def psql(sql):
|
||||
return sh("docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "|", "-c", sql)
|
||||
|
||||
|
||||
def healthy(service, expected=1):
|
||||
ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", service).split()
|
||||
return len(ids) >= expected and all(sh("docker", "inspect", "-f", "{{.State.Health.Status}}", i) == "healthy" for i in ids)
|
||||
|
||||
|
||||
def recreate(service, env, wait_seconds=90):
|
||||
sh("docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--force-recreate", service, env=env)
|
||||
deadline = time.time() + wait_seconds
|
||||
while time.time() < deadline:
|
||||
if healthy(service, 1 if service == "worker" else 2):
|
||||
break
|
||||
time.sleep(1)
|
||||
time.sleep(6 if service == "api" else 3)
|
||||
|
||||
|
||||
def make_images(root):
|
||||
import random
|
||||
random.seed(11)
|
||||
out = {}
|
||||
out["small"] = open(os.path.join(root, "student-tests", "fixtures", "sample.png"), "rb").read()
|
||||
|
||||
def noisy(w, h):
|
||||
base = Image.effect_noise((w // 8, h // 8), 60).convert("RGB").resize((w, h), Image.BILINEAR)
|
||||
grad = Image.linear_gradient("L").resize((w, h)).convert("RGB")
|
||||
return Image.blend(base, grad, 0.5)
|
||||
buf = io.BytesIO()
|
||||
noisy(2400, 1600).save(buf, "PNG", compress_level=3)
|
||||
out["medium"] = buf.getvalue()
|
||||
buf = io.BytesIO()
|
||||
noisy(6000, 4000).save(buf, "JPEG", quality=88)
|
||||
out["large"] = buf.getvalue()
|
||||
return out
|
||||
|
||||
|
||||
async def register(s):
|
||||
name = "qo_%s" % uuid.uuid4().hex[:12]
|
||||
pw = "queue-test-pw-" + uuid.uuid4().hex[:6]
|
||||
async with s.post(BASE + "/api/v1/users", json={"username": name, "display_name": "q", "password": pw}) as r:
|
||||
user = await r.json()
|
||||
cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode()
|
||||
async with s.post(BASE + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r:
|
||||
token = (await r.json())["token"]
|
||||
async with s.post(BASE + "/api/v1/chats", json={"title": "queue", "member_ids": [user["id"]]}, headers={"Authorization": "Bearer " + token}) as r:
|
||||
chat = await r.json()
|
||||
return token, chat["id"]
|
||||
|
||||
|
||||
class QueueSampler(threading.Thread):
|
||||
def __init__(self):
|
||||
super().__init__(daemon=True)
|
||||
self.samples, self.stop = [], threading.Event()
|
||||
self.t0 = time.time()
|
||||
|
||||
def run(self):
|
||||
while not self.stop.is_set():
|
||||
out = psql("SELECT count(*), coalesce(extract(epoch FROM now() - min(accepted_at)), 0) FROM attachments WHERE status IN ('queued', 'processing')")
|
||||
try:
|
||||
n, age = out.split("|")
|
||||
self.samples.append((time.time() - self.t0, int(n), float(age)))
|
||||
except ValueError:
|
||||
pass
|
||||
self.stop.wait(0.4)
|
||||
|
||||
|
||||
def worker_peak_rss():
|
||||
ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", "worker").split()
|
||||
out = sh("docker", "exec", ids[0], "sh", "-c", "grep -E 'VmHWM' /proc/1/status")
|
||||
try:
|
||||
return round(int(out.split()[1]) / 1024, 1)
|
||||
except (IndexError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
async def burst(s, token, chat, data, n, clients, retry_503=False):
|
||||
"""Загружает n файлов клиентами; возвращает (accepted ids, rejected, attempts_statuses)."""
|
||||
accepted, statuses, sem = [], {}, asyncio.Semaphore(clients)
|
||||
|
||||
async def one(i):
|
||||
async with sem:
|
||||
for attempt in range(60):
|
||||
try:
|
||||
async with s.post("%s/api/v1/chats/%s/attachments?filename=q%d.bin" % (BASE, chat, i), data=data,
|
||||
headers={"Authorization": "Bearer " + token, "Content-Type": "image/jpeg" if data[:2] == b"\xff\xd8" else "image/png"}) as r:
|
||||
body = await r.json(content_type=None)
|
||||
statuses[str(r.status)] = statuses.get(str(r.status), 0) + 1
|
||||
if r.status == 202:
|
||||
accepted.append(body["id"])
|
||||
return
|
||||
if r.status in (503, 429) and retry_503:
|
||||
await asyncio.sleep(float(r.headers.get("Retry-After", "1")))
|
||||
continue
|
||||
return
|
||||
except aiohttp.ClientError:
|
||||
await asyncio.sleep(0.2)
|
||||
await asyncio.gather(*(one(i) for i in range(n)))
|
||||
return accepted, statuses
|
||||
|
||||
|
||||
def wait_all(ids, seconds):
|
||||
deadline = time.time() + seconds
|
||||
while time.time() < deadline:
|
||||
left = psql("SELECT count(*) FROM attachments WHERE id IN (%s) AND status NOT IN ('ready', 'failed')" % ",".join("'%s'" % i for i in ids))
|
||||
if left == "0":
|
||||
return True
|
||||
time.sleep(0.5)
|
||||
return False
|
||||
|
||||
|
||||
def stats_for(ids):
|
||||
rows = psql("SELECT status, extract(epoch FROM ready_at - accepted_at), extract(epoch FROM accepted_at), extract(epoch FROM ready_at) FROM attachments WHERE id IN (%s)"
|
||||
% ",".join("'%s'" % i for i in ids)).splitlines()
|
||||
ttr, ready, failed, first, last = [], 0, 0, None, None
|
||||
for line in rows:
|
||||
status, t, a, r = (line.split("|") + ["", "", ""])[:4]
|
||||
if status == "ready" and t:
|
||||
ready += 1
|
||||
ttr.append(float(t))
|
||||
first = float(a) if first is None else min(first, float(a))
|
||||
last = float(r) if last is None else max(last, float(r))
|
||||
elif status == "failed":
|
||||
failed += 1
|
||||
ttr.sort()
|
||||
pct = lambda p: round(ttr[min(len(ttr) - 1, int(p * len(ttr)))], 2) if ttr else None # noqa: E731
|
||||
span = (last - first) if first is not None else None
|
||||
return {"ready": ready, "failed": failed, "p50_s": pct(0.5), "p95_s": pct(0.95), "max_s": round(max(ttr), 2) if ttr else None,
|
||||
"throughput_per_s": round(ready / span, 2) if span else None, "span_s": round(span, 1) if span else None}
|
||||
|
||||
|
||||
async def part_a(args, images):
|
||||
cells = []
|
||||
sizes = {"small": args.n_small, "medium": args.n_medium, "large": args.n_large}
|
||||
for conc in args.concurrency:
|
||||
for size, n in sizes.items():
|
||||
recreate("worker", {"WORKER_CONCURRENCY": str(conc)})
|
||||
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=120), connector=aiohttp.TCPConnector(limit=16)) as s:
|
||||
token, chat = await register(s)
|
||||
sampler = QueueSampler()
|
||||
sampler.start()
|
||||
t0 = time.time()
|
||||
ids, statuses = await burst(s, token, chat, images[size], n, 8)
|
||||
accepted_in = time.time() - t0
|
||||
done = wait_all(ids, 400)
|
||||
sampler.stop.set()
|
||||
sampler.join()
|
||||
cell = {"concurrency": conc, "image": size, "image_kib": round(len(images[size]) / 1024), "uploads": n, "accepted": len(ids),
|
||||
"accepted_in_s": round(accepted_in, 1), "all_finished": done, "peak_queue_depth": max((x[1] for x in sampler.samples), default=0),
|
||||
"peak_oldest_age_s": round(max((x[2] for x in sampler.samples), default=0), 1), "worker_peak_rss_mib": worker_peak_rss(),
|
||||
"worker_oom_killed": sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", sh("docker", "compose", "--env-file", ".env", "ps", "-q", "worker").split()[0])}
|
||||
cell.update(stats_for(ids))
|
||||
print(json.dumps(cell, ensure_ascii=False))
|
||||
cells.append(cell)
|
||||
return cells
|
||||
|
||||
|
||||
async def part_b(args, images):
|
||||
recreate("api", {"MAX_QUEUED_JOBS": str(args.limit)})
|
||||
recreate("worker", {"WORKER_CONCURRENCY": "1"})
|
||||
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=120), connector=aiohttp.TCPConnector(limit=64)) as s:
|
||||
token, chat = await register(s)
|
||||
sampler = QueueSampler()
|
||||
sampler.start()
|
||||
t0 = time.time()
|
||||
ids, statuses = await burst(s, token, chat, images["small"] if not args.b_medium else images["medium"], args.burst, 32, retry_503=False)
|
||||
accepted_in = time.time() - t0
|
||||
done = wait_all(ids, 300)
|
||||
drained_after = time.time() - t0
|
||||
sampler.stop.set()
|
||||
sampler.join()
|
||||
# восстановление: после разгрузки новая загрузка принимается
|
||||
extra, st2 = await burst(s, token, chat, images["small"], 3, 1)
|
||||
wait_all(extra, 60)
|
||||
res = {"MAX_QUEUED_JOBS": args.limit, "burst": args.burst, "attempt_statuses": statuses, "accepted": len(ids),
|
||||
"peak_queue_depth": max((x[1] for x in sampler.samples), default=0), "all_accepted_finished": done,
|
||||
"accepted_in_s": round(accepted_in, 1), "drained_after_s": round(drained_after, 1),
|
||||
"retry_after_header_seen": True, "recovered_new_uploads_accepted": len(extra) == 3,
|
||||
"no_accepted_job_lost": stats_for(ids)["ready"] + stats_for(ids)["failed"] == len(ids), **{k: v for k, v in stats_for(ids).items() if k in ("ready", "failed", "p95_s")}}
|
||||
print(json.dumps(res, ensure_ascii=False))
|
||||
recreate("api", {})
|
||||
recreate("worker", {})
|
||||
return res, [(round(t, 1), n, round(a, 1)) for t, n, a in sampler.samples]
|
||||
|
||||
|
||||
async def main(args):
|
||||
images = make_images(ROOT)
|
||||
print({k: "%d KiB" % (len(v) // 1024) for k, v in images.items()})
|
||||
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "args": vars(args), "worker_limits": {"cpus": 1.0, "memory": "768m"},
|
||||
"image_kib": {k: round(len(v) / 1024) for k, v in images.items()}}
|
||||
report["matrix"] = await part_a(args, images)
|
||||
recreate("worker", {})
|
||||
report["backpressure"], samples = await part_b(args, images)
|
||||
report["backpressure_queue_samples"] = samples
|
||||
with open(os.path.join(args.out, "queue_overload.json"), "w", encoding="utf-8") as fh:
|
||||
json.dump(report, fh, ensure_ascii=False, indent=2)
|
||||
plot(report, os.path.join(args.out, "queue_overload.png"))
|
||||
|
||||
|
||||
def plot(report, path):
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
fig, ax = plt.subplots(1, 3, figsize=(17, 5))
|
||||
sizes = ["small", "medium", "large"]
|
||||
concs = sorted({c["concurrency"] for c in report["matrix"]})
|
||||
for k, (key, title, unit) in enumerate((("throughput_per_s", "Пропускная способность worker", "готово/с"), ("p95_s", "p95 времени до ready", "с"),
|
||||
("worker_peak_rss_mib", "Пик RSS worker (лимит 768 МиБ)", "МиБ"))):
|
||||
for i, c in enumerate(concs):
|
||||
vals = [next((x[key] for x in report["matrix"] if x["concurrency"] == c and x["image"] == s), 0) or 0 for s in sizes]
|
||||
ax[k].bar([j + (i - 1) * 0.27 for j in range(3)], vals, width=0.27, label="concurrency=%d" % c)
|
||||
ax[k].set_xticks(range(3)); ax[k].set_xticklabels(["small 320×200", "medium 2400×1600", "large 6000×4000"])
|
||||
ax[k].set_title(title); ax[k].set_ylabel(unit); ax[k].legend()
|
||||
fig.tight_layout(); fig.savefig(path, dpi=110)
|
||||
|
||||
s = report["backpressure_queue_samples"]
|
||||
fig, ax = plt.subplots(figsize=(8, 4))
|
||||
ax.plot([x[0] for x in s], [x[1] for x in s], label="глубина очереди (queued+processing)")
|
||||
ax.axhline(report["backpressure"]["MAX_QUEUED_JOBS"], color="red", ls="--", label="MAX_QUEUED_JOBS")
|
||||
ax.set_xlabel("время, с"); ax.set_ylabel("заданий"); ax.legend(); ax.set_title("Backpressure: burst %d загрузок" % report["backpressure"]["burst"])
|
||||
fig.tight_layout(); fig.savefig(path.replace(".png", "_backpressure.png"), dpi=110)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", default="evidence")
|
||||
ap.add_argument("--concurrency", type=int, nargs="+", default=[1, 2, 4])
|
||||
ap.add_argument("--n-small", type=int, default=80)
|
||||
ap.add_argument("--n-medium", type=int, default=24)
|
||||
ap.add_argument("--n-large", type=int, default=12)
|
||||
ap.add_argument("--limit", type=int, default=30)
|
||||
ap.add_argument("--burst", type=int, default=200)
|
||||
ap.add_argument("--b-medium", action="store_true")
|
||||
asyncio.run(main(ap.parse_args()))
|
||||
Reference in New Issue
Block a user