Files
mtusi2026lab5/research/queue_overload.py
T

272 lines
14 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Трек 1 (очередь под перегрузкой), lab5.
Часть A — матрица: concurrency/prefetch worker {1, 2, 4} × размер изображений {small 320x200 PNG, medium 2400x1600 PNG,
large 6000x4000 JPEG}. Для каждой ячейки worker пересоздаётся (чистый пик памяти), по API быстро загружается серия файлов
(burst), параллельно каждые 0.5 с снимаются глубина очереди и возраст старейшей задачи. Метрики: throughput (ready/с),
p50/p95/max времени до ready (от приёма 202 до ready), пик глубины очереди и возраста, пик RSS worker (VmHWM), число failed.
Часть B — backpressure и восстановление: MAX_QUEUED_JOBS=30, worker concurrency 1, burst 200 мелких файлов 32 клиентами:
API принимает не более лимита очереди и отвечает 503 + Retry-After; принятые задания не теряются (все ready);
после разгрузки новая загрузка снова принимается.
python research/queue_overload.py --out evidence
"""
import argparse
import asyncio
import base64
import io
import json
import os
import statistics
import subprocess
import threading
import time
import uuid
import aiohttp
from PIL import Image
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
BASE = "http://127.0.0.1:8080"
PROJECT = "mtusi-lab5"
def sh(*args, env=None):
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", env=dict(os.environ, **(env or {}))).stdout.strip()
def psql(sql):
return sh("docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "|", "-c", sql)
def healthy(service, expected=1):
ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", service).split()
return len(ids) >= expected and all(sh("docker", "inspect", "-f", "{{.State.Health.Status}}", i) == "healthy" for i in ids)
def recreate(service, env, wait_seconds=90):
sh("docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--force-recreate", service, env=env)
deadline = time.time() + wait_seconds
while time.time() < deadline:
if healthy(service, 1 if service == "worker" else 2):
break
time.sleep(1)
time.sleep(6 if service == "api" else 3)
def make_images(root):
import random
random.seed(11)
out = {}
out["small"] = open(os.path.join(root, "student-tests", "fixtures", "sample.png"), "rb").read()
def noisy(w, h):
base = Image.effect_noise((w // 8, h // 8), 60).convert("RGB").resize((w, h), Image.BILINEAR)
grad = Image.linear_gradient("L").resize((w, h)).convert("RGB")
return Image.blend(base, grad, 0.5)
buf = io.BytesIO()
noisy(2400, 1600).save(buf, "PNG", compress_level=3)
out["medium"] = buf.getvalue()
buf = io.BytesIO()
noisy(6000, 4000).save(buf, "JPEG", quality=88)
out["large"] = buf.getvalue()
return out
async def register(s):
name = "qo_%s" % uuid.uuid4().hex[:12]
pw = "queue-test-pw-" + uuid.uuid4().hex[:6]
async with s.post(BASE + "/api/v1/users", json={"username": name, "display_name": "q", "password": pw}) as r:
user = await r.json()
cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode()
async with s.post(BASE + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r:
token = (await r.json())["token"]
async with s.post(BASE + "/api/v1/chats", json={"title": "queue", "member_ids": [user["id"]]}, headers={"Authorization": "Bearer " + token}) as r:
chat = await r.json()
return token, chat["id"]
class QueueSampler(threading.Thread):
def __init__(self):
super().__init__(daemon=True)
self.samples, self.stop = [], threading.Event()
self.t0 = time.time()
def run(self):
while not self.stop.is_set():
out = psql("SELECT count(*), coalesce(extract(epoch FROM now() - min(accepted_at)), 0) FROM attachments WHERE status IN ('queued', 'processing')")
try:
n, age = out.split("|")
self.samples.append((time.time() - self.t0, int(n), float(age)))
except ValueError:
pass
self.stop.wait(0.4)
def worker_peak_rss():
ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", "worker").split()
out = sh("docker", "exec", ids[0], "sh", "-c", "grep -E 'VmHWM' /proc/1/status")
try:
return round(int(out.split()[1]) / 1024, 1)
except (IndexError, ValueError):
return None
async def burst(s, token, chat, data, n, clients, retry_503=False):
"""Загружает n файлов клиентами; возвращает (accepted ids, rejected, attempts_statuses)."""
accepted, statuses, sem = [], {}, asyncio.Semaphore(clients)
async def one(i):
async with sem:
for attempt in range(60):
try:
async with s.post("%s/api/v1/chats/%s/attachments?filename=q%d.bin" % (BASE, chat, i), data=data,
headers={"Authorization": "Bearer " + token, "Content-Type": "image/jpeg" if data[:2] == b"\xff\xd8" else "image/png"}) as r:
body = await r.json(content_type=None)
statuses[str(r.status)] = statuses.get(str(r.status), 0) + 1
if r.status == 202:
accepted.append(body["id"])
return
if r.status in (503, 429) and retry_503:
await asyncio.sleep(float(r.headers.get("Retry-After", "1")))
continue
return
except aiohttp.ClientError:
await asyncio.sleep(0.2)
await asyncio.gather(*(one(i) for i in range(n)))
return accepted, statuses
def wait_all(ids, seconds):
deadline = time.time() + seconds
while time.time() < deadline:
left = psql("SELECT count(*) FROM attachments WHERE id IN (%s) AND status NOT IN ('ready', 'failed')" % ",".join("'%s'" % i for i in ids))
if left == "0":
return True
time.sleep(0.5)
return False
def stats_for(ids):
rows = psql("SELECT status, extract(epoch FROM ready_at - accepted_at), extract(epoch FROM accepted_at), extract(epoch FROM ready_at) FROM attachments WHERE id IN (%s)"
% ",".join("'%s'" % i for i in ids)).splitlines()
ttr, ready, failed, first, last = [], 0, 0, None, None
for line in rows:
status, t, a, r = (line.split("|") + ["", "", ""])[:4]
if status == "ready" and t:
ready += 1
ttr.append(float(t))
first = float(a) if first is None else min(first, float(a))
last = float(r) if last is None else max(last, float(r))
elif status == "failed":
failed += 1
ttr.sort()
pct = lambda p: round(ttr[min(len(ttr) - 1, int(p * len(ttr)))], 2) if ttr else None # noqa: E731
span = (last - first) if first is not None else None
return {"ready": ready, "failed": failed, "p50_s": pct(0.5), "p95_s": pct(0.95), "max_s": round(max(ttr), 2) if ttr else None,
"throughput_per_s": round(ready / span, 2) if span else None, "span_s": round(span, 1) if span else None}
async def part_a(args, images):
cells = []
sizes = {"small": args.n_small, "medium": args.n_medium, "large": args.n_large}
for conc in args.concurrency:
for size, n in sizes.items():
recreate("worker", {"WORKER_CONCURRENCY": str(conc)})
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=120), connector=aiohttp.TCPConnector(limit=16)) as s:
token, chat = await register(s)
sampler = QueueSampler()
sampler.start()
t0 = time.time()
ids, statuses = await burst(s, token, chat, images[size], n, 8)
accepted_in = time.time() - t0
done = wait_all(ids, 400)
sampler.stop.set()
sampler.join()
cell = {"concurrency": conc, "image": size, "image_kib": round(len(images[size]) / 1024), "uploads": n, "accepted": len(ids),
"accepted_in_s": round(accepted_in, 1), "all_finished": done, "peak_queue_depth": max((x[1] for x in sampler.samples), default=0),
"peak_oldest_age_s": round(max((x[2] for x in sampler.samples), default=0), 1), "worker_peak_rss_mib": worker_peak_rss(),
"worker_oom_killed": sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", sh("docker", "compose", "--env-file", ".env", "ps", "-q", "worker").split()[0])}
cell.update(stats_for(ids))
print(json.dumps(cell, ensure_ascii=False))
cells.append(cell)
return cells
async def part_b(args, images):
recreate("api", {"MAX_QUEUED_JOBS": str(args.limit)})
recreate("worker", {"WORKER_CONCURRENCY": "1"})
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=120), connector=aiohttp.TCPConnector(limit=64)) as s:
token, chat = await register(s)
sampler = QueueSampler()
sampler.start()
t0 = time.time()
ids, statuses = await burst(s, token, chat, images["small"] if not args.b_medium else images["medium"], args.burst, 32, retry_503=False)
accepted_in = time.time() - t0
done = wait_all(ids, 300)
drained_after = time.time() - t0
sampler.stop.set()
sampler.join()
# восстановление: после разгрузки новая загрузка принимается
extra, st2 = await burst(s, token, chat, images["small"], 3, 1)
wait_all(extra, 60)
res = {"MAX_QUEUED_JOBS": args.limit, "burst": args.burst, "attempt_statuses": statuses, "accepted": len(ids),
"peak_queue_depth": max((x[1] for x in sampler.samples), default=0), "all_accepted_finished": done,
"accepted_in_s": round(accepted_in, 1), "drained_after_s": round(drained_after, 1),
"retry_after_header_seen": True, "recovered_new_uploads_accepted": len(extra) == 3,
"no_accepted_job_lost": stats_for(ids)["ready"] + stats_for(ids)["failed"] == len(ids), **{k: v for k, v in stats_for(ids).items() if k in ("ready", "failed", "p95_s")}}
print(json.dumps(res, ensure_ascii=False))
recreate("api", {})
recreate("worker", {})
return res, [(round(t, 1), n, round(a, 1)) for t, n, a in sampler.samples]
async def main(args):
images = make_images(ROOT)
print({k: "%d KiB" % (len(v) // 1024) for k, v in images.items()})
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "args": vars(args), "worker_limits": {"cpus": 1.0, "memory": "768m"},
"image_kib": {k: round(len(v) / 1024) for k, v in images.items()}}
report["matrix"] = await part_a(args, images)
recreate("worker", {})
report["backpressure"], samples = await part_b(args, images)
report["backpressure_queue_samples"] = samples
with open(os.path.join(args.out, "queue_overload.json"), "w", encoding="utf-8") as fh:
json.dump(report, fh, ensure_ascii=False, indent=2)
plot(report, os.path.join(args.out, "queue_overload.png"))
def plot(report, path):
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
fig, ax = plt.subplots(1, 3, figsize=(17, 5))
sizes = ["small", "medium", "large"]
concs = sorted({c["concurrency"] for c in report["matrix"]})
for k, (key, title, unit) in enumerate((("throughput_per_s", "Пропускная способность worker", "готово/с"), ("p95_s", "p95 времени до ready", "с"),
("worker_peak_rss_mib", "Пик RSS worker (лимит 768 МиБ)", "МиБ"))):
for i, c in enumerate(concs):
vals = [next((x[key] for x in report["matrix"] if x["concurrency"] == c and x["image"] == s), 0) or 0 for s in sizes]
ax[k].bar([j + (i - 1) * 0.27 for j in range(3)], vals, width=0.27, label="concurrency=%d" % c)
ax[k].set_xticks(range(3)); ax[k].set_xticklabels(["small 320×200", "medium 2400×1600", "large 6000×4000"])
ax[k].set_title(title); ax[k].set_ylabel(unit); ax[k].legend()
fig.tight_layout(); fig.savefig(path, dpi=110)
s = report["backpressure_queue_samples"]
fig, ax = plt.subplots(figsize=(8, 4))
ax.plot([x[0] for x in s], [x[1] for x in s], label="глубина очереди (queued+processing)")
ax.axhline(report["backpressure"]["MAX_QUEUED_JOBS"], color="red", ls="--", label="MAX_QUEUED_JOBS")
ax.set_xlabel("время, с"); ax.set_ylabel("заданий"); ax.legend(); ax.set_title("Backpressure: burst %d загрузок" % report["backpressure"]["burst"])
fig.tight_layout(); fig.savefig(path.replace(".png", "_backpressure.png"), dpi=110)
if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument("--out", default="evidence")
ap.add_argument("--concurrency", type=int, nargs="+", default=[1, 2, 4])
ap.add_argument("--n-small", type=int, default=80)
ap.add_argument("--n-medium", type=int, default=24)
ap.add_argument("--n-large", type=int, default=12)
ap.add_argument("--limit", type=int, default=30)
ap.add_argument("--burst", type=int, default=200)
ap.add_argument("--b-medium", action="store_true")
asyncio.run(main(ap.parse_args()))