#!/usr/bin/env python3 """Трек 1 (очередь под перегрузкой), lab5. Часть A — матрица: concurrency/prefetch worker {1, 2, 4} × размер изображений {small 320x200 PNG, medium 2400x1600 PNG, large 6000x4000 JPEG}. Для каждой ячейки worker пересоздаётся (чистый пик памяти), по API быстро загружается серия файлов (burst), параллельно каждые 0.5 с снимаются глубина очереди и возраст старейшей задачи. Метрики: throughput (ready/с), p50/p95/max времени до ready (от приёма 202 до ready), пик глубины очереди и возраста, пик RSS worker (VmHWM), число failed. Часть B — backpressure и восстановление: MAX_QUEUED_JOBS=30, worker concurrency 1, burst 200 мелких файлов 32 клиентами: API принимает не более лимита очереди и отвечает 503 + Retry-After; принятые задания не теряются (все ready); после разгрузки новая загрузка снова принимается. python research/queue_overload.py --out evidence """ import argparse import asyncio import base64 import io import json import os import statistics import subprocess import threading import time import uuid import aiohttp from PIL import Image ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") BASE = "http://127.0.0.1:8080" PROJECT = "mtusi-lab5" def sh(*args, env=None): return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", env=dict(os.environ, **(env or {}))).stdout.strip() def psql(sql): return sh("docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "|", "-c", sql) def healthy(service, expected=1): ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", service).split() return len(ids) >= expected and all(sh("docker", "inspect", "-f", "{{.State.Health.Status}}", i) == "healthy" for i in ids) def recreate(service, env, wait_seconds=90): sh("docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--force-recreate", service, env=env) deadline = time.time() + wait_seconds while time.time() < deadline: if healthy(service, 1 if service == "worker" else 2): break time.sleep(1) time.sleep(6 if service == "api" else 3) def make_images(root): import random random.seed(11) out = {} out["small"] = open(os.path.join(root, "student-tests", "fixtures", "sample.png"), "rb").read() def noisy(w, h): base = Image.effect_noise((w // 8, h // 8), 60).convert("RGB").resize((w, h), Image.BILINEAR) grad = Image.linear_gradient("L").resize((w, h)).convert("RGB") return Image.blend(base, grad, 0.5) buf = io.BytesIO() noisy(2400, 1600).save(buf, "PNG", compress_level=3) out["medium"] = buf.getvalue() buf = io.BytesIO() noisy(6000, 4000).save(buf, "JPEG", quality=88) out["large"] = buf.getvalue() return out async def register(s): name = "qo_%s" % uuid.uuid4().hex[:12] pw = "queue-test-pw-" + uuid.uuid4().hex[:6] async with s.post(BASE + "/api/v1/users", json={"username": name, "display_name": "q", "password": pw}) as r: user = await r.json() cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode() async with s.post(BASE + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r: token = (await r.json())["token"] async with s.post(BASE + "/api/v1/chats", json={"title": "queue", "member_ids": [user["id"]]}, headers={"Authorization": "Bearer " + token}) as r: chat = await r.json() return token, chat["id"] class QueueSampler(threading.Thread): def __init__(self): super().__init__(daemon=True) self.samples, self.stop = [], threading.Event() self.t0 = time.time() def run(self): while not self.stop.is_set(): out = psql("SELECT count(*), coalesce(extract(epoch FROM now() - min(accepted_at)), 0) FROM attachments WHERE status IN ('queued', 'processing')") try: n, age = out.split("|") self.samples.append((time.time() - self.t0, int(n), float(age))) except ValueError: pass self.stop.wait(0.4) def worker_peak_rss(): ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", "worker").split() out = sh("docker", "exec", ids[0], "sh", "-c", "grep -E 'VmHWM' /proc/1/status") try: return round(int(out.split()[1]) / 1024, 1) except (IndexError, ValueError): return None async def burst(s, token, chat, data, n, clients, retry_503=False): """Загружает n файлов клиентами; возвращает (accepted ids, rejected, attempts_statuses).""" accepted, statuses, sem = [], {}, asyncio.Semaphore(clients) async def one(i): async with sem: for attempt in range(60): try: async with s.post("%s/api/v1/chats/%s/attachments?filename=q%d.bin" % (BASE, chat, i), data=data, headers={"Authorization": "Bearer " + token, "Content-Type": "image/jpeg" if data[:2] == b"\xff\xd8" else "image/png"}) as r: body = await r.json(content_type=None) statuses[str(r.status)] = statuses.get(str(r.status), 0) + 1 if r.status == 202: accepted.append(body["id"]) return if r.status in (503, 429) and retry_503: await asyncio.sleep(float(r.headers.get("Retry-After", "1"))) continue return except aiohttp.ClientError: await asyncio.sleep(0.2) await asyncio.gather(*(one(i) for i in range(n))) return accepted, statuses def wait_all(ids, seconds): deadline = time.time() + seconds while time.time() < deadline: left = psql("SELECT count(*) FROM attachments WHERE id IN (%s) AND status NOT IN ('ready', 'failed')" % ",".join("'%s'" % i for i in ids)) if left == "0": return True time.sleep(0.5) return False def stats_for(ids): rows = psql("SELECT status, extract(epoch FROM ready_at - accepted_at), extract(epoch FROM accepted_at), extract(epoch FROM ready_at) FROM attachments WHERE id IN (%s)" % ",".join("'%s'" % i for i in ids)).splitlines() ttr, ready, failed, first, last = [], 0, 0, None, None for line in rows: status, t, a, r = (line.split("|") + ["", "", ""])[:4] if status == "ready" and t: ready += 1 ttr.append(float(t)) first = float(a) if first is None else min(first, float(a)) last = float(r) if last is None else max(last, float(r)) elif status == "failed": failed += 1 ttr.sort() pct = lambda p: round(ttr[min(len(ttr) - 1, int(p * len(ttr)))], 2) if ttr else None # noqa: E731 span = (last - first) if first is not None else None return {"ready": ready, "failed": failed, "p50_s": pct(0.5), "p95_s": pct(0.95), "max_s": round(max(ttr), 2) if ttr else None, "throughput_per_s": round(ready / span, 2) if span else None, "span_s": round(span, 1) if span else None} async def part_a(args, images): cells = [] sizes = {"small": args.n_small, "medium": args.n_medium, "large": args.n_large} for conc in args.concurrency: for size, n in sizes.items(): recreate("worker", {"WORKER_CONCURRENCY": str(conc)}) async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=120), connector=aiohttp.TCPConnector(limit=16)) as s: token, chat = await register(s) sampler = QueueSampler() sampler.start() t0 = time.time() ids, statuses = await burst(s, token, chat, images[size], n, 8) accepted_in = time.time() - t0 done = wait_all(ids, 400) sampler.stop.set() sampler.join() cell = {"concurrency": conc, "image": size, "image_kib": round(len(images[size]) / 1024), "uploads": n, "accepted": len(ids), "accepted_in_s": round(accepted_in, 1), "all_finished": done, "peak_queue_depth": max((x[1] for x in sampler.samples), default=0), "peak_oldest_age_s": round(max((x[2] for x in sampler.samples), default=0), 1), "worker_peak_rss_mib": worker_peak_rss(), "worker_oom_killed": sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", sh("docker", "compose", "--env-file", ".env", "ps", "-q", "worker").split()[0])} cell.update(stats_for(ids)) print(json.dumps(cell, ensure_ascii=False)) cells.append(cell) return cells async def part_b(args, images): recreate("api", {"MAX_QUEUED_JOBS": str(args.limit)}) recreate("worker", {"WORKER_CONCURRENCY": "1"}) async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=120), connector=aiohttp.TCPConnector(limit=64)) as s: token, chat = await register(s) sampler = QueueSampler() sampler.start() t0 = time.time() ids, statuses = await burst(s, token, chat, images["small"] if not args.b_medium else images["medium"], args.burst, 32, retry_503=False) accepted_in = time.time() - t0 done = wait_all(ids, 300) drained_after = time.time() - t0 sampler.stop.set() sampler.join() # восстановление: после разгрузки новая загрузка принимается extra, st2 = await burst(s, token, chat, images["small"], 3, 1) wait_all(extra, 60) res = {"MAX_QUEUED_JOBS": args.limit, "burst": args.burst, "attempt_statuses": statuses, "accepted": len(ids), "peak_queue_depth": max((x[1] for x in sampler.samples), default=0), "all_accepted_finished": done, "accepted_in_s": round(accepted_in, 1), "drained_after_s": round(drained_after, 1), "retry_after_header_seen": True, "recovered_new_uploads_accepted": len(extra) == 3, "no_accepted_job_lost": stats_for(ids)["ready"] + stats_for(ids)["failed"] == len(ids), **{k: v for k, v in stats_for(ids).items() if k in ("ready", "failed", "p95_s")}} print(json.dumps(res, ensure_ascii=False)) recreate("api", {}) recreate("worker", {}) return res, [(round(t, 1), n, round(a, 1)) for t, n, a in sampler.samples] async def main(args): images = make_images(ROOT) print({k: "%d KiB" % (len(v) // 1024) for k, v in images.items()}) report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "args": vars(args), "worker_limits": {"cpus": 1.0, "memory": "768m"}, "image_kib": {k: round(len(v) / 1024) for k, v in images.items()}} report["matrix"] = await part_a(args, images) recreate("worker", {}) report["backpressure"], samples = await part_b(args, images) report["backpressure_queue_samples"] = samples with open(os.path.join(args.out, "queue_overload.json"), "w", encoding="utf-8") as fh: json.dump(report, fh, ensure_ascii=False, indent=2) plot(report, os.path.join(args.out, "queue_overload.png")) def plot(report, path): import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt fig, ax = plt.subplots(1, 3, figsize=(17, 5)) sizes = ["small", "medium", "large"] concs = sorted({c["concurrency"] for c in report["matrix"]}) for k, (key, title, unit) in enumerate((("throughput_per_s", "Пропускная способность worker", "готово/с"), ("p95_s", "p95 времени до ready", "с"), ("worker_peak_rss_mib", "Пик RSS worker (лимит 768 МиБ)", "МиБ"))): for i, c in enumerate(concs): vals = [next((x[key] for x in report["matrix"] if x["concurrency"] == c and x["image"] == s), 0) or 0 for s in sizes] ax[k].bar([j + (i - 1) * 0.27 for j in range(3)], vals, width=0.27, label="concurrency=%d" % c) ax[k].set_xticks(range(3)); ax[k].set_xticklabels(["small 320×200", "medium 2400×1600", "large 6000×4000"]) ax[k].set_title(title); ax[k].set_ylabel(unit); ax[k].legend() fig.tight_layout(); fig.savefig(path, dpi=110) s = report["backpressure_queue_samples"] fig, ax = plt.subplots(figsize=(8, 4)) ax.plot([x[0] for x in s], [x[1] for x in s], label="глубина очереди (queued+processing)") ax.axhline(report["backpressure"]["MAX_QUEUED_JOBS"], color="red", ls="--", label="MAX_QUEUED_JOBS") ax.set_xlabel("время, с"); ax.set_ylabel("заданий"); ax.legend(); ax.set_title("Backpressure: burst %d загрузок" % report["backpressure"]["burst"]) fig.tight_layout(); fig.savefig(path.replace(".png", "_backpressure.png"), dpi=110) if __name__ == "__main__": ap = argparse.ArgumentParser() ap.add_argument("--out", default="evidence") ap.add_argument("--concurrency", type=int, nargs="+", default=[1, 2, 4]) ap.add_argument("--n-small", type=int, default=80) ap.add_argument("--n-medium", type=int, default=24) ap.add_argument("--n-large", type=int, default=12) ap.add_argument("--limit", type=int, default=30) ap.add_argument("--burst", type=int, default=200) ap.add_argument("--b-medium", action="store_true") asyncio.run(main(ap.parse_args()))