#!/usr/bin/env python3 """Трек 1 (очередь под перегрузкой), lab5. Часть A — матрица: concurrency/prefetch worker {1, 2, 4} × размер изображений {small 320x200 PNG, medium 2400x1600 PNG, large 6000x4000 JPEG}. Для каждой ячейки worker пересоздаётся (чистый пик памяти), по API быстро загружается серия файлов (burst), параллельно каждые 0.5 с снимаются глубина очереди и возраст старейшей задачи. Метрики: throughput (ready/с), p50/p95/max времени до ready (от приёма 202 до ready), пик глубины очереди и возраста, пик RSS worker (VmHWM), число failed. Часть B — backpressure и восстановление: MAX_QUEUED_JOBS=30, worker concurrency 1, burst 200 мелких файлов 32 клиентами: API ограничивает приём по порогу очереди и отвечает 503 + Retry-After (возможен конкурентный overshoot); принятые задания не теряются (все ready); после разгрузки новая загрузка снова принимается. python research/queue_overload.py --out evidence """ import argparse import asyncio import base64 import io import json import math import os import statistics import subprocess import threading import time import uuid import aiohttp from PIL import Image ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") BASE = "http://127.0.0.1:8080" PROJECT = "mtusi-lab5" def sh(*args, env=None): return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True, env=dict(os.environ, **(env or {}))).stdout.strip() def psql(sql): return sh("docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "|", "-c", sql) def healthy(service, expected=1): ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", service).split() return len(ids) >= expected and all(sh("docker", "inspect", "-f", "{{.State.Health.Status}}", i) == "healthy" for i in ids) def recreate(service, env, wait_seconds=90): sh("docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--force-recreate", service, env=env) deadline = time.time() + wait_seconds while time.time() < deadline: if healthy(service, 1 if service == "worker" else 2): break time.sleep(1) else: raise RuntimeError("%s did not become healthy within %d seconds" % (service, wait_seconds)) time.sleep(6 if service == "api" else 3) def make_images(root): import random random.seed(11) out = {} out["small"] = open(os.path.join(root, "student-tests", "fixtures", "sample.png"), "rb").read() def noisy(w, h): base = Image.effect_noise((w // 8, h // 8), 60).convert("RGB").resize((w, h), Image.BILINEAR) grad = Image.linear_gradient("L").resize((w, h)).convert("RGB") return Image.blend(base, grad, 0.5) buf = io.BytesIO() noisy(2400, 1600).save(buf, "PNG", compress_level=3) out["medium"] = buf.getvalue() buf = io.BytesIO() noisy(6000, 4000).save(buf, "JPEG", quality=88) out["large"] = buf.getvalue() return out async def register(s): name = "qo_%s" % uuid.uuid4().hex[:12] pw = "queue-test-pw-" + uuid.uuid4().hex[:6] async with s.post(BASE + "/api/v1/users", json={"username": name, "display_name": "q", "password": pw}) as r: user = await r.json() cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode() async with s.post(BASE + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r: token = (await r.json())["token"] async with s.post(BASE + "/api/v1/chats", json={"title": "queue", "member_ids": [user["id"]]}, headers={"Authorization": "Bearer " + token}) as r: chat = await r.json() return token, chat["id"] class QueueSampler(threading.Thread): def __init__(self): super().__init__(daemon=True) self.samples, self.stop, self.errors = [], threading.Event(), [] self.t0 = time.time() def run(self): while not self.stop.is_set(): try: out = psql("SELECT count(*), coalesce(extract(epoch FROM now() - min(accepted_at)), 0) FROM attachments WHERE status IN ('queued', 'processing')") n, age = out.split("|") self.samples.append((time.time() - self.t0, int(n), float(age))) except Exception as exc: self.errors.append(type(exc).__name__) return self.stop.wait(0.4) def worker_peak_rss(): ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", "worker").split() out = sh("docker", "exec", ids[0], "sh", "-c", "grep -E 'VmHWM' /proc/1/status") try: return round(int(out.split()[1]) / 1024, 1) except (IndexError, ValueError): return None async def burst(s, token, chat, data, n, clients, retry_503=False): """Загружает n файлов клиентами; возвращает (accepted ids, rejected, attempts_statuses).""" accepted, statuses, sem = [], {}, asyncio.Semaphore(clients) observed = {"retry_after_values": [], "transport_errors": []} async def one(i): async with sem: for attempt in range(60): try: async with s.post("%s/api/v1/chats/%s/attachments?filename=q%d.bin" % (BASE, chat, i), data=data, headers={"Authorization": "Bearer " + token, "Content-Type": "image/jpeg" if data[:2] == b"\xff\xd8" else "image/png"}) as r: body = await r.json(content_type=None) statuses[str(r.status)] = statuses.get(str(r.status), 0) + 1 if r.status in (503, 429): observed["retry_after_values"].append(r.headers.get("Retry-After")) if r.status == 202: accepted.append(body["id"]) return if r.status in (503, 429) and retry_503: await asyncio.sleep(float(r.headers.get("Retry-After", "1"))) continue return except aiohttp.ClientError as exc: # Upload is not idempotent: an uncertain response must not trigger a hidden retry. observed["transport_errors"].append(type(exc).__name__) return await asyncio.gather(*(one(i) for i in range(n))) return accepted, statuses, observed def wait_all(ids, seconds): if not ids: raise ValueError("no accepted jobs to measure") deadline = time.time() + seconds while time.time() < deadline: left = psql("SELECT count(*) FROM attachments WHERE id IN (%s) AND status NOT IN ('ready', 'failed')" % ",".join("'%s'" % i for i in ids)) if left == "0": return True time.sleep(0.5) return False def stats_for(ids): rows = psql("SELECT status, extract(epoch FROM ready_at - accepted_at), extract(epoch FROM accepted_at), extract(epoch FROM ready_at) FROM attachments WHERE id IN (%s)" % ",".join("'%s'" % i for i in ids)).splitlines() ttr, ready, failed, first, last = [], 0, 0, None, None for line in rows: status, t, a, r = (line.split("|") + ["", "", ""])[:4] if status == "ready" and t: ready += 1 ttr.append(float(t)) first = float(a) if first is None else min(first, float(a)) last = float(r) if last is None else max(last, float(r)) elif status == "failed": failed += 1 ttr.sort() pct = lambda p: round(ttr[max(0, math.ceil(p * len(ttr)) - 1)], 2) if ttr else None # noqa: E731 span = (last - first) if first is not None else None return {"ready": ready, "failed": failed, "p50_s": pct(0.5), "p95_s": pct(0.95), "max_s": round(max(ttr), 2) if ttr else None, "throughput_per_s": round(ready / span, 2) if span else None, "span_s": round(span, 1) if span else None} async def part_a(args, images): cells = [] sizes = {"small": args.n_small, "medium": args.n_medium, "large": args.n_large} for conc in args.concurrency: for size, n in sizes.items(): recreate("worker", {"WORKER_CONCURRENCY": str(conc)}) ids_worker = sh("docker", "compose", "ps", "-q", "worker").split() assert len(ids_worker) == 1, "this experiment requires exactly one worker" limits = json.loads(sh("docker", "inspect", "-f", '{"memory_bytes":{{.HostConfig.Memory}},"nano_cpus":{{.HostConfig.NanoCpus}}}', ids_worker[0])) assert limits == {"memory_bytes": 768 * 1024 * 1024, "nano_cpus": 1000000000}, limits async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=120), connector=aiohttp.TCPConnector(limit=16, force_close=True)) as s: token, chat = await register(s) sampler = QueueSampler() sampler.start() t0 = time.time() ids, statuses, observed = await burst(s, token, chat, images[size], n, 8) accepted_in = time.time() - t0 done = wait_all(ids, 400) sampler.stop.set() sampler.join() cell = {"concurrency": conc, "image": size, "image_kib": round(len(images[size]) / 1024), "uploads": n, "accepted": len(ids), "resource_limits": limits, "attempt_statuses": statuses, "transport_errors": observed["transport_errors"], "accepted_in_s": round(accepted_in, 1), "all_finished": done, "peak_queue_depth": max((x[1] for x in sampler.samples), default=0), "peak_oldest_age_s": round(max((x[2] for x in sampler.samples), default=0), 1), "worker_peak_rss_mib": worker_peak_rss(), "worker_oom_killed": sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", sh("docker", "compose", "--env-file", ".env", "ps", "-q", "worker").split()[0])} cell.update(stats_for(ids)) print(json.dumps(cell, ensure_ascii=False)) cells.append(cell) with open(os.path.join(args.out, "matrix_progress.json"), "w", encoding="utf-8") as fh: json.dump(cells, fh, ensure_ascii=False, indent=2) assert not sampler.errors and sampler.samples, sampler.errors assert len(ids) == len(set(ids)) == n and statuses == {"202": n} and not observed["transport_errors"], cell assert done and cell["ready"] == n and cell["failed"] == 0, cell assert cell["worker_oom_killed"] == "false" and 0 < cell["worker_peak_rss_mib"] < 768, cell return cells async def part_b(args, images): recreate("api", {"MAX_QUEUED_JOBS": str(args.limit)}) recreate("worker", {"WORKER_CONCURRENCY": "1"}) async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=120), connector=aiohttp.TCPConnector(limit=64, force_close=True)) as s: token, chat = await register(s) sampler = QueueSampler() sampler.start() t0 = time.time() ids, statuses, observed = await burst(s, token, chat, images["small"] if not args.b_medium else images["medium"], args.burst, 32, retry_503=False) accepted_in = time.time() - t0 done = wait_all(ids, 300) drained_after = time.time() - t0 sampler.stop.set() sampler.join() # восстановление: после разгрузки новая загрузка принимается extra, st2, observed2 = await burst(s, token, chat, images["small"], 3, 1) extra_done = wait_all(extra, 60) throttled = observed["retry_after_values"] valid_retry_after = bool(throttled) and all(v is not None and v.isdigit() and int(v) >= 1 for v in throttled) res = {"MAX_QUEUED_JOBS": args.limit, "burst": args.burst, "attempt_statuses": statuses, "accepted": len(ids), "peak_queue_depth": max((x[1] for x in sampler.samples), default=0), "all_accepted_finished": done, "accepted_in_s": round(accepted_in, 1), "drained_after_s": round(drained_after, 1), "retry_after_header_seen": valid_retry_after, "retry_after_values": sorted(set(throttled), key=str), "transport_errors": observed["transport_errors"], "recovered_new_uploads_accepted": len(extra) == 3, "recovered_new_uploads_ready": extra_done and stats_for(extra)["ready"] == 3, "no_accepted_job_lost": stats_for(ids)["ready"] + stats_for(ids)["failed"] == len(ids), **{k: v for k, v in stats_for(ids).items() if k in ("ready", "failed", "p95_s")}} print(json.dumps(res, ensure_ascii=False)) with open(os.path.join(args.out, "backpressure_progress.json"), "w", encoding="utf-8") as fh: json.dump(res, fh, ensure_ascii=False, indent=2) assert not sampler.errors and sampler.samples, sampler.errors assert statuses.get("503", 0) > 0 and sum(statuses.values()) == args.burst and valid_retry_after, res assert set(statuses) <= {"202", "503", "429"} and not observed["transport_errors"], res assert done and res["ready"] == len(ids) and res["failed"] == 0 and res["no_accepted_job_lost"], res assert res["recovered_new_uploads_accepted"] and res["recovered_new_uploads_ready"] and not observed2["transport_errors"], res return res, [(round(t, 1), n, round(a, 1)) for t, n, a in sampler.samples] async def main(args): os.makedirs(args.out, exist_ok=True) images = make_images(ROOT) print({k: "%d KiB" % (len(v) // 1024) for k, v in images.items()}) report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "args": vars(args), "worker_limits": {"cpus": 1.0, "memory": "768m"}, "image_kib": {k: round(len(v) / 1024) for k, v in images.items()}} try: report["matrix"] = await part_a(args, images) report["backpressure"], samples = await part_b(args, images) finally: recreate("api", {}) recreate("worker", {}) report["backpressure_queue_samples"] = samples with open(os.path.join(args.out, "queue_overload.json"), "w", encoding="utf-8") as fh: json.dump(report, fh, ensure_ascii=False, indent=2) plot(report, os.path.join(args.out, "queue_overload.png")) def plot(report, path): import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt fig, ax = plt.subplots(1, 3, figsize=(17, 5)) sizes = ["small", "medium", "large"] concs = sorted({c["concurrency"] for c in report["matrix"]}) for k, (key, title, unit) in enumerate((("throughput_per_s", "Пропускная способность конвейера", "готово/с"), ("p95_s", "p95 времени до ready", "с"), ("worker_peak_rss_mib", "Пик RSS worker (лимит 768 МиБ)", "МиБ"))): for i, c in enumerate(concs): vals = [next((x[key] for x in report["matrix"] if x["concurrency"] == c and x["image"] == s), 0) or 0 for s in sizes] ax[k].bar([j + (i - 1) * 0.27 for j in range(3)], vals, width=0.27, label="concurrency=%d" % c) ax[k].set_xticks(range(3)); ax[k].set_xticklabels(["small 320×200", "medium 2400×1600", "large 6000×4000"]) ax[k].set_title(title); ax[k].set_ylabel(unit); ax[k].legend() fig.tight_layout(); fig.savefig(path, dpi=110) s = report["backpressure_queue_samples"] fig, ax = plt.subplots(figsize=(8, 4)) ax.plot([x[0] for x in s], [x[1] for x in s], label="глубина очереди (queued+processing)") ax.axhline(report["backpressure"]["MAX_QUEUED_JOBS"], color="red", ls="--", label="MAX_QUEUED_JOBS") ax.set_xlabel("время, с"); ax.set_ylabel("заданий"); ax.legend(); ax.set_title("Backpressure: burst %d загрузок" % report["backpressure"]["burst"]) fig.tight_layout(); fig.savefig(path.replace(".png", "_backpressure.png"), dpi=110) if __name__ == "__main__": ap = argparse.ArgumentParser() ap.add_argument("--out", default="evidence") ap.add_argument("--concurrency", type=int, nargs="+", default=[1, 2, 4]) ap.add_argument("--n-small", type=int, default=80) ap.add_argument("--n-medium", type=int, default=24) ap.add_argument("--n-large", type=int, default=12) ap.add_argument("--limit", type=int, default=30) ap.add_argument("--burst", type=int, default=200) ap.add_argument("--b-medium", action="store_true") asyncio.run(main(ap.parse_args()))