Files
mtusi2026lab5/research/queue_overload.py
T

310 lines
17 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Трек 1 (очередь под перегрузкой), lab5.
Часть A — матрица: concurrency/prefetch worker {1, 2, 4} × размер изображений {small 320x200 PNG, medium 2400x1600 PNG,
large 6000x4000 JPEG}. Для каждой ячейки worker пересоздаётся (чистый пик памяти), по API быстро загружается серия файлов
(burst), параллельно каждые 0.5 с снимаются глубина очереди и возраст старейшей задачи. Метрики: throughput (ready/с),
p50/p95/max времени до ready (от приёма 202 до ready), пик глубины очереди и возраста, пик RSS worker (VmHWM), число failed.
Часть B — backpressure и восстановление: MAX_QUEUED_JOBS=30, worker concurrency 1, burst 200 мелких файлов 32 клиентами:
API ограничивает приём по порогу очереди и отвечает 503 + Retry-After (возможен конкурентный overshoot);
принятые задания не теряются (все ready);
после разгрузки новая загрузка снова принимается.
python research/queue_overload.py --out evidence
"""
import argparse
import asyncio
import base64
import io
import json
import math
import os
import statistics
import subprocess
import threading
import time
import uuid
import aiohttp
from PIL import Image
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
BASE = "http://127.0.0.1:8080"
PROJECT = "mtusi-lab5"
def sh(*args, env=None):
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True,
env=dict(os.environ, **(env or {}))).stdout.strip()
def psql(sql):
return sh("docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "|", "-c", sql)
def healthy(service, expected=1):
ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", service).split()
return len(ids) >= expected and all(sh("docker", "inspect", "-f", "{{.State.Health.Status}}", i) == "healthy" for i in ids)
def recreate(service, env, wait_seconds=90):
sh("docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--force-recreate", service, env=env)
deadline = time.time() + wait_seconds
while time.time() < deadline:
if healthy(service, 1 if service == "worker" else 2):
break
time.sleep(1)
else:
raise RuntimeError("%s did not become healthy within %d seconds" % (service, wait_seconds))
time.sleep(6 if service == "api" else 3)
def make_images(root):
import random
random.seed(11)
out = {}
out["small"] = open(os.path.join(root, "student-tests", "fixtures", "sample.png"), "rb").read()
def noisy(w, h):
base = Image.effect_noise((w // 8, h // 8), 60).convert("RGB").resize((w, h), Image.BILINEAR)
grad = Image.linear_gradient("L").resize((w, h)).convert("RGB")
return Image.blend(base, grad, 0.5)
buf = io.BytesIO()
noisy(2400, 1600).save(buf, "PNG", compress_level=3)
out["medium"] = buf.getvalue()
buf = io.BytesIO()
noisy(6000, 4000).save(buf, "JPEG", quality=88)
out["large"] = buf.getvalue()
return out
async def register(s):
name = "qo_%s" % uuid.uuid4().hex[:12]
pw = "queue-test-pw-" + uuid.uuid4().hex[:6]
async with s.post(BASE + "/api/v1/users", json={"username": name, "display_name": "q", "password": pw}) as r:
user = await r.json()
cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode()
async with s.post(BASE + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r:
token = (await r.json())["token"]
async with s.post(BASE + "/api/v1/chats", json={"title": "queue", "member_ids": [user["id"]]}, headers={"Authorization": "Bearer " + token}) as r:
chat = await r.json()
return token, chat["id"]
class QueueSampler(threading.Thread):
def __init__(self):
super().__init__(daemon=True)
self.samples, self.stop, self.errors = [], threading.Event(), []
self.t0 = time.time()
def run(self):
while not self.stop.is_set():
try:
out = psql("SELECT count(*), coalesce(extract(epoch FROM now() - min(accepted_at)), 0) FROM attachments WHERE status IN ('queued', 'processing')")
n, age = out.split("|")
self.samples.append((time.time() - self.t0, int(n), float(age)))
except Exception as exc:
self.errors.append(type(exc).__name__)
return
self.stop.wait(0.4)
def worker_peak_rss():
ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", "worker").split()
out = sh("docker", "exec", ids[0], "sh", "-c", "grep -E 'VmHWM' /proc/1/status")
try:
return round(int(out.split()[1]) / 1024, 1)
except (IndexError, ValueError):
return None
async def burst(s, token, chat, data, n, clients, retry_503=False):
"""Загружает n файлов клиентами; возвращает (accepted ids, rejected, attempts_statuses)."""
accepted, statuses, sem = [], {}, asyncio.Semaphore(clients)
observed = {"retry_after_values": [], "transport_errors": []}
async def one(i):
async with sem:
for attempt in range(60):
try:
async with s.post("%s/api/v1/chats/%s/attachments?filename=q%d.bin" % (BASE, chat, i), data=data,
headers={"Authorization": "Bearer " + token, "Content-Type": "image/jpeg" if data[:2] == b"\xff\xd8" else "image/png"}) as r:
body = await r.json(content_type=None)
statuses[str(r.status)] = statuses.get(str(r.status), 0) + 1
if r.status in (503, 429):
observed["retry_after_values"].append(r.headers.get("Retry-After"))
if r.status == 202:
accepted.append(body["id"])
return
if r.status in (503, 429) and retry_503:
await asyncio.sleep(float(r.headers.get("Retry-After", "1")))
continue
return
except aiohttp.ClientError as exc:
# Upload is not idempotent: an uncertain response must not trigger a hidden retry.
observed["transport_errors"].append(type(exc).__name__)
return
await asyncio.gather(*(one(i) for i in range(n)))
return accepted, statuses, observed
def wait_all(ids, seconds):
if not ids:
raise ValueError("no accepted jobs to measure")
deadline = time.time() + seconds
while time.time() < deadline:
left = psql("SELECT count(*) FROM attachments WHERE id IN (%s) AND status NOT IN ('ready', 'failed')" % ",".join("'%s'" % i for i in ids))
if left == "0":
return True
time.sleep(0.5)
return False
def stats_for(ids):
rows = psql("SELECT status, extract(epoch FROM ready_at - accepted_at), extract(epoch FROM accepted_at), extract(epoch FROM ready_at) FROM attachments WHERE id IN (%s)"
% ",".join("'%s'" % i for i in ids)).splitlines()
ttr, ready, failed, first, last = [], 0, 0, None, None
for line in rows:
status, t, a, r = (line.split("|") + ["", "", ""])[:4]
if status == "ready" and t:
ready += 1
ttr.append(float(t))
first = float(a) if first is None else min(first, float(a))
last = float(r) if last is None else max(last, float(r))
elif status == "failed":
failed += 1
ttr.sort()
pct = lambda p: round(ttr[max(0, math.ceil(p * len(ttr)) - 1)], 2) if ttr else None # noqa: E731
span = (last - first) if first is not None else None
return {"ready": ready, "failed": failed, "p50_s": pct(0.5), "p95_s": pct(0.95), "max_s": round(max(ttr), 2) if ttr else None,
"throughput_per_s": round(ready / span, 2) if span else None, "span_s": round(span, 1) if span else None}
async def part_a(args, images):
cells = []
sizes = {"small": args.n_small, "medium": args.n_medium, "large": args.n_large}
for conc in args.concurrency:
for size, n in sizes.items():
recreate("worker", {"WORKER_CONCURRENCY": str(conc)})
ids_worker = sh("docker", "compose", "ps", "-q", "worker").split()
assert len(ids_worker) == 1, "this experiment requires exactly one worker"
limits = json.loads(sh("docker", "inspect", "-f",
'{"memory_bytes":{{.HostConfig.Memory}},"nano_cpus":{{.HostConfig.NanoCpus}}}', ids_worker[0]))
assert limits == {"memory_bytes": 768 * 1024 * 1024, "nano_cpus": 1000000000}, limits
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=120), connector=aiohttp.TCPConnector(limit=16, force_close=True)) as s:
token, chat = await register(s)
sampler = QueueSampler()
sampler.start()
t0 = time.time()
ids, statuses, observed = await burst(s, token, chat, images[size], n, 8)
accepted_in = time.time() - t0
done = wait_all(ids, 400)
sampler.stop.set()
sampler.join()
cell = {"concurrency": conc, "image": size, "image_kib": round(len(images[size]) / 1024), "uploads": n, "accepted": len(ids),
"resource_limits": limits, "attempt_statuses": statuses, "transport_errors": observed["transport_errors"],
"accepted_in_s": round(accepted_in, 1), "all_finished": done, "peak_queue_depth": max((x[1] for x in sampler.samples), default=0),
"peak_oldest_age_s": round(max((x[2] for x in sampler.samples), default=0), 1), "worker_peak_rss_mib": worker_peak_rss(),
"worker_oom_killed": sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", sh("docker", "compose", "--env-file", ".env", "ps", "-q", "worker").split()[0])}
cell.update(stats_for(ids))
print(json.dumps(cell, ensure_ascii=False))
cells.append(cell)
with open(os.path.join(args.out, "matrix_progress.json"), "w", encoding="utf-8") as fh:
json.dump(cells, fh, ensure_ascii=False, indent=2)
assert not sampler.errors and sampler.samples, sampler.errors
assert len(ids) == len(set(ids)) == n and statuses == {"202": n} and not observed["transport_errors"], cell
assert done and cell["ready"] == n and cell["failed"] == 0, cell
assert cell["worker_oom_killed"] == "false" and 0 < cell["worker_peak_rss_mib"] < 768, cell
return cells
async def part_b(args, images):
recreate("api", {"MAX_QUEUED_JOBS": str(args.limit)})
recreate("worker", {"WORKER_CONCURRENCY": "1"})
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=120), connector=aiohttp.TCPConnector(limit=64, force_close=True)) as s:
token, chat = await register(s)
sampler = QueueSampler()
sampler.start()
t0 = time.time()
ids, statuses, observed = await burst(s, token, chat, images["small"] if not args.b_medium else images["medium"], args.burst, 32, retry_503=False)
accepted_in = time.time() - t0
done = wait_all(ids, 300)
drained_after = time.time() - t0
sampler.stop.set()
sampler.join()
# восстановление: после разгрузки новая загрузка принимается
extra, st2, observed2 = await burst(s, token, chat, images["small"], 3, 1)
extra_done = wait_all(extra, 60)
throttled = observed["retry_after_values"]
valid_retry_after = bool(throttled) and all(v is not None and v.isdigit() and int(v) >= 1 for v in throttled)
res = {"MAX_QUEUED_JOBS": args.limit, "burst": args.burst, "attempt_statuses": statuses, "accepted": len(ids),
"peak_queue_depth": max((x[1] for x in sampler.samples), default=0), "all_accepted_finished": done,
"accepted_in_s": round(accepted_in, 1), "drained_after_s": round(drained_after, 1),
"retry_after_header_seen": valid_retry_after, "retry_after_values": sorted(set(throttled), key=str),
"transport_errors": observed["transport_errors"], "recovered_new_uploads_accepted": len(extra) == 3,
"recovered_new_uploads_ready": extra_done and stats_for(extra)["ready"] == 3,
"no_accepted_job_lost": stats_for(ids)["ready"] + stats_for(ids)["failed"] == len(ids), **{k: v for k, v in stats_for(ids).items() if k in ("ready", "failed", "p95_s")}}
print(json.dumps(res, ensure_ascii=False))
with open(os.path.join(args.out, "backpressure_progress.json"), "w", encoding="utf-8") as fh:
json.dump(res, fh, ensure_ascii=False, indent=2)
assert not sampler.errors and sampler.samples, sampler.errors
assert statuses.get("503", 0) > 0 and sum(statuses.values()) == args.burst and valid_retry_after, res
assert set(statuses) <= {"202", "503", "429"} and not observed["transport_errors"], res
assert done and res["ready"] == len(ids) and res["failed"] == 0 and res["no_accepted_job_lost"], res
assert res["recovered_new_uploads_accepted"] and res["recovered_new_uploads_ready"] and not observed2["transport_errors"], res
return res, [(round(t, 1), n, round(a, 1)) for t, n, a in sampler.samples]
async def main(args):
os.makedirs(args.out, exist_ok=True)
images = make_images(ROOT)
print({k: "%d KiB" % (len(v) // 1024) for k, v in images.items()})
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "args": vars(args), "worker_limits": {"cpus": 1.0, "memory": "768m"},
"image_kib": {k: round(len(v) / 1024) for k, v in images.items()}}
try:
report["matrix"] = await part_a(args, images)
report["backpressure"], samples = await part_b(args, images)
finally:
recreate("api", {})
recreate("worker", {})
report["backpressure_queue_samples"] = samples
with open(os.path.join(args.out, "queue_overload.json"), "w", encoding="utf-8") as fh:
json.dump(report, fh, ensure_ascii=False, indent=2)
plot(report, os.path.join(args.out, "queue_overload.png"))
def plot(report, path):
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
fig, ax = plt.subplots(1, 3, figsize=(17, 5))
sizes = ["small", "medium", "large"]
concs = sorted({c["concurrency"] for c in report["matrix"]})
for k, (key, title, unit) in enumerate((("throughput_per_s", "Пропускная способность конвейера", "готово/с"), ("p95_s", "p95 времени до ready", "с"),
("worker_peak_rss_mib", "Пик RSS worker (лимит 768 МиБ)", "МиБ"))):
for i, c in enumerate(concs):
vals = [next((x[key] for x in report["matrix"] if x["concurrency"] == c and x["image"] == s), 0) or 0 for s in sizes]
ax[k].bar([j + (i - 1) * 0.27 for j in range(3)], vals, width=0.27, label="concurrency=%d" % c)
ax[k].set_xticks(range(3)); ax[k].set_xticklabels(["small 320×200", "medium 2400×1600", "large 6000×4000"])
ax[k].set_title(title); ax[k].set_ylabel(unit); ax[k].legend()
fig.tight_layout(); fig.savefig(path, dpi=110)
s = report["backpressure_queue_samples"]
fig, ax = plt.subplots(figsize=(8, 4))
ax.plot([x[0] for x in s], [x[1] for x in s], label="глубина очереди (queued+processing)")
ax.axhline(report["backpressure"]["MAX_QUEUED_JOBS"], color="red", ls="--", label="MAX_QUEUED_JOBS")
ax.set_xlabel("время, с"); ax.set_ylabel("заданий"); ax.legend(); ax.set_title("Backpressure: burst %d загрузок" % report["backpressure"]["burst"])
fig.tight_layout(); fig.savefig(path.replace(".png", "_backpressure.png"), dpi=110)
if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument("--out", default="evidence")
ap.add_argument("--concurrency", type=int, nargs="+", default=[1, 2, 4])
ap.add_argument("--n-small", type=int, default=80)
ap.add_argument("--n-medium", type=int, default=24)
ap.add_argument("--n-large", type=int, default=12)
ap.add_argument("--limit", type=int, default=30)
ap.add_argument("--burst", type=int, default=200)
ap.add_argument("--b-medium", action="store_true")
asyncio.run(main(ap.parse_args()))