Files
mtusi2026lab5/student-tests/queue_load.py
T
alenchha 633fb96920 ЛР 5 — БВТ2301 — Курлова — на 5
RabbitMQ + transactional outbox, отдельный worker (thumbnail/crop/WebP), lease и heartbeat, retry с backoff+jitter, DLQ и replay, admission control; тесты отказов, трек 1 и отчёт.
2026-11-06 12:10:16 +03:00

251 lines
11 KiB
Python

#!/usr/bin/env python3
"""Track 1: the queue under load. Open-loop upload arrivals against the running stack, metrics from the DB.
Each run:
1. (re)creates the worker service with WORKER_REPLICAS / WORKER_CONCURRENCY / WORKER_PREFETCH
2. sends uploads at a FIXED arrival rate (open loop: the schedule does not wait for responses)
3. samples worker container memory (docker stats) and accepted-but-unfinished jobs once per second
4. waits until every accepted upload is ready/failed and computes from attachment_jobs:
queue lag = first_started_at - created_at (time a job waited for a worker)
time to ready = finished_at - attachment.created_at
throughput = finished jobs / (last finish - first arrival)
Rejections (503 queue_full from admission control) are counted, not retried.
python3 student-tests/queue_load.py run --size medium --rate 4 --duration 30 --concurrency 2 --prefetch 2
python3 student-tests/queue_load.py overload --max-pending 30 --size medium --rate 12 --duration 30
Results are appended as JSON lines to evidence/track1/results.jsonl.
"""
import argparse
import json
import os
import statistics
import subprocess
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from urllib.parse import quote
from stand import ROOT, Client, Failure, compose, log, sql, wait_http_ready
OUT = ROOT / "evidence/track1"
SIZES = {"small": (640, 400, "JPEG"), "medium": (4000, 3000, "JPEG"), "large": (6000, 4000, "PNG")}
GEN = r'''
import io, random, sys
from PIL import Image, ImageChops, ImageDraw
w, h, fmt = int(sys.argv[1]), int(sys.argv[2]), sys.argv[3]
if fmt == "JPEG": # photo-like noise: realistic JPEG sizes
chan = [Image.effect_noise((max(8, w // 24), max(8, h // 24)), 90).resize((w, h), Image.Resampling.BICUBIC) for _ in range(3)]
grain = Image.effect_noise((w, h), 12)
img = Image.merge("RGB", [ImageChops.add(c, grain, 1.0, -6) for c in chan])
else: # graphics-like PNG: compresses well (stays under 10 MiB) but must be fully decoded (no DCT scaling)
img = Image.linear_gradient("L").resize((w, h)).convert("RGB")
draw, rnd = ImageDraw.Draw(img), random.Random(1)
for _ in range(400):
x, y, r = rnd.randrange(w), rnd.randrange(h), rnd.randrange(20, 300)
draw.ellipse((x, y, x + r, y + r), fill=(rnd.randrange(256), rnd.randrange(256), rnd.randrange(256)))
buf = io.BytesIO()
img.save(buf, fmt, **({"quality": 90} if fmt == "JPEG" else {"compress_level": 6}))
sys.stdout.buffer.write(buf.getvalue())
'''
def image(size):
w, h, fmt = SIZES[size]
cache = OUT / ("fixture-%s.%s" % (size, fmt.lower()))
if not cache.exists():
OUT.mkdir(parents=True, exist_ok=True)
data = subprocess.run(["docker", "compose", "exec", "-T", "api-1", "python", "-c", GEN, str(w), str(h), fmt],
cwd=ROOT, capture_output=True, check=True).stdout
cache.write_bytes(data)
return cache.read_bytes(), "image/png" if fmt == "PNG" else "image/jpeg"
def set_worker(replicas, concurrency, prefetch, cpus="2.00"):
env = dict(os.environ, WORKER_REPLICAS=str(replicas), WORKER_CONCURRENCY=str(concurrency),
WORKER_PREFETCH=str(prefetch), WORKER_CPUS=cpus)
subprocess.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--wait", "worker"],
cwd=ROOT, env=env, check=True, capture_output=True, timeout=180)
def set_admission(max_pending):
env = dict(os.environ, MAX_PENDING_JOBS=str(max_pending))
subprocess.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--wait", "api-1", "api-2"],
cwd=ROOT, env=env, check=True, capture_output=True, timeout=180)
def parse_mem(text):
value = text.split("/")[0].strip()
for unit, mult in (("GiB", 1024), ("MiB", 1), ("KiB", 1 / 1024), ("B", 1 / 1048576)):
if value.endswith(unit):
return float(value[: -len(unit)]) * mult
return 0.0
class Sampler(threading.Thread):
def __init__(self, tag):
super().__init__(daemon=True)
self.tag = tag
self.stop = threading.Event()
self.rss, self.pending, self.cpu = [], [], []
def run(self):
while not self.stop.is_set():
ids = compose("ps", "-q", "worker", check_rc=False).split()
if ids:
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}|{{.CPUPerc}}", *ids],
capture_output=True, text=True).stdout
rows = [line.split("|") for line in out.splitlines() if "|" in line]
self.rss.append(sum(parse_mem(r[0]) for r in rows))
self.cpu.append(sum(float(r[1].rstrip("%") or 0) for r in rows))
try:
self.pending.append(int(sql("SELECT count(*) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id"
" WHERE a.filename LIKE 'load-%s-%%' AND j.state IN ('queued','processing')"
% self.tag)[0][0]))
except Exception:
pass
self.stop.wait(1.0)
def pct(values, q):
if not values:
return None
values = sorted(values)
return round(values[min(len(values) - 1, int(round(q * (len(values) - 1))))], 3)
def drive(c, actor, chat, body, ctype, rate, duration, tag):
"""Open-loop: request i is sent at t0 + i/rate regardless of earlier responses."""
total = int(rate * duration)
results = {"accepted": 0, "rejected_503": 0, "other_errors": 0}
lock = threading.Lock()
def one(i):
path = "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], quote("load-%s-%05d.img" % (tag, i)))
try:
r = c.call("POST", path, [202, 503], actor=actor, raw=body, headers={"Content-Type": ctype})
key = "accepted" if r["status"] == 202 else "rejected_503"
except Exception: # connection errors, timeouts: counted, never retried
key = "other_errors"
with lock:
results[key] += 1
t0 = time.monotonic()
with ThreadPoolExecutor(max_workers=64) as pool:
for i in range(total):
delay = t0 + i / rate - time.monotonic()
if delay > 0:
time.sleep(delay)
pool.submit(one, i)
results["send_seconds"] = round(time.monotonic() - t0, 2)
results["offered"] = total
return results
def wait_drained(tag, seconds):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
left = int(sql("SELECT count(*) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id"
" WHERE a.filename LIKE 'load-%s-%%' AND j.state IN ('queued','processing')" % tag)[0][0])
if left == 0:
return True
time.sleep(1)
return False
def job_metrics(tag):
rows = sql(
"SELECT extract(epoch FROM a.created_at), extract(epoch FROM j.first_started_at), extract(epoch FROM j.finished_at),"
" j.state, j.attempt FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id"
" WHERE a.filename LIKE 'load-%s-%%'" % tag)
done = [(float(a), float(s), float(f)) for a, s, f, st, _ in rows if st == "succeeded"]
lag = [s - a for a, s, f in done]
ttr = [f - a for a, s, f in done]
span = (max(f for _, _, f in done) - min(a for a, _, _ in done)) if done else 0
return {
"succeeded": len(done), "failed": sum(1 for r in rows if r[3] == "failed"),
"retries": sum(int(r[4]) - 1 for r in rows),
"throughput_jobs_s": round(len(done) / span, 3) if span else None,
"lag_p50_s": pct(lag, 0.5), "lag_p95_s": pct(lag, 0.95), "lag_max_s": pct(lag, 1.0),
"ttr_p50_s": pct(ttr, 0.5), "ttr_p95_s": pct(ttr, 0.95), "ttr_max_s": pct(ttr, 1.0),
}
def record(summary):
OUT.mkdir(parents=True, exist_ok=True)
with open(OUT / "results.jsonl", "a") as handle:
handle.write(json.dumps(summary, ensure_ascii=False) + "\n")
log(json.dumps(summary, ensure_ascii=False))
def experiment(args, label, max_pending=None):
c = Client(timeout=60)
wait_http_ready(c)
alice = c.user("load")
chat = c.call("POST", "/api/v1/chats", 201, actor=alice, body={"title": "load", "member_ids": [alice["id"]]},
schema="Chat")["json"]
body, ctype = image(args.size)
set_worker(args.replicas, args.concurrency, args.prefetch)
time.sleep(2)
tag = uuid.uuid4().hex[:8]
sampler = Sampler(tag)
sampler.start()
log("run %s tag=%s size=%s (%d KiB) rate=%s/s duration=%ss workers=%dx(c=%d,p=%d)" % (
label, tag, args.size, len(body) >> 10, args.rate, args.duration, args.replicas, args.concurrency, args.prefetch))
sent = drive(c, alice, chat, body, ctype, args.rate, args.duration, tag)
load_end = time.monotonic()
drained = wait_drained(tag, args.drain_timeout)
drain_seconds = round(time.monotonic() - load_end, 2)
sampler.stop.set()
sampler.join()
summary = {"label": label, "tag": tag, "size": args.size, "file_kib": len(body) >> 10, "rate": args.rate,
"duration": args.duration, "replicas": args.replicas, "concurrency": args.concurrency,
"prefetch": args.prefetch, "max_pending": max_pending, **sent, "drained": drained,
"drain_after_load_s": drain_seconds, **job_metrics(tag),
"pending_max": max(sampler.pending or [0]),
"worker_mem_peak_mib": round(max(sampler.rss or [0]), 1),
"worker_mem_median_mib": round(statistics.median(sampler.rss), 1) if sampler.rss else None,
"worker_cpu_peak_pct": round(max(sampler.cpu or [0]), 1),
"at": time.strftime("%Y-%m-%dT%H:%M:%S")}
return summary, c, alice, chat
def main():
parser = argparse.ArgumentParser()
parser.add_argument("mode", choices=["run", "overload"])
parser.add_argument("--size", choices=sorted(SIZES), default="medium")
parser.add_argument("--rate", type=float, default=4)
parser.add_argument("--duration", type=float, default=30)
parser.add_argument("--replicas", type=int, default=1)
parser.add_argument("--concurrency", type=int, default=2)
parser.add_argument("--prefetch", type=int, default=2)
parser.add_argument("--max-pending", type=int, default=30)
parser.add_argument("--drain-timeout", type=float, default=600)
parser.add_argument("--label", default="")
args = parser.parse_args()
if args.mode == "run":
summary, *_ = experiment(args, args.label or "c%d-p%d" % (args.concurrency, args.prefetch))
record(summary)
return
# Overload: admission limit on, offered rate far above capacity, then recovery.
set_admission(args.max_pending)
try:
summary, c, alice, chat = experiment(args, args.label or "overload", args.max_pending)
# Recovery probe: after the backlog drained, a fresh upload must be ready quickly again.
body, ctype = image("small")
t = time.monotonic()
att = c.call("POST", "/api/v1/chats/%s/attachments?filename=probe.png" % chat["id"], 202, actor=alice, raw=body,
headers={"Content-Type": ctype}, schema="Attachment")["json"]
while c.call("GET", "/api/v1/attachments/" + att["id"], actor=alice, schema="Attachment")["json"]["status"] != "ready":
time.sleep(0.2)
summary["recovery_probe_ready_s"] = round(time.monotonic() - t, 2)
record(summary)
finally:
set_admission(200)
if __name__ == "__main__":
main()