forked from mtusi2026/mtusi2026lab5
RabbitMQ + transactional outbox, отдельный worker (thumbnail/crop/WebP), lease и heartbeat, retry с backoff+jitter, DLQ и replay, admission control; тесты отказов, трек 1 и отчёт.
251 lines
11 KiB
Python
251 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Track 1: the queue under load. Open-loop upload arrivals against the running stack, metrics from the DB.
|
|
|
|
Each run:
|
|
1. (re)creates the worker service with WORKER_REPLICAS / WORKER_CONCURRENCY / WORKER_PREFETCH
|
|
2. sends uploads at a FIXED arrival rate (open loop: the schedule does not wait for responses)
|
|
3. samples worker container memory (docker stats) and accepted-but-unfinished jobs once per second
|
|
4. waits until every accepted upload is ready/failed and computes from attachment_jobs:
|
|
queue lag = first_started_at - created_at (time a job waited for a worker)
|
|
time to ready = finished_at - attachment.created_at
|
|
throughput = finished jobs / (last finish - first arrival)
|
|
Rejections (503 queue_full from admission control) are counted, not retried.
|
|
|
|
python3 student-tests/queue_load.py run --size medium --rate 4 --duration 30 --concurrency 2 --prefetch 2
|
|
python3 student-tests/queue_load.py overload --max-pending 30 --size medium --rate 12 --duration 30
|
|
Results are appended as JSON lines to evidence/track1/results.jsonl.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import statistics
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
from urllib.parse import quote
|
|
|
|
from stand import ROOT, Client, Failure, compose, log, sql, wait_http_ready
|
|
|
|
OUT = ROOT / "evidence/track1"
|
|
SIZES = {"small": (640, 400, "JPEG"), "medium": (4000, 3000, "JPEG"), "large": (6000, 4000, "PNG")}
|
|
|
|
GEN = r'''
|
|
import io, random, sys
|
|
from PIL import Image, ImageChops, ImageDraw
|
|
w, h, fmt = int(sys.argv[1]), int(sys.argv[2]), sys.argv[3]
|
|
if fmt == "JPEG": # photo-like noise: realistic JPEG sizes
|
|
chan = [Image.effect_noise((max(8, w // 24), max(8, h // 24)), 90).resize((w, h), Image.Resampling.BICUBIC) for _ in range(3)]
|
|
grain = Image.effect_noise((w, h), 12)
|
|
img = Image.merge("RGB", [ImageChops.add(c, grain, 1.0, -6) for c in chan])
|
|
else: # graphics-like PNG: compresses well (stays under 10 MiB) but must be fully decoded (no DCT scaling)
|
|
img = Image.linear_gradient("L").resize((w, h)).convert("RGB")
|
|
draw, rnd = ImageDraw.Draw(img), random.Random(1)
|
|
for _ in range(400):
|
|
x, y, r = rnd.randrange(w), rnd.randrange(h), rnd.randrange(20, 300)
|
|
draw.ellipse((x, y, x + r, y + r), fill=(rnd.randrange(256), rnd.randrange(256), rnd.randrange(256)))
|
|
buf = io.BytesIO()
|
|
img.save(buf, fmt, **({"quality": 90} if fmt == "JPEG" else {"compress_level": 6}))
|
|
sys.stdout.buffer.write(buf.getvalue())
|
|
'''
|
|
|
|
|
|
def image(size):
|
|
w, h, fmt = SIZES[size]
|
|
cache = OUT / ("fixture-%s.%s" % (size, fmt.lower()))
|
|
if not cache.exists():
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
data = subprocess.run(["docker", "compose", "exec", "-T", "api-1", "python", "-c", GEN, str(w), str(h), fmt],
|
|
cwd=ROOT, capture_output=True, check=True).stdout
|
|
cache.write_bytes(data)
|
|
return cache.read_bytes(), "image/png" if fmt == "PNG" else "image/jpeg"
|
|
|
|
|
|
def set_worker(replicas, concurrency, prefetch, cpus="2.00"):
|
|
env = dict(os.environ, WORKER_REPLICAS=str(replicas), WORKER_CONCURRENCY=str(concurrency),
|
|
WORKER_PREFETCH=str(prefetch), WORKER_CPUS=cpus)
|
|
subprocess.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--wait", "worker"],
|
|
cwd=ROOT, env=env, check=True, capture_output=True, timeout=180)
|
|
|
|
|
|
def set_admission(max_pending):
|
|
env = dict(os.environ, MAX_PENDING_JOBS=str(max_pending))
|
|
subprocess.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--wait", "api-1", "api-2"],
|
|
cwd=ROOT, env=env, check=True, capture_output=True, timeout=180)
|
|
|
|
|
|
def parse_mem(text):
|
|
value = text.split("/")[0].strip()
|
|
for unit, mult in (("GiB", 1024), ("MiB", 1), ("KiB", 1 / 1024), ("B", 1 / 1048576)):
|
|
if value.endswith(unit):
|
|
return float(value[: -len(unit)]) * mult
|
|
return 0.0
|
|
|
|
|
|
class Sampler(threading.Thread):
|
|
def __init__(self, tag):
|
|
super().__init__(daemon=True)
|
|
self.tag = tag
|
|
self.stop = threading.Event()
|
|
self.rss, self.pending, self.cpu = [], [], []
|
|
|
|
def run(self):
|
|
while not self.stop.is_set():
|
|
ids = compose("ps", "-q", "worker", check_rc=False).split()
|
|
if ids:
|
|
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}|{{.CPUPerc}}", *ids],
|
|
capture_output=True, text=True).stdout
|
|
rows = [line.split("|") for line in out.splitlines() if "|" in line]
|
|
self.rss.append(sum(parse_mem(r[0]) for r in rows))
|
|
self.cpu.append(sum(float(r[1].rstrip("%") or 0) for r in rows))
|
|
try:
|
|
self.pending.append(int(sql("SELECT count(*) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id"
|
|
" WHERE a.filename LIKE 'load-%s-%%' AND j.state IN ('queued','processing')"
|
|
% self.tag)[0][0]))
|
|
except Exception:
|
|
pass
|
|
self.stop.wait(1.0)
|
|
|
|
|
|
def pct(values, q):
|
|
if not values:
|
|
return None
|
|
values = sorted(values)
|
|
return round(values[min(len(values) - 1, int(round(q * (len(values) - 1))))], 3)
|
|
|
|
|
|
def drive(c, actor, chat, body, ctype, rate, duration, tag):
|
|
"""Open-loop: request i is sent at t0 + i/rate regardless of earlier responses."""
|
|
total = int(rate * duration)
|
|
results = {"accepted": 0, "rejected_503": 0, "other_errors": 0}
|
|
lock = threading.Lock()
|
|
|
|
def one(i):
|
|
path = "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], quote("load-%s-%05d.img" % (tag, i)))
|
|
try:
|
|
r = c.call("POST", path, [202, 503], actor=actor, raw=body, headers={"Content-Type": ctype})
|
|
key = "accepted" if r["status"] == 202 else "rejected_503"
|
|
except Exception: # connection errors, timeouts: counted, never retried
|
|
key = "other_errors"
|
|
with lock:
|
|
results[key] += 1
|
|
|
|
t0 = time.monotonic()
|
|
with ThreadPoolExecutor(max_workers=64) as pool:
|
|
for i in range(total):
|
|
delay = t0 + i / rate - time.monotonic()
|
|
if delay > 0:
|
|
time.sleep(delay)
|
|
pool.submit(one, i)
|
|
results["send_seconds"] = round(time.monotonic() - t0, 2)
|
|
results["offered"] = total
|
|
return results
|
|
|
|
|
|
def wait_drained(tag, seconds):
|
|
deadline = time.monotonic() + seconds
|
|
while time.monotonic() < deadline:
|
|
left = int(sql("SELECT count(*) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id"
|
|
" WHERE a.filename LIKE 'load-%s-%%' AND j.state IN ('queued','processing')" % tag)[0][0])
|
|
if left == 0:
|
|
return True
|
|
time.sleep(1)
|
|
return False
|
|
|
|
|
|
def job_metrics(tag):
|
|
rows = sql(
|
|
"SELECT extract(epoch FROM a.created_at), extract(epoch FROM j.first_started_at), extract(epoch FROM j.finished_at),"
|
|
" j.state, j.attempt FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id"
|
|
" WHERE a.filename LIKE 'load-%s-%%'" % tag)
|
|
done = [(float(a), float(s), float(f)) for a, s, f, st, _ in rows if st == "succeeded"]
|
|
lag = [s - a for a, s, f in done]
|
|
ttr = [f - a for a, s, f in done]
|
|
span = (max(f for _, _, f in done) - min(a for a, _, _ in done)) if done else 0
|
|
return {
|
|
"succeeded": len(done), "failed": sum(1 for r in rows if r[3] == "failed"),
|
|
"retries": sum(int(r[4]) - 1 for r in rows),
|
|
"throughput_jobs_s": round(len(done) / span, 3) if span else None,
|
|
"lag_p50_s": pct(lag, 0.5), "lag_p95_s": pct(lag, 0.95), "lag_max_s": pct(lag, 1.0),
|
|
"ttr_p50_s": pct(ttr, 0.5), "ttr_p95_s": pct(ttr, 0.95), "ttr_max_s": pct(ttr, 1.0),
|
|
}
|
|
|
|
|
|
def record(summary):
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
with open(OUT / "results.jsonl", "a") as handle:
|
|
handle.write(json.dumps(summary, ensure_ascii=False) + "\n")
|
|
log(json.dumps(summary, ensure_ascii=False))
|
|
|
|
|
|
def experiment(args, label, max_pending=None):
|
|
c = Client(timeout=60)
|
|
wait_http_ready(c)
|
|
alice = c.user("load")
|
|
chat = c.call("POST", "/api/v1/chats", 201, actor=alice, body={"title": "load", "member_ids": [alice["id"]]},
|
|
schema="Chat")["json"]
|
|
body, ctype = image(args.size)
|
|
set_worker(args.replicas, args.concurrency, args.prefetch)
|
|
time.sleep(2)
|
|
tag = uuid.uuid4().hex[:8]
|
|
sampler = Sampler(tag)
|
|
sampler.start()
|
|
log("run %s tag=%s size=%s (%d KiB) rate=%s/s duration=%ss workers=%dx(c=%d,p=%d)" % (
|
|
label, tag, args.size, len(body) >> 10, args.rate, args.duration, args.replicas, args.concurrency, args.prefetch))
|
|
sent = drive(c, alice, chat, body, ctype, args.rate, args.duration, tag)
|
|
load_end = time.monotonic()
|
|
drained = wait_drained(tag, args.drain_timeout)
|
|
drain_seconds = round(time.monotonic() - load_end, 2)
|
|
sampler.stop.set()
|
|
sampler.join()
|
|
summary = {"label": label, "tag": tag, "size": args.size, "file_kib": len(body) >> 10, "rate": args.rate,
|
|
"duration": args.duration, "replicas": args.replicas, "concurrency": args.concurrency,
|
|
"prefetch": args.prefetch, "max_pending": max_pending, **sent, "drained": drained,
|
|
"drain_after_load_s": drain_seconds, **job_metrics(tag),
|
|
"pending_max": max(sampler.pending or [0]),
|
|
"worker_mem_peak_mib": round(max(sampler.rss or [0]), 1),
|
|
"worker_mem_median_mib": round(statistics.median(sampler.rss), 1) if sampler.rss else None,
|
|
"worker_cpu_peak_pct": round(max(sampler.cpu or [0]), 1),
|
|
"at": time.strftime("%Y-%m-%dT%H:%M:%S")}
|
|
return summary, c, alice, chat
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("mode", choices=["run", "overload"])
|
|
parser.add_argument("--size", choices=sorted(SIZES), default="medium")
|
|
parser.add_argument("--rate", type=float, default=4)
|
|
parser.add_argument("--duration", type=float, default=30)
|
|
parser.add_argument("--replicas", type=int, default=1)
|
|
parser.add_argument("--concurrency", type=int, default=2)
|
|
parser.add_argument("--prefetch", type=int, default=2)
|
|
parser.add_argument("--max-pending", type=int, default=30)
|
|
parser.add_argument("--drain-timeout", type=float, default=600)
|
|
parser.add_argument("--label", default="")
|
|
args = parser.parse_args()
|
|
if args.mode == "run":
|
|
summary, *_ = experiment(args, args.label or "c%d-p%d" % (args.concurrency, args.prefetch))
|
|
record(summary)
|
|
return
|
|
# Overload: admission limit on, offered rate far above capacity, then recovery.
|
|
set_admission(args.max_pending)
|
|
try:
|
|
summary, c, alice, chat = experiment(args, args.label or "overload", args.max_pending)
|
|
# Recovery probe: after the backlog drained, a fresh upload must be ready quickly again.
|
|
body, ctype = image("small")
|
|
t = time.monotonic()
|
|
att = c.call("POST", "/api/v1/chats/%s/attachments?filename=probe.png" % chat["id"], 202, actor=alice, raw=body,
|
|
headers={"Content-Type": ctype}, schema="Attachment")["json"]
|
|
while c.call("GET", "/api/v1/attachments/" + att["id"], actor=alice, schema="Attachment")["json"]["status"] != "ready":
|
|
time.sleep(0.2)
|
|
summary["recovery_probe_ready_s"] = round(time.monotonic() - t, 2)
|
|
record(summary)
|
|
finally:
|
|
set_admission(200)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|