#!/usr/bin/env python3 """Track 1: the queue under load. Open-loop upload arrivals against the running stack, metrics from the DB. Each run: 1. (re)creates the worker service with WORKER_REPLICAS / WORKER_CONCURRENCY / WORKER_PREFETCH 2. sends uploads at a FIXED arrival rate (open loop: the schedule does not wait for responses) 3. samples worker container memory (docker stats) and accepted-but-unfinished jobs once per second 4. waits until every accepted upload is ready/failed and computes from attachment_jobs: queue lag = first_started_at - created_at (time a job waited for a worker) time to ready = finished_at - attachment.created_at throughput = finished jobs / (last finish - first arrival) Rejections (503 queue_full from admission control) are counted, not retried. python3 student-tests/queue_load.py run --size medium --rate 4 --duration 30 --concurrency 2 --prefetch 2 python3 student-tests/queue_load.py overload --max-pending 30 --size medium --rate 12 --duration 30 Results are appended as JSON lines to evidence/track1/results.jsonl. """ import argparse import json import os import statistics import subprocess import threading import time import uuid from concurrent.futures import ThreadPoolExecutor from pathlib import Path from urllib.parse import quote from stand import ROOT, Client, Failure, compose, log, sql, wait_http_ready OUT = ROOT / "evidence/track1" SIZES = {"small": (640, 400, "JPEG"), "medium": (4000, 3000, "JPEG"), "large": (6000, 4000, "PNG")} GEN = r''' import io, random, sys from PIL import Image, ImageChops, ImageDraw w, h, fmt = int(sys.argv[1]), int(sys.argv[2]), sys.argv[3] if fmt == "JPEG": # photo-like noise: realistic JPEG sizes chan = [Image.effect_noise((max(8, w // 24), max(8, h // 24)), 90).resize((w, h), Image.Resampling.BICUBIC) for _ in range(3)] grain = Image.effect_noise((w, h), 12) img = Image.merge("RGB", [ImageChops.add(c, grain, 1.0, -6) for c in chan]) else: # graphics-like PNG: compresses well (stays under 10 MiB) but must be fully decoded (no DCT scaling) img = Image.linear_gradient("L").resize((w, h)).convert("RGB") draw, rnd = ImageDraw.Draw(img), random.Random(1) for _ in range(400): x, y, r = rnd.randrange(w), rnd.randrange(h), rnd.randrange(20, 300) draw.ellipse((x, y, x + r, y + r), fill=(rnd.randrange(256), rnd.randrange(256), rnd.randrange(256))) buf = io.BytesIO() img.save(buf, fmt, **({"quality": 90} if fmt == "JPEG" else {"compress_level": 6})) sys.stdout.buffer.write(buf.getvalue()) ''' def image(size): w, h, fmt = SIZES[size] cache = OUT / ("fixture-%s.%s" % (size, fmt.lower())) if not cache.exists(): OUT.mkdir(parents=True, exist_ok=True) data = subprocess.run(["docker", "compose", "exec", "-T", "api-1", "python", "-c", GEN, str(w), str(h), fmt], cwd=ROOT, capture_output=True, check=True).stdout cache.write_bytes(data) return cache.read_bytes(), "image/png" if fmt == "PNG" else "image/jpeg" def set_worker(replicas, concurrency, prefetch, cpus="2.00"): env = dict(os.environ, WORKER_REPLICAS=str(replicas), WORKER_CONCURRENCY=str(concurrency), WORKER_PREFETCH=str(prefetch), WORKER_CPUS=cpus) subprocess.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--wait", "worker"], cwd=ROOT, env=env, check=True, capture_output=True, timeout=180) def set_admission(max_pending): env = dict(os.environ, MAX_PENDING_JOBS=str(max_pending)) subprocess.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--wait", "api-1", "api-2"], cwd=ROOT, env=env, check=True, capture_output=True, timeout=180) def parse_mem(text): value = text.split("/")[0].strip() for unit, mult in (("GiB", 1024), ("MiB", 1), ("KiB", 1 / 1024), ("B", 1 / 1048576)): if value.endswith(unit): return float(value[: -len(unit)]) * mult return 0.0 class Sampler(threading.Thread): def __init__(self, tag): super().__init__(daemon=True) self.tag = tag self.stop = threading.Event() self.rss, self.pending, self.cpu = [], [], [] def run(self): while not self.stop.is_set(): ids = compose("ps", "-q", "worker", check_rc=False).split() if ids: out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}|{{.CPUPerc}}", *ids], capture_output=True, text=True).stdout rows = [line.split("|") for line in out.splitlines() if "|" in line] self.rss.append(sum(parse_mem(r[0]) for r in rows)) self.cpu.append(sum(float(r[1].rstrip("%") or 0) for r in rows)) try: self.pending.append(int(sql("SELECT count(*) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id" " WHERE a.filename LIKE 'load-%s-%%' AND j.state IN ('queued','processing')" % self.tag)[0][0])) except Exception: pass self.stop.wait(1.0) def pct(values, q): if not values: return None values = sorted(values) return round(values[min(len(values) - 1, int(round(q * (len(values) - 1))))], 3) def drive(c, actor, chat, body, ctype, rate, duration, tag): """Open-loop: request i is sent at t0 + i/rate regardless of earlier responses.""" total = int(rate * duration) results = {"accepted": 0, "rejected_503": 0, "other_errors": 0} lock = threading.Lock() def one(i): path = "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], quote("load-%s-%05d.img" % (tag, i))) try: r = c.call("POST", path, [202, 503], actor=actor, raw=body, headers={"Content-Type": ctype}) key = "accepted" if r["status"] == 202 else "rejected_503" except Exception: # connection errors, timeouts: counted, never retried key = "other_errors" with lock: results[key] += 1 t0 = time.monotonic() with ThreadPoolExecutor(max_workers=64) as pool: for i in range(total): delay = t0 + i / rate - time.monotonic() if delay > 0: time.sleep(delay) pool.submit(one, i) results["send_seconds"] = round(time.monotonic() - t0, 2) results["offered"] = total return results def wait_drained(tag, seconds): deadline = time.monotonic() + seconds while time.monotonic() < deadline: left = int(sql("SELECT count(*) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id" " WHERE a.filename LIKE 'load-%s-%%' AND j.state IN ('queued','processing')" % tag)[0][0]) if left == 0: return True time.sleep(1) return False def job_metrics(tag): rows = sql( "SELECT extract(epoch FROM a.created_at), extract(epoch FROM j.first_started_at), extract(epoch FROM j.finished_at)," " j.state, j.attempt FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id" " WHERE a.filename LIKE 'load-%s-%%'" % tag) done = [(float(a), float(s), float(f)) for a, s, f, st, _ in rows if st == "succeeded"] lag = [s - a for a, s, f in done] ttr = [f - a for a, s, f in done] span = (max(f for _, _, f in done) - min(a for a, _, _ in done)) if done else 0 return { "succeeded": len(done), "failed": sum(1 for r in rows if r[3] == "failed"), "retries": sum(int(r[4]) - 1 for r in rows), "throughput_jobs_s": round(len(done) / span, 3) if span else None, "lag_p50_s": pct(lag, 0.5), "lag_p95_s": pct(lag, 0.95), "lag_max_s": pct(lag, 1.0), "ttr_p50_s": pct(ttr, 0.5), "ttr_p95_s": pct(ttr, 0.95), "ttr_max_s": pct(ttr, 1.0), } def record(summary): OUT.mkdir(parents=True, exist_ok=True) with open(OUT / "results.jsonl", "a") as handle: handle.write(json.dumps(summary, ensure_ascii=False) + "\n") log(json.dumps(summary, ensure_ascii=False)) def experiment(args, label, max_pending=None): c = Client(timeout=60) wait_http_ready(c) alice = c.user("load") chat = c.call("POST", "/api/v1/chats", 201, actor=alice, body={"title": "load", "member_ids": [alice["id"]]}, schema="Chat")["json"] body, ctype = image(args.size) set_worker(args.replicas, args.concurrency, args.prefetch) time.sleep(2) tag = uuid.uuid4().hex[:8] sampler = Sampler(tag) sampler.start() log("run %s tag=%s size=%s (%d KiB) rate=%s/s duration=%ss workers=%dx(c=%d,p=%d)" % ( label, tag, args.size, len(body) >> 10, args.rate, args.duration, args.replicas, args.concurrency, args.prefetch)) sent = drive(c, alice, chat, body, ctype, args.rate, args.duration, tag) load_end = time.monotonic() drained = wait_drained(tag, args.drain_timeout) drain_seconds = round(time.monotonic() - load_end, 2) sampler.stop.set() sampler.join() summary = {"label": label, "tag": tag, "size": args.size, "file_kib": len(body) >> 10, "rate": args.rate, "duration": args.duration, "replicas": args.replicas, "concurrency": args.concurrency, "prefetch": args.prefetch, "max_pending": max_pending, **sent, "drained": drained, "drain_after_load_s": drain_seconds, **job_metrics(tag), "pending_max": max(sampler.pending or [0]), "worker_mem_peak_mib": round(max(sampler.rss or [0]), 1), "worker_mem_median_mib": round(statistics.median(sampler.rss), 1) if sampler.rss else None, "worker_cpu_peak_pct": round(max(sampler.cpu or [0]), 1), "at": time.strftime("%Y-%m-%dT%H:%M:%S")} return summary, c, alice, chat def main(): parser = argparse.ArgumentParser() parser.add_argument("mode", choices=["run", "overload"]) parser.add_argument("--size", choices=sorted(SIZES), default="medium") parser.add_argument("--rate", type=float, default=4) parser.add_argument("--duration", type=float, default=30) parser.add_argument("--replicas", type=int, default=1) parser.add_argument("--concurrency", type=int, default=2) parser.add_argument("--prefetch", type=int, default=2) parser.add_argument("--max-pending", type=int, default=30) parser.add_argument("--drain-timeout", type=float, default=600) parser.add_argument("--label", default="") args = parser.parse_args() if args.mode == "run": summary, *_ = experiment(args, args.label or "c%d-p%d" % (args.concurrency, args.prefetch)) record(summary) return # Overload: admission limit on, offered rate far above capacity, then recovery. set_admission(args.max_pending) try: summary, c, alice, chat = experiment(args, args.label or "overload", args.max_pending) # Recovery probe: after the backlog drained, a fresh upload must be ready quickly again. body, ctype = image("small") t = time.monotonic() att = c.call("POST", "/api/v1/chats/%s/attachments?filename=probe.png" % chat["id"], 202, actor=alice, raw=body, headers={"Content-Type": ctype}, schema="Attachment")["json"] while c.call("GET", "/api/v1/attachments/" + att["id"], actor=alice, schema="Attachment")["json"]["status"] != "ready": time.sleep(0.2) summary["recovery_probe_ready_s"] = round(time.monotonic() - t, 2) record(summary) finally: set_admission(200) if __name__ == "__main__": main()