Files
mtusi2026lab5/scripts/check-queue-overload.py
T

244 lines
9.0 KiB
Python

#!/usr/bin/env python3
"""Track 5.1 - the attachment queue under overload.
Hypothesis: raising worker concurrency increases throughput and shortens
time-to-ready up to a point, after which CPU contention (Pillow resizing) and
memory set the ceiling; the durable queue lets the API keep accepting (202) and
bounds in-flight work, then drains and recovers afterwards.
Method (reproducible):
* for each (concurrency, image size) pair, recreate the worker with a fixed
WORKER_CONCURRENCY, submit a burst of uploads, and poll every attachment
until ready;
* record accepted/errors, throughput, p50/p95 time-to-ready, queue depth over
time (backpressure) and peak worker RSS (docker stats);
* after the largest burst, wait for the queue to drain and confirm a fresh
upload still becomes ready quickly (recovery).
Output: a JSON document plus a readable table. Run from the repository root.
"""
from __future__ import annotations
import argparse
import base64
import concurrent.futures
import json
import os
import statistics
import struct
import subprocess
import threading
import time
import urllib.error
import urllib.request
import uuid
import zlib
BASE = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/")
RESULTS = []
def _chunk(tag, data):
return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF)
def make_png(width, height):
raw = bytearray()
pixels = os.urandom(width * height * 3)
stride = width * 3
for row in range(height):
raw.append(0)
raw += pixels[row * stride : (row + 1) * stride]
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return b"\x89PNG\r\n\x1a\n" + _chunk(b"IHDR", ihdr) + _chunk(b"IDAT", zlib.compress(bytes(raw), 0)) + _chunk(b"IEND", b"")
def http(method, path, token=None, body=None, raw=None, content_type=None):
headers = {"Accept": "application/json"}
data = raw
if body is not None:
data = json.dumps(body).encode()
headers["Content-Type"] = "application/json"
if content_type:
headers["Content-Type"] = content_type
if token:
headers["Authorization"] = "Bearer " + token
request = urllib.request.Request(BASE + path, data=data, headers=headers, method=method)
try:
with urllib.request.urlopen(request, timeout=30) as response:
return response.status, response.read()
except urllib.error.HTTPError as error:
return error.code, error.read()
except (urllib.error.URLError, TimeoutError, OSError):
return 0, b""
def login():
username = "q_" + uuid.uuid4().hex[:8]
password = "T3st-" + uuid.uuid4().hex
http("POST", "/api/v1/users", body={"username": username, "display_name": "Q", "password": password})
credential = base64.b64encode(f"{username}:{password}".encode()).decode()
request = urllib.request.Request(
BASE + "/api/v1/auth/sessions",
headers={"Authorization": "Basic " + credential, "Accept": "application/json"},
method="POST",
)
with urllib.request.urlopen(request, timeout=20) as response:
token = json.loads(response.read())["token"]
user_id = json.loads(http("GET", "/api/v1/users/me", token=token)[1])["id"]
chat = json.loads(http("POST", "/api/v1/chats", token=token, body={"title": "q", "member_ids": [user_id]})[1])
return token, chat["id"]
def worker_mem_mb():
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}}|{{.MemUsage}}"],
capture_output=True, text=True).stdout
for line in out.splitlines():
name, _, usage = line.partition("|")
if "worker" not in name:
continue
value = usage.split("/")[0].strip()
number = "".join(ch for ch in value if ch.isdigit() or ch == ".")
if value.endswith("GiB"):
return float(number) * 1024
if value.endswith("MiB"):
return float(number)
return 0.0
def set_worker_concurrency(value):
env = dict(os.environ, WORKER_CONCURRENCY=str(value))
subprocess.run(["docker", "compose", "up", "-d", "--force-recreate", "worker"],
env=env, check=True, capture_output=True, text=True)
time.sleep(3)
def percentile(values, pct):
if not values:
return 0.0
ordered = sorted(values)
index = min(len(ordered) - 1, int(round((pct / 100.0) * (len(ordered) - 1))))
return ordered[index]
def run_case(concurrency, size_name, width, height, count, submit_workers=8):
set_worker_concurrency(concurrency)
token, chat_id = login()
payload = make_png(width, height)
pending = {}
codes = {}
lock = threading.Lock()
def submit(_):
status, body = http("POST", f"/api/v1/chats/{chat_id}/attachments?filename=q.png", token=token,
raw=payload, content_type="image/png")
with lock:
codes[status] = codes.get(status, 0) + 1
if status != 202:
return
attachment_id = json.loads(body)["id"]
with lock:
pending[attachment_id] = time.time()
queue_samples = []
stop = threading.Event()
peak = {"value": worker_mem_mb()}
def sampler():
while not stop.is_set():
with lock:
depth = len(pending)
queue_samples.append(depth)
peak["value"] = max(peak["value"], worker_mem_mb())
time.sleep(0.2)
thread = threading.Thread(target=sampler, daemon=True)
thread.start()
submit_start = time.time()
with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, min(submit_workers, count))) as pool:
list(pool.map(submit, range(count)))
submit_done = time.time()
latencies = []
deadline = time.time() + 300
while pending and time.time() < deadline:
with lock:
ids = list(pending.keys())
for attachment_id in ids:
status, body = http("GET", f"/api/v1/attachments/{attachment_id}", token=token)
if status != 200:
continue
state = json.loads(body)["status"]
if state in {"ready", "failed"}:
with lock:
started = pending.pop(attachment_id, None)
if started is not None:
latencies.append((time.time() - started, state))
time.sleep(0.2)
stop.set()
thread.join(timeout=2)
ready = [lat for lat, state in latencies if state == "ready"]
failed = [lat for lat, state in latencies if state == "failed"]
result = {
"concurrency": concurrency,
"size": size_name,
"bytes": len(payload),
"submitted": count,
"accepted": codes.get(202, 0),
"errors": sum(v for k, v in codes.items() if k != 202),
"codes": codes,
"ready": len(ready),
"failed": len(failed),
"submit_seconds": round(submit_done - submit_start, 2),
"drain_seconds": round(time.time() - submit_done, 2),
"throughput_per_s": round(len(ready) / max(0.001, time.time() - submit_start), 2),
"p50_ready_s": round(statistics.median(ready), 2) if ready else None,
"p95_ready_s": round(percentile(ready, 95), 2) if ready else None,
"max_queue_depth": max(queue_samples) if queue_samples else 0,
"worker_peak_mem_mb": round(peak["value"], 1),
}
RESULTS.append(result)
print(json.dumps(result, ensure_ascii=False), flush=True)
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--count", type=int, default=12)
parser.add_argument("--output", default="evidence/lab5-overload.json")
args = parser.parse_args()
os.makedirs(os.path.dirname(args.output), exist_ok=True)
plan = [
(1, "small", 640, 480, 8),
(2, "small", 640, 480, 8),
(4, "small", 640, 480, 8),
(1, "large", 2200, 1500, 4),
(2, "large", 2200, 1500, 4),
(4, "large", 2200, 1500, 4),
]
for concurrency, size_name, width, height, workers in plan:
run_case(concurrency, size_name, width, height, args.count, submit_workers=workers)
# Recovery: after the heaviest cases the queue must drain and a fresh upload
# must still be served promptly.
set_worker_concurrency(2)
token, chat_id = login()
fresh = run_case(2, "recovery", 800, 600, 1)
drained = fresh["ready"] == 1 and fresh["p95_ready_s"] is not None and fresh["p95_ready_s"] < 30
with open(args.output, "w", encoding="utf-8") as handle:
json.dump({"results": RESULTS, "recovered": drained}, handle, ensure_ascii=False, indent=2)
print("\nconcurrency size bytes accepted ready p50_s p95_s maxq peakRSS_MB")
for r in RESULTS:
print(f"{r['concurrency']:>11} {r['size']:<7} {r['bytes']:>7} {r['accepted']:>8} {r['ready']:>5} "
f"{str(r['p50_ready_s']):>5} {str(r['p95_ready_s']):>5} {r['max_queue_depth']:>5} {r['worker_peak_mem_mb']:>10}")
print(f"\nrecovery after overload: {'OK' if drained else 'FAILED'}")
print(json.dumps({"output": args.output, "recovered": drained}))
if __name__ == "__main__":
main()