244 lines
9.0 KiB
Python
244 lines
9.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Track 5.1 - the attachment queue under overload.
|
|
|
|
Hypothesis: raising worker concurrency increases throughput and shortens
|
|
time-to-ready up to a point, after which CPU contention (Pillow resizing) and
|
|
memory set the ceiling; the durable queue lets the API keep accepting (202) and
|
|
bounds in-flight work, then drains and recovers afterwards.
|
|
|
|
Method (reproducible):
|
|
* for each (concurrency, image size) pair, recreate the worker with a fixed
|
|
WORKER_CONCURRENCY, submit a burst of uploads, and poll every attachment
|
|
until ready;
|
|
* record accepted/errors, throughput, p50/p95 time-to-ready, queue depth over
|
|
time (backpressure) and peak worker RSS (docker stats);
|
|
* after the largest burst, wait for the queue to drain and confirm a fresh
|
|
upload still becomes ready quickly (recovery).
|
|
|
|
Output: a JSON document plus a readable table. Run from the repository root.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import concurrent.futures
|
|
import json
|
|
import os
|
|
import statistics
|
|
import struct
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
import zlib
|
|
|
|
BASE = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/")
|
|
RESULTS = []
|
|
|
|
|
|
def _chunk(tag, data):
|
|
return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF)
|
|
|
|
|
|
def make_png(width, height):
|
|
raw = bytearray()
|
|
pixels = os.urandom(width * height * 3)
|
|
stride = width * 3
|
|
for row in range(height):
|
|
raw.append(0)
|
|
raw += pixels[row * stride : (row + 1) * stride]
|
|
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
|
return b"\x89PNG\r\n\x1a\n" + _chunk(b"IHDR", ihdr) + _chunk(b"IDAT", zlib.compress(bytes(raw), 0)) + _chunk(b"IEND", b"")
|
|
|
|
|
|
def http(method, path, token=None, body=None, raw=None, content_type=None):
|
|
headers = {"Accept": "application/json"}
|
|
data = raw
|
|
if body is not None:
|
|
data = json.dumps(body).encode()
|
|
headers["Content-Type"] = "application/json"
|
|
if content_type:
|
|
headers["Content-Type"] = content_type
|
|
if token:
|
|
headers["Authorization"] = "Bearer " + token
|
|
request = urllib.request.Request(BASE + path, data=data, headers=headers, method=method)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=30) as response:
|
|
return response.status, response.read()
|
|
except urllib.error.HTTPError as error:
|
|
return error.code, error.read()
|
|
except (urllib.error.URLError, TimeoutError, OSError):
|
|
return 0, b""
|
|
|
|
|
|
def login():
|
|
username = "q_" + uuid.uuid4().hex[:8]
|
|
password = "T3st-" + uuid.uuid4().hex
|
|
http("POST", "/api/v1/users", body={"username": username, "display_name": "Q", "password": password})
|
|
credential = base64.b64encode(f"{username}:{password}".encode()).decode()
|
|
request = urllib.request.Request(
|
|
BASE + "/api/v1/auth/sessions",
|
|
headers={"Authorization": "Basic " + credential, "Accept": "application/json"},
|
|
method="POST",
|
|
)
|
|
with urllib.request.urlopen(request, timeout=20) as response:
|
|
token = json.loads(response.read())["token"]
|
|
user_id = json.loads(http("GET", "/api/v1/users/me", token=token)[1])["id"]
|
|
chat = json.loads(http("POST", "/api/v1/chats", token=token, body={"title": "q", "member_ids": [user_id]})[1])
|
|
return token, chat["id"]
|
|
|
|
|
|
def worker_mem_mb():
|
|
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}}|{{.MemUsage}}"],
|
|
capture_output=True, text=True).stdout
|
|
for line in out.splitlines():
|
|
name, _, usage = line.partition("|")
|
|
if "worker" not in name:
|
|
continue
|
|
value = usage.split("/")[0].strip()
|
|
number = "".join(ch for ch in value if ch.isdigit() or ch == ".")
|
|
if value.endswith("GiB"):
|
|
return float(number) * 1024
|
|
if value.endswith("MiB"):
|
|
return float(number)
|
|
return 0.0
|
|
|
|
|
|
def set_worker_concurrency(value):
|
|
env = dict(os.environ, WORKER_CONCURRENCY=str(value))
|
|
subprocess.run(["docker", "compose", "up", "-d", "--force-recreate", "worker"],
|
|
env=env, check=True, capture_output=True, text=True)
|
|
time.sleep(3)
|
|
|
|
|
|
def percentile(values, pct):
|
|
if not values:
|
|
return 0.0
|
|
ordered = sorted(values)
|
|
index = min(len(ordered) - 1, int(round((pct / 100.0) * (len(ordered) - 1))))
|
|
return ordered[index]
|
|
|
|
|
|
def run_case(concurrency, size_name, width, height, count, submit_workers=8):
|
|
set_worker_concurrency(concurrency)
|
|
token, chat_id = login()
|
|
payload = make_png(width, height)
|
|
|
|
pending = {}
|
|
codes = {}
|
|
lock = threading.Lock()
|
|
|
|
def submit(_):
|
|
status, body = http("POST", f"/api/v1/chats/{chat_id}/attachments?filename=q.png", token=token,
|
|
raw=payload, content_type="image/png")
|
|
with lock:
|
|
codes[status] = codes.get(status, 0) + 1
|
|
if status != 202:
|
|
return
|
|
attachment_id = json.loads(body)["id"]
|
|
with lock:
|
|
pending[attachment_id] = time.time()
|
|
|
|
queue_samples = []
|
|
stop = threading.Event()
|
|
peak = {"value": worker_mem_mb()}
|
|
|
|
def sampler():
|
|
while not stop.is_set():
|
|
with lock:
|
|
depth = len(pending)
|
|
queue_samples.append(depth)
|
|
peak["value"] = max(peak["value"], worker_mem_mb())
|
|
time.sleep(0.2)
|
|
|
|
thread = threading.Thread(target=sampler, daemon=True)
|
|
thread.start()
|
|
submit_start = time.time()
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, min(submit_workers, count))) as pool:
|
|
list(pool.map(submit, range(count)))
|
|
submit_done = time.time()
|
|
latencies = []
|
|
deadline = time.time() + 300
|
|
while pending and time.time() < deadline:
|
|
with lock:
|
|
ids = list(pending.keys())
|
|
for attachment_id in ids:
|
|
status, body = http("GET", f"/api/v1/attachments/{attachment_id}", token=token)
|
|
if status != 200:
|
|
continue
|
|
state = json.loads(body)["status"]
|
|
if state in {"ready", "failed"}:
|
|
with lock:
|
|
started = pending.pop(attachment_id, None)
|
|
if started is not None:
|
|
latencies.append((time.time() - started, state))
|
|
time.sleep(0.2)
|
|
stop.set()
|
|
thread.join(timeout=2)
|
|
|
|
ready = [lat for lat, state in latencies if state == "ready"]
|
|
failed = [lat for lat, state in latencies if state == "failed"]
|
|
result = {
|
|
"concurrency": concurrency,
|
|
"size": size_name,
|
|
"bytes": len(payload),
|
|
"submitted": count,
|
|
"accepted": codes.get(202, 0),
|
|
"errors": sum(v for k, v in codes.items() if k != 202),
|
|
"codes": codes,
|
|
"ready": len(ready),
|
|
"failed": len(failed),
|
|
"submit_seconds": round(submit_done - submit_start, 2),
|
|
"drain_seconds": round(time.time() - submit_done, 2),
|
|
"throughput_per_s": round(len(ready) / max(0.001, time.time() - submit_start), 2),
|
|
"p50_ready_s": round(statistics.median(ready), 2) if ready else None,
|
|
"p95_ready_s": round(percentile(ready, 95), 2) if ready else None,
|
|
"max_queue_depth": max(queue_samples) if queue_samples else 0,
|
|
"worker_peak_mem_mb": round(peak["value"], 1),
|
|
}
|
|
RESULTS.append(result)
|
|
print(json.dumps(result, ensure_ascii=False), flush=True)
|
|
return result
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--count", type=int, default=12)
|
|
parser.add_argument("--output", default="evidence/lab5-overload.json")
|
|
args = parser.parse_args()
|
|
|
|
os.makedirs(os.path.dirname(args.output), exist_ok=True)
|
|
plan = [
|
|
(1, "small", 640, 480, 8),
|
|
(2, "small", 640, 480, 8),
|
|
(4, "small", 640, 480, 8),
|
|
(1, "large", 2200, 1500, 4),
|
|
(2, "large", 2200, 1500, 4),
|
|
(4, "large", 2200, 1500, 4),
|
|
]
|
|
for concurrency, size_name, width, height, workers in plan:
|
|
run_case(concurrency, size_name, width, height, args.count, submit_workers=workers)
|
|
|
|
# Recovery: after the heaviest cases the queue must drain and a fresh upload
|
|
# must still be served promptly.
|
|
set_worker_concurrency(2)
|
|
token, chat_id = login()
|
|
fresh = run_case(2, "recovery", 800, 600, 1)
|
|
drained = fresh["ready"] == 1 and fresh["p95_ready_s"] is not None and fresh["p95_ready_s"] < 30
|
|
|
|
with open(args.output, "w", encoding="utf-8") as handle:
|
|
json.dump({"results": RESULTS, "recovered": drained}, handle, ensure_ascii=False, indent=2)
|
|
|
|
print("\nconcurrency size bytes accepted ready p50_s p95_s maxq peakRSS_MB")
|
|
for r in RESULTS:
|
|
print(f"{r['concurrency']:>11} {r['size']:<7} {r['bytes']:>7} {r['accepted']:>8} {r['ready']:>5} "
|
|
f"{str(r['p50_ready_s']):>5} {str(r['p95_ready_s']):>5} {r['max_queue_depth']:>5} {r['worker_peak_mem_mb']:>10}")
|
|
print(f"\nrecovery after overload: {'OK' if drained else 'FAILED'}")
|
|
print(json.dumps({"output": args.output, "recovered": drained}))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|