144 lines
7.1 KiB
Python
Executable File
144 lines
7.1 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Repeatable level-5 queue-overload experiment.
|
|
|
|
Run a Compose profile first, for example:
|
|
WORKER_CONCURRENCY=2 MAX_PENDING_JOBS=20 docker compose up -d --build
|
|
.venv/bin/python scripts/track1.py --run-name c2-p2 --workers 2 --prefetch 2 --images 40 --size 1600
|
|
|
|
The script records client-observed acceptance/ready latency, RabbitMQ queue depth
|
|
and Docker RSS samples. It does not print credentials or session tokens.
|
|
"""
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from io import BytesIO
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from urllib.parse import quote
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure, arguments, check, run # noqa: E402
|
|
|
|
|
|
def percentile(values, point):
|
|
values = sorted(values)
|
|
return round(values[max(0, math.ceil(len(values) * point) - 1)] * 1000, 3) if values else None
|
|
|
|
|
|
def docker_text(*args):
|
|
try:
|
|
return subprocess.run(args, cwd=ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
|
|
timeout=10, check=True).stdout.strip()
|
|
except (OSError, subprocess.SubprocessError):
|
|
return None
|
|
|
|
|
|
def broker_depth():
|
|
text = docker_text("docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "list_queues", "name", "messages_ready", "messages_unacknowledged", "--formatter", "json")
|
|
if not text:
|
|
return None
|
|
try:
|
|
rows = json.loads(text)
|
|
return next((row for row in rows if row["name"] == "attachment.process.v1"), None)
|
|
except (ValueError, KeyError, TypeError):
|
|
return None
|
|
|
|
|
|
def worker_rss():
|
|
container = docker_text("docker", "compose", "ps", "-q", "worker")
|
|
if not container:
|
|
return None
|
|
text = docker_text("docker", "stats", "--no-stream", "--format", "{{json .}}", container)
|
|
if not text:
|
|
return None
|
|
try:
|
|
return json.loads(text).get("MemUsage")
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def main():
|
|
def configure(parser):
|
|
parser.add_argument("--run-name", required=True)
|
|
parser.add_argument("--workers", type=int, required=True, help="documented WORKER_CONCURRENCY setting")
|
|
parser.add_argument("--prefetch", type=int, required=True, help="documented RabbitMQ prefetch setting")
|
|
parser.add_argument("--max-pending", type=int, required=True, help="documented MAX_PENDING_JOBS setting")
|
|
parser.add_argument("--images", type=int, default=40)
|
|
parser.add_argument("--concurrency", type=int, default=4)
|
|
parser.add_argument("--size", type=int, default=1600, help="square synthetic PNG dimensions")
|
|
parser.add_argument("--max-ready-seconds", type=float, default=120)
|
|
parser.add_argument("--output", default=None)
|
|
args = arguments("Lab 5 track 1 image-pipeline experiment", configure)
|
|
check(1 <= args.workers <= 64 and 1 <= args.prefetch <= 1024 and 1 <= args.max_pending <= 100000, "invalid worker/prefetch/max-pending")
|
|
check(1 <= args.images <= 1000 and 1 <= args.concurrency <= 128 and 1 <= args.size <= 6000, "invalid workload")
|
|
|
|
from PIL import Image
|
|
image = Image.new("RGB", (args.size, args.size), "#427aa1")
|
|
raw = BytesIO(); image.save(raw, "PNG", optimize=True); data = raw.getvalue()
|
|
client = Client(args.base_url, args.ca_file, timeout=30)
|
|
alice, bob, _eve, chat = client.bootstrap()
|
|
path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=track1.png"
|
|
barrier = threading.Barrier(args.concurrency)
|
|
queue_samples, rss_samples = [], []
|
|
stop_sampling = threading.Event()
|
|
|
|
def sampler():
|
|
while not stop_sampling.wait(1):
|
|
queue_samples.append({"seconds": round(time.monotonic() - started, 3), "queue": broker_depth()})
|
|
rss_samples.append({"seconds": round(time.monotonic() - started, 3), "rss": worker_rss()})
|
|
|
|
def upload(index):
|
|
local = Client(args.base_url, args.ca_file, timeout=30)
|
|
barrier.wait()
|
|
started_at = time.monotonic()
|
|
try:
|
|
response = local.call("POST", path + "-" + str(index), 202, actor=alice, raw=data,
|
|
headers={"Content-Type": "image/png"}, schema="Attachment")
|
|
return {"accepted": True, "id": response["json"]["id"], "accepted_seconds": time.monotonic() - started_at}
|
|
except Failure as error:
|
|
return {"accepted": False, "error": str(error), "accepted_seconds": time.monotonic() - started_at}
|
|
|
|
started = time.monotonic()
|
|
thread = threading.Thread(target=sampler, daemon=True); thread.start()
|
|
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
|
results = list(pool.map(upload, range(args.images)))
|
|
accepted = [item for item in results if item["accepted"]]
|
|
ready_latencies, failed_ready = [], 0
|
|
deadline = time.monotonic() + args.max_ready_seconds
|
|
pending = {item["id"]: started + item["accepted_seconds"] for item in accepted}
|
|
while pending and time.monotonic() < deadline:
|
|
for attachment_id, accepted_at in list(pending.items()):
|
|
try:
|
|
metadata = client.call("GET", "/api/v1/attachments/" + attachment_id, actor=bob, schema="Attachment")["json"]
|
|
except Failure:
|
|
continue
|
|
if metadata["status"] == "ready":
|
|
ready_latencies.append(time.monotonic() - accepted_at)
|
|
del pending[attachment_id]
|
|
elif metadata["status"] == "failed":
|
|
failed_ready += 1; del pending[attachment_id]
|
|
time.sleep(.2)
|
|
stop_sampling.set(); thread.join(timeout=2)
|
|
elapsed = time.monotonic() - started
|
|
report = {
|
|
"track": "queue-overload", "run_name": args.run_name,
|
|
"settings": {"worker_concurrency": args.workers, "prefetch": args.prefetch, "max_pending_jobs": args.max_pending},
|
|
"workload": {"images": args.images, "client_concurrency": args.concurrency, "png_dimensions": [args.size, args.size], "input_bytes": len(data)},
|
|
"results": {"elapsed_seconds": round(elapsed, 3), "accepted": len(accepted), "rejected_or_failed": args.images - len(accepted), "ready": len(ready_latencies), "failed": failed_ready, "timed_out": len(pending), "throughput_ready_per_second": round(len(ready_latencies) / elapsed, 4), "accept_latency_ms": {"p50": percentile([x["accepted_seconds"] for x in accepted], .5), "p95": percentile([x["accepted_seconds"] for x in accepted], .95)}, "time_to_ready_ms": {"p50": percentile(ready_latencies, .5), "p95": percentile(ready_latencies, .95)}},
|
|
"queue_samples": queue_samples, "worker_rss_samples": rss_samples,
|
|
"limitations": "Closed-loop client; queue/RSS samples are null when Docker CLI is unavailable. p95 covers ready tasks only.",
|
|
}
|
|
text = json.dumps(report, ensure_ascii=False, indent=2)
|
|
destination = Path(args.output or (ROOT / "evidence" / (args.run_name + ".json")))
|
|
destination.parent.mkdir(parents=True, exist_ok=True); destination.write_text(text + "\n", encoding="utf-8")
|
|
print(text)
|
|
check(bool(accepted), "no accepted uploads")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|