215 lines
10 KiB
Python
Executable File
215 lines
10 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Closed-loop upload burst with concurrent readiness observations and drain proof."""
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from io import BytesIO
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure, arguments, check, run
|
|
|
|
|
|
def percentile(values, point):
|
|
values = sorted(values)
|
|
return round(values[max(0, math.ceil(len(values) * point) - 1)] * 1000, 3) if values else None
|
|
|
|
|
|
def command_json(*args):
|
|
result = subprocess.run(args, cwd=ROOT, text=True, capture_output=True, timeout=20)
|
|
if result.returncode:
|
|
raise Failure("metrics command failed: " + args[0])
|
|
try:
|
|
return json.loads(result.stdout)
|
|
except ValueError:
|
|
raise Failure("metrics command returned invalid JSON") from None
|
|
|
|
|
|
def metrics(chat_id):
|
|
return command_json("docker", "compose", "exec", "-T", "worker",
|
|
"/app/.venv/bin/python", "-m", "src.pipeline_metrics", chat_id)
|
|
|
|
|
|
def broker_depth():
|
|
rows = command_json("docker", "compose", "exec", "-T", "broker",
|
|
"rabbitmqctl", "list_queues", "name", "messages_ready",
|
|
"messages_unacknowledged", "--formatter", "json")
|
|
return next(row for row in rows if row["name"] == "attachment.process.v1")
|
|
|
|
|
|
def container_info(service):
|
|
result = subprocess.run(['docker', 'compose', 'ps', '-q', service], cwd=ROOT,
|
|
capture_output=True, text=True, timeout=10, check=True)
|
|
ids = result.stdout.split()
|
|
check(len(ids) == 1, 'expected exactly one container: ' + service)
|
|
return command_json('docker', 'inspect', ids[0])[0]
|
|
|
|
|
|
def main():
|
|
def configure(p):
|
|
p.add_argument("--run-name", required=True)
|
|
p.add_argument("--workers", type=int, required=True)
|
|
p.add_argument("--prefetch", type=int, required=True)
|
|
p.add_argument("--max-pending", type=int, required=True)
|
|
p.add_argument("--images", type=int, default=40)
|
|
p.add_argument("--concurrency", type=int, default=8)
|
|
p.add_argument("--size", type=int, default=1600)
|
|
p.add_argument("--max-ready-seconds", type=float, default=180)
|
|
p.add_argument("--require-backpressure", action="store_true")
|
|
p.add_argument("--output")
|
|
args = arguments(__doc__, configure)
|
|
check(1 <= args.images <= 1000 and 1 <= args.concurrency <= 128, "invalid workload")
|
|
check(1 <= args.size <= 6000 and 0 < args.max_ready_seconds <= 3600, "invalid size/deadline")
|
|
check(args.workers > 0 and args.prefetch > 0 and args.max_pending > 0, "invalid profile")
|
|
from PIL import Image
|
|
# Reproducible, nonuniform synthetic image; no real personal data.
|
|
row = bytes(v for x in range(args.size) for v in ((x * 13) % 256, (x * 37) % 256, (x * 71) % 256))
|
|
image = Image.frombytes("RGB", (args.size, args.size), row * args.size)
|
|
stream = BytesIO()
|
|
image.save(stream, "PNG")
|
|
data = stream.getvalue()
|
|
client = Client(args.base_url, args.ca_file, timeout=30)
|
|
alice, bob, _, chat = client.bootstrap()
|
|
baseline = metrics(chat["id"])
|
|
check(baseline["concurrency"] == args.workers and baseline["prefetch"] == args.prefetch,
|
|
"worker settings do not match requested profile")
|
|
check(baseline["pending"] == 0, "use an idle isolated stack")
|
|
for service in ("api-a", "api-b"):
|
|
env = dict(item.split('=', 1) for item in container_info(service)['Config']['Env'])
|
|
check(int(env['MAX_PENDING_JOBS']) == args.max_pending,
|
|
'running API MAX_PENDING_JOBS does not match profile')
|
|
resources = container_info('worker')['HostConfig']
|
|
path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=track1.png"
|
|
pending, records, samples, errors = {}, [], [], []
|
|
lock, finished = threading.Lock(), threading.Event()
|
|
started = time.monotonic()
|
|
def observe():
|
|
local = Client(args.base_url, args.ca_file, timeout=5)
|
|
while not finished.wait(.1):
|
|
with lock:
|
|
batch = list(pending.items())
|
|
for key, record in batch:
|
|
if finished.is_set():
|
|
break
|
|
try:
|
|
att = local.call("GET", "/api/v1/attachments/" + key, actor=bob, schema="Attachment")["json"]
|
|
if att["status"] in ("ready", "failed"):
|
|
with lock:
|
|
record["terminal"] = att["status"]
|
|
record["terminal_at"] = time.monotonic()
|
|
pending.pop(key, None)
|
|
except Failure:
|
|
continue
|
|
def sample():
|
|
while not finished.is_set():
|
|
try:
|
|
sample = metrics(chat["id"])
|
|
sample["queue"] = broker_depth()
|
|
sample["seconds"] = time.monotonic() - started
|
|
with lock:
|
|
samples.append(sample)
|
|
except (Failure, OSError, subprocess.TimeoutExpired) as exc:
|
|
with lock:
|
|
errors.append(type(exc).__name__)
|
|
if finished.wait(1):
|
|
break
|
|
observer = threading.Thread(target=observe, daemon=True)
|
|
sampler = threading.Thread(target=sample, daemon=True)
|
|
observer.start()
|
|
sampler.start()
|
|
def upload(index):
|
|
local = Client(args.base_url, args.ca_file, timeout=30)
|
|
before = time.monotonic()
|
|
try:
|
|
response = local.call("POST", path, (202, 429), actor=alice, raw=data,
|
|
headers={"Content-Type": "image/png"}, schema="Attachment")
|
|
received = time.monotonic()
|
|
record = {"index": index, "status": response["status"], "upload_seconds": received - before}
|
|
if response["status"] == 202:
|
|
record.update(id=response["json"]["id"], accepted_at=received)
|
|
with lock:
|
|
records.append(record)
|
|
if response["status"] == 202:
|
|
pending[record["id"]] = record
|
|
except Failure:
|
|
with lock:
|
|
records.append({"index": index, "status": "transport_or_http_error"})
|
|
try:
|
|
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
|
list(pool.map(upload, range(args.images)))
|
|
upload_finished = time.monotonic()
|
|
deadline = upload_finished + args.max_ready_seconds
|
|
while time.monotonic() < deadline:
|
|
with lock:
|
|
empty = not pending
|
|
if empty:
|
|
break
|
|
time.sleep(.1)
|
|
finally:
|
|
measured_end = time.monotonic()
|
|
finished.set()
|
|
observer.join(timeout=10)
|
|
sampler.join(timeout=45)
|
|
elapsed = measured_end - started
|
|
final = metrics(chat["id"])
|
|
queue = broker_depth()
|
|
for _ in range(20):
|
|
if queue["messages_ready"] == 0 and queue["messages_unacknowledged"] == 0:
|
|
break
|
|
time.sleep(.25)
|
|
queue = broker_depth()
|
|
accepted = [r for r in records if r["status"] == 202]
|
|
ready = [r for r in accepted if r.get("terminal") == "ready"]
|
|
latencies = [r["terminal_at"] - r["accepted_at"] for r in ready]
|
|
waits = [r["queue_wait_seconds"] for r in final["jobs"] if r["queue_wait_seconds"] is not None]
|
|
drained = (bool(accepted) and len(ready) == len(accepted) and final["pending"] == 0
|
|
and queue["messages_ready"] == 0 and queue["messages_unacknowledged"] == 0)
|
|
recovery = False
|
|
if drained:
|
|
try:
|
|
att, _ = client.attachment(alice, chat)
|
|
client.wait_ready(bob, att, wait_seconds=args.max_ready_seconds)
|
|
recovery = True
|
|
except Failure:
|
|
pass
|
|
rejected = sum(r["status"] == 429 for r in records)
|
|
report = {
|
|
"run_name": args.run_name,
|
|
"settings": {"workers": baseline["concurrency"], "prefetch": baseline["prefetch"],
|
|
"max_pending": args.max_pending,
|
|
"worker_limits": {key: resources[key] for key in ('Memory', 'NanoCpus')}},
|
|
"workload": {"images": args.images, "concurrency": args.concurrency,
|
|
"size": args.size, "bytes": len(data), "type": "closed-loop burst"},
|
|
"results": {"accepted": len(accepted), "rejected_429": rejected, "ready": len(ready),
|
|
"failed": sum(r.get("terminal") == "failed" for r in accepted),
|
|
"timed_out": len(pending), "elapsed_seconds": elapsed,
|
|
"ready_per_second": len(ready) / elapsed,
|
|
"ready_latency_ms": {str(p): percentile(latencies, p) for p in (.5, .95, .99)},
|
|
"queue_wait_ms": {str(p): percentile(waits, p) for p in (.5, .95, .99)},
|
|
"max_sampled_rss_bytes": max((s["rss_bytes"] for s in samples), default=None),
|
|
"drained": drained, "recovery_upload_ready": recovery},
|
|
"records": [{k: v for k, v in r.items() if k not in ("accepted_at", "terminal_at")} |
|
|
({"ready_seconds": r["terminal_at"] - r["accepted_at"]} if "terminal_at" in r else {})
|
|
for r in records],
|
|
"samples": samples, "metric_errors": errors,
|
|
"limitations": "Closed-loop; ready time is polling upper bound. RSS is sum of worker/container processes, shared pages may be counted twice. Sampling may miss peaks. Queue wait is first claim minus durable job creation.",
|
|
}
|
|
target = Path(args.output or ROOT / "evidence" / (args.run_name + ".json"))
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_text(json.dumps(report, indent=2) + "\n")
|
|
print(json.dumps(report["results"], indent=2))
|
|
check(not errors and bool(samples), "incomplete metrics; result saved but not valid evidence")
|
|
check(all(r["status"] in (202, 429) for r in records), "unexpected HTTP/transport failures")
|
|
check(drained and recovery, "pipeline did not drain/recover; see saved result")
|
|
if args.require_backpressure:
|
|
check(rejected > 0, "backpressure was not observed")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|