#!/usr/bin/env python3 """Closed-loop upload burst with concurrent readiness observations and drain proof.""" from concurrent.futures import ThreadPoolExecutor, as_completed from io import BytesIO import json import math from pathlib import Path import subprocess import sys import threading import time ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client, Failure, arguments, check, run def percentile(values, point): values = sorted(values) return round(values[max(0, math.ceil(len(values) * point) - 1)] * 1000, 3) if values else None def command_json(*args): result = subprocess.run(args, cwd=ROOT, text=True, capture_output=True, timeout=20) if result.returncode: raise Failure("metrics command failed: " + args[0]) try: return json.loads(result.stdout) except ValueError: raise Failure("metrics command returned invalid JSON") from None def metrics(chat_id): return command_json("docker", "compose", "exec", "-T", "worker", "/app/.venv/bin/python", "-m", "src.pipeline_metrics", chat_id) def broker_depth(): rows = command_json("docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "list_queues", "name", "messages_ready", "messages_unacknowledged", "--formatter", "json") return next(row for row in rows if row["name"] == "attachment.process.v1") def container_info(service): result = subprocess.run(['docker', 'compose', 'ps', '-q', service], cwd=ROOT, capture_output=True, text=True, timeout=10, check=True) ids = result.stdout.split() check(len(ids) == 1, 'expected exactly one container: ' + service) return command_json('docker', 'inspect', ids[0])[0] def main(): def configure(p): p.add_argument("--run-name", required=True) p.add_argument("--workers", type=int, required=True) p.add_argument("--prefetch", type=int, required=True) p.add_argument("--max-pending", type=int, required=True) p.add_argument("--images", type=int, default=40) p.add_argument("--concurrency", type=int, default=8) p.add_argument("--size", type=int, default=1600) p.add_argument("--max-ready-seconds", type=float, default=180) p.add_argument("--require-backpressure", action="store_true") p.add_argument("--output") args = arguments(__doc__, configure) check(1 <= args.images <= 1000 and 1 <= args.concurrency <= 128, "invalid workload") check(1 <= args.size <= 6000 and 0 < args.max_ready_seconds <= 3600, "invalid size/deadline") check(args.workers > 0 and args.prefetch > 0 and args.max_pending > 0, "invalid profile") from PIL import Image # Reproducible, nonuniform synthetic image; no real personal data. row = bytes(v for x in range(args.size) for v in ((x * 13) % 256, (x * 37) % 256, (x * 71) % 256)) image = Image.frombytes("RGB", (args.size, args.size), row * args.size) stream = BytesIO() image.save(stream, "PNG") data = stream.getvalue() client = Client(args.base_url, args.ca_file, timeout=30) alice, bob, _, chat = client.bootstrap() baseline = metrics(chat["id"]) check(baseline["concurrency"] == args.workers and baseline["prefetch"] == args.prefetch, "worker settings do not match requested profile") check(baseline["pending"] == 0, "use an idle isolated stack") for service in ("api-a", "api-b"): env = dict(item.split('=', 1) for item in container_info(service)['Config']['Env']) check(int(env['MAX_PENDING_JOBS']) == args.max_pending, 'running API MAX_PENDING_JOBS does not match profile') resources = container_info('worker')['HostConfig'] path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=track1.png" pending, records, samples, errors = {}, [], [], [] lock, finished = threading.Lock(), threading.Event() started = time.monotonic() def observe(): local = Client(args.base_url, args.ca_file, timeout=5) while not finished.wait(.1): with lock: batch = list(pending.items()) for key, record in batch: if finished.is_set(): break try: att = local.call("GET", "/api/v1/attachments/" + key, actor=bob, schema="Attachment")["json"] if att["status"] in ("ready", "failed"): with lock: record["terminal"] = att["status"] record["terminal_at"] = time.monotonic() pending.pop(key, None) except Failure: continue def sample(): while not finished.is_set(): try: sample = metrics(chat["id"]) sample["queue"] = broker_depth() sample["seconds"] = time.monotonic() - started with lock: samples.append(sample) except (Failure, OSError, subprocess.TimeoutExpired) as exc: with lock: errors.append(type(exc).__name__) if finished.wait(1): break observer = threading.Thread(target=observe, daemon=True) sampler = threading.Thread(target=sample, daemon=True) observer.start() sampler.start() def upload(index): local = Client(args.base_url, args.ca_file, timeout=30) before = time.monotonic() try: response = local.call("POST", path, (202, 429), actor=alice, raw=data, headers={"Content-Type": "image/png"}, schema="Attachment") received = time.monotonic() record = {"index": index, "status": response["status"], "upload_seconds": received - before} if response["status"] == 202: record.update(id=response["json"]["id"], accepted_at=received) with lock: records.append(record) if response["status"] == 202: pending[record["id"]] = record except Failure: with lock: records.append({"index": index, "status": "transport_or_http_error"}) try: with ThreadPoolExecutor(max_workers=args.concurrency) as pool: list(pool.map(upload, range(args.images))) upload_finished = time.monotonic() deadline = upload_finished + args.max_ready_seconds while time.monotonic() < deadline: with lock: empty = not pending if empty: break time.sleep(.1) finally: measured_end = time.monotonic() finished.set() observer.join(timeout=10) sampler.join(timeout=45) elapsed = measured_end - started final = metrics(chat["id"]) queue = broker_depth() for _ in range(20): if queue["messages_ready"] == 0 and queue["messages_unacknowledged"] == 0: break time.sleep(.25) queue = broker_depth() accepted = [r for r in records if r["status"] == 202] ready = [r for r in accepted if r.get("terminal") == "ready"] latencies = [r["terminal_at"] - r["accepted_at"] for r in ready] waits = [r["queue_wait_seconds"] for r in final["jobs"] if r["queue_wait_seconds"] is not None] drained = (bool(accepted) and len(ready) == len(accepted) and final["pending"] == 0 and queue["messages_ready"] == 0 and queue["messages_unacknowledged"] == 0) recovery = False if drained: try: att, _ = client.attachment(alice, chat) client.wait_ready(bob, att, wait_seconds=args.max_ready_seconds) recovery = True except Failure: pass rejected = sum(r["status"] == 429 for r in records) report = { "run_name": args.run_name, "settings": {"workers": baseline["concurrency"], "prefetch": baseline["prefetch"], "max_pending": args.max_pending, "worker_limits": {key: resources[key] for key in ('Memory', 'NanoCpus')}}, "workload": {"images": args.images, "concurrency": args.concurrency, "size": args.size, "bytes": len(data), "type": "closed-loop burst"}, "results": {"accepted": len(accepted), "rejected_429": rejected, "ready": len(ready), "failed": sum(r.get("terminal") == "failed" for r in accepted), "timed_out": len(pending), "elapsed_seconds": elapsed, "ready_per_second": len(ready) / elapsed, "ready_latency_ms": {str(p): percentile(latencies, p) for p in (.5, .95, .99)}, "queue_wait_ms": {str(p): percentile(waits, p) for p in (.5, .95, .99)}, "max_sampled_rss_bytes": max((s["rss_bytes"] for s in samples), default=None), "drained": drained, "recovery_upload_ready": recovery}, "records": [{k: v for k, v in r.items() if k not in ("accepted_at", "terminal_at")} | ({"ready_seconds": r["terminal_at"] - r["accepted_at"]} if "terminal_at" in r else {}) for r in records], "samples": samples, "metric_errors": errors, "limitations": "Closed-loop; ready time is polling upper bound. RSS is sum of worker/container processes, shared pages may be counted twice. Sampling may miss peaks. Queue wait is first claim minus durable job creation.", } target = Path(args.output or ROOT / "evidence" / (args.run_name + ".json")) target.parent.mkdir(parents=True, exist_ok=True) target.write_text(json.dumps(report, indent=2) + "\n") print(json.dumps(report["results"], indent=2)) check(not errors and bool(samples), "incomplete metrics; result saved but not valid evidence") check(all(r["status"] in (202, 429) for r in records), "unexpected HTTP/transport failures") check(drained and recovery, "pipeline did not drain/recover; see saved result") if args.require_backpressure: check(rejected > 0, "backpressure was not observed") if __name__ == "__main__": run(main)