This commit is contained in:
Матвей
2026-10-26 15:49:31 +03:00
parent 4f6e69cfac
commit a1bb6f8bde
18 changed files with 820 additions and 195 deletions
+2 -2
View File
@@ -6,5 +6,5 @@ cd "$(dirname "$0")/.."
docker compose restart postgres s3 broker
docker compose up -d --wait postgres s3 broker
docker compose run --rm --no-deps s3-init
docker compose restart worker api-a api-b load_balancer
docker compose up -d --wait worker api-a api-b load_balancer
docker compose restart worker outbox reconcile api-a api-b load_balancer
docker compose up -d --wait worker outbox reconcile api-a api-b load_balancer
+172 -101
View File
@@ -1,13 +1,5 @@
#!/usr/bin/env python3
"""Repeatable level-5 queue-overload experiment.
Run a Compose profile first, for example:
WORKER_CONCURRENCY=2 MAX_PENDING_JOBS=20 docker compose up -d --build
.venv/bin/python scripts/track1.py --run-name c2-p2 --workers 2 --prefetch 2 --images 40 --size 1600
The script records client-observed acceptance/ready latency, RabbitMQ queue depth
and Docker RSS samples. It does not print credentials or session tokens.
"""
"""Closed-loop upload burst with concurrent readiness observations and drain proof."""
from concurrent.futures import ThreadPoolExecutor, as_completed
from io import BytesIO
import json
@@ -17,11 +9,10 @@ import subprocess
import sys
import threading
import time
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, arguments, check, run # noqa: E402
from client import Client, Failure, arguments, check, run
def percentile(values, point):
@@ -29,114 +20,194 @@ def percentile(values, point):
return round(values[max(0, math.ceil(len(values) * point) - 1)] * 1000, 3) if values else None
def docker_text(*args):
def command_json(*args):
result = subprocess.run(args, cwd=ROOT, text=True, capture_output=True, timeout=20)
if result.returncode:
raise Failure("metrics command failed: " + args[0])
try:
return subprocess.run(args, cwd=ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
timeout=10, check=True).stdout.strip()
except (OSError, subprocess.SubprocessError):
return None
return json.loads(result.stdout)
except ValueError:
raise Failure("metrics command returned invalid JSON") from None
def metrics(chat_id):
return command_json("docker", "compose", "exec", "-T", "worker",
"/app/.venv/bin/python", "-m", "src.pipeline_metrics", chat_id)
def broker_depth():
text = docker_text("docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "list_queues", "name", "messages_ready", "messages_unacknowledged", "--formatter", "json")
if not text:
return None
try:
rows = json.loads(text)
return next((row for row in rows if row["name"] == "attachment.process.v1"), None)
except (ValueError, KeyError, TypeError):
return None
rows = command_json("docker", "compose", "exec", "-T", "broker",
"rabbitmqctl", "list_queues", "name", "messages_ready",
"messages_unacknowledged", "--formatter", "json")
return next(row for row in rows if row["name"] == "attachment.process.v1")
def worker_rss():
container = docker_text("docker", "compose", "ps", "-q", "worker")
if not container:
return None
text = docker_text("docker", "stats", "--no-stream", "--format", "{{json .}}", container)
if not text:
return None
try:
return json.loads(text).get("MemUsage")
except ValueError:
return None
def container_info(service):
result = subprocess.run(['docker', 'compose', 'ps', '-q', service], cwd=ROOT,
capture_output=True, text=True, timeout=10, check=True)
ids = result.stdout.split()
check(len(ids) == 1, 'expected exactly one container: ' + service)
return command_json('docker', 'inspect', ids[0])[0]
def main():
def configure(parser):
parser.add_argument("--run-name", required=True)
parser.add_argument("--workers", type=int, required=True, help="documented WORKER_CONCURRENCY setting")
parser.add_argument("--prefetch", type=int, required=True, help="documented RabbitMQ prefetch setting")
parser.add_argument("--max-pending", type=int, required=True, help="documented MAX_PENDING_JOBS setting")
parser.add_argument("--images", type=int, default=40)
parser.add_argument("--concurrency", type=int, default=4)
parser.add_argument("--size", type=int, default=1600, help="square synthetic PNG dimensions")
parser.add_argument("--max-ready-seconds", type=float, default=120)
parser.add_argument("--output", default=None)
args = arguments("Lab 5 track 1 image-pipeline experiment", configure)
check(1 <= args.workers <= 64 and 1 <= args.prefetch <= 1024 and 1 <= args.max_pending <= 100000, "invalid worker/prefetch/max-pending")
check(1 <= args.images <= 1000 and 1 <= args.concurrency <= 128 and 1 <= args.size <= 6000, "invalid workload")
def configure(p):
p.add_argument("--run-name", required=True)
p.add_argument("--workers", type=int, required=True)
p.add_argument("--prefetch", type=int, required=True)
p.add_argument("--max-pending", type=int, required=True)
p.add_argument("--images", type=int, default=40)
p.add_argument("--concurrency", type=int, default=8)
p.add_argument("--size", type=int, default=1600)
p.add_argument("--max-ready-seconds", type=float, default=180)
p.add_argument("--require-backpressure", action="store_true")
p.add_argument("--output")
args = arguments(__doc__, configure)
check(1 <= args.images <= 1000 and 1 <= args.concurrency <= 128, "invalid workload")
check(1 <= args.size <= 6000 and 0 < args.max_ready_seconds <= 3600, "invalid size/deadline")
check(args.workers > 0 and args.prefetch > 0 and args.max_pending > 0, "invalid profile")
from PIL import Image
image = Image.new("RGB", (args.size, args.size), "#427aa1")
raw = BytesIO(); image.save(raw, "PNG", optimize=True); data = raw.getvalue()
# Reproducible, nonuniform synthetic image; no real personal data.
row = bytes(v for x in range(args.size) for v in ((x * 13) % 256, (x * 37) % 256, (x * 71) % 256))
image = Image.frombytes("RGB", (args.size, args.size), row * args.size)
stream = BytesIO()
image.save(stream, "PNG")
data = stream.getvalue()
client = Client(args.base_url, args.ca_file, timeout=30)
alice, bob, _eve, chat = client.bootstrap()
alice, bob, _, chat = client.bootstrap()
baseline = metrics(chat["id"])
check(baseline["concurrency"] == args.workers and baseline["prefetch"] == args.prefetch,
"worker settings do not match requested profile")
check(baseline["pending"] == 0, "use an idle isolated stack")
for service in ("api-a", "api-b"):
env = dict(item.split('=', 1) for item in container_info(service)['Config']['Env'])
check(int(env['MAX_PENDING_JOBS']) == args.max_pending,
'running API MAX_PENDING_JOBS does not match profile')
resources = container_info('worker')['HostConfig']
path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=track1.png"
barrier = threading.Barrier(args.concurrency)
queue_samples, rss_samples = [], []
stop_sampling = threading.Event()
def sampler():
while not stop_sampling.wait(1):
queue_samples.append({"seconds": round(time.monotonic() - started, 3), "queue": broker_depth()})
rss_samples.append({"seconds": round(time.monotonic() - started, 3), "rss": worker_rss()})
pending, records, samples, errors = {}, [], [], []
lock, finished = threading.Lock(), threading.Event()
started = time.monotonic()
def observe():
local = Client(args.base_url, args.ca_file, timeout=5)
while not finished.wait(.1):
with lock:
batch = list(pending.items())
for key, record in batch:
if finished.is_set():
break
try:
att = local.call("GET", "/api/v1/attachments/" + key, actor=bob, schema="Attachment")["json"]
if att["status"] in ("ready", "failed"):
with lock:
record["terminal"] = att["status"]
record["terminal_at"] = time.monotonic()
pending.pop(key, None)
except Failure:
continue
def sample():
while not finished.is_set():
try:
sample = metrics(chat["id"])
sample["queue"] = broker_depth()
sample["seconds"] = time.monotonic() - started
with lock:
samples.append(sample)
except (Failure, OSError, subprocess.TimeoutExpired) as exc:
with lock:
errors.append(type(exc).__name__)
if finished.wait(1):
break
observer = threading.Thread(target=observe, daemon=True)
sampler = threading.Thread(target=sample, daemon=True)
observer.start()
sampler.start()
def upload(index):
local = Client(args.base_url, args.ca_file, timeout=30)
barrier.wait()
started_at = time.monotonic()
before = time.monotonic()
try:
response = local.call("POST", path + "-" + str(index), 202, actor=alice, raw=data,
response = local.call("POST", path, (202, 429), actor=alice, raw=data,
headers={"Content-Type": "image/png"}, schema="Attachment")
return {"accepted": True, "id": response["json"]["id"], "accepted_seconds": time.monotonic() - started_at}
except Failure as error:
return {"accepted": False, "error": str(error), "accepted_seconds": time.monotonic() - started_at}
started = time.monotonic()
thread = threading.Thread(target=sampler, daemon=True); thread.start()
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
results = list(pool.map(upload, range(args.images)))
accepted = [item for item in results if item["accepted"]]
ready_latencies, failed_ready = [], 0
deadline = time.monotonic() + args.max_ready_seconds
pending = {item["id"]: started + item["accepted_seconds"] for item in accepted}
while pending and time.monotonic() < deadline:
for attachment_id, accepted_at in list(pending.items()):
try:
metadata = client.call("GET", "/api/v1/attachments/" + attachment_id, actor=bob, schema="Attachment")["json"]
except Failure:
continue
if metadata["status"] == "ready":
ready_latencies.append(time.monotonic() - accepted_at)
del pending[attachment_id]
elif metadata["status"] == "failed":
failed_ready += 1; del pending[attachment_id]
time.sleep(.2)
stop_sampling.set(); thread.join(timeout=2)
elapsed = time.monotonic() - started
received = time.monotonic()
record = {"index": index, "status": response["status"], "upload_seconds": received - before}
if response["status"] == 202:
record.update(id=response["json"]["id"], accepted_at=received)
with lock:
records.append(record)
if response["status"] == 202:
pending[record["id"]] = record
except Failure:
with lock:
records.append({"index": index, "status": "transport_or_http_error"})
try:
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
list(pool.map(upload, range(args.images)))
upload_finished = time.monotonic()
deadline = upload_finished + args.max_ready_seconds
while time.monotonic() < deadline:
with lock:
empty = not pending
if empty:
break
time.sleep(.1)
finally:
measured_end = time.monotonic()
finished.set()
observer.join(timeout=10)
sampler.join(timeout=45)
elapsed = measured_end - started
final = metrics(chat["id"])
queue = broker_depth()
for _ in range(20):
if queue["messages_ready"] == 0 and queue["messages_unacknowledged"] == 0:
break
time.sleep(.25)
queue = broker_depth()
accepted = [r for r in records if r["status"] == 202]
ready = [r for r in accepted if r.get("terminal") == "ready"]
latencies = [r["terminal_at"] - r["accepted_at"] for r in ready]
waits = [r["queue_wait_seconds"] for r in final["jobs"] if r["queue_wait_seconds"] is not None]
drained = (bool(accepted) and len(ready) == len(accepted) and final["pending"] == 0
and queue["messages_ready"] == 0 and queue["messages_unacknowledged"] == 0)
recovery = False
if drained:
try:
att, _ = client.attachment(alice, chat)
client.wait_ready(bob, att, wait_seconds=args.max_ready_seconds)
recovery = True
except Failure:
pass
rejected = sum(r["status"] == 429 for r in records)
report = {
"track": "queue-overload", "run_name": args.run_name,
"settings": {"worker_concurrency": args.workers, "prefetch": args.prefetch, "max_pending_jobs": args.max_pending},
"workload": {"images": args.images, "client_concurrency": args.concurrency, "png_dimensions": [args.size, args.size], "input_bytes": len(data)},
"results": {"elapsed_seconds": round(elapsed, 3), "accepted": len(accepted), "rejected_or_failed": args.images - len(accepted), "ready": len(ready_latencies), "failed": failed_ready, "timed_out": len(pending), "throughput_ready_per_second": round(len(ready_latencies) / elapsed, 4), "accept_latency_ms": {"p50": percentile([x["accepted_seconds"] for x in accepted], .5), "p95": percentile([x["accepted_seconds"] for x in accepted], .95)}, "time_to_ready_ms": {"p50": percentile(ready_latencies, .5), "p95": percentile(ready_latencies, .95)}},
"queue_samples": queue_samples, "worker_rss_samples": rss_samples,
"limitations": "Closed-loop client; queue/RSS samples are null when Docker CLI is unavailable. p95 covers ready tasks only.",
"run_name": args.run_name,
"settings": {"workers": baseline["concurrency"], "prefetch": baseline["prefetch"],
"max_pending": args.max_pending,
"worker_limits": {key: resources[key] for key in ('Memory', 'NanoCpus')}},
"workload": {"images": args.images, "concurrency": args.concurrency,
"size": args.size, "bytes": len(data), "type": "closed-loop burst"},
"results": {"accepted": len(accepted), "rejected_429": rejected, "ready": len(ready),
"failed": sum(r.get("terminal") == "failed" for r in accepted),
"timed_out": len(pending), "elapsed_seconds": elapsed,
"ready_per_second": len(ready) / elapsed,
"ready_latency_ms": {str(p): percentile(latencies, p) for p in (.5, .95, .99)},
"queue_wait_ms": {str(p): percentile(waits, p) for p in (.5, .95, .99)},
"max_sampled_rss_bytes": max((s["rss_bytes"] for s in samples), default=None),
"drained": drained, "recovery_upload_ready": recovery},
"records": [{k: v for k, v in r.items() if k not in ("accepted_at", "terminal_at")} |
({"ready_seconds": r["terminal_at"] - r["accepted_at"]} if "terminal_at" in r else {})
for r in records],
"samples": samples, "metric_errors": errors,
"limitations": "Closed-loop; ready time is polling upper bound. RSS is sum of worker/container processes, shared pages may be counted twice. Sampling may miss peaks. Queue wait is first claim minus durable job creation.",
}
text = json.dumps(report, ensure_ascii=False, indent=2)
destination = Path(args.output or (ROOT / "evidence" / (args.run_name + ".json")))
destination.parent.mkdir(parents=True, exist_ok=True); destination.write_text(text + "\n", encoding="utf-8")
print(text)
check(bool(accepted), "no accepted uploads")
target = Path(args.output or ROOT / "evidence" / (args.run_name + ".json"))
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(json.dumps(report, indent=2) + "\n")
print(json.dumps(report["results"], indent=2))
check(not errors and bool(samples), "incomplete metrics; result saved but not valid evidence")
check(all(r["status"] in (202, 429) for r in records), "unexpected HTTP/transport failures")
check(drained and recovery, "pipeline did not drain/recover; see saved result")
if args.require_backpressure:
check(rejected > 0, "backpressure was not observed")
if __name__ == "__main__":