#!/usr/bin/env python3 """Lab5 grade-5 track 1: queue behaviour under small vs huge images. The experiment is deliberately short and self-contained: profiles: serial (concurrency 1, prefetch 1) balanced (4/4) eager (8/32) scenarios: small_many (30 x 192x128 PNG, variants mostly fit without scaling) huge_few (12 x 2400x1800 PNG, every job needs heavy scaling) For each cell the worker is recreated with the profile, the images are uploaded concurrently, and the script records: * throughput (ready jobs / pipeline span), * queue lag: created_at -> processing_started_at (95th percentile), * time-to-ready: created_at -> finished_at (p50/p95), * worker RSS baseline/peak sampled from /proc/1/status, * upload latency and max in-flight jobs. A final backpressure run caps MAX_PENDING_JOBS at 3, stops the worker, shows 503 + Retry-After, then starts the worker and proves recovery. Outputs evidence/track1.json plus a self-contained HTML dashboard with inline SVG (no external JS/CSS/fonts). The script only drives the disposable local Compose stand. It restores the default worker/API environment in a finally block, so a failed run does not leave the stand throttled. Usage: python3 scripts/track1_experiment.py --base-url http://localhost:8080 python3 scripts/track1_experiment.py --base-url http://172.18.0.1:8080 \ --small-jobs 30 --huge-jobs 12 """ from __future__ import annotations import argparse import hashlib import html import json import math import os import platform import struct import subprocess import sys import threading import time import zlib from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) sys.path.insert(0, str(ROOT / "scripts")) from client import Client, Failure, check # noqa: E402 from lab5_resilience import ( # noqa: E402 compose, compose_capture, inspect, recreate_api, recreate_worker, restore_api, restore_worker, service_ids, wait_healthy, wait_stopped, ) PROFILES = [ {"name": "serial", "concurrency": 1, "prefetch": 1, "note": "one job at a time; minimum memory, maximum queueing"}, {"name": "balanced", "concurrency": 4, "prefetch": 4, "note": "moderate in-process parallelism and broker window"}, {"name": "eager", "concurrency": 8, "prefetch": 32, "note": "wide broker window; highest memory, lowest latency"}, ] SCENARIOS = [ {"name": "small_many", "width": 192, "height": 128, "description": "many small images; variants fit or crop without large scaling"}, {"name": "huge_few", "width": 2400, "height": 1800, "description": "few 4.32 MP images; optimized variant must scale down to 1280"}, ] # --------------------------------------------------------------------------- # Fixture generation and small helpers def png_chunk(kind, payload): crc = zlib.crc32(kind + payload) & 0xFFFFFFFF return struct.pack(">I", len(payload)) + kind + payload + struct.pack(">I", crc) # AGENT: fixtures are generated in-process so the experiment has no binary # fixtures in git and the exact bytes/pixels are recorded in the JSON. def make_png(width, height): """Deterministic RGB PNG. The pattern compresses well, but the decoder still allocates width*height*3 bytes, which is exactly what the experiment wants to stress without making upload bandwidth the bottleneck.""" ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) base = bytes(range(256)) row_len = width * 3 repeated = base * ((row_len // 256) + 2) compressor = zlib.compressobj(6) idat = bytearray() for y in range(height): offset = (y * 7) % 256 idat += compressor.compress(b"\x00" + repeated[offset : offset + row_len]) idat += compressor.flush() return ( b"\x89PNG\r\n\x1a\n" + png_chunk(b"IHDR", ihdr) + png_chunk(b"IDAT", bytes(idat)) + png_chunk(b"IEND", b"") ) def percentile(values, p): if not values: return None ordered = sorted(values) if len(ordered) == 1: return float(ordered[0]) position = (len(ordered) - 1) * (p / 100.0) low = math.floor(position) high = math.ceil(position) if low == high: return float(ordered[low]) weight = position - low return float(ordered[low] * (1 - weight) + ordered[high] * weight) def r(value, digits=2): return None if value is None else round(float(value), digits) def worker_container(): ids = service_ids("worker") if not ids: raise Failure("worker container is not running") name = inspect(ids[0]).get("Name", "").lstrip("/") if not name: raise Failure("cannot resolve worker container name") return name def psql(sql): # lab5_resilience.psql is not exported under that name in all versions; # keep the small wrapper local so this script is easy to audit. import shlex command = ( 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At -F "|" ' "-v ON_ERROR_STOP=1 -c " + shlex.quote(sql) ) out = compose_capture("exec", "-T", "postgres", "sh", "-c", command) return [line for line in out.splitlines() if line.strip()] def job_rows(attachment_ids): if not attachment_ids: return [] quoted = ",".join("'" + item + "'" for item in attachment_ids) sql = ( "SELECT attachment_id, status, attempt, coalesce(error_code,''), " "extract(epoch from created_at), " "coalesce(extract(epoch from processing_started_at), 0), " "coalesce(extract(epoch from finished_at), 0) " f"FROM attachment_jobs WHERE attachment_id IN ({quoted})" ) rows = [] for line in psql(sql): parts = line.split("|") if len(parts) != 7: continue rows.append( { "attachment_id": parts[0], "status": parts[1], "attempt": int(parts[2]), "error_code": parts[3] or None, "created_at": float(parts[4]), "processing_started_at": float(parts[5]) if parts[5] not in ("", "0") else None, "finished_at": float(parts[6]) if parts[6] not in ("", "0") else None, } ) return rows def pending_count(): lines = psql( "SELECT count(*) FROM attachment_jobs WHERE status IN ('queued','processing')" ) return int(lines[0]) if lines else 0 def wait_drained(timeout=180): deadline = time.monotonic() + timeout last = None while time.monotonic() < deadline: last = pending_count() if last == 0: return time.sleep(0.5) raise Failure(f"pipeline did not drain within {timeout}s; pending={last}") def upload_one(base_url, actor, chat_id, data, filename): client = Client(base_url, timeout=60) started = time.monotonic() try: response = client.call( "POST", f"/api/v1/chats/{chat_id}/attachments?filename={filename}", expected=(202, 503), actor=actor, raw=data, headers={"Content-Type": "image/png"}, schema="Attachment", ) except Failure as exc: return {"status": 0, "error": str(exc), "elapsed": time.monotonic() - started} result = { "status": response["status"], "elapsed": time.monotonic() - started, "retry_after": response["headers"].get("Retry-After"), } if response["status"] == 202: result["attachment_id"] = response["json"]["id"] else: result["error_code"] = response["json"]["error"]["code"] return result def upload_many(base_url, actor, chat_id, data, count, prefix): with ThreadPoolExecutor(max_workers=min(8, count)) as pool: futures = [ pool.submit(upload_one, base_url, actor, chat_id, data, f"{prefix}-{index}.png") for index in range(count) ] return [future.result() for future in futures] # --------------------------------------------------------------------------- # Worker RSS sampler # AGENT: RSS is sampled inside the worker container, not with `docker stats`, # so the metric is the process VmRSS without host/daemon noise. class RssSampler: def __init__(self, container): self.container = container self.samples = [] self._stop = threading.Event() self._process = None self._thread = None def start(self): self._process = subprocess.Popen( [ "docker", "exec", self.container, "sh", "-c", "while :; do awk '/VmRSS/{print $2}' /proc/1/status; sleep 0.1; done", ], cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, ) self._thread = threading.Thread(target=self._read, daemon=True) self._thread.start() def _read(self): if not self._process or not self._process.stdout: return for line in self._process.stdout: if self._stop.is_set(): break try: self.samples.append(int(line.strip())) except ValueError: continue def snapshot(self): return list(self.samples) def stop(self): self._stop.set() if self._process: self._process.terminate() try: self._process.wait(timeout=5) except subprocess.TimeoutExpired: self._process.kill() if self._thread: self._thread.join(timeout=2) return list(self.samples) # --------------------------------------------------------------------------- # Run phases def wait_for_terminal(attachment_ids, started_at, timeout=240): deadline = time.monotonic() + timeout timeline = [] rows = [] while time.monotonic() < deadline: rows = job_rows(attachment_ids) counts = {"queued": 0, "processing": 0, "ready": 0, "failed": 0} for row in rows: counts[row["status"]] = counts.get(row["status"], 0) + 1 timeline.append( { "t": round(time.monotonic() - started_at, 3), **counts, } ) if len(rows) == len(attachment_ids) and all( row["status"] in ("ready", "failed") for row in rows ): return rows, timeline time.sleep(0.25) raise Failure(f"jobs did not finish within {timeout}s; last={rows}") def measure_run(base_url, actor, chat_id, profile, scenario, count, data, digest, cpu_limit="1.0", mem_limit="2G"): wait_drained() recreate_worker( { "WORKER_CONCURRENCY": str(profile["concurrency"]), "WORKER_PREFETCH": str(profile["prefetch"]), "WORKER_CPU_LIMIT": str(cpu_limit), "WORKER_MEM_LIMIT": str(mem_limit), "WORKER_FAULT_MODE": "none", "WORKER_CRASH_AFTER_EFFECT": "false", "WORKER_CRASH_AFTER_CLAIM": "false", } ) time.sleep(1.0) container = worker_container() sampler = RssSampler(container) sampler.start() time.sleep(1.2) baseline_samples = sampler.snapshot() submit_started = time.monotonic() uploads = upload_many(base_url, actor, chat_id, data, count, f"track1-{profile['name']}-{scenario['name']}") submit_seconds = time.monotonic() - submit_started accepted = [item["attachment_id"] for item in uploads if item.get("attachment_id")] rejected = [item for item in uploads if item["status"] != 202] if rejected: raise Failure(f"uploads were rejected during {profile['name']}/{scenario['name']}: {rejected[:3]}") if len(accepted) != count: raise Failure(f"only {len(accepted)}/{count} uploads were accepted") rows, timeline = wait_for_terminal(accepted, submit_started) wall_seconds = time.monotonic() - submit_started time.sleep(0.5) samples = sampler.stop() failed = [row for row in rows if row["status"] != "ready"] if failed: raise Failure(f"jobs failed in {profile['name']}/{scenario['name']}: {failed[:2]}") queue_lag = [ (row["processing_started_at"] - row["created_at"]) * 1000 for row in rows if row["processing_started_at"] is not None ] time_to_ready = [ (row["finished_at"] - row["created_at"]) * 1000 for row in rows if row["finished_at"] is not None ] processing = [ (row["finished_at"] - row["processing_started_at"]) * 1000 for row in rows if row["processing_started_at"] is not None and row["finished_at"] is not None ] upload_latency = [item["elapsed"] * 1000 for item in uploads] first_created = min(row["created_at"] for row in rows) last_finished = max(row["finished_at"] for row in rows) span = max(last_finished - first_created, 0.001) throughput = len(rows) / span events = [] for row in rows: events.append((row["created_at"], 1)) events.append((row["finished_at"], -1)) backlog = 0 max_backlog = 0 for _, delta in sorted(events, key=lambda item: (item[0], -item[1])): backlog += delta max_backlog = max(max_backlog, backlog) # Actual parallel handlers: intersections of processing_started_at -> # finished_at. These are bounded by WORKER_CONCURRENCY. processing_events = [] for row in rows: if row["processing_started_at"] is None or row["finished_at"] is None: continue processing_events.append((row["processing_started_at"], 1)) processing_events.append((row["finished_at"], -1)) parallel = 0 max_parallel = 0 for _, delta in sorted(processing_events, key=lambda item: (item[0], -item[1])): parallel += delta max_parallel = max(max_parallel, parallel) baseline_kb = max(baseline_samples) if baseline_samples else None sample_array = [value for value in samples if value > 0] max_kb = max(sample_array) if sample_array else None end_kb = sample_array[-1] if sample_array else None return { "profile": profile["name"], "scenario": scenario["name"], "concurrency": profile["concurrency"], "prefetch": profile["prefetch"], "jobs": len(rows), "image_width": scenario["width"], "image_height": scenario["height"], "image_bytes": len(data), "image_sha256": digest, "throughput_jobs_per_s": r(throughput, 3), "pipeline_span_seconds": r(span, 3), "submit_seconds": r(submit_seconds, 3), "wall_seconds": r(wall_seconds, 3), "upload_p50_ms": r(percentile(upload_latency, 50), 1), "upload_p95_ms": r(percentile(upload_latency, 95), 1), "queue_lag_p50_ms": r(percentile(queue_lag, 50), 1), "queue_lag_p95_ms": r(percentile(queue_lag, 95), 1), "queue_lag_max_ms": r(max(queue_lag) if queue_lag else None, 1), "time_to_ready_p50_ms": r(percentile(time_to_ready, 50), 1), "time_to_ready_p95_ms": r(percentile(time_to_ready, 95), 1), "time_to_ready_max_ms": r(max(time_to_ready) if time_to_ready else None, 1), "processing_p50_ms": r(percentile(processing, 50), 1), "processing_p95_ms": r(percentile(processing, 95), 1), "max_backlog_jobs": max_backlog, "max_parallel_jobs": max_parallel, "rss_baseline_mib": r(baseline_kb / 1024 if baseline_kb else None, 1), "rss_peak_mib": r(max_kb / 1024 if max_kb else None, 1), "rss_end_mib": r(end_kb / 1024 if end_kb else None, 1), "rss_samples": len(sample_array), "rss_delta_mib": r((max_kb - baseline_kb) / 1024 if max_kb and baseline_kb else None, 1), "timeline": timeline, } # AGENT: backpressure is tested separately from the matrix: cap=3, worker down, # sequential uploads until 503. The accepted jobs are the evidence that bounded # acceptance did not lose already confirmed work. def measure_backpressure(base_url, actor, chat_id, data, digest): wait_drained() recreate_api({"MAX_PENDING_JOBS": "3"}) compose("stop", "worker") wait_stopped("worker") time.sleep(0.5) attempts = [] accepted_ids = [] for index in range(12): item = upload_one(base_url, actor, chat_id, data, f"track1-backpressure-{index}.png") item["index"] = index attempts.append(item) if item["status"] == 202: accepted_ids.append(item["attachment_id"]) if item["status"] == 503: break if not any(item["status"] == 503 for item in attempts): raise Failure("MAX_PENDING_JOBS=3 did not produce a 503") blocked = next(item for item in attempts if item["status"] == 503) if blocked.get("retry_after") != "5": raise Failure(f"503 did not carry Retry-After: 5 (got {blocked.get('retry_after')!r})") recovery_started = time.monotonic() compose("start", "worker") wait_healthy("worker") rows, _ = wait_for_terminal(accepted_ids, recovery_started, timeout=180) recovery_seconds = time.monotonic() - recovery_started if any(row["status"] != "ready" for row in rows): raise Failure("accepted jobs did not recover after worker start") wait_drained() probe = upload_one(base_url, actor, chat_id, data, "track1-backpressure-probe.png") if probe["status"] != 202: raise Failure(f"probe upload after recovery returned {probe['status']}") probe_ready_rows, _ = wait_for_terminal([probe["attachment_id"]], time.monotonic(), timeout=120) if probe_ready_rows[0]["status"] != "ready": raise Failure("probe upload did not become ready") return { "cap": 3, "attempts": [ { "index": item["index"], "status": item["status"], "elapsed_ms": r(item["elapsed"] * 1000, 1), "retry_after": item.get("retry_after"), "error_code": item.get("error_code"), } for item in attempts ], "accepted_before_503": len(accepted_ids), "blocked_status": blocked["status"], "blocked_retry_after": blocked.get("retry_after"), "recovery_seconds": r(recovery_seconds, 3), "recovered_jobs": len(rows), "probe_status": probe["status"], "probe_ready": True, "image_bytes": len(data), "image_sha256": digest, } # --------------------------------------------------------------------------- # HTML dashboard (inline CSS + SVG only) # AGENT: the dashboard is a single HTML file with inline CSS and SVG. No CDN, # JS framework or external font is needed, so evidence stays readable offline. COLORS = {"serial": "#64748b", "balanced": "#0284c7", "eager": "#f59e0b"} def svg_chart(title, subtitle, categories, series, values, unit, digits=1, lower_better=False): width, height = 760, 320 left, right, top, bottom = 64, 20, 52, 62 plot_w = width - left - right plot_h = height - top - bottom all_values = [v for group in values.values() for v in group if v is not None] maximum = max(all_values) if all_values else 1.0 maximum = maximum * 1.18 if maximum > 0 else 1.0 parts = [ f'', f'{html.escape(title)}', f'{html.escape(subtitle)}', ] for tick in range(5): value = maximum * tick / 4 y = top + plot_h - plot_h * tick / 4 parts.append( f'' ) parts.append( f'' f"{value:.0f}{html.escape(unit)}" ) group_width = plot_w / len(categories) bar_width = group_width / (len(series) + 1.1) for group_index, category in enumerate(categories): base_x = left + group_index * group_width + bar_width * 0.35 for series_index, name in enumerate(series): value = values[name][group_index] if value is None: continue bar_h = plot_h * value / maximum x = base_x + series_index * bar_width y = top + plot_h - bar_h parts.append( f'' ) label = f"{value:.{digits}f}{unit}" parts.append( f'{html.escape(label)}' ) parts.append( f'' f"{html.escape(category)}" ) # Legend legend_x = left for name in series: parts.append( f'' ) parts.append( f'' f"{html.escape(name)}" ) legend_x += 90 hint = "lower is better" if lower_better else "higher is better" parts.append( f'{hint}' ) parts.append("") return "\n".join(parts) def build_html(evidence): runs = evidence["runs"] scenarios = [item["name"] for item in evidence["scenarios"]] profiles = [item["name"] for item in evidence["profiles"]] lookup = {(run["profile"], run["scenario"]): run for run in runs} def series_for(metric): return { profile: [ (lookup.get((profile, scenario)) or {}).get(metric) for scenario in scenarios ] for profile in profiles } charts = [ svg_chart( "Throughput: готовые задания в секунду", "Чем выше, тем быстрее pipeline обрабатывает очередь.", scenarios, profiles, series_for("throughput_jobs_per_s"), " jobs/s", digits=2, ), svg_chart( "p95 time-to-ready: от 202 до ready", "95% заданий стали ready быстрее этого времени (меньше — лучше).", scenarios, profiles, series_for("time_to_ready_p95_ms"), " ms", digits=0, lower_better=True, ), svg_chart( "p95 queue lag: ожидание до старта обработки", "Время между durable 202 и первым claim worker-а (меньше — лучше).", scenarios, profiles, series_for("queue_lag_p95_ms"), " ms", digits=0, lower_better=True, ), svg_chart( "Пиковый RSS worker", "VmRSS процесса worker во время задания (меньше — экономнее).", scenarios, profiles, series_for("rss_peak_mib"), " MiB", digits=0, lower_better=True, ), ] def fmt(value, digits=1, suffix=""): return "—" if value is None else f"{value:.{digits}f}{suffix}" table_rows = [] for run in runs: table_rows.append( "" f"{html.escape(run['profile'])}" f"{html.escape(run['scenario'])}" f"{run['jobs']}" f"{fmt(run['throughput_jobs_per_s'], 2)}" f"{fmt(run['queue_lag_p95_ms'], 0)}" f"{fmt(run['time_to_ready_p95_ms'], 0)}" f"{fmt(run['processing_p95_ms'], 0)}" f"{fmt(run['rss_peak_mib'], 0)}" f"{run['max_parallel_jobs']}" f"{run['max_backlog_jobs']}" "" ) # Beginner-friendly observations computed from the actual numbers. observations = [] for scenario in scenarios: serial = lookup.get(("serial", scenario)) eager = lookup.get(("eager", scenario)) if serial and eager and serial["throughput_jobs_per_s"]: speedup = eager["throughput_jobs_per_s"] / serial["throughput_jobs_per_s"] if speedup >= 1.05: comparison = f"eager быстрее serial в {speedup:.1f}× по throughput" elif speedup <= 0.95: comparison = f"eager медленнее serial ({speedup:.2f}×) по throughput" else: comparison = "throughput eager и serial практически одинаков" observations.append(f"{scenario}: {comparison}.") if serial["rss_peak_mib"] and eager["rss_peak_mib"]: ratio = eager["rss_peak_mib"] / serial["rss_peak_mib"] observations.append( f"{scenario}: пиковый RSS eager относится к serial как {ratio:.1f}×." ) balanced = lookup.get(("balanced", "small_many")) huge = lookup.get(("balanced", "huge_few")) if balanced and huge and balanced["time_to_ready_p95_ms"] and huge["time_to_ready_p95_ms"]: observations.append( "При balanced p95 time-to-ready для huge_few в " f"{huge['time_to_ready_p95_ms'] / balanced['time_to_ready_p95_ms']:.1f}× " "больше, чем для small_many: размер изображения меняет стоимость задания, " "а не только число заданий." ) huge_serial = lookup.get(("serial", "huge_few")) huge_eager = lookup.get(("eager", "huge_few")) if huge_serial and huge_eager and huge_serial["throughput_jobs_per_s"]: speedup = huge_eager["throughput_jobs_per_s"] / huge_serial["throughput_jobs_per_s"] if speedup < 1.2: observations.append( "На huge_few eager почти не поднял throughput " f"({speedup:.2f}×): worker ограничен " f"{html.escape(str(evidence['environment']['worker_cpu_limit']))} CPU, " "а декодирование и масштабирование упираются в CPU. Concurrency " "снижает queue lag, но при этом растёт RSS; это и есть компромисс " "latency/ресурсы, а не бесплатное ускорение." ) else: observations.append( f"На huge_few eager поднял throughput в {speedup:.2f}× относительно " "serial, потому что несколько тяжёлых заданий перекрываются." ) observations.append( "serial ограничивает одновременно обрабатываемые задания одним, поэтому " "queue lag растёт, но RSS остаётся минимальным. eager держит больше " "заданий в работе и в prefetch-окне брокера." ) observations.append( "Backpressure: при MAX_PENDING_JOBS=3 API принял " f"{evidence['backpressure']['accepted_before_503']} задания, затем вернул 503 " f"с Retry-After={evidence['backpressure']['blocked_retry_after']}; после старта " f"worker очередь восстановилась за {evidence['backpressure']['recovery_seconds']:.1f} с." ) backpressure_rows = "".join( "" f"{item['index'] + 1}" f"{item['status']}" f"{fmt(item['elapsed_ms'], 1)}" f"{html.escape(str(item['retry_after'] or '—'))}" f"{html.escape(str(item['error_code'] or '—'))}" "" for item in evidence["backpressure"]["attempts"] ) observations_html = "".join(f"
  • {html.escape(text)}
  • " for text in observations) chart_html = "\n".join(f'
    {chart}
    ' for chart in charts) return f""" Lab5 track 1 — очередь под перегрузкой

    Lab5, трек 1: очередь под перегрузкой

    Сгенерировано {html.escape(evidence['generated_at'])}. Стенд: {html.escape(evidence['environment']['base_url'])}, {evidence['environment']['cpu_count']} CPU, Docker {html.escape(evidence['environment']['docker_version'])}.

    Гипотеза. Маленькие изображения, которые почти не требуют масштабирования, должны давать высокий throughput и низкий RSS на задание. Большие изображения требуют декодирования и downscale, поэтому каждое задание дороже: при том же числе worker-ов очередь растёт, p95 time-to-ready увеличивается, а пиковый RSS зависит от concurrency сильнее, чем у маленьких файлов. Одновременно проверяем компромисс: eager уменьшает ожидание, но повышает потребление памяти; serial экономит память, но ограничивает throughput.

    Профили и сценарии

    {''.join(f"" for p in evidence['profiles'])}
    ПрофильConcurrencyPrefetchСмысл
    {html.escape(p['name'])}{p['concurrency']}{p['prefetch']}{html.escape(p['note'])}

    Сценарии: {'; '.join(f"{html.escape(s['name'])} — {s['width']}×{s['height']}, {s['count']} jobs, {s['description']}" for s in evidence['scenarios'])}.

    Ключевые числа

    {len(runs)}
    прогонов матрицы
    {max(run['rss_peak_mib'] or 0 for run in runs):.0f} MiB
    максимальный RSS worker
    {evidence['backpressure']['accepted_before_503']}
    принято до первого 503
    {evidence['backpressure']['recovery_seconds']:.1f} s
    восстановление после backpressure
    {chart_html}

    Полная матрица

    {''.join(table_rows)}
    ПрофильСценарийJobsJobs/sQueue lag p95, msReady p95, msProcessing p95, msRSS peak, MiBMax parallelMax backlog

    Queue lag = processing_started_at - created_at. Time-to-ready = finished_at - created_at. Processing = finished_at - processing_started_at. RSS — VmRSS процесса worker, сэмплирование примерно каждые 100 мс. Max parallel — максимум одновременно обрабатываемых заданий; max backlog — максимум принятых, но ещё не готовых заданий.

    Backpressure и восстановление

    Перед этим прогоном API пересоздан с MAX_PENDING_JOBS=3, worker остановлен. Последовательные upload-ы показывают, где заканчивается принятая работа. Затем worker запускается, принятые задания должны дойти до ready.

    {backpressure_rows}
    #HTTPLatency, msRetry-AfterError code

    Recovery: {evidence['backpressure']['recovered_jobs']} принятых заданий стали ready за {evidence['backpressure']['recovery_seconds']:.2f} с после старта worker. Финальный probe upload: HTTP {evidence['backpressure']['probe_status']} → ready.

    Что это значит

    Ограничения

    """ def write_markdown(evidence, path): lines = [ "# Lab5 track 1 — queue under load", "", f"Generated: `{evidence['generated_at']}`", "", "Synthetic images; no tokens or credentials are stored.", "", "| Profile | Scenario | Jobs | Jobs/s | Queue lag p95, ms | Ready p95, ms | RSS peak, MiB |", "| --- | --- | ---: | ---: | ---: | ---: | ---: |", ] for run in evidence["runs"]: lines.append( f"| {run['profile']} | {run['scenario']} | {run['jobs']} | " f"{run['throughput_jobs_per_s']:.2f} | {run['queue_lag_p95_ms']:.0f} | " f"{run['time_to_ready_p95_ms']:.0f} | {run['rss_peak_mib']:.0f} |" ) bp = evidence["backpressure"] lines.extend( [ "", f"Backpressure: cap={bp['cap']}, accepted={bp['accepted_before_503']}, " f"first 503 Retry-After={bp['blocked_retry_after']}, " f"recovery={bp['recovery_seconds']:.2f}s, probe HTTP {bp['probe_status']}.", "", ] ) path.write_text("\n".join(lines)) # --------------------------------------------------------------------------- # Main def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080")) parser.add_argument("--small-jobs", type=int, default=30) parser.add_argument("--huge-jobs", type=int, default=12) parser.add_argument( "--worker-cpu-limit", default=os.environ.get("TRACK1_WORKER_CPU_LIMIT", "1.0"), help="Compose CPU quota applied to every experimental worker (default 1.0)", ) parser.add_argument( "--worker-mem-limit", default=os.environ.get("TRACK1_WORKER_MEM_LIMIT", "2G"), help="Compose memory limit applied to every experimental worker (default 2G)", ) parser.add_argument("--out", default=str(ROOT / "evidence" / "track1.json")) parser.add_argument("--html-out", default=str(ROOT / "evidence" / "track1.html")) parser.add_argument("--md-out", default=str(ROOT / "evidence" / "track1.md")) parser.add_argument( "--skip-backpressure", action="store_true", help="only run the profile x scenario matrix", ) args = parser.parse_args() base_url = args.base_url.rstrip("/") out = Path(args.out) html_out = Path(args.html_out) md_out = Path(args.md_out) out.parent.mkdir(parents=True, exist_ok=True) html_out.parent.mkdir(parents=True, exist_ok=True) print(f"== bootstrap {base_url}", flush=True) client = Client(base_url, timeout=60) alice, _bob, _eve, chat = client.bootstrap() scenarios = [] for item in SCENARIOS: count = args.small_jobs if item["name"] == "small_many" else args.huge_jobs data = make_png(item["width"], item["height"]) scenarios.append({**item, "count": count, "bytes": len(data)}) try: wait_drained() runs = [] for profile in PROFILES: for scenario in scenarios: data = make_png(scenario["width"], scenario["height"]) if len(data) != scenario["bytes"]: raise Failure("fixture generation is not deterministic") digest = hashlib.sha256(data).hexdigest() print( f"== run {profile['name']}/{scenario['name']} " f"({scenario['count']} jobs, {scenario['width']}x{scenario['height']})", flush=True, ) result = measure_run( base_url, alice, chat["id"], profile, scenario, scenario["count"], data, digest, cpu_limit=args.worker_cpu_limit, mem_limit=args.worker_mem_limit, ) runs.append(result) print( f" throughput={result['throughput_jobs_per_s']} jobs/s " f"queue_p95={result['queue_lag_p95_ms']} ms " f"ready_p95={result['time_to_ready_p95_ms']} ms " f"rss_peak={result['rss_peak_mib']} MiB", flush=True, ) backpressure = None if not args.skip_backpressure: print("== backpressure (cap=3, worker stopped)", flush=True) small = scenarios[0] backpressure = measure_backpressure( base_url, alice, chat["id"], make_png(small["width"], small["height"]), hashlib.sha256(make_png(small["width"], small["height"])).hexdigest(), ) print( f" accepted={backpressure['accepted_before_503']} " f"503_retry_after={backpressure['blocked_retry_after']} " f"recovery={backpressure['recovery_seconds']}s", flush=True, ) finally: # Restore the stand even if a measurement raised. try: restore_worker() except Exception as exc: # noqa: BLE001 - best effort in cleanup print(f"warning: worker restore failed: {exc}", flush=True) try: restore_api() except Exception as exc: # noqa: BLE001 - best effort in cleanup print(f"warning: API restore failed: {exc}", flush=True) if backpressure is None: backpressure = { "cap": 3, "attempts": [], "accepted_before_503": 0, "blocked_status": None, "blocked_retry_after": None, "recovery_seconds": 0.0, "recovered_jobs": 0, "probe_status": None, "probe_ready": False, } docker_version = compose_capture("version", "--short").strip() evidence = { "generated_at": datetime.now(timezone.utc).isoformat(), "environment": { "base_url": base_url, "host": platform.node(), "platform": platform.platform(), "cpu_count": os.cpu_count(), "python": platform.python_version(), "docker_version": docker_version, "compose_project": "mtusi2026lab", "worker_container": worker_container(), "worker_cpu_limit": args.worker_cpu_limit, "worker_mem_limit": args.worker_mem_limit, "worker_cpu_limit_default": "1.0", "worker_mem_limit_default": "1G", "note": ( "Experimental workers use a larger memory limit than .env so the " "eager huge-image run can hold decoded images without OOM; CPU is " "still limited so throughput is not a host-CPU artifact." ), }, "hypothesis": ( "Small images that fit the variants without large scaling should be cheaper " "per job; huge images force decode+downscale, so each job costs more memory " "and time. More concurrency should raise throughput and RSS but reduce queue lag." ), "profiles": PROFILES, "scenarios": scenarios, "runs": runs, "backpressure": backpressure, } out.write_text(json.dumps(evidence, indent=2) + "\n") html_out.write_text(build_html(evidence)) write_markdown(evidence, md_out) print(json.dumps({"ok": True, "json": str(out), "html": str(html_out), "md": str(md_out)})) if __name__ == "__main__": try: main() except (Failure, RuntimeError, subprocess.TimeoutExpired) as exc: print("FAIL:", exc) raise SystemExit(1)