#!/usr/bin/env python3
"""Track 1: queue behaviour under small, exact and huge images.
The experiment is deliberately short and self-contained:
profiles: serial (concurrency 1, prefetch 1)
balanced (4/4)
eager (8/32)
scenarios: small_many (30 x 192x128 PNG, almost no scaling)
exactly_right (18 x 1280x1280 PNG, optimized WebP at native size)
huge_few (12 x 2400x1800 PNG, every path needs heavy scaling)
For each cell the worker is recreated with the profile, the images are uploaded
concurrently, and the script records:
* throughput (ready jobs / pipeline span),
* queue lag: created_at -> processing_started_at (95th percentile),
* time-to-ready: created_at -> finished_at (p50/p95),
* worker RSS baseline/peak sampled from /proc/1/status,
* decode / scale / JPEG / WebP timing from the worker's ready log,
* variant bytes versus the original (compression ratio),
* upload latency and max in-flight jobs.
A final backpressure run caps MAX_PENDING_JOBS at 3, stops the worker, shows
503 + Retry-After, then starts the worker and proves recovery.
Outputs evidence/track1.json plus a self-contained HTML dashboard with inline
SVG (no external JS/CSS/fonts). The script only drives the disposable local
Compose stand. It restores the default worker/API environment in a finally
block, so a failed run does not leave the stand throttled.
Usage:
python3 scripts/track1_experiment.py --base-url http://localhost:8080
python3 scripts/track1_experiment.py --base-url http://172.18.0.1:8080 \
--small-jobs 30 --huge-jobs 12
"""
from __future__ import annotations
import argparse
import hashlib
import html
import json
import math
import os
import platform
import re
import struct
import subprocess
import sys
import threading
import time
import zlib
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
sys.path.insert(0, str(ROOT / "scripts"))
from client import Client, Failure, check # noqa: E402
from lab5_resilience import ( # noqa: E402
compose,
compose_capture,
inspect,
recreate_api,
recreate_worker,
restore_api,
restore_worker,
service_ids,
wait_healthy,
wait_stopped,
)
PROFILES = [
{"name": "serial", "concurrency": 1, "prefetch": 1,
"note": "one job at a time; minimum memory, maximum queueing"},
{"name": "balanced", "concurrency": 4, "prefetch": 4,
"note": "moderate in-process parallelism and broker window"},
{"name": "eager", "concurrency": 8, "prefetch": 32,
"note": "wide broker window; highest memory, lowest latency"},
]
SCENARIOS = [
{"name": "small_many", "width": 192, "height": 128,
"description": "many small images; almost no scaling"},
{"name": "exactly_right", "width": 1280, "height": 1280,
"description": "largest variant encodes at native 1280, so no scaling on the WebP path"},
{"name": "huge_few", "width": 2400, "height": 1800,
"description": "few 4.32 MP images; optimized variant must scale down to 1280"},
]
# ---------------------------------------------------------------------------
# Fixture generation and small helpers
def png_chunk(kind, payload):
crc = zlib.crc32(kind + payload) & 0xFFFFFFFF
return struct.pack(">I", len(payload)) + kind + payload + struct.pack(">I", crc)
# AGENT: fixtures are generated in-process so the experiment has no binary
# fixtures in git and the exact bytes/pixels are recorded in the JSON.
def make_png(width, height):
"""Deterministic RGB PNG. The pattern compresses well, but the decoder
still allocates width*height*3 bytes, which is exactly what the experiment
wants to stress without making upload bandwidth the bottleneck."""
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
base = bytes(range(256))
row_len = width * 3
repeated = base * ((row_len // 256) + 2)
compressor = zlib.compressobj(6)
idat = bytearray()
for y in range(height):
offset = (y * 7) % 256
idat += compressor.compress(b"\x00" + repeated[offset : offset + row_len])
idat += compressor.flush()
return (
b"\x89PNG\r\n\x1a\n"
+ png_chunk(b"IHDR", ihdr)
+ png_chunk(b"IDAT", bytes(idat))
+ png_chunk(b"IEND", b"")
)
def percentile(values, p):
if not values:
return None
ordered = sorted(values)
if len(ordered) == 1:
return float(ordered[0])
position = (len(ordered) - 1) * (p / 100.0)
low = math.floor(position)
high = math.ceil(position)
if low == high:
return float(ordered[low])
weight = position - low
return float(ordered[low] * (1 - weight) + ordered[high] * weight)
def r(value, digits=2):
return None if value is None else round(float(value), digits)
def worker_container():
ids = service_ids("worker")
if not ids:
raise Failure("worker container is not running")
name = inspect(ids[0]).get("Name", "").lstrip("/")
if not name:
raise Failure("cannot resolve worker container name")
return name
def psql(sql):
# lab5_resilience.psql is not exported under that name in all versions;
# keep the small wrapper local so this script is easy to audit.
import shlex
command = (
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At -F "|" '
"-v ON_ERROR_STOP=1 -c " + shlex.quote(sql)
)
out = compose_capture("exec", "-T", "postgres", "sh", "-c", command)
return [line for line in out.splitlines() if line.strip()]
def job_rows(attachment_ids):
if not attachment_ids:
return []
quoted = ",".join("'" + item + "'" for item in attachment_ids)
sql = (
"SELECT attachment_id, status, attempt, coalesce(error_code,''), "
"extract(epoch from created_at), "
"coalesce(extract(epoch from processing_started_at), 0), "
"coalesce(extract(epoch from finished_at), 0) "
f"FROM attachment_jobs WHERE attachment_id IN ({quoted})"
)
rows = []
for line in psql(sql):
parts = line.split("|")
if len(parts) != 7:
continue
rows.append(
{
"attachment_id": parts[0],
"status": parts[1],
"attempt": int(parts[2]),
"error_code": parts[3] or None,
"created_at": float(parts[4]),
"processing_started_at": float(parts[5]) if parts[5] not in ("", "0") else None,
"finished_at": float(parts[6]) if parts[6] not in ("", "0") else None,
}
)
return rows
def pending_count():
lines = psql(
"SELECT count(*) FROM attachment_jobs WHERE status IN ('queued','processing')"
)
return int(lines[0]) if lines else 0
def wait_drained(timeout=180):
deadline = time.monotonic() + timeout
last = None
while time.monotonic() < deadline:
last = pending_count()
if last == 0:
return
time.sleep(0.5)
raise Failure(f"pipeline did not drain within {timeout}s; pending={last}")
def upload_one(base_url, actor, chat_id, data, filename):
client = Client(base_url, timeout=60)
started = time.monotonic()
try:
response = client.call(
"POST",
f"/api/v1/chats/{chat_id}/attachments?filename={filename}",
expected=(202, 503),
actor=actor,
raw=data,
headers={"Content-Type": "image/png"},
schema="Attachment",
)
except Failure as exc:
return {"status": 0, "error": str(exc), "elapsed": time.monotonic() - started}
result = {
"status": response["status"],
"elapsed": time.monotonic() - started,
"retry_after": response["headers"].get("Retry-After"),
}
if response["status"] == 202:
result["attachment_id"] = response["json"]["id"]
else:
result["error_code"] = response["json"]["error"]["code"]
return result
def upload_many(base_url, actor, chat_id, data, count, prefix):
with ThreadPoolExecutor(max_workers=min(8, count)) as pool:
futures = [
pool.submit(upload_one, base_url, actor, chat_id, data, f"{prefix}-{index}.png")
for index in range(count)
]
return [future.result() for future in futures]
# ---------------------------------------------------------------------------
# Worker RSS sampler
# AGENT: RSS is sampled inside the worker container, not with `docker stats`,
# so the metric is the process VmRSS without host/daemon noise.
class RssSampler:
def __init__(self, container):
self.container = container
self.samples = []
self._stop = threading.Event()
self._process = None
self._thread = None
def start(self):
self._process = subprocess.Popen(
[
"docker",
"exec",
self.container,
"sh",
"-c",
"while :; do awk '/VmRSS/{print $2}' /proc/1/status; sleep 0.1; done",
],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
)
self._thread = threading.Thread(target=self._read, daemon=True)
self._thread.start()
def _read(self):
if not self._process or not self._process.stdout:
return
for line in self._process.stdout:
if self._stop.is_set():
break
try:
self.samples.append(int(line.strip()))
except ValueError:
continue
def snapshot(self):
return list(self.samples)
def stop(self):
self._stop.set()
if self._process:
self._process.terminate()
try:
self._process.wait(timeout=5)
except subprocess.TimeoutExpired:
self._process.kill()
if self._thread:
self._thread.join(timeout=2)
return list(self.samples)
# ---------------------------------------------------------------------------
# Run phases
def wait_for_terminal(attachment_ids, started_at, timeout=240):
deadline = time.monotonic() + timeout
timeline = []
rows = []
while time.monotonic() < deadline:
rows = job_rows(attachment_ids)
counts = {"queued": 0, "processing": 0, "ready": 0, "failed": 0}
for row in rows:
counts[row["status"]] = counts.get(row["status"], 0) + 1
timeline.append(
{
"t": round(time.monotonic() - started_at, 3),
**counts,
}
)
if len(rows) == len(attachment_ids) and all(
row["status"] in ("ready", "failed") for row in rows
):
return rows, timeline
time.sleep(0.25)
raise Failure(f"jobs did not finish within {timeout}s; last={rows}")
LOGFIELD_RE = re.compile(r'(\w+)=("(?:[^"\\]|\\.)*"|\S+)')
def job_timings(attachment_ids):
"""Read the worker's ready log. The worker is recreated per run, so the
lines belong to this cell; the attachment ID is still checked explicitly."""
if not attachment_ids:
return {}
wanted = set(attachment_ids)
output = compose_capture("logs", "--no-color", "--no-log-prefix", "worker")
timings = {}
for line in output.splitlines():
if 'msg="attachment ready"' not in line:
continue
fields = {key: value.strip('"') for key, value in LOGFIELD_RE.findall(line)}
attachment_id = fields.get("attachment_id")
if attachment_id not in wanted:
continue
try:
timings[attachment_id] = {
name: int(fields[name])
for name in (
"decode_us", "scale_us", "jpeg_encode_us",
"webp_encode_us", "pipeline_us",
)
}
except (KeyError, ValueError):
continue
return timings
def variant_stats(attachment_ids):
"""Variant sizes come from the committed metadata, not from the client."""
if not attachment_ids:
return {}
quoted = ",".join("'" + item + "'" for item in attachment_ids)
sql = (
"SELECT a.id, a.size_bytes, "
"coalesce(sum((v->>'size_bytes')::bigint), 0), "
"coalesce(sum((v->>'size_bytes')::bigint) FILTER (WHERE v->>'name' = 'thumbnail'), 0), "
"coalesce(sum((v->>'size_bytes')::bigint) FILTER (WHERE v->>'name' = 'crop'), 0), "
"coalesce(sum((v->>'size_bytes')::bigint) FILTER (WHERE v->>'name' = 'optimized'), 0) "
"FROM attachments a "
"LEFT JOIN LATERAL jsonb_array_elements(a.variants) v ON true "
f"WHERE a.id IN ({quoted}) "
"GROUP BY a.id, a.size_bytes"
)
result = {}
for line in psql(sql):
parts = line.split("|")
if len(parts) != 6:
continue
result[parts[0]] = {
"original_bytes": int(parts[1]),
"variants_bytes": int(parts[2]),
"thumbnail_bytes": int(parts[3]),
"crop_bytes": int(parts[4]),
"optimized_bytes": int(parts[5]),
}
return result
def measure_run(base_url, actor, chat_id, profile, scenario, count, data, digest,
cpu_limit="1.0", mem_limit="2G"):
wait_drained()
recreate_worker(
{
"WORKER_CONCURRENCY": str(profile["concurrency"]),
"WORKER_PREFETCH": str(profile["prefetch"]),
"WORKER_CPU_LIMIT": str(cpu_limit),
"WORKER_MEM_LIMIT": str(mem_limit),
"WORKER_FAULT_MODE": "none",
"WORKER_CRASH_AFTER_EFFECT": "false",
"WORKER_CRASH_AFTER_CLAIM": "false",
}
)
time.sleep(1.0)
container = worker_container()
sampler = RssSampler(container)
sampler.start()
time.sleep(1.2)
baseline_samples = sampler.snapshot()
submit_started = time.monotonic()
uploads = upload_many(base_url, actor, chat_id, data, count, f"track1-{profile['name']}-{scenario['name']}")
submit_seconds = time.monotonic() - submit_started
accepted = [item["attachment_id"] for item in uploads if item.get("attachment_id")]
rejected = [item for item in uploads if item["status"] != 202]
if rejected:
raise Failure(f"uploads were rejected during {profile['name']}/{scenario['name']}: {rejected[:3]}")
if len(accepted) != count:
raise Failure(f"only {len(accepted)}/{count} uploads were accepted")
rows, timeline = wait_for_terminal(accepted, submit_started)
wall_seconds = time.monotonic() - submit_started
time.sleep(0.5)
samples = sampler.stop()
failed = [row for row in rows if row["status"] != "ready"]
if failed:
raise Failure(f"jobs failed in {profile['name']}/{scenario['name']}: {failed[:2]}")
row_ids = [row["attachment_id"] for row in rows]
timings = job_timings(row_ids)
if len(timings) != len(rows):
raise Failure(
f"timing logs found for {len(timings)}/{len(rows)} jobs in "
f"{profile['name']}/{scenario['name']}"
)
variants = variant_stats(row_ids)
if len(variants) != len(rows):
raise Failure(
f"variant metadata found for {len(variants)}/{len(rows)} jobs in "
f"{profile['name']}/{scenario['name']}"
)
decode_ms = [timings[item]["decode_us"] / 1000 for item in row_ids]
scale_ms = [timings[item]["scale_us"] / 1000 for item in row_ids]
jpeg_ms = [timings[item]["jpeg_encode_us"] / 1000 for item in row_ids]
webp_ms = [timings[item]["webp_encode_us"] / 1000 for item in row_ids]
encode_ms = [jpeg_ms[index] + webp_ms[index] for index in range(len(row_ids))]
pipeline_ms = [timings[item]["pipeline_us"] / 1000 for item in row_ids]
cpu_us = sum(
timings[item]["decode_us"] + timings[item]["scale_us"]
+ timings[item]["jpeg_encode_us"] + timings[item]["webp_encode_us"]
for item in row_ids
)
encode_us = sum(
timings[item]["jpeg_encode_us"] + timings[item]["webp_encode_us"]
for item in row_ids
)
encode_share_pct = 100 * encode_us / cpu_us if cpu_us else None
variant_bytes = {
name: sum(item[f"{name}_bytes"] for item in variants.values())
for name in ("thumbnail", "crop", "optimized")
}
original_bytes = sum(item["original_bytes"] for item in variants.values())
variants_bytes = sum(item["variants_bytes"] for item in variants.values())
compression_ratio = variants_bytes / original_bytes if original_bytes else None
queue_lag = [
(row["processing_started_at"] - row["created_at"]) * 1000
for row in rows
if row["processing_started_at"] is not None
]
time_to_ready = [
(row["finished_at"] - row["created_at"]) * 1000
for row in rows
if row["finished_at"] is not None
]
processing = [
(row["finished_at"] - row["processing_started_at"]) * 1000
for row in rows
if row["processing_started_at"] is not None and row["finished_at"] is not None
]
upload_latency = [item["elapsed"] * 1000 for item in uploads]
first_created = min(row["created_at"] for row in rows)
last_finished = max(row["finished_at"] for row in rows)
span = max(last_finished - first_created, 0.001)
throughput = len(rows) / span
events = []
for row in rows:
events.append((row["created_at"], 1))
events.append((row["finished_at"], -1))
backlog = 0
max_backlog = 0
for _, delta in sorted(events, key=lambda item: (item[0], -item[1])):
backlog += delta
max_backlog = max(max_backlog, backlog)
# Actual parallel handlers: intersections of processing_started_at ->
# finished_at. These are bounded by WORKER_CONCURRENCY.
processing_events = []
for row in rows:
if row["processing_started_at"] is None or row["finished_at"] is None:
continue
processing_events.append((row["processing_started_at"], 1))
processing_events.append((row["finished_at"], -1))
parallel = 0
max_parallel = 0
for _, delta in sorted(processing_events, key=lambda item: (item[0], -item[1])):
parallel += delta
max_parallel = max(max_parallel, parallel)
baseline_kb = max(baseline_samples) if baseline_samples else None
sample_array = [value for value in samples if value > 0]
max_kb = max(sample_array) if sample_array else None
end_kb = sample_array[-1] if sample_array else None
return {
"profile": profile["name"],
"scenario": scenario["name"],
"concurrency": profile["concurrency"],
"prefetch": profile["prefetch"],
"jobs": len(rows),
"image_width": scenario["width"],
"image_height": scenario["height"],
"image_bytes": len(data),
"image_sha256": digest,
"throughput_jobs_per_s": r(throughput, 3),
"pipeline_span_seconds": r(span, 3),
"submit_seconds": r(submit_seconds, 3),
"wall_seconds": r(wall_seconds, 3),
"upload_p50_ms": r(percentile(upload_latency, 50), 1),
"upload_p95_ms": r(percentile(upload_latency, 95), 1),
"queue_lag_p50_ms": r(percentile(queue_lag, 50), 1),
"queue_lag_p95_ms": r(percentile(queue_lag, 95), 1),
"queue_lag_max_ms": r(max(queue_lag) if queue_lag else None, 1),
"time_to_ready_p50_ms": r(percentile(time_to_ready, 50), 1),
"time_to_ready_p95_ms": r(percentile(time_to_ready, 95), 1),
"time_to_ready_max_ms": r(max(time_to_ready) if time_to_ready else None, 1),
"processing_p50_ms": r(percentile(processing, 50), 1),
"processing_p95_ms": r(percentile(processing, 95), 1),
"decode_ms_p50": r(percentile(decode_ms, 50), 2),
"decode_ms_p95": r(percentile(decode_ms, 95), 2),
"scale_ms_p50": r(percentile(scale_ms, 50), 2),
"scale_ms_p95": r(percentile(scale_ms, 95), 2),
"jpeg_encode_ms_p50": r(percentile(jpeg_ms, 50), 2),
"jpeg_encode_ms_p95": r(percentile(jpeg_ms, 95), 2),
"webp_encode_ms_p50": r(percentile(webp_ms, 50), 2),
"webp_encode_ms_p95": r(percentile(webp_ms, 95), 2),
"encode_ms_p50": r(percentile(encode_ms, 50), 2),
"encode_ms_p95": r(percentile(encode_ms, 95), 2),
"pipeline_ms_p50": r(percentile(pipeline_ms, 50), 2),
"pipeline_ms_p95": r(percentile(pipeline_ms, 95), 2),
"encode_share_pct": r(encode_share_pct, 1),
"timing_jobs": len(timings),
"original_bytes": original_bytes,
"variants_bytes": variants_bytes,
"variant_bytes": variant_bytes,
"compression_ratio": r(compression_ratio, 3),
"max_backlog_jobs": max_backlog,
"max_parallel_jobs": max_parallel,
"rss_baseline_mib": r(baseline_kb / 1024 if baseline_kb else None, 1),
"rss_peak_mib": r(max_kb / 1024 if max_kb else None, 1),
"rss_end_mib": r(end_kb / 1024 if end_kb else None, 1),
"rss_samples": len(sample_array),
"rss_delta_mib": r((max_kb - baseline_kb) / 1024 if max_kb and baseline_kb else None, 1),
"timeline": timeline,
}
# AGENT: backpressure is tested separately from the matrix: cap=3, worker down,
# sequential uploads until 503. The accepted jobs are the evidence that bounded
# acceptance did not lose already confirmed work.
def measure_backpressure(base_url, actor, chat_id, data, digest):
wait_drained()
recreate_api({"MAX_PENDING_JOBS": "3"})
compose("stop", "worker")
wait_stopped("worker")
time.sleep(0.5)
attempts = []
accepted_ids = []
for index in range(12):
item = upload_one(base_url, actor, chat_id, data, f"track1-backpressure-{index}.png")
item["index"] = index
attempts.append(item)
if item["status"] == 202:
accepted_ids.append(item["attachment_id"])
if item["status"] == 503:
break
if not any(item["status"] == 503 for item in attempts):
raise Failure("MAX_PENDING_JOBS=3 did not produce a 503")
blocked = next(item for item in attempts if item["status"] == 503)
if blocked.get("retry_after") != "5":
raise Failure(f"503 did not carry Retry-After: 5 (got {blocked.get('retry_after')!r})")
recovery_started = time.monotonic()
compose("start", "worker")
wait_healthy("worker")
rows, _ = wait_for_terminal(accepted_ids, recovery_started, timeout=180)
recovery_seconds = time.monotonic() - recovery_started
if any(row["status"] != "ready" for row in rows):
raise Failure("accepted jobs did not recover after worker start")
wait_drained()
probe = upload_one(base_url, actor, chat_id, data, "track1-backpressure-probe.png")
if probe["status"] != 202:
raise Failure(f"probe upload after recovery returned {probe['status']}")
probe_ready_rows, _ = wait_for_terminal([probe["attachment_id"]], time.monotonic(), timeout=120)
if probe_ready_rows[0]["status"] != "ready":
raise Failure("probe upload did not become ready")
return {
"cap": 3,
"attempts": [
{
"index": item["index"],
"status": item["status"],
"elapsed_ms": r(item["elapsed"] * 1000, 1),
"retry_after": item.get("retry_after"),
"error_code": item.get("error_code"),
}
for item in attempts
],
"accepted_before_503": len(accepted_ids),
"blocked_status": blocked["status"],
"blocked_retry_after": blocked.get("retry_after"),
"recovery_seconds": r(recovery_seconds, 3),
"recovered_jobs": len(rows),
"probe_status": probe["status"],
"probe_ready": True,
"image_bytes": len(data),
"image_sha256": digest,
}
# ---------------------------------------------------------------------------
# HTML dashboard (inline CSS + SVG only)
# AGENT: the dashboard is a single HTML file with inline CSS and SVG. No CDN,
# JS framework or external font is needed, so evidence stays readable offline.
COLORS = {"serial": "#64748b", "balanced": "#0284c7", "eager": "#f59e0b"}
def svg_chart(title, subtitle, categories, series, values, unit, digits=1, lower_better=False):
width, height = 760, 320
left, right, top, bottom = 64, 20, 52, 62
plot_w = width - left - right
plot_h = height - top - bottom
all_values = [v for group in values.values() for v in group if v is not None]
maximum = max(all_values) if all_values else 1.0
maximum = maximum * 1.18 if maximum > 0 else 1.0
parts = [
f'")
return "\n".join(parts)
def build_html(evidence):
runs = evidence["runs"]
scenarios = [item["name"] for item in evidence["scenarios"]]
profiles = [item["name"] for item in evidence["profiles"]]
lookup = {(run["profile"], run["scenario"]): run for run in runs}
def series_for(metric):
return {
profile: [
(lookup.get((profile, scenario)) or {}).get(metric)
for scenario in scenarios
]
for profile in profiles
}
charts = [
svg_chart(
"Throughput: готовые задания в секунду",
"Чем выше, тем быстрее pipeline обрабатывает очередь.",
scenarios, profiles, series_for("throughput_jobs_per_s"), " jobs/s", digits=2,
),
svg_chart(
"p95 time-to-ready: от 202 до ready",
"95% заданий стали ready быстрее этого времени (меньше — лучше).",
scenarios, profiles, series_for("time_to_ready_p95_ms"), " ms", digits=0, lower_better=True,
),
svg_chart(
"p95 queue lag: ожидание до старта обработки",
"Время между durable 202 и первым claim worker-а (меньше — лучше).",
scenarios, profiles, series_for("queue_lag_p95_ms"), " ms", digits=0, lower_better=True,
),
svg_chart(
"Пиковый RSS worker",
"VmRSS процесса worker во время задания (меньше — экономнее).",
scenarios, profiles, series_for("rss_peak_mib"), " MiB", digits=0, lower_better=True,
),
svg_chart(
"Compression overhead: доля encode в pipeline CPU",
"encode / (decode + scale + encode), % (меньше — дешевле сжатие).",
scenarios, profiles, series_for("encode_share_pct"), " %", digits=1, lower_better=True,
),
]
def fmt(value, digits=1, suffix=""):
return "—" if value is None else f"{value:.{digits}f}{suffix}"
table_rows = []
for run in runs:
table_rows.append(
"
"
f"
{html.escape(run['profile'])}
"
f"
{html.escape(run['scenario'])}
"
f"
{run['jobs']}
"
f"
{fmt(run['throughput_jobs_per_s'], 2)}
"
f"
{fmt(run['queue_lag_p95_ms'], 0)}
"
f"
{fmt(run['time_to_ready_p95_ms'], 0)}
"
f"
{fmt(run['processing_p95_ms'], 0)}
"
f"
{fmt(run['encode_ms_p95'], 0)}
"
f"
{fmt(run['encode_share_pct'], 1)}
"
f"
{fmt(run['rss_peak_mib'], 0)}
"
f"
{run['max_parallel_jobs']}
"
f"
{run['max_backlog_jobs']}
"
"
"
)
# Beginner-friendly observations computed from the actual numbers.
observations = []
for scenario in scenarios:
serial = lookup.get(("serial", scenario))
eager = lookup.get(("eager", scenario))
if serial and eager and serial["throughput_jobs_per_s"]:
speedup = eager["throughput_jobs_per_s"] / serial["throughput_jobs_per_s"]
if speedup >= 1.05:
comparison = f"eager быстрее serial в {speedup:.1f}× по throughput"
elif speedup <= 0.95:
comparison = f"eager медленнее serial ({speedup:.2f}×) по throughput"
else:
comparison = "throughput eager и serial практически одинаков"
observations.append(f"{scenario}: {comparison}.")
if serial["rss_peak_mib"] and eager["rss_peak_mib"]:
ratio = eager["rss_peak_mib"] / serial["rss_peak_mib"]
observations.append(
f"{scenario}: пиковый RSS eager относится к serial как {ratio:.1f}×."
)
balanced = lookup.get(("balanced", "small_many"))
exact = lookup.get(("balanced", "exactly_right"))
huge = lookup.get(("balanced", "huge_few"))
if balanced and huge and balanced["time_to_ready_p95_ms"] and huge["time_to_ready_p95_ms"]:
observations.append(
"При balanced p95 time-to-ready для huge_few в "
f"{huge['time_to_ready_p95_ms'] / balanced['time_to_ready_p95_ms']:.1f}× "
"больше, чем для small_many: размер изображения меняет стоимость задания, "
"а не только число заданий."
)
if exact and huge and exact["encode_share_pct"] is not None and huge["encode_share_pct"] is not None:
observations.append(
"Compression vs scaling (balanced): exactly_right — encode share "
f"{exact['encode_share_pct']:.1f}%, scale p95 {exact['scale_ms_p95']:.0f} ms; "
f"huge_few — encode share {huge['encode_share_pct']:.1f}%, "
f"scale p95 {huge['scale_ms_p95']:.0f} ms. На exactly_right optimized "
"WebP кодируется в native 1280, поэтому это нижняя граница стоимости "
"сжатия; на huge_few к ней добавляется downscale."
)
huge_serial = lookup.get(("serial", "huge_few"))
huge_eager = lookup.get(("eager", "huge_few"))
if huge_serial and huge_eager and huge_serial["throughput_jobs_per_s"]:
speedup = huge_eager["throughput_jobs_per_s"] / huge_serial["throughput_jobs_per_s"]
if speedup < 1.2:
observations.append(
"На huge_few eager почти не поднял throughput "
f"({speedup:.2f}×): worker ограничен "
f"{html.escape(str(evidence['environment']['worker_cpu_limit']))} CPU, "
"а декодирование и масштабирование упираются в CPU. Concurrency "
"снижает queue lag, но при этом растёт RSS; это и есть компромисс "
"latency/ресурсы, а не бесплатное ускорение."
)
else:
observations.append(
f"На huge_few eager поднял throughput в {speedup:.2f}× относительно "
"serial, потому что несколько тяжёлых заданий перекрываются."
)
observations.append(
"serial ограничивает одновременно обрабатываемые задания одним, поэтому "
"queue lag растёт, но RSS остаётся минимальным. eager держит больше "
"заданий в работе и в prefetch-окне брокера."
)
observations.append(
"Backpressure: cap=3 выставлен намеренно, поэтому число принятых заданий — "
"следствие настройки, а не открытие. Значимо то, что API вернул 503 "
f"с Retry-After={evidence['backpressure']['blocked_retry_after']}, а после "
f"старта worker очередь восстановилась за {evidence['backpressure']['recovery_seconds']:.1f} с."
)
backpressure_rows = "".join(
"
"
f"
{item['index'] + 1}
"
f"
{item['status']}
"
f"
{fmt(item['elapsed_ms'], 1)}
"
f"
{html.escape(str(item['retry_after'] or '—'))}
"
f"
{html.escape(str(item['error_code'] or '—'))}
"
"
"
for item in evidence["backpressure"]["attempts"]
)
observations_html = "".join(f"
{html.escape(text)}
" for text in observations)
chart_html = "\n".join(f'{chart}' for chart in charts)
return f"""
Lab5 track 1 — очередь под перегрузкой
Гипотеза. Стоимость задания складывается из decode, масштабирования
и сжатия. exactly_right (1280×1280) не масштабирует optimized-вариант,
поэтому показывает нижнюю границу стоимости сжатия; huge_few добавляет
downscale, а small_many почти не масштабируется, но и пикселей в нём
мало. Большие изображения при том же числе worker-ов увеличивают очередь,
p95 time-to-ready и RSS. Одновременно проверяем компромисс: eager
уменьшает ожидание, но повышает память; serial экономит память, но
ограничивает throughput.
Профили и сценарии
Профиль
Concurrency
Prefetch
Смысл
{''.join(f"
{html.escape(p['name'])}
{p['concurrency']}
{p['prefetch']}
{html.escape(p['note'])}
" for p in evidence['profiles'])}
Сценарии:
{'; '.join(f"{html.escape(s['name'])} — {s['width']}×{s['height']}, {s['count']} jobs, {s['description']}" for s in evidence['scenarios'])}.
Ключевые числа
{len(runs)}
прогонов матрицы
{max(run['rss_peak_mib'] or 0 for run in runs):.0f} MiB
максимальный RSS worker
{evidence['backpressure']['recovery_seconds']:.1f} s
восстановление после backpressure
{chart_html}
Полная матрица
Профиль
Сценарий
Jobs
Jobs/s
Queue lag p95, ms
Ready p95, ms
Processing p95, ms
Encode p95, ms
Encode share, %
RSS peak, MiB
Max parallel
Max backlog
{''.join(table_rows)}
Queue lag = processing_started_at - created_at.
Time-to-ready = finished_at - created_at. Processing = finished_at - processing_started_at.
Encode = JPEG thumbnail + JPEG crop + WebP optimized; encode share =
encode / (decode + scale + encode) из worker-лога.
Суммы байт вариантов и оригинала остаются в JSON, но не выносятся в заголовок:
синтетический PNG сжимается слишком хорошо, чтобы отношение байт было честной
метрикой эффективности кодека.
RSS — VmRSS процесса worker, сэмплирование примерно каждые 100 мс.
Max parallel — максимум одновременно обрабатываемых заданий; max backlog —
максимум принятых, но ещё не готовых заданий.
Backpressure и восстановление
Перед этим прогоном API пересоздан с MAX_PENDING_JOBS=3, worker
остановлен. Cap выбран намеренно: важно не число принятых заданий, а то, что
API возвращает 503 с Retry-After и не теряет уже
подтверждённую работу. Затем worker запускается, принятые задания должны дойти
до ready.
#
HTTP
Latency, ms
Retry-After
Error code
{backpressure_rows}
Recovery: {evidence['backpressure']['recovered_jobs']} принятых заданий
стали ready за {evidence['backpressure']['recovery_seconds']:.2f} с после старта worker.
Финальный probe upload: HTTP {evidence['backpressure']['probe_status']} → ready.
Что это значит
{observations_html}
Ограничения
Один worker, один RabbitMQ и один PostgreSQL: это сравнение настроек, а не
HA-кластер. Числа зависят от CPU/диска конкретной машины.
PNG-фикстуры синтетические и хорошо сжимаются; absolute encode time зависит
от паттерна, но сравнение сценариев при одном паттерне остаётся корректным.
Размер в байтах указан в JSON.
Timing-поля читаются из worker-лога. Это wall-clock внутри процесса, поэтому
при concurrency > 1 в них входит и планировщик, что как раз и нужно для
сравнения профилей.
exactly_right кодирует 1.64 MP WebP, а huge_few после downscale — около
1.23 MP. Сценарий убирает scaling, но не уравнивает объём encode; это указано,
чтобы не сравнивать throughput как «одинаковую работу».
RSS сэмплируется из /proc/1/status; короткий прогон не заменяет
длительный memory profile и p99.
Backpressure cap приблизительный: каждая API-реплика считает pending-задания
по общей БД, но проверка и вставка не являются одной транзакцией.
"""
def write_markdown(evidence, path):
lines = [
"# Lab5 track 1 — queue under load",
"",
f"Generated: `{evidence['generated_at']}`",
"",
"Synthetic images; no tokens or credentials are stored.",
"",
"| Profile | Scenario | Jobs | Jobs/s | Queue lag p95, ms | Ready p95, ms | Encode p95, ms | Encode share, % | RSS peak, MiB |",
"| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
]
for run in evidence["runs"]:
lines.append(
f"| {run['profile']} | {run['scenario']} | {run['jobs']} | "
f"{run['throughput_jobs_per_s']:.2f} | {run['queue_lag_p95_ms']:.0f} | "
f"{run['time_to_ready_p95_ms']:.0f} | {run['encode_ms_p95']:.0f} | "
f"{run['encode_share_pct']:.1f} | {run['rss_peak_mib']:.0f} |"
)
bp = evidence["backpressure"]
lines.extend(
[
"",
f"Backpressure: cap={bp['cap']} (deliberate), "
f"first 503 Retry-After={bp['blocked_retry_after']}, "
f"recovery={bp['recovery_seconds']:.2f}s, probe HTTP {bp['probe_status']}.",
"",
]
)
path.write_text("\n".join(lines))
# ---------------------------------------------------------------------------
# Main
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
parser.add_argument("--small-jobs", type=int, default=30)
parser.add_argument("--exactly-jobs", type=int, default=18)
parser.add_argument("--huge-jobs", type=int, default=12)
parser.add_argument(
"--worker-cpu-limit",
default=os.environ.get("TRACK1_WORKER_CPU_LIMIT", "1.0"),
help="Compose CPU quota applied to every experimental worker (default 1.0)",
)
parser.add_argument(
"--worker-mem-limit",
default=os.environ.get("TRACK1_WORKER_MEM_LIMIT", "2G"),
help="Compose memory limit applied to every experimental worker (default 2G)",
)
parser.add_argument("--out", default=str(ROOT / "evidence" / "track1.json"))
parser.add_argument("--html-out", default=str(ROOT / "evidence" / "track1.html"))
parser.add_argument("--md-out", default=str(ROOT / "evidence" / "track1.md"))
parser.add_argument(
"--skip-backpressure",
action="store_true",
help="only run the profile x scenario matrix",
)
args = parser.parse_args()
base_url = args.base_url.rstrip("/")
out = Path(args.out)
html_out = Path(args.html_out)
md_out = Path(args.md_out)
out.parent.mkdir(parents=True, exist_ok=True)
html_out.parent.mkdir(parents=True, exist_ok=True)
print(f"== bootstrap {base_url}", flush=True)
client = Client(base_url, timeout=60)
alice, _bob, _eve, chat = client.bootstrap()
counts = {
"small_many": args.small_jobs,
"exactly_right": args.exactly_jobs,
"huge_few": args.huge_jobs,
}
scenarios = []
for item in SCENARIOS:
count = counts[item["name"]]
data = make_png(item["width"], item["height"])
scenarios.append({**item, "count": count, "bytes": len(data)})
try:
wait_drained()
runs = []
for profile in PROFILES:
for scenario in scenarios:
data = make_png(scenario["width"], scenario["height"])
if len(data) != scenario["bytes"]:
raise Failure("fixture generation is not deterministic")
digest = hashlib.sha256(data).hexdigest()
print(
f"== run {profile['name']}/{scenario['name']} "
f"({scenario['count']} jobs, {scenario['width']}x{scenario['height']})",
flush=True,
)
result = measure_run(
base_url,
alice,
chat["id"],
profile,
scenario,
scenario["count"],
data,
digest,
cpu_limit=args.worker_cpu_limit,
mem_limit=args.worker_mem_limit,
)
runs.append(result)
print(
f" throughput={result['throughput_jobs_per_s']} jobs/s "
f"queue_p95={result['queue_lag_p95_ms']} ms "
f"ready_p95={result['time_to_ready_p95_ms']} ms "
f"encode_share={result['encode_share_pct']}% "
f"rss_peak={result['rss_peak_mib']} MiB",
flush=True,
)
backpressure = None
if not args.skip_backpressure:
print("== backpressure (cap=3, worker stopped)", flush=True)
small = scenarios[0]
backpressure = measure_backpressure(
base_url,
alice,
chat["id"],
make_png(small["width"], small["height"]),
hashlib.sha256(make_png(small["width"], small["height"])).hexdigest(),
)
print(
f" accepted={backpressure['accepted_before_503']} "
f"503_retry_after={backpressure['blocked_retry_after']} "
f"recovery={backpressure['recovery_seconds']}s",
flush=True,
)
finally:
# Restore the stand even if a measurement raised.
try:
restore_worker()
except Exception as exc: # noqa: BLE001 - best effort in cleanup
print(f"warning: worker restore failed: {exc}", flush=True)
try:
restore_api()
except Exception as exc: # noqa: BLE001 - best effort in cleanup
print(f"warning: API restore failed: {exc}", flush=True)
if backpressure is None:
backpressure = {
"cap": 3,
"attempts": [],
"accepted_before_503": 0,
"blocked_status": None,
"blocked_retry_after": None,
"recovery_seconds": 0.0,
"recovered_jobs": 0,
"probe_status": None,
"probe_ready": False,
}
docker_version = compose_capture("version", "--short").strip()
evidence = {
"generated_at": datetime.now(timezone.utc).isoformat(),
"environment": {
"base_url": base_url,
"host": platform.node(),
"platform": platform.platform(),
"cpu_count": os.cpu_count(),
"python": platform.python_version(),
"docker_version": docker_version,
"compose_project": "mtusi2026lab",
"worker_container": worker_container(),
"worker_cpu_limit": args.worker_cpu_limit,
"worker_mem_limit": args.worker_mem_limit,
"worker_cpu_limit_default": "1.0",
"worker_mem_limit_default": "1G",
"note": (
"Experimental workers use a larger memory limit than .env so the "
"eager huge-image run can hold decoded images without OOM; CPU is "
"still limited so throughput is not a host-CPU artifact."
),
},
"hypothesis": (
"Job cost is decode + scale + encode. exactly_right (1280x1280) keeps the "
"optimized WebP at native size, so it measures the compression floor; "
"huge_few adds downscale, and small_many has few pixels. More concurrency "
"should raise throughput and RSS but reduce queue lag."
),
"profiles": PROFILES,
"scenarios": scenarios,
"runs": runs,
"backpressure": backpressure,
}
out.write_text(json.dumps(evidence, indent=2) + "\n")
html_out.write_text(build_html(evidence))
write_markdown(evidence, md_out)
print(json.dumps({"ok": True, "json": str(out), "html": str(html_out), "md": str(md_out)}))
if __name__ == "__main__":
try:
main()
except (Failure, RuntimeError, subprocess.TimeoutExpired) as exc:
print("FAIL:", exc)
raise SystemExit(1)