Files
mtusi2026lab4/bench/uploads.py
T

579 lines
22 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Профиль uploads: план, разбор сэмплера peak, сводка без docker. Только stdlib.
Оркестрирует прогоны `run.py` (профили `uploads`, `uploads-quick`); здесь
— чистые функции: план прогонов кругами, пиковая память по строкам
сэмплера в режиме `peak` (`bench/sampler.py`), строка `runs.csv`, сводка
по ячейкам и вердикты U1–U4 (гипотезы записаны в `REPORT.md` до замеров).
Самопроверка — `uv run python bench/uploads.py --selftest`.
"""
import random
import statistics
import sys
from collections.abc import Iterable
from dataclasses import dataclass
MIB = 2**20
VARIANTS = ("limited", "unlimited")
VARIANT_NAMES = {"limited": "с лимитом", "unlimited": "без лимита"}
ROLES = ("api", "s3", "lb")
# пороги гипотез — до замеров (REPORT.md, «Память загрузок: гипотезы»)
U1_HALF = 0.5
U1_GROWTH_MIB = 1.0
U2_BODIES_PER_CLIENT = 0.8
U2_NEAR_LIMIT = 0.9
U3_SHARE = 0.9
RUN_COLUMNS = (
"run",
"variant",
"size",
"body_bytes",
"clients",
"repeat",
"order",
"mem_before_mib",
"start_peak_mib",
"peak_mib",
"peak_source",
"anon_max_mib",
"sock_max_mib",
"oom_kill",
"oom_killed",
"exit_code",
"running_after",
"started",
"ok",
"rejected_429",
"errors_5xx",
"other_errors",
"statuses",
"uploads_per_s",
"mib_per_s",
"p50_ms",
"p95_ms",
"p99_ms",
"max_ms",
"s3_peak_mib",
"s3_anon_max_mib",
"lb_peak_mib",
"lb_sock_max_mib",
"gen_cpu_s",
"host_swapouts",
"flags",
)
def plan_runs(
variants: Iterable[str],
sizes: Iterable[str],
clients: Iterable[int],
repeats: int,
seed: int,
) -> list[dict]:
"""Прогоны кругами: в круге r — перестановка ячеек (вариант × размер ×
клиенты) генератором `Random(seed + r)`; круг не начинается с ячейки,
которой кончился прежний."""
cells = [(v, s, c) for v in variants for s in sizes for c in clients]
plan: list[dict] = []
previous = None
for round_no in range(1, repeats + 1):
order = list(range(len(cells)))
random.Random(seed + round_no).shuffle(order)
if len(order) > 1 and order[0] == previous:
order[0], order[1] = order[1], order[0]
for index in order:
variant, size, count = cells[index]
plan.append(
{
"run": f"{variant}-{size}-c{count}-r{round_no}",
"variant": variant,
"size": size,
"clients": count,
"repeat": round_no,
"order": len(plan) + 1,
},
)
previous = order[-1]
return plan
@dataclass
class Peak:
"""Память одного контейнера за окно по строкам сэмплера `peak`."""
start_peak: int | None = None
reset: bool = False
peak: int | None = None
first_current: int | None = None
max_current: int = 0
max_anon: int = 0
max_sock: int = 0
oom_first: int | None = None
oom_last: int | None = None
samples: int = 0
missing: int = 0
@property
def oom_kills(self) -> int:
if self.oom_first is None or self.oom_last is None:
return 0
return self.oom_last - self.oom_first
@property
def window_peak(self) -> int | None:
"""Пик окна: `memory.peak` своего дескриптора после сброса; без
сброса или чтения (контейнер погиб) — наибольшая выборка."""
if self.reset and self.peak is not None:
return max(self.peak, self.max_current)
return self.max_current or None
@property
def source(self) -> str:
return "reset" if self.reset and self.peak is not None else "sampled"
def parse_peak(lines: list[dict]) -> dict[str, Peak]:
"""Строки сэмплера `peak` → память по ролям."""
roles: dict[str, Peak] = {}
def role(name: str) -> Peak:
return roles.setdefault(name, Peak())
for line in lines:
if "reset" in line:
for name, value in line.get("start", {}).items():
role(name).start_peak = value
for name, ok in line["reset"].items():
role(name).reset = bool(ok)
elif "pk" in line:
for name, values in line["pk"].items():
item = role(name)
if values is None:
item.missing += 1
continue
current, anon, sock, oom = values
item.samples += 1
if item.first_current is None:
item.first_current = current
item.max_current = max(item.max_current, current)
item.max_anon = max(item.max_anon, anon)
item.max_sock = max(item.max_sock, sock)
if item.oom_first is None:
item.oom_first = oom
item.oom_last = oom
elif "peak" in line:
for name, value in line["peak"].items():
role(name).peak = value
return roles
def mib(value: int | float | None) -> float | None:
return None if value is None else round(value / MIB, 1)
def run_row(
run: dict,
*,
body_bytes: int,
peaks: dict[str, Peak],
result: dict,
state: dict,
swapouts: int | None,
) -> dict:
"""Строка `runs.csv`. Пометки: `oom` — реплику завершило ядро из-за
памяти (`OOMKilled` или `oom_kill` cgroup) — результат варианта, а не
сбой; `host_swap` — хост выгружал страницы во время прогона;
`peak_sampled` — пик окна по выборкам, а не по сброшенному
`memory.peak`; `no_result` — генератор не вернул итог."""
api = peaks.get("api", Peak())
s3 = peaks.get("s3", Peak())
lb = peaks.get("lb", Peak())
oom = bool(state.get("oom_killed")) or api.oom_kills > 0
flags = []
if oom:
flags.append("oom")
if swapouts:
flags.append("host_swap")
if api.source == "sampled":
flags.append("peak_sampled")
if not result:
flags.append("no_result")
return {
**{key: run[key] for key in ("run", "variant", "size", "clients", "repeat")},
"order": run["order"],
"body_bytes": body_bytes,
"mem_before_mib": mib(api.first_current),
"start_peak_mib": mib(api.start_peak),
"peak_mib": mib(api.window_peak),
"peak_source": api.source,
"anon_max_mib": mib(api.max_anon),
"sock_max_mib": mib(api.max_sock),
"oom_kill": api.oom_kills,
"oom_killed": bool(state.get("oom_killed")),
"exit_code": state.get("exit_code"),
"running_after": state.get("running"),
"started": result.get("started"),
"ok": result.get("ok"),
"rejected_429": result.get("rejected_429"),
"errors_5xx": result.get("errors_5xx"),
"other_errors": result.get("other_errors"),
"statuses": " ".join(
f"{k}:{v}" for k, v in sorted(result.get("statuses", {}).items())
),
**{
key: result.get(key)
for key in (
"uploads_per_s",
"mib_per_s",
"p50_ms",
"p95_ms",
"p99_ms",
"max_ms",
"gen_cpu_s",
)
},
"s3_peak_mib": mib(s3.window_peak),
"s3_anon_max_mib": mib(s3.max_anon),
"lb_peak_mib": mib(lb.window_peak),
"lb_sock_max_mib": mib(lb.max_sock),
"host_swapouts": swapouts,
"flags": " ".join(flags),
}
# --- сводка ------------------------------------------------------------------------
def num(value) -> float | None:
if value is None or value == "":
return None
try:
return float(value)
except (TypeError, ValueError):
return None
def median(values: Iterable) -> float | None:
present = [v for v in (num(x) for x in values) if v is not None]
return statistics.median(present) if present else None
def spread(values: Iterable, digits: int = 1) -> str:
present = [v for v in (num(x) for x in values) if v is not None]
if not present:
return "—"
mid = statistics.median(present)
if len(present) == 1:
return f"{mid:.{digits}f}"
return f"{mid:.{digits}f} ({min(present):.{digits}f}–{max(present):.{digits}f})"
def fmt(value: float | None, digits: int = 1) -> str:
return "—" if value is None else f"{value:.{digits}f}"
def is_oom(row: dict) -> bool:
return "oom" in str(row.get("flags") or "").split()
def cells(rows: list[dict]) -> dict[tuple[str, str, int], list[dict]]:
grouped: dict[tuple[str, str, int], list[dict]] = {}
for row in rows:
key = (row["variant"], row["size"], int(row["clients"]))
grouped.setdefault(key, []).append(row)
return grouped
def per_client_bodies(row: dict) -> float | None:
"""Рост пика окна над памятью до окна на клиента, в телах."""
peak, before = num(row.get("peak_mib")), num(row.get("mem_before_mib"))
body, clients = num(row.get("body_bytes")), int(row["clients"])
if peak is None or before is None or not body or not clients:
return None
return (peak - before) * MIB / body / clients
@dataclass(frozen=True)
class Verdict:
hypothesis: str
result: str
numbers: str
CONFIRMED = "подтверждена"
REFUTED = "опровергнута"
UNTESTED = "нет данных"
def verdicts(rows: list[dict], *, limit_mib: float, big: str) -> list[Verdict]:
grouped = cells(rows)
def group(variant: str, size: str, clients: int) -> list[dict]:
return grouped.get((variant, size, clients), [])
out = []
# U1: с лимитом пик ≤ половины лимита при C = 10, 20, 40 (большое тело),
# рост сверх C = 10 ≤ 1 MiB на клиента; OOM и 5xx нет ни в одном прогоне
levels = [c for c in (10, 20, 40) if group("limited", big, c)]
limited = [r for r in rows if r["variant"] == "limited"]
if levels:
peaks = {
c: median(r["peak_mib"] for r in group("limited", big, c)) for c in levels
}
base = peaks.get(10)
growth = {
c: (peaks[c] - base) / (c - 10)
for c in levels
if c > 10 and base is not None and peaks[c] is not None
}
ooms = sum(is_oom(r) for r in limited)
fives = sum(int(num(r.get("errors_5xx")) or 0) for r in limited)
ok = (
all(v is not None and v <= limit_mib * U1_HALF for v in peaks.values())
and all(v <= U1_GROWTH_MIB for v in growth.values())
and not ooms
and not fives
)
out.append(
Verdict(
"U1",
CONFIRMED if ok else REFUTED,
"пик, MiB: "
+ ", ".join(f"C={c} {fmt(v)}" for c, v in peaks.items())
+ f" (половина лимита {limit_mib * U1_HALF:g}); сверх C=10, MiB "
"на клиента: "
+ (", ".join(f"C={c} {fmt(v, 2)}" for c, v in growth.items()) or "—")
+ f"; OOM {ooms}, 5xx {fives} во всех прогонах «с лимитом»",
),
)
else:
out.append(Verdict("U1", UNTESTED, "нет ячеек «с лимитом» C ≥ 10"))
# U2: без лимита рост ≥ 0,8 тела на клиента при C = 10; при C = 40 —
# OOM или пик ≥ 90 % лимита
ten = group("unlimited", big, 10)
forty = group("unlimited", big, 40)
if ten or forty:
slope = median(per_client_bodies(r) for r in ten if not is_oom(r))
near = all(
is_oom(r) or (num(r["peak_mib"]) or 0) >= limit_mib * U2_NEAR_LIMIT
for r in forty
)
ok = (
slope is not None and slope >= U2_BODIES_PER_CLIENT and bool(forty) and near
)
out.append(
Verdict(
"U2",
CONFIRMED if ok else REFUTED,
f"C=10: {fmt(slope, 2)} тела на клиента (порог {U2_BODIES_PER_CLIENT}); "
f"C=40: OOM {sum(is_oom(r) for r in forty)}/{len(forty)}, пик "
f"{spread(r['peak_mib'] for r in forty)} MiB (90 % лимита "
f"{limit_mib * U2_NEAR_LIMIT:g})",
),
)
else:
out.append(Verdict("U2", UNTESTED, "нет ячеек «без лимита»"))
# U3: загрузок/с с лимитом ≥ 90 % без лимита при C ≥ 10, пока тот жив
ratios = []
for variant_size in sorted({r["size"] for r in rows}):
for clients in (10, 20, 40):
lim = group("limited", variant_size, clients)
unl = group("unlimited", variant_size, clients)
if not lim or not unl or any(is_oom(r) for r in unl):
continue
a = median(r["uploads_per_s"] for r in lim)
b = median(r["uploads_per_s"] for r in unl)
if a is not None and b:
ratios.append((variant_size, clients, a, b, a / b))
if ratios:
out.append(
Verdict(
"U3",
CONFIRMED if all(r[4] >= U3_SHARE for r in ratios) else REFUTED,
"; ".join(
f"{s} C={c}: {a:.2f} / {b:.2f} = {ratio:.0%}"
for s, c, a, b, ratio in ratios
),
),
)
else:
out.append(Verdict("U3", UNTESTED, "нет пар ячеек без OOM"))
# U4: с лимитом 429 нет при C ≤ 20 (4 места + 16 в очереди), есть при C = 40
low = [r for r in limited if int(r["clients"]) <= 20]
high = [r for r in limited if int(r["clients"]) >= 40]
if low and high:
low_429 = sum(int(num(r.get("rejected_429")) or 0) for r in low)
high_429 = [int(num(r.get("rejected_429")) or 0) for r in high]
ok = low_429 == 0 and all(v > 0 for v in high_429)
out.append(
Verdict(
"U4",
CONFIRMED if ok else REFUTED,
f"429 при C ≤ 20: {low_429} в {len(low)} прогонах; при C = 40: "
+ ", ".join(map(str, high_429)),
),
)
else:
out.append(Verdict("U4", UNTESTED, "нет ячеек «с лимитом» C ≤ 20 и C = 40"))
return out
def render_summary(meta: dict, rows: list[dict]) -> str:
profile = meta["profile"]
bodies = meta.get("bodies", {})
big = max(bodies, key=lambda k: bodies[k]["bytes"]) if bodies else ""
limit_mib = float(profile["api_memory"].rstrip("m"))
lines = [
f"# Память загрузок: метка `{meta['label']}`",
"",
f"- Статус: **{'прерван: ' + meta['aborted'] if meta.get('aborted') else 'готово'}**",
f"- Код (`src_sha256` образа): `{meta.get('code', {}).get('src_sha256', '—')}`",
f"- Профиль `{profile['name']}`: варианты "
+ ", ".join(f"«{VARIANT_NAMES[v]}»" for v in profile["variants"])
+ f"; клиенты {', '.join(map(str, profile['clients']))}; повторов "
f"{profile['repeats']}; окно {profile['window_s']:g} с после "
f"{profile['warmup_s']:g} с разгона",
"- Тела: "
+ ", ".join(
f"{name} — {item['bytes']} байт ({item['width']} × {item['height']}), "
f"SHA-256 `{item['sha256'][:16]}…`"
for name, item in bodies.items()
),
f"- Реплика {profile['api_cpus']} CPU / {profile['api_memory']} без swap, "
f"буфер приёма `{meta.get('stand', {}).get('api_tcp_rmem', '—')}`; S3 "
f"{profile['s3_cpus']} CPU / {profile['s3_memory']}; «без лимита» — "
+ ", ".join(f"`{k}={v}`" for k, v in profile["unlimited_env"]),
"- Пик — `memory.peak` cgroup, сброшенный своим дескриптором после "
"разогрева (`peak_source` = reset); без сброса — наибольшая выборка "
f"раз в {profile['sample_interval_s']:g} с (`peak_sampled`). «До окна» — "
"первая выборка. Ответы по кодам и задержки — загрузки, начатые в окне. "
"Значения — медиана (min–max) повторов. Пик S3 включает страничный кэш "
"записанных объектов и упирается в лимит; память самого шлюза — `anon`",
"",
"## Ячейки",
"",
"| Вариант | Тело | C | Пик старта, MiB | До окна, MiB | Пик, MiB | anon, MiB "
"| sock, MiB | OOM | 201 | 429 | 5xx | прочие | Загрузок/с | p50, мс | "
"p95, мс | S3 пик / anon, MiB | LB пик, MiB | Пометки |",
"|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|"
"---:|---:|---:|---|",
]
grouped = cells(rows)
for (variant, size, clients), items in sorted(
grouped.items(), key=lambda kv: (kv[0][0], kv[0][1], kv[0][2])
):
flags = sorted({f for r in items for f in str(r.get("flags") or "").split()})
lines.append(
f"| {VARIANT_NAMES.get(variant, variant)} | {size} | {clients} | "
f"{spread(r['start_peak_mib'] for r in items)} | "
f"{spread(r['mem_before_mib'] for r in items)} | "
f"{spread(r['peak_mib'] for r in items)} | "
f"{spread(r['anon_max_mib'] for r in items)} | "
f"{spread(r['sock_max_mib'] for r in items)} | "
f"{sum(is_oom(r) for r in items)}/{len(items)} | "
f"{spread((r['ok'] for r in items), 0)} | "
f"{spread((r['rejected_429'] for r in items), 0)} | "
f"{spread((r['errors_5xx'] for r in items), 0)} | "
f"{spread((r['other_errors'] for r in items), 0)} | "
f"{spread((r['uploads_per_s'] for r in items), 2)} | "
f"{spread((r['p50_ms'] for r in items), 0)} | "
f"{spread((r['p95_ms'] for r in items), 0)} | "
f"{spread(r['s3_peak_mib'] for r in items)} / "
f"{spread(r['s3_anon_max_mib'] for r in items)} | "
f"{spread(r['lb_peak_mib'] for r in items)} | {' '.join(flags) or '—'} |"
)
lines += [
"",
"## Гипотезы",
"",
"Тексты и пороги — `REPORT.md`, «Память загрузок: гипотезы» (записаны до "
f"замеров); большое тело — `{big}`, лимит памяти реплики {limit_mib:g} MiB.",
"",
"| Гипотеза | Итог | Числа |",
"|---|---|---|",
]
lines += [
f"| {v.hypothesis} | {v.result} | {v.numbers} |"
for v in verdicts(rows, limit_mib=limit_mib, big=big)
]
warnings = meta.get("host_warnings") or []
if warnings:
lines += ["", "## Предупреждения о хосте", "", *(f"- {w}" for w in warnings)]
return "\n".join(lines) + "\n"
# --- самопроверка ------------------------------------------------------------------
def selftest() -> None:
plan = plan_runs(VARIANTS, ("1mib", "10mib"), (1, 10), 2, 5)
assert [p["order"] for p in plan] == list(range(1, 17))
assert plan[7]["run"] != plan[8]["run"]
assert len({p["run"] for p in plan}) == 16
lines = [
{"start": {"api": 300 * MIB, "s3": None}, "reset": {"api": True, "s3": False}},
{"ts": 1.0, "pk": {"api": [100 * MIB, 50 * MIB, 1 * MIB, 0], "s3": None}},
{"ts": 1.2, "pk": {"api": [180 * MIB, 120 * MIB, 3 * MIB, 0], "s3": None}},
{"ts": 1.4, "pk": {"api": [150 * MIB, 90 * MIB, 2 * MIB, 1], "s3": None}},
{"ts": 1.6, "peak": {"api": 200 * MIB}, "end": True},
]
peaks = parse_peak(lines)
api = peaks["api"]
assert (api.start_peak, api.window_peak) == (300 * MIB, 200 * MIB)
assert (api.first_current, api.max_sock) == (100 * MIB, 3 * MIB)
assert (api.oom_kills, api.source) == (1, "reset")
assert (peaks["s3"].missing, peaks["s3"].source) == (3, "sampled")
run = {"run": "limited-10mib-c10-r1", "variant": "limited", "size": "10mib"}
run |= {"clients": 10, "repeat": 1, "order": 1}
result = {"ok": 40, "rejected_429": 0, "errors_5xx": 0, "statuses": {"201": 40}}
row = run_row(
run,
body_bytes=10 * MIB,
peaks=peaks,
result=result,
state={"oom_killed": False, "exit_code": 0, "running": True},
swapouts=0,
)
assert (row["peak_mib"], row["mem_before_mib"]) == (200.0, 100.0)
assert row["flags"] == "oom", row["flags"]
assert set(row) <= set(RUN_COLUMNS), set(row) - set(RUN_COLUMNS)
assert per_client_bodies(row) == 1.0
rows = []
for clients, peak in ((10, 100.0), (20, 105.0), (40, 125.0)):
rows.append(
{
"variant": "limited",
"size": "10mib",
"clients": clients,
"body_bytes": 10 * MIB,
"peak_mib": peak,
"mem_before_mib": 80.0,
"errors_5xx": 0,
"rejected_429": 0 if clients <= 20 else 7,
"uploads_per_s": 4.0,
"flags": "",
},
)
rows.append(dict(rows[0], variant="unlimited", peak_mib=200.0, uploads_per_s=4.1))
rows.append(dict(rows[2], variant="unlimited", flags="oom"))
result_by = {
v.hypothesis: v.result for v in verdicts(rows, limit_mib=512, big="10mib")
}
assert result_by == {
"U1": CONFIRMED,
"U2": CONFIRMED,
"U3": CONFIRMED,
"U4": CONFIRMED,
}, result_by
rows[2]["peak_mib"] = 400.0
assert verdicts(rows, limit_mib=512, big="10mib")[0].result == REFUTED
print("uploads selftest: ok")
if __name__ == "__main__":
if sys.argv[1:] == ["--selftest"]:
selftest()
else:
raise SystemExit("usage: uv run python bench/uploads.py --selftest")