forked from mtusi2026/mtusi2026lab3
839 lines
35 KiB
Python
839 lines
35 KiB
Python
#!/usr/bin/env python3
|
||
import argparse
|
||
import csv
|
||
import datetime as dt
|
||
import fcntl
|
||
import hashlib
|
||
import io
|
||
import json
|
||
import os
|
||
import re
|
||
import shlex
|
||
import statistics
|
||
import subprocess
|
||
import sys
|
||
import urllib.request
|
||
from pathlib import Path
|
||
from urllib.parse import quote
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
OUTPUT = ROOT / "evidence/criteria"
|
||
COMPOSE = [
|
||
"docker",
|
||
"compose",
|
||
"--project-name",
|
||
"highload-lab3",
|
||
"--env-file",
|
||
".env",
|
||
"-f",
|
||
"compose.yml",
|
||
]
|
||
BENCH = ROOT / ".memory/bench-env/bin/python"
|
||
SERIES = (
|
||
"baseline-01",
|
||
"improvement-01",
|
||
"capacity-improvement-01",
|
||
"latency-improvement-01",
|
||
)
|
||
|
||
# Each original grade-3/4 bullet has its own entry. The selected grade-5 track
|
||
# is split into six individually reviewable requirements, including its method.
|
||
CRITERIA = [
|
||
(
|
||
"3-01-stand",
|
||
"3: воспроизводимый стенд и общие хранилища",
|
||
"Запуск из репозитория; две реальные API, единый LB, одна БД и Redis, отдельный migration job.",
|
||
("startup", "inventory"),
|
||
(
|
||
"compose.yml",
|
||
"src/Dockerfile",
|
||
"scripts/init_env.py",
|
||
"src/backend/repositories/database.go",
|
||
),
|
||
),
|
||
(
|
||
"3-02-shared-session",
|
||
"3: одна сессия через разные API",
|
||
"Одна Bearer-сессия обслуживается разными контейнерами через LB без повторного входа; X-Instance-Id соответствует контейнерам.",
|
||
("inventory", "idempotency"),
|
||
(
|
||
"infra/haproxy.cfg",
|
||
"src/backend/handlers/instance.go",
|
||
"src/backend/repositories/redis.go",
|
||
"scripts/check_idempotency.py",
|
||
),
|
||
),
|
||
(
|
||
"3-03-failover",
|
||
"3: чтение и запись при остановке любой одной API",
|
||
"Каждая реплика останавливается отдельно; проверяются реальные exited-контейнеры, восстановление чтения/записи <=10 с и сохранность подтверждённых сообщений.",
|
||
("failover",),
|
||
("scripts/stop-one.sh", "scripts/check_failover.py", "infra/haproxy.cfg"),
|
||
),
|
||
(
|
||
"3-04-idempotency",
|
||
"3: общая идемпотентность сообщений",
|
||
"24 конкурентных POST с одним ключом через разные API возвращают одно сообщение; проверяются конфликт тела, область ключа и повтор после переключения реплики.",
|
||
("idempotency", "failover", "backend"),
|
||
(
|
||
"src/backend/repositories/idempotency.go",
|
||
"src/backend/repositories/migrations/000003_message_idempotency.up.sql",
|
||
"src/backend/handlers/message_http.go",
|
||
"src/backend/tests/idempotency_test.go",
|
||
),
|
||
),
|
||
(
|
||
"3-05-public-tests",
|
||
"3: публичные smoke, persistence и failover",
|
||
"Оригинальные публичные сценарии проходят, failover наблюдает две реплики; persistence использует прежнюю сессию после рестарта API/БД/Redis.",
|
||
(
|
||
"structure",
|
||
"smoke",
|
||
"backend",
|
||
"vet",
|
||
"frontend-build",
|
||
"persistence",
|
||
"public-failover",
|
||
"restore",
|
||
"final-inventory",
|
||
),
|
||
(
|
||
"tests/smoke.py",
|
||
"tests/resilience.py",
|
||
"scripts/restart.sh",
|
||
"scripts/stop-one.sh",
|
||
"src/frontend/package.json",
|
||
),
|
||
),
|
||
(
|
||
"4-01-readiness-shutdown",
|
||
"4: readiness, DNS, таймауты и SIGTERM",
|
||
"Параметры согласованы: проверки LB inter=1s/fall=2/rise=2, timeout check=1s; API прекращает приём при SIGTERM, завершает запросы за <=10 с; штатный stop_grace_period=15s. Проверка отказа использует консервативный бюджет исключения 6 с и измеряет его фактически. Rolling подтверждает graceful shutdown и новую DNS-адресацию.",
|
||
("inventory", "failover", "rolling"),
|
||
(
|
||
"infra/haproxy.cfg",
|
||
"src/backend/main.go",
|
||
"src/backend/handlers/instance.go",
|
||
"src/backend/repositories/timeout.go",
|
||
"scripts/rolling_update.py",
|
||
),
|
||
),
|
||
(
|
||
"4-02-each-replica",
|
||
"4: метрики отказа каждой API и параллельный повтор",
|
||
"Обе API останавливаются по очереди с восстановлением предыдущей. Измеряются successful/attempted RPS, ошибки, восстановление чтения/записи и исключение LB. Нагрузка проверки отказа не объявляется тестом максимальной мощности.",
|
||
("failover", "idempotency"),
|
||
(
|
||
"scripts/check_failover.py",
|
||
"scripts/check_idempotency.py",
|
||
"scripts/stop-one.sh",
|
||
),
|
||
),
|
||
(
|
||
"4-03-resources",
|
||
"4: лимиты, изоляция, секреты и масштабирование",
|
||
"Проверяются фактические CPU/RAM, отсутствие host-портов API/БД/Redis, private networks, отсутствие persistent mounts у API; .env игнорируется Git, container_name не задан.",
|
||
("resources", "inventory"),
|
||
(
|
||
"compose.yml",
|
||
"scripts/check_resources.py",
|
||
"scripts/init_env.py",
|
||
".gitignore",
|
||
),
|
||
),
|
||
(
|
||
"4-04-rolling",
|
||
"4: поочерёдная замена под нагрузкой и миграции",
|
||
"Обе API заменяются последовательно: drain, завершение запросов, SIGTERM, создание новой API, ожидание readiness/DNS. Survivor не заменяется. Одно явное применение миграций на операцию; поддерживаются только обратно совместимые миграции.",
|
||
("rolling", "final-inventory"),
|
||
(
|
||
"scripts/rolling_update.py",
|
||
"scripts/check_rolling.py",
|
||
"compose.yml",
|
||
"src/backend/repositories/database.go",
|
||
"src/backend/repositories/migrations/000003_message_idempotency.up.sql",
|
||
),
|
||
),
|
||
(
|
||
"5-01-method",
|
||
"5: гипотезы и повторяемая методика трека 1",
|
||
"Протоколы записаны до соответствующих измерений; сохранены SHA-256, seeds, порядок, фикстура, образы и архив исходников. Пилоты исключены. Анализ старых данных не является новым нагрузочным экспериментом.",
|
||
("audit-baseline", "audit-hi", "audit-hc", "audit-hl", "experiment-summary"),
|
||
(
|
||
"experiments/lab3-scale/docs/method.md",
|
||
"experiments/lab3-scale/docs/runtime-tuning.md",
|
||
"experiments/lab3-scale/run.py",
|
||
"experiments/lab3-scale/fixture.sql",
|
||
"evidence/grade5/pilots/README.md",
|
||
),
|
||
),
|
||
(
|
||
"5-02-fixed-budget",
|
||
"5: 1/2/4 API при фиксированном CPU/RAM",
|
||
"Сравниваются A1/A2/A4: общий бюджет 1 CPU/256 MiB, пул 12, единый профиль. H1 подтверждена в измеренных условиях; показаны все повторения и границы сетки.",
|
||
("analyze-baseline", "audit-baseline", "experiment-summary"),
|
||
(
|
||
"experiments/lab3-scale/common.py",
|
||
"experiments/lab3-scale/compose.yml",
|
||
"evidence/grade5/baseline-01/hypotheses.json",
|
||
"evidence/grade5/baseline-01/capacity.png",
|
||
),
|
||
),
|
||
(
|
||
"5-03-growing-budget",
|
||
"5: серия с ростом ресурсов",
|
||
"Сравниваются B1/B2/B4: по 0.5 CPU/128 MiB на API, общие зависимости фиксированы. H2 о выигрыше >=1.5 осталась неопределённой; это результат исследования, а не подтверждение гипотезы.",
|
||
("analyze-baseline", "audit-baseline", "experiment-summary"),
|
||
(
|
||
"experiments/lab3-scale/protocols/baseline.json",
|
||
"evidence/grade5/baseline-01/analysis.json",
|
||
"evidence/grade5/baseline-01/throughput.png",
|
||
"evidence/grade5/baseline-01/hypotheses.json",
|
||
),
|
||
),
|
||
(
|
||
"5-04-saturation",
|
||
"5: насыщение API/БД/пула и роль LB",
|
||
"По совместным метрикам CPU, throttling, pgx pool, WAL waits и очереди LB объясняется предел роста. Насыщение LB в диапазоне не установлено; ожидание пула само по себе не доказывает необходимость увеличения пула.",
|
||
("analyze-baseline", "analyze-hl", "experiment-summary"),
|
||
(
|
||
"experiments/lab3-scale/load.py",
|
||
"experiments/lab3-scale/analyze.py",
|
||
"src/backend/main.go",
|
||
"evidence/grade5/baseline-01/analysis.json",
|
||
"evidence/grade5/baseline-01/api-cpu.png",
|
||
"evidence/grade5/RESULTS.md",
|
||
),
|
||
),
|
||
(
|
||
"5-05-data-plots",
|
||
"5: исходные данные, графики и проверка сохранности",
|
||
"Проверены число всех 150 основных/проверочных прогонов, хеши и сохранность подтверждённых записей. Публикуются также ошибки и неопределённые результаты; сырые requests/telemetry и warmup сохранены по прогонам.",
|
||
(
|
||
"analyze-baseline",
|
||
"analyze-hi",
|
||
"analyze-hc",
|
||
"analyze-hl",
|
||
"plot-hl",
|
||
"audit-baseline",
|
||
"audit-hi",
|
||
"audit-hc",
|
||
"audit-hl",
|
||
"experiment-summary",
|
||
),
|
||
(
|
||
"evidence/grade5/baseline-01/order.json",
|
||
"evidence/grade5/baseline-01/source.tar.gz",
|
||
"evidence/grade5/baseline-01/summary.csv",
|
||
"evidence/grade5/baseline-01/success.png",
|
||
"evidence/grade5/latency-improvement-01/latency-pairs.png",
|
||
"experiments/lab3-scale/postprocess/decisions.py",
|
||
),
|
||
),
|
||
(
|
||
"5-06-improvement",
|
||
"5: подтверждённое улучшение и ограничения",
|
||
"В трёх свежих парах HL при 325 req/s GOMAXPROCS=1 снижает p95 >=15%, при >=99% успехов, p95 <=500 мс и сохранности. HI/HC остались неопределёнными и не исключены. Процент роста предельной мощности C не считается доказанным.",
|
||
(
|
||
"analyze-hi",
|
||
"analyze-hc",
|
||
"analyze-hl",
|
||
"audit-hi",
|
||
"audit-hc",
|
||
"audit-hl",
|
||
"plot-hl",
|
||
"experiment-summary",
|
||
"final-inventory",
|
||
),
|
||
(
|
||
"compose.yml",
|
||
"experiments/lab3-scale/docs/runtime-tuning.md",
|
||
"evidence/grade5/latency-improvement-01/decision.json",
|
||
"evidence/grade5/latency-improvement-01/latency-pairs.png",
|
||
"evidence/grade5/improvement-01/decision.json",
|
||
"evidence/grade5/capacity-improvement-01/decision.json",
|
||
),
|
||
),
|
||
]
|
||
|
||
|
||
def timestamp():
|
||
return dt.datetime.now(dt.timezone.utc).isoformat()
|
||
|
||
|
||
def command(args):
|
||
return subprocess.check_output(
|
||
args, cwd=ROOT, text=True, stderr=subprocess.PIPE
|
||
).strip()
|
||
|
||
|
||
def inspect_state():
|
||
expected_replicas = int(json.loads(command(COMPOSE + ["config", "--format", "json"]))["services"]["api"]["deploy"]["replicas"])
|
||
ids = command(COMPOSE + ["ps", "--all", "-q"]).split()
|
||
rows = []
|
||
template = (
|
||
'{"service":{{json (index .Config.Labels "com.docker.compose.service")}},'
|
||
'"instance":{{json .Config.Hostname}},"state":{{json .State.Status}},'
|
||
'"exit_code":{{.State.ExitCode}},"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}null{{end}},'
|
||
'"cpu":{{.HostConfig.NanoCpus}},"memory":{{.HostConfig.Memory}},'
|
||
'"ports":{{json .HostConfig.PortBindings}},"networks":{{json .NetworkSettings.Networks}},'
|
||
'"image":{{json .Image}},"started":{{json .State.StartedAt}}}'
|
||
)
|
||
for container in ids:
|
||
row = json.loads(
|
||
command(["docker", "inspect", "--format", template, container])
|
||
)
|
||
row["container_id"] = container
|
||
row["networks"] = {
|
||
name: {"address": net["IPAddress"]} for name, net in row["networks"].items()
|
||
}
|
||
if row["service"] == "api":
|
||
row["gomaxprocs"] = command(
|
||
["docker", "exec", container, "printenv", "GOMAXPROCS"]
|
||
)
|
||
rows.append(row)
|
||
api = [row for row in rows if row["service"] == "api"]
|
||
assert len(api) == expected_replicas and all(row["health"] == "healthy" for row in api), (
|
||
"All configured API replicas must be healthy"
|
||
)
|
||
for row in rows:
|
||
assert row["cpu"] > 0 and row["memory"] > 0
|
||
if row["service"] in ("api", "db", "redis"):
|
||
assert not row["ports"], "Storage/API published on host"
|
||
if row["service"] == "migrate":
|
||
assert row["state"] == "exited" and row["exit_code"] == 0
|
||
else:
|
||
assert row["state"] == "running" and row["health"] == "healthy"
|
||
with urllib.request.urlopen(
|
||
"http://127.0.0.1:8404/stats;csv", timeout=5
|
||
) as response:
|
||
stats = list(
|
||
csv.DictReader(io.StringIO(response.read().decode().removeprefix("# ")))
|
||
)
|
||
stats = [
|
||
{
|
||
k: row[k]
|
||
for k in (
|
||
"svname",
|
||
"status",
|
||
"addr",
|
||
"check_status",
|
||
"check_duration",
|
||
"scur",
|
||
"qcur",
|
||
)
|
||
}
|
||
for row in stats
|
||
if row["pxname"] == "api_replicas" and row["svname"].startswith("api")
|
||
]
|
||
assert len(stats) == expected_replicas and all(row["status"].startswith("UP") for row in stats)
|
||
addresses = {net["address"] for row in api for net in row["networks"].values()}
|
||
assert all(row["addr"].split(":")[0] in addresses for row in stats)
|
||
migration = command(
|
||
COMPOSE
|
||
+ [
|
||
"exec",
|
||
"-T",
|
||
"db",
|
||
"sh",
|
||
"-c",
|
||
'psql -X -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At -c "SELECT version,dirty FROM schema_migrations"',
|
||
]
|
||
)
|
||
assert migration == "3|f", "Expected clean migration version 3"
|
||
ignored = command(["git", "check-ignore", ".env", ".memory/"]).splitlines()
|
||
assert set(ignored) == {".env", ".memory/"}
|
||
assert "container_name:" not in ROOT.joinpath("compose.yml").read_text()
|
||
health = []
|
||
for path in ("/health/live", "/health/ready"):
|
||
with urllib.request.urlopen(
|
||
"http://127.0.0.1:8080" + path, timeout=5
|
||
) as response:
|
||
health.append(
|
||
{
|
||
"path": path,
|
||
"status": response.status,
|
||
"instance": response.headers.get("X-Instance-Id"),
|
||
}
|
||
)
|
||
print(
|
||
json.dumps(
|
||
{
|
||
"containers": rows,
|
||
"haproxy_backends": stats,
|
||
"health_through_lb": health,
|
||
"migration_version_dirty": migration,
|
||
"git_ignored": ignored,
|
||
"hook_scope": {
|
||
name: {
|
||
"sha256": hashlib.sha256(
|
||
ROOT.joinpath("scripts", name).read_bytes()
|
||
).hexdigest()
|
||
}
|
||
for name in ("restart.sh", "stop-one.sh")
|
||
},
|
||
},
|
||
ensure_ascii=False,
|
||
indent=2,
|
||
)
|
||
)
|
||
print(
|
||
"PASS: actual containers and HAProxy addresses agree; migration 3 clean; storage/API private; secrets ignored"
|
||
)
|
||
|
||
|
||
def experiment_summary():
|
||
result = {}
|
||
for name in SERIES:
|
||
directory = ROOT / "evidence/grade5" / name
|
||
analysis = json.loads(directory.joinpath("analysis.json").read_text())
|
||
groups = {}
|
||
for row in analysis["runs"]:
|
||
groups.setdefault((row["config"], row["rate"]), []).append(row)
|
||
measurements = []
|
||
for (config, rate), rows in sorted(groups.items()):
|
||
measurements.append(
|
||
{
|
||
"config": config,
|
||
"rate": rate,
|
||
"repeats": len(rows),
|
||
"medians": {
|
||
field: statistics.median(r[field] for r in rows)
|
||
for field in (
|
||
"p50_ms",
|
||
"p95_ms",
|
||
"p99_ms",
|
||
"window_goodput_rps",
|
||
"success_ratio",
|
||
"generator_cpu_cores",
|
||
"dispatch_p95_ms",
|
||
)
|
||
},
|
||
"p95_range_ms": [
|
||
min(r["p95_ms"] for r in rows),
|
||
max(r["p95_ms"] for r in rows),
|
||
],
|
||
"diagnostics": [
|
||
{"repeat": r["repeat"], **r["diagnostics"]} for r in rows
|
||
],
|
||
}
|
||
)
|
||
result[name] = {
|
||
"audit": json.loads(directory.joinpath("audit.json").read_text()),
|
||
"protocol": json.loads(directory.joinpath("protocol.json").read_text()),
|
||
"decisions": json.loads(
|
||
directory.joinpath(
|
||
"hypotheses.json" if name == "baseline-01" else "decision.json"
|
||
).read_text()
|
||
),
|
||
"capacities": analysis["capacities"],
|
||
"measurements": measurements,
|
||
}
|
||
print(
|
||
"Historical measurements from 2026-10-10; this command re-reads saved data, it does not run load."
|
||
)
|
||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||
|
||
|
||
class Recorder:
|
||
def __init__(self, run_dir):
|
||
self.directory = run_dir
|
||
self.records = {}
|
||
self.secrets = []
|
||
for path in (ROOT / ".env", ROOT / ".memory/benchmark/environment"):
|
||
if path.exists():
|
||
for line in path.read_text().splitlines():
|
||
if "=" in line and any(
|
||
word in line.split("=", 1)[0].upper()
|
||
for word in ("PASSWORD", "TOKEN", "SECRET")
|
||
):
|
||
value = line.split("=", 1)[1].strip().strip("\"'")
|
||
if value:
|
||
self.secrets.append(value)
|
||
|
||
def redact(self, value):
|
||
for secret in self.secrets:
|
||
value = value.replace(secret, "[REDACTED]")
|
||
value = re.sub(
|
||
r"(?i)(Bearer|Basic)\s+[A-Za-z0-9+/_.=:-]{16,}", r"\1 [REDACTED]", value
|
||
)
|
||
value = re.sub(
|
||
r'(?i)("(?:token|password)"\s*:\s*")[^"]+("\s*[,}])',
|
||
r"\1[REDACTED]\2",
|
||
value,
|
||
)
|
||
return value
|
||
|
||
def run(self, name, args, *, kind="fresh-check", timeout=600, artifacts=()):
|
||
record = {
|
||
"command": shlex.join(str(a) for a in args),
|
||
"kind": kind,
|
||
"started_utc": timestamp(),
|
||
}
|
||
raw = ROOT / ".memory" / (self.directory.name + "-" + name + "-output.txt")
|
||
raw.touch(mode=0o600)
|
||
try:
|
||
with raw.open("w") as file:
|
||
process = subprocess.Popen(
|
||
[str(a) for a in args],
|
||
cwd=ROOT,
|
||
stdout=file,
|
||
stderr=subprocess.STDOUT,
|
||
)
|
||
try:
|
||
record["exit_code"] = process.wait(timeout=timeout)
|
||
except BaseException:
|
||
process.terminate()
|
||
try:
|
||
process.wait(timeout=10)
|
||
except subprocess.TimeoutExpired:
|
||
process.kill()
|
||
process.wait()
|
||
record["exit_code"] = process.returncode
|
||
raise
|
||
finally:
|
||
record["finished_utc"] = timestamp()
|
||
record["output"] = self.redact(raw.read_text(errors="replace"))
|
||
raw.unlink(missing_ok=True)
|
||
record.setdefault("exit_code", -1)
|
||
self.directory.joinpath(name + ".txt").write_text(record["output"])
|
||
record["log"] = str(
|
||
self.directory.joinpath(name + ".txt").relative_to(ROOT)
|
||
)
|
||
copied = []
|
||
for relative in artifacts:
|
||
source = ROOT / relative
|
||
if source.exists():
|
||
dest = self.directory / source.name
|
||
dest.write_text(self.redact(source.read_text()))
|
||
copied.append(str(dest.relative_to(ROOT)))
|
||
record["artifacts"] = copied
|
||
self.records[name] = record
|
||
print(f"{name}: exit {record['exit_code']}", flush=True)
|
||
return record["exit_code"] == 0
|
||
|
||
|
||
def link(path, output):
|
||
target = os.path.relpath(ROOT / path, output.parent).replace(os.sep, "/")
|
||
return f"[{path}]({quote(target, safe='/._-')})"
|
||
|
||
|
||
def fenced(value, language="text"):
|
||
longest = max((len(m) for m in re.findall(r"`+", value)), default=0)
|
||
fence = "`" * max(3, longest + 1)
|
||
return f"{fence}{language}\n{value.rstrip()}\n{fence}\n"
|
||
|
||
|
||
def write_evidence(recorder, metadata, *, write_manifest=True):
|
||
index = [
|
||
"# Доказательства критериев лабораторной 3",
|
||
"",
|
||
"Генерация: `python3 scripts/generate_evidence.py`. Функциональные проверки выполняются заново; данные исследования 2026-10-10 пересчитываются без нового нагрузочного эксперимента.",
|
||
"",
|
||
f"Запуск: `{metadata['run_id']}`. Команды, время и хеши исходников сохранены в manifest запуска. Ссылки на реализацию ведут к текущим файлам проекта.",
|
||
"",
|
||
"| Пункт | Статус проверки | Доказательство |",
|
||
"|---|---|---|",
|
||
]
|
||
for slug, title, description, jobs, sources in CRITERIA:
|
||
output = OUTPUT / (slug + ".md")
|
||
records = [recorder.records.get(job) for job in jobs]
|
||
status = (
|
||
"PASS"
|
||
if all(record and record["exit_code"] == 0 for record in records)
|
||
else "FAIL / НЕ ЗАВЕРШЕНО"
|
||
)
|
||
parts = [
|
||
f"# {title}",
|
||
"",
|
||
"## Что проверяется",
|
||
"",
|
||
description,
|
||
"",
|
||
f"Статус выполнения проверок: **{status}**. Для исследования PASS означает успешную проверку данных и правил анализа; решение гипотезы указано отдельно и может быть неопределённым.",
|
||
"",
|
||
f"Запуск: `{metadata['run_id']}`. Команды выполнены из корня проекта в WSL; время и версии указаны в manifest.",
|
||
"",
|
||
"## Команды и фактический вывод",
|
||
"",
|
||
]
|
||
for job, record in zip(jobs, records):
|
||
if not record:
|
||
parts.extend(
|
||
[
|
||
f"### {job}",
|
||
"",
|
||
"НЕ ВЫПОЛНЕНО: запуск был прерван либо предыдущая проверка завершилась ошибкой.",
|
||
"",
|
||
]
|
||
)
|
||
continue
|
||
label = (
|
||
"Пересчёт/проверка исторических измерений"
|
||
if record["kind"] == "historical-analysis"
|
||
else "Проверка приложения"
|
||
)
|
||
lines = record["output"].splitlines()
|
||
excerpt = len(lines) > 120
|
||
shown = (
|
||
"\n".join(lines[:70] + ["... [пропущенные строки — в полном журнале] ..."] + lines[-20:])
|
||
if excerpt else record["output"]
|
||
)
|
||
parts.extend(
|
||
[
|
||
f"### {job}",
|
||
"",
|
||
label + ".",
|
||
"",
|
||
fenced(record["command"], "sh"),
|
||
f"Начало UTC: `{record['started_utc']}`; конец UTC: `{record['finished_utc']}`; exit code: **{record['exit_code']}**.",
|
||
"",
|
||
"Фрагмент вывода; полный журнал доступен по ссылке ниже." if excerpt else "Вывод команды:",
|
||
"",
|
||
fenced(
|
||
shown
|
||
or "Команда не вывела текст; смотрите exit code и связанные JSON-результаты."
|
||
),
|
||
"Полный сохранённый вывод: " + link(record["log"], output) + ".",
|
||
"",
|
||
]
|
||
)
|
||
parts.extend("- " + link(path, output) for path in record["artifacts"])
|
||
parts.append("")
|
||
parts.extend(["## Реализация и дополнительные доказательства", ""])
|
||
parts.extend("- " + link(path, output) for path in sources)
|
||
parts.extend(
|
||
[
|
||
"- "
|
||
+ link(
|
||
str(recorder.directory.joinpath("manifest.json").relative_to(ROOT)),
|
||
output,
|
||
),
|
||
"",
|
||
]
|
||
)
|
||
output.write_text("\n".join(parts))
|
||
index.append(f"| {title} | {status} | [{slug}]({slug}.md) |")
|
||
OUTPUT.joinpath("README.md").write_text("\n".join(index) + "\n")
|
||
manifest = {
|
||
**metadata,
|
||
"commands": {
|
||
name: {key: value for key, value in rec.items() if key != "output"}
|
||
for name, rec in recorder.records.items()
|
||
},
|
||
}
|
||
if write_manifest:
|
||
recorder.directory.joinpath("manifest.json").write_text(
|
||
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n"
|
||
)
|
||
|
||
|
||
def generate(run_id):
|
||
ROOT.joinpath(".memory").mkdir(exist_ok=True)
|
||
with ROOT.joinpath(".memory/evidence.lock").open("w") as lock:
|
||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||
run_dir = OUTPUT / "runs" / run_id
|
||
run_dir.mkdir(parents=True, exist_ok=False)
|
||
recorder = Recorder(run_dir)
|
||
metadata = {
|
||
"run_id": run_id,
|
||
"started_utc": timestamp(),
|
||
"git_commit": command(["git", "rev-parse", "HEAD"]),
|
||
"branch": command(["git", "branch", "--show-current"]),
|
||
"working_tree_status": command(["git", "status", "--short"]),
|
||
"grade5_measurement_date": "2026-10-10",
|
||
"new_load_experiment": False,
|
||
}
|
||
sources = set(path for criterion in CRITERIA for path in criterion[4]) | {
|
||
"scripts/generate_evidence.py"
|
||
}
|
||
metadata["file_sha256"] = {
|
||
path: hashlib.sha256(ROOT.joinpath(path).read_bytes()).hexdigest()
|
||
for path in sorted(sources)
|
||
if ROOT.joinpath(path).is_file()
|
||
}
|
||
passed = True
|
||
try:
|
||
jobs = [
|
||
("startup", ["make", "app-up"], ()),
|
||
("structure", ["make", "check"], ()),
|
||
(
|
||
"inventory",
|
||
["python3", "scripts/generate_evidence.py", "--inspect"],
|
||
(),
|
||
),
|
||
(
|
||
"resources",
|
||
["python3", "scripts/check_resources.py"],
|
||
(".memory/grade4/resources.json",),
|
||
),
|
||
("smoke", ["make", "test"], ()),
|
||
("backend", ["make", "test-backend"], ()),
|
||
(
|
||
"vet",
|
||
COMPOSE
|
||
+ [
|
||
"run",
|
||
"--rm",
|
||
"--no-deps",
|
||
"backend-tests",
|
||
"go",
|
||
"vet",
|
||
"./backend/...",
|
||
],
|
||
(),
|
||
),
|
||
(
|
||
"frontend-build",
|
||
COMPOSE + ["exec", "-T", "frontend", "npm", "run", "build"],
|
||
(),
|
||
),
|
||
("idempotency", ["python3", "scripts/check_idempotency.py"], ()),
|
||
(
|
||
"failover",
|
||
["python3", "scripts/check_failover.py"],
|
||
(".memory/grade4/failover.json",),
|
||
),
|
||
(
|
||
"rolling",
|
||
["python3", "scripts/check_rolling.py"],
|
||
(".memory/grade4/rolling.json",),
|
||
),
|
||
(
|
||
"persistence",
|
||
["make", "test-persistence", "ACTION_SCRIPT=scripts/restart.sh"],
|
||
(),
|
||
),
|
||
(
|
||
"public-failover",
|
||
["make", "test-failover", "ACTION_SCRIPT=scripts/stop-one.sh"],
|
||
(),
|
||
),
|
||
]
|
||
for name, args, artifacts in jobs:
|
||
if not recorder.run(name, args, artifacts=artifacts):
|
||
passed = False
|
||
break
|
||
if passed:
|
||
if not BENCH.exists():
|
||
passed = recorder.run(
|
||
"analysis-venv",
|
||
["python3", "-m", "venv", str(BENCH.parent.parent)],
|
||
)
|
||
if passed:
|
||
passed = recorder.run(
|
||
"analysis-dependencies",
|
||
[
|
||
str(BENCH),
|
||
"-m",
|
||
"pip",
|
||
"install",
|
||
"-r",
|
||
"experiments/lab3-scale/requirements-lock.txt",
|
||
],
|
||
)
|
||
for series, suffix, mode in zip(
|
||
SERIES, ("baseline", "hi", "hc", "hl"), ("audit", "hi", "hc", "hl")
|
||
):
|
||
if not passed:
|
||
break
|
||
passed = recorder.run(
|
||
"analyze-" + suffix,
|
||
[str(BENCH), "experiments/lab3-scale/analyze.py", series],
|
||
kind="historical-analysis",
|
||
)
|
||
if passed:
|
||
passed = recorder.run(
|
||
"audit-" + suffix,
|
||
[
|
||
str(BENCH),
|
||
"experiments/lab3-scale/postprocess/decisions.py",
|
||
mode,
|
||
series,
|
||
],
|
||
kind="historical-analysis",
|
||
)
|
||
if passed:
|
||
passed = recorder.run(
|
||
"plot-hl",
|
||
[
|
||
str(BENCH),
|
||
"experiments/lab3-scale/postprocess/plot_latency.py",
|
||
"latency-improvement-01",
|
||
],
|
||
kind="historical-analysis",
|
||
)
|
||
if passed:
|
||
passed = recorder.run(
|
||
"experiment-summary",
|
||
[
|
||
"python3",
|
||
"scripts/generate_evidence.py",
|
||
"--experiment-summary",
|
||
],
|
||
kind="historical-analysis",
|
||
)
|
||
except BaseException:
|
||
passed = False
|
||
raise
|
||
finally:
|
||
# The public failover hook intentionally leaves a replica stopped.
|
||
# Restore only this lab's application, preserving all volumes.
|
||
restored = recorder.run("restore", COMPOSE + ["up", "-d", "--wait"])
|
||
final = recorder.run(
|
||
"final-inventory",
|
||
["python3", "scripts/generate_evidence.py", "--inspect"],
|
||
)
|
||
metadata["finished_utc"] = timestamp()
|
||
metadata["all_commands_passed"] = passed and restored and final
|
||
write_evidence(recorder, metadata)
|
||
print("Evidence: evidence/criteria/README.md", flush=True)
|
||
return 0 if metadata["all_commands_passed"] else 1
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(__doc__)
|
||
parser.add_argument(
|
||
"--run-id",
|
||
default=dt.datetime.now(dt.timezone.utc).strftime("lab3-%Y%m%dT%H%M%S%fZ"),
|
||
)
|
||
parser.add_argument("--inspect", action="store_true", help=argparse.SUPPRESS)
|
||
parser.add_argument(
|
||
"--render", metavar="RUN_ID",
|
||
help="Update Markdown from saved logs without running checks",
|
||
)
|
||
parser.add_argument(
|
||
"--experiment-summary", action="store_true", help=argparse.SUPPRESS
|
||
)
|
||
args = parser.parse_args()
|
||
if args.render:
|
||
if not re.fullmatch(r"[A-Za-z0-9_-]+", args.render):
|
||
parser.error("render run-id must contain only letters, digits, '_' or '-'")
|
||
from types import SimpleNamespace
|
||
|
||
directory = OUTPUT / "runs" / args.render
|
||
metadata = json.loads(directory.joinpath("manifest.json").read_text())
|
||
records = metadata.pop("commands")
|
||
for record in records.values():
|
||
record["output"] = ROOT.joinpath(record["log"]).read_text()
|
||
write_evidence(
|
||
SimpleNamespace(directory=directory, records=records), metadata,
|
||
write_manifest=False,
|
||
)
|
||
return 0
|
||
if args.inspect:
|
||
inspect_state()
|
||
return 0
|
||
if args.experiment_summary:
|
||
experiment_summary()
|
||
return 0
|
||
if not re.fullmatch(r"[A-Za-z0-9_-]+", args.run_id):
|
||
parser.error("run-id must contain only letters, digits, '_' or '-'")
|
||
if not sys.platform.startswith("linux"):
|
||
parser.error("Run inside Ubuntu WSL/Linux")
|
||
return generate(args.run_id)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|