Files
mtusi2026lab3/scripts/generate_evidence.py
T

839 lines
35 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
import argparse
import csv
import datetime as dt
import fcntl
import hashlib
import io
import json
import os
import re
import shlex
import statistics
import subprocess
import sys
import urllib.request
from pathlib import Path
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
OUTPUT = ROOT / "evidence/criteria"
COMPOSE = [
"docker",
"compose",
"--project-name",
"highload-lab3",
"--env-file",
".env",
"-f",
"compose.yml",
]
BENCH = ROOT / ".memory/bench-env/bin/python"
SERIES = (
"baseline-01",
"improvement-01",
"capacity-improvement-01",
"latency-improvement-01",
)
# Each original grade-3/4 bullet has its own entry. The selected grade-5 track
# is split into six individually reviewable requirements, including its method.
CRITERIA = [
(
"3-01-stand",
"3: воспроизводимый стенд и общие хранилища",
"Запуск из репозитория; две реальные API, единый LB, одна БД и Redis, отдельный migration job.",
("startup", "inventory"),
(
"compose.yml",
"src/Dockerfile",
"scripts/init_env.py",
"src/backend/repositories/database.go",
),
),
(
"3-02-shared-session",
"3: одна сессия через разные API",
"Одна Bearer-сессия обслуживается разными контейнерами через LB без повторного входа; X-Instance-Id соответствует контейнерам.",
("inventory", "idempotency"),
(
"infra/haproxy.cfg",
"src/backend/handlers/instance.go",
"src/backend/repositories/redis.go",
"scripts/check_idempotency.py",
),
),
(
"3-03-failover",
"3: чтение и запись при остановке любой одной API",
"Каждая реплика останавливается отдельно; проверяются реальные exited-контейнеры, восстановление чтения/записи <=10 с и сохранность подтверждённых сообщений.",
("failover",),
("scripts/stop-one.sh", "scripts/check_failover.py", "infra/haproxy.cfg"),
),
(
"3-04-idempotency",
"3: общая идемпотентность сообщений",
"24 конкурентных POST с одним ключом через разные API возвращают одно сообщение; проверяются конфликт тела, область ключа и повтор после переключения реплики.",
("idempotency", "failover", "backend"),
(
"src/backend/repositories/idempotency.go",
"src/backend/repositories/migrations/000003_message_idempotency.up.sql",
"src/backend/handlers/message_http.go",
"src/backend/tests/idempotency_test.go",
),
),
(
"3-05-public-tests",
"3: публичные smoke, persistence и failover",
"Оригинальные публичные сценарии проходят, failover наблюдает две реплики; persistence использует прежнюю сессию после рестарта API/БД/Redis.",
(
"structure",
"smoke",
"backend",
"vet",
"frontend-build",
"persistence",
"public-failover",
"restore",
"final-inventory",
),
(
"tests/smoke.py",
"tests/resilience.py",
"scripts/restart.sh",
"scripts/stop-one.sh",
"src/frontend/package.json",
),
),
(
"4-01-readiness-shutdown",
"4: readiness, DNS, таймауты и SIGTERM",
"Параметры согласованы: проверки LB inter=1s/fall=2/rise=2, timeout check=1s; API прекращает приём при SIGTERM, завершает запросы за <=10 с; штатный stop_grace_period=15s. Проверка отказа использует консервативный бюджет исключения 6 с и измеряет его фактически. Rolling подтверждает graceful shutdown и новую DNS-адресацию.",
("inventory", "failover", "rolling"),
(
"infra/haproxy.cfg",
"src/backend/main.go",
"src/backend/handlers/instance.go",
"src/backend/repositories/timeout.go",
"scripts/rolling_update.py",
),
),
(
"4-02-each-replica",
"4: метрики отказа каждой API и параллельный повтор",
"Обе API останавливаются по очереди с восстановлением предыдущей. Измеряются successful/attempted RPS, ошибки, восстановление чтения/записи и исключение LB. Нагрузка проверки отказа не объявляется тестом максимальной мощности.",
("failover", "idempotency"),
(
"scripts/check_failover.py",
"scripts/check_idempotency.py",
"scripts/stop-one.sh",
),
),
(
"4-03-resources",
"4: лимиты, изоляция, секреты и масштабирование",
"Проверяются фактические CPU/RAM, отсутствие host-портов API/БД/Redis, private networks, отсутствие persistent mounts у API; .env игнорируется Git, container_name не задан.",
("resources", "inventory"),
(
"compose.yml",
"scripts/check_resources.py",
"scripts/init_env.py",
".gitignore",
),
),
(
"4-04-rolling",
"4: поочерёдная замена под нагрузкой и миграции",
"Обе API заменяются последовательно: drain, завершение запросов, SIGTERM, создание новой API, ожидание readiness/DNS. Survivor не заменяется. Одно явное применение миграций на операцию; поддерживаются только обратно совместимые миграции.",
("rolling", "final-inventory"),
(
"scripts/rolling_update.py",
"scripts/check_rolling.py",
"compose.yml",
"src/backend/repositories/database.go",
"src/backend/repositories/migrations/000003_message_idempotency.up.sql",
),
),
(
"5-01-method",
"5: гипотезы и повторяемая методика трека 1",
"Протоколы записаны до соответствующих измерений; сохранены SHA-256, seeds, порядок, фикстура, образы и архив исходников. Пилоты исключены. Анализ старых данных не является новым нагрузочным экспериментом.",
("audit-baseline", "audit-hi", "audit-hc", "audit-hl", "experiment-summary"),
(
"experiments/lab3-scale/docs/method.md",
"experiments/lab3-scale/docs/runtime-tuning.md",
"experiments/lab3-scale/run.py",
"experiments/lab3-scale/fixture.sql",
"evidence/grade5/pilots/README.md",
),
),
(
"5-02-fixed-budget",
"5: 1/2/4 API при фиксированном CPU/RAM",
"Сравниваются A1/A2/A4: общий бюджет 1 CPU/256 MiB, пул 12, единый профиль. H1 подтверждена в измеренных условиях; показаны все повторения и границы сетки.",
("analyze-baseline", "audit-baseline", "experiment-summary"),
(
"experiments/lab3-scale/common.py",
"experiments/lab3-scale/compose.yml",
"evidence/grade5/baseline-01/hypotheses.json",
"evidence/grade5/baseline-01/capacity.png",
),
),
(
"5-03-growing-budget",
"5: серия с ростом ресурсов",
"Сравниваются B1/B2/B4: по 0.5 CPU/128 MiB на API, общие зависимости фиксированы. H2 о выигрыше >=1.5 осталась неопределённой; это результат исследования, а не подтверждение гипотезы.",
("analyze-baseline", "audit-baseline", "experiment-summary"),
(
"experiments/lab3-scale/protocols/baseline.json",
"evidence/grade5/baseline-01/analysis.json",
"evidence/grade5/baseline-01/throughput.png",
"evidence/grade5/baseline-01/hypotheses.json",
),
),
(
"5-04-saturation",
"5: насыщение API/БД/пула и роль LB",
"По совместным метрикам CPU, throttling, pgx pool, WAL waits и очереди LB объясняется предел роста. Насыщение LB в диапазоне не установлено; ожидание пула само по себе не доказывает необходимость увеличения пула.",
("analyze-baseline", "analyze-hl", "experiment-summary"),
(
"experiments/lab3-scale/load.py",
"experiments/lab3-scale/analyze.py",
"src/backend/main.go",
"evidence/grade5/baseline-01/analysis.json",
"evidence/grade5/baseline-01/api-cpu.png",
"evidence/grade5/RESULTS.md",
),
),
(
"5-05-data-plots",
"5: исходные данные, графики и проверка сохранности",
"Проверены число всех 150 основных/проверочных прогонов, хеши и сохранность подтверждённых записей. Публикуются также ошибки и неопределённые результаты; сырые requests/telemetry и warmup сохранены по прогонам.",
(
"analyze-baseline",
"analyze-hi",
"analyze-hc",
"analyze-hl",
"plot-hl",
"audit-baseline",
"audit-hi",
"audit-hc",
"audit-hl",
"experiment-summary",
),
(
"evidence/grade5/baseline-01/order.json",
"evidence/grade5/baseline-01/source.tar.gz",
"evidence/grade5/baseline-01/summary.csv",
"evidence/grade5/baseline-01/success.png",
"evidence/grade5/latency-improvement-01/latency-pairs.png",
"experiments/lab3-scale/postprocess/decisions.py",
),
),
(
"5-06-improvement",
"5: подтверждённое улучшение и ограничения",
"В трёх свежих парах HL при 325 req/s GOMAXPROCS=1 снижает p95 >=15%, при >=99% успехов, p95 <=500 мс и сохранности. HI/HC остались неопределёнными и не исключены. Процент роста предельной мощности C не считается доказанным.",
(
"analyze-hi",
"analyze-hc",
"analyze-hl",
"audit-hi",
"audit-hc",
"audit-hl",
"plot-hl",
"experiment-summary",
"final-inventory",
),
(
"compose.yml",
"experiments/lab3-scale/docs/runtime-tuning.md",
"evidence/grade5/latency-improvement-01/decision.json",
"evidence/grade5/latency-improvement-01/latency-pairs.png",
"evidence/grade5/improvement-01/decision.json",
"evidence/grade5/capacity-improvement-01/decision.json",
),
),
]
def timestamp():
return dt.datetime.now(dt.timezone.utc).isoformat()
def command(args):
return subprocess.check_output(
args, cwd=ROOT, text=True, stderr=subprocess.PIPE
).strip()
def inspect_state():
expected_replicas = int(json.loads(command(COMPOSE + ["config", "--format", "json"]))["services"]["api"]["deploy"]["replicas"])
ids = command(COMPOSE + ["ps", "--all", "-q"]).split()
rows = []
template = (
'{"service":{{json (index .Config.Labels "com.docker.compose.service")}},'
'"instance":{{json .Config.Hostname}},"state":{{json .State.Status}},'
'"exit_code":{{.State.ExitCode}},"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}null{{end}},'
'"cpu":{{.HostConfig.NanoCpus}},"memory":{{.HostConfig.Memory}},'
'"ports":{{json .HostConfig.PortBindings}},"networks":{{json .NetworkSettings.Networks}},'
'"image":{{json .Image}},"started":{{json .State.StartedAt}}}'
)
for container in ids:
row = json.loads(
command(["docker", "inspect", "--format", template, container])
)
row["container_id"] = container
row["networks"] = {
name: {"address": net["IPAddress"]} for name, net in row["networks"].items()
}
if row["service"] == "api":
row["gomaxprocs"] = command(
["docker", "exec", container, "printenv", "GOMAXPROCS"]
)
rows.append(row)
api = [row for row in rows if row["service"] == "api"]
assert len(api) == expected_replicas and all(row["health"] == "healthy" for row in api), (
"All configured API replicas must be healthy"
)
for row in rows:
assert row["cpu"] > 0 and row["memory"] > 0
if row["service"] in ("api", "db", "redis"):
assert not row["ports"], "Storage/API published on host"
if row["service"] == "migrate":
assert row["state"] == "exited" and row["exit_code"] == 0
else:
assert row["state"] == "running" and row["health"] == "healthy"
with urllib.request.urlopen(
"http://127.0.0.1:8404/stats;csv", timeout=5
) as response:
stats = list(
csv.DictReader(io.StringIO(response.read().decode().removeprefix("# ")))
)
stats = [
{
k: row[k]
for k in (
"svname",
"status",
"addr",
"check_status",
"check_duration",
"scur",
"qcur",
)
}
for row in stats
if row["pxname"] == "api_replicas" and row["svname"].startswith("api")
]
assert len(stats) == expected_replicas and all(row["status"].startswith("UP") for row in stats)
addresses = {net["address"] for row in api for net in row["networks"].values()}
assert all(row["addr"].split(":")[0] in addresses for row in stats)
migration = command(
COMPOSE
+ [
"exec",
"-T",
"db",
"sh",
"-c",
'psql -X -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At -c "SELECT version,dirty FROM schema_migrations"',
]
)
assert migration == "3|f", "Expected clean migration version 3"
ignored = command(["git", "check-ignore", ".env", ".memory/"]).splitlines()
assert set(ignored) == {".env", ".memory/"}
assert "container_name:" not in ROOT.joinpath("compose.yml").read_text()
health = []
for path in ("/health/live", "/health/ready"):
with urllib.request.urlopen(
"http://127.0.0.1:8080" + path, timeout=5
) as response:
health.append(
{
"path": path,
"status": response.status,
"instance": response.headers.get("X-Instance-Id"),
}
)
print(
json.dumps(
{
"containers": rows,
"haproxy_backends": stats,
"health_through_lb": health,
"migration_version_dirty": migration,
"git_ignored": ignored,
"hook_scope": {
name: {
"sha256": hashlib.sha256(
ROOT.joinpath("scripts", name).read_bytes()
).hexdigest()
}
for name in ("restart.sh", "stop-one.sh")
},
},
ensure_ascii=False,
indent=2,
)
)
print(
"PASS: actual containers and HAProxy addresses agree; migration 3 clean; storage/API private; secrets ignored"
)
def experiment_summary():
result = {}
for name in SERIES:
directory = ROOT / "evidence/grade5" / name
analysis = json.loads(directory.joinpath("analysis.json").read_text())
groups = {}
for row in analysis["runs"]:
groups.setdefault((row["config"], row["rate"]), []).append(row)
measurements = []
for (config, rate), rows in sorted(groups.items()):
measurements.append(
{
"config": config,
"rate": rate,
"repeats": len(rows),
"medians": {
field: statistics.median(r[field] for r in rows)
for field in (
"p50_ms",
"p95_ms",
"p99_ms",
"window_goodput_rps",
"success_ratio",
"generator_cpu_cores",
"dispatch_p95_ms",
)
},
"p95_range_ms": [
min(r["p95_ms"] for r in rows),
max(r["p95_ms"] for r in rows),
],
"diagnostics": [
{"repeat": r["repeat"], **r["diagnostics"]} for r in rows
],
}
)
result[name] = {
"audit": json.loads(directory.joinpath("audit.json").read_text()),
"protocol": json.loads(directory.joinpath("protocol.json").read_text()),
"decisions": json.loads(
directory.joinpath(
"hypotheses.json" if name == "baseline-01" else "decision.json"
).read_text()
),
"capacities": analysis["capacities"],
"measurements": measurements,
}
print(
"Historical measurements from 2026-10-10; this command re-reads saved data, it does not run load."
)
print(json.dumps(result, ensure_ascii=False, indent=2))
class Recorder:
def __init__(self, run_dir):
self.directory = run_dir
self.records = {}
self.secrets = []
for path in (ROOT / ".env", ROOT / ".memory/benchmark/environment"):
if path.exists():
for line in path.read_text().splitlines():
if "=" in line and any(
word in line.split("=", 1)[0].upper()
for word in ("PASSWORD", "TOKEN", "SECRET")
):
value = line.split("=", 1)[1].strip().strip("\"'")
if value:
self.secrets.append(value)
def redact(self, value):
for secret in self.secrets:
value = value.replace(secret, "[REDACTED]")
value = re.sub(
r"(?i)(Bearer|Basic)\s+[A-Za-z0-9+/_.=:-]{16,}", r"\1 [REDACTED]", value
)
value = re.sub(
r'(?i)("(?:token|password)"\s*:\s*")[^"]+("\s*[,}])',
r"\1[REDACTED]\2",
value,
)
return value
def run(self, name, args, *, kind="fresh-check", timeout=600, artifacts=()):
record = {
"command": shlex.join(str(a) for a in args),
"kind": kind,
"started_utc": timestamp(),
}
raw = ROOT / ".memory" / (self.directory.name + "-" + name + "-output.txt")
raw.touch(mode=0o600)
try:
with raw.open("w") as file:
process = subprocess.Popen(
[str(a) for a in args],
cwd=ROOT,
stdout=file,
stderr=subprocess.STDOUT,
)
try:
record["exit_code"] = process.wait(timeout=timeout)
except BaseException:
process.terminate()
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
process.kill()
process.wait()
record["exit_code"] = process.returncode
raise
finally:
record["finished_utc"] = timestamp()
record["output"] = self.redact(raw.read_text(errors="replace"))
raw.unlink(missing_ok=True)
record.setdefault("exit_code", -1)
self.directory.joinpath(name + ".txt").write_text(record["output"])
record["log"] = str(
self.directory.joinpath(name + ".txt").relative_to(ROOT)
)
copied = []
for relative in artifacts:
source = ROOT / relative
if source.exists():
dest = self.directory / source.name
dest.write_text(self.redact(source.read_text()))
copied.append(str(dest.relative_to(ROOT)))
record["artifacts"] = copied
self.records[name] = record
print(f"{name}: exit {record['exit_code']}", flush=True)
return record["exit_code"] == 0
def link(path, output):
target = os.path.relpath(ROOT / path, output.parent).replace(os.sep, "/")
return f"[{path}]({quote(target, safe='/._-')})"
def fenced(value, language="text"):
longest = max((len(m) for m in re.findall(r"`+", value)), default=0)
fence = "`" * max(3, longest + 1)
return f"{fence}{language}\n{value.rstrip()}\n{fence}\n"
def write_evidence(recorder, metadata, *, write_manifest=True):
index = [
"# Доказательства критериев лабораторной 3",
"",
"Генерация: `python3 scripts/generate_evidence.py`. Функциональные проверки выполняются заново; данные исследования 2026-10-10 пересчитываются без нового нагрузочного эксперимента.",
"",
f"Запуск: `{metadata['run_id']}`. Команды, время и хеши исходников сохранены в manifest запуска. Ссылки на реализацию ведут к текущим файлам проекта.",
"",
"| Пункт | Статус проверки | Доказательство |",
"|---|---|---|",
]
for slug, title, description, jobs, sources in CRITERIA:
output = OUTPUT / (slug + ".md")
records = [recorder.records.get(job) for job in jobs]
status = (
"PASS"
if all(record and record["exit_code"] == 0 for record in records)
else "FAIL / НЕ ЗАВЕРШЕНО"
)
parts = [
f"# {title}",
"",
"## Что проверяется",
"",
description,
"",
f"Статус выполнения проверок: **{status}**. Для исследования PASS означает успешную проверку данных и правил анализа; решение гипотезы указано отдельно и может быть неопределённым.",
"",
f"Запуск: `{metadata['run_id']}`. Команды выполнены из корня проекта в WSL; время и версии указаны в manifest.",
"",
"## Команды и фактический вывод",
"",
]
for job, record in zip(jobs, records):
if not record:
parts.extend(
[
f"### {job}",
"",
"НЕ ВЫПОЛНЕНО: запуск был прерван либо предыдущая проверка завершилась ошибкой.",
"",
]
)
continue
label = (
"Пересчёт/проверка исторических измерений"
if record["kind"] == "historical-analysis"
else "Проверка приложения"
)
lines = record["output"].splitlines()
excerpt = len(lines) > 120
shown = (
"\n".join(lines[:70] + ["... [пропущенные строки — в полном журнале] ..."] + lines[-20:])
if excerpt else record["output"]
)
parts.extend(
[
f"### {job}",
"",
label + ".",
"",
fenced(record["command"], "sh"),
f"Начало UTC: `{record['started_utc']}`; конец UTC: `{record['finished_utc']}`; exit code: **{record['exit_code']}**.",
"",
"Фрагмент вывода; полный журнал доступен по ссылке ниже." if excerpt else "Вывод команды:",
"",
fenced(
shown
or "Команда не вывела текст; смотрите exit code и связанные JSON-результаты."
),
"Полный сохранённый вывод: " + link(record["log"], output) + ".",
"",
]
)
parts.extend("- " + link(path, output) for path in record["artifacts"])
parts.append("")
parts.extend(["## Реализация и дополнительные доказательства", ""])
parts.extend("- " + link(path, output) for path in sources)
parts.extend(
[
"- "
+ link(
str(recorder.directory.joinpath("manifest.json").relative_to(ROOT)),
output,
),
"",
]
)
output.write_text("\n".join(parts))
index.append(f"| {title} | {status} | [{slug}]({slug}.md) |")
OUTPUT.joinpath("README.md").write_text("\n".join(index) + "\n")
manifest = {
**metadata,
"commands": {
name: {key: value for key, value in rec.items() if key != "output"}
for name, rec in recorder.records.items()
},
}
if write_manifest:
recorder.directory.joinpath("manifest.json").write_text(
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n"
)
def generate(run_id):
ROOT.joinpath(".memory").mkdir(exist_ok=True)
with ROOT.joinpath(".memory/evidence.lock").open("w") as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
run_dir = OUTPUT / "runs" / run_id
run_dir.mkdir(parents=True, exist_ok=False)
recorder = Recorder(run_dir)
metadata = {
"run_id": run_id,
"started_utc": timestamp(),
"git_commit": command(["git", "rev-parse", "HEAD"]),
"branch": command(["git", "branch", "--show-current"]),
"working_tree_status": command(["git", "status", "--short"]),
"grade5_measurement_date": "2026-10-10",
"new_load_experiment": False,
}
sources = set(path for criterion in CRITERIA for path in criterion[4]) | {
"scripts/generate_evidence.py"
}
metadata["file_sha256"] = {
path: hashlib.sha256(ROOT.joinpath(path).read_bytes()).hexdigest()
for path in sorted(sources)
if ROOT.joinpath(path).is_file()
}
passed = True
try:
jobs = [
("startup", ["make", "app-up"], ()),
("structure", ["make", "check"], ()),
(
"inventory",
["python3", "scripts/generate_evidence.py", "--inspect"],
(),
),
(
"resources",
["python3", "scripts/check_resources.py"],
(".memory/grade4/resources.json",),
),
("smoke", ["make", "test"], ()),
("backend", ["make", "test-backend"], ()),
(
"vet",
COMPOSE
+ [
"run",
"--rm",
"--no-deps",
"backend-tests",
"go",
"vet",
"./backend/...",
],
(),
),
(
"frontend-build",
COMPOSE + ["exec", "-T", "frontend", "npm", "run", "build"],
(),
),
("idempotency", ["python3", "scripts/check_idempotency.py"], ()),
(
"failover",
["python3", "scripts/check_failover.py"],
(".memory/grade4/failover.json",),
),
(
"rolling",
["python3", "scripts/check_rolling.py"],
(".memory/grade4/rolling.json",),
),
(
"persistence",
["make", "test-persistence", "ACTION_SCRIPT=scripts/restart.sh"],
(),
),
(
"public-failover",
["make", "test-failover", "ACTION_SCRIPT=scripts/stop-one.sh"],
(),
),
]
for name, args, artifacts in jobs:
if not recorder.run(name, args, artifacts=artifacts):
passed = False
break
if passed:
if not BENCH.exists():
passed = recorder.run(
"analysis-venv",
["python3", "-m", "venv", str(BENCH.parent.parent)],
)
if passed:
passed = recorder.run(
"analysis-dependencies",
[
str(BENCH),
"-m",
"pip",
"install",
"-r",
"experiments/lab3-scale/requirements-lock.txt",
],
)
for series, suffix, mode in zip(
SERIES, ("baseline", "hi", "hc", "hl"), ("audit", "hi", "hc", "hl")
):
if not passed:
break
passed = recorder.run(
"analyze-" + suffix,
[str(BENCH), "experiments/lab3-scale/analyze.py", series],
kind="historical-analysis",
)
if passed:
passed = recorder.run(
"audit-" + suffix,
[
str(BENCH),
"experiments/lab3-scale/postprocess/decisions.py",
mode,
series,
],
kind="historical-analysis",
)
if passed:
passed = recorder.run(
"plot-hl",
[
str(BENCH),
"experiments/lab3-scale/postprocess/plot_latency.py",
"latency-improvement-01",
],
kind="historical-analysis",
)
if passed:
passed = recorder.run(
"experiment-summary",
[
"python3",
"scripts/generate_evidence.py",
"--experiment-summary",
],
kind="historical-analysis",
)
except BaseException:
passed = False
raise
finally:
# The public failover hook intentionally leaves a replica stopped.
# Restore only this lab's application, preserving all volumes.
restored = recorder.run("restore", COMPOSE + ["up", "-d", "--wait"])
final = recorder.run(
"final-inventory",
["python3", "scripts/generate_evidence.py", "--inspect"],
)
metadata["finished_utc"] = timestamp()
metadata["all_commands_passed"] = passed and restored and final
write_evidence(recorder, metadata)
print("Evidence: evidence/criteria/README.md", flush=True)
return 0 if metadata["all_commands_passed"] else 1
def main():
parser = argparse.ArgumentParser(__doc__)
parser.add_argument(
"--run-id",
default=dt.datetime.now(dt.timezone.utc).strftime("lab3-%Y%m%dT%H%M%S%fZ"),
)
parser.add_argument("--inspect", action="store_true", help=argparse.SUPPRESS)
parser.add_argument(
"--render", metavar="RUN_ID",
help="Update Markdown from saved logs without running checks",
)
parser.add_argument(
"--experiment-summary", action="store_true", help=argparse.SUPPRESS
)
args = parser.parse_args()
if args.render:
if not re.fullmatch(r"[A-Za-z0-9_-]+", args.render):
parser.error("render run-id must contain only letters, digits, '_' or '-'")
from types import SimpleNamespace
directory = OUTPUT / "runs" / args.render
metadata = json.loads(directory.joinpath("manifest.json").read_text())
records = metadata.pop("commands")
for record in records.values():
record["output"] = ROOT.joinpath(record["log"]).read_text()
write_evidence(
SimpleNamespace(directory=directory, records=records), metadata,
write_manifest=False,
)
return 0
if args.inspect:
inspect_state()
return 0
if args.experiment_summary:
experiment_summary()
return 0
if not re.fullmatch(r"[A-Za-z0-9_-]+", args.run_id):
parser.error("run-id must contain only letters, digits, '_' or '-'")
if not sys.platform.startswith("linux"):
parser.error("Run inside Ubuntu WSL/Linux")
return generate(args.run_id)
if __name__ == "__main__":
sys.exit(main())