Files
mtusi2026lab4/scripts/generate_evidence.py
T
2026-10-19 00:20:43 +03:00

364 lines
22 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Run criterion checks and publish sanitized Markdown evidence; never commit."""
import argparse
import datetime as dt
import fcntl
import hashlib
import json
import os
from pathlib import Path
import platform
import re
import shlex
import statistics
import subprocess
import sys
import tarfile
import time
ROOT = Path(__file__).resolve().parents[1]
OUTPUT = ROOT / "evidence/criteria"
COMPOSE = ["docker", "compose", "--project-name", "highload-lab4",
"--env-file", ".env", "-f", "compose.yml"]
# One file per criterion in REPORT.md. Shared checks are executed only once.
CRITERIA = [
("public-smoke", "Публичный smoke и структура контракта", ["structure", "smoke"], [],
["tests/smoke.py", "contracts/openapi.json"]),
("baseline", "Возможности lab3", ["baseline", "backend"],
["TestStoredPasswordUsesArgon2id", "TestShortSessionTTLAndLogout", "TestIdempotencyConcurrentStoresAndScopes", "TestChatTransactionRollsBackAfterMemberInsertFailure", "TestListMessagesPagination"],
["src/backend/repositories/idempotency.go", "src/backend/repositories/redis.go", "src/backend/services/password.go"]),
("shared-storage", "Две API, PostgreSQL и приватный S3 в общем стенде", ["startup", "inventory", "schema", "isolation"], [],
["compose.yml", "infra/minio.Dockerfile", "src/backend/repositories/migrations/000004_attachments.up.sql"]),
("proxy-attachments", "PNG/JPEG, ready, сообщение и побайтный download", ["proxy", "backend"],
["TestAttachmentAccessHTTP"], ["src/backend/services/attachment.go", "src/backend/handlers/attachment_http.go", "src/backend/repositories/attachment.go"]),
("validation-access", "Лимиты, MIME, filename и доступ к чужим вложениям", ["proxy", "backend"],
["TestAttachmentValidationHTTP", "TestAttachmentImageValidation", "TestAttachmentFilenameValidation", "TestAttachmentAccessHTTP"],
["src/backend/tests/attachment_http_test.go", "src/backend/services/attachment.go"]),
("replica-failover", "Оригинал после остановки обслужившей API", ["proxy-failures", "direct-failures", "public-failover", "restore"], [],
["scripts/check_grade3.py", "scripts/check_direct_upload.py", "scripts/stop-one.sh"]),
("persistence", "Сессия, метаданные и оригинал после restart", ["before-restart", "persistence", "after-restart", "restart-audit", "direct-failures"], [],
["scripts/restart.sh", "tests/resilience.py"]),
("upload-recovery", "Обрыв HTTP и отказ между S3 PUT и COMMIT", ["backend"],
["TestUploadRecoveryHTTPDisconnect", "TestUploadRecoveryAfterPutFailure", "TestUploadRecoveryMetadataRejection", "TestUploadRecoveryDatabaseConnectionLoss", "TestUploadRecoveryProcessExit"],
["src/backend/repositories/upload.go", "src/backend/tests/upload_recovery_test.go"]),
("safe-cleanup", "Grace period, блокировки, защита готовых объектов и повтор", ["backend", "cleanup-dry-run"],
["TestAttachmentCleanupGraceLocksProtectionAndRetry", "TestAttachmentCleanupInvalidKeyIsNeverDeleted"],
["src/backend/cleanup.go", "src/backend/repositories/cleanup_config.go", "src/backend/repositories/upload.go"]),
("bounded-memory", "Потоковая обработка, лимиты и ≥10 клиентов", ["inventory", "experiment"], [],
["src/backend/services/attachment.go", "src/backend/services/direct_upload.go", "experiments/config/protocol.json"]),
("s3-permissions", "Повторяемый bootstrap и ограниченные credentials", ["backend", "isolation"],
["TestS3PermissionsAndBootstrap", "TestDirectUploadAccessPermissionsAndLifecycle"],
["src/backend/repositories/s3_admin.go", "scripts/check_s3_isolation.py"]),
("direct-flow", "Presigned POST и неизменяемый готовый оригинал", ["direct", "backend"],
["TestDirectUploadFlow", "TestDirectUploadRecoveryConcurrentComplete"],
["src/backend/services/direct_upload.go", "src/backend/repositories/s3_direct.go"]),
("direct-validation-access", "Подпись, владелец, размер, SHA-256, MIME и CORS", ["backend"],
["TestDirectUploadValidationHTTP", "TestDirectUploadValidationContents", "TestDirectUploadAccess", "TestDirectUploadAccessCORS"],
["src/backend/handlers/direct_upload_http.go", "src/backend/tests/direct_upload_test.go", "src/backend/tests/direct_permissions_test.go"]),
("direct-recovery", "Повторы complete, crash и отмена ожидания", ["backend", "direct-failures"],
["TestDirectUploadRecoveryFailures", "TestDirectUploadRecoveryProcessExit", "TestDirectUploadRecoverySharedSlotAndCancellation"],
["src/backend/repositories/direct_upload.go", "src/frontend/src/directUpload.ts"]),
("direct-cleanup", "Незавершённый staging, replay, dry-run и lifecycle", ["backend", "cleanup-dry-run"],
["TestDirectUploadCleanupSafetyAndReplay", "TestDirectUploadCleanupDependencyFailureAndMalformedKey", "TestDirectUploadAccessPermissionsAndLifecycle"],
["src/backend/repositories/direct_cleanup.go", "src/backend/repositories/s3_admin.go"]),
("upload-comparison", "Гипотеза и сравнение трафика/RAM на четырёх профилях", ["experiment"], [],
["experiments/docs/HYPOTHESIS.md", "experiments/docs/METHODOLOGY.md", "experiments/docs/LIMITATIONS.md", "experiments/run.py"]),
("frontend", "Сборка React UI", ["frontend-build"], [],
["src/frontend/src/App.tsx", "src/frontend/src/AttachmentImage.tsx", "src/frontend/src/directUpload.ts"]),
]
def load(path):
return json.loads(path.read_text())
def digest(path):
h = hashlib.sha256()
with path.open("rb") as stream:
for block in iter(lambda: stream.read(65536), b""):
h.update(block)
return h.hexdigest()
def sanitize(text):
# Redact actual local secrets as well as common credential representations.
values = []
for path in (ROOT / "secrets").glob("*"):
if path.is_file():
values.append(path.read_text().strip())
env = ROOT / ".env"
if env.exists():
values += [line.split("=", 1)[1].strip().strip("\"'")
for line in env.read_text().splitlines()
if line.startswith("POSTGRES_PASSWORD=")]
for value in sorted(set(values), key=len, reverse=True):
if value:
text = text.replace(value, "[REDACTED]")
text = re.sub(r"(?i)(Bearer|Basic)\s+[A-Za-z0-9+/=_.-]+", r"\1 [REDACTED]", text)
text = re.sub(r"(?i)(postgres(?:ql)?|redis)://[^\s\"']+", r"\1://[REDACTED]", text)
text = re.sub(r'(?i)("(?:token|password|secret_access_key|x-amz-signature|x-amz-credential|policy)"\s*:\s*)"[^"]*"', r'\1"[REDACTED]"', text)
return text
def capture(args):
return subprocess.check_output(args, cwd=ROOT, text=True, stderr=subprocess.STDOUT)
def inventory():
ids = capture(COMPOSE + ["ps", "--all", "-q"]).splitlines()
assert ids, "stack is absent"
result = []
for c in json.loads(capture(["docker", "inspect", *ids])):
if c["Config"]["Labels"].get("com.docker.compose.oneoff") == "True":
continue
result.append({
"id": c["Id"], "hostname": c["Config"]["Hostname"],
"service": c["Config"]["Labels"]["com.docker.compose.service"],
"image": c["Image"], "state": c["State"]["Status"],
"health": c["State"].get("Health", {}).get("Status"),
"exit_code": c["State"]["ExitCode"], "started_at": c["State"]["StartedAt"],
"restarts": c["RestartCount"], "memory": c["HostConfig"]["Memory"],
"nano_cpus": c["HostConfig"]["NanoCpus"],
"ports": c["HostConfig"]["PortBindings"],
"tmpfs": c["HostConfig"].get("Tmpfs", {}),
"volumes": sorted(m["Name"] for m in c["Mounts"] if m["Type"] == "volume"),
})
api = [c for c in result if c["service"] == "api"]
assert len(api) == 2 and all(c["health"] == "healthy" for c in api)
assert all(c["memory"] == 512 * 1048576 and not c["volumes"] for c in api)
assert all(not c["ports"] for c in api)
for service in ("db", "redis", "s3"):
assert any(c["service"] == service and c["volumes"] and c["health"] == "healthy" for c in result)
for service in ("migrate", "s3-bootstrap"):
assert any(c["service"] == service and c["state"] == "exited" and c["exit_code"] == 0 for c in result)
print(json.dumps(result, indent=2))
def audit_restart(before, after):
a, b = load(before), load(after)
for role in ("db", "redis", "s3", "api", "upload-cleanup"):
old, new = [c for c in a if c["service"] == role], [c for c in b if c["service"] == role]
assert old and len(old) == len(new)
assert {c["id"] for c in old} == {c["id"] for c in new}
for first in old:
second = next(c for c in new if c["id"] == first["id"])
assert first["started_at"] != second["started_at"], role + " was not restarted"
assert first["volumes"] == second["volumes"], role + " changed volumes"
print("PASS: real restart " + role + "; same containers and persistent volumes")
def audit_experiment(campaign):
campaign = campaign.resolve()
assert campaign.parent == (ROOT / "evidence/upload-comparison").resolve()
sys.path.insert(0, str(ROOT / "experiments/src"))
from common import check_seal
from analysis import case_metrics, decision, json_lines
check_seal(campaign)
protocol, plan, manifest = load(campaign / "protocol.json"), load(campaign / "plan.json"), load(campaign / "source-manifest.json")
assert protocol["purpose"] == "confirmatory" and protocol["clients"] >= 10
assert len({p["size_bytes"] for p in protocol["profiles"]}) >= 2
# Verify archived code, not today's edited docs/plotter, against the frozen manifest.
with tarfile.open(campaign / "source.tar.gz") as archive:
for name, expected in manifest.items():
member = archive.extractfile(name)
assert member and hashlib.sha256(member.read()).hexdigest() == expected, name
for name in ("experiments/src/analysis.py", "experiments/src/common.py"):
assert digest(ROOT / name) == manifest[name], "use frozen analysis code"
fixtures = {f["name"]: f for f in load(campaign / "fixtures.json")}
metrics = []
for case in plan:
folder = campaign / "cases" / case["directory"]
recorded = load(folder / "case.json")
assert all(recorded.get(key) == value for key, value in case.items())
requests, checks = json_lines(folder / "requests.jsonl"), json_lines(folder / "verification.jsonl")
assert len(requests) == protocol["clients"] * protocol["waves"]
assert len(json_lines(folder / "warmup.jsonl")) == protocol["warmup_clients"]
ids = [r["attachment"]["id"] for r in requests if r["ready"]]
assert len(set(ids)) == len(ids) and sorted(ids) == sorted(v["id"] for v in checks)
for v in checks:
assert v["verified"] and v["size_bytes"] == case["size_bytes"]
assert v["sha256"] == fixtures[case["profile"]]["sha256"]
metrics.append(case_metrics(folder, protocol))
saved = load(campaign / "metrics.json")
actual = decision(protocol, metrics, plan)
assert saved["cases"] == metrics and saved["decision"] == actual, "summary differs from raw evidence"
assert not (campaign / "execution-failure.json").exists()
assert actual["criterion_status"] == "confirmed_in_tested_conditions", actual["criterion_status"]
# The running backend must match the one actually measured.
changed_backend = [n for n, h in manifest.items() if n.startswith(("src/backend/", "src/go.", "src/Dockerfile", "infra/minio.Dockerfile")) and digest(ROOT / n) != h]
assert not changed_backend, "measured backend differs from current backend"
print("PASS: sealed protocol, archive, fixtures, raw records and recomputed decision match")
print("Campaign: " + str(campaign.relative_to(ROOT)))
print("Ready/verified:", sum(c["ready"] for c in metrics), sum(c["verified"] for c in metrics))
print("Errors:", sum(c["errors"] for c in metrics))
for profile in actual["profiles"]:
print(json.dumps(profile))
for name in [p["name"] for p in protocol["profiles"]]:
for mode in ("proxy", "direct"):
cases = [c for c in metrics if c["profile"] == name and c["mode"] == mode]
print(json.dumps({"profile": name, "mode": mode,
"median_api_MiB_per_ready": statistics.median(c["api_bytes_per_ready"] for c in cases) / 1048576,
"median_api_window_peak_MiB": statistics.median(c["api_sampled_peak_window_max"] for c in cases) / 1048576,
"max_api_lifetime_peak_MiB": max(c["api_peak_lifetime_max"] for c in cases) / 1048576,
"median_ready_rps": statistics.median(c["ready_rps"] for c in cases),
"median_p50_s": statistics.median(c["latency_seconds"]["p50"] for c in cases),
"median_p95_s": statistics.median(c["latency_seconds"]["p95"] for c in cases),
"median_p99_s": statistics.median(c["latency_seconds"]["p99"] for c in cases)}))
print("Decision:", actual["criterion_status"])
def report_ids():
return re.findall(r"\]\(evidence/criteria/([a-z0-9-]+)\.md\)", (ROOT / "REPORT.md").read_text())
def write_pages(directory, results, campaign):
status = {}
for key, title, checks, names, sources in CRITERIA:
records = [results[n] for n in checks]
missing = [n for n in names if "--- PASS: " + n + " (" not in results["backend"]["output"]]
passed = all(r["exit_code"] == 0 for r in records) and not missing
status[key] = "PASS" if passed else "FAIL / НЕ ДОКАЗАНО"
lines = ["# " + title, "", "[Отчёт](../../REPORT.md) · [Все критерии](README.md)", "",
"Статус: **" + status[key] + "**. Запуск: `" + directory.name + "`.", "",
"Тесты создают синтетические данные. Интеграционные Go-тесты используют отдельные схемы БД и собственные S3-ключи.", ""]
if names:
lines += ["Проверенные сценарии: " + ", ".join("`" + n + "`" for n in names) + ".", ""]
if missing:
lines += ["Не найдены успешные результаты: " + ", ".join(missing), ""]
for check in checks:
r = results[check]
lines += ["## " + check, "", "```sh", r["command"], "```", "",
f"Начало: {r['started_utc']}; длительность: {r['seconds']:.3f} с; exit code: {r['exit_code']}.", ""]
if check == "backend" and names:
shown = "\n".join(l for l in r["output"].splitlines() if any(n in l for n in names) or l.startswith(("FAIL", "ok\t", "PASS")))
lines += ["Фрагмент; полный вывод: [backend.txt](runs/" + directory.name + "/backend.txt).", ""]
else:
shown = r["output"]
lines += ["````text", shown.strip(), "````", ""]
lines += ["## Реализация", "", *["- [" + s + "](../../" + s + ")" for s in sources], ""]
if "experiment" in checks:
rel = str(campaign.relative_to(ROOT))
lines += ["Измерения не запускались повторно: проверены сохранённые сырые данные.", "",
"[Серия](../../" + rel + "/metrics.json) · [Протокол](../../" + rel + "/protocol.json) · [Графики](../../" + rel + "/plots/)", ""]
text = "\n".join(lines)
(OUTPUT / (key + ".md")).write_text(text)
history = text.replace("](../../", "](../../../../")
history = history.replace("](README.md)", "](../../README.md)")
history = history.replace("](runs/" + directory.name + "/backend.txt)", "](backend.txt)")
(directory / (key + ".md")).write_text(history)
index = ["# Доказательства критериев", "", "[REPORT.md](../../REPORT.md)", "", "Последний запуск: `" + directory.name + "`. Предыдущие попытки сохранены в `runs/`.", "", "| Критерий | Результат |", "| --- | --- |"]
index += [f"| [{title}]({key}.md) | {status[key]} |" for key, title, *_ in CRITERIA]
(OUTPUT / "README.md").write_text("\n".join(index) + "\n")
return status
def generate(campaign):
assert set(report_ids()) == {c[0] for c in CRITERIA}, "REPORT.md criteria and generator differ"
OUTPUT.mkdir(parents=True, exist_ok=True)
with (OUTPUT / ".lock").open("a") as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
# Never run fault checks alongside an experiment campaign.
with (ROOT / "evidence/upload-comparison/.lock").open("a") as experiment_lock:
fcntl.flock(experiment_lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
return run_checks(campaign)
def run_checks(campaign):
stamp = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ")
directory = OUTPUT / "runs" / stamp
directory.mkdir(parents=True)
results = {}
self_command = [sys.executable, "scripts/generate_evidence.py", "--probe"]
def run(name, args, timeout=900):
print("Running " + name, flush=True)
started, clock = dt.datetime.now(dt.timezone.utc).isoformat(), time.monotonic()
try:
proc = subprocess.run(args, cwd=ROOT, text=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, timeout=timeout)
output, code = proc.stdout, proc.returncode
except subprocess.TimeoutExpired as error:
output = error.stdout or b""
if isinstance(output, bytes):
output = output.decode(errors="replace")
output += "\nTIMEOUT: check did not finish within the declared limit."
code = 124
except OSError as error:
output, code = type(error).__name__, 127
result = dict(command=shlex.join(args), started_utc=started,
seconds=time.monotonic() - clock, exit_code=code, output=sanitize(output))
results[name] = result
(directory / (name + ".txt")).write_text(result["output"])
(directory / "commands.json").write_text(json.dumps(results, indent=2) + "\n")
print(name + ": " + ("PASS" if code == 0 else "FAIL"), flush=True)
return result
try:
run("startup", ["make", "app-up"], 1800)
run("inventory", self_command + ["inventory"])
run("schema", COMPOSE + ["exec", "-T", "db", "sh", "-c", 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "SELECT version, dirty FROM schema_migrations;"'])
run("structure", ["make", "check"])
run("smoke", ["make", "test"])
run("baseline", ["make", "test-baseline"])
run("backend", ["make", "test-backend"], 1800)
run("proxy", ["make", "test-grade3"])
run("direct", [sys.executable, "scripts/check_direct_upload.py"])
run("isolation", [sys.executable, "scripts/check_s3_isolation.py"])
run("cleanup-dry-run", ["make", "attachments-cleanup"])
run("frontend-build", COMPOSE + ["exec", "-T", "frontend", "npm", "run", "build"])
run("proxy-failures", ["make", "test-grade3-failures"])
run("before-restart", self_command + ["inventory"])
run("persistence", ["make", "test-persistence", "ACTION_SCRIPT=scripts/restart.sh"])
run("after-restart", self_command + ["inventory"])
run("restart-audit", self_command + ["restart", "--before", str(directory / "before-restart.txt"), "--after", str(directory / "after-restart.txt")])
run("direct-failures", [sys.executable, "scripts/check_direct_upload.py", "--failures", "--restart"])
run("public-failover", ["make", "test-failover", "ACTION_SCRIPT=scripts/stop-one.sh"])
run("experiment", self_command + ["experiment", "--campaign", str(campaign)])
finally:
run("restore", ["make", "app-start"])
run("final-inventory", self_command + ["inventory"])
for _, _, checks, _, _ in CRITERIA:
for check in checks:
results.setdefault(check, dict(command="not executed", started_utc=stamp,
seconds=0, exit_code=125, output="NOT RUN"))
state = write_pages(directory, results, campaign)
meta = {"run": stamp, "python": platform.python_version(), "platform": platform.platform(),
"script_sha256": digest(Path(__file__)), "report_sha256": digest(ROOT / "REPORT.md"),
"campaign": str(campaign.relative_to(ROOT)), "criteria": state,
"restored": results["restore"]["exit_code"] == results["final-inventory"]["exit_code"] == 0}
(directory / "summary.json").write_text(json.dumps(meta, indent=2) + "\n")
print("Evidence: " + str(OUTPUT.relative_to(ROOT)))
return 0 if all(v == "PASS" for v in state.values()) and meta["restored"] else 1
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--campaign", type=Path, help="completed confirmatory campaign; otherwise latest completed")
parser.add_argument("--probe", choices=["inventory", "restart", "experiment"])
parser.add_argument("--before", type=Path)
parser.add_argument("--after", type=Path)
args = parser.parse_args()
if args.probe == "inventory":
inventory()
elif args.probe == "restart":
audit_restart(args.before, args.after)
else:
campaign = args.campaign
if campaign is None:
candidates = sorted(d for d in (ROOT / "evidence/upload-comparison").iterdir()
if (d / "metrics.json").exists()
and load(d / "protocol.json")["purpose"] == "confirmatory")
if not candidates:
parser.error("no completed formal campaign; pass --campaign")
campaign = candidates[-1]
campaign = campaign.resolve()
if args.probe == "experiment":
audit_experiment(campaign)
else:
raise SystemExit(generate(campaign))
if __name__ == "__main__":
main()