Files
mtusi2026lab5-fork/scripts/generate_evidence.py
T

243 lines
14 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Run lab5 checks and write sanitized criterion evidence. Run in WSL."""
import argparse
import datetime as dt
import fcntl
import hashlib
import ast
import json
import os
from pathlib import Path
import re
import shlex
import statistics
import subprocess
import sys
import time
import zipfile
from local_config import api_base_url
ROOT = Path(__file__).resolve().parents[1]
OUTPUT = ROOT / "evidence/criteria"
COMPOSE = ["docker", "compose", "--project-name", "highload-lab5", "--env-file", ".env", "-f", "compose.yml"]
CRITERIA = [
("public-smoke", "Публичный smoke", ["structure", "smoke"], "tests/smoke.py"),
("baseline", "Сохранение возможностей lab4", ["backend", "persistence", "failover"], "src/backend/tests"),
("durable-upload", "Устойчивый приём до 202", ["grade3"], "src/backend/repositories/jobs.go"),
("broker-worker", "Отдельные RabbitMQ и worker, durable delivery и ack", ["inventory", "grade3"], "src/backend/repositories/broker.go"),
("states-variants", "Состояния, оригинал и thumbnail", ["smoke", "grade3"], "src/backend/workers/attachment.go"),
("retry-idempotency", "Ограниченный retry и повтор job", ["grade3", "grade4"], "src/backend/repositories/jobs.go"),
("restart-recovery", "Worker off, restart API/broker, восстановление", ["grade3", "persistence", "restart-audit"], "scripts/restart.sh"),
("variants-limits", "Crop, WebP, валидация и лимиты worker", ["grade4", "backend"], "src/backend/workers/image_process.go"),
("outbox", "Восстановление между COMMIT и publish", ["grade4", "experiment"], "src/backend/repositories/jobs.go"),
("retry-replay", "Backoff/jitter, failed registry и безопасный replay", ["grade4"], "src/backend/repositories/job_admin.go"),
("lease-cleanup", "Crash после side effect, lease, prefetch и staging", ["grade4", "experiment"], "src/backend/workers/pipeline.go"),
("versioned-pipeline", "Трек 2: versioned pipeline и fault injection", ["experiment", "backend"], "src/backend/repositories/pipeline_versions.go"),
]
def load(path):
return json.loads(path.read_text())
def sha(data):
return hashlib.sha256(data).hexdigest()
def sanitize(value):
for path in (ROOT / "secrets").glob("*"):
if path.is_file():
secret = path.read_text().strip()
if secret:
value = value.replace(secret, "[REDACTED]")
value = re.sub(r"(?i)(Bearer|Basic)\s+[A-Za-z0-9+/=_.-]+", r"\1 [REDACTED]", value)
return re.sub(r"(?i)(amqps?|postgres(?:ql)?|redis)://[^\s\"']+", r"\1://[REDACTED]", value)
def inventory():
ids = subprocess.check_output(COMPOSE + ["ps", "-aq"], cwd=ROOT, text=True).split()
assert ids, "No containers in highload-lab5"
containers = json.loads(subprocess.check_output(["docker", "inspect", *ids], text=True))
rows = []
for c in containers:
assert c["Config"]["Labels"]["com.docker.compose.project"] == "highload-lab5"
rows.append(dict(id=c["Id"], service=c["Config"]["Labels"]["com.docker.compose.service"],
image=c["Image"], status=c["State"]["Status"],
started=c["State"]["StartedAt"], oom=c["State"]["OOMKilled"],
memory=c["HostConfig"]["Memory"], nano_cpus=c["HostConfig"]["NanoCpus"],
health=c["State"].get("Health", {}).get("Status"),
mounts=[dict(type=m["Type"], target=m["Destination"], name=m.get("Name")) for m in c["Mounts"]]))
assert sum(r["service"] == "api" and r["health"] == "healthy" for r in rows) == 2
for service in ("db", "redis", "s3", "broker", "worker", "dispatcher"):
assert any(r["service"] == service and r["status"] == "running" for r in rows), service
assert not any(r["oom"] for r in rows)
print(json.dumps(rows, indent=2))
def audit_experiment(campaign):
assert campaign.parent == (ROOT / "evidence/idempotency").resolve()
frozen = load(campaign / "seal/manifest.json")
with zipfile.ZipFile(campaign / "seal/source.zip") as archive:
for name, expected in frozen["files"].items():
assert sha(archive.read(name)) == expected, name
analysis_source = archive.read("experiments/run.py")
# Run the sealed analysis function; changes to packaging/Git rules must not
# invalidate a completed experiment or rewrite its original summary.
tree = ast.parse(analysis_source)
function = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "analyze")
namespace = {"json": json, "dump": lambda *args: None}
exec(compile(ast.Module(body=[function], type_ignores=[]), "<sealed analyze>", "exec"), namespace)
summary = namespace["analyze"](campaign)
sys.path.insert(0, str(ROOT / "experiments/src"))
assert summary == load(campaign / "summary.json"), "Saved summary differs from raw data"
assert summary["outcome"] == "supported_in_tested_conditions", summary
assert sha((campaign / "protocol.json").read_bytes()) == frozen["protocol_sha256"]
manifest = load(campaign / "manifest.json")
assert manifest["protocol_sha256"] == frozen["protocol_sha256"]
from driver import webp_size
from image_probe import jpeg_size
records = [json.loads(line) for line in (campaign / "cases.jsonl").read_text().splitlines()]
objects = 0
for row in records:
folder = (campaign / row["directory"]).resolve()
assert folder.is_relative_to(campaign / "cases")
assert load(folder / "result.json") == row
final = load(folder / "database-final.json")
assert final["status"] == "ready" and final["desired"] == final["published"] == row["version"]
versions = [p["version"] for p in final["publications"]]
assert versions == sorted(set(versions)) and versions[-1] == row["version"]
events = [json.loads(line) for line in (folder / "events.jsonl").read_text().splitlines()]
assert any(e["kind"] == "accepted_202" for e in events)
assert any(e["kind"] == "case_complete" for e in events)
for path in folder.glob("verify-*-manifest.json"):
observation = path.name.removesuffix("-manifest.json")
version = int(observation.rsplit("v", 1)[1])
publication = next(p for p in final["publications"] if p["version"] == version)
hashes = load(path)
assert len(hashes) == 3
for key, checksum in hashes.items():
data = (folder / "objects" / observation / Path(key).name).read_bytes()
name = Path(key).stem
metadata = next(v for v in publication["variants"] if v["name"] == name)
assert sha(data) == checksum == metadata["sha256"]
assert len(data) == metadata["size_bytes"]
dimensions = webp_size(data) if name == "optimized" else jpeg_size(data)
assert dimensions == (metadata["width"], metadata["height"])
objects += 1
assert row["manifest"] == load(sorted(folder.glob("verify-*-manifest.json"))[-1])
for repeat in range(1, 6):
environment = load(campaign / f"environment-r{repeat}.json")
for service in ("api", "worker"):
rows = [r for r in environment if r["service"] == service]
assert len(rows) == 2 and all(r["image"] == manifest["application_image"] for r in rows)
print(json.dumps(dict(campaign=str(campaign.relative_to(ROOT)), completed=len(records),
checked_variant_files=objects, outcome=summary["outcome"],
violations=summary["violations"], oom=summary["oom_events"],
protocol_sha256=frozen["protocol_sha256"], source_archive_verified=True), indent=2))
print("\n| Сценарий | Случаи | Время сценария min / median / max, с |\n| --- | --- | --- |")
for scenario in load(campaign / "protocol.json")["scenarios"]:
times = [r["elapsed_seconds"] for r in records if r["scenario"] == scenario]
print(f"| {scenario} | {len(times)} | {min(times):.2f} / {statistics.median(times):.2f} / {max(times):.2f} |")
def generate(campaign):
OUTPUT.mkdir(parents=True, exist_ok=True)
lock = (ROOT / "evidence/idempotency/.run.lock").open("a+")
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
stamp = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ")
directory = OUTPUT / "runs" / stamp
directory.mkdir(parents=True)
results = {}
def run(name, command, timeout=900):
print(name, flush=True)
start = time.monotonic()
try:
result = subprocess.run(command, cwd=ROOT, text=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, timeout=timeout)
code, output = result.returncode, result.stdout
except subprocess.TimeoutExpired as exc:
code, output = 124, (exc.stdout or b"").decode() if isinstance(exc.stdout, bytes) else (exc.stdout or "")
output = sanitize(output)
if name == "backend" and ("--- SKIP:" in output or "--- PASS:" not in output):
code = code or 1
output += "\nBackend evidence requires executed tests without skips.\n"
results[name] = dict(command=shlex.join(command), exit_code=code, seconds=round(time.monotonic()-start, 3))
(directory / (name + ".txt")).write_text(output)
(directory / (name + ".json")).write_text(json.dumps(results[name], indent=2) + "\n")
return code
self_cmd = [sys.executable, str(Path(__file__).resolve())]
try:
run("structure", ["make", "check"])
run("experiment", self_cmd + ["--probe", "experiment", "--campaign", str(campaign)])
if run("startup", COMPOSE + ["up", "-d", "--build", "--wait"], 1800) == 0:
run("inventory", self_cmd + ["--probe", "inventory"])
run("smoke", ["make", "test", "BASE_URL=" + api_base_url()])
run("backend", COMPOSE + ["--profile", "test", "run", "--rm", "--build", "backend-tests"], 1800)
run("grade3", [sys.executable, "scripts/check_async_grade3.py"])
run("grade4", [sys.executable, "scripts/check_async_grade4.py"])
run("before-restart", self_cmd + ["--probe", "inventory"])
run("persistence", ["make", "test-persistence", "BASE_URL=" + api_base_url(), "ACTION_SCRIPT=scripts/restart.sh"])
run("after-restart", self_cmd + ["--probe", "inventory"])
before = {r["id"]: r for r in load(directory / "before-restart.txt")} if results["before-restart"]["exit_code"] == 0 else {}
after = {r["id"]: r for r in load(directory / "after-restart.txt")} if results["after-restart"]["exit_code"] == 0 else {}
services = {"db", "redis", "s3", "broker", "api", "worker", "dispatcher", "upload-cleanup"}
audited = [r for r in before.values() if r["service"] in services]
passed = bool(audited) and services <= {r["service"] for r in audited} and all(
r["id"] in after and after[r["id"]]["started"] != r["started"] and after[r["id"]]["mounts"] == r["mounts"] for r in audited)
results["restart-audit"] = dict(command="compare before-restart / after-restart: StartedAt and mounts", exit_code=0 if passed else 1, seconds=0)
(directory / "restart-audit.txt").write_text(json.dumps(dict(processes_restarted_with_same_mounts=passed, containers=len(audited)), indent=2))
run("failover", ["make", "test-failover", "BASE_URL=" + api_base_url(), "ACTION_SCRIPT=scripts/stop-one.sh"])
finally:
run("restore", COMPOSE + ["up", "-d", "--wait"], 300)
run("final-inventory", self_cmd + ["--probe", "inventory"])
index = ["# Доказательства критериев", "", f"Запуск: `{stamp}`. Серия: [{campaign.name}](../idempotency/{campaign.name}/summary.json).", "", "| Критерий | Результат |", "| --- | --- |"]
states = {}
for slug, title, checks, source in CRITERIA:
checks = ["startup", *checks, "restore", "final-inventory"]
status = "PASS" if all(results.get(c, {}).get("exit_code", 125) == 0 for c in checks) else "FAIL / NOT RUN"
states[slug] = status
page = [f"# {title}", "", f"Результат: **{status}**. Запуск: `{stamp}`.", "", f"Код: [{source}](../../{source}).", ""]
for check in checks:
result = results.get(check, dict(command="not run", exit_code=125, seconds=0))
output = (directory / (check + ".txt")).read_text() if (directory / (check + ".txt")).exists() else "NOT RUN"
page += [f"## {check}", "", f"Команда: `{result['command']}`", "", f"Exit code: {result['exit_code']}; время: {result['seconds']} с.", "", f"[Полный вывод](runs/{stamp}/{check}.txt)", "", "````text", output.rstrip(), "````", ""]
content = "\n".join(page)
(OUTPUT / (slug + ".md")).write_text(content)
historical = re.sub(r"\]\(([^)]+)\)", lambda match: "](" + os.path.relpath(
(OUTPUT / match.group(1)).resolve(), directory).replace(os.sep, "/") + ")", content)
(directory / (slug + ".md")).write_text(historical)
index.append(f"| [{title}]({slug}.md) | {status} |")
(OUTPUT / "README.md").write_text("\n".join(index) + "\n")
(directory / "summary.json").write_text(json.dumps(dict(run=stamp, campaign=campaign.name, criteria=states,
generator_sha256=sha(Path(__file__).read_bytes()), results=results), indent=2) + "\n")
lock.close()
print("Evidence:", OUTPUT)
return 0 if all(s == "PASS" for s in states.values()) else 1
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--campaign", type=Path)
parser.add_argument("--probe", choices=("inventory", "experiment"))
args = parser.parse_args()
if args.probe == "inventory":
inventory()
return 0
campaign = args.campaign
if campaign is None:
candidates = sorted((ROOT / "evidence/idempotency").glob("main-*/manifest.json"))
if not candidates:
parser.error("No main campaign; specify --campaign")
campaign = candidates[-1].parent # Latest attempt, never select by success.
campaign = campaign.resolve()
if args.probe == "experiment":
audit_experiment(campaign)
return 0
return generate(campaign)
if __name__ == "__main__":
sys.exit(main())