forked from mtusi2026/mtusi2026lab5
Добавить генератор доказательств критериев лабораторной
This commit is contained in:
@@ -0,0 +1,242 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run lab5 checks and write sanitized criterion evidence. Run in WSL."""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import fcntl
|
||||
import hashlib
|
||||
import ast
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shlex
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import zipfile
|
||||
from local_config import api_base_url
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUTPUT = ROOT / "evidence/criteria"
|
||||
COMPOSE = ["docker", "compose", "--project-name", "highload-lab5", "--env-file", ".env", "-f", "compose.yml"]
|
||||
CRITERIA = [
|
||||
("public-smoke", "Публичный smoke", ["structure", "smoke"], "tests/smoke.py"),
|
||||
("baseline", "Сохранение возможностей lab4", ["backend", "persistence", "failover"], "src/backend/tests"),
|
||||
("durable-upload", "Устойчивый приём до 202", ["grade3"], "src/backend/repositories/jobs.go"),
|
||||
("broker-worker", "Отдельные RabbitMQ и worker, durable delivery и ack", ["inventory", "grade3"], "src/backend/repositories/broker.go"),
|
||||
("states-variants", "Состояния, оригинал и thumbnail", ["smoke", "grade3"], "src/backend/workers/attachment.go"),
|
||||
("retry-idempotency", "Ограниченный retry и повтор job", ["grade3", "grade4"], "src/backend/repositories/jobs.go"),
|
||||
("restart-recovery", "Worker off, restart API/broker, восстановление", ["grade3", "persistence", "restart-audit"], "scripts/restart.sh"),
|
||||
("variants-limits", "Crop, WebP, валидация и лимиты worker", ["grade4", "backend"], "src/backend/workers/image_process.go"),
|
||||
("outbox", "Восстановление между COMMIT и publish", ["grade4", "experiment"], "src/backend/repositories/jobs.go"),
|
||||
("retry-replay", "Backoff/jitter, failed registry и безопасный replay", ["grade4"], "src/backend/repositories/job_admin.go"),
|
||||
("lease-cleanup", "Crash после side effect, lease, prefetch и staging", ["grade4", "experiment"], "src/backend/workers/pipeline.go"),
|
||||
("versioned-pipeline", "Трек 2: versioned pipeline и fault injection", ["experiment", "backend"], "src/backend/repositories/pipeline_versions.go"),
|
||||
]
|
||||
|
||||
|
||||
def load(path):
|
||||
return json.loads(path.read_text())
|
||||
|
||||
|
||||
def sha(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def sanitize(value):
|
||||
for path in (ROOT / "secrets").glob("*"):
|
||||
if path.is_file():
|
||||
secret = path.read_text().strip()
|
||||
if secret:
|
||||
value = value.replace(secret, "[REDACTED]")
|
||||
value = re.sub(r"(?i)(Bearer|Basic)\s+[A-Za-z0-9+/=_.-]+", r"\1 [REDACTED]", value)
|
||||
return re.sub(r"(?i)(amqps?|postgres(?:ql)?|redis)://[^\s\"']+", r"\1://[REDACTED]", value)
|
||||
|
||||
|
||||
def inventory():
|
||||
ids = subprocess.check_output(COMPOSE + ["ps", "-aq"], cwd=ROOT, text=True).split()
|
||||
assert ids, "No containers in highload-lab5"
|
||||
containers = json.loads(subprocess.check_output(["docker", "inspect", *ids], text=True))
|
||||
rows = []
|
||||
for c in containers:
|
||||
assert c["Config"]["Labels"]["com.docker.compose.project"] == "highload-lab5"
|
||||
rows.append(dict(id=c["Id"], service=c["Config"]["Labels"]["com.docker.compose.service"],
|
||||
image=c["Image"], status=c["State"]["Status"],
|
||||
started=c["State"]["StartedAt"], oom=c["State"]["OOMKilled"],
|
||||
memory=c["HostConfig"]["Memory"], nano_cpus=c["HostConfig"]["NanoCpus"],
|
||||
health=c["State"].get("Health", {}).get("Status"),
|
||||
mounts=[dict(type=m["Type"], target=m["Destination"], name=m.get("Name")) for m in c["Mounts"]]))
|
||||
assert sum(r["service"] == "api" and r["health"] == "healthy" for r in rows) == 2
|
||||
for service in ("db", "redis", "s3", "broker", "worker", "dispatcher"):
|
||||
assert any(r["service"] == service and r["status"] == "running" for r in rows), service
|
||||
assert not any(r["oom"] for r in rows)
|
||||
print(json.dumps(rows, indent=2))
|
||||
|
||||
|
||||
def audit_experiment(campaign):
|
||||
assert campaign.parent == (ROOT / "evidence/idempotency").resolve()
|
||||
frozen = load(campaign / "seal/manifest.json")
|
||||
with zipfile.ZipFile(campaign / "seal/source.zip") as archive:
|
||||
for name, expected in frozen["files"].items():
|
||||
assert sha(archive.read(name)) == expected, name
|
||||
analysis_source = archive.read("experiments/run.py")
|
||||
# Run the sealed analysis function; changes to packaging/Git rules must not
|
||||
# invalidate a completed experiment or rewrite its original summary.
|
||||
tree = ast.parse(analysis_source)
|
||||
function = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "analyze")
|
||||
namespace = {"json": json, "dump": lambda *args: None}
|
||||
exec(compile(ast.Module(body=[function], type_ignores=[]), "<sealed analyze>", "exec"), namespace)
|
||||
summary = namespace["analyze"](campaign)
|
||||
sys.path.insert(0, str(ROOT / "experiments/src"))
|
||||
assert summary == load(campaign / "summary.json"), "Saved summary differs from raw data"
|
||||
assert summary["outcome"] == "supported_in_tested_conditions", summary
|
||||
assert sha((campaign / "protocol.json").read_bytes()) == frozen["protocol_sha256"]
|
||||
manifest = load(campaign / "manifest.json")
|
||||
assert manifest["protocol_sha256"] == frozen["protocol_sha256"]
|
||||
from driver import webp_size
|
||||
from image_probe import jpeg_size
|
||||
records = [json.loads(line) for line in (campaign / "cases.jsonl").read_text().splitlines()]
|
||||
objects = 0
|
||||
for row in records:
|
||||
folder = (campaign / row["directory"]).resolve()
|
||||
assert folder.is_relative_to(campaign / "cases")
|
||||
assert load(folder / "result.json") == row
|
||||
final = load(folder / "database-final.json")
|
||||
assert final["status"] == "ready" and final["desired"] == final["published"] == row["version"]
|
||||
versions = [p["version"] for p in final["publications"]]
|
||||
assert versions == sorted(set(versions)) and versions[-1] == row["version"]
|
||||
events = [json.loads(line) for line in (folder / "events.jsonl").read_text().splitlines()]
|
||||
assert any(e["kind"] == "accepted_202" for e in events)
|
||||
assert any(e["kind"] == "case_complete" for e in events)
|
||||
for path in folder.glob("verify-*-manifest.json"):
|
||||
observation = path.name.removesuffix("-manifest.json")
|
||||
version = int(observation.rsplit("v", 1)[1])
|
||||
publication = next(p for p in final["publications"] if p["version"] == version)
|
||||
hashes = load(path)
|
||||
assert len(hashes) == 3
|
||||
for key, checksum in hashes.items():
|
||||
data = (folder / "objects" / observation / Path(key).name).read_bytes()
|
||||
name = Path(key).stem
|
||||
metadata = next(v for v in publication["variants"] if v["name"] == name)
|
||||
assert sha(data) == checksum == metadata["sha256"]
|
||||
assert len(data) == metadata["size_bytes"]
|
||||
dimensions = webp_size(data) if name == "optimized" else jpeg_size(data)
|
||||
assert dimensions == (metadata["width"], metadata["height"])
|
||||
objects += 1
|
||||
assert row["manifest"] == load(sorted(folder.glob("verify-*-manifest.json"))[-1])
|
||||
for repeat in range(1, 6):
|
||||
environment = load(campaign / f"environment-r{repeat}.json")
|
||||
for service in ("api", "worker"):
|
||||
rows = [r for r in environment if r["service"] == service]
|
||||
assert len(rows) == 2 and all(r["image"] == manifest["application_image"] for r in rows)
|
||||
print(json.dumps(dict(campaign=str(campaign.relative_to(ROOT)), completed=len(records),
|
||||
checked_variant_files=objects, outcome=summary["outcome"],
|
||||
violations=summary["violations"], oom=summary["oom_events"],
|
||||
protocol_sha256=frozen["protocol_sha256"], source_archive_verified=True), indent=2))
|
||||
print("\n| Сценарий | Случаи | Время сценария min / median / max, с |\n| --- | --- | --- |")
|
||||
for scenario in load(campaign / "protocol.json")["scenarios"]:
|
||||
times = [r["elapsed_seconds"] for r in records if r["scenario"] == scenario]
|
||||
print(f"| {scenario} | {len(times)} | {min(times):.2f} / {statistics.median(times):.2f} / {max(times):.2f} |")
|
||||
|
||||
|
||||
def generate(campaign):
|
||||
OUTPUT.mkdir(parents=True, exist_ok=True)
|
||||
lock = (ROOT / "evidence/idempotency/.run.lock").open("a+")
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
stamp = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ")
|
||||
directory = OUTPUT / "runs" / stamp
|
||||
directory.mkdir(parents=True)
|
||||
results = {}
|
||||
|
||||
def run(name, command, timeout=900):
|
||||
print(name, flush=True)
|
||||
start = time.monotonic()
|
||||
try:
|
||||
result = subprocess.run(command, cwd=ROOT, text=True, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT, timeout=timeout)
|
||||
code, output = result.returncode, result.stdout
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
code, output = 124, (exc.stdout or b"").decode() if isinstance(exc.stdout, bytes) else (exc.stdout or "")
|
||||
output = sanitize(output)
|
||||
if name == "backend" and ("--- SKIP:" in output or "--- PASS:" not in output):
|
||||
code = code or 1
|
||||
output += "\nBackend evidence requires executed tests without skips.\n"
|
||||
results[name] = dict(command=shlex.join(command), exit_code=code, seconds=round(time.monotonic()-start, 3))
|
||||
(directory / (name + ".txt")).write_text(output)
|
||||
(directory / (name + ".json")).write_text(json.dumps(results[name], indent=2) + "\n")
|
||||
return code
|
||||
|
||||
self_cmd = [sys.executable, str(Path(__file__).resolve())]
|
||||
try:
|
||||
run("structure", ["make", "check"])
|
||||
run("experiment", self_cmd + ["--probe", "experiment", "--campaign", str(campaign)])
|
||||
if run("startup", COMPOSE + ["up", "-d", "--build", "--wait"], 1800) == 0:
|
||||
run("inventory", self_cmd + ["--probe", "inventory"])
|
||||
run("smoke", ["make", "test", "BASE_URL=" + api_base_url()])
|
||||
run("backend", COMPOSE + ["--profile", "test", "run", "--rm", "--build", "backend-tests"], 1800)
|
||||
run("grade3", [sys.executable, "scripts/check_async_grade3.py"])
|
||||
run("grade4", [sys.executable, "scripts/check_async_grade4.py"])
|
||||
run("before-restart", self_cmd + ["--probe", "inventory"])
|
||||
run("persistence", ["make", "test-persistence", "BASE_URL=" + api_base_url(), "ACTION_SCRIPT=scripts/restart.sh"])
|
||||
run("after-restart", self_cmd + ["--probe", "inventory"])
|
||||
before = {r["id"]: r for r in load(directory / "before-restart.txt")} if results["before-restart"]["exit_code"] == 0 else {}
|
||||
after = {r["id"]: r for r in load(directory / "after-restart.txt")} if results["after-restart"]["exit_code"] == 0 else {}
|
||||
services = {"db", "redis", "s3", "broker", "api", "worker", "dispatcher", "upload-cleanup"}
|
||||
audited = [r for r in before.values() if r["service"] in services]
|
||||
passed = bool(audited) and services <= {r["service"] for r in audited} and all(
|
||||
r["id"] in after and after[r["id"]]["started"] != r["started"] and after[r["id"]]["mounts"] == r["mounts"] for r in audited)
|
||||
results["restart-audit"] = dict(command="compare before-restart / after-restart: StartedAt and mounts", exit_code=0 if passed else 1, seconds=0)
|
||||
(directory / "restart-audit.txt").write_text(json.dumps(dict(processes_restarted_with_same_mounts=passed, containers=len(audited)), indent=2))
|
||||
run("failover", ["make", "test-failover", "BASE_URL=" + api_base_url(), "ACTION_SCRIPT=scripts/stop-one.sh"])
|
||||
finally:
|
||||
run("restore", COMPOSE + ["up", "-d", "--wait"], 300)
|
||||
run("final-inventory", self_cmd + ["--probe", "inventory"])
|
||||
index = ["# Доказательства критериев", "", f"Запуск: `{stamp}`. Серия: [{campaign.name}](../idempotency/{campaign.name}/summary.json).", "", "| Критерий | Результат |", "| --- | --- |"]
|
||||
states = {}
|
||||
for slug, title, checks, source in CRITERIA:
|
||||
checks = ["startup", *checks, "restore", "final-inventory"]
|
||||
status = "PASS" if all(results.get(c, {}).get("exit_code", 125) == 0 for c in checks) else "FAIL / NOT RUN"
|
||||
states[slug] = status
|
||||
page = [f"# {title}", "", f"Результат: **{status}**. Запуск: `{stamp}`.", "", f"Код: [{source}](../../{source}).", ""]
|
||||
for check in checks:
|
||||
result = results.get(check, dict(command="not run", exit_code=125, seconds=0))
|
||||
output = (directory / (check + ".txt")).read_text() if (directory / (check + ".txt")).exists() else "NOT RUN"
|
||||
page += [f"## {check}", "", f"Команда: `{result['command']}`", "", f"Exit code: {result['exit_code']}; время: {result['seconds']} с.", "", f"[Полный вывод](runs/{stamp}/{check}.txt)", "", "````text", output.rstrip(), "````", ""]
|
||||
content = "\n".join(page)
|
||||
(OUTPUT / (slug + ".md")).write_text(content)
|
||||
historical = re.sub(r"\]\(([^)]+)\)", lambda match: "](" + os.path.relpath(
|
||||
(OUTPUT / match.group(1)).resolve(), directory).replace(os.sep, "/") + ")", content)
|
||||
(directory / (slug + ".md")).write_text(historical)
|
||||
index.append(f"| [{title}]({slug}.md) | {status} |")
|
||||
(OUTPUT / "README.md").write_text("\n".join(index) + "\n")
|
||||
(directory / "summary.json").write_text(json.dumps(dict(run=stamp, campaign=campaign.name, criteria=states,
|
||||
generator_sha256=sha(Path(__file__).read_bytes()), results=results), indent=2) + "\n")
|
||||
lock.close()
|
||||
print("Evidence:", OUTPUT)
|
||||
return 0 if all(s == "PASS" for s in states.values()) else 1
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--campaign", type=Path)
|
||||
parser.add_argument("--probe", choices=("inventory", "experiment"))
|
||||
args = parser.parse_args()
|
||||
if args.probe == "inventory":
|
||||
inventory()
|
||||
return 0
|
||||
campaign = args.campaign
|
||||
if campaign is None:
|
||||
candidates = sorted((ROOT / "evidence/idempotency").glob("main-*/manifest.json"))
|
||||
if not candidates:
|
||||
parser.error("No main campaign; specify --campaign")
|
||||
campaign = candidates[-1].parent # Latest attempt, never select by success.
|
||||
campaign = campaign.resolve()
|
||||
if args.probe == "experiment":
|
||||
audit_experiment(campaign)
|
||||
return 0
|
||||
return generate(campaign)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user