#!/usr/bin/env python3 """Run lab5 checks and write sanitized criterion evidence. Run in WSL.""" import argparse import datetime as dt import fcntl import hashlib import ast import json import os from pathlib import Path import re import shlex import statistics import subprocess import sys import time import zipfile from local_config import api_base_url ROOT = Path(__file__).resolve().parents[1] OUTPUT = ROOT / "evidence/criteria" COMPOSE = ["docker", "compose", "--project-name", "highload-lab5", "--env-file", ".env", "-f", "compose.yml"] CRITERIA = [ ("public-smoke", "Публичный smoke", ["structure", "smoke"], "tests/smoke.py"), ("baseline", "Сохранение возможностей lab4", ["backend", "persistence", "failover"], "src/backend/tests"), ("durable-upload", "Устойчивый приём до 202", ["grade3"], "src/backend/repositories/jobs.go"), ("broker-worker", "Отдельные RabbitMQ и worker, durable delivery и ack", ["inventory", "grade3"], "src/backend/repositories/broker.go"), ("states-variants", "Состояния, оригинал и thumbnail", ["smoke", "grade3"], "src/backend/workers/attachment.go"), ("retry-idempotency", "Ограниченный retry и повтор job", ["grade3", "grade4"], "src/backend/repositories/jobs.go"), ("restart-recovery", "Worker off, restart API/broker, восстановление", ["grade3", "persistence", "restart-audit"], "scripts/restart.sh"), ("variants-limits", "Crop, WebP, валидация и лимиты worker", ["grade4", "backend"], "src/backend/workers/image_process.go"), ("outbox", "Восстановление между COMMIT и publish", ["grade4", "experiment"], "src/backend/repositories/jobs.go"), ("retry-replay", "Backoff/jitter, failed registry и безопасный replay", ["grade4"], "src/backend/repositories/job_admin.go"), ("lease-cleanup", "Crash после side effect, lease, prefetch и staging", ["grade4", "experiment"], "src/backend/workers/pipeline.go"), ("versioned-pipeline", "Трек 2: versioned pipeline и fault injection", ["experiment", "backend"], "src/backend/repositories/pipeline_versions.go"), ] def load(path): return json.loads(path.read_text()) def sha(data): return hashlib.sha256(data).hexdigest() def sanitize(value): for path in (ROOT / "secrets").glob("*"): if path.is_file(): secret = path.read_text().strip() if secret: value = value.replace(secret, "[REDACTED]") value = re.sub(r"(?i)(Bearer|Basic)\s+[A-Za-z0-9+/=_.-]+", r"\1 [REDACTED]", value) return re.sub(r"(?i)(amqps?|postgres(?:ql)?|redis)://[^\s\"']+", r"\1://[REDACTED]", value) def inventory(): ids = subprocess.check_output(COMPOSE + ["ps", "-aq"], cwd=ROOT, text=True).split() assert ids, "No containers in highload-lab5" containers = json.loads(subprocess.check_output(["docker", "inspect", *ids], text=True)) rows = [] for c in containers: assert c["Config"]["Labels"]["com.docker.compose.project"] == "highload-lab5" rows.append(dict(id=c["Id"], service=c["Config"]["Labels"]["com.docker.compose.service"], image=c["Image"], status=c["State"]["Status"], started=c["State"]["StartedAt"], oom=c["State"]["OOMKilled"], memory=c["HostConfig"]["Memory"], nano_cpus=c["HostConfig"]["NanoCpus"], health=c["State"].get("Health", {}).get("Status"), mounts=[dict(type=m["Type"], target=m["Destination"], name=m.get("Name")) for m in c["Mounts"]])) assert sum(r["service"] == "api" and r["health"] == "healthy" for r in rows) == 2 for service in ("db", "redis", "s3", "broker", "worker", "dispatcher"): assert any(r["service"] == service and r["status"] == "running" for r in rows), service assert not any(r["oom"] for r in rows) print(json.dumps(rows, indent=2)) def audit_experiment(campaign): assert campaign.parent == (ROOT / "evidence/idempotency").resolve() frozen = load(campaign / "seal/manifest.json") with zipfile.ZipFile(campaign / "seal/source.zip") as archive: for name, expected in frozen["files"].items(): assert sha(archive.read(name)) == expected, name analysis_source = archive.read("experiments/run.py") # Run the sealed analysis function; changes to packaging/Git rules must not # invalidate a completed experiment or rewrite its original summary. tree = ast.parse(analysis_source) function = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "analyze") namespace = {"json": json, "dump": lambda *args: None} exec(compile(ast.Module(body=[function], type_ignores=[]), "", "exec"), namespace) summary = namespace["analyze"](campaign) sys.path.insert(0, str(ROOT / "experiments/src")) assert summary == load(campaign / "summary.json"), "Saved summary differs from raw data" assert summary["outcome"] == "supported_in_tested_conditions", summary assert sha((campaign / "protocol.json").read_bytes()) == frozen["protocol_sha256"] manifest = load(campaign / "manifest.json") assert manifest["protocol_sha256"] == frozen["protocol_sha256"] from driver import webp_size from image_probe import jpeg_size records = [json.loads(line) for line in (campaign / "cases.jsonl").read_text().splitlines()] objects = 0 for row in records: folder = (campaign / row["directory"]).resolve() assert folder.is_relative_to(campaign / "cases") assert load(folder / "result.json") == row final = load(folder / "database-final.json") assert final["status"] == "ready" and final["desired"] == final["published"] == row["version"] versions = [p["version"] for p in final["publications"]] assert versions == sorted(set(versions)) and versions[-1] == row["version"] events = [json.loads(line) for line in (folder / "events.jsonl").read_text().splitlines()] assert any(e["kind"] == "accepted_202" for e in events) assert any(e["kind"] == "case_complete" for e in events) for path in folder.glob("verify-*-manifest.json"): observation = path.name.removesuffix("-manifest.json") version = int(observation.rsplit("v", 1)[1]) publication = next(p for p in final["publications"] if p["version"] == version) hashes = load(path) assert len(hashes) == 3 for key, checksum in hashes.items(): data = (folder / "objects" / observation / Path(key).name).read_bytes() name = Path(key).stem metadata = next(v for v in publication["variants"] if v["name"] == name) assert sha(data) == checksum == metadata["sha256"] assert len(data) == metadata["size_bytes"] dimensions = webp_size(data) if name == "optimized" else jpeg_size(data) assert dimensions == (metadata["width"], metadata["height"]) objects += 1 assert row["manifest"] == load(sorted(folder.glob("verify-*-manifest.json"))[-1]) for repeat in range(1, 6): environment = load(campaign / f"environment-r{repeat}.json") for service in ("api", "worker"): rows = [r for r in environment if r["service"] == service] assert len(rows) == 2 and all(r["image"] == manifest["application_image"] for r in rows) print(json.dumps(dict(campaign=str(campaign.relative_to(ROOT)), completed=len(records), checked_variant_files=objects, outcome=summary["outcome"], violations=summary["violations"], oom=summary["oom_events"], protocol_sha256=frozen["protocol_sha256"], source_archive_verified=True), indent=2)) print("\n| Сценарий | Случаи | Время сценария min / median / max, с |\n| --- | --- | --- |") for scenario in load(campaign / "protocol.json")["scenarios"]: times = [r["elapsed_seconds"] for r in records if r["scenario"] == scenario] print(f"| {scenario} | {len(times)} | {min(times):.2f} / {statistics.median(times):.2f} / {max(times):.2f} |") def generate(campaign): OUTPUT.mkdir(parents=True, exist_ok=True) lock = (ROOT / "evidence/idempotency/.run.lock").open("a+") fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) stamp = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ") directory = OUTPUT / "runs" / stamp directory.mkdir(parents=True) results = {} def run(name, command, timeout=900): print(name, flush=True) start = time.monotonic() try: result = subprocess.run(command, cwd=ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout) code, output = result.returncode, result.stdout except subprocess.TimeoutExpired as exc: code, output = 124, (exc.stdout or b"").decode() if isinstance(exc.stdout, bytes) else (exc.stdout or "") output = sanitize(output) if name == "backend" and ("--- SKIP:" in output or "--- PASS:" not in output): code = code or 1 output += "\nBackend evidence requires executed tests without skips.\n" results[name] = dict(command=shlex.join(command), exit_code=code, seconds=round(time.monotonic()-start, 3)) (directory / (name + ".txt")).write_text(output) (directory / (name + ".json")).write_text(json.dumps(results[name], indent=2) + "\n") return code self_cmd = [sys.executable, str(Path(__file__).resolve())] try: run("structure", ["make", "check"]) run("experiment", self_cmd + ["--probe", "experiment", "--campaign", str(campaign)]) if run("startup", COMPOSE + ["up", "-d", "--build", "--wait"], 1800) == 0: run("inventory", self_cmd + ["--probe", "inventory"]) run("smoke", ["make", "test", "BASE_URL=" + api_base_url()]) run("backend", COMPOSE + ["--profile", "test", "run", "--rm", "--build", "backend-tests"], 1800) run("grade3", [sys.executable, "scripts/check_async_grade3.py"]) run("grade4", [sys.executable, "scripts/check_async_grade4.py"]) run("before-restart", self_cmd + ["--probe", "inventory"]) run("persistence", ["make", "test-persistence", "BASE_URL=" + api_base_url(), "ACTION_SCRIPT=scripts/restart.sh"]) run("after-restart", self_cmd + ["--probe", "inventory"]) before = {r["id"]: r for r in load(directory / "before-restart.txt")} if results["before-restart"]["exit_code"] == 0 else {} after = {r["id"]: r for r in load(directory / "after-restart.txt")} if results["after-restart"]["exit_code"] == 0 else {} services = {"db", "redis", "s3", "broker", "api", "worker", "dispatcher", "upload-cleanup"} audited = [r for r in before.values() if r["service"] in services] passed = bool(audited) and services <= {r["service"] for r in audited} and all( r["id"] in after and after[r["id"]]["started"] != r["started"] and after[r["id"]]["mounts"] == r["mounts"] for r in audited) results["restart-audit"] = dict(command="compare before-restart / after-restart: StartedAt and mounts", exit_code=0 if passed else 1, seconds=0) (directory / "restart-audit.txt").write_text(json.dumps(dict(processes_restarted_with_same_mounts=passed, containers=len(audited)), indent=2)) run("failover", ["make", "test-failover", "BASE_URL=" + api_base_url(), "ACTION_SCRIPT=scripts/stop-one.sh"]) finally: run("restore", COMPOSE + ["up", "-d", "--wait"], 300) run("final-inventory", self_cmd + ["--probe", "inventory"]) index = ["# Доказательства критериев", "", f"Запуск: `{stamp}`. Серия: [{campaign.name}](../idempotency/{campaign.name}/summary.json).", "", "| Критерий | Результат |", "| --- | --- |"] states = {} for slug, title, checks, source in CRITERIA: checks = ["startup", *checks, "restore", "final-inventory"] status = "PASS" if all(results.get(c, {}).get("exit_code", 125) == 0 for c in checks) else "FAIL / NOT RUN" states[slug] = status page = [f"# {title}", "", f"Результат: **{status}**. Запуск: `{stamp}`.", "", f"Код: [{source}](../../{source}).", ""] for check in checks: result = results.get(check, dict(command="not run", exit_code=125, seconds=0)) output = (directory / (check + ".txt")).read_text() if (directory / (check + ".txt")).exists() else "NOT RUN" page += [f"## {check}", "", f"Команда: `{result['command']}`", "", f"Exit code: {result['exit_code']}; время: {result['seconds']} с.", "", f"[Полный вывод](runs/{stamp}/{check}.txt)", "", "````text", output.rstrip(), "````", ""] content = "\n".join(page) (OUTPUT / (slug + ".md")).write_text(content) historical = re.sub(r"\]\(([^)]+)\)", lambda match: "](" + os.path.relpath( (OUTPUT / match.group(1)).resolve(), directory).replace(os.sep, "/") + ")", content) (directory / (slug + ".md")).write_text(historical) index.append(f"| [{title}]({slug}.md) | {status} |") (OUTPUT / "README.md").write_text("\n".join(index) + "\n") (directory / "summary.json").write_text(json.dumps(dict(run=stamp, campaign=campaign.name, criteria=states, generator_sha256=sha(Path(__file__).read_bytes()), results=results), indent=2) + "\n") lock.close() print("Evidence:", OUTPUT) return 0 if all(s == "PASS" for s in states.values()) else 1 def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--campaign", type=Path) parser.add_argument("--probe", choices=("inventory", "experiment")) args = parser.parse_args() if args.probe == "inventory": inventory() return 0 campaign = args.campaign if campaign is None: candidates = sorted((ROOT / "evidence/idempotency").glob("main-*/manifest.json")) if not candidates: parser.error("No main campaign; specify --campaign") campaign = candidates[-1].parent # Latest attempt, never select by success. campaign = campaign.resolve() if args.probe == "experiment": audit_experiment(campaign) return 0 return generate(campaign) if __name__ == "__main__": sys.exit(main())