From 2dbd25cfdc6244fdd17c01d8f66ffbf51127c3f4 Mon Sep 17 00:00:00 2001 From: TAskMAster339 Date: Thu, 5 Nov 2026 23:48:50 +0300 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20=D0=B3=D0=B5=D0=BD=D0=B5=D1=80=D0=B0=D1=82=D0=BE=D1=80?= =?UTF-8?q?=20=D0=B4=D0=BE=D0=BA=D0=B0=D0=B7=D0=B0=D1=82=D0=B5=D0=BB=D1=8C?= =?UTF-8?q?=D1=81=D1=82=D0=B2=20=D0=BA=D1=80=D0=B8=D1=82=D0=B5=D1=80=D0=B8?= =?UTF-8?q?=D0=B5=D0=B2=20=D0=BB=D0=B0=D0=B1=D0=BE=D1=80=D0=B0=D1=82=D0=BE?= =?UTF-8?q?=D1=80=D0=BD=D0=BE=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/generate_evidence.py | 242 +++++++++++++++++++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100644 scripts/generate_evidence.py diff --git a/scripts/generate_evidence.py b/scripts/generate_evidence.py new file mode 100644 index 0000000..a473549 --- /dev/null +++ b/scripts/generate_evidence.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +"""Run lab5 checks and write sanitized criterion evidence. Run in WSL.""" +import argparse +import datetime as dt +import fcntl +import hashlib +import ast +import json +import os +from pathlib import Path +import re +import shlex +import statistics +import subprocess +import sys +import time +import zipfile +from local_config import api_base_url + +ROOT = Path(__file__).resolve().parents[1] +OUTPUT = ROOT / "evidence/criteria" +COMPOSE = ["docker", "compose", "--project-name", "highload-lab5", "--env-file", ".env", "-f", "compose.yml"] +CRITERIA = [ + ("public-smoke", "Публичный smoke", ["structure", "smoke"], "tests/smoke.py"), + ("baseline", "Сохранение возможностей lab4", ["backend", "persistence", "failover"], "src/backend/tests"), + ("durable-upload", "Устойчивый приём до 202", ["grade3"], "src/backend/repositories/jobs.go"), + ("broker-worker", "Отдельные RabbitMQ и worker, durable delivery и ack", ["inventory", "grade3"], "src/backend/repositories/broker.go"), + ("states-variants", "Состояния, оригинал и thumbnail", ["smoke", "grade3"], "src/backend/workers/attachment.go"), + ("retry-idempotency", "Ограниченный retry и повтор job", ["grade3", "grade4"], "src/backend/repositories/jobs.go"), + ("restart-recovery", "Worker off, restart API/broker, восстановление", ["grade3", "persistence", "restart-audit"], "scripts/restart.sh"), + ("variants-limits", "Crop, WebP, валидация и лимиты worker", ["grade4", "backend"], "src/backend/workers/image_process.go"), + ("outbox", "Восстановление между COMMIT и publish", ["grade4", "experiment"], "src/backend/repositories/jobs.go"), + ("retry-replay", "Backoff/jitter, failed registry и безопасный replay", ["grade4"], "src/backend/repositories/job_admin.go"), + ("lease-cleanup", "Crash после side effect, lease, prefetch и staging", ["grade4", "experiment"], "src/backend/workers/pipeline.go"), + ("versioned-pipeline", "Трек 2: versioned pipeline и fault injection", ["experiment", "backend"], "src/backend/repositories/pipeline_versions.go"), +] + + +def load(path): + return json.loads(path.read_text()) + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def sanitize(value): + for path in (ROOT / "secrets").glob("*"): + if path.is_file(): + secret = path.read_text().strip() + if secret: + value = value.replace(secret, "[REDACTED]") + value = re.sub(r"(?i)(Bearer|Basic)\s+[A-Za-z0-9+/=_.-]+", r"\1 [REDACTED]", value) + return re.sub(r"(?i)(amqps?|postgres(?:ql)?|redis)://[^\s\"']+", r"\1://[REDACTED]", value) + + +def inventory(): + ids = subprocess.check_output(COMPOSE + ["ps", "-aq"], cwd=ROOT, text=True).split() + assert ids, "No containers in highload-lab5" + containers = json.loads(subprocess.check_output(["docker", "inspect", *ids], text=True)) + rows = [] + for c in containers: + assert c["Config"]["Labels"]["com.docker.compose.project"] == "highload-lab5" + rows.append(dict(id=c["Id"], service=c["Config"]["Labels"]["com.docker.compose.service"], + image=c["Image"], status=c["State"]["Status"], + started=c["State"]["StartedAt"], oom=c["State"]["OOMKilled"], + memory=c["HostConfig"]["Memory"], nano_cpus=c["HostConfig"]["NanoCpus"], + health=c["State"].get("Health", {}).get("Status"), + mounts=[dict(type=m["Type"], target=m["Destination"], name=m.get("Name")) for m in c["Mounts"]])) + assert sum(r["service"] == "api" and r["health"] == "healthy" for r in rows) == 2 + for service in ("db", "redis", "s3", "broker", "worker", "dispatcher"): + assert any(r["service"] == service and r["status"] == "running" for r in rows), service + assert not any(r["oom"] for r in rows) + print(json.dumps(rows, indent=2)) + + +def audit_experiment(campaign): + assert campaign.parent == (ROOT / "evidence/idempotency").resolve() + frozen = load(campaign / "seal/manifest.json") + with zipfile.ZipFile(campaign / "seal/source.zip") as archive: + for name, expected in frozen["files"].items(): + assert sha(archive.read(name)) == expected, name + analysis_source = archive.read("experiments/run.py") + # Run the sealed analysis function; changes to packaging/Git rules must not + # invalidate a completed experiment or rewrite its original summary. + tree = ast.parse(analysis_source) + function = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "analyze") + namespace = {"json": json, "dump": lambda *args: None} + exec(compile(ast.Module(body=[function], type_ignores=[]), "", "exec"), namespace) + summary = namespace["analyze"](campaign) + sys.path.insert(0, str(ROOT / "experiments/src")) + assert summary == load(campaign / "summary.json"), "Saved summary differs from raw data" + assert summary["outcome"] == "supported_in_tested_conditions", summary + assert sha((campaign / "protocol.json").read_bytes()) == frozen["protocol_sha256"] + manifest = load(campaign / "manifest.json") + assert manifest["protocol_sha256"] == frozen["protocol_sha256"] + from driver import webp_size + from image_probe import jpeg_size + records = [json.loads(line) for line in (campaign / "cases.jsonl").read_text().splitlines()] + objects = 0 + for row in records: + folder = (campaign / row["directory"]).resolve() + assert folder.is_relative_to(campaign / "cases") + assert load(folder / "result.json") == row + final = load(folder / "database-final.json") + assert final["status"] == "ready" and final["desired"] == final["published"] == row["version"] + versions = [p["version"] for p in final["publications"]] + assert versions == sorted(set(versions)) and versions[-1] == row["version"] + events = [json.loads(line) for line in (folder / "events.jsonl").read_text().splitlines()] + assert any(e["kind"] == "accepted_202" for e in events) + assert any(e["kind"] == "case_complete" for e in events) + for path in folder.glob("verify-*-manifest.json"): + observation = path.name.removesuffix("-manifest.json") + version = int(observation.rsplit("v", 1)[1]) + publication = next(p for p in final["publications"] if p["version"] == version) + hashes = load(path) + assert len(hashes) == 3 + for key, checksum in hashes.items(): + data = (folder / "objects" / observation / Path(key).name).read_bytes() + name = Path(key).stem + metadata = next(v for v in publication["variants"] if v["name"] == name) + assert sha(data) == checksum == metadata["sha256"] + assert len(data) == metadata["size_bytes"] + dimensions = webp_size(data) if name == "optimized" else jpeg_size(data) + assert dimensions == (metadata["width"], metadata["height"]) + objects += 1 + assert row["manifest"] == load(sorted(folder.glob("verify-*-manifest.json"))[-1]) + for repeat in range(1, 6): + environment = load(campaign / f"environment-r{repeat}.json") + for service in ("api", "worker"): + rows = [r for r in environment if r["service"] == service] + assert len(rows) == 2 and all(r["image"] == manifest["application_image"] for r in rows) + print(json.dumps(dict(campaign=str(campaign.relative_to(ROOT)), completed=len(records), + checked_variant_files=objects, outcome=summary["outcome"], + violations=summary["violations"], oom=summary["oom_events"], + protocol_sha256=frozen["protocol_sha256"], source_archive_verified=True), indent=2)) + print("\n| Сценарий | Случаи | Время сценария min / median / max, с |\n| --- | --- | --- |") + for scenario in load(campaign / "protocol.json")["scenarios"]: + times = [r["elapsed_seconds"] for r in records if r["scenario"] == scenario] + print(f"| {scenario} | {len(times)} | {min(times):.2f} / {statistics.median(times):.2f} / {max(times):.2f} |") + + +def generate(campaign): + OUTPUT.mkdir(parents=True, exist_ok=True) + lock = (ROOT / "evidence/idempotency/.run.lock").open("a+") + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + stamp = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ") + directory = OUTPUT / "runs" / stamp + directory.mkdir(parents=True) + results = {} + + def run(name, command, timeout=900): + print(name, flush=True) + start = time.monotonic() + try: + result = subprocess.run(command, cwd=ROOT, text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, timeout=timeout) + code, output = result.returncode, result.stdout + except subprocess.TimeoutExpired as exc: + code, output = 124, (exc.stdout or b"").decode() if isinstance(exc.stdout, bytes) else (exc.stdout or "") + output = sanitize(output) + if name == "backend" and ("--- SKIP:" in output or "--- PASS:" not in output): + code = code or 1 + output += "\nBackend evidence requires executed tests without skips.\n" + results[name] = dict(command=shlex.join(command), exit_code=code, seconds=round(time.monotonic()-start, 3)) + (directory / (name + ".txt")).write_text(output) + (directory / (name + ".json")).write_text(json.dumps(results[name], indent=2) + "\n") + return code + + self_cmd = [sys.executable, str(Path(__file__).resolve())] + try: + run("structure", ["make", "check"]) + run("experiment", self_cmd + ["--probe", "experiment", "--campaign", str(campaign)]) + if run("startup", COMPOSE + ["up", "-d", "--build", "--wait"], 1800) == 0: + run("inventory", self_cmd + ["--probe", "inventory"]) + run("smoke", ["make", "test", "BASE_URL=" + api_base_url()]) + run("backend", COMPOSE + ["--profile", "test", "run", "--rm", "--build", "backend-tests"], 1800) + run("grade3", [sys.executable, "scripts/check_async_grade3.py"]) + run("grade4", [sys.executable, "scripts/check_async_grade4.py"]) + run("before-restart", self_cmd + ["--probe", "inventory"]) + run("persistence", ["make", "test-persistence", "BASE_URL=" + api_base_url(), "ACTION_SCRIPT=scripts/restart.sh"]) + run("after-restart", self_cmd + ["--probe", "inventory"]) + before = {r["id"]: r for r in load(directory / "before-restart.txt")} if results["before-restart"]["exit_code"] == 0 else {} + after = {r["id"]: r for r in load(directory / "after-restart.txt")} if results["after-restart"]["exit_code"] == 0 else {} + services = {"db", "redis", "s3", "broker", "api", "worker", "dispatcher", "upload-cleanup"} + audited = [r for r in before.values() if r["service"] in services] + passed = bool(audited) and services <= {r["service"] for r in audited} and all( + r["id"] in after and after[r["id"]]["started"] != r["started"] and after[r["id"]]["mounts"] == r["mounts"] for r in audited) + results["restart-audit"] = dict(command="compare before-restart / after-restart: StartedAt and mounts", exit_code=0 if passed else 1, seconds=0) + (directory / "restart-audit.txt").write_text(json.dumps(dict(processes_restarted_with_same_mounts=passed, containers=len(audited)), indent=2)) + run("failover", ["make", "test-failover", "BASE_URL=" + api_base_url(), "ACTION_SCRIPT=scripts/stop-one.sh"]) + finally: + run("restore", COMPOSE + ["up", "-d", "--wait"], 300) + run("final-inventory", self_cmd + ["--probe", "inventory"]) + index = ["# Доказательства критериев", "", f"Запуск: `{stamp}`. Серия: [{campaign.name}](../idempotency/{campaign.name}/summary.json).", "", "| Критерий | Результат |", "| --- | --- |"] + states = {} + for slug, title, checks, source in CRITERIA: + checks = ["startup", *checks, "restore", "final-inventory"] + status = "PASS" if all(results.get(c, {}).get("exit_code", 125) == 0 for c in checks) else "FAIL / NOT RUN" + states[slug] = status + page = [f"# {title}", "", f"Результат: **{status}**. Запуск: `{stamp}`.", "", f"Код: [{source}](../../{source}).", ""] + for check in checks: + result = results.get(check, dict(command="not run", exit_code=125, seconds=0)) + output = (directory / (check + ".txt")).read_text() if (directory / (check + ".txt")).exists() else "NOT RUN" + page += [f"## {check}", "", f"Команда: `{result['command']}`", "", f"Exit code: {result['exit_code']}; время: {result['seconds']} с.", "", f"[Полный вывод](runs/{stamp}/{check}.txt)", "", "````text", output.rstrip(), "````", ""] + content = "\n".join(page) + (OUTPUT / (slug + ".md")).write_text(content) + historical = re.sub(r"\]\(([^)]+)\)", lambda match: "](" + os.path.relpath( + (OUTPUT / match.group(1)).resolve(), directory).replace(os.sep, "/") + ")", content) + (directory / (slug + ".md")).write_text(historical) + index.append(f"| [{title}]({slug}.md) | {status} |") + (OUTPUT / "README.md").write_text("\n".join(index) + "\n") + (directory / "summary.json").write_text(json.dumps(dict(run=stamp, campaign=campaign.name, criteria=states, + generator_sha256=sha(Path(__file__).read_bytes()), results=results), indent=2) + "\n") + lock.close() + print("Evidence:", OUTPUT) + return 0 if all(s == "PASS" for s in states.values()) else 1 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--campaign", type=Path) + parser.add_argument("--probe", choices=("inventory", "experiment")) + args = parser.parse_args() + if args.probe == "inventory": + inventory() + return 0 + campaign = args.campaign + if campaign is None: + candidates = sorted((ROOT / "evidence/idempotency").glob("main-*/manifest.json")) + if not candidates: + parser.error("No main campaign; specify --campaign") + campaign = candidates[-1].parent # Latest attempt, never select by success. + campaign = campaign.resolve() + if args.probe == "experiment": + audit_experiment(campaign) + return 0 + return generate(campaign) + + +if __name__ == "__main__": + sys.exit(main())