243 lines
14 KiB
Python
243 lines
14 KiB
Python
#!/usr/bin/env python3
|
||
"""Run lab5 checks and write sanitized criterion evidence. Run in WSL."""
|
||
import argparse
|
||
import datetime as dt
|
||
import fcntl
|
||
import hashlib
|
||
import ast
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
import re
|
||
import shlex
|
||
import statistics
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import zipfile
|
||
from local_config import api_base_url
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
OUTPUT = ROOT / "evidence/criteria"
|
||
COMPOSE = ["docker", "compose", "--project-name", "highload-lab5", "--env-file", ".env", "-f", "compose.yml"]
|
||
CRITERIA = [
|
||
("public-smoke", "Публичный smoke", ["structure", "smoke"], "tests/smoke.py"),
|
||
("baseline", "Сохранение возможностей lab4", ["backend", "persistence", "failover"], "src/backend/tests"),
|
||
("durable-upload", "Устойчивый приём до 202", ["grade3"], "src/backend/repositories/jobs.go"),
|
||
("broker-worker", "Отдельные RabbitMQ и worker, durable delivery и ack", ["inventory", "grade3"], "src/backend/repositories/broker.go"),
|
||
("states-variants", "Состояния, оригинал и thumbnail", ["smoke", "grade3"], "src/backend/workers/attachment.go"),
|
||
("retry-idempotency", "Ограниченный retry и повтор job", ["grade3", "grade4"], "src/backend/repositories/jobs.go"),
|
||
("restart-recovery", "Worker off, restart API/broker, восстановление", ["grade3", "persistence", "restart-audit"], "scripts/restart.sh"),
|
||
("variants-limits", "Crop, WebP, валидация и лимиты worker", ["grade4", "backend"], "src/backend/workers/image_process.go"),
|
||
("outbox", "Восстановление между COMMIT и publish", ["grade4", "experiment"], "src/backend/repositories/jobs.go"),
|
||
("retry-replay", "Backoff/jitter, failed registry и безопасный replay", ["grade4"], "src/backend/repositories/job_admin.go"),
|
||
("lease-cleanup", "Crash после side effect, lease, prefetch и staging", ["grade4", "experiment"], "src/backend/workers/pipeline.go"),
|
||
("versioned-pipeline", "Трек 2: versioned pipeline и fault injection", ["experiment", "backend"], "src/backend/repositories/pipeline_versions.go"),
|
||
]
|
||
|
||
|
||
def load(path):
|
||
return json.loads(path.read_text())
|
||
|
||
|
||
def sha(data):
|
||
return hashlib.sha256(data).hexdigest()
|
||
|
||
|
||
def sanitize(value):
|
||
for path in (ROOT / "secrets").glob("*"):
|
||
if path.is_file():
|
||
secret = path.read_text().strip()
|
||
if secret:
|
||
value = value.replace(secret, "[REDACTED]")
|
||
value = re.sub(r"(?i)(Bearer|Basic)\s+[A-Za-z0-9+/=_.-]+", r"\1 [REDACTED]", value)
|
||
return re.sub(r"(?i)(amqps?|postgres(?:ql)?|redis)://[^\s\"']+", r"\1://[REDACTED]", value)
|
||
|
||
|
||
def inventory():
|
||
ids = subprocess.check_output(COMPOSE + ["ps", "-aq"], cwd=ROOT, text=True).split()
|
||
assert ids, "No containers in highload-lab5"
|
||
containers = json.loads(subprocess.check_output(["docker", "inspect", *ids], text=True))
|
||
rows = []
|
||
for c in containers:
|
||
assert c["Config"]["Labels"]["com.docker.compose.project"] == "highload-lab5"
|
||
rows.append(dict(id=c["Id"], service=c["Config"]["Labels"]["com.docker.compose.service"],
|
||
image=c["Image"], status=c["State"]["Status"],
|
||
started=c["State"]["StartedAt"], oom=c["State"]["OOMKilled"],
|
||
memory=c["HostConfig"]["Memory"], nano_cpus=c["HostConfig"]["NanoCpus"],
|
||
health=c["State"].get("Health", {}).get("Status"),
|
||
mounts=[dict(type=m["Type"], target=m["Destination"], name=m.get("Name")) for m in c["Mounts"]]))
|
||
assert sum(r["service"] == "api" and r["health"] == "healthy" for r in rows) == 2
|
||
for service in ("db", "redis", "s3", "broker", "worker", "dispatcher"):
|
||
assert any(r["service"] == service and r["status"] == "running" for r in rows), service
|
||
assert not any(r["oom"] for r in rows)
|
||
print(json.dumps(rows, indent=2))
|
||
|
||
|
||
def audit_experiment(campaign):
|
||
assert campaign.parent == (ROOT / "evidence/idempotency").resolve()
|
||
frozen = load(campaign / "seal/manifest.json")
|
||
with zipfile.ZipFile(campaign / "seal/source.zip") as archive:
|
||
for name, expected in frozen["files"].items():
|
||
assert sha(archive.read(name)) == expected, name
|
||
analysis_source = archive.read("experiments/run.py")
|
||
# Run the sealed analysis function; changes to packaging/Git rules must not
|
||
# invalidate a completed experiment or rewrite its original summary.
|
||
tree = ast.parse(analysis_source)
|
||
function = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "analyze")
|
||
namespace = {"json": json, "dump": lambda *args: None}
|
||
exec(compile(ast.Module(body=[function], type_ignores=[]), "<sealed analyze>", "exec"), namespace)
|
||
summary = namespace["analyze"](campaign)
|
||
sys.path.insert(0, str(ROOT / "experiments/src"))
|
||
assert summary == load(campaign / "summary.json"), "Saved summary differs from raw data"
|
||
assert summary["outcome"] == "supported_in_tested_conditions", summary
|
||
assert sha((campaign / "protocol.json").read_bytes()) == frozen["protocol_sha256"]
|
||
manifest = load(campaign / "manifest.json")
|
||
assert manifest["protocol_sha256"] == frozen["protocol_sha256"]
|
||
from driver import webp_size
|
||
from image_probe import jpeg_size
|
||
records = [json.loads(line) for line in (campaign / "cases.jsonl").read_text().splitlines()]
|
||
objects = 0
|
||
for row in records:
|
||
folder = (campaign / row["directory"]).resolve()
|
||
assert folder.is_relative_to(campaign / "cases")
|
||
assert load(folder / "result.json") == row
|
||
final = load(folder / "database-final.json")
|
||
assert final["status"] == "ready" and final["desired"] == final["published"] == row["version"]
|
||
versions = [p["version"] for p in final["publications"]]
|
||
assert versions == sorted(set(versions)) and versions[-1] == row["version"]
|
||
events = [json.loads(line) for line in (folder / "events.jsonl").read_text().splitlines()]
|
||
assert any(e["kind"] == "accepted_202" for e in events)
|
||
assert any(e["kind"] == "case_complete" for e in events)
|
||
for path in folder.glob("verify-*-manifest.json"):
|
||
observation = path.name.removesuffix("-manifest.json")
|
||
version = int(observation.rsplit("v", 1)[1])
|
||
publication = next(p for p in final["publications"] if p["version"] == version)
|
||
hashes = load(path)
|
||
assert len(hashes) == 3
|
||
for key, checksum in hashes.items():
|
||
data = (folder / "objects" / observation / Path(key).name).read_bytes()
|
||
name = Path(key).stem
|
||
metadata = next(v for v in publication["variants"] if v["name"] == name)
|
||
assert sha(data) == checksum == metadata["sha256"]
|
||
assert len(data) == metadata["size_bytes"]
|
||
dimensions = webp_size(data) if name == "optimized" else jpeg_size(data)
|
||
assert dimensions == (metadata["width"], metadata["height"])
|
||
objects += 1
|
||
assert row["manifest"] == load(sorted(folder.glob("verify-*-manifest.json"))[-1])
|
||
for repeat in range(1, 6):
|
||
environment = load(campaign / f"environment-r{repeat}.json")
|
||
for service in ("api", "worker"):
|
||
rows = [r for r in environment if r["service"] == service]
|
||
assert len(rows) == 2 and all(r["image"] == manifest["application_image"] for r in rows)
|
||
print(json.dumps(dict(campaign=str(campaign.relative_to(ROOT)), completed=len(records),
|
||
checked_variant_files=objects, outcome=summary["outcome"],
|
||
violations=summary["violations"], oom=summary["oom_events"],
|
||
protocol_sha256=frozen["protocol_sha256"], source_archive_verified=True), indent=2))
|
||
print("\n| Сценарий | Случаи | Время сценария min / median / max, с |\n| --- | --- | --- |")
|
||
for scenario in load(campaign / "protocol.json")["scenarios"]:
|
||
times = [r["elapsed_seconds"] for r in records if r["scenario"] == scenario]
|
||
print(f"| {scenario} | {len(times)} | {min(times):.2f} / {statistics.median(times):.2f} / {max(times):.2f} |")
|
||
|
||
|
||
def generate(campaign):
|
||
OUTPUT.mkdir(parents=True, exist_ok=True)
|
||
lock = (ROOT / "evidence/idempotency/.run.lock").open("a+")
|
||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||
stamp = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ")
|
||
directory = OUTPUT / "runs" / stamp
|
||
directory.mkdir(parents=True)
|
||
results = {}
|
||
|
||
def run(name, command, timeout=900):
|
||
print(name, flush=True)
|
||
start = time.monotonic()
|
||
try:
|
||
result = subprocess.run(command, cwd=ROOT, text=True, stdout=subprocess.PIPE,
|
||
stderr=subprocess.STDOUT, timeout=timeout)
|
||
code, output = result.returncode, result.stdout
|
||
except subprocess.TimeoutExpired as exc:
|
||
code, output = 124, (exc.stdout or b"").decode() if isinstance(exc.stdout, bytes) else (exc.stdout or "")
|
||
output = sanitize(output)
|
||
if name == "backend" and ("--- SKIP:" in output or "--- PASS:" not in output):
|
||
code = code or 1
|
||
output += "\nBackend evidence requires executed tests without skips.\n"
|
||
results[name] = dict(command=shlex.join(command), exit_code=code, seconds=round(time.monotonic()-start, 3))
|
||
(directory / (name + ".txt")).write_text(output)
|
||
(directory / (name + ".json")).write_text(json.dumps(results[name], indent=2) + "\n")
|
||
return code
|
||
|
||
self_cmd = [sys.executable, str(Path(__file__).resolve())]
|
||
try:
|
||
run("structure", ["make", "check"])
|
||
run("experiment", self_cmd + ["--probe", "experiment", "--campaign", str(campaign)])
|
||
if run("startup", COMPOSE + ["up", "-d", "--build", "--wait"], 1800) == 0:
|
||
run("inventory", self_cmd + ["--probe", "inventory"])
|
||
run("smoke", ["make", "test", "BASE_URL=" + api_base_url()])
|
||
run("backend", COMPOSE + ["--profile", "test", "run", "--rm", "--build", "backend-tests"], 1800)
|
||
run("grade3", [sys.executable, "scripts/check_async_grade3.py"])
|
||
run("grade4", [sys.executable, "scripts/check_async_grade4.py"])
|
||
run("before-restart", self_cmd + ["--probe", "inventory"])
|
||
run("persistence", ["make", "test-persistence", "BASE_URL=" + api_base_url(), "ACTION_SCRIPT=scripts/restart.sh"])
|
||
run("after-restart", self_cmd + ["--probe", "inventory"])
|
||
before = {r["id"]: r for r in load(directory / "before-restart.txt")} if results["before-restart"]["exit_code"] == 0 else {}
|
||
after = {r["id"]: r for r in load(directory / "after-restart.txt")} if results["after-restart"]["exit_code"] == 0 else {}
|
||
services = {"db", "redis", "s3", "broker", "api", "worker", "dispatcher", "upload-cleanup"}
|
||
audited = [r for r in before.values() if r["service"] in services]
|
||
passed = bool(audited) and services <= {r["service"] for r in audited} and all(
|
||
r["id"] in after and after[r["id"]]["started"] != r["started"] and after[r["id"]]["mounts"] == r["mounts"] for r in audited)
|
||
results["restart-audit"] = dict(command="compare before-restart / after-restart: StartedAt and mounts", exit_code=0 if passed else 1, seconds=0)
|
||
(directory / "restart-audit.txt").write_text(json.dumps(dict(processes_restarted_with_same_mounts=passed, containers=len(audited)), indent=2))
|
||
run("failover", ["make", "test-failover", "BASE_URL=" + api_base_url(), "ACTION_SCRIPT=scripts/stop-one.sh"])
|
||
finally:
|
||
run("restore", COMPOSE + ["up", "-d", "--wait"], 300)
|
||
run("final-inventory", self_cmd + ["--probe", "inventory"])
|
||
index = ["# Доказательства критериев", "", f"Запуск: `{stamp}`. Серия: [{campaign.name}](../idempotency/{campaign.name}/summary.json).", "", "| Критерий | Результат |", "| --- | --- |"]
|
||
states = {}
|
||
for slug, title, checks, source in CRITERIA:
|
||
checks = ["startup", *checks, "restore", "final-inventory"]
|
||
status = "PASS" if all(results.get(c, {}).get("exit_code", 125) == 0 for c in checks) else "FAIL / NOT RUN"
|
||
states[slug] = status
|
||
page = [f"# {title}", "", f"Результат: **{status}**. Запуск: `{stamp}`.", "", f"Код: [{source}](../../{source}).", ""]
|
||
for check in checks:
|
||
result = results.get(check, dict(command="not run", exit_code=125, seconds=0))
|
||
output = (directory / (check + ".txt")).read_text() if (directory / (check + ".txt")).exists() else "NOT RUN"
|
||
page += [f"## {check}", "", f"Команда: `{result['command']}`", "", f"Exit code: {result['exit_code']}; время: {result['seconds']} с.", "", f"[Полный вывод](runs/{stamp}/{check}.txt)", "", "````text", output.rstrip(), "````", ""]
|
||
content = "\n".join(page)
|
||
(OUTPUT / (slug + ".md")).write_text(content)
|
||
historical = re.sub(r"\]\(([^)]+)\)", lambda match: "](" + os.path.relpath(
|
||
(OUTPUT / match.group(1)).resolve(), directory).replace(os.sep, "/") + ")", content)
|
||
(directory / (slug + ".md")).write_text(historical)
|
||
index.append(f"| [{title}]({slug}.md) | {status} |")
|
||
(OUTPUT / "README.md").write_text("\n".join(index) + "\n")
|
||
(directory / "summary.json").write_text(json.dumps(dict(run=stamp, campaign=campaign.name, criteria=states,
|
||
generator_sha256=sha(Path(__file__).read_bytes()), results=results), indent=2) + "\n")
|
||
lock.close()
|
||
print("Evidence:", OUTPUT)
|
||
return 0 if all(s == "PASS" for s in states.values()) else 1
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument("--campaign", type=Path)
|
||
parser.add_argument("--probe", choices=("inventory", "experiment"))
|
||
args = parser.parse_args()
|
||
if args.probe == "inventory":
|
||
inventory()
|
||
return 0
|
||
campaign = args.campaign
|
||
if campaign is None:
|
||
candidates = sorted((ROOT / "evidence/idempotency").glob("main-*/manifest.json"))
|
||
if not candidates:
|
||
parser.error("No main campaign; specify --campaign")
|
||
campaign = candidates[-1].parent # Latest attempt, never select by success.
|
||
campaign = campaign.resolve()
|
||
if args.probe == "experiment":
|
||
audit_experiment(campaign)
|
||
return 0
|
||
return generate(campaign)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|