From e779ff1c49ee61570bfbf8269ecd7da8223f9120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=81=D0=BB=D0=BE=D0=B2?= Date: Sun, 15 Nov 2026 19:12:35 +0300 Subject: [PATCH] test: verify cluster failures and clean clone reproducibility --- .gitignore | 2 + research/analyze.py | 117 ++++++++++++++++++++++ scripts/verify_clean.py | 201 ++++++++++++++++++++++++++++++++++++++ scripts/verify_cluster.py | 52 +++++++++- 4 files changed, 371 insertions(+), 1 deletion(-) create mode 100644 research/analyze.py create mode 100644 scripts/verify_clean.py diff --git a/.gitignore b/.gitignore index fe66e37d..673137ba 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,5 @@ coverage/ .work/ .pytest_cache/ .ruff_cache/ + +.work-*.txt diff --git a/research/analyze.py b/research/analyze.py new file mode 100644 index 00000000..07e1a4ea --- /dev/null +++ b/research/analyze.py @@ -0,0 +1,117 @@ +import argparse +import csv +import json +import statistics +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("directory") + args = parser.parse_args() + directory = Path(args.directory) + meta = json.loads((directory / "run.json").read_text()) + assert meta["complete"] + rows = [ + json.loads((directory / name).read_text()) for name in meta["cells"] + ] + keys = [ + "series", + "replicas", + "concurrency", + "repeat", + "pool_size", + "successful_rps", + "errors", + "error_ratio", + "p50_ms", + "p95_ms", + "p99_ms", + ] + with (directory / "summary.csv").open("w", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=keys) + writer.writeheader() + writer.writerows({k: r[k] for k in keys} for r in rows) + fig, axes = plt.subplots(1, 2, figsize=(11, 4), constrained_layout=True) + for ax, series in zip(axes, ["fixed", "growing"], strict=True): + for concurrency in [1, 8, 32]: + values = [ + statistics.median( + r["successful_rps"] + for r in rows + if r["series"] == series + and r["replicas"] == n + and r["concurrency"] == concurrency + ) + for n in [1, 2, 4] + ] + ax.plot([1, 2, 4], values, marker="o", label=f"C={concurrency}") + ax.set( + title=series, + xlabel="API replicas", + ylabel="Successful HTTP requests/s", + xticks=[1, 2, 4], + ) + ax.legend() + ax.grid(alpha=0.3) + fig.savefig(directory / "scaling.png", dpi=160) + plt.close(fig) + pool = { + n: statistics.median( + r["successful_rps"] + for r in rows + if r["series"] == "pool" and r["pool_size"] == n + ) + for n in [1, 10] + } + lines = [ + "# Измерения масштабирования", + "", + f"Source commit: `{meta['source_commit']}`.", + "", + "Медианы повторов; latency percentile внутри каждого прогона относится только к успешным ответам.", + "", + "| Серия | API | C | RPS | p95, ms | Ошибки |", + "|---|---:|---:|---:|---:|---:|", + ] + for series in ["fixed", "growing"]: + for n in [1, 2, 4]: + for c in [1, 8, 32]: + group = [ + r + for r in rows + if r["series"] == series + and r["replicas"] == n + and r["concurrency"] == c + ] + lines.append( + f"| {series} | {n} | {c} | {statistics.median(r['successful_rps'] for r in group):.1f} | {statistics.median(r['p95_ms'] for r in group):.2f} | {sum(r['errors'] for r in group)} |" + ) + lines.extend( + [ + "", + f"Пул1 → пул10: {pool[1]:.1f} → {pool[10]:.1f} успешных RPS (N2/C32, три повтора).", + "", + "Настройка считается подтверждённым улучшением только при наблюдаемом выигрыше; это ограниченная локальная серия.", + ] + ) + (directory / "analysis.md").write_text("\n".join(lines) + "\n") + print( + json.dumps( + { + "cells": len(rows), + "success": sum(r["success"] for r in rows), + "errors": sum(r["errors"] for r in rows), + "pool_rps": pool, + } + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_clean.py b/scripts/verify_clean.py new file mode 100644 index 00000000..896c1cbc --- /dev/null +++ b/scripts/verify_clean.py @@ -0,0 +1,201 @@ +import argparse +import atexit +import json +import os +import socket +import subprocess +import tempfile +from datetime import UTC, datetime +from pathlib import Path +from uuid import uuid4 + +ROOT = Path(__file__).resolve().parents[1] +PROGRESS = {} + + +def save_incomplete(): + receipt = PROGRESS.get("receipt") + if receipt and receipt.get("result") != "PASS": + target = PROGRESS["output"].with_name( + PROGRESS["output"].stem + "-failed.json" + ) + if target.exists(): + target = target.with_name( + target.stem + "-" + uuid4().hex[:8] + target.suffix + ) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text( + json.dumps( + {**receipt, "result": "INCOMPLETE"}, + ensure_ascii=False, + indent=2, + ) + + "\n" + ) + + +atexit.register(save_incomplete) + + +def run(args, *, cwd, env, expected=0): + result = subprocess.run( + args, cwd=cwd, env=env, capture_output=True, text=True + ) + + def redact(text): + return ( + text.replace(str(cwd), "") + .replace(str(ROOT), "") + .replace(str(Path.home()), "") + ) + + if result.returncode != expected: + if "receipt" in PROGRESS: + PROGRESS["receipt"]["failed_action"] = redact(" ".join(args)) + raise RuntimeError(redact(result.stdout + result.stderr)) + return redact(result.stdout.strip()) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--output", default="evidence/clean-clone.json") + args = parser.parse_args() + output = ROOT / args.output + if output.exists(): + raise SystemExit("Choose new output path") + assert not subprocess.check_output( + ["git", "status", "--porcelain"], cwd=ROOT + ) + sha = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=ROOT, text=True + ).strip() + identity = uuid4().hex[:10] + project = "mtusi-lab3-clean-" + identity + with socket.socket() as reservation: + reservation.bind(("127.0.0.1", 0)) + public_port = str(reservation.getsockname()[1]) + env = { + **os.environ, + "COMPOSE_PROJECT_NAME": project, + "API_IMAGE": "mtusi-messenger:lab3-clean-" + identity, + "PUBLIC_PORT": public_port, + "VCS_REF": sha, + "NO_PROXY": "localhost,127.0.0.1", + "no_proxy": "localhost,127.0.0.1", + } + receipt = { + "date": datetime.now(UTC).isoformat(), + "source_commit": sha, + "fresh_secrets_environment_and_volumes": True, + "command": "python scripts/verify_clean.py", + "fixed_host_port_for_restart": int(public_port), + } + PROGRESS.update(receipt=receipt, output=output) + with tempfile.TemporaryDirectory(prefix="mtusi-lab3-clean-") as temporary: + clone = Path(temporary) / "lab3" + run( + [ + "git", + "clone", + "--no-hardlinks", + "--branch", + "solution/bvt2301-maslov", + str(ROOT), + str(clone), + ], + cwd=ROOT, + env=env, + ) + for name in [".env", ".venv", "secrets"]: + assert not (clone / name).exists() + run(["uv", "sync", "--locked"], cwd=clone, env=env) + receipt["static_checks"] = run( + ["make", "check", "validate", "lint", "PYTHON=.venv/bin/python"], + cwd=clone, + env=env, + ) + run(["make", "bootstrap"], cwd=clone, env=env) + try: + run( + ["docker", "compose", "up", "-d", "--build", "--wait"], + cwd=clone, + env=env, + ) + env["BASE_URL"] = "http://127.0.0.1:" + public_port + receipt["smoke"] = run( + ["make", "test", "BASE_URL=" + env["BASE_URL"]], + cwd=clone, + env=env, + ) + receipt["http_tests"] = run( + [".venv/bin/pytest", "student-tests", "-q"], cwd=clone, env=env + ) + run( + [ + ".venv/bin/python", + "-m", + "scripts.verify_cluster", + "--output", + "evidence/clean-cluster.json", + ], + cwd=clone, + env=env, + ) + receipt["cluster"] = json.loads( + (clone / "evidence/clean-cluster.json").read_text() + ) + import hashlib + + expected = { + p.name: hashlib.sha256(p.read_bytes()).hexdigest() + for p in (clone / "src").glob("*.py") + } + ids = run( + ["docker", "compose", "ps", "-q", "api"], cwd=clone, env=env + ).split() + assert len(ids) == 2 + code = ( + "import hashlib,json,pathlib; print(json.dumps({p.name:" + "hashlib.sha256(p.read_bytes()).hexdigest() for p in pathlib.Path('/app/src').glob('*.py')}))" + ) + for cid in ids: + actual = json.loads( + run( + ["docker", "exec", cid, "python", "-c", code], + cwd=clone, + env=env, + ) + ) + assert actual == expected + info = json.loads( + run(["docker", "inspect", cid], cwd=clone, env=env) + )[0] + assert ( + info["Config"]["Labels"][ + "org.opencontainers.image.revision" + ] + == sha + ) + assert not info["HostConfig"]["PortBindings"] + receipt["both_replicas_runtime_hashes_and_revision_match"] = True + receipt["fsck"] = run( + ["git", "fsck", "--full"], cwd=clone, env=env + ) + receipt["checks_completed"] = True + finally: + assert project.startswith("mtusi-lab3-clean-") + run( + ["docker", "compose", "down", "--volumes", "--remove-orphans"], + cwd=clone, + env=env, + ) + receipt["result"] = "PASS" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + "\n") + print( + "PASS: fresh clone, build, HTTP tests, failover, rolling, persistence and runtime hashes" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_cluster.py b/scripts/verify_cluster.py index 1eb09607..872696bd 100644 --- a/scripts/verify_cluster.py +++ b/scripts/verify_cluster.py @@ -14,7 +14,7 @@ from uuid import uuid4 import httpx from scripts import cluster -from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready +from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql def snapshot(): @@ -179,6 +179,56 @@ def main(): "messages": 1, "instances": sorted({m[1] for m in repeated}), } + # Сбой второй записи транзакции не должен оставить сообщение. + fault_key = uuid4().hex + fault_text = "Atomic rollback " + fault_key + sql( + "CREATE FUNCTION student_idempo_fault() RETURNS trigger " + "LANGUAGE plpgsql AS $$ BEGIN IF NEW.key='" + + fault_key + + "' THEN RAISE EXCEPTION 'synthetic'; END IF; RETURN NEW; END $$; " + "CREATE TRIGGER student_idempo_fault BEFORE INSERT ON idempotency " + "FOR EACH ROW EXECUTE FUNCTION student_idempo_fault()" + ) + try: + failed = client.post( + path, + headers={**auth(session), "Idempotency-Key": fault_key}, + json={"text": fault_text}, + ) + assert failed.status_code == 500 + assert ( + sql( + "SELECT count(*) FROM messages WHERE text='" + + fault_text + + "'" + ) + == "0" + ) + finally: + sql( + "DROP TRIGGER student_idempo_fault ON idempotency; " + "DROP FUNCTION student_idempo_fault()" + ) + retried = client.post( + path, + headers={**auth(session), "Idempotency-Key": fault_key}, + json={"text": fault_text}, + ) + assert retried.status_code == 201 + report["atomic_rollback_and_5xx_retry"] = True + sql( + "UPDATE idempotency SET expires_at=clock_timestamp()-interval '1 second' " + "WHERE key='" + fault_key + "'" + ) + expired = client.post( + path, + headers={**auth(session), "Idempotency-Key": fault_key}, + json={"text": "New retention window"}, + ) + assert expired.status_code == 201 + assert expired.json()["id"] != retried.json()["id"] + report["expired_key_can_be_reused"] = True # Отдельный чат каждой серии: проверки дубликатов не смешивают тексты. report["failovers"] = [] for api in apis: