test: verify cluster failures and clean clone reproducibility

This commit is contained in:
2026-11-15 19:12:35 +03:00
parent 7927228810
commit e779ff1c49
4 changed files with 371 additions and 1 deletions
+2
View File
@@ -34,3 +34,5 @@ coverage/
.work/
.pytest_cache/
.ruff_cache/
.work-*.txt
+117
View File
@@ -0,0 +1,117 @@
import argparse
import csv
import json
import statistics
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
def main():
parser = argparse.ArgumentParser()
parser.add_argument("directory")
args = parser.parse_args()
directory = Path(args.directory)
meta = json.loads((directory / "run.json").read_text())
assert meta["complete"]
rows = [
json.loads((directory / name).read_text()) for name in meta["cells"]
]
keys = [
"series",
"replicas",
"concurrency",
"repeat",
"pool_size",
"successful_rps",
"errors",
"error_ratio",
"p50_ms",
"p95_ms",
"p99_ms",
]
with (directory / "summary.csv").open("w", newline="") as stream:
writer = csv.DictWriter(stream, fieldnames=keys)
writer.writeheader()
writer.writerows({k: r[k] for k in keys} for r in rows)
fig, axes = plt.subplots(1, 2, figsize=(11, 4), constrained_layout=True)
for ax, series in zip(axes, ["fixed", "growing"], strict=True):
for concurrency in [1, 8, 32]:
values = [
statistics.median(
r["successful_rps"]
for r in rows
if r["series"] == series
and r["replicas"] == n
and r["concurrency"] == concurrency
)
for n in [1, 2, 4]
]
ax.plot([1, 2, 4], values, marker="o", label=f"C={concurrency}")
ax.set(
title=series,
xlabel="API replicas",
ylabel="Successful HTTP requests/s",
xticks=[1, 2, 4],
)
ax.legend()
ax.grid(alpha=0.3)
fig.savefig(directory / "scaling.png", dpi=160)
plt.close(fig)
pool = {
n: statistics.median(
r["successful_rps"]
for r in rows
if r["series"] == "pool" and r["pool_size"] == n
)
for n in [1, 10]
}
lines = [
"# Измерения масштабирования",
"",
f"Source commit: `{meta['source_commit']}`.",
"",
"Медианы повторов; latency percentile внутри каждого прогона относится только к успешным ответам.",
"",
"| Серия | API | C | RPS | p95, ms | Ошибки |",
"|---|---:|---:|---:|---:|---:|",
]
for series in ["fixed", "growing"]:
for n in [1, 2, 4]:
for c in [1, 8, 32]:
group = [
r
for r in rows
if r["series"] == series
and r["replicas"] == n
and r["concurrency"] == c
]
lines.append(
f"| {series} | {n} | {c} | {statistics.median(r['successful_rps'] for r in group):.1f} | {statistics.median(r['p95_ms'] for r in group):.2f} | {sum(r['errors'] for r in group)} |"
)
lines.extend(
[
"",
f"Пул1 → пул10: {pool[1]:.1f} → {pool[10]:.1f} успешных RPS (N2/C32, три повтора).",
"",
"Настройка считается подтверждённым улучшением только при наблюдаемом выигрыше; это ограниченная локальная серия.",
]
)
(directory / "analysis.md").write_text("\n".join(lines) + "\n")
print(
json.dumps(
{
"cells": len(rows),
"success": sum(r["success"] for r in rows),
"errors": sum(r["errors"] for r in rows),
"pool_rps": pool,
}
)
)
if __name__ == "__main__":
main()
+201
View File
@@ -0,0 +1,201 @@
import argparse
import atexit
import json
import os
import socket
import subprocess
import tempfile
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4
ROOT = Path(__file__).resolve().parents[1]
PROGRESS = {}
def save_incomplete():
receipt = PROGRESS.get("receipt")
if receipt and receipt.get("result") != "PASS":
target = PROGRESS["output"].with_name(
PROGRESS["output"].stem + "-failed.json"
)
if target.exists():
target = target.with_name(
target.stem + "-" + uuid4().hex[:8] + target.suffix
)
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(
json.dumps(
{**receipt, "result": "INCOMPLETE"},
ensure_ascii=False,
indent=2,
)
+ "\n"
)
atexit.register(save_incomplete)
def run(args, *, cwd, env, expected=0):
result = subprocess.run(
args, cwd=cwd, env=env, capture_output=True, text=True
)
def redact(text):
return (
text.replace(str(cwd), "<clean-clone>")
.replace(str(ROOT), "<repository>")
.replace(str(Path.home()), "<home>")
)
if result.returncode != expected:
if "receipt" in PROGRESS:
PROGRESS["receipt"]["failed_action"] = redact(" ".join(args))
raise RuntimeError(redact(result.stdout + result.stderr))
return redact(result.stdout.strip())
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/clean-clone.json")
args = parser.parse_args()
output = ROOT / args.output
if output.exists():
raise SystemExit("Choose new output path")
assert not subprocess.check_output(
["git", "status", "--porcelain"], cwd=ROOT
)
sha = subprocess.check_output(
["git", "rev-parse", "HEAD"], cwd=ROOT, text=True
).strip()
identity = uuid4().hex[:10]
project = "mtusi-lab3-clean-" + identity
with socket.socket() as reservation:
reservation.bind(("127.0.0.1", 0))
public_port = str(reservation.getsockname()[1])
env = {
**os.environ,
"COMPOSE_PROJECT_NAME": project,
"API_IMAGE": "mtusi-messenger:lab3-clean-" + identity,
"PUBLIC_PORT": public_port,
"VCS_REF": sha,
"NO_PROXY": "localhost,127.0.0.1",
"no_proxy": "localhost,127.0.0.1",
}
receipt = {
"date": datetime.now(UTC).isoformat(),
"source_commit": sha,
"fresh_secrets_environment_and_volumes": True,
"command": "python scripts/verify_clean.py",
"fixed_host_port_for_restart": int(public_port),
}
PROGRESS.update(receipt=receipt, output=output)
with tempfile.TemporaryDirectory(prefix="mtusi-lab3-clean-") as temporary:
clone = Path(temporary) / "lab3"
run(
[
"git",
"clone",
"--no-hardlinks",
"--branch",
"solution/bvt2301-maslov",
str(ROOT),
str(clone),
],
cwd=ROOT,
env=env,
)
for name in [".env", ".venv", "secrets"]:
assert not (clone / name).exists()
run(["uv", "sync", "--locked"], cwd=clone, env=env)
receipt["static_checks"] = run(
["make", "check", "validate", "lint", "PYTHON=.venv/bin/python"],
cwd=clone,
env=env,
)
run(["make", "bootstrap"], cwd=clone, env=env)
try:
run(
["docker", "compose", "up", "-d", "--build", "--wait"],
cwd=clone,
env=env,
)
env["BASE_URL"] = "http://127.0.0.1:" + public_port
receipt["smoke"] = run(
["make", "test", "BASE_URL=" + env["BASE_URL"]],
cwd=clone,
env=env,
)
receipt["http_tests"] = run(
[".venv/bin/pytest", "student-tests", "-q"], cwd=clone, env=env
)
run(
[
".venv/bin/python",
"-m",
"scripts.verify_cluster",
"--output",
"evidence/clean-cluster.json",
],
cwd=clone,
env=env,
)
receipt["cluster"] = json.loads(
(clone / "evidence/clean-cluster.json").read_text()
)
import hashlib
expected = {
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
for p in (clone / "src").glob("*.py")
}
ids = run(
["docker", "compose", "ps", "-q", "api"], cwd=clone, env=env
).split()
assert len(ids) == 2
code = (
"import hashlib,json,pathlib; print(json.dumps({p.name:"
"hashlib.sha256(p.read_bytes()).hexdigest() for p in pathlib.Path('/app/src').glob('*.py')}))"
)
for cid in ids:
actual = json.loads(
run(
["docker", "exec", cid, "python", "-c", code],
cwd=clone,
env=env,
)
)
assert actual == expected
info = json.loads(
run(["docker", "inspect", cid], cwd=clone, env=env)
)[0]
assert (
info["Config"]["Labels"][
"org.opencontainers.image.revision"
]
== sha
)
assert not info["HostConfig"]["PortBindings"]
receipt["both_replicas_runtime_hashes_and_revision_match"] = True
receipt["fsck"] = run(
["git", "fsck", "--full"], cwd=clone, env=env
)
receipt["checks_completed"] = True
finally:
assert project.startswith("mtusi-lab3-clean-")
run(
["docker", "compose", "down", "--volumes", "--remove-orphans"],
cwd=clone,
env=env,
)
receipt["result"] = "PASS"
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + "\n")
print(
"PASS: fresh clone, build, HTTP tests, failover, rolling, persistence and runtime hashes"
)
if __name__ == "__main__":
main()
+51 -1
View File
@@ -14,7 +14,7 @@ from uuid import uuid4
import httpx
from scripts import cluster
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
def snapshot():
@@ -179,6 +179,56 @@ def main():
"messages": 1,
"instances": sorted({m[1] for m in repeated}),
}
# Сбой второй записи транзакции не должен оставить сообщение.
fault_key = uuid4().hex
fault_text = "Atomic rollback " + fault_key
sql(
"CREATE FUNCTION student_idempo_fault() RETURNS trigger "
"LANGUAGE plpgsql AS $$ BEGIN IF NEW.key='"
+ fault_key
+ "' THEN RAISE EXCEPTION 'synthetic'; END IF; RETURN NEW; END $$; "
"CREATE TRIGGER student_idempo_fault BEFORE INSERT ON idempotency "
"FOR EACH ROW EXECUTE FUNCTION student_idempo_fault()"
)
try:
failed = client.post(
path,
headers={**auth(session), "Idempotency-Key": fault_key},
json={"text": fault_text},
)
assert failed.status_code == 500
assert (
sql(
"SELECT count(*) FROM messages WHERE text='"
+ fault_text
+ "'"
)
== "0"
)
finally:
sql(
"DROP TRIGGER student_idempo_fault ON idempotency; "
"DROP FUNCTION student_idempo_fault()"
)
retried = client.post(
path,
headers={**auth(session), "Idempotency-Key": fault_key},
json={"text": fault_text},
)
assert retried.status_code == 201
report["atomic_rollback_and_5xx_retry"] = True
sql(
"UPDATE idempotency SET expires_at=clock_timestamp()-interval '1 second' "
"WHERE key='" + fault_key + "'"
)
expired = client.post(
path,
headers={**auth(session), "Idempotency-Key": fault_key},
json={"text": "New retention window"},
)
assert expired.status_code == 201
assert expired.json()["id"] != retried.json()["id"]
report["expired_key_can_be_reused"] = True
# Отдельный чат каждой серии: проверки дубликатов не смешивают тексты.
report["failovers"] = []
for api in apis: