test: verify cluster failures and clean clone reproducibility
This commit is contained in:
@@ -0,0 +1,201 @@
|
||||
import argparse
|
||||
import atexit
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PROGRESS = {}
|
||||
|
||||
|
||||
def save_incomplete():
|
||||
receipt = PROGRESS.get("receipt")
|
||||
if receipt and receipt.get("result") != "PASS":
|
||||
target = PROGRESS["output"].with_name(
|
||||
PROGRESS["output"].stem + "-failed.json"
|
||||
)
|
||||
if target.exists():
|
||||
target = target.with_name(
|
||||
target.stem + "-" + uuid4().hex[:8] + target.suffix
|
||||
)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(
|
||||
json.dumps(
|
||||
{**receipt, "result": "INCOMPLETE"},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
|
||||
atexit.register(save_incomplete)
|
||||
|
||||
|
||||
def run(args, *, cwd, env, expected=0):
|
||||
result = subprocess.run(
|
||||
args, cwd=cwd, env=env, capture_output=True, text=True
|
||||
)
|
||||
|
||||
def redact(text):
|
||||
return (
|
||||
text.replace(str(cwd), "<clean-clone>")
|
||||
.replace(str(ROOT), "<repository>")
|
||||
.replace(str(Path.home()), "<home>")
|
||||
)
|
||||
|
||||
if result.returncode != expected:
|
||||
if "receipt" in PROGRESS:
|
||||
PROGRESS["receipt"]["failed_action"] = redact(" ".join(args))
|
||||
raise RuntimeError(redact(result.stdout + result.stderr))
|
||||
return redact(result.stdout.strip())
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", default="evidence/clean-clone.json")
|
||||
args = parser.parse_args()
|
||||
output = ROOT / args.output
|
||||
if output.exists():
|
||||
raise SystemExit("Choose new output path")
|
||||
assert not subprocess.check_output(
|
||||
["git", "status", "--porcelain"], cwd=ROOT
|
||||
)
|
||||
sha = subprocess.check_output(
|
||||
["git", "rev-parse", "HEAD"], cwd=ROOT, text=True
|
||||
).strip()
|
||||
identity = uuid4().hex[:10]
|
||||
project = "mtusi-lab3-clean-" + identity
|
||||
with socket.socket() as reservation:
|
||||
reservation.bind(("127.0.0.1", 0))
|
||||
public_port = str(reservation.getsockname()[1])
|
||||
env = {
|
||||
**os.environ,
|
||||
"COMPOSE_PROJECT_NAME": project,
|
||||
"API_IMAGE": "mtusi-messenger:lab3-clean-" + identity,
|
||||
"PUBLIC_PORT": public_port,
|
||||
"VCS_REF": sha,
|
||||
"NO_PROXY": "localhost,127.0.0.1",
|
||||
"no_proxy": "localhost,127.0.0.1",
|
||||
}
|
||||
receipt = {
|
||||
"date": datetime.now(UTC).isoformat(),
|
||||
"source_commit": sha,
|
||||
"fresh_secrets_environment_and_volumes": True,
|
||||
"command": "python scripts/verify_clean.py",
|
||||
"fixed_host_port_for_restart": int(public_port),
|
||||
}
|
||||
PROGRESS.update(receipt=receipt, output=output)
|
||||
with tempfile.TemporaryDirectory(prefix="mtusi-lab3-clean-") as temporary:
|
||||
clone = Path(temporary) / "lab3"
|
||||
run(
|
||||
[
|
||||
"git",
|
||||
"clone",
|
||||
"--no-hardlinks",
|
||||
"--branch",
|
||||
"solution/bvt2301-maslov",
|
||||
str(ROOT),
|
||||
str(clone),
|
||||
],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
)
|
||||
for name in [".env", ".venv", "secrets"]:
|
||||
assert not (clone / name).exists()
|
||||
run(["uv", "sync", "--locked"], cwd=clone, env=env)
|
||||
receipt["static_checks"] = run(
|
||||
["make", "check", "validate", "lint", "PYTHON=.venv/bin/python"],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
run(["make", "bootstrap"], cwd=clone, env=env)
|
||||
try:
|
||||
run(
|
||||
["docker", "compose", "up", "-d", "--build", "--wait"],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
env["BASE_URL"] = "http://127.0.0.1:" + public_port
|
||||
receipt["smoke"] = run(
|
||||
["make", "test", "BASE_URL=" + env["BASE_URL"]],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
receipt["http_tests"] = run(
|
||||
[".venv/bin/pytest", "student-tests", "-q"], cwd=clone, env=env
|
||||
)
|
||||
run(
|
||||
[
|
||||
".venv/bin/python",
|
||||
"-m",
|
||||
"scripts.verify_cluster",
|
||||
"--output",
|
||||
"evidence/clean-cluster.json",
|
||||
],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
receipt["cluster"] = json.loads(
|
||||
(clone / "evidence/clean-cluster.json").read_text()
|
||||
)
|
||||
import hashlib
|
||||
|
||||
expected = {
|
||||
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
for p in (clone / "src").glob("*.py")
|
||||
}
|
||||
ids = run(
|
||||
["docker", "compose", "ps", "-q", "api"], cwd=clone, env=env
|
||||
).split()
|
||||
assert len(ids) == 2
|
||||
code = (
|
||||
"import hashlib,json,pathlib; print(json.dumps({p.name:"
|
||||
"hashlib.sha256(p.read_bytes()).hexdigest() for p in pathlib.Path('/app/src').glob('*.py')}))"
|
||||
)
|
||||
for cid in ids:
|
||||
actual = json.loads(
|
||||
run(
|
||||
["docker", "exec", cid, "python", "-c", code],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
)
|
||||
assert actual == expected
|
||||
info = json.loads(
|
||||
run(["docker", "inspect", cid], cwd=clone, env=env)
|
||||
)[0]
|
||||
assert (
|
||||
info["Config"]["Labels"][
|
||||
"org.opencontainers.image.revision"
|
||||
]
|
||||
== sha
|
||||
)
|
||||
assert not info["HostConfig"]["PortBindings"]
|
||||
receipt["both_replicas_runtime_hashes_and_revision_match"] = True
|
||||
receipt["fsck"] = run(
|
||||
["git", "fsck", "--full"], cwd=clone, env=env
|
||||
)
|
||||
receipt["checks_completed"] = True
|
||||
finally:
|
||||
assert project.startswith("mtusi-lab3-clean-")
|
||||
run(
|
||||
["docker", "compose", "down", "--volumes", "--remove-orphans"],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
receipt["result"] = "PASS"
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + "\n")
|
||||
print(
|
||||
"PASS: fresh clone, build, HTTP tests, failover, rolling, persistence and runtime hashes"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -14,7 +14,7 @@ from uuid import uuid4
|
||||
import httpx
|
||||
|
||||
from scripts import cluster
|
||||
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
|
||||
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
|
||||
|
||||
|
||||
def snapshot():
|
||||
@@ -179,6 +179,56 @@ def main():
|
||||
"messages": 1,
|
||||
"instances": sorted({m[1] for m in repeated}),
|
||||
}
|
||||
# Сбой второй записи транзакции не должен оставить сообщение.
|
||||
fault_key = uuid4().hex
|
||||
fault_text = "Atomic rollback " + fault_key
|
||||
sql(
|
||||
"CREATE FUNCTION student_idempo_fault() RETURNS trigger "
|
||||
"LANGUAGE plpgsql AS $$ BEGIN IF NEW.key='"
|
||||
+ fault_key
|
||||
+ "' THEN RAISE EXCEPTION 'synthetic'; END IF; RETURN NEW; END $$; "
|
||||
"CREATE TRIGGER student_idempo_fault BEFORE INSERT ON idempotency "
|
||||
"FOR EACH ROW EXECUTE FUNCTION student_idempo_fault()"
|
||||
)
|
||||
try:
|
||||
failed = client.post(
|
||||
path,
|
||||
headers={**auth(session), "Idempotency-Key": fault_key},
|
||||
json={"text": fault_text},
|
||||
)
|
||||
assert failed.status_code == 500
|
||||
assert (
|
||||
sql(
|
||||
"SELECT count(*) FROM messages WHERE text='"
|
||||
+ fault_text
|
||||
+ "'"
|
||||
)
|
||||
== "0"
|
||||
)
|
||||
finally:
|
||||
sql(
|
||||
"DROP TRIGGER student_idempo_fault ON idempotency; "
|
||||
"DROP FUNCTION student_idempo_fault()"
|
||||
)
|
||||
retried = client.post(
|
||||
path,
|
||||
headers={**auth(session), "Idempotency-Key": fault_key},
|
||||
json={"text": fault_text},
|
||||
)
|
||||
assert retried.status_code == 201
|
||||
report["atomic_rollback_and_5xx_retry"] = True
|
||||
sql(
|
||||
"UPDATE idempotency SET expires_at=clock_timestamp()-interval '1 second' "
|
||||
"WHERE key='" + fault_key + "'"
|
||||
)
|
||||
expired = client.post(
|
||||
path,
|
||||
headers={**auth(session), "Idempotency-Key": fault_key},
|
||||
json={"text": "New retention window"},
|
||||
)
|
||||
assert expired.status_code == 201
|
||||
assert expired.json()["id"] != retried.json()["id"]
|
||||
report["expired_key_can_be_reused"] = True
|
||||
# Отдельный чат каждой серии: проверки дубликатов не смешивают тексты.
|
||||
report["failovers"] = []
|
||||
for api in apis:
|
||||
|
||||
Reference in New Issue
Block a user