Files
mtusi2026lab6/scripts/verify_clean.py

340 lines
11 KiB
Python

import argparse
import atexit
import json
import os
import socket
import subprocess
import tempfile
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4
ROOT = Path(__file__).resolve().parents[1]
PROGRESS = {}
def save_incomplete():
receipt = PROGRESS.get("receipt")
if receipt and receipt.get("result") != "PASS":
target = PROGRESS["output"].with_name(
PROGRESS["output"].stem + "-failed.json"
)
if target.exists():
target = target.with_name(
target.stem + "-" + uuid4().hex[:8] + target.suffix
)
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(
json.dumps(
{**receipt, "result": "INCOMPLETE"},
ensure_ascii=False,
indent=2,
)
+ "\n"
)
atexit.register(save_incomplete)
def redact(text, cwd):
directory = cwd / "secrets"
if directory.exists():
for path in directory.iterdir():
if path.is_file():
value = path.read_text().strip()
if len(value) >= 12:
text = text.replace(value, "<secret>")
return (
text.replace(str(cwd), "<clean-clone>")
.replace(str(ROOT), "<repository>")
.replace(str(Path.home()), "<home>")
)
def run(args, *, cwd, env, expected=0):
print("CHECK: " + redact(" ".join(args[:3]), cwd), flush=True)
result = subprocess.run(
args, cwd=cwd, env=env, capture_output=True, text=True
)
if result.returncode != expected:
if "receipt" in PROGRESS:
PROGRESS["receipt"]["failed_action"] = redact(" ".join(args), cwd)
raise RuntimeError(redact(result.stdout + result.stderr, cwd))
return redact(result.stdout.strip(), cwd)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/clean-clone.json")
args = parser.parse_args()
output = ROOT / args.output
if output.exists():
raise SystemExit("Choose new output path")
assert not subprocess.check_output(
["git", "status", "--porcelain"], cwd=ROOT
)
sha = subprocess.check_output(
["git", "rev-parse", "HEAD"], cwd=ROOT, text=True
).strip()
identity = uuid4().hex[:10]
project = "mtusi-lab6-clean-" + identity
with socket.socket() as reservation:
reservation.bind(("127.0.0.1", 0))
public_port = str(reservation.getsockname()[1])
with socket.socket() as grafana_reservation:
grafana_reservation.bind(("127.0.0.1", 0))
grafana_port = str(grafana_reservation.getsockname()[1])
env = {
**os.environ,
"COMPOSE_PROJECT_NAME": project,
"API_IMAGE": "mtusi-messenger:lab6-clean-" + identity,
"PUBLIC_PORT": public_port,
"GRAFANA_PORT": grafana_port,
"VCS_REF": sha,
"COMPOSE_FILE": "compose.yaml:compose.test.yaml",
"NO_PROXY": "localhost,127.0.0.1",
"no_proxy": "localhost,127.0.0.1",
}
receipt = {
"date": datetime.now(UTC).isoformat(),
"source_commit": sha,
"fresh_secrets_environment_and_volumes": True,
"command": "python scripts/verify_clean.py",
"fixed_host_port_for_restart": int(public_port),
}
PROGRESS.update(receipt=receipt, output=output)
with tempfile.TemporaryDirectory(prefix="mtusi-lab6-clean-") as temporary:
clone = Path(temporary) / "lab6"
run(
[
"git",
"clone",
"--no-hardlinks",
"--branch",
"solution/bvt2301-maslov",
str(ROOT),
str(clone),
],
cwd=ROOT,
env=env,
)
for name in [".env", ".venv", "secrets"]:
assert not (clone / name).exists()
run(["uv", "sync", "--locked"], cwd=clone, env=env)
receipt["static_checks"] = run(
["make", "check", "validate", "lint", "PYTHON=.venv/bin/python"],
cwd=clone,
env=env,
)
receipt["unit_tests"] = run(["make", "unit"], cwd=clone, env=env)
run(["make", "bootstrap"], cwd=clone, env=env)
try:
run(
["docker", "compose", "up", "-d", "--build", "--wait"],
cwd=clone,
env=env,
)
env["BASE_URL"] = "http://127.0.0.1:" + public_port
receipt["smoke"] = run(
["make", "test", "BASE_URL=" + env["BASE_URL"]],
cwd=clone,
env=env,
)
run(
[
".venv/bin/python",
"-m",
"scripts.verify_cluster",
"--output",
"evidence/clean-cluster.json",
],
cwd=clone,
env=env,
)
for command_args in [
[
".venv/bin/python",
"-m",
"scripts.verify_cleanup",
"seed",
"--seed",
"evidence/clean-gc-seed.json",
],
[
".venv/bin/python",
"-m",
"scripts.verify_pipeline",
"--output",
"evidence/clean-pipeline.json",
],
[
".venv/bin/python",
"-m",
"scripts.verify_cleanup",
"verify",
"--seed",
"evidence/clean-gc-seed.json",
"--output",
"evidence/clean-cleanup.json",
],
]:
run(command_args, cwd=clone, env=env)
receipt["pipeline"] = json.loads(
(clone / "evidence/clean-pipeline.json").read_text()
)
receipt["cleanup"] = json.loads(
(clone / "evidence/clean-cleanup.json").read_text()
)
receipt["cluster"] = json.loads(
(clone / "evidence/clean-cluster.json").read_text()
)
receipt["http_tests"] = run(
[".venv/bin/pytest", "student-tests", "-q"], cwd=clone, env=env
)
for module, extra in [
("verify_telemetry", []),
(
"verify_observability_persistence",
["--causal-evidence", "evidence/clean-telemetry.json"],
),
]:
name = (
"telemetry"
if module == "verify_telemetry"
else "observability-persistence"
)
run(
[
".venv/bin/python",
"-m",
"scripts." + module,
"--output",
"evidence/clean-" + name + ".json",
*extra,
],
cwd=clone,
env=env,
)
receipt[name] = json.loads(
(clone / ("evidence/clean-" + name + ".json")).read_text()
)
import hashlib
expected = {
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
for p in (clone / "src").glob("*.py")
}
ids = run(
["docker", "compose", "ps", "-q", "api"], cwd=clone, env=env
).split()
assert len(ids) == 2
code = (
"import hashlib,json,pathlib; print(json.dumps({p.name:"
"hashlib.sha256(p.read_bytes()).hexdigest() for p in pathlib.Path('/app/src').glob('*.py')}))"
)
ids += run(
[
"docker",
"compose",
"ps",
"-q",
"worker",
"relay",
"reconciler",
"controller",
"observer",
],
cwd=clone,
env=env,
).split()
assert len(ids) == 7
for cid in ids:
actual = json.loads(
run(
["docker", "exec", cid, "python", "-c", code],
cwd=clone,
env=env,
)
)
assert actual == expected
info = json.loads(
run(["docker", "inspect", cid], cwd=clone, env=env)
)[0]
assert (
info["Config"]["Labels"][
"org.opencontainers.image.revision"
]
== sha
)
assert not info["HostConfig"]["PortBindings"]
receipt[
"all_seven_application_roles_runtime_hashes_and_revision_match"
] = True
receipt["fsck"] = run(
["git", "fsck", "--full"], cwd=clone, env=env
)
receipt["checks_completed"] = True
except BaseException:
receipt["partial_evidence"] = {}
for name in [
"telemetry",
"observability-persistence",
"cluster",
"pipeline",
"cleanup",
"gc-seed",
]:
partial = clone / "evidence" / ("clean-" + name + ".json")
if partial.exists():
receipt["partial_evidence"][name] = json.loads(
redact(partial.read_text(), clone)
)
receipt["failure_diagnostics"] = {}
for label, arguments in [
("services", ["ps", "-a", "--format", "json"]),
(
"logs",
[
"logs",
"--no-color",
"--tail",
"120",
"lb",
"api",
"s3",
"migrate",
"cache",
],
),
]:
diagnostic = subprocess.run(
["docker", "compose", *arguments],
cwd=clone,
env=env,
capture_output=True,
text=True,
)
receipt["failure_diagnostics"][label] = redact(
diagnostic.stdout + diagnostic.stderr, clone
)
raise
finally:
assert project.startswith("mtusi-lab6-clean-")
run(
["docker", "compose", "down", "--volumes", "--remove-orphans"],
cwd=clone,
env=env,
)
receipt["result"] = "PASS"
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + "\n")
print(
"PASS: fresh clone, build, HTTP tests, failover, rolling, persistence and runtime hashes"
)
if __name__ == "__main__":
main()