Files
mtusi2026lab3/scripts/verify.py
T

151 lines
7.2 KiB
Python

"""Reproduce grade-3 evidence against the lab3 Compose stack; no credentials in output."""
import argparse
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import subprocess
import sys
import threading
import time
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure
COMPOSE = ["docker", "compose", "--env-file", ".env"]
def command(args, capture=False):
result = subprocess.run(args, cwd=ROOT, text=True, encoding="utf-8", errors="replace",
stdout=subprocess.PIPE if capture else sys.stdout,
stderr=subprocess.STDOUT, timeout=180)
if result.returncode:
raise RuntimeError("Command failed: " + args[0])
return result.stdout
def sql(query):
return command(COMPOSE + ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-Atc", query], True).strip()
def state():
result = {}
for service in ("api1", "api2", "postgres", "lb"):
cid = command(COMPOSE + ["ps", "-a", "-q", service], True).strip()
data = json.loads(command(["docker", "inspect", cid], True))[0]
result[service] = {"id": cid[:12], "status": data["State"]["Status"],
"started_at": data["State"]["StartedAt"],
"volumes": [m["Name"] for m in data["Mounts"] if m["Type"] == "volume"],
"ports": data["HostConfig"]["PortBindings"],
"memory_limit": data["HostConfig"]["Memory"],
"nano_cpus": data["HostConfig"]["NanoCpus"]}
return result
def wait_replicas(c, actor):
deadline, seen = time.monotonic() + 60, set()
while time.monotonic() < deadline:
try:
seen.add(c.call("GET", "/api/v1/users/me", actor=actor, schema="User")["headers"]["X-Instance-Id"])
if len(seen) >= 2:
return seen
except Failure:
pass
time.sleep(0.1)
raise AssertionError("Two replicas did not become ready")
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--base-url", default="http://127.0.0.1:8080")
p.add_argument("--output")
args = p.parse_args()
if args.output:
sys.stdout = open(ROOT / args.output, "w", encoding="utf-8", buffering=1)
sys.stderr = sys.stdout
os.environ.update(BASE_URL=args.base_url, NO_PROXY="localhost,127.0.0.1", no_proxy="localhost,127.0.0.1")
print("UTC " + datetime.now(timezone.utc).isoformat(), flush=True)
print("COMMAND python scripts/verify.py --base-url " + args.base_url, flush=True)
command(["docker", "version", "--format", "Client={{.Client.Version}} Server={{.Server.Version}}"])
command(["docker", "info", "--format", "Docker CPUs={{.NCPU}} memory_bytes={{.MemTotal}} OS={{.OperatingSystem}}"])
command(COMPOSE + ["config", "--quiet"])
command([sys.executable, "scripts/check_contract.py"])
command([sys.executable, "tests/smoke.py", "--base-url", args.base_url])
command([sys.executable, "-m", "unittest", "discover", "-s", "student-tests", "-v"])
c = Client(args.base_url, timeout=2)
a, b, _, chat = c.bootstrap()
seen = wait_replicas(c, a)
print("OBSERVED_INSTANCES " + json.dumps(sorted(seen)), flush=True)
first = c.message(a, chat, "survives restart", key="restart-key")["json"]
second = c.message(b, chat, "cursor after restart")["json"]
path = "/api/v1/chats/" + chat["id"] + "/messages"
page = c.call("GET", path + "?limit=1", actor=a, schema="MessagePage")["json"]
counts = "SELECT (SELECT count(*) FROM users),(SELECT count(*) FROM chats),(SELECT count(*) FROM memberships),(SELECT count(*) FROM messages),(SELECT count(*) FROM sessions),(SELECT count(*) FROM message_keys),(SELECT count(*) FROM schema_migrations)"
before_counts = sql(counts)
for _ in range(2):
command(COMPOSE + ["run", "--rm", "--no-deps", "migrate"])
assert sql(counts) == before_counts
print("PASS: repeated migrations preserve all table counts " + before_counts, flush=True)
before = state()
print("BEFORE_RESTART " + json.dumps(before), flush=True)
hook = "scripts/restart.cmd" if os.name == "nt" else "scripts/restart.sh"
command([sys.executable, "tests/resilience.py", "--mode", "persistence", "--action-script", hook, "--base-url", args.base_url])
after = state()
print("AFTER_RESTART " + json.dumps(after), flush=True)
for service in before:
assert before[service]["started_at"] != after[service]["started_at"]
assert before[service]["volumes"] == after[service]["volumes"]
assert c.message(a, chat, "survives restart", key="restart-key")["json"] == first
rest = c.call("GET", path + "?limit=1&cursor=" + quote(page["next_cursor"]), actor=a, schema="MessagePage")["json"]
assert page["items"] == [first] and rest["items"] == [second]
print("PASS: four containers restarted; volume, sessions, cursor and idempotency result preserved", flush=True)
stopped = set()
hook = "scripts/stop-one.cmd" if os.name == "nt" else "scripts/stop-one.sh"
# Run the unchanged public test until it has selected each real replica.
# Read concurrently with its POST stream to also measure read availability.
for attempt in range(6):
if stopped == {"api1", "api2"}:
break
command(COMPOSE + ["up", "-d", "--wait", "api1", "api2"], True)
wait_replicas(c, a)
done, reads = threading.Event(), {"ok": 0, "errors": 0}
def read_during_failure():
reader = Client(args.base_url, timeout=2)
while not done.is_set():
try:
result = reader.call("GET", path, actor=b, schema="MessagePage")["json"]
assert first in result["items"]
reads["ok"] += 1
except Failure:
reads["errors"] += 1
done.wait(0.1)
worker = threading.Thread(target=read_during_failure)
worker.start()
try:
print("FAILOVER_RUN " + str(attempt + 1), flush=True)
command([sys.executable, "tests/resilience.py", "--mode", "failover", "--action-script", hook, "--base-url", args.base_url])
current = state()
down = {s for s in ("api1", "api2") if current[s]["status"] == "exited"}
assert len(down) == 1
stopped.update(down)
print("AFTER_FAILOVER " + json.dumps(current), flush=True)
assert reads["ok"] >= 5
print("CONCURRENT_READS " + json.dumps(reads), flush=True)
assert c.message(a, chat, "survives restart", key="restart-key")["json"] == first
finally:
done.set()
worker.join(timeout=5)
command(COMPOSE + ["up", "-d", "--wait", "api1", "api2"], True)
assert stopped == {"api1", "api2"}, "Both replicas must be tested"
wait_replicas(c, a)
assert sql("SELECT count(*) FROM chats c WHERE NOT EXISTS (SELECT 1 FROM memberships m WHERE m.chat_id=c.id)") == "0"
print("PASS: each API stopped separately; reads/writes and original session survive; both replicas restored", flush=True)
print("PASS: grade-3 verification complete", flush=True)
if __name__ == "__main__":
main()