Files
mtusi2026lab4/scripts/verify.py

238 lines
12 KiB
Python

"""Reproduce grade-3 evidence against the lab4 Compose stack; no credentials in output."""
import argparse
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import subprocess
import sys
import threading
import time
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure
COMPOSE = ["docker", "compose", "--env-file", ".env"]
def command(args, capture=False):
result = subprocess.run(args, cwd=ROOT, text=True, encoding="utf-8", errors="replace",
stdout=subprocess.PIPE if capture else sys.stdout,
stderr=subprocess.STDOUT, timeout=180)
if result.returncode:
raise RuntimeError("Command failed: " + args[0])
return result.stdout
def sql(query):
return command(COMPOSE + ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-Atc", query], True).strip()
def state():
result = {}
for service in ("api1", "api2", "postgres", "s3", "lb"):
cid = command(COMPOSE + ["ps", "-a", "-q", service], True).strip()
data = json.loads(command(["docker", "inspect", cid], True))[0]
result[service] = {"id": cid[:12], "status": data["State"]["Status"],
"started_at": data["State"]["StartedAt"],
"volumes": [m["Name"] for m in data["Mounts"] if m["Type"] == "volume"],
"ports": data["HostConfig"]["PortBindings"],
"memory_limit": data["HostConfig"]["Memory"],
"nano_cpus": data["HostConfig"]["NanoCpus"]}
return result
def wait_replicas(c, actor):
deadline, seen = time.monotonic() + 60, set()
while time.monotonic() < deadline:
try:
seen.add(c.call("GET", "/api/v1/users/me", actor=actor, schema="User")["headers"]["X-Instance-Id"])
if len(seen) >= 2:
return seen
except Failure:
pass
time.sleep(0.1)
raise AssertionError("Two replicas did not become ready")
def verify_upload_failover(c, a, b, chat):
import hashlib
content = (ROOT / "tests/fixtures/sample.png").read_bytes()
for service in ("api1", "api2"):
wait_replicas(c, a)
for _ in range(30):
response = c.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=failover.png",
201, actor=a, raw=content, headers={"Content-Type": "image/png"}, schema="Attachment")
target = response["headers"]["X-Instance-Id"]
if target.startswith(service + "-"):
break
else:
raise AssertionError("Upload did not reach expected replica")
att = response["json"]
message = c.message(a, chat, "", attachments=[att["id"]])["json"]
env = dict(os.environ, TARGET_INSTANCE_ID=target)
subprocess.run([sys.executable, "scripts/stop_one.py"], cwd=ROOT, env=env, check=True, timeout=20, stdout=sys.stdout, stderr=subprocess.STDOUT)
try:
assert state()[service]["status"] == "exited"
result = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=b)
assert result["bytes"] == content
assert result["headers"]["X-Instance-Id"] != target
assert c.call("GET", "/api/v1/attachments/" + att["id"], actor=b, schema="Attachment")["json"] == att
print("UPLOAD_FAILOVER " + json.dumps({"stopped": target, "download_instance": result["headers"]["X-Instance-Id"],
"sha256": hashlib.sha256(result["bytes"]).hexdigest(),
"message_id": message["id"]}), flush=True)
finally:
command(COMPOSE + ["up", "-d", "--wait", "api1", "api2"], True)
# Inspect mounts: original bytes are never persisted in an API volume.
assert all(not state()[service]["volumes"] for service in ("api1", "api2"))
key = sql("SELECT object_key FROM attachments WHERE id='" + att["id"] + "'")
probe = """import urllib.request, urllib.error, sys, os
try:
urllib.request.urlopen(os.environ['S3_ENDPOINT'] + '/' + os.environ['S3_BUCKET'] + '/' + sys.argv[1],timeout=3)
raise AssertionError('Anonymous S3 GET allowed')
except urllib.error.HTTPError as exc:
assert exc.code == 403, exc.code
print('PASS: anonymous S3 GET of existing object denied (403)')
"""
command(COMPOSE + ["exec", "-T", "api1", "python", "-c", probe, key])
def verify_s3_failure(c, a, b, chat):
att, content = c.attachment(a, chat)
before = sql("SELECT count(*) FROM attachments")
# Execute inside the API network namespace: LB correctly removes unready APIs.
# Program travels through stdin; bearer never appears in command arguments/output.
program = """import urllib.request, urllib.error, json, base64
actor = ACTOR
attachment = ATTACHMENT
chat_id = CHAT_ID
content = base64.b64decode(CONTENT)
def request(path, expected, data=None, mime=None):
headers = {'Authorization': 'Bearer ' + actor['token']}
if mime: headers['Content-Type'] = mime
req = urllib.request.Request('http://127.0.0.1:8080' + path, data=data, headers=headers)
try: response = urllib.request.urlopen(req, timeout=8)
except urllib.error.HTTPError as exc: response = exc
with response:
assert response.code == expected, (path, response.code)
result = response.read()
return result
request('/health/live', 200)
request('/health/ready', 503)
request('/api/v1/attachments/' + attachment['id'], 200)
request('/api/v1/attachments/' + attachment['id'] + '/content', 503)
request('/api/v1/chats/' + chat_id + '/attachments?filename=outage.png', 503, content, 'image/png')
print('PASS: S3 down: live=200 ready=503 metadata=200 download=503 upload=503')
"""
import base64
program = program.replace("ACTOR", repr({"token": a["token"]})).replace("ATTACHMENT", repr(att)).replace("CHAT_ID", repr(chat["id"])).replace("CONTENT", repr(base64.b64encode(content).decode()))
command(COMPOSE + ["stop", "--timeout", "10", "s3"], True)
try:
result = subprocess.run(COMPOSE + ["exec", "-T", "api1", "python", "-"], cwd=ROOT,
input=program, text=True, capture_output=True, timeout=30)
assert result.returncode == 0, "S3 outage check failed (details suppressed to protect session)"
print(result.stdout.strip(), flush=True)
assert sql("SELECT count(*) FROM attachments") == before
finally:
command(COMPOSE + ["start", "s3"], True)
wait_replicas(c, a)
assert c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=b)["bytes"] == content
print("PASS: S3 recovered; original bytes preserved; failed upload created no metadata", flush=True)
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--base-url", default="http://127.0.0.1:8080")
p.add_argument("--output")
args = p.parse_args()
if args.output:
sys.stdout = open(ROOT / args.output, "w", encoding="utf-8", buffering=1)
sys.stderr = sys.stdout
os.environ.update(BASE_URL=args.base_url, NO_PROXY="localhost,127.0.0.1", no_proxy="localhost,127.0.0.1")
print("UTC " + datetime.now(timezone.utc).isoformat(), flush=True)
print("COMMAND python scripts/verify.py --base-url " + args.base_url, flush=True)
command(["docker", "version", "--format", "Client={{.Client.Version}} Server={{.Server.Version}}"])
command(["docker", "info", "--format", "Docker CPUs={{.NCPU}} memory_bytes={{.MemTotal}} OS={{.OperatingSystem}}"])
command(COMPOSE + ["config", "--quiet"])
command([sys.executable, "scripts/check_contract.py"])
command([sys.executable, "tests/smoke.py", "--base-url", args.base_url])
command([sys.executable, "-m", "unittest", "discover", "-s", "student-tests", "-v"])
c = Client(args.base_url, timeout=5)
a, b, _, chat = c.bootstrap()
seen = wait_replicas(c, a)
print("OBSERVED_INSTANCES " + json.dumps(sorted(seen)), flush=True)
first = c.message(a, chat, "survives restart", key="restart-key")["json"]
second = c.message(b, chat, "cursor after restart")["json"]
path = "/api/v1/chats/" + chat["id"] + "/messages"
page = c.call("GET", path + "?limit=1", actor=a, schema="MessagePage")["json"]
counts = "SELECT (SELECT count(*) FROM users),(SELECT count(*) FROM chats),(SELECT count(*) FROM memberships),(SELECT count(*) FROM messages),(SELECT count(*) FROM sessions),(SELECT count(*) FROM message_keys),(SELECT count(*) FROM schema_migrations),(SELECT count(*) FROM attachments)"
before_counts = sql(counts)
for _ in range(2):
command(COMPOSE + ["run", "--rm", "--no-deps", "migrate"])
assert sql(counts) == before_counts
print("PASS: repeated migrations preserve all table counts " + before_counts, flush=True)
before = state()
print("BEFORE_RESTART " + json.dumps(before), flush=True)
hook = "scripts/restart.cmd" if os.name == "nt" else "scripts/restart.sh"
command([sys.executable, "tests/resilience.py", "--mode", "persistence", "--action-script", hook, "--base-url", args.base_url])
after = state()
print("AFTER_RESTART " + json.dumps(after), flush=True)
for service in before:
assert before[service]["started_at"] != after[service]["started_at"]
assert before[service]["volumes"] == after[service]["volumes"]
assert c.message(a, chat, "survives restart", key="restart-key")["json"] == first
rest = c.call("GET", path + "?limit=1&cursor=" + quote(page["next_cursor"]), actor=a, schema="MessagePage")["json"]
assert page["items"] == [first] and rest["items"] == [second]
print("PASS: five containers restarted; volume, sessions, cursor and idempotency result preserved", flush=True)
verify_upload_failover(c, a, b, chat)
verify_s3_failure(c, a, b, chat)
stopped = set()
hook = "scripts/stop-one.cmd" if os.name == "nt" else "scripts/stop-one.sh"
# Run the unchanged public test until it has selected each real replica.
# Read concurrently with its POST stream to also measure read availability.
for attempt in range(6):
if stopped == {"api1", "api2"}:
break
command(COMPOSE + ["up", "-d", "--wait", "api1", "api2"], True)
wait_replicas(c, a)
done, reads = threading.Event(), {"ok": 0, "errors": 0}
def read_during_failure():
reader = Client(args.base_url, timeout=2)
while not done.is_set():
try:
result = reader.call("GET", path, actor=b, schema="MessagePage")["json"]
assert first in result["items"]
reads["ok"] += 1
except Failure:
reads["errors"] += 1
done.wait(0.1)
worker = threading.Thread(target=read_during_failure)
worker.start()
try:
print("FAILOVER_RUN " + str(attempt + 1), flush=True)
command([sys.executable, "tests/resilience.py", "--mode", "failover", "--action-script", hook, "--base-url", args.base_url])
current = state()
down = {s for s in ("api1", "api2") if current[s]["status"] == "exited"}
assert len(down) == 1
stopped.update(down)
print("AFTER_FAILOVER " + json.dumps(current), flush=True)
assert reads["ok"] >= 5
print("CONCURRENT_READS " + json.dumps(reads), flush=True)
assert c.message(a, chat, "survives restart", key="restart-key")["json"] == first
finally:
done.set()
worker.join(timeout=5)
command(COMPOSE + ["up", "-d", "--wait", "api1", "api2"], True)
assert stopped == {"api1", "api2"}, "Both replicas must be tested"
wait_replicas(c, a)
assert sql("SELECT count(*) FROM chats c WHERE NOT EXISTS (SELECT 1 FROM memberships m WHERE m.chat_id=c.id)") == "0"
print("PASS: each API stopped separately; reads/writes and original session survive; both replicas restored; S3 bytes checked", flush=True)
print("PASS: grade-3 verification complete", flush=True)
if __name__ == "__main__":
main()