238 lines
12 KiB
Python
238 lines
12 KiB
Python
"""Reproduce grade-3 evidence against the lab4 Compose stack; no credentials in output."""
|
|
import argparse
|
|
from datetime import datetime, timezone
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from urllib.parse import quote
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure
|
|
|
|
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
|
|
|
|
|
def command(args, capture=False):
|
|
result = subprocess.run(args, cwd=ROOT, text=True, encoding="utf-8", errors="replace",
|
|
stdout=subprocess.PIPE if capture else sys.stdout,
|
|
stderr=subprocess.STDOUT, timeout=180)
|
|
if result.returncode:
|
|
raise RuntimeError("Command failed: " + args[0])
|
|
return result.stdout
|
|
|
|
|
|
def sql(query):
|
|
return command(COMPOSE + ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-Atc", query], True).strip()
|
|
|
|
|
|
def state():
|
|
result = {}
|
|
for service in ("api1", "api2", "postgres", "s3", "lb"):
|
|
cid = command(COMPOSE + ["ps", "-a", "-q", service], True).strip()
|
|
data = json.loads(command(["docker", "inspect", cid], True))[0]
|
|
result[service] = {"id": cid[:12], "status": data["State"]["Status"],
|
|
"started_at": data["State"]["StartedAt"],
|
|
"volumes": [m["Name"] for m in data["Mounts"] if m["Type"] == "volume"],
|
|
"ports": data["HostConfig"]["PortBindings"],
|
|
"memory_limit": data["HostConfig"]["Memory"],
|
|
"nano_cpus": data["HostConfig"]["NanoCpus"]}
|
|
return result
|
|
|
|
|
|
def wait_replicas(c, actor):
|
|
deadline, seen = time.monotonic() + 60, set()
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
seen.add(c.call("GET", "/api/v1/users/me", actor=actor, schema="User")["headers"]["X-Instance-Id"])
|
|
if len(seen) >= 2:
|
|
return seen
|
|
except Failure:
|
|
pass
|
|
time.sleep(0.1)
|
|
raise AssertionError("Two replicas did not become ready")
|
|
|
|
|
|
def verify_upload_failover(c, a, b, chat):
|
|
import hashlib
|
|
content = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
for service in ("api1", "api2"):
|
|
wait_replicas(c, a)
|
|
for _ in range(30):
|
|
response = c.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=failover.png",
|
|
201, actor=a, raw=content, headers={"Content-Type": "image/png"}, schema="Attachment")
|
|
target = response["headers"]["X-Instance-Id"]
|
|
if target.startswith(service + "-"):
|
|
break
|
|
else:
|
|
raise AssertionError("Upload did not reach expected replica")
|
|
att = response["json"]
|
|
message = c.message(a, chat, "", attachments=[att["id"]])["json"]
|
|
env = dict(os.environ, TARGET_INSTANCE_ID=target)
|
|
subprocess.run([sys.executable, "scripts/stop_one.py"], cwd=ROOT, env=env, check=True, timeout=20, stdout=sys.stdout, stderr=subprocess.STDOUT)
|
|
try:
|
|
assert state()[service]["status"] == "exited"
|
|
result = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=b)
|
|
assert result["bytes"] == content
|
|
assert result["headers"]["X-Instance-Id"] != target
|
|
assert c.call("GET", "/api/v1/attachments/" + att["id"], actor=b, schema="Attachment")["json"] == att
|
|
print("UPLOAD_FAILOVER " + json.dumps({"stopped": target, "download_instance": result["headers"]["X-Instance-Id"],
|
|
"sha256": hashlib.sha256(result["bytes"]).hexdigest(),
|
|
"message_id": message["id"]}), flush=True)
|
|
finally:
|
|
command(COMPOSE + ["up", "-d", "--wait", "api1", "api2"], True)
|
|
# Inspect mounts: original bytes are never persisted in an API volume.
|
|
assert all(not state()[service]["volumes"] for service in ("api1", "api2"))
|
|
key = sql("SELECT object_key FROM attachments WHERE id='" + att["id"] + "'")
|
|
probe = """import urllib.request, urllib.error, sys, os
|
|
try:
|
|
urllib.request.urlopen(os.environ['S3_ENDPOINT'] + '/' + os.environ['S3_BUCKET'] + '/' + sys.argv[1],timeout=3)
|
|
raise AssertionError('Anonymous S3 GET allowed')
|
|
except urllib.error.HTTPError as exc:
|
|
assert exc.code == 403, exc.code
|
|
print('PASS: anonymous S3 GET of existing object denied (403)')
|
|
"""
|
|
command(COMPOSE + ["exec", "-T", "api1", "python", "-c", probe, key])
|
|
|
|
|
|
def verify_s3_failure(c, a, b, chat):
|
|
att, content = c.attachment(a, chat)
|
|
before = sql("SELECT count(*) FROM attachments")
|
|
# Execute inside the API network namespace: LB correctly removes unready APIs.
|
|
# Program travels through stdin; bearer never appears in command arguments/output.
|
|
program = """import urllib.request, urllib.error, json, base64
|
|
actor = ACTOR
|
|
attachment = ATTACHMENT
|
|
chat_id = CHAT_ID
|
|
content = base64.b64decode(CONTENT)
|
|
def request(path, expected, data=None, mime=None):
|
|
headers = {'Authorization': 'Bearer ' + actor['token']}
|
|
if mime: headers['Content-Type'] = mime
|
|
req = urllib.request.Request('http://127.0.0.1:8080' + path, data=data, headers=headers)
|
|
try: response = urllib.request.urlopen(req, timeout=8)
|
|
except urllib.error.HTTPError as exc: response = exc
|
|
with response:
|
|
assert response.code == expected, (path, response.code)
|
|
result = response.read()
|
|
return result
|
|
request('/health/live', 200)
|
|
request('/health/ready', 503)
|
|
request('/api/v1/attachments/' + attachment['id'], 200)
|
|
request('/api/v1/attachments/' + attachment['id'] + '/content', 503)
|
|
request('/api/v1/chats/' + chat_id + '/attachments?filename=outage.png', 503, content, 'image/png')
|
|
print('PASS: S3 down: live=200 ready=503 metadata=200 download=503 upload=503')
|
|
"""
|
|
import base64
|
|
program = program.replace("ACTOR", repr({"token": a["token"]})).replace("ATTACHMENT", repr(att)).replace("CHAT_ID", repr(chat["id"])).replace("CONTENT", repr(base64.b64encode(content).decode()))
|
|
command(COMPOSE + ["stop", "--timeout", "10", "s3"], True)
|
|
try:
|
|
result = subprocess.run(COMPOSE + ["exec", "-T", "api1", "python", "-"], cwd=ROOT,
|
|
input=program, text=True, capture_output=True, timeout=30)
|
|
assert result.returncode == 0, "S3 outage check failed (details suppressed to protect session)"
|
|
print(result.stdout.strip(), flush=True)
|
|
assert sql("SELECT count(*) FROM attachments") == before
|
|
finally:
|
|
command(COMPOSE + ["start", "s3"], True)
|
|
wait_replicas(c, a)
|
|
assert c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=b)["bytes"] == content
|
|
print("PASS: S3 recovered; original bytes preserved; failed upload created no metadata", flush=True)
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument("--base-url", default="http://127.0.0.1:8080")
|
|
p.add_argument("--output")
|
|
args = p.parse_args()
|
|
if args.output:
|
|
sys.stdout = open(ROOT / args.output, "w", encoding="utf-8", buffering=1)
|
|
sys.stderr = sys.stdout
|
|
os.environ.update(BASE_URL=args.base_url, NO_PROXY="localhost,127.0.0.1", no_proxy="localhost,127.0.0.1")
|
|
print("UTC " + datetime.now(timezone.utc).isoformat(), flush=True)
|
|
print("COMMAND python scripts/verify.py --base-url " + args.base_url, flush=True)
|
|
command(["docker", "version", "--format", "Client={{.Client.Version}} Server={{.Server.Version}}"])
|
|
command(["docker", "info", "--format", "Docker CPUs={{.NCPU}} memory_bytes={{.MemTotal}} OS={{.OperatingSystem}}"])
|
|
command(COMPOSE + ["config", "--quiet"])
|
|
command([sys.executable, "scripts/check_contract.py"])
|
|
command([sys.executable, "tests/smoke.py", "--base-url", args.base_url])
|
|
command([sys.executable, "-m", "unittest", "discover", "-s", "student-tests", "-v"])
|
|
c = Client(args.base_url, timeout=5)
|
|
a, b, _, chat = c.bootstrap()
|
|
seen = wait_replicas(c, a)
|
|
print("OBSERVED_INSTANCES " + json.dumps(sorted(seen)), flush=True)
|
|
first = c.message(a, chat, "survives restart", key="restart-key")["json"]
|
|
second = c.message(b, chat, "cursor after restart")["json"]
|
|
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
|
page = c.call("GET", path + "?limit=1", actor=a, schema="MessagePage")["json"]
|
|
counts = "SELECT (SELECT count(*) FROM users),(SELECT count(*) FROM chats),(SELECT count(*) FROM memberships),(SELECT count(*) FROM messages),(SELECT count(*) FROM sessions),(SELECT count(*) FROM message_keys),(SELECT count(*) FROM schema_migrations),(SELECT count(*) FROM attachments)"
|
|
before_counts = sql(counts)
|
|
for _ in range(2):
|
|
command(COMPOSE + ["run", "--rm", "--no-deps", "migrate"])
|
|
assert sql(counts) == before_counts
|
|
print("PASS: repeated migrations preserve all table counts " + before_counts, flush=True)
|
|
before = state()
|
|
print("BEFORE_RESTART " + json.dumps(before), flush=True)
|
|
hook = "scripts/restart.cmd" if os.name == "nt" else "scripts/restart.sh"
|
|
command([sys.executable, "tests/resilience.py", "--mode", "persistence", "--action-script", hook, "--base-url", args.base_url])
|
|
after = state()
|
|
print("AFTER_RESTART " + json.dumps(after), flush=True)
|
|
for service in before:
|
|
assert before[service]["started_at"] != after[service]["started_at"]
|
|
assert before[service]["volumes"] == after[service]["volumes"]
|
|
assert c.message(a, chat, "survives restart", key="restart-key")["json"] == first
|
|
rest = c.call("GET", path + "?limit=1&cursor=" + quote(page["next_cursor"]), actor=a, schema="MessagePage")["json"]
|
|
assert page["items"] == [first] and rest["items"] == [second]
|
|
print("PASS: five containers restarted; volume, sessions, cursor and idempotency result preserved", flush=True)
|
|
|
|
verify_upload_failover(c, a, b, chat)
|
|
verify_s3_failure(c, a, b, chat)
|
|
stopped = set()
|
|
hook = "scripts/stop-one.cmd" if os.name == "nt" else "scripts/stop-one.sh"
|
|
# Run the unchanged public test until it has selected each real replica.
|
|
# Read concurrently with its POST stream to also measure read availability.
|
|
for attempt in range(6):
|
|
if stopped == {"api1", "api2"}:
|
|
break
|
|
command(COMPOSE + ["up", "-d", "--wait", "api1", "api2"], True)
|
|
wait_replicas(c, a)
|
|
done, reads = threading.Event(), {"ok": 0, "errors": 0}
|
|
def read_during_failure():
|
|
reader = Client(args.base_url, timeout=2)
|
|
while not done.is_set():
|
|
try:
|
|
result = reader.call("GET", path, actor=b, schema="MessagePage")["json"]
|
|
assert first in result["items"]
|
|
reads["ok"] += 1
|
|
except Failure:
|
|
reads["errors"] += 1
|
|
done.wait(0.1)
|
|
worker = threading.Thread(target=read_during_failure)
|
|
worker.start()
|
|
try:
|
|
print("FAILOVER_RUN " + str(attempt + 1), flush=True)
|
|
command([sys.executable, "tests/resilience.py", "--mode", "failover", "--action-script", hook, "--base-url", args.base_url])
|
|
current = state()
|
|
down = {s for s in ("api1", "api2") if current[s]["status"] == "exited"}
|
|
assert len(down) == 1
|
|
stopped.update(down)
|
|
print("AFTER_FAILOVER " + json.dumps(current), flush=True)
|
|
assert reads["ok"] >= 5
|
|
print("CONCURRENT_READS " + json.dumps(reads), flush=True)
|
|
assert c.message(a, chat, "survives restart", key="restart-key")["json"] == first
|
|
finally:
|
|
done.set()
|
|
worker.join(timeout=5)
|
|
command(COMPOSE + ["up", "-d", "--wait", "api1", "api2"], True)
|
|
assert stopped == {"api1", "api2"}, "Both replicas must be tested"
|
|
wait_replicas(c, a)
|
|
assert sql("SELECT count(*) FROM chats c WHERE NOT EXISTS (SELECT 1 FROM memberships m WHERE m.chat_id=c.id)") == "0"
|
|
print("PASS: each API stopped separately; reads/writes and original session survive; both replicas restored; S3 bytes checked", flush=True)
|
|
print("PASS: grade-3 verification complete", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|