663 lines
30 KiB
Python
663 lines
30 KiB
Python
import argparse
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import secrets
|
|
import shutil
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from urllib.error import URLError
|
|
from urllib.request import ProxyHandler, build_opener
|
|
import venv
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
LOG = []
|
|
LOCAL_HTTP = build_opener(ProxyHandler({}))
|
|
|
|
|
|
def settings():
|
|
path = ROOT / ".env"
|
|
result = {}
|
|
if path.exists():
|
|
for line in path.read_text(encoding="utf-8-sig").splitlines():
|
|
key, separator, value = line.partition("=")
|
|
if separator and not key.strip().startswith("#"):
|
|
result[key.strip()] = value.strip()
|
|
return result
|
|
|
|
|
|
def environment(args):
|
|
result = os.environ.copy()
|
|
result.update(PYTHONUTF8="1", PYTHONIOENCODING="utf-8", PYTHONDONTWRITEBYTECODE="1",
|
|
COMPOSE_PROJECT_NAME=args.project, HTTP_PORT=str(args.port),
|
|
SESSION_TTL_SECONDS=str(args.ttl), LAB4_PROJECT=args.project,
|
|
LAB4_PORT=str(args.port), LAB4_TTL=str(args.ttl))
|
|
for key in ("NO_PROXY", "no_proxy"):
|
|
result[key] = ",".join(filter(None, [result.get(key), "localhost", "127.0.0.1"]))
|
|
return result
|
|
|
|
|
|
def say(value):
|
|
line = str(value).replace(str(ROOT), ".")
|
|
print(line, flush=True)
|
|
LOG.append(line)
|
|
|
|
|
|
def run(args, command, quiet=False):
|
|
command = [str(item) for item in command]
|
|
if not quiet:
|
|
say("$ " + subprocess.list2cmdline(command))
|
|
with subprocess.Popen(command, cwd=ROOT, env=environment(args), stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace") as process:
|
|
lines = []
|
|
for line in process.stdout:
|
|
lines.append(line)
|
|
if not quiet:
|
|
say(line.rstrip())
|
|
code = process.wait()
|
|
if code:
|
|
raise RuntimeError(f"{command[0]} exited with code {code}")
|
|
return "".join(lines)
|
|
|
|
|
|
def compose(args, *command, quiet=False):
|
|
return run(args, ["docker", "compose", "--project-name", args.project, "--env-file", ".env", *command], quiet)
|
|
|
|
|
|
def interpreter():
|
|
return ROOT / ".venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
|
|
|
|
|
|
def evidence(name, value):
|
|
path = ROOT / "evidence" / name
|
|
path.parent.mkdir(exist_ok=True)
|
|
content = json.dumps(value, ensure_ascii=False, indent=2) if not isinstance(value, str) else value
|
|
path.write_text(content.rstrip() + "\n", encoding="utf-8", newline="\n")
|
|
|
|
|
|
def setup(args):
|
|
if not (ROOT / ".env").exists():
|
|
shutil.copyfile(ROOT / ".env.example", ROOT / ".env")
|
|
folder = ROOT / "secrets"
|
|
folder.mkdir(exist_ok=True)
|
|
password_file = folder / "postgres_password"
|
|
if not password_file.exists():
|
|
password_file.write_text(secrets.token_hex(32) + "\n", encoding="ascii")
|
|
password = password_file.read_text(encoding="ascii").strip()
|
|
values = {"database_url": f"postgresql://messenger:{password}@db:5432/messenger",
|
|
"cursor_key": secrets.token_hex(32)}
|
|
for name, value in values.items():
|
|
path = folder / name
|
|
if not path.exists():
|
|
path.write_text(value + "\n", encoding="ascii")
|
|
if os.name != "nt":
|
|
path.chmod(0o644)
|
|
if os.name != "nt":
|
|
folder.chmod(0o700)
|
|
password_file.chmod(0o644)
|
|
for name in ("s3_access_key_id", "s3_secret_access_key", "s3_admin_access_key_id", "s3_admin_secret_access_key"):
|
|
path = folder / name
|
|
if not path.exists():
|
|
path.write_text(secrets.token_hex(32) + "\n", encoding="ascii")
|
|
if os.name != "nt":
|
|
path.chmod(0o644)
|
|
config = folder / "s3_config.json"
|
|
if not config.exists():
|
|
def key(name):
|
|
return (folder / name).read_text(encoding="ascii").strip()
|
|
bucket = settings().get("S3_BUCKET", "messenger-attachments")
|
|
identities = [
|
|
{"name": "bootstrap", "credentials": [{"accessKey": key("s3_admin_access_key_id"),
|
|
"secretKey": key("s3_admin_secret_access_key")}], "actions": ["Admin", "Read", "Write", "List"]},
|
|
{"name": "messenger", "credentials": [{"accessKey": key("s3_access_key_id"),
|
|
"secretKey": key("s3_secret_access_key")}],
|
|
"actions": [action + ":" + bucket for action in ("Read", "Write", "List")]},
|
|
]
|
|
config.write_text(json.dumps({"identities": identities}) + "\n", encoding="ascii")
|
|
if os.name != "nt":
|
|
config.chmod(0o644)
|
|
if not interpreter().exists():
|
|
venv.EnvBuilder(with_pip=True).create(ROOT / ".venv")
|
|
lockfiles = [ROOT / "requirements.lock", ROOT / "requirements-validation.lock"]
|
|
fingerprint = hashlib.sha256(b"".join(path.read_bytes() for path in lockfiles)).hexdigest()
|
|
marker = ROOT / ".venv/.lab-dependencies"
|
|
if not marker.exists() or marker.read_text(encoding="ascii") != fingerprint:
|
|
run(args, [interpreter(), "-X", "utf8", "-m", "pip", "install", "--disable-pip-version-check",
|
|
"--no-input", "--progress-bar", "off", "--quiet", "-r", lockfiles[0], "-r", lockfiles[1]])
|
|
marker.write_text(fingerprint, encoding="ascii")
|
|
say("Environment ready; existing settings and secrets preserved")
|
|
|
|
|
|
def base_url(args):
|
|
return f"http://127.0.0.1:{args.port}"
|
|
|
|
|
|
def wait_storage(args, timeout=90):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
container = compose(args, "ps", "--all", "--quiet", "s3-storage", quiet=True).strip()
|
|
if container:
|
|
status = run(args, ["docker", "inspect", "--format", "{{.State.Health.Status}}", container], quiet=True).strip()
|
|
if status == "healthy":
|
|
return
|
|
time.sleep(0.5)
|
|
raise RuntimeError("S3 storage readiness timeout")
|
|
|
|
|
|
def wait_ready(args, timeout=90):
|
|
wait_storage(args, timeout)
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with LOCAL_HTTP.open(base_url(args) + "/health/ready", timeout=4) as response:
|
|
if response.status == 200 and json.load(response) == {"status": "ok"}:
|
|
return
|
|
except (URLError, OSError, ValueError):
|
|
pass
|
|
time.sleep(0.3)
|
|
raise RuntimeError("API readiness timeout")
|
|
|
|
|
|
def build(args):
|
|
compose(args, "config", "--quiet")
|
|
compose(args, "build", "api1")
|
|
compose(args, "run", "--rm", "--no-deps", "lb", "haproxy", "-c", "-f",
|
|
"/usr/local/etc/haproxy/haproxy.cfg")
|
|
|
|
|
|
def start(args):
|
|
current = compose(args, "ps", "--all", "--quiet", "lb", quiet=True).strip()
|
|
if not current:
|
|
with socket.socket() as probe:
|
|
try:
|
|
probe.bind(("127.0.0.1", args.port))
|
|
except OSError:
|
|
raise RuntimeError(f"Port {args.port} is busy; stop its container or use --port") from None
|
|
compose(args, "up", "-d", "--no-build", "db", "migrate", "s3-storage", "s3", "bootstrap", "api1", "api2", "cleanup", "lb")
|
|
wait_ready(args)
|
|
wait_replicas(args)
|
|
say("Two API replicas ready through LB: " + base_url(args))
|
|
|
|
|
|
def states(args):
|
|
services = ("api1", "api2", "db", "lb", "s3", "s3-storage", "cleanup")
|
|
containers = compose(args, "ps", "--all", "--quiet", *services, quiet=True).split()
|
|
if len(set(containers)) != len(services):
|
|
raise RuntimeError("Missing runtime container")
|
|
result = {}
|
|
for data in json.loads(run(args, ["docker", "inspect", *containers], quiet=True)):
|
|
labels = data["Config"].get("Labels", {})
|
|
service = labels.get("com.docker.compose.service")
|
|
if labels.get("com.docker.compose.project") != args.project or service not in services or service in result:
|
|
raise RuntimeError("Unexpected container ownership")
|
|
result[service] = {
|
|
"id": data["Id"], "image": data["Image"], "started_at": data["State"]["StartedAt"],
|
|
"running": data["State"]["Running"], "user": data["Config"]["User"],
|
|
"volumes": [mount["Name"] for mount in data["Mounts"] if mount["Type"] == "volume"],
|
|
}
|
|
return result
|
|
|
|
|
|
def restart(args):
|
|
before = states(args)
|
|
if not all(item["running"] for item in before.values()):
|
|
raise RuntimeError("All runtime containers must be running before restart")
|
|
compose(args, "restart", "--timeout", "10", "db", "s3-storage", "s3", "api1", "api2", "cleanup", "lb")
|
|
wait_storage(args)
|
|
compose(args, "run", "--rm", "--no-deps", "bootstrap")
|
|
after = states(args)
|
|
for service in before:
|
|
if before[service]["started_at"] == after[service]["started_at"]:
|
|
raise RuntimeError("Container was not restarted: " + service)
|
|
if before[service]["volumes"] != after[service]["volumes"]:
|
|
raise RuntimeError("Volume changed during restart")
|
|
evidence("restart.json", {"project": args.project, "before": before, "after": after})
|
|
say("PASS: both APIs, PostgreSQL, S3 and LB restarted, volumes preserved")
|
|
|
|
|
|
def contract_tests(args):
|
|
run(args, [interpreter(), "-X", "utf8", "scripts/check_contract.py"])
|
|
run(args, [interpreter(), "-X", "utf8", "scripts/validate_openapi.py"])
|
|
compose(args, "config", "--quiet")
|
|
|
|
|
|
def api_tests(args):
|
|
wait_ready(args)
|
|
run(args, [interpreter(), "-X", "utf8", "tests/smoke.py", "--base-url", base_url(args)])
|
|
os.environ["BASE_URL"] = base_url(args)
|
|
run(args, [interpreter(), "-X", "utf8", "-m", "unittest", "discover", "-s", "student-tests", "-v"])
|
|
|
|
|
|
def client(args):
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client
|
|
for key in ("NO_PROXY", "no_proxy"):
|
|
os.environ[key] = environment(args)[key]
|
|
return Client(base_url(args))
|
|
|
|
|
|
def database_stats(args):
|
|
sql = """SELECT json_build_object(
|
|
'users',(SELECT count(*) FROM users),'chats',(SELECT count(*) FROM chats),
|
|
'members',(SELECT count(*) FROM chat_members),'messages',(SELECT count(*) FROM messages),
|
|
'sessions',(SELECT count(*) FROM sessions),'migrations',(SELECT count(*) FROM schema_migrations),
|
|
'idempotency_keys',(SELECT count(*) FROM idempotency_keys),
|
|
'attachments',(SELECT count(*) FROM attachments),'message_attachments',(SELECT count(*) FROM message_attachments),
|
|
'upload_intents',(SELECT count(*) FROM upload_intents),
|
|
'expiry_fingerprint',(SELECT md5(COALESCE(string_agg(user_id::text || created_at::text || expires_at::text,
|
|
',' ORDER BY token_hash),'')) FROM sessions),
|
|
'passwords_are_argon2id',(SELECT bool_and(password_hash LIKE '$argon2id$%') FROM users),
|
|
'session_hashes_only',(SELECT bool_and(token_hash ~ '^[0-9a-f]{64}$') FROM sessions))"""
|
|
result = compose(args, "exec", "-T", "db", "psql", "-U", "messenger", "-d", "messenger",
|
|
"-At", "-c", sql, quiet=True)
|
|
return json.loads(result)
|
|
|
|
|
|
def restart_check(args):
|
|
from urllib.parse import urlencode
|
|
c = client(args)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
|
replay_key = secrets.token_hex(16)
|
|
original = c.message(alice, chat, "first", key=replay_key)["json"]
|
|
last = c.message(bob, chat, "second")["json"]
|
|
page = c.call("GET", path + "?limit=1", actor=alice, schema="MessagePage")["json"]
|
|
revoked = c.user("revoked")
|
|
c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=revoked)
|
|
hook = ROOT / "scripts" / ("restart.cmd" if os.name == "nt" else "restart.sh")
|
|
run(args, [interpreter(), "-X", "utf8", "tests/resilience.py", "--mode", "persistence",
|
|
"--action-script", hook, "--base-url", base_url(args)])
|
|
continued = c.call("GET", path + "?" + urlencode({"cursor": page["next_cursor"]}),
|
|
actor=alice, schema="MessagePage")["json"]
|
|
if continued != {"items": [last], "next_cursor": None}:
|
|
raise RuntimeError("Old cursor did not survive restart")
|
|
c.call("GET", "/api/v1/users/me", 401, actor=revoked)
|
|
c.call("GET", "/api/v1/chats/" + chat["id"], 404, actor=eve)
|
|
before = database_stats(args)
|
|
compose(args, "run", "--rm", "--no-deps", "migrate")
|
|
compose(args, "run", "--rm", "--no-deps", "migrate")
|
|
after = database_stats(args)
|
|
stable_before = {key: value for key, value in before.items() if key != "upload_intents"}
|
|
stable_after = {key: value for key, value in after.items() if key != "upload_intents"}
|
|
if stable_before != stable_after or after["migrations"] != 3:
|
|
raise RuntimeError("Migration replay changed existing data")
|
|
if not after["passwords_are_argon2id"] or not after["session_hashes_only"]:
|
|
raise RuntimeError("Unexpected credential representation in DB")
|
|
repeated = c.message(alice, chat, "first", key=replay_key)["json"]
|
|
if repeated != original:
|
|
raise RuntimeError("Idempotency result was not preserved")
|
|
result = {"idempotency_after_restart": "PASS", "public_persistence": "PASS", "cursor_after_restart": "PASS",
|
|
"logout_after_restart": "PASS", "migration_replay": "PASS", "database": after}
|
|
evidence("persistence.json", result)
|
|
say("PASS: old session/cursor, durable logout and repeatable migrations")
|
|
return result
|
|
|
|
|
|
def ttl_check(args):
|
|
test = argparse.Namespace(project=args.project + "-ttl", port=args.test_port, ttl=2,
|
|
test_port=args.test_port, action="start")
|
|
try:
|
|
start(test)
|
|
c = client(test)
|
|
actor = c.user("expiry")
|
|
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
|
before = database_stats(test)["expiry_fingerprint"]
|
|
compose(test, "restart", "--timeout", "3", "api1", "api2")
|
|
wait_ready(test)
|
|
after = database_stats(test)["expiry_fingerprint"]
|
|
if before != after:
|
|
raise RuntimeError("Session expiry changed during restart")
|
|
time.sleep(2.2)
|
|
c.call("GET", "/api/v1/users/me", 401, actor=actor)
|
|
registration = actor["registration"]
|
|
import base64
|
|
basic = base64.b64encode((registration["username"] + ":" + registration["password"]).encode()).decode()
|
|
c.call("POST", "/api/v1/auth/sessions", 201, headers={"Authorization": "Basic " + basic}, schema="Session")
|
|
say("PASS: TTL=2, expired token rejected after API restart; fresh login works")
|
|
result = {"ttl_seconds": 2, "expiry_unchanged_on_restart": True, "before_restart": 200,
|
|
"after_expiry_and_restart": 401, "new_login": 201}
|
|
evidence("expiry.json", result)
|
|
return result
|
|
finally:
|
|
compose(test, "stop", "--timeout", "10", "lb", "api1", "api2", "cleanup", "s3", "s3-storage", "db")
|
|
|
|
|
|
def dependency_check(args):
|
|
c = client(args)
|
|
actor = c.user("dbcheck")
|
|
try:
|
|
compose(args, "stop", "--timeout", "10", "db")
|
|
c.call("GET", "/health/live", schema="Health")
|
|
outage_response(args, "/health/ready")
|
|
outage_response(args, "/api/v1/users/me", actor["token"])
|
|
finally:
|
|
compose(args, "start", "db")
|
|
wait_ready(args)
|
|
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
|
say("PASS: DB outage -> live=200, ready/business=503; session works after recovery")
|
|
return {"live_without_db": 200, "ready_without_db": 503, "business_without_db": 503, "recovered": 200}
|
|
|
|
|
|
def verify(args):
|
|
summary = {"date_utc": datetime.now(timezone.utc).isoformat(), "platform": platform.platform(),
|
|
"source_commit": run(args, ["git", "rev-parse", "HEAD"], quiet=True).strip(),
|
|
"worktree_dirty_at_start": bool(run(args, ["git", "status", "--porcelain"], quiet=True).strip()),
|
|
"project": args.project, "status": "FAIL"}
|
|
try:
|
|
setup(args)
|
|
contract_tests(args)
|
|
build(args)
|
|
start(args)
|
|
api_tests(args)
|
|
summary["cleanup"] = cleanup_check(args)
|
|
summary["persistence"] = restart_check(args)
|
|
summary["failover"] = failover_check(args)
|
|
summary["attachments_failover"] = attachment_failover_check(args)
|
|
summary["expiry"] = ttl_check(args)
|
|
summary["memory"] = memory_check(args)
|
|
summary["dependency"] = dependency_check(args)
|
|
run(args, [interpreter(), "-X", "utf8", "tests/smoke.py", "--base-url", base_url(args)])
|
|
summary["containers"] = states(args)
|
|
summary["docker"] = run(args, ["docker", "version", "--format", "{{.Server.Version}}"], quiet=True).strip()
|
|
summary["resources"] = json.loads(run(
|
|
args, ["docker", "info", "--format", '{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}'], quiet=True))
|
|
summary["status"] = "PASS"
|
|
say("PASS: full lab4 verification")
|
|
finally:
|
|
evidence("verification.txt", "\n".join(LOG))
|
|
evidence("summary.json", summary)
|
|
|
|
|
|
def instance_map(args):
|
|
current = states(args)
|
|
result = {}
|
|
for service in ("api1", "api2"):
|
|
item = current[service]
|
|
if not item["running"]:
|
|
continue
|
|
code = (
|
|
"import urllib.request; "
|
|
"o=urllib.request.build_opener(urllib.request.ProxyHandler({})); "
|
|
"r=o.open('http://127.0.0.1:8080/health/live',timeout=1); "
|
|
"print(r.headers['X-Instance-Id'])"
|
|
)
|
|
alias = run(args, ["docker", "exec", item["id"], "python", "-c", code], quiet=True).strip()
|
|
import re
|
|
if not re.fullmatch(r"[0-9a-f]{32}", alias):
|
|
raise RuntimeError("Invalid API instance alias")
|
|
result[service] = {"instance_id": alias, "container_id": item["id"]}
|
|
return result
|
|
|
|
|
|
def wait_replicas(args):
|
|
deadline = time.monotonic() + 90
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
mapping = instance_map(args)
|
|
if len(mapping) == 2:
|
|
c = client(args)
|
|
seen = {c.call("GET", "/health/ready", schema="Health")["headers"]["X-Instance-Id"]
|
|
for _ in range(8)}
|
|
if seen == {item["instance_id"] for item in mapping.values()}:
|
|
return mapping
|
|
except (RuntimeError, OSError, AssertionError):
|
|
pass
|
|
time.sleep(0.3)
|
|
raise RuntimeError("Two healthy API replicas were not observed through LB")
|
|
|
|
|
|
def restore(args):
|
|
compose(args, "start", "api1", "api2")
|
|
wait_ready(args)
|
|
return wait_replicas(args)
|
|
|
|
|
|
def stop_one(args):
|
|
target = os.environ.get("TARGET_INSTANCE_ID", "")
|
|
before = instance_map(args)
|
|
matches = [(name, item) for name, item in before.items() if item["instance_id"] == target]
|
|
if len(before) != 2 or len(matches) != 1:
|
|
raise RuntimeError("Target must match exactly one of two running API replicas")
|
|
service, item = matches[0]
|
|
started = time.monotonic()
|
|
run(args, ["docker", "stop", "--timeout", "3", item["container_id"]])
|
|
after = states(args)
|
|
if after[service]["running"] or not after["api1" if service == "api2" else "api2"]["running"]:
|
|
raise RuntimeError("Unexpected API state after stop")
|
|
elapsed = time.monotonic() - started
|
|
if elapsed >= 10:
|
|
raise RuntimeError("Stopping the selected API exceeded 10 seconds")
|
|
evidence("stop-one.json", {"service": service, "target_instance": target, "before": before,
|
|
"after": after, "stop_seconds": round(elapsed, 3)})
|
|
say("Stopped only " + service)
|
|
|
|
|
|
def failover_check(args):
|
|
wait_replicas(args)
|
|
hook = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh")
|
|
try:
|
|
output = run(args, [interpreter(), "-X", "utf8", "tests/resilience.py", "--mode", "failover",
|
|
"--action-script", hook, "--base-url", base_url(args)])
|
|
first = json.loads((ROOT / "evidence/stop-one.json").read_text(encoding="utf-8"))
|
|
first["metrics"] = next(json.loads(line) for line in output.splitlines()
|
|
if line.startswith("{") and "first_recovery_seconds" in line)
|
|
if first["after"][first["service"]]["running"]:
|
|
raise RuntimeError("Public hook did not leave the target stopped")
|
|
finally:
|
|
restore(args)
|
|
other_service = "api2" if first["service"] == "api1" else "api1"
|
|
mapping = instance_map(args)
|
|
target = mapping[other_service]["instance_id"]
|
|
c = client(args)
|
|
c.timeout = 2
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
for _ in range(10):
|
|
replay_key = secrets.token_hex(16)
|
|
replay_text = "before-failure-" + replay_key
|
|
response = c.message(alice, chat, replay_text, key=replay_key)
|
|
if response["headers"]["X-Instance-Id"] == target:
|
|
original = response["json"]
|
|
break
|
|
else:
|
|
raise RuntimeError("Second API was not reached through LB")
|
|
env = environment(args)
|
|
env["TARGET_INSTANCE_ID"] = target
|
|
started = time.monotonic()
|
|
process = subprocess.Popen([str(hook)], cwd=ROOT, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
from client import Failure
|
|
from resilience import history
|
|
confirmations, after_instances = [], []
|
|
failures = 0
|
|
recovered = None
|
|
key = secrets.token_hex(16)
|
|
text = "during-failure-" + key
|
|
try:
|
|
while time.monotonic() - started < 15:
|
|
code = process.poll()
|
|
if code not in (None, 0):
|
|
raise RuntimeError("Second stop hook failed")
|
|
if code is None and time.monotonic() - started >= 10:
|
|
raise RuntimeError("Second stop hook exceeded 10 seconds")
|
|
try:
|
|
response = c.message(alice, chat, text, key=key)
|
|
confirmations.append(response["json"])
|
|
if code == 0:
|
|
if recovered is None:
|
|
recovered = time.monotonic() - started
|
|
after_instances.append(response["headers"]["X-Instance-Id"])
|
|
key = secrets.token_hex(16)
|
|
text = "during-failure-" + key
|
|
except Failure:
|
|
failures += 1
|
|
time.sleep(0.1)
|
|
if process.poll() != 0 or recovered is None or recovered > 10:
|
|
raise RuntimeError("Second API failure did not recover within 10 seconds")
|
|
if len(after_instances) < 5 or target in after_instances[-5:]:
|
|
raise RuntimeError("Stopped API still serves requests")
|
|
replayed = c.message(alice, chat, replay_text, key=replay_key)
|
|
if replayed["json"] != original or replayed["headers"]["X-Instance-Id"] == target:
|
|
raise RuntimeError("Message replay through the survivor changed the result")
|
|
messages = history(c, bob, chat)
|
|
ids = [item["id"] for item in messages]
|
|
texts = [item["text"] for item in messages]
|
|
if len(ids) != len(set(ids)) or len(texts) != len(set(texts)):
|
|
raise RuntimeError("Duplicates appeared during failover")
|
|
if any(item not in messages for item in [original, *confirmations]):
|
|
raise RuntimeError("A confirmed message was lost")
|
|
second = json.loads((ROOT / "evidence/stop-one.json").read_text(encoding="utf-8"))
|
|
if second["service"] != other_service:
|
|
raise RuntimeError("Wrong second API was stopped")
|
|
second["metrics"] = {"successful_writes": len(confirmations), "failed_attempts": failures,
|
|
"first_recovery_seconds": round(recovered, 3),
|
|
"replay_on_survivor": "PASS"}
|
|
result = {"public": first, "second_replica": second, "status": "PASS"}
|
|
evidence("failover.json", result)
|
|
say("PASS: both physical API replicas stopped separately; no lost messages or duplicates")
|
|
return result
|
|
finally:
|
|
if process.poll() is None:
|
|
process.terminate()
|
|
process.wait(timeout=5)
|
|
restore(args)
|
|
|
|
|
|
def outage_response(args, path, token=None):
|
|
from urllib.error import HTTPError
|
|
from urllib.request import Request
|
|
from client import shape
|
|
headers = {"Authorization": "Bearer " + token} if token else {}
|
|
try:
|
|
response = LOCAL_HTTP.open(Request(base_url(args) + path, headers=headers), timeout=10)
|
|
except HTTPError as error:
|
|
response = error
|
|
with response:
|
|
payload = json.load(response)
|
|
if response.status != 503 or not response.headers.get("X-Request-Id"):
|
|
raise RuntimeError("Unexpected dependency failure response")
|
|
shape(payload, "Error")
|
|
if payload["error"]["request_id"] != response.headers["X-Request-Id"]:
|
|
raise RuntimeError("LB error request ID mismatch")
|
|
|
|
|
|
|
|
def cleanup_check(args):
|
|
run(args, [interpreter(), "-X", "utf8", "student-tests/check_cleanup.py"])
|
|
return json.loads((ROOT / "evidence/cleanup.json").read_text(encoding="utf-8"))
|
|
|
|
|
|
def memory_check(args):
|
|
run(args, [interpreter(), "-X", "utf8", "student-tests/check_memory.py",
|
|
"--project", args.project + "-memory", "--port", str(args.test_port + 1)])
|
|
return json.loads((ROOT / "evidence/memory.json").read_text(encoding="utf-8"))
|
|
|
|
|
|
def attachment_failover_check(args):
|
|
import hashlib
|
|
results = []
|
|
raw = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
try:
|
|
for service in ("api1", "api2"):
|
|
mapping = wait_replicas(args)
|
|
c = client(args)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
target = mapping[service]["instance_id"]
|
|
for _ in range(8):
|
|
uploaded = c.call(
|
|
"POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=failover.png", 201,
|
|
actor=alice, raw=raw, headers={"Content-Type": "image/png"}, schema="Attachment",
|
|
)
|
|
if uploaded["headers"]["X-Instance-Id"] == target:
|
|
break
|
|
else:
|
|
raise RuntimeError("Selected upload replica was not observed")
|
|
attachment = uploaded["json"]
|
|
linked = c.message(alice, chat, "", attachments=[attachment["id"]])["json"]
|
|
hook = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh")
|
|
previous = os.environ.get("TARGET_INSTANCE_ID")
|
|
os.environ["TARGET_INSTANCE_ID"] = target
|
|
started = time.monotonic()
|
|
try:
|
|
run(args, [str(hook)])
|
|
finally:
|
|
if previous is None:
|
|
os.environ.pop("TARGET_INSTANCE_ID", None)
|
|
else:
|
|
os.environ["TARGET_INSTANCE_ID"] = previous
|
|
from client import Client
|
|
reads = Client(base_url(args), timeout=2)
|
|
errors = 0
|
|
while True:
|
|
try:
|
|
downloaded = reads.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob)
|
|
history_response = reads.call(
|
|
"GET", "/api/v1/chats/" + chat["id"] + "/messages", actor=bob, schema="MessagePage",
|
|
)
|
|
break
|
|
except (AssertionError, OSError) as error:
|
|
if isinstance(error, AssertionError) and not any(
|
|
text in str(error) for text in ("received 502", "received 503", "connection/TLS failure")
|
|
):
|
|
raise
|
|
errors += 1
|
|
if time.monotonic() - started >= 10:
|
|
raise RuntimeError("File and history did not recover within 10 seconds") from None
|
|
time.sleep(0.1)
|
|
recovered = time.monotonic() - started
|
|
if downloaded["bytes"] != raw or hashlib.sha256(raw).hexdigest() != attachment["sha256"]:
|
|
raise RuntimeError("Original file changed after API failure")
|
|
if downloaded["headers"]["X-Instance-Id"] == target or recovered > 10:
|
|
raise RuntimeError("Upload replica was not replaced")
|
|
history = history_response["json"]["items"]
|
|
if history_response["headers"]["X-Instance-Id"] == target:
|
|
raise RuntimeError("Stopped API returned history")
|
|
if history != [linked]:
|
|
raise RuntimeError("Attached message changed during failover")
|
|
results.append({"stopped_service": service, "upload_instance": target,
|
|
"download_instance": downloaded["headers"]["X-Instance-Id"],
|
|
"recovery_seconds": round(recovered, 3), "transitional_errors": errors,
|
|
"same_original": True, "message_preserved": True})
|
|
restore(args)
|
|
result = {"status": "PASS", "cases": results}
|
|
evidence("attachment-failover.json", result)
|
|
say("PASS: stopped each uploader; other API downloaded the same original")
|
|
return result
|
|
finally:
|
|
restore(args)
|
|
|
|
|
|
def main():
|
|
saved = settings()
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("action", choices=["setup", "build", "start", "test", "restart", "stop-one",
|
|
"restart-check", "failover-check", "cleanup-check", "memory-check", "restore", "stop", "verify"])
|
|
parser.add_argument("--project", default=os.environ.get("LAB4_PROJECT", saved.get("COMPOSE_PROJECT_NAME", "mtusi-lab4")))
|
|
parser.add_argument("--port", type=int, default=int(os.environ.get("LAB4_PORT", saved.get("HTTP_PORT", "8080"))))
|
|
parser.add_argument("--ttl", type=int, default=int(os.environ.get("LAB4_TTL", saved.get("SESSION_TTL_SECONDS", "3600"))))
|
|
parser.add_argument("--test-port", type=int, default=18082)
|
|
args = parser.parse_args()
|
|
if not 1 <= args.port <= 65535 or not 1 <= args.test_port <= 65534 or not 1 <= args.ttl <= 86400:
|
|
parser.error("Invalid port or session TTL")
|
|
if args.action in ("setup", "build", "start", "test", "restart-check", "failover-check", "cleanup-check", "memory-check"):
|
|
setup(args)
|
|
actions = {"setup": lambda item: None, "build": build, "start": start,
|
|
"test": lambda item: (contract_tests(item), api_tests(item)),
|
|
"restart": restart, "restart-check": restart_check, "verify": verify,
|
|
"stop-one": stop_one, "restore": restore, "failover-check": failover_check,
|
|
"cleanup-check": cleanup_check, "memory-check": memory_check,
|
|
"stop": lambda item: compose(item, "stop", "--timeout", "10", "lb", "api1", "api2", "cleanup", "s3", "s3-storage", "db")}
|
|
actions[args.action](args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except (RuntimeError, OSError, ValueError) as error:
|
|
print("FAIL: " + str(error), file=sys.stderr)
|
|
raise SystemExit(1)
|