forked from mtusi2026/mtusi2026lab2
375 lines
13 KiB
Python
375 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""Grade-5 verification scenarios for Lab 2. Uses only synthetic records."""
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
from datetime import datetime, timezone
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
|
|
from client import Client, Failure, check, run
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def env_file():
|
|
result = {}
|
|
path = ROOT / ".env"
|
|
if path.exists():
|
|
for line in path.read_text(encoding="utf-8-sig").splitlines():
|
|
line = line.strip()
|
|
if not line or line.startswith("#") or "=" not in line:
|
|
continue
|
|
key, value = line.split("=", 1)
|
|
result[key.strip()] = value.strip().strip('"').strip("'")
|
|
return result
|
|
|
|
|
|
ENV = env_file()
|
|
|
|
|
|
def docker(*args, input_text=None, check_ok=True, timeout=60):
|
|
proc = subprocess.run(
|
|
["docker", "compose", *args],
|
|
cwd=ROOT,
|
|
input=input_text,
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=timeout,
|
|
)
|
|
if check_ok and proc.returncode:
|
|
raise Failure(
|
|
"docker compose " + " ".join(args)
|
|
+ " failed: " + (proc.stderr or proc.stdout)[-1500:]
|
|
)
|
|
return proc
|
|
|
|
|
|
def psql(sql: str) -> str:
|
|
user = ENV.get("POSTGRES_USER", "messenger")
|
|
database = ENV.get("POSTGRES_DB", "messenger")
|
|
proc = subprocess.run(
|
|
[
|
|
"docker", "compose", "exec", "-T", "postgres",
|
|
"psql", "-X", "-A", "-t", "-v", "ON_ERROR_STOP=1",
|
|
"-U", user, "-d", database,
|
|
],
|
|
cwd=ROOT,
|
|
input=sql,
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=30,
|
|
)
|
|
if proc.returncode:
|
|
raise Failure("PostgreSQL test query failed: " + proc.stderr[-1200:])
|
|
return proc.stdout.strip()
|
|
|
|
|
|
def wait_api(base_url: str, timeout: int = 90):
|
|
deadline = time.monotonic() + timeout
|
|
last = None
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
Client(base_url, timeout=2).call(
|
|
"GET", "/health/ready", schema="Health"
|
|
)
|
|
return
|
|
except Exception as exc:
|
|
last = exc
|
|
time.sleep(1)
|
|
raise Failure(f"API did not become ready: {last}")
|
|
|
|
|
|
def wait_redis(timeout: int = 60):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
proc = docker("exec", "-T", "redis", "redis-cli", "ping", check_ok=False)
|
|
if proc.returncode == 0 and "PONG" in proc.stdout:
|
|
return
|
|
time.sleep(1)
|
|
raise Failure("Redis did not recover")
|
|
|
|
|
|
def registration_race(base_url: str):
|
|
client = Client(base_url, timeout=15)
|
|
username = "race_" + uuid.uuid4().hex[:16]
|
|
password = "Race-" + uuid.uuid4().hex
|
|
body = {
|
|
"username": username,
|
|
"display_name": "Concurrent registration",
|
|
"password": password,
|
|
}
|
|
workers = 12
|
|
barrier = threading.Barrier(workers)
|
|
|
|
def attempt(_):
|
|
barrier.wait(timeout=10)
|
|
return client.call(
|
|
"POST", "/api/v1/users", expected=[201, 409],
|
|
body=body, schema="User",
|
|
)["status"]
|
|
|
|
with ThreadPoolExecutor(max_workers=workers) as pool:
|
|
statuses = list(pool.map(attempt, range(workers)))
|
|
|
|
check(statuses.count(201) == 1, f"registration race: expected 1 success, got {statuses.count(201)}")
|
|
check(statuses.count(409) == workers - 1, f"registration race: expected {workers-1} conflicts, got {statuses}")
|
|
count = psql(
|
|
"SELECT count(*) FROM users WHERE username = "
|
|
+ "'" + username + "';"
|
|
)
|
|
check(count == "1", f"registration race left {count} database rows")
|
|
print(f"PASS registration race: {workers} concurrent requests, 1 created, {workers-1} conflicts, 1 DB row")
|
|
|
|
|
|
def chat_rollback(base_url: str):
|
|
client = Client(base_url)
|
|
alice, bob, _, _ = client.bootstrap()
|
|
before = int(psql("SELECT count(*) FROM chats;"))
|
|
|
|
psql("""
|
|
CREATE OR REPLACE FUNCTION lab2_test_reject_membership()
|
|
RETURNS trigger LANGUAGE plpgsql AS $$
|
|
BEGIN
|
|
RAISE EXCEPTION 'lab2 injected membership failure';
|
|
END;
|
|
$$;
|
|
DROP TRIGGER IF EXISTS lab2_test_reject_membership_trigger ON chat_members;
|
|
CREATE TRIGGER lab2_test_reject_membership_trigger
|
|
BEFORE INSERT ON chat_members
|
|
FOR EACH ROW EXECUTE FUNCTION lab2_test_reject_membership();
|
|
""")
|
|
try:
|
|
response = client.call(
|
|
"POST", "/api/v1/chats", expected=500,
|
|
actor=alice,
|
|
body={
|
|
"title": "Rollback verification",
|
|
"member_ids": [alice["id"], bob["id"]],
|
|
},
|
|
)
|
|
check(response["status"] == 500, "injected membership failure did not fail request")
|
|
finally:
|
|
psql("""
|
|
DROP TRIGGER IF EXISTS lab2_test_reject_membership_trigger ON chat_members;
|
|
DROP FUNCTION IF EXISTS lab2_test_reject_membership();
|
|
""")
|
|
after = int(psql("SELECT count(*) FROM chats;"))
|
|
check(after == before, f"chat rollback failed: chat count changed {before} -> {after}")
|
|
print("PASS chat rollback: injected membership failure left no partial chat")
|
|
|
|
|
|
def short_ttl(base_url: str):
|
|
override = ROOT / "compose.ttl-test.yaml"
|
|
override.write_text(
|
|
'services:\n api:\n environment:\n SESSION_TTL_SECONDS: "3"\n',
|
|
encoding="utf-8",
|
|
)
|
|
try:
|
|
docker("-f", "compose.yaml", "-f", "compose.ttl-test.yaml",
|
|
"up", "-d", "--force-recreate", "api", timeout=180)
|
|
wait_api(base_url)
|
|
client = Client(base_url, timeout=10)
|
|
actor = client.user("shortttl")
|
|
time.sleep(4)
|
|
expired = client.call(
|
|
"GET", "/api/v1/users/me", expected=401, actor=actor
|
|
)
|
|
check(expired["status"] == 401, "expired Bearer session was accepted")
|
|
print("PASS short TTL: 3-second session was rejected after expiry")
|
|
finally:
|
|
try:
|
|
override.unlink(missing_ok=True)
|
|
finally:
|
|
docker("up", "-d", "--force-recreate", "api", timeout=180)
|
|
wait_api(base_url)
|
|
|
|
|
|
def redis_outage(base_url: str):
|
|
client = Client(base_url, timeout=10)
|
|
alice, _, _, chat = client.bootstrap()
|
|
client.message(alice, chat, "redis-outage-history-check")
|
|
docker("stop", "redis")
|
|
try:
|
|
response = client.call(
|
|
"GET",
|
|
f"/api/v1/chats/{chat['id']}/messages?limit=50",
|
|
actor=alice,
|
|
schema="MessagePage",
|
|
)
|
|
texts = [item["text"] for item in response["json"]["items"]]
|
|
check("redis-outage-history-check" in texts, "history unavailable while Redis is down")
|
|
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")
|
|
print("PASS Redis outage: history and Bearer authentication remained available via PostgreSQL")
|
|
finally:
|
|
docker("start", "redis")
|
|
wait_redis()
|
|
wait_api(base_url)
|
|
|
|
|
|
|
|
def postgres_outage(base_url: str):
|
|
client = Client(base_url, timeout=5)
|
|
actor = client.user("dboutage")
|
|
docker("stop", "postgres")
|
|
try:
|
|
health = client.call("GET", "/health/ready", expected=503)
|
|
check(health["status"] == 503, "readiness did not report PostgreSQL outage")
|
|
protected = client.call(
|
|
"GET", "/api/v1/users/me", expected=[500, 503], actor=actor
|
|
)
|
|
check(
|
|
protected["status"] in (500, 503),
|
|
"protected request unexpectedly succeeded without PostgreSQL",
|
|
)
|
|
print("PASS PostgreSQL outage: readiness returned 503 and protected request failed closed")
|
|
finally:
|
|
docker("start", "postgres")
|
|
deadline = time.monotonic() + 90
|
|
user = ENV.get("POSTGRES_USER", "messenger")
|
|
database = ENV.get("POSTGRES_DB", "messenger")
|
|
while time.monotonic() < deadline:
|
|
proc = docker(
|
|
"exec", "-T", "postgres", "pg_isready",
|
|
"-U", user, "-d", database, check_ok=False
|
|
)
|
|
if proc.returncode == 0:
|
|
break
|
|
time.sleep(1)
|
|
else:
|
|
raise Failure("PostgreSQL did not recover within 90 seconds")
|
|
wait_api(base_url)
|
|
|
|
|
|
def redis_key_cleanup(chat_id: str):
|
|
patterns = [f"messages:{chat_id}:*", f"lock:messages:{chat_id}:*"]
|
|
for pattern in patterns:
|
|
scan = docker("exec", "-T", "redis", "redis-cli", "--scan", "--pattern", pattern)
|
|
keys = [line.strip() for line in scan.stdout.splitlines() if line.strip()]
|
|
if keys:
|
|
docker("exec", "-T", "redis", "redis-cli", "DEL", *keys)
|
|
|
|
|
|
def cache_stampede(base_url: str):
|
|
previous = psql("SHOW log_min_duration_statement;")
|
|
psql("ALTER SYSTEM SET log_min_duration_statement = 0; SELECT pg_reload_conf();")
|
|
|
|
def logged_message_selects(since: str):
|
|
logs = subprocess.run(
|
|
["docker", "logs", "--since", since, "mtusi-lab2-postgres"],
|
|
cwd=ROOT,
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=30,
|
|
)
|
|
lines = (logs.stdout + "\n" + logs.stderr).splitlines()
|
|
return [
|
|
line for line in lines
|
|
if re.search(r"statement:\s*SELECT", line, re.I)
|
|
and re.search(r"\bFROM\s+messages\b", line, re.I)
|
|
]
|
|
|
|
try:
|
|
client = Client(base_url, timeout=15)
|
|
alice, _, _, chat = client.bootstrap()
|
|
for i in range(5):
|
|
client.message(alice, chat, f"stampede-seed-{i}")
|
|
|
|
path = f"/api/v1/chats/{chat['id']}/messages?limit=50"
|
|
redis_key_cleanup(chat["id"])
|
|
workers = 12
|
|
barrier = threading.Barrier(workers)
|
|
|
|
def get_history(_):
|
|
barrier.wait(timeout=10)
|
|
return client.call("GET", path, actor=alice, schema="MessagePage")
|
|
|
|
cold_since = datetime.now(timezone.utc).isoformat(timespec="milliseconds")
|
|
started = time.monotonic()
|
|
with ThreadPoolExecutor(max_workers=workers) as pool:
|
|
cold_results = list(pool.map(get_history, range(workers)))
|
|
cold_elapsed = (time.monotonic() - started) * 1000
|
|
check(all(r["status"] == 200 for r in cold_results), "cold concurrent history request failed")
|
|
cold_selects = logged_message_selects(cold_since)
|
|
|
|
warm_since = datetime.now(timezone.utc).isoformat(timespec="milliseconds")
|
|
started = time.monotonic()
|
|
with ThreadPoolExecutor(max_workers=workers) as pool:
|
|
warm_results = list(pool.map(get_history, range(workers)))
|
|
warm_elapsed = (time.monotonic() - started) * 1000
|
|
check(all(r["status"] == 200 for r in warm_results), "warm concurrent history request failed")
|
|
warm_selects = logged_message_selects(warm_since)
|
|
|
|
check(
|
|
len(cold_selects) <= 2,
|
|
f"cache stampede protection ineffective: {len(cold_selects)} message SELECTs for {workers} simultaneous misses",
|
|
)
|
|
check(
|
|
len(warm_selects) == 0,
|
|
f"warm cache unexpectedly issued {len(warm_selects)} message SELECTs",
|
|
)
|
|
|
|
result = {
|
|
"experiment": "concurrent cache miss / stampede",
|
|
"workers": workers,
|
|
"cold_concurrent_requests": len(cold_results),
|
|
"warm_concurrent_requests": len(warm_results),
|
|
"cold_batch_elapsed_ms": round(cold_elapsed, 3),
|
|
"warm_batch_elapsed_ms": round(warm_elapsed, 3),
|
|
"cold_message_select_statements": len(cold_selects),
|
|
"warm_message_select_statements": len(warm_selects),
|
|
"result": "PASS",
|
|
"note": "PostgreSQL log_min_duration_statement was temporarily set to 0 and restored after the experiment. Counts are filtered to SELECT statements whose SQL contains FROM messages.",
|
|
}
|
|
out = ROOT / "evidence" / "cache-stampede.json"
|
|
out.parent.mkdir(exist_ok=True)
|
|
out.write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
(ROOT / "evidence" / "cache-stampede-cold.log").write_text(
|
|
"\n".join(cold_selects) + "\n", encoding="utf-8"
|
|
)
|
|
(ROOT / "evidence" / "cache-stampede-warm.log").write_text(
|
|
"\n".join(warm_selects) + "\n", encoding="utf-8"
|
|
)
|
|
print(json.dumps(result, indent=2, ensure_ascii=False))
|
|
print("PASS cache stampede: concurrent cold misses collapsed; warm cache issued no message SELECT")
|
|
finally:
|
|
psql("ALTER SYSTEM SET log_min_duration_statement = " + previous + "; SELECT pg_reload_conf();")
|
|
|
|
|
|
def main():
|
|
def configure(parser):
|
|
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
|
parser.add_argument(
|
|
"--only",
|
|
choices=["all", "registration", "rollback", "ttl", "redis", "postgres", "stampede"],
|
|
default="all",
|
|
)
|
|
import argparse
|
|
parser = argparse.ArgumentParser(description="Run Lab 2 grade-5 verification scenarios")
|
|
configure(parser)
|
|
args = parser.parse_args()
|
|
|
|
if args.only in ("all", "registration"):
|
|
registration_race(args.base_url)
|
|
if args.only in ("all", "rollback"):
|
|
chat_rollback(args.base_url)
|
|
if args.only in ("all", "ttl"):
|
|
short_ttl(args.base_url)
|
|
if args.only in ("all", "redis"):
|
|
redis_outage(args.base_url)
|
|
if args.only in ("all", "postgres"):
|
|
postgres_outage(args.base_url)
|
|
if args.only in ("all", "stampede"):
|
|
cache_stampede(args.base_url)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|