#!/usr/bin/env python3 """Explicit, scoped Docker failure checks. Bearer tokens stay in memory only.""" import argparse import json import os import subprocess import sys import time from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime from pathlib import Path from uuid import uuid4 ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client, check # noqa: E402 def compose(*args): return subprocess.run( ["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, capture_output=True, text=True, timeout=60, check=True, ).stdout def states(): values = {} for service in ("api", "postgres", "valkey"): container = compose("ps", "-q", service).strip() value = subprocess.run( [ "docker", "inspect", "--format", '{"id":{{json .Id}},"started_at":{{json .State.StartedAt}},' '"pid":{{.State.Pid}},"image":{{json .Image}},' '"memory_limit":{{.HostConfig.Memory}},"nano_cpus":{{.HostConfig.NanoCpus}}}', container, ], capture_output=True, text=True, check=True, ) values[service] = json.loads(value.stdout) return values def wait_ready(client): deadline = time.monotonic() + 60 while True: try: client.call("GET", "/health/ready", schema="Health") return except AssertionError: if time.monotonic() >= deadline: raise time.sleep(0.3) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--output", type=Path, default=ROOT / "evidence/lifecycle.json") parser.add_argument("--base-url", default="http://127.0.0.1:8082") args = parser.parse_args() if args.output.exists(): raise SystemExit("Choose a new evidence file") os.environ["NO_PROXY"] = os.environ["no_proxy"] = "127.0.0.1,localhost" client = Client(args.base_url, timeout=8) before = states() hook = subprocess.run( [ "make", "test-persistence", "ACTION_SCRIPT=scripts/restart.sh", "BASE_URL=" + args.base_url, ], cwd=ROOT, capture_output=True, text=True, timeout=150, check=True, ) (ROOT / "evidence/persistence.txt").write_text(hook.stdout) after = states() for service in before: check( before[service]["started_at"] != after[service]["started_at"], "Persistence hook did not restart " + service, ) alice, bob, eve, room = client.bootstrap() path = "/api/v1/chats/" + room["id"] + "/messages" canary = "PRIVATE_CANARY_" + uuid4().hex def send(i): return client.call( "POST", path, 201, actor=alice if i % 2 else bob, body={"text": canary + ":" + str(i)}, schema="Message", )["json"] with ThreadPoolExecutor(max_workers=10) as pool: sent = list(pool.map(send, range(100))) page = client.call("GET", path + "?limit=100", actor=bob, schema="MessagePage")["json"] check( len(page["items"]) == len({x["id"] for x in page["items"]}) == 100, "Concurrent sends lost or duplicated messages", ) check({x["id"] for x in sent} == {x["id"] for x in page["items"]}, "Message set differs") results = { "at": datetime.now(UTC).isoformat(), "before_restart": before, "after_restart": after, "concurrent_clients": 10, "unique_messages": 100, "faults": [], } try: # Cache loss is harmless; PG decides auth even when a stale key remains. for _ in range(2): client.call("GET", "/api/v1/users/me", actor=bob, schema="User") compose("stop", "--timeout", "10", "valkey") valid = client.call("GET", "/api/v1/users/me", actor=bob, schema="User") check(valid["elapsed"] < 3, "Cache fallback exceeded request budget") client.call("GET", "/api/v1/users/me", 401) client.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice) client.call("GET", "/api/v1/users/me", 401, actor=alice) results["faults"].append( { "dependency": "valkey", "valid_session_status": 200, "anonymous_status": 401, "revoked_status": 401, "fallback_seconds": valid["elapsed"], } ) compose("start", "valkey") compose("exec", "-T", "valkey", "valkey-cli", "FLUSHDB") client.call("GET", "/api/v1/users/me", actor=bob, schema="User") client.call("GET", "/api/v1/users/me", 401, actor=alice) compose("stop", "--timeout", "10", "postgres") client.call("GET", "/health/live", schema="Health") ready = client.call("GET", "/health/ready", 503) denied = client.call("GET", "/api/v1/users/me", 503, actor=bob) check(denied["elapsed"] < 6, "DB failure exceeded request budget") client.call("GET", "/api/v1/users/me", 401) results["faults"].append( { "dependency": "postgres", "liveness": 200, "readiness": 503, "cached_session_status": 503, "anonymous_status": 401, "request_seconds": denied["elapsed"], "readiness_seconds": ready["elapsed"], } ) finally: compose("start", "postgres", "valkey") wait_ready(client) recovered = client.call("GET", path + "?limit=100", actor=bob, schema="MessagePage")["json"] check(page == recovered, "Confirmed history changed across dependency failures") client.call("GET", "/api/v1/users/me", 401, actor=alice) client.call("GET", "/api/v1/users/me", actor=bob, schema="User") client.call("GET", "/api/v1/chats/" + room["id"], 404, actor=eve) compose("run", "--rm", "--no-deps", "migrate") logs = compose("logs", "--no-color", "api") sensitive = [canary] + [a["token"] for a in (alice, bob, eve)] sensitive += [a["registration"]["password"] for a in (alice, bob, eve)] check(not any(value in logs for value in sensitive), "Sensitive content appeared in logs") results["secrets_absent_from_api_logs"] = True results["migration_repeat_ok"] = True results["api_uid"] = int(compose("exec", "-T", "api", "id", "-u").strip()) check(results["api_uid"] == 10001, "API must run non-root") results["history_preserved"] = True args.output.write_text(json.dumps(results, ensure_ascii=False, indent=2) + "\n") print("PASS: full restart, concurrent writes, dependency failures, cache flush and revocation") if __name__ == "__main__": main()