#!/usr/bin/env python3 """Operator-run rotation, expiry/revoke, sealed-store and restart recovery tests.""" # ruff: noqa: E402 -- repository scripts are imported after setting their path. import json import subprocess import sys import time import urllib.error import urllib.request from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts")) from client import Client from securitylib import LOCAL, OPERATOR, compose, request, root_token, tls def wait_ready(timeout=90): deadline = time.monotonic() + timeout while time.monotonic() < deadline: try: with urllib.request.urlopen( "https://localhost:8443/health/ready", context=tls(None), timeout=3 ) as response: if response.status == 200: return except (OSError, urllib.error.URLError): pass time.sleep(1) raise AssertionError("HTTPS did not recover") def worker_children(): # PID 1 is supervisor (or tini); readiness checks the actual application child. code = "from pathlib import Path; print(sum(b'src.pipeline.worker' in p.read_bytes() and b'src.secure.runtime' not in p.read_bytes() for p in Path('/proc').glob('[0-9]*/cmdline')))" return int(compose("exec", "-T", "worker", "python", "-c", code, capture=True)) def main(): rows = [] started = time.monotonic() subprocess.run([sys.executable, "scripts/lab7.py", "rotate-secret"], cwd=ROOT, check=True) record = json.loads((OPERATOR / "last-worker-rotation.json").read_text()) code = """import asyncio,json,sys,asyncpg from src.secure.transport import context async def main(): x=json.load(sys.stdin) try:c=await asyncpg.connect(x['url'],ssl=context(),direct_tls=True,timeout=3) except asyncpg.InvalidAuthorizationSpecificationError:print('OLD_DENIED');return else:await c.close();raise AssertionError('Old DB credential still accepted') asyncio.run(main())""" result = subprocess.check_output( ["docker", "compose", "run", "--rm", "--no-deps", "-T", "migration", "python", "-c", code], input=json.dumps({"url": record["old_url"]}), text=True, cwd=ROOT, ) assert "OLD_DENIED" in result rows.append( { "check": "db-secret-rotation-old-login-denied", "result": "PASS", "seconds": round(time.monotonic() - started, 2), } ) before = set(OPERATOR.glob("revoked-worker-*")) subprocess.run( [sys.executable, "scripts/lab7.py", "rotate-cert", "--role", "worker"], cwd=ROOT, check=True ) archived = (set(OPERATOR.glob("revoked-worker-*")) - before).pop() # New TLS connection with revoked certificate must fail, not only an old socket. code = """import socket,ssl ctx=ssl.create_default_context(cafile='/run/trust/ca.crt') ctx.load_cert_chain('/run/revoked/service.crt','/run/revoked/service.key') try: with socket.create_connection(('broker',6379),timeout=3) as raw: with ctx.wrap_socket(raw,server_hostname='broker') as s: s.sendall(b'*1\\r\\n$4\\r\\nPING\\r\\n');data=s.recv(100) assert not data,'Revoked client certificate accepted' except (ssl.SSLError,ConnectionResetError):pass print('OLD_CERT_DENIED')""" result = compose( "run", "--rm", "--no-deps", "-v", str(archived) + ":/run/revoked:ro", "migration", "python", "-c", code, capture=True, ) assert "OLD_CERT_DENIED" in result deadline = time.monotonic() + 60 while worker_children() == 0 and time.monotonic() < deadline: time.sleep(1) assert worker_children() > 0 rows.append({"check": "leaf-rotation-crl-old-cert-new-connection-denied", "result": "PASS"}) wait_ready() c = Client("https://localhost:8443", str(LOCAL / "trust/ca.crt")) alice, bob, eve, chat = c.bootstrap() attachment, _ = c.attachment(alice, chat) c.wait_ready(alice, attachment) rows.append({"check": "pipeline-after-credential-and-certificate-rotation", "result": "PASS"}) # Sealed Bao: cached credentials can run only until last renewable token TTL. token = root_token() request("sys/seal", {}, token=token) try: deadline = time.monotonic() + 80 while worker_children() > 0 and time.monotonic() < deadline: time.sleep(2) assert worker_children() == 0, "Worker kept running past its Bao access deadline" rows.append({"check": "sealed-bao-stops-worker-within-ttl", "result": "PASS"}) api_stopped = False deadline = time.monotonic() + 25 while time.monotonic() < deadline: try: with urllib.request.urlopen( "https://localhost:8443/health/ready", context=tls(None), timeout=4 ) as _response: pass except urllib.error.HTTPError as error: assert error.code == 503 api_stopped = True break time.sleep(1) assert api_stopped rows.append({"check": "sealed-bao-api-fails-closed", "result": "PASS"}) finally: subprocess.run([sys.executable, "scripts/lab7.py", "unseal"], cwd=ROOT, check=True) wait_ready() deadline = time.monotonic() + 30 while worker_children() == 0 and time.monotonic() < deadline: time.sleep(1) assert worker_children() > 0 c.call("GET", "/api/v1/chats/" + chat["id"], actor=alice, schema="Chat") attachment, _ = c.attachment(alice, chat) c.wait_ready(alice, attachment) rows.append({"check": "unseal-recovers-api-session-and-pipeline", "result": "PASS"}) (ROOT / "evidence/lifecycle-summary.json").write_text( json.dumps({"checks": rows, "passed": len(rows)}, indent=2) + "\n" ) print("PASS:", len(rows), "lifecycle checks") if __name__ == "__main__": main()