forked from mtusi2026/mtusi2026lab7
151 lines
5.8 KiB
Python
151 lines
5.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Operator-run rotation, expiry/revoke, sealed-store and restart recovery tests."""
|
|
|
|
# ruff: noqa: E402 -- repository scripts are imported after setting their path.
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
from client import Client
|
|
from securitylib import LOCAL, OPERATOR, compose, request, root_token, tls
|
|
|
|
|
|
def wait_ready(timeout=90):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with urllib.request.urlopen(
|
|
"https://localhost:8443/health/ready", context=tls(None), timeout=3
|
|
) as response:
|
|
if response.status == 200:
|
|
return
|
|
except (OSError, urllib.error.URLError):
|
|
pass
|
|
time.sleep(1)
|
|
raise AssertionError("HTTPS did not recover")
|
|
|
|
|
|
def worker_children():
|
|
# PID 1 is supervisor (or tini); readiness checks the actual application child.
|
|
code = "from pathlib import Path; print(sum(b'src.pipeline.worker' in p.read_bytes() and b'src.secure.runtime' not in p.read_bytes() for p in Path('/proc').glob('[0-9]*/cmdline')))"
|
|
return int(compose("exec", "-T", "worker", "python", "-c", code, capture=True))
|
|
|
|
|
|
def main():
|
|
rows = []
|
|
started = time.monotonic()
|
|
subprocess.run([sys.executable, "scripts/lab7.py", "rotate-secret"], cwd=ROOT, check=True)
|
|
record = json.loads((OPERATOR / "last-worker-rotation.json").read_text())
|
|
code = """import asyncio,json,sys,asyncpg
|
|
from src.secure.transport import context
|
|
async def main():
|
|
x=json.load(sys.stdin)
|
|
try:c=await asyncpg.connect(x['url'],ssl=context(),direct_tls=True,timeout=3)
|
|
except asyncpg.InvalidAuthorizationSpecificationError:print('OLD_DENIED');return
|
|
else:await c.close();raise AssertionError('Old DB credential still accepted')
|
|
asyncio.run(main())"""
|
|
result = subprocess.check_output(
|
|
["docker", "compose", "run", "--rm", "--no-deps", "-T", "migration", "python", "-c", code],
|
|
input=json.dumps({"url": record["old_url"]}),
|
|
text=True,
|
|
cwd=ROOT,
|
|
)
|
|
assert "OLD_DENIED" in result
|
|
rows.append(
|
|
{
|
|
"check": "db-secret-rotation-old-login-denied",
|
|
"result": "PASS",
|
|
"seconds": round(time.monotonic() - started, 2),
|
|
}
|
|
)
|
|
before = set(OPERATOR.glob("revoked-worker-*"))
|
|
subprocess.run(
|
|
[sys.executable, "scripts/lab7.py", "rotate-cert", "--role", "worker"], cwd=ROOT, check=True
|
|
)
|
|
archived = (set(OPERATOR.glob("revoked-worker-*")) - before).pop()
|
|
# New TLS connection with revoked certificate must fail, not only an old socket.
|
|
code = """import socket,ssl
|
|
ctx=ssl.create_default_context(cafile='/run/trust/ca.crt')
|
|
ctx.load_cert_chain('/run/revoked/service.crt','/run/revoked/service.key')
|
|
try:
|
|
with socket.create_connection(('broker',6379),timeout=3) as raw:
|
|
with ctx.wrap_socket(raw,server_hostname='broker') as s:
|
|
s.sendall(b'*1\\r\\n$4\\r\\nPING\\r\\n');data=s.recv(100)
|
|
assert not data,'Revoked client certificate accepted'
|
|
except (ssl.SSLError,ConnectionResetError):pass
|
|
print('OLD_CERT_DENIED')"""
|
|
result = compose(
|
|
"run",
|
|
"--rm",
|
|
"--no-deps",
|
|
"-v",
|
|
str(archived) + ":/run/revoked:ro",
|
|
"migration",
|
|
"python",
|
|
"-c",
|
|
code,
|
|
capture=True,
|
|
)
|
|
assert "OLD_CERT_DENIED" in result
|
|
deadline = time.monotonic() + 60
|
|
while worker_children() == 0 and time.monotonic() < deadline:
|
|
time.sleep(1)
|
|
assert worker_children() > 0
|
|
rows.append({"check": "leaf-rotation-crl-old-cert-new-connection-denied", "result": "PASS"})
|
|
wait_ready()
|
|
c = Client("https://localhost:8443", str(LOCAL / "trust/ca.crt"))
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
attachment, _ = c.attachment(alice, chat)
|
|
c.wait_ready(alice, attachment)
|
|
rows.append({"check": "pipeline-after-credential-and-certificate-rotation", "result": "PASS"})
|
|
# Sealed Bao: cached credentials can run only until last renewable token TTL.
|
|
token = root_token()
|
|
request("sys/seal", {}, token=token)
|
|
try:
|
|
deadline = time.monotonic() + 80
|
|
while worker_children() > 0 and time.monotonic() < deadline:
|
|
time.sleep(2)
|
|
assert worker_children() == 0, "Worker kept running past its Bao access deadline"
|
|
rows.append({"check": "sealed-bao-stops-worker-within-ttl", "result": "PASS"})
|
|
api_stopped = False
|
|
deadline = time.monotonic() + 25
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with urllib.request.urlopen(
|
|
"https://localhost:8443/health/ready", context=tls(None), timeout=4
|
|
) as _response:
|
|
pass
|
|
except urllib.error.HTTPError as error:
|
|
assert error.code == 503
|
|
api_stopped = True
|
|
break
|
|
time.sleep(1)
|
|
assert api_stopped
|
|
rows.append({"check": "sealed-bao-api-fails-closed", "result": "PASS"})
|
|
finally:
|
|
subprocess.run([sys.executable, "scripts/lab7.py", "unseal"], cwd=ROOT, check=True)
|
|
wait_ready()
|
|
deadline = time.monotonic() + 30
|
|
while worker_children() == 0 and time.monotonic() < deadline:
|
|
time.sleep(1)
|
|
assert worker_children() > 0
|
|
c.call("GET", "/api/v1/chats/" + chat["id"], actor=alice, schema="Chat")
|
|
attachment, _ = c.attachment(alice, chat)
|
|
c.wait_ready(alice, attachment)
|
|
rows.append({"check": "unseal-recovers-api-session-and-pipeline", "result": "PASS"})
|
|
(ROOT / "evidence/lifecycle-summary.json").write_text(
|
|
json.dumps({"checks": rows, "passed": len(rows)}, indent=2) + "\n"
|
|
)
|
|
print("PASS:", len(rows), "lifecycle checks")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|