Files
2026-11-20 15:30:33 +03:00

151 lines
5.8 KiB
Python

#!/usr/bin/env python3
"""Operator-run rotation, expiry/revoke, sealed-store and restart recovery tests."""
# ruff: noqa: E402 -- repository scripts are imported after setting their path.
import json
import subprocess
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
from client import Client
from securitylib import LOCAL, OPERATOR, compose, request, root_token, tls
def wait_ready(timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(
"https://localhost:8443/health/ready", context=tls(None), timeout=3
) as response:
if response.status == 200:
return
except (OSError, urllib.error.URLError):
pass
time.sleep(1)
raise AssertionError("HTTPS did not recover")
def worker_children():
# PID 1 is supervisor (or tini); readiness checks the actual application child.
code = "from pathlib import Path; print(sum(b'src.pipeline.worker' in p.read_bytes() and b'src.secure.runtime' not in p.read_bytes() for p in Path('/proc').glob('[0-9]*/cmdline')))"
return int(compose("exec", "-T", "worker", "python", "-c", code, capture=True))
def main():
rows = []
started = time.monotonic()
subprocess.run([sys.executable, "scripts/lab7.py", "rotate-secret"], cwd=ROOT, check=True)
record = json.loads((OPERATOR / "last-worker-rotation.json").read_text())
code = """import asyncio,json,sys,asyncpg
from src.secure.transport import context
async def main():
x=json.load(sys.stdin)
try:c=await asyncpg.connect(x['url'],ssl=context(),direct_tls=True,timeout=3)
except asyncpg.InvalidAuthorizationSpecificationError:print('OLD_DENIED');return
else:await c.close();raise AssertionError('Old DB credential still accepted')
asyncio.run(main())"""
result = subprocess.check_output(
["docker", "compose", "run", "--rm", "--no-deps", "-T", "migration", "python", "-c", code],
input=json.dumps({"url": record["old_url"]}),
text=True,
cwd=ROOT,
)
assert "OLD_DENIED" in result
rows.append(
{
"check": "db-secret-rotation-old-login-denied",
"result": "PASS",
"seconds": round(time.monotonic() - started, 2),
}
)
before = set(OPERATOR.glob("revoked-worker-*"))
subprocess.run(
[sys.executable, "scripts/lab7.py", "rotate-cert", "--role", "worker"], cwd=ROOT, check=True
)
archived = (set(OPERATOR.glob("revoked-worker-*")) - before).pop()
# New TLS connection with revoked certificate must fail, not only an old socket.
code = """import socket,ssl
ctx=ssl.create_default_context(cafile='/run/trust/ca.crt')
ctx.load_cert_chain('/run/revoked/service.crt','/run/revoked/service.key')
try:
with socket.create_connection(('broker',6379),timeout=3) as raw:
with ctx.wrap_socket(raw,server_hostname='broker') as s:
s.sendall(b'*1\\r\\n$4\\r\\nPING\\r\\n');data=s.recv(100)
assert not data,'Revoked client certificate accepted'
except (ssl.SSLError,ConnectionResetError):pass
print('OLD_CERT_DENIED')"""
result = compose(
"run",
"--rm",
"--no-deps",
"-v",
str(archived) + ":/run/revoked:ro",
"migration",
"python",
"-c",
code,
capture=True,
)
assert "OLD_CERT_DENIED" in result
deadline = time.monotonic() + 60
while worker_children() == 0 and time.monotonic() < deadline:
time.sleep(1)
assert worker_children() > 0
rows.append({"check": "leaf-rotation-crl-old-cert-new-connection-denied", "result": "PASS"})
wait_ready()
c = Client("https://localhost:8443", str(LOCAL / "trust/ca.crt"))
alice, bob, eve, chat = c.bootstrap()
attachment, _ = c.attachment(alice, chat)
c.wait_ready(alice, attachment)
rows.append({"check": "pipeline-after-credential-and-certificate-rotation", "result": "PASS"})
# Sealed Bao: cached credentials can run only until last renewable token TTL.
token = root_token()
request("sys/seal", {}, token=token)
try:
deadline = time.monotonic() + 80
while worker_children() > 0 and time.monotonic() < deadline:
time.sleep(2)
assert worker_children() == 0, "Worker kept running past its Bao access deadline"
rows.append({"check": "sealed-bao-stops-worker-within-ttl", "result": "PASS"})
api_stopped = False
deadline = time.monotonic() + 25
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(
"https://localhost:8443/health/ready", context=tls(None), timeout=4
) as _response:
pass
except urllib.error.HTTPError as error:
assert error.code == 503
api_stopped = True
break
time.sleep(1)
assert api_stopped
rows.append({"check": "sealed-bao-api-fails-closed", "result": "PASS"})
finally:
subprocess.run([sys.executable, "scripts/lab7.py", "unseal"], cwd=ROOT, check=True)
wait_ready()
deadline = time.monotonic() + 30
while worker_children() == 0 and time.monotonic() < deadline:
time.sleep(1)
assert worker_children() > 0
c.call("GET", "/api/v1/chats/" + chat["id"], actor=alice, schema="Chat")
attachment, _ = c.attachment(alice, chat)
c.wait_ready(alice, attachment)
rows.append({"check": "unseal-recovers-api-session-and-pipeline", "result": "PASS"})
(ROOT / "evidence/lifecycle-summary.json").write_text(
json.dumps({"checks": rows, "passed": len(rows)}, indent=2) + "\n"
)
print("PASS:", len(rows), "lifecycle checks")
if __name__ == "__main__":
main()