"""Explicit operator PKI lifecycle; trust is never disabled.""" import argparse import json import time from uuid import uuid4 from cryptography import x509 from cryptography.hazmat.primitives import hashes from scripts.bao_operator import Operator from scripts.bootstrap_security import ( OPERATOR, ROLES, SECRET, TRUST, ca, key_bytes, make_root, public, sign_intermediate, sign_leaf, write_private, ) from scripts.credentials import install, repair from scripts.stand import compose REQUEST = ( b"GET /health/live HTTP/1.1\r\nHost: api\r\nConnection: close\r\n\r\n" ) def fixture_check(role, *, reject=False): expression = f"exchange('api',8443,{role!r},{REQUEST!r})" assertion = ( "denied(lambda:" + expression + ")" if reject else "b' 200 ' in " + expression ) code = ( "from src.security_probe import exchange,denied;assert " + assertion + ";print('Fresh TLS fixture check passed')" ) compile(code, "fixture-check", "exec") return compose( "--profile", "operator", "run", "--rm", "--no-deps", "operator-probe", "-c", code, ) META = """import json,pathlib from cryptography import x509 result={} for directory in pathlib.Path('/identities').iterdir(): try: cert=x509.load_pem_x509_certificate((directory/'current/cert.pem').read_bytes()) result[directory.name]={'serial':format(cert.serial_number,'x'),'issuer':cert.issuer.rfc4514_string(),'expires_at':cert.not_valid_after_utc.timestamp()} except FileNotFoundError:pass print(json.dumps(result)) """ def metadata(): return json.loads( compose( "--profile", "operator", "run", "--rm", "--no-deps", "operator-probe", "-c", META, ) ) def reload_lb(): compose( "exec", "-T", "identity-lb", "python", "-c", "import os,signal;os.kill(1,signal.SIGUSR2)", ) def preserve_lb(): data = json.loads( compose( "exec", "-T", "identity-lb", "python", "-c", "import json,pathlib;p=pathlib.Path('/run/identity/current');print(json.dumps({name:(p/name).read_text() for name in ['cert.pem','key.pem']}))", ) ) directory = SECRET / "probe/revoked-lb" for name, value in data.items(): write_private(directory / name, value, mode=0o644) leaf = x509.load_pem_x509_certificate(data["cert.pem"].encode()) return { "serial": format(leaf.serial_number, "x"), "expires_at": leaf.not_valid_after_utc.timestamp(), } def revoke(operator, leaf): assert leaf["expires_at"] > time.time() + 5, ( "Revocation must be tested before natural expiry" ) serial = leaf["serial"] serial = ("0" + serial) if len(serial) % 2 else serial colon = ":".join(serial[i : i + 2] for i in range(0, len(serial), 2)) operator.authorized("POST", "pki/revoke", {"serial_number": colon}) response = operator.http.get( "pki/crl/pem", headers={"X-Vault-Token": operator.token or operator.authenticate()}, ) response.raise_for_status() operator.authorized( "PUT", "auth/cert/crls/service-issuer", {"crl": response.text} ) revoked = json.loads((TRUST / "revoked.json").read_text()) revoked[leaf["serial"]] = True write_private( TRUST / "revoked.json", json.dumps(revoked) + "\n", mode=0o644 ) fixture_check("revoked-lb", reject=True) return { "revoked_serial": leaf["serial"], "still_unexpired_at_check": leaf["expires_at"] > time.time(), "new_tls_connection_denied": True, "bao_crl_updated": True, } def reload_bao(): compose("restart", "openbao") last = None for _ in range(60): operator = Operator() try: operator.unseal() operator.authenticate() return except Exception as error: last = type(error).__name__ finally: operator.close() time.sleep(1) raise TimeoutError("Verified Bao restart/unseal unavailable: " + str(last)) def recover_bao(): # Explicit operator recovery after the server leaf itself has expired. key, root = ca() leaf_key, leaf = sign_leaf( key, root, "openbao", ["openbao", "localhost"], ["127.0.0.1"], seconds=120, ) install( "openbao", { "certificate": public(leaf).decode(), "private_key": key_bytes(leaf_key).decode(), "ca_chain": [public(root).decode()], }, ) time.sleep(6) operator = Operator() try: operator.unseal() for role in ROLES: if role != "migrate": repair(operator, role) finally: operator.close() def rollover(): old_key, old_root = ca() new_key, new_root = make_root() directory = OPERATOR / ("rollover-" + uuid4().hex[:12]) directory.mkdir(mode=0o700) fixture_key, fixture = sign_leaf( old_key, old_root, "lb", ["lb"], seconds=900 ) write_private( SECRET / "probe/retired-ca-lb/cert.pem", public(fixture) + public(old_root), mode=0o644, ) write_private( SECRET / "probe/retired-ca-lb/key.pem", key_bytes(fixture_key), mode=0o644, ) fixture_check("retired-ca-lb") for name, value in [ ("old-key.pem", key_bytes(old_key)), ("old-ca.pem", public(old_root)), ("new-key.pem", key_bytes(new_key)), ("new-ca.pem", public(new_root)), ]: write_private(directory / name, value) journal = { "phase": "dual_trust", "old_ca_sha256": old_root.fingerprint(hashes.SHA256()).hex(), "new_ca_sha256": new_root.fingerprint(hashes.SHA256()).hex(), } write_private(directory / "state.json", json.dumps(journal) + "\n") write_private( TRUST / "ca.pem", public(old_root) + public(new_root), mode=0o644 ) reload_lb() # Load both roots before any API server leaf changes issuer. reload_bao() # Native client CA requires an explicit listener restart. operator = Operator() try: for role in [*ROLES, "operator"]: operator.authorized( "POST", "auth/cert/certs/" + role + "-v2", { "display_name": role, "certificate": public(new_root).decode(), "allowed_common_names": [role], "allowed_uri_sans": ["spiffe://messenger/" + role], "token_policies": ["messenger-" + role], "token_ttl": "900s" if role == "operator" else "120s", "token_max_ttl": "3600s" if role == "operator" else "600s", "token_no_default_policy": True, }, ) csr = operator.authorized( "POST", "pki/intermediate/generate/internal", { "common_name": "Messenger rollover intermediate " + uuid4().hex[:12], "key_type": "ec", "key_bits": 256, }, )["data"]["csr"] intermediate = sign_intermediate( new_key, new_root, x509.load_pem_x509_csr(csr.encode()) ) imported = operator.authorized( "POST", "pki/intermediate/set-signed", { "certificate": ( public(intermediate) + public(new_root) ).decode() }, )["data"]["imported_issuers"] chosen = None for issuer in imported: details = operator.authorized("GET", "pki/issuer/" + issuer)[ "data" ] if ( x509.load_pem_x509_certificate( details["certificate"].encode() ).serial_number == intermediate.serial_number ): chosen = issuer assert chosen operator.authorized( "POST", "pki/config/issuers", {"default": chosen, "default_follows_latest_issuer": False}, ) journal["phase"] = "new_issuer_active" write_private(directory / "state.json", json.dumps(journal) + "\n") deadline = time.monotonic() + 120 active = set(ROLES) - {"migrate"} while time.monotonic() < deadline: observed = metadata() if active <= observed.keys() and all( observed[r]["issuer"] == intermediate.subject.rfc4514_string() for r in active ): break time.sleep(3) else: raise TimeoutError( "Not all active services renewed into new issuer" ) fixture_check("retired-ca-lb") # Allow old L4 connections to expire, then remove old trust everywhere. time.sleep(46) for role in [*ROLES, "operator"]: operator.authorized("DELETE", "auth/cert/certs/" + role + "-v1") client_key, client_cert = sign_leaf( new_key, new_root, "operator", ["operator"], seconds=86400 ) write_private(OPERATOR / "client/key.pem", key_bytes(client_key)) write_private( OPERATOR / "client/cert.pem", public(client_cert) + public(new_root), ) write_private(OPERATOR / "ca-key.pem", key_bytes(new_key)) write_private(OPERATOR / "ca.pem", public(new_root)) write_private(TRUST / "ca.pem", public(new_root), mode=0o644) journal["phase"] = "old_trust_retired" write_private(directory / "state.json", json.dumps(journal) + "\n") finally: operator.close() reload_lb() reload_bao() fixture_check("retired-ca-lb", reject=True) fixture_check("lb") journal["phase"] = "complete" write_private(directory / "state.json", json.dumps(journal) + "\n") return { **journal, "issuer_id": chosen, "active_service_count": len(active), "old_l4_connections_waited_seconds": 46, "old_ca_fixture_valid_during_dual_trust": True, "old_ca_new_connection_denied_after_retirement": True, "services": metadata(), "image_rebuilt": False, } def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "action", choices=["metadata", "rollover", "recover-bao"] ) args = parser.parse_args() if args.action == "metadata": result = metadata() elif args.action == "rollover": result = rollover() else: recover_bao() result = {"recovered": True} print(json.dumps(result)) if __name__ == "__main__": main()