forked from mtusi2026/mtusi2026lab7
208 lines
7.4 KiB
Python
208 lines
7.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Rotate the LB client leaf, pin the replacement, and prove the prior leaf is rejected."""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import ssl
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
import urllib.request
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
PKI_DIR = ROOT / "secrets" / "pki"
|
|
CA_CERT = PKI_DIR / "ca.crt"
|
|
CA_KEY = PKI_DIR / "ca.key"
|
|
CERT_PATH = PKI_DIR / "lb-client.crt"
|
|
KEY_PATH = PKI_DIR / "lb-client.key"
|
|
BUNDLE_PATH = PKI_DIR / "lb-client.pem"
|
|
API_PROBE = """
|
|
import json, ssl, sys, tempfile, urllib.request, urllib.error
|
|
value = json.load(sys.stdin)
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
cert = directory + "/client.crt"
|
|
key = directory + "/client.key"
|
|
open(cert, "w").write(value["cert"])
|
|
open(key, "w").write(value["key"])
|
|
context = ssl.create_default_context(cafile="/run/secrets/ca_crt")
|
|
context.load_cert_chain(cert, key)
|
|
try:
|
|
with urllib.request.urlopen("https://127.0.0.1:8080/health/ready", context=context, timeout=5) as response:
|
|
result = {"accepted": response.status == 200}
|
|
except (urllib.error.URLError, ssl.SSLError, OSError):
|
|
result = {"accepted": False}
|
|
print(json.dumps(result))
|
|
"""
|
|
|
|
|
|
def run(command, *, input_text=None):
|
|
return subprocess.run(
|
|
command,
|
|
cwd=ROOT,
|
|
input=input_text,
|
|
text=True,
|
|
check=True,
|
|
capture_output=True,
|
|
)
|
|
|
|
|
|
def compose(*args, input_text=None):
|
|
return run(["docker", "compose", *args], input_text=input_text)
|
|
|
|
|
|
def write_atomic(path: Path, content: bytes, mode: int) -> None:
|
|
temporary = path.with_suffix(path.suffix + ".rotation-tmp")
|
|
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode)
|
|
with os.fdopen(descriptor, "wb") as output:
|
|
output.write(content)
|
|
os.chmod(temporary, mode)
|
|
os.replace(temporary, path)
|
|
|
|
|
|
def probe(cert: bytes, key: bytes) -> bool:
|
|
payload = json.dumps(
|
|
{
|
|
"cert": cert.decode("ascii"),
|
|
"key": key.decode("ascii"),
|
|
}
|
|
)
|
|
result = compose("exec", "-T", "api", "python3", "-c", API_PROBE, input_text=payload)
|
|
return json.loads(result.stdout)["accepted"]
|
|
|
|
|
|
def image_ids() -> set:
|
|
container_ids = compose("ps", "-q", "api").stdout.split()
|
|
if len(container_ids) != 2:
|
|
raise RuntimeError("expected exactly two running API replicas")
|
|
return {
|
|
run(["docker", "inspect", "--format", "{{.Image}}", container_id]).stdout.strip()
|
|
for container_id in container_ids
|
|
}
|
|
|
|
|
|
def wait_healthy(timeout: int = 90) -> None:
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
containers = compose("ps", "-q", "api").stdout.split()
|
|
healthy = len(containers) == 2
|
|
for container_id in containers:
|
|
state = json.loads(
|
|
run(["docker", "inspect", "--format", "{{json .State.Health}}", container_id]).stdout
|
|
)
|
|
healthy = healthy and state.get("Status") == "healthy"
|
|
lb = compose("ps", "-q", "load_balancer").stdout.split()
|
|
healthy = healthy and len(lb) == 1
|
|
if healthy:
|
|
state = json.loads(
|
|
run(["docker", "inspect", "--format", "{{json .State.Health}}", lb[0]]).stdout
|
|
)
|
|
healthy = state.get("Status") == "healthy"
|
|
if healthy:
|
|
return
|
|
time.sleep(1)
|
|
raise RuntimeError("API and load balancer did not become healthy after certificate rotation")
|
|
|
|
|
|
def create_replacement(directory: Path) -> tuple[bytes, bytes, bytes]:
|
|
request_path = directory / "lb-client.csr"
|
|
cert_path = directory / "lb-client.crt"
|
|
key_path = directory / "lb-client.key"
|
|
extensions_path = directory / "lb-client.ext"
|
|
subprocess.run(
|
|
[
|
|
"openssl", "req", "-new", "-newkey", "rsa:2048", "-nodes",
|
|
"-keyout", str(key_path), "-out", str(request_path),
|
|
"-subj", "/CN=lab7-load-balancer",
|
|
],
|
|
check=True,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
extensions_path.write_text(
|
|
"basicConstraints=critical,CA:FALSE\n"
|
|
"keyUsage=critical,digitalSignature,keyEncipherment\n"
|
|
"extendedKeyUsage=clientAuth\n"
|
|
"subjectAltName=URI:spiffe://mtusi2026lab7/load-balancer\n",
|
|
encoding="ascii",
|
|
)
|
|
subprocess.run(
|
|
[
|
|
"openssl", "x509", "-req", "-in", str(request_path),
|
|
"-CA", str(CA_CERT), "-CAkey", str(CA_KEY), "-CAcreateserial",
|
|
"-out", str(cert_path), "-days", "30", "-sha256",
|
|
"-extfile", str(extensions_path),
|
|
],
|
|
check=True,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
cert = cert_path.read_bytes()
|
|
key = key_path.read_bytes()
|
|
return cert, key, cert + key
|
|
|
|
|
|
def main() -> None:
|
|
if not CA_CERT.is_file() or not CA_KEY.is_file():
|
|
raise RuntimeError("local CA certificate and operator-held private key are required")
|
|
old_cert = CERT_PATH.read_bytes()
|
|
old_key = KEY_PATH.read_bytes()
|
|
old_bundle = BUNDLE_PATH.read_bytes()
|
|
before_images = image_ids()
|
|
if not probe(old_cert, old_key):
|
|
raise RuntimeError("current LB certificate was not accepted before rotation")
|
|
|
|
replaced = False
|
|
try:
|
|
with tempfile.TemporaryDirectory(prefix="lab7-lb-leaf-") as directory_name:
|
|
new_cert, new_key, new_bundle = create_replacement(Path(directory_name))
|
|
new_fingerprint = hashlib.sha256(ssl.PEM_cert_to_DER_cert(new_cert.decode("ascii"))).hexdigest()
|
|
write_atomic(CERT_PATH, new_cert, 0o644)
|
|
write_atomic(KEY_PATH, new_key, 0o600)
|
|
write_atomic(BUNDLE_PATH, new_bundle, 0o600)
|
|
replaced = True
|
|
|
|
compose("up", "-d", "--no-deps", "--force-recreate", "api", "load_balancer")
|
|
wait_healthy()
|
|
if probe(old_cert, old_key):
|
|
raise RuntimeError("previous LB certificate remains authorized after rotation")
|
|
if not probe(new_cert, new_key):
|
|
raise RuntimeError("replacement LB certificate was not accepted")
|
|
|
|
context = ssl.create_default_context(cafile=str(CA_CERT))
|
|
with urllib.request.urlopen(
|
|
"https://localhost:8443/health/ready",
|
|
context=context,
|
|
timeout=10,
|
|
) as response:
|
|
if response.status != 200:
|
|
raise RuntimeError("HTTPS ingress did not recover after leaf rotation")
|
|
|
|
after_images = image_ids()
|
|
if before_images != after_images:
|
|
raise RuntimeError("container image changed; expected rotation without rebuild")
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"result": "PASS",
|
|
"old_leaf_rejected": True,
|
|
"new_leaf_accepted": True,
|
|
"https_ingress_healthy": True,
|
|
"images_unchanged": True,
|
|
"new_leaf_sha256": new_fingerprint,
|
|
},
|
|
sort_keys=True,
|
|
)
|
|
)
|
|
except Exception:
|
|
if replaced:
|
|
write_atomic(CERT_PATH, old_cert, 0o644)
|
|
write_atomic(KEY_PATH, old_key, 0o600)
|
|
write_atomic(BUNDLE_PATH, old_bundle, 0o600)
|
|
compose("up", "-d", "--no-deps", "--force-recreate", "api", "load_balancer")
|
|
raise
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|