#!/usr/bin/env python3 """Reproducible negative checks and worker revocation on the local lab7 stand.""" import argparse import base64 import hashlib import json import secrets import shutil import socket import ssl import subprocess import sys import tempfile import time import uuid from pathlib import Path from urllib.error import HTTPError, URLError from urllib.request import Request, urlopen ROOT = Path(__file__).resolve().parents[1] LOCAL = ROOT / "secrets" CA = LOCAL / "tls/trust/ca.crt" INSIDE_CA = "/run/tls/trust/ca.crt" def checked(*command, **kwargs): try: return subprocess.run(command, cwd=ROOT, check=True, **kwargs) except subprocess.CalledProcessError as exc: if exc.stderr: sys.stderr.write(exc.stderr) raise def compose(*args, **kwargs): return checked("docker", "compose", *args, **kwargs) def context(ca, certificate=None): ctx = ssl.create_default_context(cafile=ca) if certificate: ctx.load_cert_chain(certificate / "service.crt", certificate / "service.key") return ctx def response(url, ctx, data=None, headers=None, method=None): request = Request(url, data, headers or {}, method=method) try: with urlopen(request, context=ctx, timeout=4) as r: return r.status, r.read(262144) except HTTPError as e: return e.code, b"" def denied(action): try: status = action() assert status in (401, 403) or status == b"", "Unexpected success in negative probe" except (ssl.SSLError, ConnectionResetError, BrokenPipeError, URLError) as exc: # Preserve DNS/connectivity errors as real test failures. cause = exc.reason if isinstance(exc, URLError) else exc if not isinstance(cause, (ssl.SSLError, ConnectionResetError, BrokenPipeError)): raise def redis_ping(ctx): with socket.create_connection(("broker", 6380), timeout=3) as raw: with ctx.wrap_socket(raw, server_hostname="broker") as tls: tls.sendall(b"*1\r\n$4\r\nPING\r\n") return tls.recv(256) def audit_tls(): results = {} endpoints = { "api": (lambda ctx: response("https://api:8080/health/ready", ctx)[0], "lb-client"), "collector": ( lambda ctx: response( "https://otel-collector:4319/v1/traces", ctx, b"", {"Content-Type": "application/x-protobuf"}, )[0], "worker-client", ), "broker": (redis_ping, "worker-client"), } for name, (probe, identity) in endpoints.items(): status = probe(context(INSIDE_CA, Path("/probe") / identity)) assert status == 200 or status.startswith(b"-NOAUTH") for cert in (None, Path("/probe/probe-client"), Path("/probe/rogue")): denied(lambda probe=probe, cert=cert: probe(context(INSIDE_CA, cert))) results[name] = [ "allowed", "no_client_denied", "same_ca_wrong_role_denied", "rogue_ca_denied", ] for host, port in ( ("broker", 6379), ("otel-collector", 4318), ("prometheus", 19090), ("loki", 13100), ("tempo", 13200), ("tempo", 14318), ("tempo", 9095), ("loki", 9095), ): try: with socket.create_connection((host, port), timeout=1): raise AssertionError("Reachable plaintext " + host) except (ConnectionRefusedError, TimeoutError): pass results["network_plaintext_bypass"] = "denied" results["grafana_backends"] = {} for host, port, path in ( ("prometheus", 9090, "/api/v1/status/buildinfo"), ("loki", 3100, "/loki/api/v1/status/buildinfo"), ("tempo", 3200, "/api/echo"), ): endpoint = f"https://{host}:{port}{path}" assert response(endpoint, context(INSIDE_CA, Path("/probe/grafana-client")))[0] == 200 denied( lambda endpoint=endpoint: response( endpoint, context(INSIDE_CA, Path("/probe/probe-client")) )[0] ) results["grafana_backends"][host] = "verified_tls_identity_allow_and_deny" return results def worker_scope(): import psycopg import redis from botocore.exceptions import ClientError from messenger.settings import Settings from messenger.storage import ObjectStore settings = Settings.from_env() with psycopg.connect(settings.database_url, autocommit=True) as conn: assert conn.execute("SELECT ssl FROM pg_stat_ssl WHERE pid=pg_backend_pid()").fetchone()[0] conn.execute("SELECT count(*) FROM attachment_jobs").fetchone() for query in ( "SELECT count(*) FROM users", "SELECT count(*) FROM messages", "SELECT count(*) FROM sessions", "CREATE TEMP TABLE forbidden_probe(id int)", "UPDATE attachments SET owner_id=owner_id WHERE false", "SET ROLE messenger", ): try: conn.execute(query) except psycopg.errors.InsufficientPrivilege: continue raise AssertionError("Unexpected database grant") broker = redis.Redis.from_url(settings.broker_url) assert broker.ping() for op in ( lambda: broker.config_get("appendonly"), lambda: broker.xrange("unrelated-key"), lambda: broker.get("session-user:synthetic"), ): try: op() except redis.exceptions.NoPermissionError: continue raise AssertionError("Unexpected broker grant") store = ObjectStore(settings) suffix = uuid.uuid4().hex key = "objects/audit/" + suffix store.client.put_object(Bucket=store.bucket, Key=key, Body=b"synthetic") assert store.client.head_object(Bucket=store.bucket, Key=key)["ContentLength"] == 9 for op in ( lambda: store.client.put_object( Bucket=store.bucket, Key="staging/audit/" + suffix, Body=b"x" ), lambda: store.client.delete_object(Bucket=store.bucket, Key=key), lambda: store.client.get_bucket_policy(Bucket=store.bucket), ): try: op() except ClientError as exc: assert exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403 continue raise AssertionError("Unexpected S3 grant") cert = Path("/run/tls/client") ctx = context(INSIDE_CA, cert) base = "https://openbao:8200/v1/" ct = {"Content-Type": "application/json"} code, raw = response(base + "auth/cert/login", ctx, b'{"name":"worker"}', ct) assert code == 200 auth = json.loads(raw)["auth"] token = auth["client_token"] headers = {**ct, "X-Vault-Token": token} assert 0 < auth["lease_duration"] <= 120 assert response(base + "services/data/worker", ctx, headers=headers)[0] == 200 assert response(base + "services/data/api", ctx, headers=headers)[0] == 403 assert response(base + "services/data/worker", ctx, b'{"data":{}}', headers)[0] == 403 assert response(base + "auth/token/revoke-self", ctx, b"{}", headers)[0] == 204 assert response(base + "services/data/worker", ctx, headers=headers)[0] == 403 return { "postgres": {"tls": True, "job_read": True, "denied_operations": 6}, "broker": {"tls": True, "ping": True, "denied_operations": 3}, "s3": {"put_objects": True, "denied_operations": 3}, "openbao": { "own_read": True, "foreign_read_and_write_denied": True, "token_ttl_s": auth["lease_duration"], "revoked_token_denied": True, }, } def old_identity(): ctx = context(INSIDE_CA, Path("/probe/old")) denied(lambda: redis_ping(ctx)) denied( lambda: response( "https://otel-collector:4319/v1/traces", ctx, b"", {"Content-Type": "application/x-protobuf"}, )[0] ) code, _ = response( "https://openbao:8200/v1/auth/cert/login", ctx, b'{"name":"worker"}', {"Content-Type": "application/json"}, ) assert code in (400, 403) return {"broker": "revoked", "collector": "revoked", "openbao": "revoked"} def worker_probe(): raw = compose( "run", "--rm", "--no-deps", "-T", "-v", f"{Path(__file__).resolve()}:/test/audit.py:ro", "--entrypoint", "python", "worker", "/test/audit.py", "--inside", "worker", capture_output=True, text=True, ).stdout return json.loads(raw) def isolated(mode, mounts): cmd = [ "docker", "run", "--rm", "--network", "mtusi-lab7_backend", "--read-only", "--cap-drop", "ALL", "--security-opt", "no-new-privileges:true", "--memory", "256m", "-v", f"{Path(__file__).resolve()}:/test/audit.py:ro", "-v", f"{LOCAL / 'tls/trust'}:/run/tls/trust:ro", ] for name, source in mounts.items(): cmd += ["-v", f"{source}:/probe/{name}:ro"] image = compose("images", "-q", "api", capture_output=True, text=True).stdout.split()[0] raw = checked( *cmd, image, "python", "/test/audit.py", "--inside", mode, capture_output=True, text=True ).stdout return json.loads(raw) def check_api(client): alice, bob, eve, chat = client.bootstrap() attachment, _ = client.attachment(alice, chat) client.wait_ready(bob, attachment) path = "/api/v1/attachments/" + attachment["id"] client.call("GET", path, 404, actor=eve, headers={"X-User-Id": alice["id"]}) client.call("GET", path + "/content", 404, actor=eve) client.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=bob) client.call("GET", "/api/v1/users/me", 401, actor=bob) reg = alice["registration"] bad = base64.b64encode((reg["username"] + ":invalid-password-123").encode()).decode() statuses, replicas, times = [], set(), [] for _ in range(6): r = client.call( "POST", "/api/v1/auth/sessions", [401, 429], headers={"Authorization": "Basic " + bad} ) statuses.append(r["status"]) times.append(r["elapsed"] * 1000) replicas.add(r["headers"]["X-Instance-Id"]) if r["status"] == 429: assert int(r["headers"]["Retry-After"]) > 0 assert 429 in statuses and 401 in statuses and len(replicas) >= 2 too_big = client.call( "POST", "/api/v1/users", 413, raw=b"x" * 66000, headers={"Content-Type": "application/json"} ) assert too_big["status"] == 413 return { "statuses": statuses, "replicas": len(replicas), "latency_ms": times, "outsider_denied": True, "logout_revoked": True, "oversize_json_denied": True, } def rotate(client, temp): alice, _, _, chat = client.bootstrap() old = temp / "old" shutil.copytree(LOCAL / "tls/worker-client", old) compose("stop", "worker") accepted = [] start = time.monotonic() try: for _ in range(5): att, data = client.attachment(alice, chat) accepted.append((att, data)) checked(sys.executable, "scripts/bao_operator.py", "--revoke", "worker") checked(sys.executable, "scripts/pki.py", "--rotate", "worker-client") password = LOCAL / "broker_worker_password" password.write_text(secrets.token_hex(24) + "\n") checked(sys.executable, "scripts/bootstrap.py") # Load the changed ACL without restarting durable Valkey or erasing its stream. compose( "exec", "-T", "broker", "sh", "-ec", """p="$(awk '/^user operator / {print substr($NF,2)}' /run/secrets/broker.acl)" export REDISCLI_AUTH="$p" valkey-cli --user operator ACL LOAD | grep -q '^OK$' valkey-cli --user operator CLIENT KILL USER worker >/dev/null""", ) compose("restart", "broker-tls", "otel-collector-tls") result = isolated("old", {"old": old}) finally: checked(sys.executable, "scripts/bao_operator.py") compose("up", "-d", "--no-deps", "--force-recreate", "worker") for attachment, original in accepted: item = client.wait_ready(alice, attachment) file = client.call("GET", f"/api/v1/attachments/{item['id']}/content", actor=alice) assert hashlib.sha256(file["bytes"]).digest() == hashlib.sha256(original).digest() return { "old_identity": result, "accepted": len(accepted), "ready_with_original": len(accepted), "elapsed_seconds": round(time.monotonic() - start, 3), "broker_acl_reloaded": True, "images_rebuilt": False, "new_worker": worker_probe(), } def check_sealed(client): client.call("GET", "/health/ready", schema="Health") start = time.monotonic() checked(sys.executable, "scripts/bao_operator.py", "--seal") try: deadline = start + 42 status = 200 while time.monotonic() < deadline: status, _ = response("https://localhost:8443/health/ready", context(CA)) if status == 503: break time.sleep(1) assert status == 503, "API did not stop serving after the credential grace period" unavailable = time.monotonic() - start finally: checked(sys.executable, "scripts/bao_operator.py") deadline = time.monotonic() + 22 while time.monotonic() < deadline: status, _ = response("https://localhost:8443/health/ready", context(CA)) if status == 200: return { "unready_after_seconds": round(unavailable, 3), "recovered_after_seconds": round(time.monotonic() - start, 3), } time.sleep(1) raise AssertionError("API readiness did not recover after unseal") def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--inside", choices=("tls", "worker", "old")) parser.add_argument("--output", type=Path) parser.add_argument("--rotate", action="store_true") parser.add_argument("--seal", action="store_true") args = parser.parse_args() if args.inside: print( json.dumps( {"tls": audit_tls, "worker": worker_scope, "old": old_identity}[args.inside]() ) ) return if not args.output or args.output.exists(): parser.error("choose a new --output") sys.path.insert(0, str(ROOT / "tests")) from client import Client client = Client("https://localhost:8443", str(CA)) with tempfile.TemporaryDirectory(dir=LOCAL) as folder: tmp = Path(folder) tmp.chmod(0o755) rogue = tmp / "rogue" rogue.mkdir(mode=0o755) checked( "openssl", "req", "-x509", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:P-256", "-nodes", "-days", "1", "-subj", "/CN=worker-client", "-addext", "extendedKeyUsage=clientAuth", "-keyout", str(rogue / "service.key"), "-out", str(rogue / "service.crt"), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) (rogue / "service.key").chmod(0o644) identities = { name: LOCAL / "tls" / name for name in ("lb-client", "worker-client", "probe-client", "grafana-client") } result = { "tls": isolated("tls", {**identities, "rogue": rogue}), "worker_scope": worker_probe(), "api": check_api(client), } print("PASS: verified TLS and distinct service privileges", flush=True) if args.rotate: result["worker_compromise"] = rotate(client, tmp) print("PASS: old identity denied; all accepted uploads recovered", flush=True) if args.seal: result["sealed"] = check_sealed(client) print( "PASS: sealed OpenBao makes API unready within a bounded grace period", flush=True ) args.output.write_text(json.dumps(result, indent=2) + "\n") if __name__ == "__main__": main()