import asyncio import json import socket import ssl import sys import urllib.error import urllib.request from pathlib import Path from urllib.parse import urlencode, urlsplit, urlunsplit import aio_pika import redis.asyncio as redis from botocore.auth import S3SigV4Auth from botocore.awsrequest import AWSRequest from botocore.credentials import Credentials from sqlalchemy import text from sqlalchemy.exc import DBAPIError from sqlalchemy.ext.asyncio import create_async_engine PKI = Path("/audit-pki") RESULTS = [] def record(name, passed, **details): RESULTS.append({"test": name, "passed": bool(passed), **details}) def context(role=None, trusted=True): ctx = ssl.create_default_context(cafile=str(PKI / "operator/ca.crt") if trusted else None) if role: ctx.load_cert_chain(PKI / role / "tls.crt", PKI / role / "tls.key") return ctx def tls(host, port, role, payload, *, hostname=None, trusted=True): try: with ( socket.create_connection((host, port), timeout=3) as raw, context(role, trusted).wrap_socket(raw, server_hostname=hostname or host) as stream, ): stream.sendall(payload) return bool(stream.recv(2048)) except (OSError, ssl.SSLError): return False def http(role, url, payload=None, token=None, *, method=None, headers=None): head = {"Content-Type": "application/json", **(headers or {})} if token: head["X-Vault-Token"] = token data = json.dumps(payload).encode() if isinstance(payload, dict) else payload req = urllib.request.Request(url, data=data, headers=head, method=method) try: with urllib.request.urlopen(req, context=context(role), timeout=5) as response: return response.status, response.read() except urllib.error.HTTPError as error: return error.code, error.read() def credentials(role): status, body = http(role, "https://openbao:8200/v1/auth/cert/login", {"name": role}) if status != 200: raise RuntimeError("Service authentication failed") token = json.loads(body)["auth"]["client_token"] status, body = http(role, "https://openbao:8200/v1/secret/data/" + role, token=token) if status != 200: raise RuntimeError("Scoped configuration unavailable") return token, json.loads(body)["data"]["data"] def transport_matrix(): targets = [ ("api", 8443, "lb"), ("s3", 9443, "api"), ("broker", 5671, "worker"), ("otel-collector", 44318, "api"), ("prometheus", 9443, "grafana"), ("loki", 9443, "grafana"), ("tempo", 9443, "grafana"), ("tempo", 44318, "otel-collector"), ("grafana", 9443, "operator"), ("valkey", 6380, "api"), ] for host, port, role in targets: payload = ( b"AMQP\x00\x00\x09\x01" if host == "broker" else b"*1\r\n$4\r\nPING\r\n" if host == "valkey" else b"GET / HTTP/1.1\r\nHost: " + host.encode() + b"\r\nConnection: close\r\n\r\n" ) for identity, expected in [ (role, True), (None, False), ("outsider", False), ("foreign", False), ]: observed = tls(host, port, identity, payload) record(f"mtls:{host}:{port}:{identity}", observed == expected) record( f"hostname:{host}:{port}", not tls(host, port, role, payload, hostname="incorrect.invalid"), ) for host, port in [ ("api", 8080), ("s3", 9000), ("broker", 5672), ("broker", 15672), ("broker", 15692), ("broker", 25672), ("broker", 4369), ("valkey", 6379), ("otel-collector", 4318), ("otel-collector", 8888), ("otel-collector", 13133), ("prometheus", 9090), ("loki", 3100), ("loki", 9096), ("tempo", 3200), ("tempo", 4318), ("tempo", 9095), ("grafana", 3000), ]: try: with socket.create_connection((host, port), timeout=1): accessible = True except OSError: accessible = False record(f"plaintext-closed:{host}:{port}", not accessible) for role in ["api", "worker", "relay"]: token, _ = credentials(role) own = http(role, "https://openbao:8200/v1/secret/data/" + role, token=token)[0] other = http(role, "https://openbao:8200/v1/secret/data/migrate", token=token)[0] admin = http( role, "https://openbao:8200/v1/sys/policies/acl/forbidden", {"policy": 'path "*" { capabilities=["sudo"] }'}, token, )[0] record( "bao-scope:" + role, own == 200 and other == 403 and admin == 403, own=own, foreign=other, admin=admin, ) def signed_s3(data, method, key, body=b"", *, role="worker"): url = "https://s3:9443/messenger-attachments/" + key request = AWSRequest(method=method, url=url, data=body) S3SigV4Auth( Credentials(data["S3_ACCESS_KEY_ID"], data["S3_SECRET_ACCESS_KEY"]), "s3", "us-east-1" ).add_auth(request) return http(role, url, body, method=method, headers=dict(request.headers))[0] async def worker_scope(): _, data = credentials("worker") dsn = data["DATABASE_URL"].replace("postgresql://", "postgresql+asyncpg://") engine = create_async_engine(dsn, connect_args={"ssl": context()}, hide_parameters=True) probes = [ ("jobs-readable", "SELECT count(*) FROM jobs", True), ("users-denied", "SELECT count(*) FROM users", False), ("sessions-denied", "SELECT count(*) FROM sessions", False), ("ddl-denied", "CREATE TABLE security_escape (id integer)", False), ] for name, sql, expected in probes: try: async with engine.connect() as connection: await connection.execute(text(sql)) await connection.rollback() success = True except DBAPIError: success = False record("worker-db:" + name, success == expected) await engine.dispose() for method, key, expected in [ ("PUT", "variants/security-probe", 200), ("GET", "variants/security-probe", 200), ("PUT", "staging/security-probe", 403), ("GET", "private/security-probe", 403), ("DELETE", "variants/security-probe", 403), ]: status = signed_s3(data, method, key, b"probe" if method == "PUT" else b"") record("worker-s3:" + method + ":" + key, status == expected, status=status) parsed = urlsplit(data["BROKER_URL"]) url = urlunsplit( ("amqps", parsed.netloc.replace("127.0.0.1:5672", "broker:5671"), parsed.path, "", "") ) conn = await aio_pika.connect(url, ssl_context=context("worker"), timeout=5) try: channel = await conn.channel() queue = await channel.get_queue("attachments.process") message = await queue.get(fail=False) if message: await message.nack(requeue=True) record("worker-queue:consume", True) channel = await conn.channel() try: exchange = await channel.get_exchange("attachments", ensure=False) await exchange.publish(aio_pika.Message(b"forbidden"), routing_key="process") denied = False except aio_pika.exceptions.AMQPException: denied = True record("worker-queue:publish-denied", denied) channel = await conn.channel() try: await channel.declare_queue("security-escape") denied = False except aio_pika.exceptions.AMQPException: denied = True record("worker-queue:configure-denied", denied) finally: await conn.close() async def dependency_acl(): _, data = credentials("api") cache = redis.from_url( data["CACHE_URL"].replace("redis://", "rediss://").replace("127.0.0.1:6379", "valkey:6380"), ssl_ca_certs=str(PKI / "api/ca.crt"), ssl_certfile=str(PKI / "api/tls.crt"), ssl_keyfile=str(PKI / "api/tls.key"), socket_timeout=3, ) await cache.set("session:security-probe", "probe", ex=30) record("valkey:own-prefix", await cache.get("session:security-probe") == b"probe") for name, action in [ ("foreign-prefix", lambda: cache.get("foreign:security-probe")), ("admin", lambda: cache.config_get("*")), ]: try: await action() denied = False except redis.ResponseError: denied = True record("valkey:" + name + "-denied", denied) await cache.delete("session:security-probe") await cache.aclose() dsn = data["DATABASE_URL"].replace("postgresql://", "postgresql+asyncpg://") for label, ssl_mode in [("plaintext", False), ("untrusted-ca", ssl.create_default_context())]: engine = create_async_engine(dsn, connect_args={"ssl": ssl_mode}, hide_parameters=True) try: async with engine.connect(): pass denied = False except Exception: denied = True finally: await engine.dispose() record("postgres:" + label + "-denied", denied) def telemetry(): for host, path in [ ("prometheus", "/api/v1/query?query=up"), ("loki", "/loki/api/v1/labels"), ("tempo", "/api/search?limit=10"), ]: status, body = http("grafana", "https://" + host + ":9443" + path) record("telemetry-query:" + host, status == 200, bytes=len(body)) for host, path in [("prometheus", "/api/v1/query?query=up"), ("loki", "/loki/api/v1/labels")]: status, _ = http("otel-collector", "https://" + host + ":9443" + path) record("collector-read-denied:" + host, status == 403) for host, path in [("prometheus", "/api/v1/otlp/v1/metrics"), ("loki", "/otlp/v1/logs")]: status, _ = http("grafana", "https://" + host + ":9443" + path, b"", method="POST") record("grafana-write-denied:" + host, status == 403) for address in sorted( {item[4][0] for item in socket.getaddrinfo("api", 8443, type=socket.SOCK_STREAM)} ): payload = b"GET /health/ready HTTP/1.1\r\nHost: api\r\nConnection: close\r\n\r\n" for role, expected in [ ("lb", True), (None, False), ("outsider", False), ("foreign", False), ]: record( "api-replica:" + address + ":" + str(role), tls(address, 8443, role, payload, hostname="api") == expected, ) def telemetry_dump(): queries = { "metrics": "https://prometheus:9443/api/v1/query?" + urlencode({"query": '{__name__=~"http_server.*|messenger.*"}'}), "logs": "https://loki:9443/loki/api/v1/query_range?" + urlencode({"query": '{service_name=~"messenger-.+"}', "limit": 1000}), "traces": "https://tempo:9443/api/search?limit=20", } data = {name: json.loads(http("operator", url)[1]) for name, url in queries.items()} data["trace_details"] = [ json.loads(http("operator", "https://tempo:9443/api/traces/" + trace["traceID"])[1]) for trace in data["traces"].get("traces", []) ] print(json.dumps(data)) async def revoked(): data = json.loads((PKI / "previous-worker/credentials.json").read_text()) for host, port, payload in [ ("broker", 5671, b"AMQP\x00\x00\x09\x01"), ("s3", 9443, b"GET / HTTP/1.0\r\n\r\n"), ("otel-collector", 44318, b"GET / HTTP/1.0\r\n\r\n"), ]: record("revoked-cert:" + host, not tls(host, port, "previous-worker", payload)) status, _ = http( "previous-worker", "https://openbao:8200/v1/auth/cert/login", {"name": "worker"} ) record("revoked-bao-cert", status in {400, 403}, status=status) engine = create_async_engine( data["DATABASE_URL"].replace("postgresql://", "postgresql+asyncpg://"), connect_args={"ssl": context()}, hide_parameters=True, ) try: async with engine.connect(): pass denied = False except Exception: denied = True finally: await engine.dispose() record("rotated-db-password", denied) if __name__ == "__main__": if "telemetry-dump" in sys.argv: telemetry_dump() raise SystemExit(0) if "revoked" in sys.argv: asyncio.run(revoked()) print( json.dumps( {"checks": RESULTS, "passed": all(item["passed"] for item in RESULTS)}, indent=2 ) ) raise SystemExit(0 if all(item["passed"] for item in RESULTS) else 1) transport_matrix() telemetry() asyncio.run(worker_scope()) asyncio.run(dependency_acl()) print( json.dumps({"checks": RESULTS, "passed": all(item["passed"] for item in RESULTS)}, indent=2) ) raise SystemExit(0 if all(item["passed"] for item in RESULTS) else 1)