#!/usr/bin/env python3 """Real TLS/identity, ACL, secret lifecycle and application regression checks.""" import argparse import asyncio import base64 import io import json import os import socket import ssl import subprocess import sys import time import urllib.error import urllib.request from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) ROWS = [] def ok(name, **data): ROWS.append({"check": name, "result": "PASS", **data}) def context(role=None): ctx = ssl.create_default_context(cafile="/run/trust/ca.crt") ctx.minimum_version = ssl.TLSVersion.TLSv1_3 if role: directory = Path("/run/test-identities") / role ctx.load_cert_chain(directory / "service.crt", directory / "service.key") return ctx def bao(path, role="operator", body=None, token=None, method=None): headers = {"Content-Type": "application/json"} if token: headers["X-Vault-Token"] = token req = urllib.request.Request( "https://openbao:8200/v1/" + path, headers=headers, data=json.dumps(body).encode() if body is not None else None, method=method, ) with urllib.request.urlopen(req, context=context(role), timeout=4) as response: raw = response.read() return json.loads(raw) if raw else {} def forbidden(name, call, exception, *, code=None): try: call() except exception as error: if code is not None: assert error.code == code, name ok(name) else: raise AssertionError("Forbidden operation succeeded: " + name) def wire(host, port, role, data): with socket.create_connection((host, port), timeout=3) as raw: with context(role).wrap_socket(raw, server_hostname=host) as secured: secured.sendall(data) return secured.recv(4096) def denied_wire(host, port, role, data): try: result = wire(host, port, role, data) assert not result or result.startswith(b"HTTP/1.1 403"), (host, port, role) except ssl.SSLError: pass except ConnectionResetError: pass # Timeouts, DNS errors and connection refused deliberately do not count. ok(f"mtls-deny:{host}:{port}:{role or 'no-cert'}") async def internal(): import asyncpg import redis import urllib3 from minio import Minio from minio.error import S3Error token = json.loads(Path("/run/operator/bao-init.json").read_text())["root_token"] secrets = {} for role in ["api", "worker", "dispatcher", "reconciler"]: auth = bao("auth/cert/login", role, {"name": role})["auth"] assert role in auth["policies"] and "root" not in auth["policies"] value = bao("secret/data/messenger/" + role, role, token=auth["client_token"]) secrets[role] = value["data"]["data"] ok("bao-own-secret:" + role) other = "worker" if role == "api" else "api" forbidden( "bao-other-secret:" + role, lambda: bao("secret/data/messenger/" + other, role, token=auth["client_token"]), urllib.error.HTTPError, code=403, ) forbidden( "bao-policy-write:" + role, lambda: bao( "sys/policies/acl/forbidden", role, {"policy": 'path "*" { capabilities=["sudo"] }'}, token=auth["client_token"], ), urllib.error.HTTPError, code=403, ) renewal = bao("auth/token/renew-self", role, {}, token=auth["client_token"]) assert renewal["auth"]["lease_duration"] > 0 ok("bao-renew:" + role) # Full chain/identity rejection at each mandatory grade-4 mTLS boundary. for host, port, right, payload in [ ( "api", 8443, "lb", b"GET /health/ready HTTP/1.1\r\nHost: api\r\nConnection: close\r\n\r\n", ), ("broker", 6379, "worker", b"*1\r\n$4\r\nPING\r\n"), ( "otel-collector", 4318, "api", b"POST /v1/traces HTTP/1.1\r\nHost: otel-collector\r\nContent-Length: 0\r\nContent-Type: application/x-protobuf\r\nConnection: close\r\n\r\n", ), ]: result = wire(host, port, right, payload) assert result and (b"200" in result[:40] if host != "broker" else b"NOAUTH" in result) ok(f"mtls-allow:{host}:{port}:{right}") for wrong in [None, "intruder", "foreign", "expired"] + ( ["worker"] if host == "api" else [] ): denied_wire(host, port, wrong, payload) # Loopback-only plaintext listeners cannot be reached from another container. for host, port in [ ("api", 8080), ("postgres", 15432), ("redis", 16379), ("broker", 16379), ("otel-collector", 14318), ("otel-collector", 18889), ("loki", 13100), ("tempo", 13200), ("tempo", 14318), ("prometheus", 19090), ]: try: connection = socket.create_connection((host, port), timeout=2) except ConnectionRefusedError: ok(f"no-plaintext-bypass:{host}:{port}") else: connection.close() raise AssertionError("Plaintext listener is reachable: " + host) # Each remaining matrix row has a real positive TLS probe. for host, port, role, path in [ ("redis", 6379, "api", None), ("postgres", 5432, "operator", None), ("loki", 3100, "grafana", "/ready"), ("tempo", 3200, "grafana", "/ready"), ("prometheus", 9090, "grafana", "/-/ready"), ("otel-collector", 8889, "prometheus", "/metrics"), ("otel-collector", 8888, "prometheus", "/metrics"), ("otel-collector", 13133, "operator", "/"), ("tempo", 4318, "otel-collector", "/v1/traces"), ]: if host == "postgres": connection = await asyncpg.connect( os.environ["DATABASE_URL"], ssl=context("operator"), direct_tls=True ) assert await connection.fetchval("SELECT 1") == 1 await connection.close() elif host == "redis": result = wire(host, port, role, b"*1\r\n$4\r\nPING\r\n") assert b"NOAUTH" in result else: data = f"GET {path} HTTP/1.1\r\nHost: {host}\r\nConnection: close\r\n\r\n".encode() if host == "tempo" and port == 4318: data = b"POST /v1/traces HTTP/1.1\r\nHost: tempo\r\nContent-Length: 0\r\nContent-Type: application/x-protobuf\r\nConnection: close\r\n\r\n" result = wire(host, port, role, data) assert b"200" in result[:40] or b"503" in result[:40], host ok(f"tls-matrix:{host}:{port}:{role}") for role in ["worker", "dispatcher"]: connection = await asyncpg.connect( secrets[role]["DATABASE_URL"], ssl=context(role), direct_tls=True ) try: await connection.fetch("SELECT id FROM attachment_jobs LIMIT 1") for name, sql in [ ("users", "SELECT password_hash FROM users LIMIT 1"), ("ddl", "CREATE TABLE forbidden(id int)"), ("roles", "CREATE ROLE forbidden"), ]: try: await connection.execute(sql) except asyncpg.InsufficientPrivilegeError: ok(f"db-deny:{role}:{name}") else: raise AssertionError("Excessive DB privilege") finally: await connection.close() def redis_client(role, kind): url = secrets[role]["BROKER_URL" if kind == "broker" else "REDIS_URL"] folder = Path("/run/test-identities") / role return redis.Redis.from_url( url, ssl_ca_certs="/run/trust/ca.crt", ssl_certfile=str(folder / "service.crt"), ssl_keyfile=str(folder / "service.key"), ssl_check_hostname=True, socket_timeout=3, ) for role, kind in [("api", "sessions"), ("worker", "broker"), ("dispatcher", "broker")]: client = redis_client(role, kind) assert client.ping() forbidden( f"redis-admin-deny:{role}", lambda: client.config_get("*"), redis.exceptions.NoPermissionError, ) forbidden( f"redis-prefix-deny:{role}", lambda: client.get("operator:secret"), redis.exceptions.NoPermissionError, ) if role == "worker": forbidden( "consumer-cannot-publish", lambda: client.xadd("attachment-jobs-v1", {"job": "{}"}), redis.exceptions.NoPermissionError, ) if role == "dispatcher": forbidden( "producer-cannot-consume", lambda: client.xreadgroup( "image-workers-v1", "probe", {"attachment-jobs-v1": ">"}, count=1 ), redis.exceptions.NoPermissionError, ) client.close() for role, prefix in [("api", "S3"), ("worker", "S3_WORKER")]: data = secrets[role] client = Minio( "minio:9000", access_key=data[prefix + "_ACCESS_KEY_ID"], secret_key=data[prefix + "_SECRET_ACCESS_KEY"], secure=True, http_client=urllib3.PoolManager( ca_certs="/run/trust/ca.crt", cert_reqs="CERT_REQUIRED" ), ) assert client.bucket_exists("messenger-attachments") try: client.put_object("messenger-attachments", "admin/forbidden", io.BytesIO(b"x"), 1) except S3Error as error: assert error.code == "AccessDenied" ok("s3-prefix-deny:" + role) else: raise AssertionError("Forbidden S3 prefix allowed") if role == "api": try: client.put_object( "messenger-attachments", "variants/forbidden", io.BytesIO(b"x"), 1 ) except S3Error as error: assert error.code == "AccessDenied" ok("api-cannot-write-variants") else: raise AssertionError("API can overwrite worker variants") for host, port, path in [ ("loki", 3100, "/otlp/v1/logs"), ("tempo", 3200, "/api/traces/anything"), ("prometheus", 9090, "/api/v1/admin/tsdb/delete_series"), ]: data = f"POST {path} HTTP/1.1\r\nHost: {host}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".encode() assert b"403" in wire(host, port, "grafana", data)[:40] ok("grafana-write-deny:" + host) short = bao( "auth/token/create", body={"policies": ["worker"], "ttl": "1s", "explicit_max_ttl": "1s"}, token=token, )["auth"]["client_token"] time.sleep(1.5) forbidden( "bao-expired-token", lambda: bao("secret/data/messenger/worker", "worker", token=short), urllib.error.HTTPError, code=403, ) short = bao("auth/cert/login", "worker", {"name": "worker"})["auth"]["client_token"] bao("auth/token/revoke", body={"token": short}, token=token) forbidden( "bao-revoked-token", lambda: bao("secret/data/messenger/worker", "worker", token=short), urllib.error.HTTPError, code=403, ) print(json.dumps(ROWS)) def host_checks(): from client import Client sys.path.insert(0, str(ROOT / "scripts")) from securitylib import LOCAL, compose probe = [ "run", "--rm", "--no-deps", "-v", str(ROOT / "tests") + ":/app/tests:ro", "-v", str(LOCAL / "identities") + ":/run/test-identities:ro", "-v", str(LOCAL / "operator") + ":/run/operator:ro", "migration", "python", "tests/lab7_security.py", "--internal", ] result = compose(*probe, capture=True) ROWS.extend(json.loads(result.splitlines()[-1])) (ROOT / "evidence/security-internal-summary.json").write_text( json.dumps({"checks": ROWS, "passed": len(ROWS)}, indent=2) + "\n" ) c = Client("https://localhost:8443", str(LOCAL / "trust/ca.crt")) with socket.create_connection(("localhost", 8443), timeout=3) as connection: try: tls_context = ssl.create_default_context(cafile=str(LOCAL / "trust/ca.crt")) tls_context.wrap_socket(connection, server_hostname="wrong.invalid") except ssl.SSLCertVerificationError: ok("external-wrong-hostname-denied") else: raise AssertionError("Wrong hostname accepted") actor = c.user("security") # Body forged user identity never replaces verified session identity. response = c.call( "GET", "/api/v1/users/me", actor=actor, headers={"X-User-Id": "00000000-0000-0000-0000-000000000000"}, schema="User", ) assert response["json"]["id"] == actor["id"] ok("bearer-not-x-user-id") username = actor["registration"]["username"] password = actor["registration"]["password"] dbcode = """import asyncio,os,json,sys import asyncpg from argon2 import PasswordHasher from src.secure.transport import context async def main(): c=await asyncpg.connect(os.environ['DATABASE_URL'],ssl=context(),direct_tls=True) data=json.load(sys.stdin) if data['action']=='old': value=PasswordHasher(memory_cost=19456,time_cost=2,parallelism=1).hash(data['password']) await c.execute('UPDATE users SET password_hash=$1 WHERE username=$2',value,data['username']) else: value=await c.fetchval('SELECT password_hash FROM users WHERE username=$1',data['username']) assert not PasswordHasher(memory_cost=65536,time_cost=3,parallelism=4).check_needs_rehash(value) await c.close() asyncio.run(main())""" def db(action): subprocess.run( [ "docker", "compose", "run", "--rm", "--no-deps", "-T", "migration", "python", "-c", dbcode, ], cwd=ROOT, input=json.dumps({"action": action, "username": username, "password": password}), text=True, check=True, stdout=subprocess.DEVNULL, ) db("old") credentials = base64.b64encode((username + ":" + password).encode()).decode() c.call( "POST", "/api/v1/auth/sessions", 201, headers={"Authorization": "Basic " + credentials}, schema="Session", ) db("check") ok("successful-login-rehash") # Account limit must hold while spoofed forwarding headers change. observed = set() blocked = False bad = base64.b64encode((username + ":WrongPassword123!").encode()).decode() for index in range(14): response = c.call( "POST", "/api/v1/auth/sessions", [401, 429], headers={ "Authorization": "Basic " + bad, "X-Real-IP": f"198.51.100.{index}", "X-Forwarded-For": f"198.51.100.{index}", }, ) observed.add(response["headers"]["X-Instance-Id"]) if response["status"] == 429: assert int(response["headers"]["Retry-After"]) > 0 blocked = True assert blocked and len(observed) >= 2 ok("shared-account-rate-limit", replicas=len(observed)) seed_ip = """import os,redis r=redis.Redis.from_url(os.environ['REDIS_URL'],ssl_ca_certs='/run/trust/ca.crt',ssl_certfile='/run/identity/service.crt',ssl_keyfile='/run/identity/service.key',ssl_check_hostname=True) keys=list(r.scan_iter('rl:login:ip:*')) assert keys for key in keys:r.set(key,120,px=3000) """ subprocess.run( [ "docker", "compose", "run", "--rm", "--no-deps", "-T", "migration", "python", "-c", seed_ip, ], cwd=ROOT, check=True, stdout=subprocess.DEVNULL, ) login = "ipprobe_" + os.urandom(4).hex() credential = base64.b64encode((login + ":WrongPassword123!").encode()).decode() response = c.call( "POST", "/api/v1/auth/sessions", 429, headers={ "Authorization": "Basic " + credential, "X-Real-IP": "203.0.113.55", "X-Forwarded-For": "203.0.113.55", }, ) assert int(response["headers"]["Retry-After"]) > 0 ok("shared-ip-rate-limit-spoofed-headers-denied") time.sleep(3.1) # Let the synthetic IP saturation expire before later tests. # Expire a real user session in Redis; the same bearer must stop working. expire = """import hashlib,os,sys,json,redis from src.secure.transport import context x=json.load(sys.stdin) r=redis.Redis.from_url(x['url'],ssl_ca_certs='/run/trust/ca.crt',ssl_certfile='/run/identity/service.crt',ssl_keyfile='/run/identity/service.key',ssl_check_hostname=True) r.pexpire('session:v1:'+hashlib.sha256(x['token'].encode()).hexdigest(),1)""" from securitylib import request, root_token api = request("secret/data/messenger/api", token=root_token())["data"]["data"] # The operator TLS role is allowed; session Redis auth uses the scoped API account. subprocess.run( [ "docker", "compose", "run", "--rm", "--no-deps", "-T", "migration", "python", "-c", expire, ], cwd=ROOT, input=json.dumps({"url": api["REDIS_URL"], "token": actor["token"]}), text=True, check=True, stdout=subprocess.DEVNULL, ) time.sleep(0.05) c.call("GET", "/api/v1/users/me", 401, actor=actor) ok("session-expiry") (ROOT / "evidence/security-summary.json").write_text( json.dumps({"checks": ROWS, "passed": len(ROWS)}, indent=2) + "\n" ) print("PASS:", len(ROWS), "security checks; evidence/security-summary.json") if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--internal", action="store_true") args = parser.parse_args() if args.internal: asyncio.run(internal()) else: host_checks()