Files
mtusi2026lab7/scripts/security_probe.py

483 lines
17 KiB
Python

#!/usr/bin/env python3
"""Reproducible negative checks and worker revocation on the local lab7 stand."""
import argparse
import base64
import hashlib
import json
import secrets
import shutil
import socket
import ssl
import subprocess
import sys
import tempfile
import time
import uuid
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
ROOT = Path(__file__).resolve().parents[1]
LOCAL = ROOT / "secrets"
CA = LOCAL / "tls/trust/ca.crt"
INSIDE_CA = "/run/tls/trust/ca.crt"
def checked(*command, **kwargs):
try:
return subprocess.run(command, cwd=ROOT, check=True, **kwargs)
except subprocess.CalledProcessError as exc:
if exc.stderr:
sys.stderr.write(exc.stderr)
raise
def compose(*args, **kwargs):
return checked("docker", "compose", *args, **kwargs)
def context(ca, certificate=None):
ctx = ssl.create_default_context(cafile=ca)
if certificate:
ctx.load_cert_chain(certificate / "service.crt", certificate / "service.key")
return ctx
def response(url, ctx, data=None, headers=None, method=None):
request = Request(url, data, headers or {}, method=method)
try:
with urlopen(request, context=ctx, timeout=4) as r:
return r.status, r.read(262144)
except HTTPError as e:
return e.code, b""
def denied(action):
try:
status = action()
assert status in (401, 403) or status == b"", "Unexpected success in negative probe"
except (ssl.SSLError, ConnectionResetError, BrokenPipeError, URLError) as exc:
# Preserve DNS/connectivity errors as real test failures.
cause = exc.reason if isinstance(exc, URLError) else exc
if not isinstance(cause, (ssl.SSLError, ConnectionResetError, BrokenPipeError)):
raise
def redis_ping(ctx):
with socket.create_connection(("broker", 6380), timeout=3) as raw:
with ctx.wrap_socket(raw, server_hostname="broker") as tls:
tls.sendall(b"*1\r\n$4\r\nPING\r\n")
return tls.recv(256)
def audit_tls():
results = {}
endpoints = {
"api": (lambda ctx: response("https://api:8080/health/ready", ctx)[0], "lb-client"),
"collector": (
lambda ctx: response(
"https://otel-collector:4319/v1/traces",
ctx,
b"",
{"Content-Type": "application/x-protobuf"},
)[0],
"worker-client",
),
"broker": (redis_ping, "worker-client"),
}
for name, (probe, identity) in endpoints.items():
status = probe(context(INSIDE_CA, Path("/probe") / identity))
assert status == 200 or status.startswith(b"-NOAUTH")
for cert in (None, Path("/probe/probe-client"), Path("/probe/rogue")):
denied(lambda probe=probe, cert=cert: probe(context(INSIDE_CA, cert)))
results[name] = [
"allowed",
"no_client_denied",
"same_ca_wrong_role_denied",
"rogue_ca_denied",
]
for host, port in (
("broker", 6379),
("otel-collector", 4318),
("prometheus", 19090),
("loki", 13100),
("tempo", 13200),
("tempo", 14318),
("tempo", 9095),
("loki", 9095),
):
try:
with socket.create_connection((host, port), timeout=1):
raise AssertionError("Reachable plaintext " + host)
except (ConnectionRefusedError, TimeoutError):
pass
results["network_plaintext_bypass"] = "denied"
results["grafana_backends"] = {}
for host, port, path in (
("prometheus", 9090, "/api/v1/status/buildinfo"),
("loki", 3100, "/loki/api/v1/status/buildinfo"),
("tempo", 3200, "/api/echo"),
):
endpoint = f"https://{host}:{port}{path}"
assert response(endpoint, context(INSIDE_CA, Path("/probe/grafana-client")))[0] == 200
denied(
lambda endpoint=endpoint: response(
endpoint, context(INSIDE_CA, Path("/probe/probe-client"))
)[0]
)
results["grafana_backends"][host] = "verified_tls_identity_allow_and_deny"
return results
def worker_scope():
import psycopg
import redis
from botocore.exceptions import ClientError
from messenger.settings import Settings
from messenger.storage import ObjectStore
settings = Settings.from_env()
with psycopg.connect(settings.database_url, autocommit=True) as conn:
assert conn.execute("SELECT ssl FROM pg_stat_ssl WHERE pid=pg_backend_pid()").fetchone()[0]
conn.execute("SELECT count(*) FROM attachment_jobs").fetchone()
for query in (
"SELECT count(*) FROM users",
"SELECT count(*) FROM messages",
"SELECT count(*) FROM sessions",
"CREATE TEMP TABLE forbidden_probe(id int)",
"UPDATE attachments SET owner_id=owner_id WHERE false",
"SET ROLE messenger",
):
try:
conn.execute(query)
except psycopg.errors.InsufficientPrivilege:
continue
raise AssertionError("Unexpected database grant")
broker = redis.Redis.from_url(settings.broker_url)
assert broker.ping()
for op in (
lambda: broker.config_get("appendonly"),
lambda: broker.xrange("unrelated-key"),
lambda: broker.get("session-user:synthetic"),
):
try:
op()
except redis.exceptions.NoPermissionError:
continue
raise AssertionError("Unexpected broker grant")
store = ObjectStore(settings)
suffix = uuid.uuid4().hex
key = "objects/audit/" + suffix
store.client.put_object(Bucket=store.bucket, Key=key, Body=b"synthetic")
assert store.client.head_object(Bucket=store.bucket, Key=key)["ContentLength"] == 9
for op in (
lambda: store.client.put_object(
Bucket=store.bucket, Key="staging/audit/" + suffix, Body=b"x"
),
lambda: store.client.delete_object(Bucket=store.bucket, Key=key),
lambda: store.client.get_bucket_policy(Bucket=store.bucket),
):
try:
op()
except ClientError as exc:
assert exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403
continue
raise AssertionError("Unexpected S3 grant")
cert = Path("/run/tls/client")
ctx = context(INSIDE_CA, cert)
base = "https://openbao:8200/v1/"
ct = {"Content-Type": "application/json"}
code, raw = response(base + "auth/cert/login", ctx, b'{"name":"worker"}', ct)
assert code == 200
auth = json.loads(raw)["auth"]
token = auth["client_token"]
headers = {**ct, "X-Vault-Token": token}
assert 0 < auth["lease_duration"] <= 120
assert response(base + "services/data/worker", ctx, headers=headers)[0] == 200
assert response(base + "services/data/api", ctx, headers=headers)[0] == 403
assert response(base + "services/data/worker", ctx, b'{"data":{}}', headers)[0] == 403
assert response(base + "auth/token/revoke-self", ctx, b"{}", headers)[0] == 204
assert response(base + "services/data/worker", ctx, headers=headers)[0] == 403
return {
"postgres": {"tls": True, "job_read": True, "denied_operations": 6},
"broker": {"tls": True, "ping": True, "denied_operations": 3},
"s3": {"put_objects": True, "denied_operations": 3},
"openbao": {
"own_read": True,
"foreign_read_and_write_denied": True,
"token_ttl_s": auth["lease_duration"],
"revoked_token_denied": True,
},
}
def old_identity():
ctx = context(INSIDE_CA, Path("/probe/old"))
denied(lambda: redis_ping(ctx))
denied(
lambda: response(
"https://otel-collector:4319/v1/traces",
ctx,
b"",
{"Content-Type": "application/x-protobuf"},
)[0]
)
code, _ = response(
"https://openbao:8200/v1/auth/cert/login",
ctx,
b'{"name":"worker"}',
{"Content-Type": "application/json"},
)
assert code in (400, 403)
return {"broker": "revoked", "collector": "revoked", "openbao": "revoked"}
def worker_probe():
raw = compose(
"run",
"--rm",
"--no-deps",
"-T",
"-v",
f"{Path(__file__).resolve()}:/test/audit.py:ro",
"--entrypoint",
"python",
"worker",
"/test/audit.py",
"--inside",
"worker",
capture_output=True,
text=True,
).stdout
return json.loads(raw)
def isolated(mode, mounts):
cmd = [
"docker",
"run",
"--rm",
"--network",
"mtusi-lab7_backend",
"--read-only",
"--cap-drop",
"ALL",
"--security-opt",
"no-new-privileges:true",
"--memory",
"256m",
"-v",
f"{Path(__file__).resolve()}:/test/audit.py:ro",
"-v",
f"{LOCAL / 'tls/trust'}:/run/tls/trust:ro",
]
for name, source in mounts.items():
cmd += ["-v", f"{source}:/probe/{name}:ro"]
image = compose("images", "-q", "api", capture_output=True, text=True).stdout.split()[0]
raw = checked(
*cmd, image, "python", "/test/audit.py", "--inside", mode, capture_output=True, text=True
).stdout
return json.loads(raw)
def verify_hostname():
with socket.create_connection(("127.0.0.1", 8443), timeout=3) as raw:
try:
with context(CA).wrap_socket(raw, server_hostname="wrong-name.invalid"):
raise AssertionError("Wrong TLS hostname accepted")
except ssl.SSLCertVerificationError as exc:
assert exc.verify_code == 62, "Expected a hostname mismatch"
return "wrong_hostname_rejected"
def check_api(client):
alice, bob, eve, chat = client.bootstrap()
attachment, _ = client.attachment(alice, chat)
client.wait_ready(bob, attachment)
path = "/api/v1/attachments/" + attachment["id"]
client.call("GET", path, 404, actor=eve, headers={"X-User-Id": alice["id"]})
client.call("GET", path + "/content", 404, actor=eve)
client.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=bob)
client.call("GET", "/api/v1/users/me", 401, actor=bob)
reg = alice["registration"]
bad = base64.b64encode((reg["username"] + ":invalid-password-123").encode()).decode()
statuses, replicas, times = [], set(), []
for _ in range(6):
r = client.call(
"POST", "/api/v1/auth/sessions", [401, 429], headers={"Authorization": "Basic " + bad}
)
statuses.append(r["status"])
times.append(r["elapsed"] * 1000)
replicas.add(r["headers"]["X-Instance-Id"])
if r["status"] == 429:
assert int(r["headers"]["Retry-After"]) > 0
assert 429 in statuses and 401 in statuses and len(replicas) >= 2
too_big = client.call(
"POST", "/api/v1/users", 413, raw=b"x" * 66000, headers={"Content-Type": "application/json"}
)
assert too_big["status"] == 413
return {
"statuses": statuses,
"replicas": len(replicas),
"latency_ms": times,
"outsider_denied": True,
"logout_revoked": True,
"oversize_json_denied": True,
}
def rotate(client, temp):
alice, _, _, chat = client.bootstrap()
old = temp / "old"
shutil.copytree(LOCAL / "tls/worker-client", old)
compose("stop", "worker")
accepted = []
start = time.monotonic()
try:
for _ in range(5):
att, data = client.attachment(alice, chat)
accepted.append((att, data))
checked(sys.executable, "scripts/bao_operator.py", "--revoke", "worker")
checked(sys.executable, "scripts/pki.py", "--rotate", "worker-client")
password = LOCAL / "broker_worker_password"
password.write_text(secrets.token_hex(24) + "\n")
checked(sys.executable, "scripts/bootstrap.py")
# Load the changed ACL without restarting durable Valkey or erasing its stream.
compose(
"exec",
"-T",
"broker",
"sh",
"-ec",
"""p="$(awk '/^user operator / {print substr($NF,2)}' /run/secrets/broker.acl)"
export REDISCLI_AUTH="$p"
valkey-cli --user operator ACL LOAD | grep -q '^OK$'
valkey-cli --user operator CLIENT KILL USER worker >/dev/null""",
)
compose("restart", "broker-tls", "otel-collector-tls")
result = isolated("old", {"old": old})
finally:
checked(sys.executable, "scripts/bao_operator.py")
compose("up", "-d", "--no-deps", "--force-recreate", "worker")
for attachment, original in accepted:
item = client.wait_ready(alice, attachment)
file = client.call("GET", f"/api/v1/attachments/{item['id']}/content", actor=alice)
assert hashlib.sha256(file["bytes"]).digest() == hashlib.sha256(original).digest()
return {
"old_identity": result,
"accepted": len(accepted),
"ready_with_original": len(accepted),
"elapsed_seconds": round(time.monotonic() - start, 3),
"broker_acl_reloaded": True,
"images_rebuilt": False,
"new_worker": worker_probe(),
}
def check_sealed(client):
client.call("GET", "/health/ready", schema="Health")
start = time.monotonic()
checked(sys.executable, "scripts/bao_operator.py", "--seal")
try:
deadline = start + 42
status = 200
while time.monotonic() < deadline:
status, _ = response("https://localhost:8443/health/ready", context(CA))
if status == 503:
break
time.sleep(1)
assert status == 503, "API did not stop serving after the credential grace period"
unavailable = time.monotonic() - start
finally:
checked(sys.executable, "scripts/bao_operator.py")
deadline = time.monotonic() + 22
while time.monotonic() < deadline:
status, _ = response("https://localhost:8443/health/ready", context(CA))
if status == 200:
return {
"unready_after_seconds": round(unavailable, 3),
"recovered_after_seconds": round(time.monotonic() - start, 3),
}
time.sleep(1)
raise AssertionError("API readiness did not recover after unseal")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--inside", choices=("tls", "worker", "old"))
parser.add_argument("--output", type=Path)
parser.add_argument("--rotate", action="store_true")
parser.add_argument("--seal", action="store_true")
args = parser.parse_args()
if args.inside:
print(
json.dumps(
{"tls": audit_tls, "worker": worker_scope, "old": old_identity}[args.inside]()
)
)
return
if not args.output or args.output.exists():
parser.error("choose a new --output")
sys.path.insert(0, str(ROOT / "tests"))
from client import Client
client = Client("https://localhost:8443", str(CA))
with tempfile.TemporaryDirectory(dir=LOCAL) as folder:
tmp = Path(folder)
tmp.chmod(0o755)
rogue = tmp / "rogue"
rogue.mkdir(mode=0o755)
checked(
"openssl",
"req",
"-x509",
"-newkey",
"ec",
"-pkeyopt",
"ec_paramgen_curve:P-256",
"-nodes",
"-days",
"1",
"-subj",
"/CN=worker-client",
"-addext",
"extendedKeyUsage=clientAuth",
"-keyout",
str(rogue / "service.key"),
"-out",
str(rogue / "service.crt"),
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
(rogue / "service.key").chmod(0o644)
identities = {
name: LOCAL / "tls" / name
for name in ("lb-client", "worker-client", "probe-client", "grafana-client")
}
result = {
"tls": isolated("tls", {**identities, "rogue": rogue}),
"worker_scope": worker_probe(),
"api": check_api(client),
"external_hostname": verify_hostname(),
}
print("PASS: verified TLS and distinct service privileges", flush=True)
if args.rotate:
result["worker_compromise"] = rotate(client, tmp)
print("PASS: old identity denied; all accepted uploads recovered", flush=True)
if args.seal:
result["sealed"] = check_sealed(client)
print(
"PASS: sealed OpenBao makes API unready within a bounded grace period", flush=True
)
args.output.write_text(json.dumps(result, indent=2) + "\n")
if __name__ == "__main__":
main()