Files
mtusi2026lab7/scripts/security_probe.py

349 lines
13 KiB
Python

import asyncio
import json
import socket
import ssl
import sys
import urllib.error
import urllib.request
from pathlib import Path
from urllib.parse import urlencode, urlsplit, urlunsplit
import aio_pika
import redis.asyncio as redis
from botocore.auth import S3SigV4Auth
from botocore.awsrequest import AWSRequest
from botocore.credentials import Credentials
from sqlalchemy import text
from sqlalchemy.exc import DBAPIError
from sqlalchemy.ext.asyncio import create_async_engine
PKI = Path("/audit-pki")
RESULTS = []
def record(name, passed, **details):
RESULTS.append({"test": name, "passed": bool(passed), **details})
def context(role=None, trusted=True):
ctx = ssl.create_default_context(cafile=str(PKI / "operator/ca.crt") if trusted else None)
if role:
ctx.load_cert_chain(PKI / role / "tls.crt", PKI / role / "tls.key")
return ctx
def tls(host, port, role, payload, *, hostname=None, trusted=True):
try:
with (
socket.create_connection((host, port), timeout=3) as raw,
context(role, trusted).wrap_socket(raw, server_hostname=hostname or host) as stream,
):
stream.sendall(payload)
return bool(stream.recv(2048))
except (OSError, ssl.SSLError):
return False
def http(role, url, payload=None, token=None, *, method=None, headers=None):
head = {"Content-Type": "application/json", **(headers or {})}
if token:
head["X-Vault-Token"] = token
data = json.dumps(payload).encode() if isinstance(payload, dict) else payload
req = urllib.request.Request(url, data=data, headers=head, method=method)
try:
with urllib.request.urlopen(req, context=context(role), timeout=5) as response:
return response.status, response.read()
except urllib.error.HTTPError as error:
return error.code, error.read()
def credentials(role):
status, body = http(role, "https://openbao:8200/v1/auth/cert/login", {"name": role})
if status != 200:
raise RuntimeError("Service authentication failed")
token = json.loads(body)["auth"]["client_token"]
status, body = http(role, "https://openbao:8200/v1/secret/data/" + role, token=token)
if status != 200:
raise RuntimeError("Scoped configuration unavailable")
return token, json.loads(body)["data"]["data"]
def transport_matrix():
targets = [
("api", 8443, "lb"),
("s3", 9443, "api"),
("broker", 5671, "worker"),
("otel-collector", 44318, "api"),
("prometheus", 9443, "grafana"),
("loki", 9443, "grafana"),
("tempo", 9443, "grafana"),
("tempo", 44318, "otel-collector"),
("grafana", 9443, "operator"),
("valkey", 6380, "api"),
]
for host, port, role in targets:
payload = (
b"AMQP\x00\x00\x09\x01"
if host == "broker"
else b"*1\r\n$4\r\nPING\r\n"
if host == "valkey"
else b"GET / HTTP/1.1\r\nHost: " + host.encode() + b"\r\nConnection: close\r\n\r\n"
)
for identity, expected in [
(role, True),
(None, False),
("outsider", False),
("foreign", False),
]:
observed = tls(host, port, identity, payload)
record(f"mtls:{host}:{port}:{identity}", observed == expected)
record(
f"hostname:{host}:{port}",
not tls(host, port, role, payload, hostname="incorrect.invalid"),
)
for host, port in [
("api", 8080),
("s3", 9000),
("broker", 5672),
("broker", 15672),
("broker", 15692),
("broker", 25672),
("broker", 4369),
("valkey", 6379),
("otel-collector", 4318),
("otel-collector", 8888),
("otel-collector", 13133),
("prometheus", 9090),
("loki", 3100),
("loki", 9096),
("tempo", 3200),
("tempo", 4318),
("tempo", 9095),
("grafana", 3000),
]:
try:
with socket.create_connection((host, port), timeout=1):
accessible = True
except OSError:
accessible = False
record(f"plaintext-closed:{host}:{port}", not accessible)
for role in ["api", "worker", "relay"]:
token, _ = credentials(role)
own = http(role, "https://openbao:8200/v1/secret/data/" + role, token=token)[0]
other = http(role, "https://openbao:8200/v1/secret/data/migrate", token=token)[0]
admin = http(
role,
"https://openbao:8200/v1/sys/policies/acl/forbidden",
{"policy": 'path "*" { capabilities=["sudo"] }'},
token,
)[0]
record(
"bao-scope:" + role,
own == 200 and other == 403 and admin == 403,
own=own,
foreign=other,
admin=admin,
)
def signed_s3(data, method, key, body=b"", *, role="worker"):
url = "https://s3:9443/messenger-attachments/" + key
request = AWSRequest(method=method, url=url, data=body)
S3SigV4Auth(
Credentials(data["S3_ACCESS_KEY_ID"], data["S3_SECRET_ACCESS_KEY"]), "s3", "us-east-1"
).add_auth(request)
return http(role, url, body, method=method, headers=dict(request.headers))[0]
async def worker_scope():
_, data = credentials("worker")
dsn = data["DATABASE_URL"].replace("postgresql://", "postgresql+asyncpg://")
engine = create_async_engine(dsn, connect_args={"ssl": context()}, hide_parameters=True)
probes = [
("jobs-readable", "SELECT count(*) FROM jobs", True),
("users-denied", "SELECT count(*) FROM users", False),
("sessions-denied", "SELECT count(*) FROM sessions", False),
("ddl-denied", "CREATE TABLE security_escape (id integer)", False),
]
for name, sql, expected in probes:
try:
async with engine.connect() as connection:
await connection.execute(text(sql))
await connection.rollback()
success = True
except DBAPIError:
success = False
record("worker-db:" + name, success == expected)
await engine.dispose()
for method, key, expected in [
("PUT", "variants/security-probe", 200),
("GET", "variants/security-probe", 200),
("PUT", "staging/security-probe", 403),
("GET", "private/security-probe", 403),
("DELETE", "variants/security-probe", 403),
]:
status = signed_s3(data, method, key, b"probe" if method == "PUT" else b"")
record("worker-s3:" + method + ":" + key, status == expected, status=status)
parsed = urlsplit(data["BROKER_URL"])
url = urlunsplit(
("amqps", parsed.netloc.replace("127.0.0.1:5672", "broker:5671"), parsed.path, "", "")
)
conn = await aio_pika.connect(url, ssl_context=context("worker"), timeout=5)
try:
channel = await conn.channel()
queue = await channel.get_queue("attachments.process")
message = await queue.get(fail=False)
if message:
await message.nack(requeue=True)
record("worker-queue:consume", True)
channel = await conn.channel()
try:
exchange = await channel.get_exchange("attachments", ensure=False)
await exchange.publish(aio_pika.Message(b"forbidden"), routing_key="process")
denied = False
except aio_pika.exceptions.AMQPException:
denied = True
record("worker-queue:publish-denied", denied)
channel = await conn.channel()
try:
await channel.declare_queue("security-escape")
denied = False
except aio_pika.exceptions.AMQPException:
denied = True
record("worker-queue:configure-denied", denied)
finally:
await conn.close()
async def dependency_acl():
_, data = credentials("api")
cache = redis.from_url(
data["CACHE_URL"].replace("redis://", "rediss://").replace("127.0.0.1:6379", "valkey:6380"),
ssl_ca_certs=str(PKI / "api/ca.crt"),
ssl_certfile=str(PKI / "api/tls.crt"),
ssl_keyfile=str(PKI / "api/tls.key"),
socket_timeout=3,
)
await cache.set("session:security-probe", "probe", ex=30)
record("valkey:own-prefix", await cache.get("session:security-probe") == b"probe")
for name, action in [
("foreign-prefix", lambda: cache.get("foreign:security-probe")),
("admin", lambda: cache.config_get("*")),
]:
try:
await action()
denied = False
except redis.ResponseError:
denied = True
record("valkey:" + name + "-denied", denied)
await cache.delete("session:security-probe")
await cache.aclose()
dsn = data["DATABASE_URL"].replace("postgresql://", "postgresql+asyncpg://")
for label, ssl_mode in [("plaintext", False), ("untrusted-ca", ssl.create_default_context())]:
engine = create_async_engine(dsn, connect_args={"ssl": ssl_mode}, hide_parameters=True)
try:
async with engine.connect():
pass
denied = False
except Exception:
denied = True
finally:
await engine.dispose()
record("postgres:" + label + "-denied", denied)
def telemetry():
for host, path in [
("prometheus", "/api/v1/query?query=up"),
("loki", "/loki/api/v1/labels"),
("tempo", "/api/search?limit=10"),
]:
status, body = http("grafana", "https://" + host + ":9443" + path)
record("telemetry-query:" + host, status == 200, bytes=len(body))
for host, path in [("prometheus", "/api/v1/query?query=up"), ("loki", "/loki/api/v1/labels")]:
status, _ = http("otel-collector", "https://" + host + ":9443" + path)
record("collector-read-denied:" + host, status == 403)
for host, path in [("prometheus", "/api/v1/otlp/v1/metrics"), ("loki", "/otlp/v1/logs")]:
status, _ = http("grafana", "https://" + host + ":9443" + path, b"", method="POST")
record("grafana-write-denied:" + host, status == 403)
for address in sorted(
{item[4][0] for item in socket.getaddrinfo("api", 8443, type=socket.SOCK_STREAM)}
):
payload = b"GET /health/ready HTTP/1.1\r\nHost: api\r\nConnection: close\r\n\r\n"
for role, expected in [
("lb", True),
(None, False),
("outsider", False),
("foreign", False),
]:
record(
"api-replica:" + address + ":" + str(role),
tls(address, 8443, role, payload, hostname="api") == expected,
)
def telemetry_dump():
queries = {
"metrics": "https://prometheus:9443/api/v1/query?"
+ urlencode({"query": '{__name__=~"http_server.*|messenger.*"}'}),
"logs": "https://loki:9443/loki/api/v1/query_range?"
+ urlencode({"query": '{service_name=~"messenger-.+"}', "limit": 1000}),
"traces": "https://tempo:9443/api/search?limit=20",
}
data = {name: json.loads(http("operator", url)[1]) for name, url in queries.items()}
data["trace_details"] = [
json.loads(http("operator", "https://tempo:9443/api/traces/" + trace["traceID"])[1])
for trace in data["traces"].get("traces", [])
]
print(json.dumps(data))
async def revoked():
data = json.loads((PKI / "previous-worker/credentials.json").read_text())
for host, port, payload in [
("broker", 5671, b"AMQP\x00\x00\x09\x01"),
("s3", 9443, b"GET / HTTP/1.0\r\n\r\n"),
("otel-collector", 44318, b"GET / HTTP/1.0\r\n\r\n"),
]:
record("revoked-cert:" + host, not tls(host, port, "previous-worker", payload))
status, _ = http(
"previous-worker", "https://openbao:8200/v1/auth/cert/login", {"name": "worker"}
)
record("revoked-bao-cert", status in {400, 403}, status=status)
engine = create_async_engine(
data["DATABASE_URL"].replace("postgresql://", "postgresql+asyncpg://"),
connect_args={"ssl": context()},
hide_parameters=True,
)
try:
async with engine.connect():
pass
denied = False
except Exception:
denied = True
finally:
await engine.dispose()
record("rotated-db-password", denied)
if __name__ == "__main__":
if "telemetry-dump" in sys.argv:
telemetry_dump()
raise SystemExit(0)
if "revoked" in sys.argv:
asyncio.run(revoked())
print(
json.dumps(
{"checks": RESULTS, "passed": all(item["passed"] for item in RESULTS)}, indent=2
)
)
raise SystemExit(0 if all(item["passed"] for item in RESULTS) else 1)
transport_matrix()
telemetry()
asyncio.run(worker_scope())
asyncio.run(dependency_acl())
print(
json.dumps({"checks": RESULTS, "passed": all(item["passed"] for item in RESULTS)}, indent=2)
)
raise SystemExit(0 if all(item["passed"] for item in RESULTS) else 1)