forked from mtusi2026/mtusi2026lab7
349 lines
13 KiB
Python
349 lines
13 KiB
Python
import asyncio
|
|
import json
|
|
import socket
|
|
import ssl
|
|
import sys
|
|
import urllib.error
|
|
import urllib.request
|
|
from pathlib import Path
|
|
from urllib.parse import urlencode, urlsplit, urlunsplit
|
|
|
|
import aio_pika
|
|
import redis.asyncio as redis
|
|
from botocore.auth import S3SigV4Auth
|
|
from botocore.awsrequest import AWSRequest
|
|
from botocore.credentials import Credentials
|
|
from sqlalchemy import text
|
|
from sqlalchemy.exc import DBAPIError
|
|
from sqlalchemy.ext.asyncio import create_async_engine
|
|
|
|
PKI = Path("/audit-pki")
|
|
RESULTS = []
|
|
|
|
|
|
def record(name, passed, **details):
|
|
RESULTS.append({"test": name, "passed": bool(passed), **details})
|
|
|
|
|
|
def context(role=None, trusted=True):
|
|
ctx = ssl.create_default_context(cafile=str(PKI / "operator/ca.crt") if trusted else None)
|
|
if role:
|
|
ctx.load_cert_chain(PKI / role / "tls.crt", PKI / role / "tls.key")
|
|
return ctx
|
|
|
|
|
|
def tls(host, port, role, payload, *, hostname=None, trusted=True):
|
|
try:
|
|
with (
|
|
socket.create_connection((host, port), timeout=3) as raw,
|
|
context(role, trusted).wrap_socket(raw, server_hostname=hostname or host) as stream,
|
|
):
|
|
stream.sendall(payload)
|
|
return bool(stream.recv(2048))
|
|
except (OSError, ssl.SSLError):
|
|
return False
|
|
|
|
|
|
def http(role, url, payload=None, token=None, *, method=None, headers=None):
|
|
head = {"Content-Type": "application/json", **(headers or {})}
|
|
if token:
|
|
head["X-Vault-Token"] = token
|
|
data = json.dumps(payload).encode() if isinstance(payload, dict) else payload
|
|
req = urllib.request.Request(url, data=data, headers=head, method=method)
|
|
try:
|
|
with urllib.request.urlopen(req, context=context(role), timeout=5) as response:
|
|
return response.status, response.read()
|
|
except urllib.error.HTTPError as error:
|
|
return error.code, error.read()
|
|
|
|
|
|
def credentials(role):
|
|
status, body = http(role, "https://openbao:8200/v1/auth/cert/login", {"name": role})
|
|
if status != 200:
|
|
raise RuntimeError("Service authentication failed")
|
|
token = json.loads(body)["auth"]["client_token"]
|
|
status, body = http(role, "https://openbao:8200/v1/secret/data/" + role, token=token)
|
|
if status != 200:
|
|
raise RuntimeError("Scoped configuration unavailable")
|
|
return token, json.loads(body)["data"]["data"]
|
|
|
|
|
|
def transport_matrix():
|
|
targets = [
|
|
("api", 8443, "lb"),
|
|
("s3", 9443, "api"),
|
|
("broker", 5671, "worker"),
|
|
("otel-collector", 44318, "api"),
|
|
("prometheus", 9443, "grafana"),
|
|
("loki", 9443, "grafana"),
|
|
("tempo", 9443, "grafana"),
|
|
("tempo", 44318, "otel-collector"),
|
|
("grafana", 9443, "operator"),
|
|
("valkey", 6380, "api"),
|
|
]
|
|
for host, port, role in targets:
|
|
payload = (
|
|
b"AMQP\x00\x00\x09\x01"
|
|
if host == "broker"
|
|
else b"*1\r\n$4\r\nPING\r\n"
|
|
if host == "valkey"
|
|
else b"GET / HTTP/1.1\r\nHost: " + host.encode() + b"\r\nConnection: close\r\n\r\n"
|
|
)
|
|
for identity, expected in [
|
|
(role, True),
|
|
(None, False),
|
|
("outsider", False),
|
|
("foreign", False),
|
|
]:
|
|
observed = tls(host, port, identity, payload)
|
|
record(f"mtls:{host}:{port}:{identity}", observed == expected)
|
|
record(
|
|
f"hostname:{host}:{port}",
|
|
not tls(host, port, role, payload, hostname="incorrect.invalid"),
|
|
)
|
|
for host, port in [
|
|
("api", 8080),
|
|
("s3", 9000),
|
|
("broker", 5672),
|
|
("broker", 15672),
|
|
("broker", 15692),
|
|
("broker", 25672),
|
|
("broker", 4369),
|
|
("valkey", 6379),
|
|
("otel-collector", 4318),
|
|
("otel-collector", 8888),
|
|
("otel-collector", 13133),
|
|
("prometheus", 9090),
|
|
("loki", 3100),
|
|
("loki", 9096),
|
|
("tempo", 3200),
|
|
("tempo", 4318),
|
|
("tempo", 9095),
|
|
("grafana", 3000),
|
|
]:
|
|
try:
|
|
with socket.create_connection((host, port), timeout=1):
|
|
accessible = True
|
|
except OSError:
|
|
accessible = False
|
|
record(f"plaintext-closed:{host}:{port}", not accessible)
|
|
for role in ["api", "worker", "relay"]:
|
|
token, _ = credentials(role)
|
|
own = http(role, "https://openbao:8200/v1/secret/data/" + role, token=token)[0]
|
|
other = http(role, "https://openbao:8200/v1/secret/data/migrate", token=token)[0]
|
|
admin = http(
|
|
role,
|
|
"https://openbao:8200/v1/sys/policies/acl/forbidden",
|
|
{"policy": 'path "*" { capabilities=["sudo"] }'},
|
|
token,
|
|
)[0]
|
|
record(
|
|
"bao-scope:" + role,
|
|
own == 200 and other == 403 and admin == 403,
|
|
own=own,
|
|
foreign=other,
|
|
admin=admin,
|
|
)
|
|
|
|
|
|
def signed_s3(data, method, key, body=b"", *, role="worker"):
|
|
url = "https://s3:9443/messenger-attachments/" + key
|
|
request = AWSRequest(method=method, url=url, data=body)
|
|
S3SigV4Auth(
|
|
Credentials(data["S3_ACCESS_KEY_ID"], data["S3_SECRET_ACCESS_KEY"]), "s3", "us-east-1"
|
|
).add_auth(request)
|
|
return http(role, url, body, method=method, headers=dict(request.headers))[0]
|
|
|
|
|
|
async def worker_scope():
|
|
_, data = credentials("worker")
|
|
dsn = data["DATABASE_URL"].replace("postgresql://", "postgresql+asyncpg://")
|
|
engine = create_async_engine(dsn, connect_args={"ssl": context()}, hide_parameters=True)
|
|
probes = [
|
|
("jobs-readable", "SELECT count(*) FROM jobs", True),
|
|
("users-denied", "SELECT count(*) FROM users", False),
|
|
("sessions-denied", "SELECT count(*) FROM sessions", False),
|
|
("ddl-denied", "CREATE TABLE security_escape (id integer)", False),
|
|
]
|
|
for name, sql, expected in probes:
|
|
try:
|
|
async with engine.connect() as connection:
|
|
await connection.execute(text(sql))
|
|
await connection.rollback()
|
|
success = True
|
|
except DBAPIError:
|
|
success = False
|
|
record("worker-db:" + name, success == expected)
|
|
await engine.dispose()
|
|
for method, key, expected in [
|
|
("PUT", "variants/security-probe", 200),
|
|
("GET", "variants/security-probe", 200),
|
|
("PUT", "staging/security-probe", 403),
|
|
("GET", "private/security-probe", 403),
|
|
("DELETE", "variants/security-probe", 403),
|
|
]:
|
|
status = signed_s3(data, method, key, b"probe" if method == "PUT" else b"")
|
|
record("worker-s3:" + method + ":" + key, status == expected, status=status)
|
|
parsed = urlsplit(data["BROKER_URL"])
|
|
url = urlunsplit(
|
|
("amqps", parsed.netloc.replace("127.0.0.1:5672", "broker:5671"), parsed.path, "", "")
|
|
)
|
|
conn = await aio_pika.connect(url, ssl_context=context("worker"), timeout=5)
|
|
try:
|
|
channel = await conn.channel()
|
|
queue = await channel.get_queue("attachments.process")
|
|
message = await queue.get(fail=False)
|
|
if message:
|
|
await message.nack(requeue=True)
|
|
record("worker-queue:consume", True)
|
|
channel = await conn.channel()
|
|
try:
|
|
exchange = await channel.get_exchange("attachments", ensure=False)
|
|
await exchange.publish(aio_pika.Message(b"forbidden"), routing_key="process")
|
|
denied = False
|
|
except aio_pika.exceptions.AMQPException:
|
|
denied = True
|
|
record("worker-queue:publish-denied", denied)
|
|
channel = await conn.channel()
|
|
try:
|
|
await channel.declare_queue("security-escape")
|
|
denied = False
|
|
except aio_pika.exceptions.AMQPException:
|
|
denied = True
|
|
record("worker-queue:configure-denied", denied)
|
|
finally:
|
|
await conn.close()
|
|
|
|
|
|
async def dependency_acl():
|
|
_, data = credentials("api")
|
|
cache = redis.from_url(
|
|
data["CACHE_URL"].replace("redis://", "rediss://").replace("127.0.0.1:6379", "valkey:6380"),
|
|
ssl_ca_certs=str(PKI / "api/ca.crt"),
|
|
ssl_certfile=str(PKI / "api/tls.crt"),
|
|
ssl_keyfile=str(PKI / "api/tls.key"),
|
|
socket_timeout=3,
|
|
)
|
|
await cache.set("session:security-probe", "probe", ex=30)
|
|
record("valkey:own-prefix", await cache.get("session:security-probe") == b"probe")
|
|
for name, action in [
|
|
("foreign-prefix", lambda: cache.get("foreign:security-probe")),
|
|
("admin", lambda: cache.config_get("*")),
|
|
]:
|
|
try:
|
|
await action()
|
|
denied = False
|
|
except redis.ResponseError:
|
|
denied = True
|
|
record("valkey:" + name + "-denied", denied)
|
|
await cache.delete("session:security-probe")
|
|
await cache.aclose()
|
|
dsn = data["DATABASE_URL"].replace("postgresql://", "postgresql+asyncpg://")
|
|
for label, ssl_mode in [("plaintext", False), ("untrusted-ca", ssl.create_default_context())]:
|
|
engine = create_async_engine(dsn, connect_args={"ssl": ssl_mode}, hide_parameters=True)
|
|
try:
|
|
async with engine.connect():
|
|
pass
|
|
denied = False
|
|
except Exception:
|
|
denied = True
|
|
finally:
|
|
await engine.dispose()
|
|
record("postgres:" + label + "-denied", denied)
|
|
|
|
|
|
def telemetry():
|
|
for host, path in [
|
|
("prometheus", "/api/v1/query?query=up"),
|
|
("loki", "/loki/api/v1/labels"),
|
|
("tempo", "/api/search?limit=10"),
|
|
]:
|
|
status, body = http("grafana", "https://" + host + ":9443" + path)
|
|
record("telemetry-query:" + host, status == 200, bytes=len(body))
|
|
for host, path in [("prometheus", "/api/v1/query?query=up"), ("loki", "/loki/api/v1/labels")]:
|
|
status, _ = http("otel-collector", "https://" + host + ":9443" + path)
|
|
record("collector-read-denied:" + host, status == 403)
|
|
for host, path in [("prometheus", "/api/v1/otlp/v1/metrics"), ("loki", "/otlp/v1/logs")]:
|
|
status, _ = http("grafana", "https://" + host + ":9443" + path, b"", method="POST")
|
|
record("grafana-write-denied:" + host, status == 403)
|
|
for address in sorted(
|
|
{item[4][0] for item in socket.getaddrinfo("api", 8443, type=socket.SOCK_STREAM)}
|
|
):
|
|
payload = b"GET /health/ready HTTP/1.1\r\nHost: api\r\nConnection: close\r\n\r\n"
|
|
for role, expected in [
|
|
("lb", True),
|
|
(None, False),
|
|
("outsider", False),
|
|
("foreign", False),
|
|
]:
|
|
record(
|
|
"api-replica:" + address + ":" + str(role),
|
|
tls(address, 8443, role, payload, hostname="api") == expected,
|
|
)
|
|
|
|
|
|
def telemetry_dump():
|
|
queries = {
|
|
"metrics": "https://prometheus:9443/api/v1/query?"
|
|
+ urlencode({"query": '{__name__=~"http_server.*|messenger.*"}'}),
|
|
"logs": "https://loki:9443/loki/api/v1/query_range?"
|
|
+ urlencode({"query": '{service_name=~"messenger-.+"}', "limit": 1000}),
|
|
"traces": "https://tempo:9443/api/search?limit=20",
|
|
}
|
|
data = {name: json.loads(http("operator", url)[1]) for name, url in queries.items()}
|
|
data["trace_details"] = [
|
|
json.loads(http("operator", "https://tempo:9443/api/traces/" + trace["traceID"])[1])
|
|
for trace in data["traces"].get("traces", [])
|
|
]
|
|
print(json.dumps(data))
|
|
|
|
|
|
async def revoked():
|
|
data = json.loads((PKI / "previous-worker/credentials.json").read_text())
|
|
for host, port, payload in [
|
|
("broker", 5671, b"AMQP\x00\x00\x09\x01"),
|
|
("s3", 9443, b"GET / HTTP/1.0\r\n\r\n"),
|
|
("otel-collector", 44318, b"GET / HTTP/1.0\r\n\r\n"),
|
|
]:
|
|
record("revoked-cert:" + host, not tls(host, port, "previous-worker", payload))
|
|
status, _ = http(
|
|
"previous-worker", "https://openbao:8200/v1/auth/cert/login", {"name": "worker"}
|
|
)
|
|
record("revoked-bao-cert", status in {400, 403}, status=status)
|
|
engine = create_async_engine(
|
|
data["DATABASE_URL"].replace("postgresql://", "postgresql+asyncpg://"),
|
|
connect_args={"ssl": context()},
|
|
hide_parameters=True,
|
|
)
|
|
try:
|
|
async with engine.connect():
|
|
pass
|
|
denied = False
|
|
except Exception:
|
|
denied = True
|
|
finally:
|
|
await engine.dispose()
|
|
record("rotated-db-password", denied)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if "telemetry-dump" in sys.argv:
|
|
telemetry_dump()
|
|
raise SystemExit(0)
|
|
if "revoked" in sys.argv:
|
|
asyncio.run(revoked())
|
|
print(
|
|
json.dumps(
|
|
{"checks": RESULTS, "passed": all(item["passed"] for item in RESULTS)}, indent=2
|
|
)
|
|
)
|
|
raise SystemExit(0 if all(item["passed"] for item in RESULTS) else 1)
|
|
transport_matrix()
|
|
telemetry()
|
|
asyncio.run(worker_scope())
|
|
asyncio.run(dependency_acl())
|
|
print(
|
|
json.dumps({"checks": RESULTS, "passed": all(item["passed"] for item in RESULTS)}, indent=2)
|
|
)
|
|
raise SystemExit(0 if all(item["passed"] for item in RESULTS) else 1)
|