forked from mtusi2026/mtusi2026lab7
252 lines
12 KiB
Python
252 lines
12 KiB
Python
"""Operator-only integration probe, on the Compose network with synthetic credentials."""
|
|
import http.client
|
|
import json
|
|
import socket
|
|
import ssl
|
|
from pathlib import Path
|
|
import sys
|
|
import time
|
|
from urllib.error import HTTPError
|
|
|
|
import boto3
|
|
from botocore.exceptions import ClientError
|
|
import psycopg
|
|
import redis
|
|
|
|
sys.path.insert(0, "/app")
|
|
from src.bao import client, request
|
|
|
|
PKI = Path("/audit/runtime")
|
|
CA = str(PKI / "api" / "ca.crt")
|
|
SEEN = []
|
|
|
|
|
|
def check(label, condition):
|
|
if not condition:
|
|
raise AssertionError(label)
|
|
SEEN.append(label)
|
|
print("PASS", label, flush=True)
|
|
|
|
|
|
def context(role=None, ca=CA):
|
|
ctx = ssl.create_default_context(cafile=ca)
|
|
if role == "foreign":
|
|
ctx.load_cert_chain("/audit/foreign/client/service.crt", "/audit/foreign/client/service.key")
|
|
elif role == "lb-old":
|
|
ctx.load_cert_chain("/audit/rotation/previous.pem")
|
|
elif role == "collector-previous":
|
|
ctx.load_cert_chain("/audit/rotation/collector-previous.pem")
|
|
elif role == "collector-export":
|
|
ctx.load_cert_chain(PKI / "collector/client/service.crt", PKI / "collector/client/service.key")
|
|
elif role == "lb-active":
|
|
ctx.load_cert_chain(PKI / "lb" / "client.pem")
|
|
elif role:
|
|
ctx.load_cert_chain(PKI / role / "service.crt", PKI / role / "service.key")
|
|
return ctx
|
|
|
|
|
|
def https(host, port, path, role=None, method="GET", body=None, address=None, ca=CA, content_type="application/json"):
|
|
conn = http.client.HTTPSConnection(host, port, context=context(role, ca), timeout=4)
|
|
if address:
|
|
conn._create_connection = lambda _addr, timeout, source_address: socket.create_connection(
|
|
(address, port), timeout, source_address)
|
|
try:
|
|
conn.request(method, path, body=body, headers={"Content-Type": content_type})
|
|
res = conn.getresponse()
|
|
return res.status, res.read(), dict(res.getheaders())
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def rejected_http(*args, **kwargs):
|
|
try:
|
|
return https(*args, **kwargs)[0] == 403
|
|
except (ssl.SSLError, ConnectionResetError, http.client.RemoteDisconnected):
|
|
return True
|
|
|
|
|
|
def denied(action, error, verify=lambda _exc: True):
|
|
try:
|
|
action()
|
|
except error as exc:
|
|
return verify(exc)
|
|
return False
|
|
|
|
|
|
def bao_client(role):
|
|
path = PKI / role
|
|
return client(str(path / "service.crt"), str(path / "service.key"), CA)
|
|
|
|
|
|
def bao(role, path, data=None, token=None, method=None):
|
|
return request(bao_client(role), "https://openbao:8200", path, data, token, method)
|
|
|
|
|
|
def get_auth(role):
|
|
return bao(role, "auth/cert/login", {"name": role})["auth"]
|
|
|
|
|
|
def get_values(role, token=None):
|
|
return bao(role, "messenger/data/" + role, token=token or get_auth(role)["client_token"])["data"]["data"]
|
|
|
|
|
|
def broker(url, role):
|
|
folder = (Path("/audit/foreign/client") if role == "foreign" else PKI / role) if role else None
|
|
return redis.Redis.from_url(url, socket_timeout=3, socket_connect_timeout=3,
|
|
ssl_ca_certs=CA, ssl_check_hostname=True,
|
|
ssl_certfile=str(folder / "service.crt") if folder else None,
|
|
ssl_keyfile=str(folder / "service.key") if folder else None)
|
|
|
|
|
|
def s3(values):
|
|
return boto3.client("s3", endpoint_url="https://s3:9000", region_name="us-east-1",
|
|
verify=CA, aws_access_key_id=values["S3_ACCESS_KEY_ID"],
|
|
aws_secret_access_key=values["S3_SECRET_ACCESS_KEY"])
|
|
|
|
|
|
def verify_revocation():
|
|
addresses = {item[4][0] for item in socket.getaddrinfo("api", 8444, socket.AF_INET)}
|
|
check("rotation tests both API addresses", len(addresses) == 2)
|
|
for index, address in enumerate(sorted(addresses), 1):
|
|
args = ("api", 8444, "/health/ready")
|
|
check(f"API {index} rejects previous same-SAN leaf on new TLS connection",
|
|
rejected_http(*args, "lb-old", address=address))
|
|
check(f"API {index} permits replacement leaf on new TLS connection",
|
|
https(*args, "lb-active", address=address)[0] == 200)
|
|
|
|
|
|
def collector_status(role):
|
|
from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest
|
|
payload = ExportLogsServiceRequest()
|
|
logs = payload.resource_logs.add()
|
|
logs.resource.attributes.add(key="service.name").value.string_value = "lab7-audit"
|
|
record = logs.scope_logs.add().log_records.add(time_unix_nano=time.time_ns(), severity_text="INFO")
|
|
record.body.string_value = "TLS audit event"
|
|
loki = https("loki", 3100, "/otlp/v1/logs", role, "POST",
|
|
payload.SerializeToString(), content_type="application/x-protobuf")[0]
|
|
tempo = https("tempo", 4318, "/v1/traces", role, "POST", b"{}")[0]
|
|
return loki, tempo
|
|
|
|
|
|
def main():
|
|
addresses = sorted({item[4][0] for item in socket.getaddrinfo("api", 8444, socket.AF_INET)})
|
|
check("two API addresses resolved", len(addresses) == 2)
|
|
for index, address in enumerate(addresses, 1):
|
|
args = ("api", 8444, "/health/ready")
|
|
check(f"API {index} allows active LB", https(*args, "lb-active", address=address)[0] == 200)
|
|
for role in (None, "foreign", "intruder"):
|
|
check(f"API {index} rejects {role or 'missing cert'}",
|
|
rejected_http(*args, role, address=address))
|
|
|
|
for role in (None, "foreign", "intruder"):
|
|
check(f"Collector rejects {role or 'missing cert'}", rejected_http(
|
|
"collector", 4318, "/v1/metrics", role, "POST", b"{}"))
|
|
for role in ("api", "worker", "dispatcher", "reconciler"):
|
|
check(f"Collector allows {role} OTLP", https(
|
|
"collector", 4318, "/v1/metrics", role, "POST", b"{}")[0] == 200)
|
|
for port in (8888, 8889):
|
|
check(f"Collector scrape {port} permits only scraper",
|
|
https("collector", port, "/metrics", "scraper")[0] == 200
|
|
and https("collector", port, "/metrics", "api")[0] == 403)
|
|
|
|
api = get_values("api")
|
|
worker = get_values("worker")
|
|
dispatcher = get_values("dispatcher")
|
|
for role, values in (("worker", worker), ("dispatcher", dispatcher)):
|
|
url = values["BROKER_URL"]
|
|
check(f"broker allows {role}", broker(url, role).ping())
|
|
for wrong in (None, "foreign", "intruder", "api"):
|
|
check(f"broker rejects {wrong or 'missing cert'} with valid {role} Redis password",
|
|
denied(lambda: broker(url, wrong).ping(), redis.ConnectionError))
|
|
check("Redis producer cannot read stream", denied(
|
|
lambda: broker(dispatcher["BROKER_URL"], "dispatcher").xrange("attachments.jobs"),
|
|
redis.ResponseError, lambda e: "no permissions" in str(e).lower()))
|
|
check("Redis worker cannot publish", denied(
|
|
lambda: broker(worker["BROKER_URL"], "worker").xadd("attachments.jobs", {"audit": "denied"}),
|
|
redis.ResponseError, lambda e: "no permissions" in str(e).lower()))
|
|
check("Redis worker cannot write foreign key", denied(
|
|
lambda: broker(worker["BROKER_URL"], "worker").xack("other.stream", "image-workers", "0-0"),
|
|
redis.ResponseError, lambda e: "no permissions" in str(e).lower()))
|
|
|
|
for role, values in (("api", api), ("worker", worker), ("dispatcher", dispatcher),
|
|
("reconciler", get_values("reconciler"))):
|
|
dsn = values["DATABASE_URL"]
|
|
with psycopg.connect(dsn) as db:
|
|
info = db.execute("SELECT current_user, ssl, version FROM pg_stat_ssl WHERE pid=pg_backend_pid()").fetchone()
|
|
check(f"DB verify-full and distinct {role} login", info[0] == "lab_" + role and info[1])
|
|
check(f"DB rejects plaintext {role} login", denied(
|
|
lambda: psycopg.connect(dsn, sslmode="disable"), psycopg.OperationalError,
|
|
lambda e: "no encryption" in str(e).lower()))
|
|
check("DB rejects mismatched SAN", denied(lambda: psycopg.connect(
|
|
api["DATABASE_URL"], host="intruder", hostaddr=socket.gethostbyname("db")),
|
|
psycopg.OperationalError, lambda e: "does not match" in str(e).lower()))
|
|
with psycopg.connect(worker["DATABASE_URL"]) as db:
|
|
check("worker cannot read users/passwords", denied(
|
|
lambda: db.execute("SELECT password_hash FROM users"), psycopg.errors.InsufficientPrivilege))
|
|
|
|
bucket = "messenger-attachments"
|
|
original = s3(api).list_objects_v2(Bucket=bucket, Prefix="originals/", MaxKeys=1)
|
|
check("S3 API allowed to list originals via verified HTTPS",
|
|
original["ResponseMetadata"]["HTTPStatusCode"] == 200)
|
|
check("S3 application cannot list all buckets (operator privilege)", denied(
|
|
lambda: s3(api).list_buckets(), ClientError, lambda e: e.response["ResponseMetadata"]["HTTPStatusCode"] == 403))
|
|
check("S3 public health verified over TLS", https("s3", 9000, "/minio/health/live")[0] == 200)
|
|
if original.get("Contents"):
|
|
key = original["Contents"][0]["Key"]
|
|
check("S3 original is private even with a known object key", https("s3", 9000, f"/{bucket}/{key}")[0] == 403)
|
|
for label, fn in [
|
|
("S3 API cannot put original", lambda: s3(api).put_object(Bucket=bucket, Key="originals/deny", Body=b"deny")),
|
|
("S3 worker cannot list bucket", lambda: s3(worker).list_objects_v2(Bucket=bucket)),
|
|
("S3 worker cannot put arbitrary prefix", lambda: s3(worker).put_object(Bucket=bucket, Key="deny/object", Body=b"deny")),
|
|
]:
|
|
check(label, denied(fn, ClientError, lambda e: e.response["ResponseMetadata"]["HTTPStatusCode"] == 403))
|
|
|
|
for role in ("api", "worker", "dispatcher", "reconciler"):
|
|
auth = get_auth(role)
|
|
token = auth["client_token"]
|
|
check(f"OpenBao {role} scoped TTL and secret", auth["policies"] == [role]
|
|
and 0 < auth["lease_duration"] <= 90 and "DATABASE_URL" in get_values(role, token))
|
|
for path, data in (("messenger/data/operator", None), ("sys/policies/acl/escalate", {"policy": 'path "*" {capabilities=["sudo"]}'})):
|
|
check(f"OpenBao {role} denied {path}", denied(
|
|
lambda: bao(role, path, data, token), HTTPError, lambda e: e.code == 403))
|
|
other = "worker" if role == "api" else "api"
|
|
check(f"OpenBao {role} denied existing {other} secret", denied(
|
|
lambda: bao(role, "messenger/data/" + other, token=token), HTTPError, lambda e: e.code == 403))
|
|
check("OpenBao same-CA intruder cannot assume API role", denied(
|
|
lambda: bao("intruder", "auth/cert/login", {"name": "api"}), HTTPError,
|
|
lambda e: e.code in (400, 403)))
|
|
check("OpenBao worker credential has no operator access",
|
|
set(worker) == {"DATABASE_URL", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY", "BROKER_URL"})
|
|
|
|
for host, port, path in (
|
|
("prometheus", 9090, "/api/v1/query?query=up"),
|
|
("loki", 3100, "/loki/api/v1/labels"),
|
|
("tempo", 3200, "/api/search?limit=1"),
|
|
):
|
|
check(f"{host} data source read role", https(host, port, path, "grafana")[0] == 200)
|
|
check(f"{host} denies application reads", https(host, port, path, "api")[0] == 403)
|
|
check("Loki denies Grafana ingest", https("loki", 3100, "/otlp/v1/logs", "grafana", "POST", b"{}")[0] == 403)
|
|
check("Tempo denies Grafana ingest", https("tempo", 4318, "/v1/traces", "grafana", "POST", b"{}")[0] == 403)
|
|
loki, tempo = collector_status("collector-export")
|
|
check("Loki permits Collector ingest", loki in (200, 204))
|
|
check("Tempo permits Collector ingest", tempo == 200)
|
|
for host, port in (("broker", 6379), ("collector", 14318), ("collector", 18889),
|
|
("prometheus", 19090), ("loki", 13100), ("tempo", 13200), ("tempo", 14318)):
|
|
check(f"closed plaintext port {host}:{port}", denied(
|
|
lambda: socket.create_connection((host, port), timeout=2), ConnectionRefusedError))
|
|
print(f"PASS {len(SEEN)} TLS and ACL assertions")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if sys.argv[1:] == ["rotation"]:
|
|
verify_revocation()
|
|
elif sys.argv[1:] == ["collector-current"]:
|
|
loki, tempo = collector_status("collector-export")
|
|
check("active Collector exporter leaf accepted by Loki and Tempo", loki in (200, 204) and tempo == 200)
|
|
elif sys.argv[1:] == ["collector-rotation"]:
|
|
check("previous Collector leaf denied by Loki and Tempo", collector_status("collector-previous") == (403, 403))
|
|
loki, tempo = collector_status("collector-export")
|
|
check("replacement Collector leaf accepted by Loki and Tempo", loki in (200, 204) and tempo == 200)
|
|
else:
|
|
main()
|