Files
mtusi2026lab7/student-tests/trust.py
T
2026-11-11 13:53:51 +03:00

252 lines
12 KiB
Python

"""Operator-only integration probe, on the Compose network with synthetic credentials."""
import http.client
import json
import socket
import ssl
from pathlib import Path
import sys
import time
from urllib.error import HTTPError
import boto3
from botocore.exceptions import ClientError
import psycopg
import redis
sys.path.insert(0, "/app")
from src.bao import client, request
PKI = Path("/audit/runtime")
CA = str(PKI / "api" / "ca.crt")
SEEN = []
def check(label, condition):
if not condition:
raise AssertionError(label)
SEEN.append(label)
print("PASS", label, flush=True)
def context(role=None, ca=CA):
ctx = ssl.create_default_context(cafile=ca)
if role == "foreign":
ctx.load_cert_chain("/audit/foreign/client/service.crt", "/audit/foreign/client/service.key")
elif role == "lb-old":
ctx.load_cert_chain("/audit/rotation/previous.pem")
elif role == "collector-previous":
ctx.load_cert_chain("/audit/rotation/collector-previous.pem")
elif role == "collector-export":
ctx.load_cert_chain(PKI / "collector/client/service.crt", PKI / "collector/client/service.key")
elif role == "lb-active":
ctx.load_cert_chain(PKI / "lb" / "client.pem")
elif role:
ctx.load_cert_chain(PKI / role / "service.crt", PKI / role / "service.key")
return ctx
def https(host, port, path, role=None, method="GET", body=None, address=None, ca=CA, content_type="application/json"):
conn = http.client.HTTPSConnection(host, port, context=context(role, ca), timeout=4)
if address:
conn._create_connection = lambda _addr, timeout, source_address: socket.create_connection(
(address, port), timeout, source_address)
try:
conn.request(method, path, body=body, headers={"Content-Type": content_type})
res = conn.getresponse()
return res.status, res.read(), dict(res.getheaders())
finally:
conn.close()
def rejected_http(*args, **kwargs):
try:
return https(*args, **kwargs)[0] == 403
except (ssl.SSLError, ConnectionResetError, http.client.RemoteDisconnected):
return True
def denied(action, error, verify=lambda _exc: True):
try:
action()
except error as exc:
return verify(exc)
return False
def bao_client(role):
path = PKI / role
return client(str(path / "service.crt"), str(path / "service.key"), CA)
def bao(role, path, data=None, token=None, method=None):
return request(bao_client(role), "https://openbao:8200", path, data, token, method)
def get_auth(role):
return bao(role, "auth/cert/login", {"name": role})["auth"]
def get_values(role, token=None):
return bao(role, "messenger/data/" + role, token=token or get_auth(role)["client_token"])["data"]["data"]
def broker(url, role):
folder = (Path("/audit/foreign/client") if role == "foreign" else PKI / role) if role else None
return redis.Redis.from_url(url, socket_timeout=3, socket_connect_timeout=3,
ssl_ca_certs=CA, ssl_check_hostname=True,
ssl_certfile=str(folder / "service.crt") if folder else None,
ssl_keyfile=str(folder / "service.key") if folder else None)
def s3(values):
return boto3.client("s3", endpoint_url="https://s3:9000", region_name="us-east-1",
verify=CA, aws_access_key_id=values["S3_ACCESS_KEY_ID"],
aws_secret_access_key=values["S3_SECRET_ACCESS_KEY"])
def verify_revocation():
addresses = {item[4][0] for item in socket.getaddrinfo("api", 8444, socket.AF_INET)}
check("rotation tests both API addresses", len(addresses) == 2)
for index, address in enumerate(sorted(addresses), 1):
args = ("api", 8444, "/health/ready")
check(f"API {index} rejects previous same-SAN leaf on new TLS connection",
rejected_http(*args, "lb-old", address=address))
check(f"API {index} permits replacement leaf on new TLS connection",
https(*args, "lb-active", address=address)[0] == 200)
def collector_status(role):
from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest
payload = ExportLogsServiceRequest()
logs = payload.resource_logs.add()
logs.resource.attributes.add(key="service.name").value.string_value = "lab7-audit"
record = logs.scope_logs.add().log_records.add(time_unix_nano=time.time_ns(), severity_text="INFO")
record.body.string_value = "TLS audit event"
loki = https("loki", 3100, "/otlp/v1/logs", role, "POST",
payload.SerializeToString(), content_type="application/x-protobuf")[0]
tempo = https("tempo", 4318, "/v1/traces", role, "POST", b"{}")[0]
return loki, tempo
def main():
addresses = sorted({item[4][0] for item in socket.getaddrinfo("api", 8444, socket.AF_INET)})
check("two API addresses resolved", len(addresses) == 2)
for index, address in enumerate(addresses, 1):
args = ("api", 8444, "/health/ready")
check(f"API {index} allows active LB", https(*args, "lb-active", address=address)[0] == 200)
for role in (None, "foreign", "intruder"):
check(f"API {index} rejects {role or 'missing cert'}",
rejected_http(*args, role, address=address))
for role in (None, "foreign", "intruder"):
check(f"Collector rejects {role or 'missing cert'}", rejected_http(
"collector", 4318, "/v1/metrics", role, "POST", b"{}"))
for role in ("api", "worker", "dispatcher", "reconciler"):
check(f"Collector allows {role} OTLP", https(
"collector", 4318, "/v1/metrics", role, "POST", b"{}")[0] == 200)
for port in (8888, 8889):
check(f"Collector scrape {port} permits only scraper",
https("collector", port, "/metrics", "scraper")[0] == 200
and https("collector", port, "/metrics", "api")[0] == 403)
api = get_values("api")
worker = get_values("worker")
dispatcher = get_values("dispatcher")
for role, values in (("worker", worker), ("dispatcher", dispatcher)):
url = values["BROKER_URL"]
check(f"broker allows {role}", broker(url, role).ping())
for wrong in (None, "foreign", "intruder", "api"):
check(f"broker rejects {wrong or 'missing cert'} with valid {role} Redis password",
denied(lambda: broker(url, wrong).ping(), redis.ConnectionError))
check("Redis producer cannot read stream", denied(
lambda: broker(dispatcher["BROKER_URL"], "dispatcher").xrange("attachments.jobs"),
redis.ResponseError, lambda e: "no permissions" in str(e).lower()))
check("Redis worker cannot publish", denied(
lambda: broker(worker["BROKER_URL"], "worker").xadd("attachments.jobs", {"audit": "denied"}),
redis.ResponseError, lambda e: "no permissions" in str(e).lower()))
check("Redis worker cannot write foreign key", denied(
lambda: broker(worker["BROKER_URL"], "worker").xack("other.stream", "image-workers", "0-0"),
redis.ResponseError, lambda e: "no permissions" in str(e).lower()))
for role, values in (("api", api), ("worker", worker), ("dispatcher", dispatcher),
("reconciler", get_values("reconciler"))):
dsn = values["DATABASE_URL"]
with psycopg.connect(dsn) as db:
info = db.execute("SELECT current_user, ssl, version FROM pg_stat_ssl WHERE pid=pg_backend_pid()").fetchone()
check(f"DB verify-full and distinct {role} login", info[0] == "lab_" + role and info[1])
check(f"DB rejects plaintext {role} login", denied(
lambda: psycopg.connect(dsn, sslmode="disable"), psycopg.OperationalError,
lambda e: "no encryption" in str(e).lower()))
check("DB rejects mismatched SAN", denied(lambda: psycopg.connect(
api["DATABASE_URL"], host="intruder", hostaddr=socket.gethostbyname("db")),
psycopg.OperationalError, lambda e: "does not match" in str(e).lower()))
with psycopg.connect(worker["DATABASE_URL"]) as db:
check("worker cannot read users/passwords", denied(
lambda: db.execute("SELECT password_hash FROM users"), psycopg.errors.InsufficientPrivilege))
bucket = "messenger-attachments"
original = s3(api).list_objects_v2(Bucket=bucket, Prefix="originals/", MaxKeys=1)
check("S3 API allowed to list originals via verified HTTPS",
original["ResponseMetadata"]["HTTPStatusCode"] == 200)
check("S3 application cannot list all buckets (operator privilege)", denied(
lambda: s3(api).list_buckets(), ClientError, lambda e: e.response["ResponseMetadata"]["HTTPStatusCode"] == 403))
check("S3 public health verified over TLS", https("s3", 9000, "/minio/health/live")[0] == 200)
if original.get("Contents"):
key = original["Contents"][0]["Key"]
check("S3 original is private even with a known object key", https("s3", 9000, f"/{bucket}/{key}")[0] == 403)
for label, fn in [
("S3 API cannot put original", lambda: s3(api).put_object(Bucket=bucket, Key="originals/deny", Body=b"deny")),
("S3 worker cannot list bucket", lambda: s3(worker).list_objects_v2(Bucket=bucket)),
("S3 worker cannot put arbitrary prefix", lambda: s3(worker).put_object(Bucket=bucket, Key="deny/object", Body=b"deny")),
]:
check(label, denied(fn, ClientError, lambda e: e.response["ResponseMetadata"]["HTTPStatusCode"] == 403))
for role in ("api", "worker", "dispatcher", "reconciler"):
auth = get_auth(role)
token = auth["client_token"]
check(f"OpenBao {role} scoped TTL and secret", auth["policies"] == [role]
and 0 < auth["lease_duration"] <= 90 and "DATABASE_URL" in get_values(role, token))
for path, data in (("messenger/data/operator", None), ("sys/policies/acl/escalate", {"policy": 'path "*" {capabilities=["sudo"]}'})):
check(f"OpenBao {role} denied {path}", denied(
lambda: bao(role, path, data, token), HTTPError, lambda e: e.code == 403))
other = "worker" if role == "api" else "api"
check(f"OpenBao {role} denied existing {other} secret", denied(
lambda: bao(role, "messenger/data/" + other, token=token), HTTPError, lambda e: e.code == 403))
check("OpenBao same-CA intruder cannot assume API role", denied(
lambda: bao("intruder", "auth/cert/login", {"name": "api"}), HTTPError,
lambda e: e.code in (400, 403)))
check("OpenBao worker credential has no operator access",
set(worker) == {"DATABASE_URL", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY", "BROKER_URL"})
for host, port, path in (
("prometheus", 9090, "/api/v1/query?query=up"),
("loki", 3100, "/loki/api/v1/labels"),
("tempo", 3200, "/api/search?limit=1"),
):
check(f"{host} data source read role", https(host, port, path, "grafana")[0] == 200)
check(f"{host} denies application reads", https(host, port, path, "api")[0] == 403)
check("Loki denies Grafana ingest", https("loki", 3100, "/otlp/v1/logs", "grafana", "POST", b"{}")[0] == 403)
check("Tempo denies Grafana ingest", https("tempo", 4318, "/v1/traces", "grafana", "POST", b"{}")[0] == 403)
loki, tempo = collector_status("collector-export")
check("Loki permits Collector ingest", loki in (200, 204))
check("Tempo permits Collector ingest", tempo == 200)
for host, port in (("broker", 6379), ("collector", 14318), ("collector", 18889),
("prometheus", 19090), ("loki", 13100), ("tempo", 13200), ("tempo", 14318)):
check(f"closed plaintext port {host}:{port}", denied(
lambda: socket.create_connection((host, port), timeout=2), ConnectionRefusedError))
print(f"PASS {len(SEEN)} TLS and ACL assertions")
if __name__ == "__main__":
if sys.argv[1:] == ["rotation"]:
verify_revocation()
elif sys.argv[1:] == ["collector-current"]:
loki, tempo = collector_status("collector-export")
check("active Collector exporter leaf accepted by Loki and Tempo", loki in (200, 204) and tempo == 200)
elif sys.argv[1:] == ["collector-rotation"]:
check("previous Collector leaf denied by Loki and Tempo", collector_status("collector-previous") == (403, 403))
loki, tempo = collector_status("collector-export")
check("replacement Collector leaf accepted by Loki and Tempo", loki in (200, 204) and tempo == 200)
else:
main()