forked from mtusi2026/mtusi2026lab7
472 lines
16 KiB
Python
472 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""Reproducible negative checks and worker revocation on the local lab7 stand."""
|
|
|
|
import argparse
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import secrets
|
|
import shutil
|
|
import socket
|
|
import ssl
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.request import Request, urlopen
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
LOCAL = ROOT / "secrets"
|
|
CA = LOCAL / "tls/trust/ca.crt"
|
|
INSIDE_CA = "/run/tls/trust/ca.crt"
|
|
|
|
|
|
def checked(*command, **kwargs):
|
|
try:
|
|
return subprocess.run(command, cwd=ROOT, check=True, **kwargs)
|
|
except subprocess.CalledProcessError as exc:
|
|
if exc.stderr:
|
|
sys.stderr.write(exc.stderr)
|
|
raise
|
|
|
|
|
|
def compose(*args, **kwargs):
|
|
return checked("docker", "compose", *args, **kwargs)
|
|
|
|
|
|
def context(ca, certificate=None):
|
|
ctx = ssl.create_default_context(cafile=ca)
|
|
if certificate:
|
|
ctx.load_cert_chain(certificate / "service.crt", certificate / "service.key")
|
|
return ctx
|
|
|
|
|
|
def response(url, ctx, data=None, headers=None, method=None):
|
|
request = Request(url, data, headers or {}, method=method)
|
|
try:
|
|
with urlopen(request, context=ctx, timeout=4) as r:
|
|
return r.status, r.read(262144)
|
|
except HTTPError as e:
|
|
return e.code, b""
|
|
|
|
|
|
def denied(action):
|
|
try:
|
|
status = action()
|
|
assert status in (401, 403) or status == b"", "Unexpected success in negative probe"
|
|
except (ssl.SSLError, ConnectionResetError, BrokenPipeError, URLError) as exc:
|
|
# Preserve DNS/connectivity errors as real test failures.
|
|
cause = exc.reason if isinstance(exc, URLError) else exc
|
|
if not isinstance(cause, (ssl.SSLError, ConnectionResetError, BrokenPipeError)):
|
|
raise
|
|
|
|
|
|
def redis_ping(ctx):
|
|
with socket.create_connection(("broker", 6380), timeout=3) as raw:
|
|
with ctx.wrap_socket(raw, server_hostname="broker") as tls:
|
|
tls.sendall(b"*1\r\n$4\r\nPING\r\n")
|
|
return tls.recv(256)
|
|
|
|
|
|
def audit_tls():
|
|
results = {}
|
|
endpoints = {
|
|
"api": (lambda ctx: response("https://api:8080/health/ready", ctx)[0], "lb-client"),
|
|
"collector": (
|
|
lambda ctx: response(
|
|
"https://otel-collector:4319/v1/traces",
|
|
ctx,
|
|
b"",
|
|
{"Content-Type": "application/x-protobuf"},
|
|
)[0],
|
|
"worker-client",
|
|
),
|
|
"broker": (redis_ping, "worker-client"),
|
|
}
|
|
for name, (probe, identity) in endpoints.items():
|
|
status = probe(context(INSIDE_CA, Path("/probe") / identity))
|
|
assert status == 200 or status.startswith(b"-NOAUTH")
|
|
for cert in (None, Path("/probe/probe-client"), Path("/probe/rogue")):
|
|
denied(lambda probe=probe, cert=cert: probe(context(INSIDE_CA, cert)))
|
|
results[name] = [
|
|
"allowed",
|
|
"no_client_denied",
|
|
"same_ca_wrong_role_denied",
|
|
"rogue_ca_denied",
|
|
]
|
|
|
|
for host, port in (
|
|
("broker", 6379),
|
|
("otel-collector", 4318),
|
|
("prometheus", 19090),
|
|
("loki", 13100),
|
|
("tempo", 13200),
|
|
("tempo", 14318),
|
|
("tempo", 9095),
|
|
("loki", 9095),
|
|
):
|
|
try:
|
|
with socket.create_connection((host, port), timeout=1):
|
|
raise AssertionError("Reachable plaintext " + host)
|
|
except (ConnectionRefusedError, TimeoutError):
|
|
pass
|
|
results["network_plaintext_bypass"] = "denied"
|
|
results["grafana_backends"] = {}
|
|
for host, port, path in (
|
|
("prometheus", 9090, "/api/v1/status/buildinfo"),
|
|
("loki", 3100, "/loki/api/v1/status/buildinfo"),
|
|
("tempo", 3200, "/api/echo"),
|
|
):
|
|
endpoint = f"https://{host}:{port}{path}"
|
|
assert response(endpoint, context(INSIDE_CA, Path("/probe/grafana-client")))[0] == 200
|
|
denied(
|
|
lambda endpoint=endpoint: response(
|
|
endpoint, context(INSIDE_CA, Path("/probe/probe-client"))
|
|
)[0]
|
|
)
|
|
results["grafana_backends"][host] = "verified_tls_identity_allow_and_deny"
|
|
return results
|
|
|
|
|
|
def worker_scope():
|
|
import psycopg
|
|
import redis
|
|
from botocore.exceptions import ClientError
|
|
|
|
from messenger.settings import Settings
|
|
from messenger.storage import ObjectStore
|
|
|
|
settings = Settings.from_env()
|
|
with psycopg.connect(settings.database_url, autocommit=True) as conn:
|
|
assert conn.execute("SELECT ssl FROM pg_stat_ssl WHERE pid=pg_backend_pid()").fetchone()[0]
|
|
conn.execute("SELECT count(*) FROM attachment_jobs").fetchone()
|
|
for query in (
|
|
"SELECT count(*) FROM users",
|
|
"SELECT count(*) FROM messages",
|
|
"SELECT count(*) FROM sessions",
|
|
"CREATE TEMP TABLE forbidden_probe(id int)",
|
|
"UPDATE attachments SET owner_id=owner_id WHERE false",
|
|
"SET ROLE messenger",
|
|
):
|
|
try:
|
|
conn.execute(query)
|
|
except psycopg.errors.InsufficientPrivilege:
|
|
continue
|
|
raise AssertionError("Unexpected database grant")
|
|
|
|
broker = redis.Redis.from_url(settings.broker_url)
|
|
assert broker.ping()
|
|
for op in (
|
|
lambda: broker.config_get("appendonly"),
|
|
lambda: broker.xrange("unrelated-key"),
|
|
lambda: broker.get("session-user:synthetic"),
|
|
):
|
|
try:
|
|
op()
|
|
except redis.exceptions.NoPermissionError:
|
|
continue
|
|
raise AssertionError("Unexpected broker grant")
|
|
|
|
store = ObjectStore(settings)
|
|
suffix = uuid.uuid4().hex
|
|
key = "objects/audit/" + suffix
|
|
store.client.put_object(Bucket=store.bucket, Key=key, Body=b"synthetic")
|
|
assert store.client.head_object(Bucket=store.bucket, Key=key)["ContentLength"] == 9
|
|
for op in (
|
|
lambda: store.client.put_object(
|
|
Bucket=store.bucket, Key="staging/audit/" + suffix, Body=b"x"
|
|
),
|
|
lambda: store.client.delete_object(Bucket=store.bucket, Key=key),
|
|
lambda: store.client.get_bucket_policy(Bucket=store.bucket),
|
|
):
|
|
try:
|
|
op()
|
|
except ClientError as exc:
|
|
assert exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403
|
|
continue
|
|
raise AssertionError("Unexpected S3 grant")
|
|
|
|
cert = Path("/run/tls/client")
|
|
ctx = context(INSIDE_CA, cert)
|
|
base = "https://openbao:8200/v1/"
|
|
ct = {"Content-Type": "application/json"}
|
|
code, raw = response(base + "auth/cert/login", ctx, b'{"name":"worker"}', ct)
|
|
assert code == 200
|
|
auth = json.loads(raw)["auth"]
|
|
token = auth["client_token"]
|
|
headers = {**ct, "X-Vault-Token": token}
|
|
assert 0 < auth["lease_duration"] <= 120
|
|
assert response(base + "services/data/worker", ctx, headers=headers)[0] == 200
|
|
assert response(base + "services/data/api", ctx, headers=headers)[0] == 403
|
|
assert response(base + "services/data/worker", ctx, b'{"data":{}}', headers)[0] == 403
|
|
assert response(base + "auth/token/revoke-self", ctx, b"{}", headers)[0] == 204
|
|
assert response(base + "services/data/worker", ctx, headers=headers)[0] == 403
|
|
return {
|
|
"postgres": {"tls": True, "job_read": True, "denied_operations": 6},
|
|
"broker": {"tls": True, "ping": True, "denied_operations": 3},
|
|
"s3": {"put_objects": True, "denied_operations": 3},
|
|
"openbao": {
|
|
"own_read": True,
|
|
"foreign_read_and_write_denied": True,
|
|
"token_ttl_s": auth["lease_duration"],
|
|
"revoked_token_denied": True,
|
|
},
|
|
}
|
|
|
|
|
|
def old_identity():
|
|
ctx = context(INSIDE_CA, Path("/probe/old"))
|
|
denied(lambda: redis_ping(ctx))
|
|
denied(
|
|
lambda: response(
|
|
"https://otel-collector:4319/v1/traces",
|
|
ctx,
|
|
b"",
|
|
{"Content-Type": "application/x-protobuf"},
|
|
)[0]
|
|
)
|
|
code, _ = response(
|
|
"https://openbao:8200/v1/auth/cert/login",
|
|
ctx,
|
|
b'{"name":"worker"}',
|
|
{"Content-Type": "application/json"},
|
|
)
|
|
assert code in (400, 403)
|
|
return {"broker": "revoked", "collector": "revoked", "openbao": "revoked"}
|
|
|
|
|
|
def worker_probe():
|
|
raw = compose(
|
|
"run",
|
|
"--rm",
|
|
"--no-deps",
|
|
"-T",
|
|
"-v",
|
|
f"{Path(__file__).resolve()}:/test/audit.py:ro",
|
|
"--entrypoint",
|
|
"python",
|
|
"worker",
|
|
"/test/audit.py",
|
|
"--inside",
|
|
"worker",
|
|
capture_output=True,
|
|
text=True,
|
|
).stdout
|
|
return json.loads(raw)
|
|
|
|
|
|
def isolated(mode, mounts):
|
|
cmd = [
|
|
"docker",
|
|
"run",
|
|
"--rm",
|
|
"--network",
|
|
"mtusi-lab7_backend",
|
|
"--read-only",
|
|
"--cap-drop",
|
|
"ALL",
|
|
"--security-opt",
|
|
"no-new-privileges:true",
|
|
"--memory",
|
|
"256m",
|
|
"-v",
|
|
f"{Path(__file__).resolve()}:/test/audit.py:ro",
|
|
"-v",
|
|
f"{LOCAL / 'tls/trust'}:/run/tls/trust:ro",
|
|
]
|
|
for name, source in mounts.items():
|
|
cmd += ["-v", f"{source}:/probe/{name}:ro"]
|
|
image = compose("images", "-q", "api", capture_output=True, text=True).stdout.split()[0]
|
|
raw = checked(
|
|
*cmd, image, "python", "/test/audit.py", "--inside", mode, capture_output=True, text=True
|
|
).stdout
|
|
return json.loads(raw)
|
|
|
|
|
|
def check_api(client):
|
|
alice, bob, eve, chat = client.bootstrap()
|
|
attachment, _ = client.attachment(alice, chat)
|
|
client.wait_ready(bob, attachment)
|
|
path = "/api/v1/attachments/" + attachment["id"]
|
|
client.call("GET", path, 404, actor=eve, headers={"X-User-Id": alice["id"]})
|
|
client.call("GET", path + "/content", 404, actor=eve)
|
|
client.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=bob)
|
|
client.call("GET", "/api/v1/users/me", 401, actor=bob)
|
|
reg = alice["registration"]
|
|
bad = base64.b64encode((reg["username"] + ":invalid-password-123").encode()).decode()
|
|
statuses, replicas, times = [], set(), []
|
|
for _ in range(6):
|
|
r = client.call(
|
|
"POST", "/api/v1/auth/sessions", [401, 429], headers={"Authorization": "Basic " + bad}
|
|
)
|
|
statuses.append(r["status"])
|
|
times.append(r["elapsed"] * 1000)
|
|
replicas.add(r["headers"]["X-Instance-Id"])
|
|
if r["status"] == 429:
|
|
assert int(r["headers"]["Retry-After"]) > 0
|
|
assert 429 in statuses and 401 in statuses and len(replicas) >= 2
|
|
too_big = client.call(
|
|
"POST", "/api/v1/users", 413, raw=b"x" * 66000, headers={"Content-Type": "application/json"}
|
|
)
|
|
assert too_big["status"] == 413
|
|
return {
|
|
"statuses": statuses,
|
|
"replicas": len(replicas),
|
|
"latency_ms": times,
|
|
"outsider_denied": True,
|
|
"logout_revoked": True,
|
|
"oversize_json_denied": True,
|
|
}
|
|
|
|
|
|
def rotate(client, temp):
|
|
alice, _, _, chat = client.bootstrap()
|
|
old = temp / "old"
|
|
shutil.copytree(LOCAL / "tls/worker-client", old)
|
|
compose("stop", "worker")
|
|
accepted = []
|
|
start = time.monotonic()
|
|
try:
|
|
for _ in range(5):
|
|
att, data = client.attachment(alice, chat)
|
|
accepted.append((att, data))
|
|
checked(sys.executable, "scripts/bao_operator.py", "--revoke", "worker")
|
|
checked(sys.executable, "scripts/pki.py", "--rotate", "worker-client")
|
|
password = LOCAL / "broker_worker_password"
|
|
password.write_text(secrets.token_hex(24) + "\n")
|
|
checked(sys.executable, "scripts/bootstrap.py")
|
|
# Load the changed ACL without restarting durable Valkey or erasing its stream.
|
|
compose(
|
|
"exec",
|
|
"-T",
|
|
"broker",
|
|
"sh",
|
|
"-ec",
|
|
"""p="$(awk '/^user operator / {print substr($NF,2)}' /run/secrets/broker.acl)"
|
|
export REDISCLI_AUTH="$p"
|
|
valkey-cli --user operator ACL LOAD | grep -q '^OK$'
|
|
valkey-cli --user operator CLIENT KILL USER worker >/dev/null""",
|
|
)
|
|
compose("restart", "broker-tls", "otel-collector-tls")
|
|
result = isolated("old", {"old": old})
|
|
finally:
|
|
checked(sys.executable, "scripts/bao_operator.py")
|
|
compose("up", "-d", "--no-deps", "--force-recreate", "worker")
|
|
for attachment, original in accepted:
|
|
item = client.wait_ready(alice, attachment)
|
|
file = client.call("GET", f"/api/v1/attachments/{item['id']}/content", actor=alice)
|
|
assert hashlib.sha256(file["bytes"]).digest() == hashlib.sha256(original).digest()
|
|
return {
|
|
"old_identity": result,
|
|
"accepted": len(accepted),
|
|
"ready_with_original": len(accepted),
|
|
"elapsed_seconds": round(time.monotonic() - start, 3),
|
|
"broker_acl_reloaded": True,
|
|
"images_rebuilt": False,
|
|
"new_worker": worker_probe(),
|
|
}
|
|
|
|
|
|
def check_sealed(client):
|
|
client.call("GET", "/health/ready", schema="Health")
|
|
start = time.monotonic()
|
|
checked(sys.executable, "scripts/bao_operator.py", "--seal")
|
|
try:
|
|
deadline = start + 42
|
|
status = 200
|
|
while time.monotonic() < deadline:
|
|
status, _ = response("https://localhost:8443/health/ready", context(CA))
|
|
if status == 503:
|
|
break
|
|
time.sleep(1)
|
|
assert status == 503, "API did not stop serving after the credential grace period"
|
|
unavailable = time.monotonic() - start
|
|
finally:
|
|
checked(sys.executable, "scripts/bao_operator.py")
|
|
deadline = time.monotonic() + 22
|
|
while time.monotonic() < deadline:
|
|
status, _ = response("https://localhost:8443/health/ready", context(CA))
|
|
if status == 200:
|
|
return {
|
|
"unready_after_seconds": round(unavailable, 3),
|
|
"recovered_after_seconds": round(time.monotonic() - start, 3),
|
|
}
|
|
time.sleep(1)
|
|
raise AssertionError("API readiness did not recover after unseal")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--inside", choices=("tls", "worker", "old"))
|
|
parser.add_argument("--output", type=Path)
|
|
parser.add_argument("--rotate", action="store_true")
|
|
parser.add_argument("--seal", action="store_true")
|
|
args = parser.parse_args()
|
|
if args.inside:
|
|
print(
|
|
json.dumps(
|
|
{"tls": audit_tls, "worker": worker_scope, "old": old_identity}[args.inside]()
|
|
)
|
|
)
|
|
return
|
|
if not args.output or args.output.exists():
|
|
parser.error("choose a new --output")
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client
|
|
|
|
client = Client("https://localhost:8443", str(CA))
|
|
with tempfile.TemporaryDirectory(dir=LOCAL) as folder:
|
|
tmp = Path(folder)
|
|
tmp.chmod(0o755)
|
|
rogue = tmp / "rogue"
|
|
rogue.mkdir(mode=0o755)
|
|
checked(
|
|
"openssl",
|
|
"req",
|
|
"-x509",
|
|
"-newkey",
|
|
"ec",
|
|
"-pkeyopt",
|
|
"ec_paramgen_curve:P-256",
|
|
"-nodes",
|
|
"-days",
|
|
"1",
|
|
"-subj",
|
|
"/CN=worker-client",
|
|
"-addext",
|
|
"extendedKeyUsage=clientAuth",
|
|
"-keyout",
|
|
str(rogue / "service.key"),
|
|
"-out",
|
|
str(rogue / "service.crt"),
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
(rogue / "service.key").chmod(0o644)
|
|
identities = {
|
|
name: LOCAL / "tls" / name
|
|
for name in ("lb-client", "worker-client", "probe-client", "grafana-client")
|
|
}
|
|
result = {
|
|
"tls": isolated("tls", {**identities, "rogue": rogue}),
|
|
"worker_scope": worker_probe(),
|
|
"api": check_api(client),
|
|
}
|
|
print("PASS: verified TLS and distinct service privileges", flush=True)
|
|
if args.rotate:
|
|
result["worker_compromise"] = rotate(client, tmp)
|
|
print("PASS: old identity denied; all accepted uploads recovered", flush=True)
|
|
if args.seal:
|
|
result["sealed"] = check_sealed(client)
|
|
print(
|
|
"PASS: sealed OpenBao makes API unready within a bounded grace period", flush=True
|
|
)
|
|
args.output.write_text(json.dumps(result, indent=2) + "\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|