forked from mtusi2026/mtusi2026lab7
Защитить мессенджер TLS и OpenBao и проверить отзыв прав worker
This commit is contained in:
@@ -0,0 +1,471 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reproducible negative checks and worker revocation on the local lab7 stand."""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import secrets
|
||||
import shutil
|
||||
import socket
|
||||
import ssl
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
LOCAL = ROOT / "secrets"
|
||||
CA = LOCAL / "tls/trust/ca.crt"
|
||||
INSIDE_CA = "/run/tls/trust/ca.crt"
|
||||
|
||||
|
||||
def checked(*command, **kwargs):
|
||||
try:
|
||||
return subprocess.run(command, cwd=ROOT, check=True, **kwargs)
|
||||
except subprocess.CalledProcessError as exc:
|
||||
if exc.stderr:
|
||||
sys.stderr.write(exc.stderr)
|
||||
raise
|
||||
|
||||
|
||||
def compose(*args, **kwargs):
|
||||
return checked("docker", "compose", *args, **kwargs)
|
||||
|
||||
|
||||
def context(ca, certificate=None):
|
||||
ctx = ssl.create_default_context(cafile=ca)
|
||||
if certificate:
|
||||
ctx.load_cert_chain(certificate / "service.crt", certificate / "service.key")
|
||||
return ctx
|
||||
|
||||
|
||||
def response(url, ctx, data=None, headers=None, method=None):
|
||||
request = Request(url, data, headers or {}, method=method)
|
||||
try:
|
||||
with urlopen(request, context=ctx, timeout=4) as r:
|
||||
return r.status, r.read(262144)
|
||||
except HTTPError as e:
|
||||
return e.code, b""
|
||||
|
||||
|
||||
def denied(action):
|
||||
try:
|
||||
status = action()
|
||||
assert status in (401, 403) or status == b"", "Unexpected success in negative probe"
|
||||
except (ssl.SSLError, ConnectionResetError, BrokenPipeError, URLError) as exc:
|
||||
# Preserve DNS/connectivity errors as real test failures.
|
||||
cause = exc.reason if isinstance(exc, URLError) else exc
|
||||
if not isinstance(cause, (ssl.SSLError, ConnectionResetError, BrokenPipeError)):
|
||||
raise
|
||||
|
||||
|
||||
def redis_ping(ctx):
|
||||
with socket.create_connection(("broker", 6380), timeout=3) as raw:
|
||||
with ctx.wrap_socket(raw, server_hostname="broker") as tls:
|
||||
tls.sendall(b"*1\r\n$4\r\nPING\r\n")
|
||||
return tls.recv(256)
|
||||
|
||||
|
||||
def audit_tls():
|
||||
results = {}
|
||||
endpoints = {
|
||||
"api": (lambda ctx: response("https://api:8080/health/ready", ctx)[0], "lb-client"),
|
||||
"collector": (
|
||||
lambda ctx: response(
|
||||
"https://otel-collector:4319/v1/traces",
|
||||
ctx,
|
||||
b"",
|
||||
{"Content-Type": "application/x-protobuf"},
|
||||
)[0],
|
||||
"worker-client",
|
||||
),
|
||||
"broker": (redis_ping, "worker-client"),
|
||||
}
|
||||
for name, (probe, identity) in endpoints.items():
|
||||
status = probe(context(INSIDE_CA, Path("/probe") / identity))
|
||||
assert status == 200 or status.startswith(b"-NOAUTH")
|
||||
for cert in (None, Path("/probe/probe-client"), Path("/probe/rogue")):
|
||||
denied(lambda probe=probe, cert=cert: probe(context(INSIDE_CA, cert)))
|
||||
results[name] = [
|
||||
"allowed",
|
||||
"no_client_denied",
|
||||
"same_ca_wrong_role_denied",
|
||||
"rogue_ca_denied",
|
||||
]
|
||||
|
||||
for host, port in (
|
||||
("broker", 6379),
|
||||
("otel-collector", 4318),
|
||||
("prometheus", 19090),
|
||||
("loki", 13100),
|
||||
("tempo", 13200),
|
||||
("tempo", 14318),
|
||||
("tempo", 9095),
|
||||
("loki", 9095),
|
||||
):
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=1):
|
||||
raise AssertionError("Reachable plaintext " + host)
|
||||
except (ConnectionRefusedError, TimeoutError):
|
||||
pass
|
||||
results["network_plaintext_bypass"] = "denied"
|
||||
results["grafana_backends"] = {}
|
||||
for host, port, path in (
|
||||
("prometheus", 9090, "/api/v1/status/buildinfo"),
|
||||
("loki", 3100, "/loki/api/v1/status/buildinfo"),
|
||||
("tempo", 3200, "/api/echo"),
|
||||
):
|
||||
endpoint = f"https://{host}:{port}{path}"
|
||||
assert response(endpoint, context(INSIDE_CA, Path("/probe/grafana-client")))[0] == 200
|
||||
denied(
|
||||
lambda endpoint=endpoint: response(
|
||||
endpoint, context(INSIDE_CA, Path("/probe/probe-client"))
|
||||
)[0]
|
||||
)
|
||||
results["grafana_backends"][host] = "verified_tls_identity_allow_and_deny"
|
||||
return results
|
||||
|
||||
|
||||
def worker_scope():
|
||||
import psycopg
|
||||
import redis
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from messenger.settings import Settings
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
settings = Settings.from_env()
|
||||
with psycopg.connect(settings.database_url, autocommit=True) as conn:
|
||||
assert conn.execute("SELECT ssl FROM pg_stat_ssl WHERE pid=pg_backend_pid()").fetchone()[0]
|
||||
conn.execute("SELECT count(*) FROM attachment_jobs").fetchone()
|
||||
for query in (
|
||||
"SELECT count(*) FROM users",
|
||||
"SELECT count(*) FROM messages",
|
||||
"SELECT count(*) FROM sessions",
|
||||
"CREATE TEMP TABLE forbidden_probe(id int)",
|
||||
"UPDATE attachments SET owner_id=owner_id WHERE false",
|
||||
"SET ROLE messenger",
|
||||
):
|
||||
try:
|
||||
conn.execute(query)
|
||||
except psycopg.errors.InsufficientPrivilege:
|
||||
continue
|
||||
raise AssertionError("Unexpected database grant")
|
||||
|
||||
broker = redis.Redis.from_url(settings.broker_url)
|
||||
assert broker.ping()
|
||||
for op in (
|
||||
lambda: broker.config_get("appendonly"),
|
||||
lambda: broker.xrange("unrelated-key"),
|
||||
lambda: broker.get("session-user:synthetic"),
|
||||
):
|
||||
try:
|
||||
op()
|
||||
except redis.exceptions.NoPermissionError:
|
||||
continue
|
||||
raise AssertionError("Unexpected broker grant")
|
||||
|
||||
store = ObjectStore(settings)
|
||||
suffix = uuid.uuid4().hex
|
||||
key = "objects/audit/" + suffix
|
||||
store.client.put_object(Bucket=store.bucket, Key=key, Body=b"synthetic")
|
||||
assert store.client.head_object(Bucket=store.bucket, Key=key)["ContentLength"] == 9
|
||||
for op in (
|
||||
lambda: store.client.put_object(
|
||||
Bucket=store.bucket, Key="staging/audit/" + suffix, Body=b"x"
|
||||
),
|
||||
lambda: store.client.delete_object(Bucket=store.bucket, Key=key),
|
||||
lambda: store.client.get_bucket_policy(Bucket=store.bucket),
|
||||
):
|
||||
try:
|
||||
op()
|
||||
except ClientError as exc:
|
||||
assert exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403
|
||||
continue
|
||||
raise AssertionError("Unexpected S3 grant")
|
||||
|
||||
cert = Path("/run/tls/client")
|
||||
ctx = context(INSIDE_CA, cert)
|
||||
base = "https://openbao:8200/v1/"
|
||||
ct = {"Content-Type": "application/json"}
|
||||
code, raw = response(base + "auth/cert/login", ctx, b'{"name":"worker"}', ct)
|
||||
assert code == 200
|
||||
auth = json.loads(raw)["auth"]
|
||||
token = auth["client_token"]
|
||||
headers = {**ct, "X-Vault-Token": token}
|
||||
assert 0 < auth["lease_duration"] <= 120
|
||||
assert response(base + "services/data/worker", ctx, headers=headers)[0] == 200
|
||||
assert response(base + "services/data/api", ctx, headers=headers)[0] == 403
|
||||
assert response(base + "services/data/worker", ctx, b'{"data":{}}', headers)[0] == 403
|
||||
assert response(base + "auth/token/revoke-self", ctx, b"{}", headers)[0] == 204
|
||||
assert response(base + "services/data/worker", ctx, headers=headers)[0] == 403
|
||||
return {
|
||||
"postgres": {"tls": True, "job_read": True, "denied_operations": 6},
|
||||
"broker": {"tls": True, "ping": True, "denied_operations": 3},
|
||||
"s3": {"put_objects": True, "denied_operations": 3},
|
||||
"openbao": {
|
||||
"own_read": True,
|
||||
"foreign_read_and_write_denied": True,
|
||||
"token_ttl_s": auth["lease_duration"],
|
||||
"revoked_token_denied": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def old_identity():
|
||||
ctx = context(INSIDE_CA, Path("/probe/old"))
|
||||
denied(lambda: redis_ping(ctx))
|
||||
denied(
|
||||
lambda: response(
|
||||
"https://otel-collector:4319/v1/traces",
|
||||
ctx,
|
||||
b"",
|
||||
{"Content-Type": "application/x-protobuf"},
|
||||
)[0]
|
||||
)
|
||||
code, _ = response(
|
||||
"https://openbao:8200/v1/auth/cert/login",
|
||||
ctx,
|
||||
b'{"name":"worker"}',
|
||||
{"Content-Type": "application/json"},
|
||||
)
|
||||
assert code in (400, 403)
|
||||
return {"broker": "revoked", "collector": "revoked", "openbao": "revoked"}
|
||||
|
||||
|
||||
def worker_probe():
|
||||
raw = compose(
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"-T",
|
||||
"-v",
|
||||
f"{Path(__file__).resolve()}:/test/audit.py:ro",
|
||||
"--entrypoint",
|
||||
"python",
|
||||
"worker",
|
||||
"/test/audit.py",
|
||||
"--inside",
|
||||
"worker",
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
def isolated(mode, mounts):
|
||||
cmd = [
|
||||
"docker",
|
||||
"run",
|
||||
"--rm",
|
||||
"--network",
|
||||
"mtusi-lab7_backend",
|
||||
"--read-only",
|
||||
"--cap-drop",
|
||||
"ALL",
|
||||
"--security-opt",
|
||||
"no-new-privileges:true",
|
||||
"--memory",
|
||||
"256m",
|
||||
"-v",
|
||||
f"{Path(__file__).resolve()}:/test/audit.py:ro",
|
||||
"-v",
|
||||
f"{LOCAL / 'tls/trust'}:/run/tls/trust:ro",
|
||||
]
|
||||
for name, source in mounts.items():
|
||||
cmd += ["-v", f"{source}:/probe/{name}:ro"]
|
||||
image = compose("images", "-q", "api", capture_output=True, text=True).stdout.split()[0]
|
||||
raw = checked(
|
||||
*cmd, image, "python", "/test/audit.py", "--inside", mode, capture_output=True, text=True
|
||||
).stdout
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
def check_api(client):
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
attachment, _ = client.attachment(alice, chat)
|
||||
client.wait_ready(bob, attachment)
|
||||
path = "/api/v1/attachments/" + attachment["id"]
|
||||
client.call("GET", path, 404, actor=eve, headers={"X-User-Id": alice["id"]})
|
||||
client.call("GET", path + "/content", 404, actor=eve)
|
||||
client.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=bob)
|
||||
client.call("GET", "/api/v1/users/me", 401, actor=bob)
|
||||
reg = alice["registration"]
|
||||
bad = base64.b64encode((reg["username"] + ":invalid-password-123").encode()).decode()
|
||||
statuses, replicas, times = [], set(), []
|
||||
for _ in range(6):
|
||||
r = client.call(
|
||||
"POST", "/api/v1/auth/sessions", [401, 429], headers={"Authorization": "Basic " + bad}
|
||||
)
|
||||
statuses.append(r["status"])
|
||||
times.append(r["elapsed"] * 1000)
|
||||
replicas.add(r["headers"]["X-Instance-Id"])
|
||||
if r["status"] == 429:
|
||||
assert int(r["headers"]["Retry-After"]) > 0
|
||||
assert 429 in statuses and 401 in statuses and len(replicas) >= 2
|
||||
too_big = client.call(
|
||||
"POST", "/api/v1/users", 413, raw=b"x" * 66000, headers={"Content-Type": "application/json"}
|
||||
)
|
||||
assert too_big["status"] == 413
|
||||
return {
|
||||
"statuses": statuses,
|
||||
"replicas": len(replicas),
|
||||
"latency_ms": times,
|
||||
"outsider_denied": True,
|
||||
"logout_revoked": True,
|
||||
"oversize_json_denied": True,
|
||||
}
|
||||
|
||||
|
||||
def rotate(client, temp):
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
old = temp / "old"
|
||||
shutil.copytree(LOCAL / "tls/worker-client", old)
|
||||
compose("stop", "worker")
|
||||
accepted = []
|
||||
start = time.monotonic()
|
||||
try:
|
||||
for _ in range(5):
|
||||
att, data = client.attachment(alice, chat)
|
||||
accepted.append((att, data))
|
||||
checked(sys.executable, "scripts/bao_operator.py", "--revoke", "worker")
|
||||
checked(sys.executable, "scripts/pki.py", "--rotate", "worker-client")
|
||||
password = LOCAL / "broker_worker_password"
|
||||
password.write_text(secrets.token_hex(24) + "\n")
|
||||
checked(sys.executable, "scripts/bootstrap.py")
|
||||
# Load the changed ACL without restarting durable Valkey or erasing its stream.
|
||||
compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"broker",
|
||||
"sh",
|
||||
"-ec",
|
||||
"""p="$(awk '/^user operator / {print substr($NF,2)}' /run/secrets/broker.acl)"
|
||||
export REDISCLI_AUTH="$p"
|
||||
valkey-cli --user operator ACL LOAD | grep -q '^OK$'
|
||||
valkey-cli --user operator CLIENT KILL USER worker >/dev/null""",
|
||||
)
|
||||
compose("restart", "broker-tls", "otel-collector-tls")
|
||||
result = isolated("old", {"old": old})
|
||||
finally:
|
||||
checked(sys.executable, "scripts/bao_operator.py")
|
||||
compose("up", "-d", "--no-deps", "--force-recreate", "worker")
|
||||
for attachment, original in accepted:
|
||||
item = client.wait_ready(alice, attachment)
|
||||
file = client.call("GET", f"/api/v1/attachments/{item['id']}/content", actor=alice)
|
||||
assert hashlib.sha256(file["bytes"]).digest() == hashlib.sha256(original).digest()
|
||||
return {
|
||||
"old_identity": result,
|
||||
"accepted": len(accepted),
|
||||
"ready_with_original": len(accepted),
|
||||
"elapsed_seconds": round(time.monotonic() - start, 3),
|
||||
"broker_acl_reloaded": True,
|
||||
"images_rebuilt": False,
|
||||
"new_worker": worker_probe(),
|
||||
}
|
||||
|
||||
|
||||
def check_sealed(client):
|
||||
client.call("GET", "/health/ready", schema="Health")
|
||||
start = time.monotonic()
|
||||
checked(sys.executable, "scripts/bao_operator.py", "--seal")
|
||||
try:
|
||||
deadline = start + 42
|
||||
status = 200
|
||||
while time.monotonic() < deadline:
|
||||
status, _ = response("https://localhost:8443/health/ready", context(CA))
|
||||
if status == 503:
|
||||
break
|
||||
time.sleep(1)
|
||||
assert status == 503, "API did not stop serving after the credential grace period"
|
||||
unavailable = time.monotonic() - start
|
||||
finally:
|
||||
checked(sys.executable, "scripts/bao_operator.py")
|
||||
deadline = time.monotonic() + 22
|
||||
while time.monotonic() < deadline:
|
||||
status, _ = response("https://localhost:8443/health/ready", context(CA))
|
||||
if status == 200:
|
||||
return {
|
||||
"unready_after_seconds": round(unavailable, 3),
|
||||
"recovered_after_seconds": round(time.monotonic() - start, 3),
|
||||
}
|
||||
time.sleep(1)
|
||||
raise AssertionError("API readiness did not recover after unseal")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--inside", choices=("tls", "worker", "old"))
|
||||
parser.add_argument("--output", type=Path)
|
||||
parser.add_argument("--rotate", action="store_true")
|
||||
parser.add_argument("--seal", action="store_true")
|
||||
args = parser.parse_args()
|
||||
if args.inside:
|
||||
print(
|
||||
json.dumps(
|
||||
{"tls": audit_tls, "worker": worker_scope, "old": old_identity}[args.inside]()
|
||||
)
|
||||
)
|
||||
return
|
||||
if not args.output or args.output.exists():
|
||||
parser.error("choose a new --output")
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client
|
||||
|
||||
client = Client("https://localhost:8443", str(CA))
|
||||
with tempfile.TemporaryDirectory(dir=LOCAL) as folder:
|
||||
tmp = Path(folder)
|
||||
tmp.chmod(0o755)
|
||||
rogue = tmp / "rogue"
|
||||
rogue.mkdir(mode=0o755)
|
||||
checked(
|
||||
"openssl",
|
||||
"req",
|
||||
"-x509",
|
||||
"-newkey",
|
||||
"ec",
|
||||
"-pkeyopt",
|
||||
"ec_paramgen_curve:P-256",
|
||||
"-nodes",
|
||||
"-days",
|
||||
"1",
|
||||
"-subj",
|
||||
"/CN=worker-client",
|
||||
"-addext",
|
||||
"extendedKeyUsage=clientAuth",
|
||||
"-keyout",
|
||||
str(rogue / "service.key"),
|
||||
"-out",
|
||||
str(rogue / "service.crt"),
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
(rogue / "service.key").chmod(0o644)
|
||||
identities = {
|
||||
name: LOCAL / "tls" / name
|
||||
for name in ("lb-client", "worker-client", "probe-client", "grafana-client")
|
||||
}
|
||||
result = {
|
||||
"tls": isolated("tls", {**identities, "rogue": rogue}),
|
||||
"worker_scope": worker_probe(),
|
||||
"api": check_api(client),
|
||||
}
|
||||
print("PASS: verified TLS and distinct service privileges", flush=True)
|
||||
if args.rotate:
|
||||
result["worker_compromise"] = rotate(client, tmp)
|
||||
print("PASS: old identity denied; all accepted uploads recovered", flush=True)
|
||||
if args.seal:
|
||||
result["sealed"] = check_sealed(client)
|
||||
print(
|
||||
"PASS: sealed OpenBao makes API unready within a bounded grace period", flush=True
|
||||
)
|
||||
args.output.write_text(json.dumps(result, indent=2) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user