Files
2026-11-20 15:30:33 +03:00

511 lines
18 KiB
Python

#!/usr/bin/env python3
"""Real TLS/identity, ACL, secret lifecycle and application regression checks."""
import argparse
import asyncio
import base64
import io
import json
import os
import socket
import ssl
import subprocess
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
ROWS = []
def ok(name, **data):
ROWS.append({"check": name, "result": "PASS", **data})
def context(role=None):
ctx = ssl.create_default_context(cafile="/run/trust/ca.crt")
ctx.minimum_version = ssl.TLSVersion.TLSv1_3
if role:
directory = Path("/run/test-identities") / role
ctx.load_cert_chain(directory / "service.crt", directory / "service.key")
return ctx
def bao(path, role="operator", body=None, token=None, method=None):
headers = {"Content-Type": "application/json"}
if token:
headers["X-Vault-Token"] = token
req = urllib.request.Request(
"https://openbao:8200/v1/" + path,
headers=headers,
data=json.dumps(body).encode() if body is not None else None,
method=method,
)
with urllib.request.urlopen(req, context=context(role), timeout=4) as response:
raw = response.read()
return json.loads(raw) if raw else {}
def forbidden(name, call, exception, *, code=None):
try:
call()
except exception as error:
if code is not None:
assert error.code == code, name
ok(name)
else:
raise AssertionError("Forbidden operation succeeded: " + name)
def wire(host, port, role, data):
with socket.create_connection((host, port), timeout=3) as raw:
with context(role).wrap_socket(raw, server_hostname=host) as secured:
secured.sendall(data)
return secured.recv(4096)
def denied_wire(host, port, role, data):
try:
result = wire(host, port, role, data)
assert not result or result.startswith(b"HTTP/1.1 403"), (host, port, role)
except ssl.SSLError:
pass
except ConnectionResetError:
pass
# Timeouts, DNS errors and connection refused deliberately do not count.
ok(f"mtls-deny:{host}:{port}:{role or 'no-cert'}")
async def internal():
import asyncpg
import redis
import urllib3
from minio import Minio
from minio.error import S3Error
token = json.loads(Path("/run/operator/bao-init.json").read_text())["root_token"]
secrets = {}
for role in ["api", "worker", "dispatcher", "reconciler"]:
auth = bao("auth/cert/login", role, {"name": role})["auth"]
assert role in auth["policies"] and "root" not in auth["policies"]
value = bao("secret/data/messenger/" + role, role, token=auth["client_token"])
secrets[role] = value["data"]["data"]
ok("bao-own-secret:" + role)
other = "worker" if role == "api" else "api"
forbidden(
"bao-other-secret:" + role,
lambda: bao("secret/data/messenger/" + other, role, token=auth["client_token"]),
urllib.error.HTTPError,
code=403,
)
forbidden(
"bao-policy-write:" + role,
lambda: bao(
"sys/policies/acl/forbidden",
role,
{"policy": 'path "*" { capabilities=["sudo"] }'},
token=auth["client_token"],
),
urllib.error.HTTPError,
code=403,
)
renewal = bao("auth/token/renew-self", role, {}, token=auth["client_token"])
assert renewal["auth"]["lease_duration"] > 0
ok("bao-renew:" + role)
# Full chain/identity rejection at each mandatory grade-4 mTLS boundary.
for host, port, right, payload in [
(
"api",
8443,
"lb",
b"GET /health/ready HTTP/1.1\r\nHost: api\r\nConnection: close\r\n\r\n",
),
("broker", 6379, "worker", b"*1\r\n$4\r\nPING\r\n"),
(
"otel-collector",
4318,
"api",
b"POST /v1/traces HTTP/1.1\r\nHost: otel-collector\r\nContent-Length: 0\r\nContent-Type: application/x-protobuf\r\nConnection: close\r\n\r\n",
),
]:
result = wire(host, port, right, payload)
assert result and (b"200" in result[:40] if host != "broker" else b"NOAUTH" in result)
ok(f"mtls-allow:{host}:{port}:{right}")
for wrong in [None, "intruder", "foreign", "expired"] + (
["worker"] if host == "api" else []
):
denied_wire(host, port, wrong, payload)
# Loopback-only plaintext listeners cannot be reached from another container.
for host, port in [
("api", 8080),
("postgres", 15432),
("redis", 16379),
("broker", 16379),
("otel-collector", 14318),
("otel-collector", 18889),
("loki", 13100),
("tempo", 13200),
("tempo", 14318),
("prometheus", 19090),
]:
try:
connection = socket.create_connection((host, port), timeout=2)
except ConnectionRefusedError:
ok(f"no-plaintext-bypass:{host}:{port}")
else:
connection.close()
raise AssertionError("Plaintext listener is reachable: " + host)
# Each remaining matrix row has a real positive TLS probe.
for host, port, role, path in [
("redis", 6379, "api", None),
("postgres", 5432, "operator", None),
("loki", 3100, "grafana", "/ready"),
("tempo", 3200, "grafana", "/ready"),
("prometheus", 9090, "grafana", "/-/ready"),
("otel-collector", 8889, "prometheus", "/metrics"),
("otel-collector", 8888, "prometheus", "/metrics"),
("otel-collector", 13133, "operator", "/"),
("tempo", 4318, "otel-collector", "/v1/traces"),
]:
if host == "postgres":
connection = await asyncpg.connect(
os.environ["DATABASE_URL"], ssl=context("operator"), direct_tls=True
)
assert await connection.fetchval("SELECT 1") == 1
await connection.close()
elif host == "redis":
result = wire(host, port, role, b"*1\r\n$4\r\nPING\r\n")
assert b"NOAUTH" in result
else:
data = f"GET {path} HTTP/1.1\r\nHost: {host}\r\nConnection: close\r\n\r\n".encode()
if host == "tempo" and port == 4318:
data = b"POST /v1/traces HTTP/1.1\r\nHost: tempo\r\nContent-Length: 0\r\nContent-Type: application/x-protobuf\r\nConnection: close\r\n\r\n"
result = wire(host, port, role, data)
assert b"200" in result[:40] or b"503" in result[:40], host
ok(f"tls-matrix:{host}:{port}:{role}")
for role in ["worker", "dispatcher"]:
connection = await asyncpg.connect(
secrets[role]["DATABASE_URL"], ssl=context(role), direct_tls=True
)
try:
await connection.fetch("SELECT id FROM attachment_jobs LIMIT 1")
for name, sql in [
("users", "SELECT password_hash FROM users LIMIT 1"),
("ddl", "CREATE TABLE forbidden(id int)"),
("roles", "CREATE ROLE forbidden"),
]:
try:
await connection.execute(sql)
except asyncpg.InsufficientPrivilegeError:
ok(f"db-deny:{role}:{name}")
else:
raise AssertionError("Excessive DB privilege")
finally:
await connection.close()
def redis_client(role, kind):
url = secrets[role]["BROKER_URL" if kind == "broker" else "REDIS_URL"]
folder = Path("/run/test-identities") / role
return redis.Redis.from_url(
url,
ssl_ca_certs="/run/trust/ca.crt",
ssl_certfile=str(folder / "service.crt"),
ssl_keyfile=str(folder / "service.key"),
ssl_check_hostname=True,
socket_timeout=3,
)
for role, kind in [("api", "sessions"), ("worker", "broker"), ("dispatcher", "broker")]:
client = redis_client(role, kind)
assert client.ping()
forbidden(
f"redis-admin-deny:{role}",
lambda: client.config_get("*"),
redis.exceptions.NoPermissionError,
)
forbidden(
f"redis-prefix-deny:{role}",
lambda: client.get("operator:secret"),
redis.exceptions.NoPermissionError,
)
if role == "worker":
forbidden(
"consumer-cannot-publish",
lambda: client.xadd("attachment-jobs-v1", {"job": "{}"}),
redis.exceptions.NoPermissionError,
)
if role == "dispatcher":
forbidden(
"producer-cannot-consume",
lambda: client.xreadgroup(
"image-workers-v1", "probe", {"attachment-jobs-v1": ">"}, count=1
),
redis.exceptions.NoPermissionError,
)
client.close()
for role, prefix in [("api", "S3"), ("worker", "S3_WORKER")]:
data = secrets[role]
client = Minio(
"minio:9000",
access_key=data[prefix + "_ACCESS_KEY_ID"],
secret_key=data[prefix + "_SECRET_ACCESS_KEY"],
secure=True,
http_client=urllib3.PoolManager(
ca_certs="/run/trust/ca.crt", cert_reqs="CERT_REQUIRED"
),
)
assert client.bucket_exists("messenger-attachments")
try:
client.put_object("messenger-attachments", "admin/forbidden", io.BytesIO(b"x"), 1)
except S3Error as error:
assert error.code == "AccessDenied"
ok("s3-prefix-deny:" + role)
else:
raise AssertionError("Forbidden S3 prefix allowed")
if role == "api":
try:
client.put_object(
"messenger-attachments", "variants/forbidden", io.BytesIO(b"x"), 1
)
except S3Error as error:
assert error.code == "AccessDenied"
ok("api-cannot-write-variants")
else:
raise AssertionError("API can overwrite worker variants")
for host, port, path in [
("loki", 3100, "/otlp/v1/logs"),
("tempo", 3200, "/api/traces/anything"),
("prometheus", 9090, "/api/v1/admin/tsdb/delete_series"),
]:
data = f"POST {path} HTTP/1.1\r\nHost: {host}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".encode()
assert b"403" in wire(host, port, "grafana", data)[:40]
ok("grafana-write-deny:" + host)
short = bao(
"auth/token/create",
body={"policies": ["worker"], "ttl": "1s", "explicit_max_ttl": "1s"},
token=token,
)["auth"]["client_token"]
time.sleep(1.5)
forbidden(
"bao-expired-token",
lambda: bao("secret/data/messenger/worker", "worker", token=short),
urllib.error.HTTPError,
code=403,
)
short = bao("auth/cert/login", "worker", {"name": "worker"})["auth"]["client_token"]
bao("auth/token/revoke", body={"token": short}, token=token)
forbidden(
"bao-revoked-token",
lambda: bao("secret/data/messenger/worker", "worker", token=short),
urllib.error.HTTPError,
code=403,
)
print(json.dumps(ROWS))
def host_checks():
from client import Client
sys.path.insert(0, str(ROOT / "scripts"))
from securitylib import LOCAL, compose
probe = [
"run",
"--rm",
"--no-deps",
"-v",
str(ROOT / "tests") + ":/app/tests:ro",
"-v",
str(LOCAL / "identities") + ":/run/test-identities:ro",
"-v",
str(LOCAL / "operator") + ":/run/operator:ro",
"migration",
"python",
"tests/lab7_security.py",
"--internal",
]
result = compose(*probe, capture=True)
ROWS.extend(json.loads(result.splitlines()[-1]))
(ROOT / "evidence/security-internal-summary.json").write_text(
json.dumps({"checks": ROWS, "passed": len(ROWS)}, indent=2) + "\n"
)
c = Client("https://localhost:8443", str(LOCAL / "trust/ca.crt"))
with socket.create_connection(("localhost", 8443), timeout=3) as connection:
try:
tls_context = ssl.create_default_context(cafile=str(LOCAL / "trust/ca.crt"))
tls_context.wrap_socket(connection, server_hostname="wrong.invalid")
except ssl.SSLCertVerificationError:
ok("external-wrong-hostname-denied")
else:
raise AssertionError("Wrong hostname accepted")
actor = c.user("security")
# Body forged user identity never replaces verified session identity.
response = c.call(
"GET",
"/api/v1/users/me",
actor=actor,
headers={"X-User-Id": "00000000-0000-0000-0000-000000000000"},
schema="User",
)
assert response["json"]["id"] == actor["id"]
ok("bearer-not-x-user-id")
username = actor["registration"]["username"]
password = actor["registration"]["password"]
dbcode = """import asyncio,os,json,sys
import asyncpg
from argon2 import PasswordHasher
from src.secure.transport import context
async def main():
c=await asyncpg.connect(os.environ['DATABASE_URL'],ssl=context(),direct_tls=True)
data=json.load(sys.stdin)
if data['action']=='old':
value=PasswordHasher(memory_cost=19456,time_cost=2,parallelism=1).hash(data['password'])
await c.execute('UPDATE users SET password_hash=$1 WHERE username=$2',value,data['username'])
else:
value=await c.fetchval('SELECT password_hash FROM users WHERE username=$1',data['username'])
assert not PasswordHasher(memory_cost=65536,time_cost=3,parallelism=4).check_needs_rehash(value)
await c.close()
asyncio.run(main())"""
def db(action):
subprocess.run(
[
"docker",
"compose",
"run",
"--rm",
"--no-deps",
"-T",
"migration",
"python",
"-c",
dbcode,
],
cwd=ROOT,
input=json.dumps({"action": action, "username": username, "password": password}),
text=True,
check=True,
stdout=subprocess.DEVNULL,
)
db("old")
credentials = base64.b64encode((username + ":" + password).encode()).decode()
c.call(
"POST",
"/api/v1/auth/sessions",
201,
headers={"Authorization": "Basic " + credentials},
schema="Session",
)
db("check")
ok("successful-login-rehash")
# Account limit must hold while spoofed forwarding headers change.
observed = set()
blocked = False
bad = base64.b64encode((username + ":WrongPassword123!").encode()).decode()
for index in range(14):
response = c.call(
"POST",
"/api/v1/auth/sessions",
[401, 429],
headers={
"Authorization": "Basic " + bad,
"X-Real-IP": f"198.51.100.{index}",
"X-Forwarded-For": f"198.51.100.{index}",
},
)
observed.add(response["headers"]["X-Instance-Id"])
if response["status"] == 429:
assert int(response["headers"]["Retry-After"]) > 0
blocked = True
assert blocked and len(observed) >= 2
ok("shared-account-rate-limit", replicas=len(observed))
seed_ip = """import os,redis
r=redis.Redis.from_url(os.environ['REDIS_URL'],ssl_ca_certs='/run/trust/ca.crt',ssl_certfile='/run/identity/service.crt',ssl_keyfile='/run/identity/service.key',ssl_check_hostname=True)
keys=list(r.scan_iter('rl:login:ip:*'))
assert keys
for key in keys:r.set(key,120,px=3000)
"""
subprocess.run(
[
"docker",
"compose",
"run",
"--rm",
"--no-deps",
"-T",
"migration",
"python",
"-c",
seed_ip,
],
cwd=ROOT,
check=True,
stdout=subprocess.DEVNULL,
)
login = "ipprobe_" + os.urandom(4).hex()
credential = base64.b64encode((login + ":WrongPassword123!").encode()).decode()
response = c.call(
"POST",
"/api/v1/auth/sessions",
429,
headers={
"Authorization": "Basic " + credential,
"X-Real-IP": "203.0.113.55",
"X-Forwarded-For": "203.0.113.55",
},
)
assert int(response["headers"]["Retry-After"]) > 0
ok("shared-ip-rate-limit-spoofed-headers-denied")
time.sleep(3.1) # Let the synthetic IP saturation expire before later tests.
# Expire a real user session in Redis; the same bearer must stop working.
expire = """import hashlib,os,sys,json,redis
from src.secure.transport import context
x=json.load(sys.stdin)
r=redis.Redis.from_url(x['url'],ssl_ca_certs='/run/trust/ca.crt',ssl_certfile='/run/identity/service.crt',ssl_keyfile='/run/identity/service.key',ssl_check_hostname=True)
r.pexpire('session:v1:'+hashlib.sha256(x['token'].encode()).hexdigest(),1)"""
from securitylib import request, root_token
api = request("secret/data/messenger/api", token=root_token())["data"]["data"]
# The operator TLS role is allowed; session Redis auth uses the scoped API account.
subprocess.run(
[
"docker",
"compose",
"run",
"--rm",
"--no-deps",
"-T",
"migration",
"python",
"-c",
expire,
],
cwd=ROOT,
input=json.dumps({"url": api["REDIS_URL"], "token": actor["token"]}),
text=True,
check=True,
stdout=subprocess.DEVNULL,
)
time.sleep(0.05)
c.call("GET", "/api/v1/users/me", 401, actor=actor)
ok("session-expiry")
(ROOT / "evidence/security-summary.json").write_text(
json.dumps({"checks": ROWS, "passed": len(ROWS)}, indent=2) + "\n"
)
print("PASS:", len(ROWS), "security checks; evidence/security-summary.json")
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--internal", action="store_true")
args = parser.parse_args()
if args.internal:
asyncio.run(internal())
else:
host_checks()