Files

448 lines
25 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""lab7: TLS/mTLS/identity/ACL-аудит ВСЕЙ заявленной матрицы (contracts/trust-matrix.md).
Запуск (контейнер `audit` подключён к внутренним сетям как посторонний клиент и получает
только тестовые материалы оператора — CA, intruder и сертификаты/пароли ролей):
python scripts/ops.py tls-audit # = docker compose --profile audit run --rm audit student-tests/security/tls_audit.py
Для каждой связи: положительная проверка (правильный CA/имя/identity -> успех) и
отрицательные: нет клиентского сертификата, чужой CA (с «правильным» CN), сертификат
нашего CA с запрещённой identity (intruder и чужие роли), неверное имя сервера,
plaintext на TLS-порт, ACL-нарушения. Отказ засчитывается только как TLS/соединение/
авторизационная ошибка — таймаут считается провалом проверки (не доказывает отказ).
Вывод — JSON; код возврата 1, если хоть одна проверка не совпала с ожиданием.
"""
import datetime as dt
import json
import os
import socket
import ssl
import sys
import tempfile
import time
from pathlib import Path
SEC = Path(os.environ.get("AUDIT_SECRETS", "/run/secrets"))
CA = str(SEC / "ca_crt")
RESULTS = []
# Порты можно переопределить (AUDIT_PORT_LB=...) — только для запуска вне Docker.
PORT = {k: int(os.environ.get("AUDIT_PORT_" + k, v)) for k, v in
{"LB": 8443, "API": 8443, "CACHE": 6379, "SELF": 8888}.items()}
def f(name):
return str(SEC / name)
def secret(name):
return (SEC / name).read_text(encoding="utf-8").strip()
def record(link, check, expected, ok, detail=""):
RESULTS.append({"link": link, "check": check, "expected": expected, "ok": bool(ok), "observed": str(detail)[:200]})
# ---------------------------------------------------------------- материалы
def rogue_identity(cn, sans):
"""Чужой CA и leaf с «правильным» CN/SAN (например, lb) — должен быть отклонён цепочкой."""
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import ec
from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID
now = dt.datetime.now(dt.timezone.utc)
ca_key = ec.generate_private_key(ec.SECP256R1())
ca_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "Rogue CA")])
ca = (x509.CertificateBuilder().subject_name(ca_name).issuer_name(ca_name).public_key(ca_key.public_key())
.serial_number(x509.random_serial_number()).not_valid_before(now - dt.timedelta(minutes=5))
.not_valid_after(now + dt.timedelta(days=1))
.add_extension(x509.BasicConstraints(ca=True, path_length=0), critical=True).sign(ca_key, hashes.SHA256()))
key = ec.generate_private_key(ec.SECP256R1())
leaf = (x509.CertificateBuilder().subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, cn)]))
.issuer_name(ca_name).public_key(key.public_key()).serial_number(x509.random_serial_number())
.not_valid_before(now - dt.timedelta(minutes=5)).not_valid_after(now + dt.timedelta(days=1))
.add_extension(x509.SubjectAlternativeName([x509.DNSName(s) for s in sans]), critical=False)
.add_extension(x509.ExtendedKeyUsage([ExtendedKeyUsageOID.CLIENT_AUTH, ExtendedKeyUsageOID.SERVER_AUTH]),
critical=False)
.sign(ca_key, hashes.SHA256()))
d = Path(tempfile.mkdtemp(prefix="rogue-"))
(d / "ca.crt").write_bytes(ca.public_bytes(serialization.Encoding.PEM))
(d / "leaf.crt").write_bytes(leaf.public_bytes(serialization.Encoding.PEM))
(d / "leaf.key").write_bytes(key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8,
serialization.NoEncryption()))
return str(d / "leaf.crt"), str(d / "leaf.key"), str(d / "ca.crt")
# ---------------------------------------------------------------- транспорт
def https(host, port, server_name, path="/", cert=None, key=None, cafile=CA, method="GET", body=b"",
ctype=None, timeout=5):
"""-> (kind, value): kind in http|verify_error|tls_error|reset|closed|refused|dns_error|timeout."""
ctx = ssl.create_default_context(cafile=cafile) if cafile else ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
if not cafile:
ctx.verify_mode = ssl.CERT_REQUIRED
ctx.check_hostname = True
if cert:
ctx.load_cert_chain(cert, key)
try:
raw = socket.create_connection((host, port), timeout=timeout)
except socket.gaierror as exc:
return "dns_error", exc.strerror
except ConnectionRefusedError:
return "refused", "connection refused"
except socket.timeout:
return "timeout", "connect timeout"
except OSError as exc:
return "refused", exc.strerror
try:
s = ctx.wrap_socket(raw, server_hostname=server_name)
head = "%s %s HTTP/1.1\r\nHost: %s\r\nConnection: close\r\nContent-Length: %d\r\n" % (
method, path, server_name, len(body))
if ctype:
head += "Content-Type: %s\r\n" % ctype
s.sendall(head.encode() + b"\r\n" + body)
data = b""
while True:
chunk = s.recv(65536)
if not chunk:
break
data += chunk
if data.startswith(b"HTTP/"):
return "http", int(data.split(b" ", 2)[1])
return "closed", "connection closed without HTTP response"
except ssl.SSLCertVerificationError as exc:
return "verify_error", exc.verify_message
except ssl.SSLError as exc:
return "tls_error", exc.reason or str(exc)
except (ConnectionResetError, BrokenPipeError, ConnectionAbortedError) as exc:
return "reset", type(exc).__name__
except socket.timeout:
return "timeout", "read timeout"
finally:
raw.close()
def https_body(host, port, server_name, path, cert, key):
"""Текст ответа (причина 503 у Loki/Tempo /ready) — для диагностики в отчёте."""
import urllib.request
ctx = ssl.create_default_context(cafile=CA)
ctx.load_cert_chain(cert, key)
try:
urllib.request.urlopen("https://%s:%d%s" % (server_name, port, path), context=ctx, timeout=5)
return ""
except Exception as exc: # HTTPError 503 содержит тело с причиной
body = getattr(exc, "read", lambda: b"")()
return body.decode("utf-8", "replace").strip()[:150]
def plaintext(host, port, timeout=4):
"""HTTP без TLS на TLS-порт: не должно получиться успешного HTTP-ответа 2xx."""
try:
s = socket.create_connection((host, port), timeout=timeout)
except OSError as exc:
return "refused", str(exc)
try:
s.sendall(b"GET / HTTP/1.0\r\nHost: x\r\n\r\n")
data = s.recv(4096)
if data.startswith(b"HTTP/"):
return "http", int(data.split(b" ", 2)[1])
return "closed", repr(data[:20])
except OSError as exc:
return "reset", type(exc).__name__
finally:
s.close()
REJECTED = ("verify_error", "tls_error", "reset", "closed")
def expect_rejected(link, check, res):
record(link, check, "rejected (TLS/handshake/close)", res[0] in REJECTED, res)
def expect_status(link, check, res, status=200):
record(link, check, "HTTP %d" % status, res == ("http", status), res)
def expect_no_success(link, check, res):
ok = not (res[0] == "http" and 200 <= res[1] < 300)
record(link, check, "no successful plaintext response", ok, res)
# ---------------------------------------------------------------- связи
def audit_ingress():
link = "client -> lb (public TLS)"
expect_status(link, "trusted CA + hostname lb/localhost SAN", https("lb", PORT["LB"], "lb", "/health/ready"))
res = https("lb", PORT["LB"], "lb", "/health/ready", cafile=None)
record(link, "empty trust store", "verify_error", res[0] == "verify_error", res)
res = https("lb", PORT["LB"], "evil.example", "/health/ready")
record(link, "wrong hostname", "verify_error", res[0] == "verify_error", res)
expect_no_success(link, "plaintext HTTP on :8443", plaintext("lb", PORT["LB"]))
res = https("lb", 8404, "lb", "/stats")
record(link, "HAProxy stats not exposed on network", "refused", res[0] in ("refused", "timeout"), res)
def audit_lb_api(rogue):
link = "lb -> api (mTLS, allowlist lb + CRL)"
expect_status(link, "identity lb (allowed)", https("api", PORT["API"], "api", "/health/ready", f("lb_client_pem")))
expect_rejected(link, "no client certificate", https("api", PORT["API"], "api", "/health/ready"))
expect_rejected(link, "rogue CA with CN=lb", https("api", PORT["API"], "api", "/health/ready", rogue[0], rogue[1]))
expect_rejected(link, "same CA, identity intruder", https("api", PORT["API"], "api", "/health/ready",
f("intruder_crt"), f("intruder_key")))
expect_rejected(link, "same CA, identity worker", https("api", PORT["API"], "api", "/health/ready",
f("worker_crt"), f("worker_key")))
expect_rejected(link, "same CA, identity api (replica as client)", https("api", PORT["API"], "api", "/health/ready",
f("api_crt"), f("api_key")))
expect_no_success(link, "plaintext HTTP on :8443", plaintext("api", PORT["API"]))
res = plaintext("api", 8081)
record(link, "health listener 8081 not reachable from network (loopback only)", "refused",
res[0] == "refused", res)
res = https("api", PORT["API"], "evil.example", "/health/ready", f("lb_client_pem"))
record(link, "LB-side check: server name must be api", "verify_error", res[0] == "verify_error", res)
def audit_postgres(rogue):
import psycopg
link = "api/worker -> postgres (TLS verify-full, roles)"
api_pw, wrk_pw = secret("db_api_password"), secret("db_worker_password")
def connect(user, pw, sslmode="verify-full", root=CA, host="postgres"):
return psycopg.connect(host=host, dbname="messenger", user=user, password=pw, sslmode=sslmode,
sslrootcert=root, connect_timeout=5)
try:
with connect("messenger_api", api_pw) as c:
v = c.execute("SELECT ssl, version FROM pg_stat_ssl WHERE pid = pg_backend_pid()").fetchone()
record(link, "messenger_api verify-full", "TLS session", v and v[0], v)
except Exception as exc:
record(link, "messenger_api verify-full", "TLS session", False, exc)
for name, kwargs, needle in (
("plaintext (sslmode=disable)", dict(sslmode="disable"), "no encryption"),
("rogue CA as trust root", dict(root=rogue[2]), "certificate verify failed"),
("wrong hostname (IP instead of name)", dict(host=socket.gethostbyname("postgres")), "does not match")):
try:
connect("messenger_api", api_pw, **kwargs).close()
record(link, name, "rejected", False, "connected")
except Exception as exc:
record(link, name, "rejected", needle in str(exc), str(exc).splitlines()[0])
for user, pw, sql, allowed in (
("messenger_worker", wrk_pw, "SELECT count(*) FROM attachments", True),
("messenger_worker", wrk_pw, "SELECT password_hash FROM users LIMIT 1", False),
("messenger_worker", wrk_pw, "SELECT token_hash FROM sessions LIMIT 1", False),
("messenger_api", api_pw, "CREATE TABLE audit_ddl (i int)", False),
("messenger_api", api_pw, "TRUNCATE users", False)):
try:
with connect(user, pw) as c:
c.execute(sql)
c.rollback()
record(link, "%s: %s" % (user, sql), "allowed" if allowed else "permission denied", allowed, "executed")
except Exception as exc:
record(link, "%s: %s" % (user, sql), "allowed" if allowed else "permission denied",
(not allowed) and "permission denied" in str(exc), str(exc).splitlines()[0])
def redis_client(host, port, cert=None, key=None, user=None, password=None, tls=True, cafile=CA):
import redis
kw = dict(host=host, port=port, socket_timeout=4, socket_connect_timeout=4, username=user, password=password)
if tls:
kw.update(ssl=True, ssl_ca_certs=cafile, ssl_cert_reqs="required", ssl_check_hostname=True)
if cert:
kw.update(ssl_certfile=cert, ssl_keyfile=key)
return redis.Redis(**kw)
def redis_try(client, *cmd):
import redis
try:
return "ok", client.execute_command(*cmd)
except redis.exceptions.NoPermissionError as exc:
return "noperm", str(exc)
except redis.exceptions.AuthenticationError as exc:
return "noauth", str(exc)
except redis.exceptions.ResponseError as exc:
text = str(exc)
return ("noauth" if "NOAUTH" in text or "WRONGPASS" in text else
"noperm" if "NOPERM" in text else "error"), text
except (redis.exceptions.ConnectionError, redis.exceptions.TimeoutError, OSError) as exc:
return "conn_error", str(exc)[:120]
def audit_cache(rogue):
link = "api -> redis cache (mTLS + ACL)"
pw = secret("cache_api_password")
api = redis_client("redis", PORT["CACHE"], f("api_crt"), f("api_key"), "messenger_api", pw)
res = redis_try(api, "SET", "sess:audit-probe", "1", "PX", "1000")
record(link, "messenger_api SET sess:*", "ok", res[0] == "ok", res)
for cmd in (("SET", "other:key", "1"), ("FLUSHALL",), ("CONFIG", "GET", "*"), ("KEYS", "*")):
res = redis_try(api, *cmd)
record(link, "messenger_api %s" % " ".join(cmd), "NOPERM", res[0] == "noperm", res)
res = redis_try(redis_client("redis", PORT["CACHE"], f("api_crt"), f("api_key")), "GET", "sess:x")
record(link, "cert without AUTH (default user off)", "NOAUTH", res[0] == "noauth", res)
res = redis_try(redis_client("redis", PORT["CACHE"], user="messenger_api", password=pw), "PING")
record(link, "no client certificate", "connection rejected", res[0] == "conn_error", res)
res = redis_try(redis_client("redis", PORT["CACHE"], rogue[0], rogue[1], "messenger_api", pw), "PING")
record(link, "rogue CA client certificate", "connection rejected", res[0] == "conn_error", res)
res = redis_try(redis_client("redis", PORT["CACHE"], user="messenger_api", password=pw, tls=False), "PING")
record(link, "plaintext", "connection rejected", res[0] == "conn_error", res)
def audit_broker(rogue):
link = "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL"
ap, wp = secret("broker_api_password"), secret("broker_worker_password")
for name, cert, key in (("no client certificate", None, None), ("rogue CA with CN=api", rogue[0], rogue[1]),
("same CA, identity intruder", f("intruder_crt"), f("intruder_key")),
("same CA, identity lb", None, None), ("same CA, identity prometheus",
f("prometheus_crt"), f("prometheus_key"))):
if name.endswith("identity lb"):
cert = key = f("lb_client_pem")
res = redis_try(redis_client("broker", 6380, cert, key, "api_producer", ap), "PING")
record(link, name + " (with valid producer password)", "connection rejected", res[0] == "conn_error", res)
res = redis_try(redis_client("broker", 6380, f("api_crt"), f("api_key")), "PING")
record(link, "identity api without AUTH", "NOAUTH", res[0] == "noauth", res)
producer = redis_client("broker", 6380, f("api_crt"), f("api_key"), "api_producer", ap)
res = redis_try(producer, "PING")
record(link, "api_producer PING", "ok", res[0] == "ok", res)
for cmd in (("XREADGROUP", "GROUP", "attachment-workers", "audit", "COUNT", "1", "STREAMS", "attachments.jobs", ">"),
("XACK", "attachments.jobs", "attachment-workers", "0-1"), ("XDEL", "attachments.jobs", "0-1"),
("DEL", "attachments.jobs"), ("FLUSHALL",), ("XADD", "audit.other", "*", "k", "v")):
res = redis_try(producer, *cmd)
record(link, "api_producer %s" % cmd[0] + (" other key" if cmd[1:2] == ("audit.other",) else ""),
"NOPERM", res[0] == "noperm", res)
consumer = redis_client("broker", 6380, f("worker_crt"), f("worker_key"), "worker_consumer", wp)
res = redis_try(consumer, "XLEN", "attachments.jobs")
record(link, "worker_consumer XLEN", "ok", res[0] == "ok", res)
for cmd in (("SET", "x", "1"), ("FLUSHALL",), ("CONFIG", "GET", "*"), ("DEL", "attachments.jobs")):
res = redis_try(consumer, *cmd)
record(link, "worker_consumer %s" % cmd[0], "NOPERM", res[0] == "noperm", res)
res = redis_try(redis_client("broker", 6379, user="api_producer", password=ap, tls=False), "PING")
record(link, "Redis port 6379 not reachable from network (loopback only)", "connection refused",
res[0] == "conn_error", res)
res = redis_try(redis_client("broker", 6380, user="api_producer", password=ap, tls=False), "PING")
record(link, "plaintext to gate", "connection rejected", res[0] == "conn_error", res)
def audit_s3():
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError
link = "api/worker -> s3 (TLS, scoped identities)"
expect_status(link, "TLS verify-full GET /healthz", https("s3", 8333, "s3", "/healthz"))
res = https("s3", 8333, "evil.example", "/healthz")
record(link, "wrong hostname", "verify_error", res[0] == "verify_error", res)
expect_no_success(link, "plaintext on :8333", plaintext("s3", 8333))
for port, what in ((8334, "S3 API behind gate"), (18334, "S3 gateway gRPC (unauthenticated)"),
(18333, "S3 gRPC default port"), (9333, "master"), (8888, "filer"), (18888, "filer gRPC")):
res = plaintext("s3", port)
record(link, "%s :%d not reachable from network" % (what, port), "refused", res[0] == "refused", res)
def client(prefix):
return boto3.client("s3", endpoint_url="https://s3:8333", region_name="us-east-1", verify=CA,
aws_access_key_id=secret(prefix + "access_key_id"),
aws_secret_access_key=secret(prefix + "secret_access_key"),
config=Config(signature_version="s3v4", s3={"addressing_style": "path"},
retries={"max_attempts": 1}))
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
api, wrk = client("s3_"), client("s3_worker_")
cases = (
("api PUT staging/", api, lambda c: c.put_object(Bucket=bucket, Key="staging/_audit-probe", Body=b"a"), True),
("api DELETE staging/", api, lambda c: c.delete_object(Bucket=bucket, Key="staging/_audit-probe"), True),
("api PUT final/ (worker prefix)", api,
lambda c: c.put_object(Bucket=bucket, Key="final/_audit-forged", Body=b"a"), False),
("api LIST bucket", api, lambda c: c.list_objects_v2(Bucket=bucket, MaxKeys=1), False),
("api CREATE bucket", api, lambda c: c.create_bucket(Bucket="audit-forbidden"), False),
("worker LIST bucket", wrk, lambda c: c.list_objects_v2(Bucket=bucket, MaxKeys=1), True),
("worker PUT outside its prefixes", wrk, lambda c: c.put_object(Bucket=bucket, Key="_other/_audit", Body=b"a"),
False),
("worker DELETE bucket", wrk, lambda c: c.delete_bucket(Bucket=bucket), False),
)
for label, c, op, allowed in cases:
try:
op(c)
record(link, label, "allowed" if allowed else "AccessDenied", allowed, "allowed")
except ClientError as exc:
code = exc.response.get("Error", {}).get("Code")
record(link, label, "allowed" if allowed else "AccessDenied",
(not allowed) and code in ("AccessDenied", "Forbidden"), code)
def audit_collector(rogue):
link = "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector"
body, ct = b"{}", "application/json"
for name, cert, key in (("identity api", f("api_crt"), f("api_key")),
("identity worker", f("worker_crt"), f("worker_key"))):
expect_status(link, name + " POST /v1/logs", https("otel-collector", 4318, "otel-collector", "/v1/logs",
cert, key, method="POST", body=body, ctype=ct))
for name, cert, key in (("no client certificate", None, None), ("rogue CA with CN=api", rogue[0], rogue[1]),
("same CA, identity intruder", f("intruder_crt"), f("intruder_key")),
("same CA, identity prometheus", f("prometheus_crt"), f("prometheus_key")),
("same CA, identity lb", f("lb_client_pem"), None)):
expect_rejected(link, name + " POST /v1/logs", https("otel-collector", 4318, "otel-collector", "/v1/logs",
cert, key, method="POST", body=body, ctype=ct))
expect_no_success(link, "plaintext on :4318", plaintext("otel-collector", 4318))
link2 = "prometheus -> otel-gate metrics (allowlist prometheus)"
expect_status(link2, "identity prometheus /metrics :8889",
https("otel-collector", 8889, "otel-collector", "/metrics", f("prometheus_crt"), f("prometheus_key")))
expect_status(link2, "identity prometheus /metrics :8888 (self)",
https("otel-collector", PORT["SELF"], "otel-collector", "/metrics", f("prometheus_crt"), f("prometheus_key")))
expect_rejected(link2, "identity api /metrics :8889",
https("otel-collector", 8889, "otel-collector", "/metrics", f("api_crt"), f("api_key")))
for port in (14318, 18889, 18888, 13133):
res = plaintext("otel-collector", port)
record(link2, "collector loopback port %d not reachable" % port, "refused", res[0] == "refused", res)
def audit_backends(rogue):
for host, port, path, link in (
("loki", 3100, "/ready", "collector/grafana -> loki (mTLS)"),
("tempo", 3200, "/ready", "grafana -> tempo (mTLS)"),
("prometheus", 9090, "/-/ready", "grafana -> prometheus (mTLS)")):
# Loki/Tempo после старта несколько десятков секунд отвечают 503 (кольцо ingester'а
# прогревается). Это не про TLS: ждём готовности до 90 с, иначе показываем причину.
res = https(host, port, host, path, f("grafana_crt"), f("grafana_key"))
deadline = time.monotonic() + 90
while res == ("http", 503) and time.monotonic() < deadline:
time.sleep(5)
res = https(host, port, host, path, f("grafana_crt"), f("grafana_key"))
if res == ("http", 503):
res = ("http", 503, https_body(host, port, host, path, f("grafana_crt"), f("grafana_key")))
expect_status(link, "client cert grafana (TLS ok, service ready)", res)
expect_rejected(link, "no client certificate", https(host, port, host, path))
expect_rejected(link, "rogue CA client certificate", https(host, port, host, path, rogue[0], rogue[1]))
expect_no_success(link, "plaintext", plaintext(host, port))
link = "collector -> tempo OTLP gRPC (mTLS)"
expect_rejected(link, "no client certificate", https("tempo", 4317, "tempo", "/"))
def audit_openbao():
link = "agents -> openbao (TLS + AppRole)"
res = https("openbao", 8200, "openbao", "/v1/sys/health")
record(link, "TLS verify-full /v1/sys/health", "HTTP 200 (unsealed)", res == ("http", 200), res)
res = https("openbao", 8200, "evil.example", "/v1/sys/health")
record(link, "wrong hostname", "verify_error", res[0] == "verify_error", res)
expect_no_success(link, "plaintext on :8200", plaintext("openbao", 8200))
res = https("openbao", 8200, "openbao", "/v1/kv/data/messenger/api")
record(link, "read KV without token", "HTTP 403", res == ("http", 403), res)
def main():
rogue = rogue_identity("lb", ["lb", "api"])
for fn in (audit_ingress, lambda: audit_lb_api(rogue), lambda: audit_postgres(rogue), lambda: audit_cache(rogue),
lambda: audit_broker(rogue), audit_s3, lambda: audit_collector(rogue), lambda: audit_backends(rogue),
audit_openbao):
try:
fn()
except Exception as exc: # одна сломанная связь не прячет остальные
record("audit", getattr(fn, "__name__", "section"), "no exception", False, repr(exc))
failed = [r for r in RESULTS if not r["ok"]]
print(json.dumps({"ok": not failed, "total": len(RESULTS), "failed": len(failed), "checks": RESULTS},
ensure_ascii=False, indent=2))
for r in RESULTS:
print("%s %-62s %-52s %s" % ("PASS" if r["ok"] else "FAIL", r["link"][:62], r["check"][:52], r["observed"][:60]),
file=sys.stderr)
return 1 if failed else 0
if __name__ == "__main__":
sys.exit(main())