Skeleton contracts / contracts (pull_request) Canceled after 0s
Закрывает обязательный минимум и трек 1: ротация leaf и секрета, sealed OpenBao и сохранность queued-задачи.
332 lines
14 KiB
Python
332 lines
14 KiB
Python
"""Negative checks for the trust matrix. Prints statuses only, never secret values."""
|
|
import json
|
|
import socket
|
|
import ssl
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
|
|
import pika
|
|
|
|
|
|
CA = "/probe/ca.crt"
|
|
RESULTS = []
|
|
|
|
|
|
def record(name, ok, detail):
|
|
RESULTS.append({"name": name, "ok": bool(ok), "detail": detail})
|
|
print(("PASS " if ok else "FAIL ") + name + " " + detail)
|
|
|
|
|
|
def tls_context(cert=None, key=None, ca=CA):
|
|
ctx = ssl.create_default_context(cafile=ca)
|
|
ctx.minimum_version = ssl.TLSVersion.TLSv1_2
|
|
ctx.check_hostname = True
|
|
ctx.verify_mode = ssl.CERT_REQUIRED
|
|
if cert and key:
|
|
ctx.load_cert_chain(cert, key)
|
|
return ctx
|
|
|
|
|
|
def http(url, cert=None, key=None, ca=CA, data=None, method="GET", headers=None, hostname=None):
|
|
ctx = tls_context(cert, key, ca)
|
|
request = urllib.request.Request(url, data=data, headers=headers or {}, method=method)
|
|
return urllib.request.urlopen(request, context=ctx, timeout=8)
|
|
|
|
|
|
def _handshake_rejected(exc):
|
|
if isinstance(exc, urllib.error.URLError) and exc.reason is not None:
|
|
exc = exc.reason
|
|
if isinstance(exc, (ssl.SSLError, ConnectionResetError, ConnectionAbortedError, BrokenPipeError)):
|
|
return True
|
|
text = str(exc).lower()
|
|
return "certificate" in text or "ssl" in text or "tls" in text
|
|
|
|
|
|
def expect_ssl(name, func):
|
|
try:
|
|
func()
|
|
except urllib.error.HTTPError as exc:
|
|
record(name, exc.code == 400, "http " + str(exc.code))
|
|
return
|
|
except Exception as exc:
|
|
record(name, _handshake_rejected(exc), type(exc).__name__)
|
|
return
|
|
record(name, False, "connection succeeded")
|
|
|
|
|
|
def api_identity():
|
|
try:
|
|
with http("https://api-1:8443/health/ready", "/probe/lb.crt", "/probe/lb.key") as response:
|
|
record("lb-identity-allowed", response.status == 200, "http " + str(response.status))
|
|
except Exception as exc:
|
|
record("lb-identity-allowed", False, type(exc).__name__)
|
|
try:
|
|
with http("https://api-1:8443/health/ready", "/probe/worker.crt", "/probe/worker.key") as response:
|
|
record("worker-cert-rejected-by-api", response.status == 403, "http " + str(response.status))
|
|
except urllib.error.HTTPError as exc:
|
|
record("worker-cert-rejected-by-api", exc.code == 403, "http " + str(exc.code))
|
|
except Exception as exc:
|
|
record("worker-cert-rejected-by-api", False, type(exc).__name__)
|
|
try:
|
|
with http("https://api-1:8443/health/ready", "/probe/stranger.crt", "/probe/stranger.key") as response:
|
|
record("stranger-san-rejected", response.status == 403, "http " + str(response.status))
|
|
except urllib.error.HTTPError as exc:
|
|
record("stranger-san-rejected", exc.code == 403, "http " + str(exc.code))
|
|
except Exception as exc:
|
|
record("stranger-san-rejected", False, type(exc).__name__)
|
|
expect_ssl("api-without-client-cert", lambda: http("https://api-1:8443/health/ready"))
|
|
expect_ssl(
|
|
"api-foreign-ca",
|
|
lambda: http("https://api-1:8443/health/ready", "/probe/other.crt", "/probe/other.key", ca="/probe/other-ca.crt"),
|
|
)
|
|
expect_ssl("api-expired-leaf", lambda: http("https://api-1:8443/health/ready", "/probe/expired.crt", "/probe/expired.key"))
|
|
|
|
|
|
def bypass():
|
|
sock = socket.socket()
|
|
sock.settimeout(3)
|
|
try:
|
|
sock.connect(("api-1", 8080))
|
|
record("api-local-port-not-routable", False, "connected")
|
|
except OSError as exc:
|
|
record("api-local-port-not-routable", True, type(exc).__name__)
|
|
finally:
|
|
sock.close()
|
|
|
|
|
|
def bao(cert, key):
|
|
ctx = tls_context(cert, key)
|
|
request = urllib.request.Request(
|
|
"https://openbao:8200/v1/auth/cert/login",
|
|
data=b"{}",
|
|
headers={"Content-Type": "application/json"},
|
|
method="POST",
|
|
)
|
|
with urllib.request.urlopen(request, context=ctx, timeout=8) as response:
|
|
payload = json.loads(response.read().decode("utf-8"))
|
|
token = (payload.get("auth") or {}).get("client_token")
|
|
policies = (payload.get("auth") or {}).get("policies") or []
|
|
if not token or "root" in policies:
|
|
raise RuntimeError("unexpected token shape")
|
|
return token
|
|
|
|
|
|
def bao_status(token, path, method="GET", body=None):
|
|
data = None if body is None else json.dumps(body).encode("utf-8")
|
|
request = urllib.request.Request(
|
|
"https://openbao:8200/v1/" + path,
|
|
data=data,
|
|
headers={"X-Vault-Token": token, "Content-Type": "application/json"},
|
|
method=method,
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(request, context=tls_context("/probe/api.crt", "/probe/api.key"), timeout=8) as response:
|
|
raw = response.read()
|
|
return response.status, json.loads(raw.decode("utf-8")) if raw else {}
|
|
except urllib.error.HTTPError as exc:
|
|
exc.read(256)
|
|
return exc.code, {}
|
|
|
|
|
|
def openbao_checks():
|
|
try:
|
|
token = bao("/probe/api.crt", "/probe/api.key")
|
|
status, payload = bao_status(token, "secret/data/api/db")
|
|
username = (((payload.get("data") or {}).get("data") or {}).get("username"))
|
|
record("api-reads-own-db-secret", status == 200 and username == "api_login", "http " + str(status))
|
|
denied, _ = bao_status(token, "secret/data/worker/db")
|
|
record("api-cannot-read-worker-secret", denied == 403, "http " + str(denied))
|
|
written, _ = bao_status(token, "secret/data/api/db", "POST", {"data": {"username": "nope"}})
|
|
record("api-cannot-write-secret", written in {403, 405}, "http " + str(written))
|
|
except Exception as exc:
|
|
record("api-openbao-login", False, type(exc).__name__)
|
|
try:
|
|
token = bao("/probe/worker.crt", "/probe/worker.key")
|
|
status, payload = bao_status(token, "secret/data/worker/db")
|
|
# bao_status uses the api client cert for TLS, while the token is the worker token.
|
|
username = (((payload.get("data") or {}).get("data") or {}).get("username"))
|
|
record("worker-reads-own-db-secret", status == 200 and username == "worker_login", "http " + str(status))
|
|
denied, _ = bao_status(token, "secret/data/api/db")
|
|
record("worker-cannot-read-api-secret", denied == 403, "http " + str(denied))
|
|
except Exception as exc:
|
|
record("worker-openbao-login", False, type(exc).__name__)
|
|
try:
|
|
bao("/probe/stranger.crt", "/probe/stranger.key")
|
|
record("stranger-openbao-login-denied", False, "login succeeded")
|
|
except urllib.error.HTTPError as exc:
|
|
record("stranger-openbao-login-denied", exc.code in {400, 403}, "http " + str(exc.code))
|
|
except Exception as exc:
|
|
record("stranger-openbao-login-denied", False, type(exc).__name__)
|
|
expect_ssl("openbao-without-client-cert", lambda: bao(None, None))
|
|
|
|
|
|
def rabbit_params(cert, key):
|
|
ctx = tls_context(cert, key)
|
|
return pika.ConnectionParameters(
|
|
host="rabbitmq",
|
|
port=5671,
|
|
virtual_host="/",
|
|
credentials=pika.credentials.ExternalCredentials(),
|
|
ssl_options=pika.SSLOptions(ctx, "rabbitmq"),
|
|
socket_timeout=5,
|
|
blocked_connection_timeout=5,
|
|
connection_attempts=1,
|
|
)
|
|
|
|
|
|
def rabbit_checks():
|
|
try:
|
|
connection = pika.BlockingConnection(rabbit_params("/probe/api.crt", "/probe/api.key"))
|
|
channel = connection.channel()
|
|
channel.queue_declare(queue="attachment.process.v1", passive=True)
|
|
record("api-broker-mtls", True, "passive declare")
|
|
try:
|
|
channel.basic_get("attachment.process.v1", auto_ack=False)
|
|
record("api-cannot-consume", False, "basic_get succeeded")
|
|
except pika.exceptions.ChannelClosedByBroker as exc:
|
|
record("api-cannot-consume", exc.reply_code == 403, "reply " + str(exc.reply_code))
|
|
connection.close()
|
|
except Exception as exc:
|
|
record("api-broker-mtls", False, type(exc).__name__)
|
|
try:
|
|
connection = pika.BlockingConnection(rabbit_params("/probe/worker.crt", "/probe/worker.key"))
|
|
channel = connection.channel()
|
|
channel.confirm_delivery()
|
|
try:
|
|
channel.basic_publish(exchange="", routing_key="attachment.process.v1", body=b"{}")
|
|
record("worker-cannot-publish", False, "publish succeeded")
|
|
except pika.exceptions.ChannelClosedByBroker as exc:
|
|
record("worker-cannot-publish", exc.reply_code == 403, "reply " + str(exc.reply_code))
|
|
except pika.exceptions.AMQPError as exc:
|
|
record("worker-cannot-publish", "403" in str(exc) or "access_refused" in str(exc).lower(), type(exc).__name__)
|
|
connection.close()
|
|
except Exception as exc:
|
|
record("worker-cannot-publish", False, type(exc).__name__)
|
|
try:
|
|
pika.BlockingConnection(rabbit_params("/probe/stranger.crt", "/probe/stranger.key"))
|
|
record("stranger-broker-rejected", False, "connected")
|
|
except pika.exceptions.AMQPConnectionError:
|
|
record("stranger-broker-rejected", True, "amqp auth failed")
|
|
except ssl.SSLError as exc:
|
|
record("stranger-broker-rejected", False, type(exc).__name__)
|
|
except Exception as exc:
|
|
record("stranger-broker-rejected", True, type(exc).__name__)
|
|
expect_ssl("broker-without-client-cert", lambda: pika.BlockingConnection(rabbit_params(None, None)))
|
|
expect_ssl(
|
|
"broker-foreign-ca",
|
|
lambda: pika.BlockingConnection(
|
|
pika.ConnectionParameters(
|
|
host="rabbitmq",
|
|
port=5671,
|
|
credentials=pika.credentials.ExternalCredentials(),
|
|
ssl_options=pika.SSLOptions(tls_context("/probe/other.crt", "/probe/other.key", "/probe/other-ca.crt"), "rabbitmq"),
|
|
socket_timeout=5,
|
|
connection_attempts=1,
|
|
)
|
|
),
|
|
)
|
|
|
|
|
|
def otel_checks():
|
|
body = b"{}"
|
|
try:
|
|
with http("https://otel-edge:4318/v1/traces", "/probe/api.crt", "/probe/api.key", data=body, method="POST", headers={"Content-Type": "application/json"}) as response:
|
|
record("api-collector-mtls", response.status < 500, "http " + str(response.status))
|
|
except urllib.error.HTTPError as exc:
|
|
record("api-collector-mtls", exc.code not in {401, 403}, "http " + str(exc.code))
|
|
except Exception as exc:
|
|
record("api-collector-mtls", False, type(exc).__name__)
|
|
try:
|
|
with http("https://otel-edge:4318/v1/traces", "/probe/stranger.crt", "/probe/stranger.key", data=body, method="POST", headers={"Content-Type": "application/json"}) as response:
|
|
record("stranger-collector-rejected", response.status == 403, "http " + str(response.status))
|
|
except urllib.error.HTTPError as exc:
|
|
record("stranger-collector-rejected", exc.code == 403, "http " + str(exc.code))
|
|
except Exception as exc:
|
|
record("stranger-collector-rejected", False, type(exc).__name__)
|
|
expect_ssl("collector-without-client-cert", lambda: http("https://otel-edge:4318/v1/traces", data=body, method="POST"))
|
|
|
|
|
|
def plain_backend():
|
|
sock = socket.socket()
|
|
sock.settimeout(3)
|
|
try:
|
|
sock.connect(("loki", 3100))
|
|
sock.sendall(b"GET /ready HTTP/1.0\r\nHost: loki\r\n\r\n")
|
|
data = sock.recv(32)
|
|
record("loki-plaintext-rejected", not data.startswith(b"HTTP/1.1 200") and not data.startswith(b"HTTP/1.0 200"), "bytes " + str(len(data)))
|
|
except OSError as exc:
|
|
record("loki-plaintext-rejected", False, type(exc).__name__)
|
|
finally:
|
|
sock.close()
|
|
try:
|
|
with http("https://loki:3100/ready") as response:
|
|
record("loki-tls-server-verified", response.status in {200, 503}, "http " + str(response.status))
|
|
except urllib.error.HTTPError as exc:
|
|
record("loki-tls-server-verified", exc.code in {200, 401, 403, 503}, "http " + str(exc.code))
|
|
except Exception as exc:
|
|
record("loki-tls-server-verified", False, type(exc).__name__)
|
|
|
|
|
|
def public_limits():
|
|
try:
|
|
with http("https://nginx:8443/health/ready") as response:
|
|
record("ingress-trusted-https", response.status == 200, "http " + str(response.status))
|
|
except Exception as exc:
|
|
record("ingress-trusted-https", False, type(exc).__name__)
|
|
name = "lim" + uuid.uuid4().hex[:8]
|
|
password = "correct-horse-1"
|
|
body = json.dumps({"username": name, "display_name": "Limit", "password": password}).encode("utf-8")
|
|
try:
|
|
with http("https://nginx:8443/api/v1/users", data=body, method="POST", headers={"Content-Type": "application/json"}) as response:
|
|
record("register-via-ingress", response.status == 201, "http " + str(response.status))
|
|
except Exception as exc:
|
|
record("register-via-ingress", False, type(exc).__name__)
|
|
return
|
|
import base64
|
|
|
|
auth = base64.b64encode((name + ":" + password).encode("utf-8")).decode("ascii")
|
|
saw_429 = False
|
|
retry = ""
|
|
for _ in range(12):
|
|
request = urllib.request.Request(
|
|
"https://nginx:8443/api/v1/auth/sessions",
|
|
headers={"Authorization": "Basic " + auth},
|
|
method="POST",
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(request, context=tls_context(), timeout=15) as response:
|
|
response.read()
|
|
except urllib.error.HTTPError as exc:
|
|
if exc.code == 429:
|
|
saw_429 = True
|
|
retry = exc.headers.get("Retry-After") or ""
|
|
break
|
|
record("login-rate-limit", saw_429 and retry.isdigit(), "retry " + ("set" if retry else "missing"))
|
|
huge = json.dumps({"username": "abc", "display_name": "n", "password": "p" * 70000}).encode("utf-8")
|
|
try:
|
|
http("https://nginx:8443/api/v1/users", data=huge, method="POST", headers={"Content-Type": "application/json"})
|
|
record("json-body-limit", False, "accepted")
|
|
except urllib.error.HTTPError as exc:
|
|
record("json-body-limit", exc.code == 413, "http " + str(exc.code))
|
|
except Exception as exc:
|
|
record("json-body-limit", False, type(exc).__name__)
|
|
|
|
|
|
def main():
|
|
api_identity()
|
|
bypass()
|
|
openbao_checks()
|
|
rabbit_checks()
|
|
otel_checks()
|
|
plain_backend()
|
|
public_limits()
|
|
failed = [item for item in RESULTS if not item["ok"]]
|
|
print("SUMMARY passed=%d failed=%d" % (len(RESULTS) - len(failed), len(failed)))
|
|
if failed:
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|