Защитить мессенджер TLS и OpenBao и проверить отзыв прав worker

This commit is contained in:
2026-11-06 18:32:19 +03:00
parent 20afc2c790
commit 8f82949094
89 changed files with 8323 additions and 76 deletions
+145
View File
@@ -0,0 +1,145 @@
#!/usr/bin/env python3
"""Operator-only init, unseal, service policies and scoped secrets."""
import argparse
import json
import os
import ssl
import time
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
from bootstrap import ROLES, SECRETS
ADDRESS = f"https://localhost:{int(os.getenv('BAO_HOST_PORT', '8207'))}/v1/"
CA = SECRETS / "tls/trust/ca.crt"
RECEIPT = SECRETS / "operator/bao.json"
def call(path, data=None, *, token=None, method=None):
headers = {"Content-Type": "application/json"}
if token:
headers["X-Vault-Token"] = token
req = Request(
ADDRESS + path,
json.dumps(data).encode() if data is not None else None,
headers=headers,
method=method,
)
with urlopen(req, context=ssl.create_default_context(cafile=CA), timeout=30) as response:
body = response.read()
return json.loads(body) if body else {}
def secret(name):
return (SECRETS / name).read_text().strip()
def config(role):
ca = "/run/tls/trust/ca.crt"
db = (
f"postgresql://{role}:{secret(f'db_{role}_password')}@postgres:5432/messenger"
f"?sslmode=verify-full&sslrootcert={ca}"
)
cache = (
f"rediss://{role}:{secret(f'cache_{role}_password')}@valkey:6379/0"
f"?ssl_ca_certs={ca}&ssl_check_hostname=True"
if role == "api"
else ""
)
broker = (
f"rediss://{role}:{secret(f'broker_{role}_password')}@broker:6380/0"
f"?ssl_ca_certs={ca}&ssl_check_hostname=True&ssl_certfile=/run/tls/client/service.crt"
"&ssl_keyfile=/run/tls/client/service.key"
if role in ("api", "worker", "relay")
else ""
)
return {
"database_url": db,
"cache_url": cache,
"broker_url": broker,
"s3_access_key": secret(f"s3_{role}_access_key_id") if role != "relay" else "",
"s3_secret_key": secret(f"s3_{role}_secret_access_key") if role != "relay" else "",
}
def setup():
os.umask(0o077)
for attempt in range(20):
try:
state = call("sys/init")
break
except URLError:
if attempt == 19:
raise
time.sleep(0.5)
if not state["initialized"]:
result = call("sys/init", {"secret_shares": 1, "secret_threshold": 1}, method="PUT")
RECEIPT.write_text(
json.dumps({"root_token": result["root_token"], "unseal_key": result["keys_base64"][0]})
)
if not RECEIPT.exists():
raise SystemExit("This OpenBao volume requires its original operator receipt")
receipt = json.loads(RECEIPT.read_text())
if call("sys/seal-status")["sealed"]:
call("sys/unseal", {"key": receipt["unseal_key"]}, method="PUT")
for attempt in range(30):
try:
call("sys/health")
break
except HTTPError as exc:
if exc.code not in (429, 503) or attempt == 29:
raise
time.sleep(0.5)
token = receipt["root_token"]
if "cert/" not in call("sys/auth", token=token):
call("sys/auth/cert", {"type": "cert"}, token=token)
if "services/" not in call("sys/mounts", token=token):
call("sys/mounts/services", {"type": "kv", "options": {"version": "2"}}, token=token)
for role in ROLES:
policy = (
f'path "services/data/{role}" {{ capabilities = ["read"] }}\n'
'path "auth/token/revoke-self" { capabilities = ["update"] }\n'
)
call("sys/policies/acl/" + role, {"policy": policy}, token=token, method="PUT")
cert = secret(f"tls/{role}-client/service.crt")
call(
f"auth/cert/certs/{role}",
{
"certificate": cert,
"allowed_uri_sans": f"urn:mtusi:lab7:{role}-client",
"token_policies": [role],
"token_ttl": "90s",
"token_max_ttl": "120s",
"token_no_default_policy": True,
},
token=token,
)
call(f"services/data/{role}", {"data": config(role)}, token=token)
print("OpenBao initialized and unsealed; four scoped identities configured.")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--revoke", choices=ROLES)
parser.add_argument("--seal", action="store_true")
args = parser.parse_args()
try:
if args.revoke or args.seal:
token = json.loads(RECEIPT.read_text())["root_token"]
if args.revoke:
call(f"auth/cert/certs/{args.revoke}", token=token, method="DELETE")
print(f"Certificate role {args.revoke} revoked.")
else:
call("sys/seal", {}, token=token, method="PUT")
print("OpenBao sealed.")
else:
setup()
except HTTPError as exc:
raise SystemExit(f"OpenBao operator action failed (HTTP {exc.code})") from None
except (URLError, TimeoutError):
raise SystemExit("OpenBao unavailable over verified TLS") from None
if __name__ == "__main__":
main()
+125
View File
@@ -0,0 +1,125 @@
#!/usr/bin/env python3
"""Generate local operator credentials and TLS files for this stand."""
import json
import os
import secrets
import shutil
from pathlib import Path
import pki
ROOT = Path(__file__).resolve().parents[1]
SECRETS = ROOT / "secrets"
ROLES = ("api", "worker", "relay", "reconciler")
S3_ROLES = ("api", "worker", "reconciler", "test")
def ensure(name, value):
path = SECRETS / name
if not path.exists():
path.write_text(value + "\n")
path.chmod(0o644)
return path.read_text().strip()
def initialize():
os.umask(0o077)
SECRETS.mkdir(mode=0o700, exist_ok=True)
pki.init_ca()
for name in (*pki.SERVERS, *pki.CLIENTS):
pki.issue(name)
env = ROOT / ".env"
if not env.exists():
shutil.copyfile(ROOT / ".env.example", env)
for name in ("postgres_password", "grafana_password", "s3_root_password"):
ensure(name, secrets.token_hex(24))
ensure("s3_root_user", "operator-" + secrets.token_hex(8))
for role in S3_ROLES:
ensure(f"s3_{role}_access_key_id", "svc-" + secrets.token_hex(8))
ensure(f"s3_{role}_secret_access_key", secrets.token_hex(24))
for role in ROLES:
for store in ("db", "cache", "broker"):
ensure(f"{store}_{role}_password", secrets.token_hex(24))
write_acls()
provision_datasources()
print("Local PKI, credentials and provisioning ready.")
def write_acls():
cache_ops = "reset on ~session-user:* +get +set +del +ping +hello +client|setinfo"
broker_shared = "+ping +hello +client|setinfo"
worker_ops = (
"~attachment-jobs +xreadgroup +xautoclaim +xgroup|create "
"+xack +xdel +eval +multi +exec (~invalid-jobs +xadd)"
)
for store, rules in {
"cache": {"api": cache_ops, "operator": "on ~* +@all"},
"broker": {
"api": f"on {broker_shared}",
"relay": f"on ~attachment-jobs +xadd {broker_shared}",
"worker": f"on {worker_ops} {broker_shared}",
"operator": "on ~* +@all",
},
}.items():
lines = ["user default off"]
for role, permissions in rules.items():
password = ensure(f"{store}_{role}_password", secrets.token_hex(24))
lines.append(f"user {role} {permissions} >{password}")
target = SECRETS / f"{store}.acl"
target.write_text("\n".join(lines) + "\n")
target.chmod(0o644)
def provision_datasources():
trust = (pki.TLS / "trust" / "ca.crt").read_text()
identity = pki.TLS / "grafana-client"
secure = {
"tlsCACert": trust,
"tlsClientCert": (identity / "service.crt").read_text(),
"tlsClientKey": (identity / "service.key").read_text(),
}
datasources = []
for name, port in (("prometheus", 9090), ("loki", 3100), ("tempo", 3200)):
config = {"tlsAuth": True, "tlsAuthWithCACert": True}
if name == "loki":
config["derivedFields"] = [
{
"name": "Trace",
"matcherType": "label",
"matcherRegex": "trace_id",
"datasourceUid": "tempo",
"url": "$${__value.raw}",
}
]
if name == "tempo":
config["tracesToLogsV2"] = {
"datasourceUid": "loki",
"spanStartTimeShift": "-2m",
"spanEndTimeShift": "2m",
"filterByTraceID": True,
"tags": [{"key": "service.name", "value": "service_name"}],
}
datasources.append(
{
"name": name.title(),
"uid": name,
"type": name,
"access": "proxy",
"url": f"https://{name}:{port}",
"editable": False,
"isDefault": name == "prometheus",
"jsonData": config,
"secureJsonData": secure,
}
)
directory = SECRETS / "grafana-datasources"
directory.mkdir(exist_ok=True)
directory.chmod(0o755)
target = directory / "local.yaml"
target.write_text(json.dumps({"apiVersion": 1, "datasources": datasources}, indent=2))
target.chmod(0o644)
if __name__ == "__main__":
initialize()
+81
View File
@@ -0,0 +1,81 @@
"""Read-only discovery and scoped Compose operations for the local lab5 stack."""
import json
import ssl
import subprocess
import time
import urllib.error
import urllib.request
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def compose(*args, env=None):
return subprocess.run(
["docker", "compose", "--env-file", ".env", *args],
cwd=ROOT,
env=env,
check=True,
capture_output=True,
text=True,
timeout=90,
).stdout
def inspect(container):
template = (
'{"id":{{json .Id}},"started_at":{{json .State.StartedAt}},'
'"pid":{{.State.Pid}},"running":{{json .State.Running}},'
'"image":{{json .Image}},"memory":{{.HostConfig.Memory}},'
'"nano_cpus":{{.HostConfig.NanoCpus}},'
'"published_ports":{{json .HostConfig.PortBindings}}}'
)
r = subprocess.run(
["docker", "inspect", "--format", template, container],
capture_output=True,
text=True,
check=True,
timeout=5,
)
return json.loads(r.stdout)
def instance(container):
code = "from pathlib import Path; print(Path('/tmp/api-instance-id').read_text())"
r = subprocess.run(
["docker", "exec", container, "python", "-c", code],
capture_output=True,
text=True,
check=True,
timeout=3,
)
value = inspect(container)
value["instance"] = r.stdout.strip()
return value
def instances():
ids = compose("ps", "-q", "api").split()
with ThreadPoolExecutor(max_workers=8) as pool:
return list(pool.map(instance, ids))
def wait_lb(base="https://localhost:8443", count=2, timeout=45):
deadline, seen = time.monotonic() + timeout, set()
tls = ssl.create_default_context(cafile=ROOT / "secrets/tls/trust/ca.crt")
while time.monotonic() < deadline:
try:
current = {value["instance"] for value in instances()}
if len(current) != count:
time.sleep(0.2)
continue
with urllib.request.urlopen(base + "/health/ready", timeout=2, context=tls) as response:
seen.add(response.headers["X-Instance-Id"])
if current <= seen:
return sorted(current)
except (OSError, urllib.error.URLError, subprocess.SubprocessError):
pass
time.sleep(0.1)
raise RuntimeError("Expected healthy API replicas were not observed through LB")
+190
View File
@@ -0,0 +1,190 @@
#!/usr/bin/env python3
"""Operator PKI. Only service-specific files are mounted into containers."""
import argparse
import os
import shutil
import subprocess
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1] / "secrets"
CA = ROOT / "operator" / "ca"
TLS = ROOT / "tls"
SERVERS = {
"lb": "DNS:localhost,DNS:lb,IP:127.0.0.1",
"api": "DNS:api",
"postgres": "DNS:postgres",
"valkey": "DNS:valkey",
"broker": "DNS:broker",
"s3": "DNS:s3",
"openbao": "DNS:openbao,DNS:localhost,IP:127.0.0.1",
"otel-collector": "DNS:otel-collector",
"prometheus": "DNS:prometheus",
"loki": "DNS:loki",
"tempo": "DNS:tempo",
"grafana": "DNS:grafana,DNS:localhost,IP:127.0.0.1",
}
CLIENTS = (
"lb-client",
"api-client",
"worker-client",
"relay-client",
"reconciler-client",
"collector-client",
"prometheus-client",
"grafana-client",
"probe-client",
)
def openssl(*args):
subprocess.run(
["openssl", *map(str, args)],
cwd=CA,
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
def init_ca():
CA.mkdir(parents=True, exist_ok=True)
TLS.mkdir(parents=True, exist_ok=True)
if (CA / "ca.key").exists():
return
(CA / "issued").mkdir(exist_ok=True)
for name, value in {
"index.txt": "",
"index.txt.attr": "unique_subject = no\n",
"serial": "1000\n",
"crlnumber": "1000\n",
}.items():
(CA / name).write_text(value)
openssl(
"req",
"-new",
"-newkey",
"ec",
"-pkeyopt",
"ec_paramgen_curve:P-256",
"-nodes",
"-x509",
"-days",
90,
"-subj",
"/CN=MTUSI lab7 local root",
"-addext",
"basicConstraints=critical,CA:TRUE,pathlen:0",
"-addext",
"keyUsage=critical,keyCertSign,cRLSign",
"-keyout",
"ca.key",
"-out",
"ca.crt",
)
(CA / "ca.cnf").write_text("""[ca]
default_ca = signer
[signer]
database = index.txt
new_certs_dir = issued
certificate = ca.crt
private_key = ca.key
serial = serial
crlnumber = crlnumber
default_md = sha256
default_days = 7
default_crl_days = 30
policy = policy
unique_subject = no
[policy]
commonName = supplied
""")
refresh_crl()
def refresh_crl():
openssl("ca", "-config", "ca.cnf", "-gencrl", "-out", "current.crl")
trust = TLS / "trust"
trust.mkdir(exist_ok=True)
trust.chmod(0o755)
for source, name in (("ca.crt", "ca.crt"), ("current.crl", "revoked.crl")):
data = (CA / source).read_bytes()
(trust / name).write_bytes(data)
(trust / name).chmod(0o644)
def issue(name, *, rotate=False):
target = TLS / name
target.mkdir(exist_ok=True)
target.chmod(0o755)
key, cert = target / "service.key", target / "service.crt"
if cert.exists() and not rotate:
return
if rotate and cert.exists():
openssl("ca", "-config", "ca.cnf", "-revoke", cert)
purpose = "serverAuth" if name in SERVERS else "clientAuth"
san = SERVERS.get(name, "URI:urn:mtusi:lab7:" + name)
(CA / "leaf.cnf").write_text(
"[leaf]\nbasicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature\n"
f"extendedKeyUsage={purpose}\nsubjectAltName={san}\n"
"subjectKeyIdentifier=hash\nauthorityKeyIdentifier=keyid,issuer\n"
)
openssl(
"req",
"-new",
"-newkey",
"ec",
"-pkeyopt",
"ec_paramgen_curve:P-256",
"-nodes",
"-subj",
"/CN=" + name,
"-keyout",
"leaf.key",
"-out",
"leaf.csr",
)
openssl(
"ca",
"-config",
"ca.cnf",
"-batch",
"-notext",
"-extfile",
"leaf.cnf",
"-extensions",
"leaf",
"-in",
"leaf.csr",
"-out",
"leaf.crt",
)
# A directory mount sees atomic leaf replacement. Host parent remains 0700.
for source, dest in (("leaf.key", key), ("leaf.crt", cert)):
temp = dest.with_suffix(".new")
shutil.move(CA / source, temp)
temp.chmod(0o644)
temp.replace(dest)
(target / "service.pem").write_bytes(cert.read_bytes() + key.read_bytes())
(target / "service.pem").chmod(0o644)
if rotate:
refresh_crl()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--rotate", choices=[*SERVERS, *CLIENTS])
args = parser.parse_args()
os.umask(0o077)
ROOT.mkdir(exist_ok=True, mode=0o700)
init_ca()
if args.rotate:
issue(args.rotate, rotate=True)
else:
for name in (*SERVERS, *CLIENTS):
issue(name)
print("Local PKI ready.")
if __name__ == "__main__":
main()
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose stop api worker relay lb
docker compose restart --timeout 7 postgres valkey s3 broker openbao
python3 scripts/bao_operator.py
docker compose restart broker-tls
docker compose up -d --no-build --wait --wait-timeout 90 api worker relay lb
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env python3
"""Sequential process replacement; at least one existing API stays running."""
import argparse
import json
import subprocess
from cluster import compose, instances, wait_lb
def replace_all(base="http://127.0.0.1:8084"):
original = instances()
if len(original) < 2:
raise RuntimeError("Rolling replacement requires at least two healthy replicas")
wait_lb(base, len(original))
replacements = []
for current in original:
subprocess.run(
["docker", "stop", "--time", "5", current["id"]],
check=True,
capture_output=True,
timeout=8,
)
subprocess.run(["docker", "rm", current["id"]], check=True, capture_output=True)
compose(
"up", "-d", "--no-deps", "--no-recreate", "--scale", "api=" + str(len(original)), "api"
)
aliases = wait_lb(base, len(original))
if current["instance"] in aliases:
raise RuntimeError("Old API process is still serving after replacement")
replacements.append(
{
"old_container": current["id"],
"old_instance": current["instance"],
"live_instances": aliases,
}
)
return replacements
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8084")
args = parser.parse_args()
print(json.dumps(replace_all(args.base_url)))
if __name__ == "__main__":
main()
+471
View File
@@ -0,0 +1,471 @@
#!/usr/bin/env python3
"""Reproducible negative checks and worker revocation on the local lab7 stand."""
import argparse
import base64
import hashlib
import json
import secrets
import shutil
import socket
import ssl
import subprocess
import sys
import tempfile
import time
import uuid
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
ROOT = Path(__file__).resolve().parents[1]
LOCAL = ROOT / "secrets"
CA = LOCAL / "tls/trust/ca.crt"
INSIDE_CA = "/run/tls/trust/ca.crt"
def checked(*command, **kwargs):
try:
return subprocess.run(command, cwd=ROOT, check=True, **kwargs)
except subprocess.CalledProcessError as exc:
if exc.stderr:
sys.stderr.write(exc.stderr)
raise
def compose(*args, **kwargs):
return checked("docker", "compose", *args, **kwargs)
def context(ca, certificate=None):
ctx = ssl.create_default_context(cafile=ca)
if certificate:
ctx.load_cert_chain(certificate / "service.crt", certificate / "service.key")
return ctx
def response(url, ctx, data=None, headers=None, method=None):
request = Request(url, data, headers or {}, method=method)
try:
with urlopen(request, context=ctx, timeout=4) as r:
return r.status, r.read(262144)
except HTTPError as e:
return e.code, b""
def denied(action):
try:
status = action()
assert status in (401, 403) or status == b"", "Unexpected success in negative probe"
except (ssl.SSLError, ConnectionResetError, BrokenPipeError, URLError) as exc:
# Preserve DNS/connectivity errors as real test failures.
cause = exc.reason if isinstance(exc, URLError) else exc
if not isinstance(cause, (ssl.SSLError, ConnectionResetError, BrokenPipeError)):
raise
def redis_ping(ctx):
with socket.create_connection(("broker", 6380), timeout=3) as raw:
with ctx.wrap_socket(raw, server_hostname="broker") as tls:
tls.sendall(b"*1\r\n$4\r\nPING\r\n")
return tls.recv(256)
def audit_tls():
results = {}
endpoints = {
"api": (lambda ctx: response("https://api:8080/health/ready", ctx)[0], "lb-client"),
"collector": (
lambda ctx: response(
"https://otel-collector:4319/v1/traces",
ctx,
b"",
{"Content-Type": "application/x-protobuf"},
)[0],
"worker-client",
),
"broker": (redis_ping, "worker-client"),
}
for name, (probe, identity) in endpoints.items():
status = probe(context(INSIDE_CA, Path("/probe") / identity))
assert status == 200 or status.startswith(b"-NOAUTH")
for cert in (None, Path("/probe/probe-client"), Path("/probe/rogue")):
denied(lambda probe=probe, cert=cert: probe(context(INSIDE_CA, cert)))
results[name] = [
"allowed",
"no_client_denied",
"same_ca_wrong_role_denied",
"rogue_ca_denied",
]
for host, port in (
("broker", 6379),
("otel-collector", 4318),
("prometheus", 19090),
("loki", 13100),
("tempo", 13200),
("tempo", 14318),
("tempo", 9095),
("loki", 9095),
):
try:
with socket.create_connection((host, port), timeout=1):
raise AssertionError("Reachable plaintext " + host)
except (ConnectionRefusedError, TimeoutError):
pass
results["network_plaintext_bypass"] = "denied"
results["grafana_backends"] = {}
for host, port, path in (
("prometheus", 9090, "/api/v1/status/buildinfo"),
("loki", 3100, "/loki/api/v1/status/buildinfo"),
("tempo", 3200, "/api/echo"),
):
endpoint = f"https://{host}:{port}{path}"
assert response(endpoint, context(INSIDE_CA, Path("/probe/grafana-client")))[0] == 200
denied(
lambda endpoint=endpoint: response(
endpoint, context(INSIDE_CA, Path("/probe/probe-client"))
)[0]
)
results["grafana_backends"][host] = "verified_tls_identity_allow_and_deny"
return results
def worker_scope():
import psycopg
import redis
from botocore.exceptions import ClientError
from messenger.settings import Settings
from messenger.storage import ObjectStore
settings = Settings.from_env()
with psycopg.connect(settings.database_url, autocommit=True) as conn:
assert conn.execute("SELECT ssl FROM pg_stat_ssl WHERE pid=pg_backend_pid()").fetchone()[0]
conn.execute("SELECT count(*) FROM attachment_jobs").fetchone()
for query in (
"SELECT count(*) FROM users",
"SELECT count(*) FROM messages",
"SELECT count(*) FROM sessions",
"CREATE TEMP TABLE forbidden_probe(id int)",
"UPDATE attachments SET owner_id=owner_id WHERE false",
"SET ROLE messenger",
):
try:
conn.execute(query)
except psycopg.errors.InsufficientPrivilege:
continue
raise AssertionError("Unexpected database grant")
broker = redis.Redis.from_url(settings.broker_url)
assert broker.ping()
for op in (
lambda: broker.config_get("appendonly"),
lambda: broker.xrange("unrelated-key"),
lambda: broker.get("session-user:synthetic"),
):
try:
op()
except redis.exceptions.NoPermissionError:
continue
raise AssertionError("Unexpected broker grant")
store = ObjectStore(settings)
suffix = uuid.uuid4().hex
key = "objects/audit/" + suffix
store.client.put_object(Bucket=store.bucket, Key=key, Body=b"synthetic")
assert store.client.head_object(Bucket=store.bucket, Key=key)["ContentLength"] == 9
for op in (
lambda: store.client.put_object(
Bucket=store.bucket, Key="staging/audit/" + suffix, Body=b"x"
),
lambda: store.client.delete_object(Bucket=store.bucket, Key=key),
lambda: store.client.get_bucket_policy(Bucket=store.bucket),
):
try:
op()
except ClientError as exc:
assert exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403
continue
raise AssertionError("Unexpected S3 grant")
cert = Path("/run/tls/client")
ctx = context(INSIDE_CA, cert)
base = "https://openbao:8200/v1/"
ct = {"Content-Type": "application/json"}
code, raw = response(base + "auth/cert/login", ctx, b'{"name":"worker"}', ct)
assert code == 200
auth = json.loads(raw)["auth"]
token = auth["client_token"]
headers = {**ct, "X-Vault-Token": token}
assert 0 < auth["lease_duration"] <= 120
assert response(base + "services/data/worker", ctx, headers=headers)[0] == 200
assert response(base + "services/data/api", ctx, headers=headers)[0] == 403
assert response(base + "services/data/worker", ctx, b'{"data":{}}', headers)[0] == 403
assert response(base + "auth/token/revoke-self", ctx, b"{}", headers)[0] == 204
assert response(base + "services/data/worker", ctx, headers=headers)[0] == 403
return {
"postgres": {"tls": True, "job_read": True, "denied_operations": 6},
"broker": {"tls": True, "ping": True, "denied_operations": 3},
"s3": {"put_objects": True, "denied_operations": 3},
"openbao": {
"own_read": True,
"foreign_read_and_write_denied": True,
"token_ttl_s": auth["lease_duration"],
"revoked_token_denied": True,
},
}
def old_identity():
ctx = context(INSIDE_CA, Path("/probe/old"))
denied(lambda: redis_ping(ctx))
denied(
lambda: response(
"https://otel-collector:4319/v1/traces",
ctx,
b"",
{"Content-Type": "application/x-protobuf"},
)[0]
)
code, _ = response(
"https://openbao:8200/v1/auth/cert/login",
ctx,
b'{"name":"worker"}',
{"Content-Type": "application/json"},
)
assert code in (400, 403)
return {"broker": "revoked", "collector": "revoked", "openbao": "revoked"}
def worker_probe():
raw = compose(
"run",
"--rm",
"--no-deps",
"-T",
"-v",
f"{Path(__file__).resolve()}:/test/audit.py:ro",
"--entrypoint",
"python",
"worker",
"/test/audit.py",
"--inside",
"worker",
capture_output=True,
text=True,
).stdout
return json.loads(raw)
def isolated(mode, mounts):
cmd = [
"docker",
"run",
"--rm",
"--network",
"mtusi-lab7_backend",
"--read-only",
"--cap-drop",
"ALL",
"--security-opt",
"no-new-privileges:true",
"--memory",
"256m",
"-v",
f"{Path(__file__).resolve()}:/test/audit.py:ro",
"-v",
f"{LOCAL / 'tls/trust'}:/run/tls/trust:ro",
]
for name, source in mounts.items():
cmd += ["-v", f"{source}:/probe/{name}:ro"]
image = compose("images", "-q", "api", capture_output=True, text=True).stdout.split()[0]
raw = checked(
*cmd, image, "python", "/test/audit.py", "--inside", mode, capture_output=True, text=True
).stdout
return json.loads(raw)
def check_api(client):
alice, bob, eve, chat = client.bootstrap()
attachment, _ = client.attachment(alice, chat)
client.wait_ready(bob, attachment)
path = "/api/v1/attachments/" + attachment["id"]
client.call("GET", path, 404, actor=eve, headers={"X-User-Id": alice["id"]})
client.call("GET", path + "/content", 404, actor=eve)
client.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=bob)
client.call("GET", "/api/v1/users/me", 401, actor=bob)
reg = alice["registration"]
bad = base64.b64encode((reg["username"] + ":invalid-password-123").encode()).decode()
statuses, replicas, times = [], set(), []
for _ in range(6):
r = client.call(
"POST", "/api/v1/auth/sessions", [401, 429], headers={"Authorization": "Basic " + bad}
)
statuses.append(r["status"])
times.append(r["elapsed"] * 1000)
replicas.add(r["headers"]["X-Instance-Id"])
if r["status"] == 429:
assert int(r["headers"]["Retry-After"]) > 0
assert 429 in statuses and 401 in statuses and len(replicas) >= 2
too_big = client.call(
"POST", "/api/v1/users", 413, raw=b"x" * 66000, headers={"Content-Type": "application/json"}
)
assert too_big["status"] == 413
return {
"statuses": statuses,
"replicas": len(replicas),
"latency_ms": times,
"outsider_denied": True,
"logout_revoked": True,
"oversize_json_denied": True,
}
def rotate(client, temp):
alice, _, _, chat = client.bootstrap()
old = temp / "old"
shutil.copytree(LOCAL / "tls/worker-client", old)
compose("stop", "worker")
accepted = []
start = time.monotonic()
try:
for _ in range(5):
att, data = client.attachment(alice, chat)
accepted.append((att, data))
checked(sys.executable, "scripts/bao_operator.py", "--revoke", "worker")
checked(sys.executable, "scripts/pki.py", "--rotate", "worker-client")
password = LOCAL / "broker_worker_password"
password.write_text(secrets.token_hex(24) + "\n")
checked(sys.executable, "scripts/bootstrap.py")
# Load the changed ACL without restarting durable Valkey or erasing its stream.
compose(
"exec",
"-T",
"broker",
"sh",
"-ec",
"""p="$(awk '/^user operator / {print substr($NF,2)}' /run/secrets/broker.acl)"
export REDISCLI_AUTH="$p"
valkey-cli --user operator ACL LOAD | grep -q '^OK$'
valkey-cli --user operator CLIENT KILL USER worker >/dev/null""",
)
compose("restart", "broker-tls", "otel-collector-tls")
result = isolated("old", {"old": old})
finally:
checked(sys.executable, "scripts/bao_operator.py")
compose("up", "-d", "--no-deps", "--force-recreate", "worker")
for attachment, original in accepted:
item = client.wait_ready(alice, attachment)
file = client.call("GET", f"/api/v1/attachments/{item['id']}/content", actor=alice)
assert hashlib.sha256(file["bytes"]).digest() == hashlib.sha256(original).digest()
return {
"old_identity": result,
"accepted": len(accepted),
"ready_with_original": len(accepted),
"elapsed_seconds": round(time.monotonic() - start, 3),
"broker_acl_reloaded": True,
"images_rebuilt": False,
"new_worker": worker_probe(),
}
def check_sealed(client):
client.call("GET", "/health/ready", schema="Health")
start = time.monotonic()
checked(sys.executable, "scripts/bao_operator.py", "--seal")
try:
deadline = start + 42
status = 200
while time.monotonic() < deadline:
status, _ = response("https://localhost:8443/health/ready", context(CA))
if status == 503:
break
time.sleep(1)
assert status == 503, "API did not stop serving after the credential grace period"
unavailable = time.monotonic() - start
finally:
checked(sys.executable, "scripts/bao_operator.py")
deadline = time.monotonic() + 22
while time.monotonic() < deadline:
status, _ = response("https://localhost:8443/health/ready", context(CA))
if status == 200:
return {
"unready_after_seconds": round(unavailable, 3),
"recovered_after_seconds": round(time.monotonic() - start, 3),
}
time.sleep(1)
raise AssertionError("API readiness did not recover after unseal")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--inside", choices=("tls", "worker", "old"))
parser.add_argument("--output", type=Path)
parser.add_argument("--rotate", action="store_true")
parser.add_argument("--seal", action="store_true")
args = parser.parse_args()
if args.inside:
print(
json.dumps(
{"tls": audit_tls, "worker": worker_scope, "old": old_identity}[args.inside]()
)
)
return
if not args.output or args.output.exists():
parser.error("choose a new --output")
sys.path.insert(0, str(ROOT / "tests"))
from client import Client
client = Client("https://localhost:8443", str(CA))
with tempfile.TemporaryDirectory(dir=LOCAL) as folder:
tmp = Path(folder)
tmp.chmod(0o755)
rogue = tmp / "rogue"
rogue.mkdir(mode=0o755)
checked(
"openssl",
"req",
"-x509",
"-newkey",
"ec",
"-pkeyopt",
"ec_paramgen_curve:P-256",
"-nodes",
"-days",
"1",
"-subj",
"/CN=worker-client",
"-addext",
"extendedKeyUsage=clientAuth",
"-keyout",
str(rogue / "service.key"),
"-out",
str(rogue / "service.crt"),
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
(rogue / "service.key").chmod(0o644)
identities = {
name: LOCAL / "tls" / name
for name in ("lb-client", "worker-client", "probe-client", "grafana-client")
}
result = {
"tls": isolated("tls", {**identities, "rogue": rogue}),
"worker_scope": worker_probe(),
"api": check_api(client),
}
print("PASS: verified TLS and distinct service privileges", flush=True)
if args.rotate:
result["worker_compromise"] = rotate(client, tmp)
print("PASS: old identity denied; all accepted uploads recovered", flush=True)
if args.seal:
result["sealed"] = check_sealed(client)
print(
"PASS: sealed OpenBao makes API unready within a bounded grace period", flush=True
)
args.output.write_text(json.dumps(result, indent=2) + "\n")
if __name__ == "__main__":
main()
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env python3
"""Map a process alias to ONE container belonging to this Compose service."""
import json
import os
import re
import subprocess
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def alias(container):
code = "from pathlib import Path; print(Path('/tmp/api-instance-id').read_text())"
r = subprocess.run(
["docker", "exec", container, "python", "-c", code],
cwd=ROOT,
capture_output=True,
text=True,
timeout=3,
check=True,
)
return container, r.stdout.strip()
def main():
target = os.environ.get("TARGET_INSTANCE_ID", "")
if not re.fullmatch(r"[a-f0-9]{32}", target):
raise SystemExit("TARGET_INSTANCE_ID must name a live API process")
ids = subprocess.check_output(
["docker", "compose", "ps", "-q", "api"], cwd=ROOT, text=True
).split()
with ThreadPoolExecutor(max_workers=8) as pool:
matches = [cid for cid, identity in pool.map(alias, ids) if identity == target]
if len(matches) != 1:
raise SystemExit("Expected exactly one matching API in this project")
subprocess.run(
["docker", "stop", "--time", "5", matches[0]], check=True, capture_output=True, timeout=7
)
print(json.dumps({"stopped_container": matches[0], "instance": target}))
if __name__ == "__main__":
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/stop-one.py
+97
View File
@@ -0,0 +1,97 @@
#!/usr/bin/env python3
"""Use an operator connection only to seed a legacy hash and an expired test session."""
import base64
import hashlib
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
def inside():
import psycopg
from messenger.db_roles import admin_conninfo
fixture = json.load(sys.stdin)
with psycopg.connect(admin_conninfo()) as db:
if fixture["mode"] == "seed":
db.execute(
"UPDATE users SET password_hash=%s WHERE id=%s",
(fixture["hash"], fixture["user"]),
)
elif fixture["mode"] == "check":
row = db.execute(
"SELECT password_hash FROM users WHERE id=%s", (fixture["user"],)
).fetchone()
assert row and row[0].startswith("$argon2id$")
assert row[0].split("$")[3] == "m=19456,t=2,p=1"
db.execute(
"UPDATE sessions SET expires_at=created_at+interval '1 millisecond' "
"WHERE token_hash=%s",
(bytes.fromhex(fixture["digest"]),),
)
else:
raise AssertionError("Unknown fixture mode")
def db_fixture(**data):
subprocess.run(
[
"docker",
"compose",
"run",
"--rm",
"--no-deps",
"-T",
"-v",
f"{Path(__file__).resolve()}:/test/verify_sessions.py:ro",
"--entrypoint",
"python",
"migrate",
"/test/verify_sessions.py",
"--inside",
],
cwd=ROOT,
text=True,
input=json.dumps(data),
check=True,
)
def main():
if "--inside" in sys.argv:
inside()
return
from argon2 import PasswordHasher
sys.path.insert(0, str(ROOT / "tests"))
from client import Client
client = Client("https://localhost:8443", str(ROOT / "secrets/tls/trust/ca.crt"))
actor = client.user("legacy")
reg = actor["registration"]
legacy = PasswordHasher(memory_cost=65536, time_cost=3, parallelism=4).hash(reg["password"])
db_fixture(mode="seed", user=actor["id"], hash=legacy)
auth = base64.b64encode((reg["username"] + ":" + reg["password"]).encode()).decode()
session = client.call(
"POST",
"/api/v1/auth/sessions",
201,
schema="Session",
headers={"Authorization": "Basic " + auth},
)["json"]
actor["token"] = session["token"]
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
db_fixture(
mode="check", user=actor["id"], digest=hashlib.sha256(actor["token"].encode()).hexdigest()
)
client.call("GET", "/api/v1/users/me", 401, actor=actor)
print("PASS: successful login rehashed old Argon2id parameters; expired session rejected.")
if __name__ == "__main__":
main()
+95
View File
@@ -0,0 +1,95 @@
#!/usr/bin/env python3
"""Verify the real Grafana data paths over authenticated HTTPS."""
import base64
import json
import ssl
import sys
import time
import uuid
from pathlib import Path
from urllib.error import HTTPError
from urllib.parse import urlencode
from urllib.request import Request, urlopen
ROOT = Path(__file__).resolve().parents[1]
CA = ROOT / "secrets/tls/trust/ca.crt"
def grafana(uid, path, **params):
password = (ROOT / "secrets/grafana_password").read_text().strip()
basic = base64.b64encode(("admin:" + password).encode()).decode()
url = f"https://localhost:3007/api/datasources/proxy/uid/{uid}{path}"
if params:
url += "?" + urlencode(params)
request = Request(url, headers={"Authorization": "Basic " + basic})
with urlopen(request, context=ssl.create_default_context(cafile=CA), timeout=5) as response:
return json.load(response)
def until(fn):
end = time.monotonic() + 35
while time.monotonic() < end:
try:
value = fn()
except HTTPError as exc:
if exc.code not in (404, 502, 503):
raise
else:
if value:
return value
time.sleep(0.5)
raise AssertionError("Timed out waiting for Grafana datasource results")
def logs(service, field, value):
# The query is restricted to a generated ID, which is validated by the probe.
return grafana(
"loki",
"/loki/api/v1/query_range",
query=f'{{service_name="messenger-{service}"}} | {field}="{value}"',
start=str(time.time_ns() - 120_000_000_000),
limit=10,
)["data"]["result"]
def main():
sys.path.insert(0, str(ROOT / "tests"))
from client import Client
client = Client("https://localhost:8443", str(CA))
actor, _, _, chat = client.bootstrap()
marker = "probe_" + uuid.uuid4().hex
client.message(actor, chat, marker)
upload = client.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename={marker}.png",
202,
actor=actor,
headers={"Content-Type": "image/png"},
raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
schema="Attachment",
)
client.wait_ready(actor, upload["json"])
request_id = str(uuid.UUID(upload["headers"]["X-Request-Id"]))
api_logs = until(lambda: logs("api", "request_id", request_id))
stream = api_logs[0]
value = stream["values"][0]
trace_id = (value[2] if len(value) > 2 else stream["stream"])["trace_id"]
assert len(trace_id) == 32 and all(ch in "0123456789abcdef" for ch in trace_id)
worker_logs = until(lambda: logs("worker", "trace_id", trace_id))
trace = until(lambda: grafana("tempo", "/api/traces/" + trace_id))
collected = json.dumps([api_logs, worker_logs, trace])
assert "jobs.consume" in collected and "s3.put" in collected and "db.transaction" in collected
for forbidden in (marker, actor["registration"]["password"], actor["token"]):
assert forbidden not in collected
until(
lambda: grafana("prometheus", "/api/v1/query", query="sum(messenger_http_requests_total)")[
"data"
]["result"]
)
print("PASS: three Grafana datasources over mTLS; correlated payloads are clean.")
if __name__ == "__main__":
main()