forked from mtusi2026/mtusi2026lab7
Добавить стенд лабораторной 7 с mTLS, OpenBao и least privilege worker.
Закрывает обязательный минимум и трек 1: ротация leaf и секрета, sealed OpenBao и сохранность queued-задачи.
This commit is contained in:
@@ -0,0 +1,544 @@
|
||||
"""Create local CA, secrets and, with --up, initialize the stand. Never prints secret values."""
|
||||
import datetime
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PKI = ROOT / "secrets" / "pki"
|
||||
OPERATOR = ROOT / "secrets" / "operator"
|
||||
|
||||
|
||||
def ensure_cryptography():
|
||||
try:
|
||||
import cryptography # noqa: F401
|
||||
return
|
||||
except ImportError:
|
||||
pass
|
||||
venv = ROOT / ".venv-bootstrap"
|
||||
py = venv / "Scripts" / "python.exe"
|
||||
if not py.exists():
|
||||
subprocess.check_call([sys.executable, "-m", "venv", str(venv)])
|
||||
subprocess.check_call([str(py), "-m", "pip", "install", "cryptography==44.0.2"])
|
||||
os.execv(str(py), [str(py), str(Path(__file__).resolve()), *sys.argv[1:]])
|
||||
|
||||
|
||||
def write_pem(path, data):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
|
||||
|
||||
def build_material():
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID
|
||||
|
||||
def private():
|
||||
return rsa.generate_private_key(public_exponent=65537, key_size=2048)
|
||||
|
||||
def dump_key(key):
|
||||
return key.private_bytes(
|
||||
serialization.Encoding.PEM,
|
||||
serialization.PrivateFormat.PKCS8,
|
||||
serialization.NoEncryption(),
|
||||
)
|
||||
|
||||
def dump_cert(cert):
|
||||
return cert.public_bytes(serialization.Encoding.PEM)
|
||||
|
||||
def name(cn):
|
||||
return x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, cn)])
|
||||
|
||||
now = datetime.datetime.now(datetime.timezone.utc)
|
||||
|
||||
def certificate(cn, sans, ips, eku, days, issuer_key, issuer_cert, expired=False):
|
||||
key = private()
|
||||
if expired:
|
||||
start = now - datetime.timedelta(days=3)
|
||||
end = now - datetime.timedelta(days=1)
|
||||
else:
|
||||
start = now - datetime.timedelta(minutes=10)
|
||||
end = now + datetime.timedelta(days=days)
|
||||
alt = [x509.DNSName(item) for item in sans]
|
||||
alt += [x509.IPAddress(ipaddress.ip_address(item)) for item in ips]
|
||||
uses = []
|
||||
if "client" in eku:
|
||||
uses.append(ExtendedKeyUsageOID.CLIENT_AUTH)
|
||||
if "server" in eku:
|
||||
uses.append(ExtendedKeyUsageOID.SERVER_AUTH)
|
||||
builder = (
|
||||
x509.CertificateBuilder()
|
||||
.subject_name(name(cn))
|
||||
.issuer_name(issuer_cert.subject if issuer_cert is not None else name(cn))
|
||||
.public_key(key.public_key())
|
||||
.serial_number(x509.random_serial_number())
|
||||
.not_valid_before(start)
|
||||
.not_valid_after(end)
|
||||
.add_extension(x509.BasicConstraints(ca=issuer_cert is None, path_length=0 if issuer_cert is None else None), critical=True)
|
||||
)
|
||||
if issuer_cert is None:
|
||||
builder = builder.add_extension(
|
||||
x509.KeyUsage(
|
||||
digital_signature=True,
|
||||
key_cert_sign=True,
|
||||
crl_sign=True,
|
||||
content_commitment=False,
|
||||
key_encipherment=False,
|
||||
data_encipherment=False,
|
||||
key_agreement=False,
|
||||
encipher_only=False,
|
||||
decipher_only=False,
|
||||
),
|
||||
critical=True,
|
||||
)
|
||||
else:
|
||||
builder = builder.add_extension(x509.SubjectAlternativeName(alt), critical=False)
|
||||
builder = builder.add_extension(x509.ExtendedKeyUsage(uses), critical=False)
|
||||
signed = builder.sign(issuer_key if issuer_cert is not None else key, hashes.SHA256())
|
||||
return key, signed
|
||||
|
||||
PKI.mkdir(parents=True, exist_ok=True)
|
||||
if not (PKI / "ca.crt").exists():
|
||||
ca_key, ca_cert = certificate("messenger-lab-ca", [], [], [], 30, None, None)
|
||||
other_key, other_cert = certificate("foreign-lab-ca", [], [], [], 30, None, None)
|
||||
write_pem(PKI / "ca.key", dump_key(ca_key))
|
||||
write_pem(PKI / "ca.crt", dump_cert(ca_cert))
|
||||
write_pem(PKI / "other-ca.key", dump_key(other_key))
|
||||
write_pem(PKI / "other-ca.crt", dump_cert(other_cert))
|
||||
leaves = {
|
||||
"ingress": ("localhost", ["localhost", "nginx"], ["127.0.0.1"], ["server"], 14, False, ca_key, ca_cert),
|
||||
"lb": ("lb.messenger.internal", ["lb.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert),
|
||||
"api-server": ("api.messenger.internal", ["messenger_api", "api-1", "api-2", "api.messenger.internal"], [], ["server"], 14, False, ca_key, ca_cert),
|
||||
"api-client": ("api.messenger.internal", ["api.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert),
|
||||
"worker-client": ("worker.messenger.internal", ["worker.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert),
|
||||
"stranger": ("stranger.messenger.internal", ["stranger.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert),
|
||||
"expired": ("lb.messenger.internal", ["lb.messenger.internal"], [], ["client"], 1, True, ca_key, ca_cert),
|
||||
"other-client": ("lb.messenger.internal", ["lb.messenger.internal"], [], ["client"], 14, False, other_key, other_cert),
|
||||
"postgres": ("postgres", ["postgres"], [], ["server"], 14, False, ca_key, ca_cert),
|
||||
"minio": ("minio", ["minio"], [], ["server"], 14, False, ca_key, ca_cert),
|
||||
"rabbitmq": ("rabbitmq", ["rabbitmq"], [], ["server"], 14, False, ca_key, ca_cert),
|
||||
"valkey": ("valkey", ["valkey"], [], ["server", "client"], 14, False, ca_key, ca_cert),
|
||||
"openbao": ("openbao", ["openbao", "localhost"], ["127.0.0.1"], ["server"], 14, False, ca_key, ca_cert),
|
||||
"health": ("health.messenger.internal", ["health.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert),
|
||||
"operator": ("operator.messenger.internal", ["operator.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert),
|
||||
"otel-edge": ("otel-edge", ["otel-edge"], [], ["server"], 14, False, ca_key, ca_cert),
|
||||
"collector": ("otel-collector", ["otel-collector"], [], ["server", "client"], 14, False, ca_key, ca_cert),
|
||||
"prometheus": ("prometheus", ["prometheus"], [], ["server"], 14, False, ca_key, ca_cert),
|
||||
"loki": ("loki", ["loki"], [], ["server"], 14, False, ca_key, ca_cert),
|
||||
"tempo": ("tempo", ["tempo"], [], ["server"], 14, False, ca_key, ca_cert),
|
||||
"grafana": ("grafana", ["grafana", "localhost"], ["127.0.0.1"], ["server"], 14, False, ca_key, ca_cert),
|
||||
}
|
||||
serials = {}
|
||||
for stem, spec in leaves.items():
|
||||
cn, sans, ips, eku, days, expired, issuer_key, issuer_cert = spec
|
||||
key, cert = certificate(cn, sans, ips, eku, days, issuer_key, issuer_cert, expired)
|
||||
write_pem(PKI / (stem + ".key"), dump_key(key))
|
||||
write_pem(PKI / (stem + ".crt"), dump_cert(cert))
|
||||
serials[stem] = cert.serial_number
|
||||
minio_dir = PKI / "minio"
|
||||
minio_dir.mkdir(exist_ok=True)
|
||||
shutil.copyfile(PKI / "minio.crt", minio_dir / "public.crt")
|
||||
shutil.copyfile(PKI / "minio.key", minio_dir / "private.key")
|
||||
(PKI / "serials.json").write_text(json.dumps({key: format(value, "X") for key, value in serials.items()}), encoding="utf-8")
|
||||
return json.loads((PKI / "serials.json").read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def password():
|
||||
return secrets.token_hex(24)
|
||||
|
||||
|
||||
def material_passwords():
|
||||
OPERATOR.mkdir(parents=True, exist_ok=True)
|
||||
path = OPERATOR / "passwords.json"
|
||||
current = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
|
||||
defaults = {
|
||||
"postgres_password": password(),
|
||||
"api_db_password": password(),
|
||||
"worker_db_password": password(),
|
||||
"minio_root_user": "lab-root",
|
||||
"minio_root_password": password(),
|
||||
"api_s3_key": "api-" + secrets.token_hex(6),
|
||||
"api_s3_secret": password(),
|
||||
"worker_s3_key": "worker-" + secrets.token_hex(6),
|
||||
"worker_s3_secret": password(),
|
||||
"valkey_password": password(),
|
||||
"grafana_password": password(),
|
||||
"s3_admin_key": "admin-" + secrets.token_hex(6),
|
||||
"s3_admin_secret": password(),
|
||||
}
|
||||
changed = False
|
||||
for key, value in defaults.items():
|
||||
if key not in current:
|
||||
current[key] = value
|
||||
changed = True
|
||||
if changed:
|
||||
path.write_text(json.dumps(current), encoding="utf-8")
|
||||
return current
|
||||
|
||||
|
||||
def colon(serial):
|
||||
hexed = serial if len(serial) % 2 == 0 else "0" + serial
|
||||
return ":".join(hexed[i : i + 2] for i in range(0, len(hexed), 2))
|
||||
|
||||
|
||||
def write_configs(pw, serials):
|
||||
(ROOT / "secrets" / "db").mkdir(parents=True, exist_ok=True)
|
||||
(ROOT / "secrets" / "db" / "01-roles.sql").write_text(
|
||||
"CREATE ROLE api_login LOGIN PASSWORD '{api}';\nCREATE ROLE worker_login LOGIN PASSWORD '{worker}';\n".format(
|
||||
api=pw["api_db_password"], worker=pw["worker_db_password"]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(ROOT / "secrets" / "postgres.env").write_text(
|
||||
"POSTGRES_USER=messenger\nPOSTGRES_PASSWORD={password}\nPOSTGRES_DB=messenger\n".format(password=pw["postgres_password"]),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(ROOT / "secrets" / "minio.env").write_text(
|
||||
"MINIO_ROOT_USER={user}\nMINIO_ROOT_PASSWORD={password}\n".format(
|
||||
user=pw["minio_root_user"], password=pw["minio_root_password"]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(ROOT / "secrets" / "grafana.env").write_text(
|
||||
"GF_SECURITY_ADMIN_PASSWORD={password}\n".format(password=pw["grafana_password"]),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(ROOT / "secrets" / "valkey").mkdir(parents=True, exist_ok=True)
|
||||
(ROOT / "secrets" / "valkey" / "users.acl").write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"user default off",
|
||||
"user health on nopass +ping",
|
||||
"user api on >{password} ~login:* +incr +incrby +expire +ttl +ping +get".format(password=pw["valkey_password"]),
|
||||
"",
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(PKI / "lb-allowed-serials.txt").write_text(serials["lb"] + "\n" + colon(serials["lb"]) + "\n", encoding="utf-8")
|
||||
seen = {}
|
||||
for stem in ("api-client", "worker-client"):
|
||||
raw = serials[stem].upper()
|
||||
for item in (raw, colon(raw)):
|
||||
seen[item.lower()] = item
|
||||
lines = ["map $ssl_client_s_dn $otel_cn_ok {", " default 0;", ' "~*CN=api\\.messenger\\.internal" 1;', ' "~*CN=worker\\.messenger\\.internal" 1;', "}"]
|
||||
lines.append("map $ssl_client_serial $otel_serial_ok {")
|
||||
lines.append(" default 0;")
|
||||
for item in seen.values():
|
||||
lines.append(' "%s" 1;' % item)
|
||||
lines.append("}")
|
||||
nginx = ROOT / "secrets" / "nginx"
|
||||
nginx.mkdir(parents=True, exist_ok=True)
|
||||
(nginx / "serials.conf").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
(ROOT / ".env").write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"SESSION_TTL_SECONDS=3600",
|
||||
"IDEMPOTENCY_TTL_SECONDS=86400",
|
||||
"WORKER_CONCURRENCY=2",
|
||||
"JOB_MAX_ATTEMPTS=5",
|
||||
"JOB_TIMEOUT_SECONDS=60",
|
||||
"STAGING_RETENTION_SECONDS=86400",
|
||||
"",
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
for name in ("api-1", "api-2", "worker"):
|
||||
(ROOT / "run" / "instances" / name).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def compose(*args, check=True, env=None):
|
||||
command = ["docker", "compose", "--env-file", ".env", *args]
|
||||
return subprocess.run(command, cwd=ROOT, check=check, env=env)
|
||||
|
||||
|
||||
def bao(*args, token=None):
|
||||
command = [
|
||||
"docker",
|
||||
"compose",
|
||||
"--env-file",
|
||||
".env",
|
||||
"exec",
|
||||
"-T",
|
||||
"-e",
|
||||
"BAO_ADDR=https://127.0.0.1:8200",
|
||||
"-e",
|
||||
"BAO_CACERT=/openbao/tls/ca.crt",
|
||||
"-e",
|
||||
"BAO_CLIENT_CERT=/openbao/tls/operator.crt",
|
||||
"-e",
|
||||
"BAO_CLIENT_KEY=/openbao/tls/operator.key",
|
||||
]
|
||||
if token:
|
||||
command += ["-e", "BAO_TOKEN=" + token]
|
||||
command += ["openbao", "bao", *args]
|
||||
return subprocess.run(command, cwd=ROOT, capture_output=True, text=True)
|
||||
|
||||
|
||||
def parse_json(text):
|
||||
start = text.find("{")
|
||||
end = text.rfind("}")
|
||||
if start < 0 or end < start:
|
||||
return None
|
||||
return json.loads(text[start : end + 1])
|
||||
|
||||
|
||||
def wait_status():
|
||||
for _ in range(40):
|
||||
result = bao("status", "-format=json")
|
||||
payload = parse_json(result.stdout or "")
|
||||
if payload is not None and "initialized" in payload:
|
||||
return payload
|
||||
time.sleep(2)
|
||||
raise SystemExit("OpenBao did not start")
|
||||
|
||||
|
||||
def init_openbao(pw):
|
||||
status = wait_status()
|
||||
init_path = OPERATOR / "init.json"
|
||||
if not status.get("initialized"):
|
||||
result = bao("operator", "init", "-key-shares=3", "-key-threshold=2", "-format=json")
|
||||
payload = parse_json(result.stdout or "")
|
||||
if result.returncode != 0 or payload is None or "root_token" not in payload:
|
||||
raise SystemExit("OpenBao init failed")
|
||||
init_path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
status = wait_status()
|
||||
if status.get("sealed", True):
|
||||
keys = json.loads(init_path.read_text(encoding="utf-8"))["unseal_keys_b64"]
|
||||
for key in keys[:2]:
|
||||
result = bao("operator", "unseal", key)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit("OpenBao unseal failed")
|
||||
root = json.loads(init_path.read_text(encoding="utf-8"))["root_token"]
|
||||
bao("secrets", "enable", "-path=secret", "kv-v2", token=root)
|
||||
bao("policy", "write", "api", "/openbao/policies/api.hcl", token=root)
|
||||
bao("policy", "write", "worker", "/openbao/policies/worker.hcl", token=root)
|
||||
bao("auth", "enable", "cert", token=root)
|
||||
for role, cn in (("api", "api.messenger.internal"), ("worker", "worker.messenger.internal")):
|
||||
result = bao(
|
||||
"write",
|
||||
"auth/cert/certs/" + role,
|
||||
"display_name=" + role,
|
||||
"policies=" + role,
|
||||
"certificate=@/openbao/tls/ca.crt",
|
||||
"allowed_common_names=" + cn,
|
||||
"token_ttl=20m",
|
||||
"token_max_ttl=2h",
|
||||
token=root,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit("OpenBao cert role " + role + " failed")
|
||||
writes = {
|
||||
"secret/api/db": ["username=api_login", "password=" + pw["api_db_password"], "host=postgres", "dbname=messenger"],
|
||||
"secret/worker/db": ["username=worker_login", "password=" + pw["worker_db_password"], "host=postgres", "dbname=messenger"],
|
||||
"secret/api/s3": ["access_key=" + pw["api_s3_key"], "secret_key=" + pw["api_s3_secret"]],
|
||||
"secret/worker/s3": ["access_key=" + pw["worker_s3_key"], "secret_key=" + pw["worker_s3_secret"]],
|
||||
"secret/api/valkey": ["username=api", "password=" + pw["valkey_password"], "host=valkey"],
|
||||
}
|
||||
for path, fields in writes.items():
|
||||
result = bao("kv", "put", path, *fields, token=root)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit("OpenBao kv put failed for " + path.split("/")[1])
|
||||
|
||||
|
||||
def wait_healthy(*services):
|
||||
for _ in range(50):
|
||||
result = subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", "ps", "--format", "{{.Service}} {{.Health}}", *services],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
rows = [line.strip() for line in result.stdout.splitlines() if line.strip()]
|
||||
if len(rows) >= len(services) and all(line.endswith(" healthy") for line in rows):
|
||||
return
|
||||
time.sleep(3)
|
||||
raise SystemExit("services did not become healthy: " + " ".join(services))
|
||||
|
||||
|
||||
def write_s3_config(pw):
|
||||
bucket = "messenger-attachments"
|
||||
config = {
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [{"accessKey": pw["s3_admin_key"], "secretKey": pw["s3_admin_secret"]}],
|
||||
"actions": ["Admin", "Read", "Write", "List"],
|
||||
},
|
||||
{
|
||||
"name": "api",
|
||||
"credentials": [{"accessKey": pw["api_s3_key"], "secretKey": pw["api_s3_secret"]}],
|
||||
"actions": [
|
||||
"Read:{0}/staging/*".format(bucket),
|
||||
"Write:{0}/staging/*".format(bucket),
|
||||
"List:{0}/staging/*".format(bucket),
|
||||
"Read:{0}/originals/*".format(bucket),
|
||||
"List:{0}/originals/*".format(bucket),
|
||||
"Read:{0}/variants/*".format(bucket),
|
||||
"List:{0}/variants/*".format(bucket),
|
||||
],
|
||||
},
|
||||
{
|
||||
"name": "worker",
|
||||
"credentials": [{"accessKey": pw["worker_s3_key"], "secretKey": pw["worker_s3_secret"]}],
|
||||
"actions": [
|
||||
"Read:{0}/staging/*".format(bucket),
|
||||
"Write:{0}/staging/*".format(bucket),
|
||||
"List:{0}/staging/*".format(bucket),
|
||||
"Read:{0}/originals/*".format(bucket),
|
||||
"Write:{0}/originals/*".format(bucket),
|
||||
"List:{0}/originals/*".format(bucket),
|
||||
"Read:{0}/variants/*".format(bucket),
|
||||
"Write:{0}/variants/*".format(bucket),
|
||||
"List:{0}/variants/*".format(bucket),
|
||||
],
|
||||
},
|
||||
]
|
||||
}
|
||||
target = ROOT / "secrets" / "seaweed"
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
(target / "s3.json").write_text(json.dumps(config), encoding="utf-8")
|
||||
|
||||
|
||||
def configure_bucket(pw):
|
||||
admin = ROOT / "secrets" / "seaweed" / "admin.json"
|
||||
admin.write_text(
|
||||
json.dumps({"access_key": pw["s3_admin_key"], "secret_key": pw["s3_admin_secret"]}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
result = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"--rm",
|
||||
"--network",
|
||||
"mtusi-lab7_app",
|
||||
"-v",
|
||||
str(PKI / "ca.crt") + ":/certs/ca.crt:ro",
|
||||
"-v",
|
||||
str(admin) + ":/certs/admin.json:ro",
|
||||
"--entrypoint",
|
||||
"python",
|
||||
"mtusi-lab7-messenger:1.7.0",
|
||||
"-c",
|
||||
(
|
||||
"import json,boto3; from botocore.config import Config; "
|
||||
"admin=json.load(open('/certs/admin.json')); "
|
||||
"c=boto3.client('s3',endpoint_url='https://minio:8333',aws_access_key_id=admin['access_key'],"
|
||||
"aws_secret_access_key=admin['secret_key'],region_name='us-east-1',verify='/certs/ca.crt',"
|
||||
"config=Config(signature_version='s3v4',s3={'addressing_style':'path'})); "
|
||||
"buckets=[b['Name'] for b in c.list_buckets().get('Buckets',[])]; "
|
||||
"c.create_bucket(Bucket='messenger-attachments') if 'messenger-attachments' not in buckets else None; "
|
||||
"c.put_object(Bucket='messenger-attachments',Key='canary/marker',Body=b'canary-not-a-secret'); "
|
||||
"print('s3-bootstrap-ok')"
|
||||
),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
admin.unlink(missing_ok=True)
|
||||
if result.returncode != 0 or "s3-bootstrap-ok" not in (result.stdout or ""):
|
||||
raise SystemExit("s3 bootstrap failed")
|
||||
|
||||
|
||||
def configure_rabbit():
|
||||
users = {
|
||||
"api.messenger.internal": ("^attachment\\.process(\\.v1)?$", "^attachment\\.process$", "^$"),
|
||||
"worker.messenger.internal": ("^attachment\\.process\\.v1$", "^$", "^attachment\\.process\\.v1$"),
|
||||
}
|
||||
subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "await_online_nodes", "1"],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
for user, perms in users.items():
|
||||
last = None
|
||||
for _ in range(10):
|
||||
subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "add_user", user, password()],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
last = subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "set_permissions", "-p", "/", user, *perms],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if last.returncode == 0:
|
||||
break
|
||||
time.sleep(3)
|
||||
if last is None or last.returncode != 0:
|
||||
raise SystemExit("rabbit permissions failed for a service user")
|
||||
expressions = [
|
||||
'rabbit_exchange:declare({resource, <<"/">>, exchange, <<"attachment.process">>}, direct, true, false, false, [], <<"bootstrap">>).',
|
||||
'rabbit_binding:add({binding, {resource, <<"/">>, exchange, <<"attachment.process">>}, <<"attachment.process.v1">>, {resource, <<"/">>, queue, <<"attachment.process.v1">>}, []}, <<"bootstrap">>).',
|
||||
]
|
||||
for expression in expressions:
|
||||
topology = subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "eval", expression],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if topology.returncode != 0:
|
||||
raise SystemExit("rabbit topology failed")
|
||||
subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "delete_user", "guest"],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def wait_ready():
|
||||
import ssl
|
||||
|
||||
ctx = ssl.create_default_context(cafile=str(PKI / "ca.crt"))
|
||||
for _ in range(40):
|
||||
try:
|
||||
with urllib.request.urlopen("https://127.0.0.1:8443/health/ready", context=ctx, timeout=5) as response:
|
||||
if response.status == 200:
|
||||
print("ready https://127.0.0.1:8443/health/ready")
|
||||
return
|
||||
except Exception:
|
||||
time.sleep(3)
|
||||
raise SystemExit("public HTTPS endpoint did not become ready")
|
||||
|
||||
|
||||
def up(pw):
|
||||
write_s3_config(pw)
|
||||
compose("up", "-d", "openbao")
|
||||
init_openbao(pw)
|
||||
compose("build", "api-1", "postgres", "rabbitmq")
|
||||
compose("up", "-d", "--build", "postgres", "minio", "rabbitmq", "valkey")
|
||||
wait_healthy("postgres", "minio", "rabbitmq", "valkey")
|
||||
configure_bucket(pw)
|
||||
configure_rabbit()
|
||||
compose("up", "-d", "--build", "api-1", "api-2", "worker", "nginx", "otel-edge", "otel-collector", "prometheus", "loki", "tempo", "grafana")
|
||||
wait_ready()
|
||||
|
||||
|
||||
def main():
|
||||
ensure_cryptography()
|
||||
serials = build_material()
|
||||
pw = material_passwords()
|
||||
write_configs(pw, serials)
|
||||
print("local material is in secrets/ and .env; private keys were not printed")
|
||||
if "--up" in sys.argv:
|
||||
up(pw)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/bin/sh
|
||||
# Restart user-data stores and the application. OpenBao stays up: restarting it seals the vault.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
wait_healthy() {
|
||||
i=0
|
||||
while [ "$i" -lt 40 ]; do
|
||||
ok=1
|
||||
for svc in "$@"; do
|
||||
health=$(docker compose --env-file .env ps --format '{{.Service}} {{.Health}}' "$svc" | awk 'NR==1 {print $2}')
|
||||
if [ "$health" != "healthy" ]; then
|
||||
ok=0
|
||||
fi
|
||||
done
|
||||
if [ "$ok" -eq 1 ]; then
|
||||
return 0
|
||||
fi
|
||||
i=$((i + 1))
|
||||
sleep 2
|
||||
done
|
||||
echo "timed out waiting for healthy: $*" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
docker compose --env-file .env restart -t 8 postgres minio rabbitmq valkey
|
||||
wait_healthy postgres minio rabbitmq valkey
|
||||
docker compose --env-file .env restart -t 8 api-1 api-2 worker
|
||||
wait_healthy api-1 api-2 worker
|
||||
docker compose --env-file .env restart -t 3 nginx
|
||||
@@ -0,0 +1,23 @@
|
||||
#!/bin/sh
|
||||
# Map the public random alias to the compose service and stop only that replica.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
found=""
|
||||
for dir in run/instances/*; do
|
||||
[ -f "$dir/id" ] || continue
|
||||
current=$(tr -d '\r\n' < "$dir/id")
|
||||
if [ "$current" = "$TARGET_INSTANCE_ID" ]; then
|
||||
found=$(basename "$dir")
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ -z "$found" ]; then
|
||||
echo "unknown instance alias" >&2
|
||||
exit 1
|
||||
fi
|
||||
case "$found" in
|
||||
api-1|api-2) ;;
|
||||
*) echo "alias does not belong to an API replica" >&2; exit 1 ;;
|
||||
esac
|
||||
docker compose --env-file .env stop -t 2 "$found"
|
||||
@@ -0,0 +1,312 @@
|
||||
"""Track 1: least privilege before and after, then revoke, rotate, and keep a queued job.
|
||||
|
||||
The worker is stopped only for the durability section and started again before exit.
|
||||
RabbitMQ identity and the worker S3 secret are restored to a working pair.
|
||||
"""
|
||||
import json
|
||||
import secrets
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
S3_CONFIG = ROOT / "secrets" / "seaweed" / "s3.json"
|
||||
PASSWORDS = ROOT / "secrets" / "operator" / "passwords.json"
|
||||
EVIDENCE = ROOT / "evidence"
|
||||
BUCKET = "messenger-attachments"
|
||||
|
||||
|
||||
def run(command, check=True, input_text=None):
|
||||
result = subprocess.run(command, cwd=ROOT, input=input_text, capture_output=True, text=True)
|
||||
if check and result.returncode != 0:
|
||||
detail = ((result.stderr or result.stdout or "")[:180]).replace("\n", " ")
|
||||
raise SystemExit("command failed: " + detail)
|
||||
return result
|
||||
|
||||
|
||||
def psql(sql):
|
||||
return run(
|
||||
["docker", "exec", "-i", "mtusi-lab7-postgres-1", "psql", "-U", "messenger", "-d", "messenger", "-t", "-A"],
|
||||
input_text=sql,
|
||||
)
|
||||
|
||||
|
||||
def bao_put_worker_s3(access_key, secret_key):
|
||||
token = json.loads((ROOT / "secrets" / "operator" / "init.json").read_text(encoding="utf-8"))["root_token"]
|
||||
command = [
|
||||
"docker",
|
||||
"compose",
|
||||
"--env-file",
|
||||
".env",
|
||||
"exec",
|
||||
"-T",
|
||||
"-e",
|
||||
"BAO_ADDR=https://127.0.0.1:8200",
|
||||
"-e",
|
||||
"BAO_CACERT=/openbao/tls/ca.crt",
|
||||
"-e",
|
||||
"BAO_CLIENT_CERT=/openbao/tls/operator.crt",
|
||||
"-e",
|
||||
"BAO_CLIENT_KEY=/openbao/tls/operator.key",
|
||||
"-e",
|
||||
"BAO_TOKEN=" + token,
|
||||
"openbao",
|
||||
"bao",
|
||||
"kv",
|
||||
"put",
|
||||
"secret/worker/s3",
|
||||
"access_key=" + access_key,
|
||||
"secret_key=" + secret_key,
|
||||
]
|
||||
run(command)
|
||||
|
||||
|
||||
def restart_minio():
|
||||
run(["docker", "compose", "--env-file", ".env", "restart", "-t", "5", "minio"])
|
||||
for _ in range(30):
|
||||
result = run(["docker", "inspect", "--format", "{{.State.Health.Status}}", "mtusi-lab7-minio-1"], check=False)
|
||||
if (result.stdout or "").strip() == "healthy":
|
||||
return
|
||||
time.sleep(2)
|
||||
raise SystemExit("minio did not become healthy")
|
||||
|
||||
|
||||
def track1():
|
||||
result = run(
|
||||
["docker", "compose", "--env-file", ".env", "--profile", "tools", "run", "--rm", "--entrypoint", "python", "probe", "-m", "app.probes.track1"],
|
||||
check=False,
|
||||
)
|
||||
line = ""
|
||||
for item in (result.stdout or "").splitlines():
|
||||
if item.startswith("{"):
|
||||
line = item
|
||||
if not line:
|
||||
raise SystemExit("track1 probe produced no report")
|
||||
return json.loads(line), result.returncode
|
||||
|
||||
|
||||
def widen_s3(config):
|
||||
for identity in config["identities"]:
|
||||
if identity["name"] == "worker":
|
||||
identity["actions"] = [
|
||||
"Admin",
|
||||
"Read",
|
||||
"Write",
|
||||
"List",
|
||||
"Read:%s/canary/*" % BUCKET,
|
||||
"Write:%s/canary/*" % BUCKET,
|
||||
"List:%s/canary/*" % BUCKET,
|
||||
]
|
||||
return config
|
||||
|
||||
|
||||
def worker_secret(config):
|
||||
for identity in config["identities"]:
|
||||
if identity["name"] == "worker":
|
||||
return identity["credentials"][0]["accessKey"], identity["credentials"][0]["secretKey"]
|
||||
raise SystemExit("worker s3 identity is missing")
|
||||
|
||||
|
||||
def set_worker_secret(config, secret_key):
|
||||
for identity in config["identities"]:
|
||||
if identity["name"] == "worker":
|
||||
identity["credentials"][0]["secretKey"] = secret_key
|
||||
return config
|
||||
|
||||
|
||||
def old_secret_denied(access_key, secret_key):
|
||||
admin = ROOT / "secrets" / "seaweed" / "old-secret.json"
|
||||
admin.write_text(json.dumps({"access_key": access_key, "secret_key": secret_key}), encoding="utf-8")
|
||||
code = (
|
||||
"import json,boto3\n"
|
||||
"from botocore.config import Config\n"
|
||||
"from botocore.exceptions import ClientError\n"
|
||||
"cred=json.load(open('/c/cred.json'))\n"
|
||||
"client=boto3.client('s3',endpoint_url='https://minio:8333',aws_access_key_id=cred['access_key'],"
|
||||
"aws_secret_access_key=cred['secret_key'],region_name='us-east-1',verify='/c/ca.crt',"
|
||||
"config=Config(signature_version='s3v4',s3={'addressing_style':'path'}))\n"
|
||||
"try:\n"
|
||||
" client.put_object(Bucket='messenger-attachments',Key='staging/old-secret',Body=b'x')\n"
|
||||
" print('allow')\n"
|
||||
"except ClientError as exc:\n"
|
||||
" print('deny:'+str(exc.response.get('Error',{}).get('Code','error')))\n"
|
||||
"except Exception as exc:\n"
|
||||
" print('deny:'+type(exc).__name__)\n"
|
||||
)
|
||||
try:
|
||||
result = run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"--rm",
|
||||
"--network",
|
||||
"mtusi-lab7_app",
|
||||
"-v",
|
||||
str(ROOT / "secrets" / "pki" / "ca.crt") + ":/c/ca.crt:ro",
|
||||
"-v",
|
||||
str(admin) + ":/c/cred.json:ro",
|
||||
"--entrypoint",
|
||||
"python",
|
||||
"mtusi-lab7-messenger:1.7.0",
|
||||
"-c",
|
||||
code,
|
||||
]
|
||||
)
|
||||
return (result.stdout or "").strip().splitlines()[-1]
|
||||
finally:
|
||||
admin.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def restore_rabbit():
|
||||
run(
|
||||
["docker", "exec", "mtusi-lab7-rabbitmq-1", "rabbitmqctl", "add_user", "worker.messenger.internal", "unused-password"],
|
||||
check=False,
|
||||
)
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
"mtusi-lab7-rabbitmq-1",
|
||||
"rabbitmqctl",
|
||||
"set_permissions",
|
||||
"-p",
|
||||
"/",
|
||||
"worker.messenger.internal",
|
||||
"^attachment\\.process\\.v1$",
|
||||
"^$",
|
||||
"^attachment\\.process\\.v1$",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def queue_depth():
|
||||
result = run(["docker", "exec", "mtusi-lab7-rabbitmq-1", "rabbitmqctl", "list_queues", "name", "messages"])
|
||||
for line in (result.stdout or "").splitlines():
|
||||
if line.startswith("attachment.process.v1"):
|
||||
parts = line.split()
|
||||
return int(parts[-1])
|
||||
return 0
|
||||
|
||||
|
||||
def wait_queue(minimum, timeout=25):
|
||||
last = 0
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
last = queue_depth()
|
||||
if last >= minimum:
|
||||
return last
|
||||
time.sleep(1)
|
||||
return last
|
||||
|
||||
|
||||
def upload_while_worker_stopped():
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from tests.client import Client
|
||||
|
||||
client = Client("https://127.0.0.1:8443", str(ROOT / "secrets" / "pki" / "ca.crt"), timeout=15)
|
||||
alice, _bob, _eve, chat = client.bootstrap()
|
||||
attachment, _content = client.attachment(alice, chat)
|
||||
return alice, attachment["id"]
|
||||
|
||||
|
||||
def wait_status(actor, attachment_id):
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from tests.client import Client
|
||||
|
||||
client = Client("https://127.0.0.1:8443", str(ROOT / "secrets" / "pki" / "ca.crt"), timeout=15)
|
||||
deadline = time.monotonic() + 70
|
||||
last = ""
|
||||
while time.monotonic() < deadline:
|
||||
current = client.call("GET", "/api/v1/attachments/" + attachment_id, actor=actor, schema="Attachment")["json"]
|
||||
last = current["status"]
|
||||
if last in {"ready", "failed"}:
|
||||
return last
|
||||
time.sleep(1)
|
||||
return last or "timeout"
|
||||
|
||||
|
||||
def main():
|
||||
EVIDENCE.mkdir(exist_ok=True)
|
||||
original = S3_CONFIG.read_text(encoding="utf-8")
|
||||
original_config = json.loads(original)
|
||||
access_key, old_secret = worker_secret(original_config)
|
||||
report = {}
|
||||
committed = False
|
||||
try:
|
||||
least, code = track1()
|
||||
report["least_privilege_exit"] = code
|
||||
report["least_privilege"] = least
|
||||
psql("GRANT SELECT ON users TO worker_login;\n")
|
||||
broad = widen_s3(json.loads(original))
|
||||
S3_CONFIG.write_text(json.dumps(broad), encoding="utf-8")
|
||||
restart_minio()
|
||||
widened, widened_code = track1()
|
||||
report["broad_exit"] = widened_code
|
||||
report["broad"] = widened
|
||||
S3_CONFIG.write_text(original, encoding="utf-8")
|
||||
psql("REVOKE SELECT ON users FROM worker_login;\n")
|
||||
restart_minio()
|
||||
restored, restored_code = track1()
|
||||
report["restored_exit"] = restored_code
|
||||
report["restored"] = restored
|
||||
|
||||
run(["docker", "compose", "--env-file", ".env", "stop", "-t", "5", "worker"])
|
||||
actor, attachment_id = upload_while_worker_stopped()
|
||||
uuid.UUID(attachment_id)
|
||||
report["queued_before_revoke"] = psql(
|
||||
"SELECT status FROM attachments WHERE id = '%s';\n" % attachment_id
|
||||
).stdout.strip()
|
||||
report["queue_before_revoke"] = wait_queue(1)
|
||||
run(["docker", "exec", "mtusi-lab7-rabbitmq-1", "rabbitmqctl", "delete_user", "worker.messenger.internal"])
|
||||
report["queue_after_revoke"] = queue_depth()
|
||||
new_secret = secrets.token_hex(24)
|
||||
rotated = set_worker_secret(json.loads(original), new_secret)
|
||||
S3_CONFIG.write_text(json.dumps(rotated), encoding="utf-8")
|
||||
passwords = json.loads(PASSWORDS.read_text(encoding="utf-8"))
|
||||
passwords["worker_s3_secret"] = new_secret
|
||||
PASSWORDS.write_text(json.dumps(passwords), encoding="utf-8")
|
||||
bao_put_worker_s3(access_key, new_secret)
|
||||
committed = True
|
||||
restart_minio()
|
||||
report["old_secret"] = old_secret_denied(access_key, old_secret)
|
||||
restore_rabbit()
|
||||
run(["docker", "compose", "--env-file", ".env", "start", "worker"])
|
||||
for _ in range(30):
|
||||
health = run(["docker", "inspect", "--format", "{{.State.Health.Status}}", "mtusi-lab7-worker-1"], check=False)
|
||||
if (health.stdout or "").strip() == "healthy":
|
||||
break
|
||||
time.sleep(2)
|
||||
report["after_restore"] = wait_status(actor, attachment_id)
|
||||
report["attachment_id_prefix"] = attachment_id[:8]
|
||||
finally:
|
||||
if not committed:
|
||||
S3_CONFIG.write_text(original, encoding="utf-8")
|
||||
psql("REVOKE SELECT ON users FROM worker_login;\n")
|
||||
restore_rabbit()
|
||||
run(["docker", "compose", "--env-file", ".env", "start", "worker"], check=False)
|
||||
(EVIDENCE / "track1-run.json").write_text(json.dumps(report, indent=2), encoding="utf-8")
|
||||
print(json.dumps({key: report[key] for key in report if key != "least_privilege" and key != "broad" and key != "restored"}))
|
||||
print("least", report.get("least_privilege"))
|
||||
print("broad", report.get("broad"))
|
||||
print("restored", report.get("restored"))
|
||||
required = (
|
||||
report.get("least_privilege_exit") == 0
|
||||
and report.get("broad", {}).get("db_select_password_hash") == "allow"
|
||||
and str(report.get("broad", {}).get("s3_get_canary", "")).startswith("allow")
|
||||
and report.get("restored_exit") == 0
|
||||
and report.get("queued_before_revoke") == "queued"
|
||||
and report.get("queue_after_revoke", 0) >= 1
|
||||
and str(report.get("old_secret", "")).startswith("deny")
|
||||
and report.get("after_restore") == "ready"
|
||||
)
|
||||
if not required:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Unseal a restarted OpenBao from the operator's local init file. Prints only seal state."""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
INIT = ROOT / "secrets" / "operator" / "init.json"
|
||||
|
||||
|
||||
def bao(*args):
|
||||
command = [
|
||||
"docker",
|
||||
"compose",
|
||||
"--env-file",
|
||||
".env",
|
||||
"exec",
|
||||
"-T",
|
||||
"-e",
|
||||
"BAO_ADDR=https://127.0.0.1:8200",
|
||||
"-e",
|
||||
"BAO_CACERT=/openbao/tls/ca.crt",
|
||||
"-e",
|
||||
"BAO_CLIENT_CERT=/openbao/tls/operator.crt",
|
||||
"-e",
|
||||
"BAO_CLIENT_KEY=/openbao/tls/operator.key",
|
||||
"openbao",
|
||||
"bao",
|
||||
*args,
|
||||
]
|
||||
return subprocess.run(command, cwd=ROOT, capture_output=True, text=True)
|
||||
|
||||
|
||||
def status():
|
||||
result = bao("status", "-format=json")
|
||||
text = result.stdout or ""
|
||||
start = text.find("{")
|
||||
end = text.rfind("}")
|
||||
if start < 0 or end < start:
|
||||
raise SystemExit("OpenBao status is not available")
|
||||
return json.loads(text[start : end + 1])
|
||||
|
||||
|
||||
def main():
|
||||
current = status()
|
||||
if not current.get("initialized"):
|
||||
raise SystemExit("OpenBao is not initialized")
|
||||
if not current.get("sealed"):
|
||||
print("sealed=false")
|
||||
return
|
||||
if not INIT.is_file():
|
||||
raise SystemExit("operator init file is missing")
|
||||
keys = json.loads(INIT.read_text(encoding="utf-8")).get("unseal_keys_b64") or []
|
||||
for key in keys[:2]:
|
||||
result = bao("operator", "unseal", key)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit("OpenBao unseal failed")
|
||||
print("sealed=" + str(bool(status().get("sealed"))).lower())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,325 @@
|
||||
"""Local grade-4 proofs: rehash, session expiry, LB leaf rotation, sealed OpenBao, telemetry.
|
||||
|
||||
Prints statuses only. Restores certificates, the allowlist and the vault before exiting.
|
||||
"""
|
||||
import json
|
||||
import ssl
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PKI = ROOT / "secrets" / "pki"
|
||||
OPERATOR = ROOT / "secrets" / "operator"
|
||||
EVIDENCE = ROOT / "evidence"
|
||||
BASE = "https://127.0.0.1:8443"
|
||||
CA = PKI / "ca.crt"
|
||||
|
||||
|
||||
def run(command, check=True, input_text=None):
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
input=input_text,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if check and result.returncode != 0:
|
||||
detail = ((result.stderr or result.stdout or "")[:240]).replace("\n", " ")
|
||||
raise SystemExit("command failed: " + detail)
|
||||
return result
|
||||
|
||||
|
||||
def bao(*args, token=None):
|
||||
command = [
|
||||
"docker",
|
||||
"compose",
|
||||
"--env-file",
|
||||
".env",
|
||||
"exec",
|
||||
"-T",
|
||||
"-e",
|
||||
"BAO_ADDR=https://127.0.0.1:8200",
|
||||
"-e",
|
||||
"BAO_CACERT=/openbao/tls/ca.crt",
|
||||
"-e",
|
||||
"BAO_CLIENT_CERT=/openbao/tls/operator.crt",
|
||||
"-e",
|
||||
"BAO_CLIENT_KEY=/openbao/tls/operator.key",
|
||||
]
|
||||
if token:
|
||||
command += ["-e", "BAO_TOKEN=" + token]
|
||||
command += ["openbao", "bao", *args]
|
||||
return run(command, check=False)
|
||||
|
||||
|
||||
def bao_status():
|
||||
result = bao("status", "-format=json")
|
||||
text = result.stdout or ""
|
||||
start = text.find("{")
|
||||
end = text.rfind("}")
|
||||
if start < 0 or end < start:
|
||||
return None
|
||||
return json.loads(text[start : end + 1])
|
||||
|
||||
|
||||
def root_token():
|
||||
return json.loads((OPERATOR / "init.json").read_text(encoding="utf-8"))["root_token"]
|
||||
|
||||
|
||||
def https(path, cert=None, key=None, data=None, method=None, headers=None):
|
||||
ctx = ssl.create_default_context(cafile=str(CA))
|
||||
if cert and key:
|
||||
ctx.load_cert_chain(str(cert), str(key))
|
||||
request = urllib.request.Request(BASE + path, data=data, headers=headers or {}, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(request, context=ctx, timeout=8) as response:
|
||||
return response.status, response.read()
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code, exc.read()
|
||||
|
||||
|
||||
def psql(sql):
|
||||
result = run(
|
||||
["docker", "exec", "-i", "mtusi-lab7-postgres-1", "psql", "-U", "messenger", "-d", "messenger", "-t", "-A"],
|
||||
input_text=sql,
|
||||
)
|
||||
return (result.stdout or "").strip()
|
||||
|
||||
|
||||
def mtls_ready(cert, key):
|
||||
code = (
|
||||
"import ssl,urllib.request,urllib.error\n"
|
||||
"ctx=ssl.create_default_context(cafile='/c/ca.crt')\n"
|
||||
"ctx.load_cert_chain('/c/client.crt','/c/client.key')\n"
|
||||
"try:\n"
|
||||
" r=urllib.request.urlopen('https://api-1:8443/health/ready', context=ctx, timeout=8)\n"
|
||||
" print(r.status)\n"
|
||||
"except urllib.error.HTTPError as exc:\n"
|
||||
" print('http', exc.code)\n"
|
||||
"except Exception as exc:\n"
|
||||
" print(type(exc).__name__)\n"
|
||||
)
|
||||
result = run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"--rm",
|
||||
"--network",
|
||||
"mtusi-lab7_app",
|
||||
"-v",
|
||||
str(CA) + ":/c/ca.crt:ro",
|
||||
"-v",
|
||||
str(cert) + ":/c/client.crt:ro",
|
||||
"-v",
|
||||
str(key) + ":/c/client.key:ro",
|
||||
"--entrypoint",
|
||||
"python",
|
||||
"mtusi-lab7-messenger:1.7.0",
|
||||
"-c",
|
||||
code,
|
||||
]
|
||||
)
|
||||
return (result.stdout or "").strip()
|
||||
|
||||
|
||||
def issue_lb_leaf():
|
||||
ca_key = serialization.load_pem_private_key((PKI / "ca.key").read_bytes(), password=None)
|
||||
ca_cert = x509.load_pem_x509_certificate((PKI / "ca.crt").read_bytes())
|
||||
key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
|
||||
now = __import__("datetime").datetime.now(__import__("datetime").timezone.utc)
|
||||
cert = (
|
||||
x509.CertificateBuilder()
|
||||
.subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "lb.messenger.internal")]))
|
||||
.issuer_name(ca_cert.subject)
|
||||
.public_key(key.public_key())
|
||||
.serial_number(x509.random_serial_number())
|
||||
.not_valid_before(now - __import__("datetime").timedelta(minutes=5))
|
||||
.not_valid_after(now + __import__("datetime").timedelta(days=7))
|
||||
.add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True)
|
||||
.add_extension(x509.SubjectAlternativeName([x509.DNSName("lb.messenger.internal")]), critical=False)
|
||||
.add_extension(x509.ExtendedKeyUsage([ExtendedKeyUsageOID.CLIENT_AUTH]), critical=False)
|
||||
.sign(ca_key, hashes.SHA256())
|
||||
)
|
||||
leaf_key = PKI / "lb-rotated.key"
|
||||
leaf_crt = PKI / "lb-rotated.crt"
|
||||
leaf_key.write_bytes(
|
||||
key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption())
|
||||
)
|
||||
leaf_crt.write_bytes(cert.public_bytes(serialization.Encoding.PEM))
|
||||
return leaf_crt, leaf_key, format(cert.serial_number, "X")
|
||||
|
||||
|
||||
def reload_nginx():
|
||||
run(["docker", "exec", "mtusi-lab7-nginx-1", "nginx", "-s", "reload"])
|
||||
|
||||
|
||||
def count_leaks():
|
||||
patterns = ("Bearer ", "password=", "postgresql://", "aws_secret_access_key", "BEGIN PRIVATE")
|
||||
counts = {pattern: 0 for pattern in patterns}
|
||||
for name in ("mtusi-lab7-api-1-1", "mtusi-lab7-api-2-1", "mtusi-lab7-worker-1", "mtusi-lab7-loki-1"):
|
||||
result = run(["docker", "logs", name, "--since", "20m"], check=False)
|
||||
text = (result.stdout or "") + (result.stderr or "")
|
||||
for pattern in patterns:
|
||||
counts[pattern] += text.count(pattern)
|
||||
return counts
|
||||
|
||||
|
||||
def rehash_and_expiry():
|
||||
from argon2 import PasswordHasher
|
||||
|
||||
username = "rehash" + uuid.uuid4().hex[:8]
|
||||
password = "rotate-horse-2"
|
||||
body = json.dumps({"username": username, "display_name": "Rehash", "password": password}).encode("utf-8")
|
||||
status, _ = https("/api/v1/users", data=body, method="POST", headers={"Content-Type": "application/json"})
|
||||
if status != 201:
|
||||
raise SystemExit("rehash register failed")
|
||||
old = PasswordHasher(time_cost=1, memory_cost=8192, parallelism=1).hash(password)
|
||||
psql("UPDATE users SET password_hash = %s WHERE username = %s;\n" % (_quote(old), _quote(username)))
|
||||
before = psql("SELECT substring(password_hash from 'm=[0-9]+,t=[0-9]+,p=[0-9]+') FROM users WHERE username = %s;\n" % _quote(username))
|
||||
import base64
|
||||
|
||||
basic = base64.b64encode((username + ":" + password).encode("utf-8")).decode("ascii")
|
||||
login_status, raw = https("/api/v1/auth/sessions", method="POST", headers={"Authorization": "Basic " + basic})
|
||||
after = psql("SELECT substring(password_hash from 'm=[0-9]+,t=[0-9]+,p=[0-9]+') FROM users WHERE username = %s;\n" % _quote(username))
|
||||
token = json.loads(raw.decode("utf-8")).get("token") if login_status == 201 else ""
|
||||
expired = None
|
||||
if token:
|
||||
psql(
|
||||
"UPDATE sessions SET expires_at = NOW() - INTERVAL '1 minute' "
|
||||
"WHERE user_id = (SELECT id FROM users WHERE username = %s);\n" % _quote(username)
|
||||
)
|
||||
expired, _ = https("/api/v1/users/me", headers={"Authorization": "Bearer " + token})
|
||||
psql("DELETE FROM sessions WHERE user_id IN (SELECT id FROM users WHERE username = %s);\n" % _quote(username))
|
||||
psql("DELETE FROM users WHERE username = %s;\n" % _quote(username))
|
||||
return {
|
||||
"login": login_status,
|
||||
"params_before": before,
|
||||
"params_after": after,
|
||||
"expired_session": expired,
|
||||
}
|
||||
|
||||
|
||||
def _quote(value):
|
||||
return "'" + value.replace("'", "''") + "'"
|
||||
|
||||
|
||||
def _until(action, expected, attempts=8):
|
||||
last = ""
|
||||
for _ in range(attempts):
|
||||
last = action()
|
||||
if last == expected or last.startswith(expected):
|
||||
return last
|
||||
time.sleep(1)
|
||||
return last
|
||||
|
||||
|
||||
def rotate_leaf():
|
||||
allow = PKI / "lb-allowed-serials.txt"
|
||||
backup_allow = allow.read_bytes()
|
||||
backup_crt = (PKI / "lb.crt").read_bytes()
|
||||
backup_key = (PKI / "lb.key").read_bytes()
|
||||
leaf_crt, leaf_key, serial = issue_lb_leaf()
|
||||
try:
|
||||
allow.write_text(serial + "\n", encoding="ascii")
|
||||
rotated = _until(lambda: mtls_ready(leaf_crt, leaf_key), "200")
|
||||
retired = _until(lambda: mtls_ready(PKI / "lb.crt", PKI / "lb.key"), "http 403")
|
||||
(PKI / "lb.crt").write_bytes(leaf_crt.read_bytes())
|
||||
(PKI / "lb.key").write_bytes(leaf_key.read_bytes())
|
||||
reload_nginx()
|
||||
time.sleep(1)
|
||||
public, _ = https("/health/ready")
|
||||
return {"new_serial_ready": rotated, "old_serial": retired, "public_after_reload": public}
|
||||
finally:
|
||||
allow.write_bytes(backup_allow)
|
||||
(PKI / "lb.crt").write_bytes(backup_crt)
|
||||
(PKI / "lb.key").write_bytes(backup_key)
|
||||
leaf_crt.unlink(missing_ok=True)
|
||||
leaf_key.unlink(missing_ok=True)
|
||||
reload_nginx()
|
||||
|
||||
|
||||
def seal_cycle():
|
||||
before, _ = https("/health/ready")
|
||||
open_login, _ = https(
|
||||
"/api/v1/auth/sessions",
|
||||
method="POST",
|
||||
headers={"Authorization": "Basic YmFkOmJhZA=="},
|
||||
)
|
||||
sealed = bao("operator", "seal", token=root_token())
|
||||
if sealed.returncode != 0:
|
||||
raise SystemExit("OpenBao seal failed")
|
||||
report = {"ready_before": before, "login_before_seal": open_login, "login_while_sealed": None, "secret_env": []}
|
||||
try:
|
||||
time.sleep(20)
|
||||
deadline = time.monotonic() + 30
|
||||
denied = None
|
||||
while time.monotonic() < deadline:
|
||||
denied, _ = https(
|
||||
"/api/v1/auth/sessions",
|
||||
method="POST",
|
||||
headers={"Authorization": "Basic YmFkOmJhZA=="},
|
||||
)
|
||||
if denied == 503:
|
||||
break
|
||||
time.sleep(2)
|
||||
env = run(["docker", "exec", "mtusi-lab7-api-1-1", "printenv"], check=False)
|
||||
leaked = [
|
||||
line.split("=", 1)[0]
|
||||
for line in (env.stdout or "").splitlines()
|
||||
if line.startswith(("DATABASE_URL=", "S3_SECRET_ACCESS_KEY=", "VALKEY_PASSWORD=", "BAO_TOKEN="))
|
||||
]
|
||||
report["login_while_sealed"] = denied
|
||||
report["secret_env"] = leaked
|
||||
finally:
|
||||
run(["python", "scripts/unseal.py"])
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
status, _ = https("/health/ready")
|
||||
if status == 200:
|
||||
break
|
||||
time.sleep(2)
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
EVIDENCE.mkdir(exist_ok=True)
|
||||
report = {
|
||||
"rehash": rehash_and_expiry(),
|
||||
"leaf": rotate_leaf(),
|
||||
"telemetry_leak_counts": count_leaks(),
|
||||
}
|
||||
sealed = seal_cycle()
|
||||
report["sealed"] = sealed if isinstance(sealed, dict) else {"restored": True}
|
||||
ready, _ = https("/health/ready")
|
||||
report["ready_after_unseal"] = ready
|
||||
(EVIDENCE / "verify-local.json").write_text(json.dumps(report, indent=2), encoding="utf-8")
|
||||
print(json.dumps(report))
|
||||
ok = (
|
||||
report["rehash"]["login"] == 201
|
||||
and report["rehash"]["params_before"].startswith("m=8192")
|
||||
and report["rehash"]["params_after"] == "m=19456,t=2,p=1"
|
||||
and report["rehash"]["expired_session"] == 401
|
||||
and report["leaf"]["new_serial_ready"] == "200"
|
||||
and report["leaf"]["old_serial"].startswith("http 403")
|
||||
and report["leaf"]["public_after_reload"] == 200
|
||||
and report["sealed"]["login_before_seal"] == 401
|
||||
and report["sealed"]["login_while_sealed"] == 503
|
||||
and report["sealed"]["secret_env"] == []
|
||||
and ready == 200
|
||||
and all(value == 0 for value in report["telemetry_leak_counts"].values())
|
||||
)
|
||||
if not ok:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user