Добавить стенд лабораторной 7 с mTLS, OpenBao и least privilege worker.

Закрывает обязательный минимум и трек 1: ротация leaf и секрета, sealed OpenBao и сохранность queued-задачи.
This commit is contained in:
2026-11-07 20:46:02 +03:00
parent 20afc2c790
commit baafe2a3ae
76 changed files with 6473 additions and 65 deletions
+544
View File
@@ -0,0 +1,544 @@
"""Create local CA, secrets and, with --up, initialize the stand. Never prints secret values."""
import datetime
import ipaddress
import json
import os
import secrets
import shutil
import subprocess
import sys
import time
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
PKI = ROOT / "secrets" / "pki"
OPERATOR = ROOT / "secrets" / "operator"
def ensure_cryptography():
try:
import cryptography # noqa: F401
return
except ImportError:
pass
venv = ROOT / ".venv-bootstrap"
py = venv / "Scripts" / "python.exe"
if not py.exists():
subprocess.check_call([sys.executable, "-m", "venv", str(venv)])
subprocess.check_call([str(py), "-m", "pip", "install", "cryptography==44.0.2"])
os.execv(str(py), [str(py), str(Path(__file__).resolve()), *sys.argv[1:]])
def write_pem(path, data):
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
def build_material():
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID
def private():
return rsa.generate_private_key(public_exponent=65537, key_size=2048)
def dump_key(key):
return key.private_bytes(
serialization.Encoding.PEM,
serialization.PrivateFormat.PKCS8,
serialization.NoEncryption(),
)
def dump_cert(cert):
return cert.public_bytes(serialization.Encoding.PEM)
def name(cn):
return x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, cn)])
now = datetime.datetime.now(datetime.timezone.utc)
def certificate(cn, sans, ips, eku, days, issuer_key, issuer_cert, expired=False):
key = private()
if expired:
start = now - datetime.timedelta(days=3)
end = now - datetime.timedelta(days=1)
else:
start = now - datetime.timedelta(minutes=10)
end = now + datetime.timedelta(days=days)
alt = [x509.DNSName(item) for item in sans]
alt += [x509.IPAddress(ipaddress.ip_address(item)) for item in ips]
uses = []
if "client" in eku:
uses.append(ExtendedKeyUsageOID.CLIENT_AUTH)
if "server" in eku:
uses.append(ExtendedKeyUsageOID.SERVER_AUTH)
builder = (
x509.CertificateBuilder()
.subject_name(name(cn))
.issuer_name(issuer_cert.subject if issuer_cert is not None else name(cn))
.public_key(key.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(start)
.not_valid_after(end)
.add_extension(x509.BasicConstraints(ca=issuer_cert is None, path_length=0 if issuer_cert is None else None), critical=True)
)
if issuer_cert is None:
builder = builder.add_extension(
x509.KeyUsage(
digital_signature=True,
key_cert_sign=True,
crl_sign=True,
content_commitment=False,
key_encipherment=False,
data_encipherment=False,
key_agreement=False,
encipher_only=False,
decipher_only=False,
),
critical=True,
)
else:
builder = builder.add_extension(x509.SubjectAlternativeName(alt), critical=False)
builder = builder.add_extension(x509.ExtendedKeyUsage(uses), critical=False)
signed = builder.sign(issuer_key if issuer_cert is not None else key, hashes.SHA256())
return key, signed
PKI.mkdir(parents=True, exist_ok=True)
if not (PKI / "ca.crt").exists():
ca_key, ca_cert = certificate("messenger-lab-ca", [], [], [], 30, None, None)
other_key, other_cert = certificate("foreign-lab-ca", [], [], [], 30, None, None)
write_pem(PKI / "ca.key", dump_key(ca_key))
write_pem(PKI / "ca.crt", dump_cert(ca_cert))
write_pem(PKI / "other-ca.key", dump_key(other_key))
write_pem(PKI / "other-ca.crt", dump_cert(other_cert))
leaves = {
"ingress": ("localhost", ["localhost", "nginx"], ["127.0.0.1"], ["server"], 14, False, ca_key, ca_cert),
"lb": ("lb.messenger.internal", ["lb.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert),
"api-server": ("api.messenger.internal", ["messenger_api", "api-1", "api-2", "api.messenger.internal"], [], ["server"], 14, False, ca_key, ca_cert),
"api-client": ("api.messenger.internal", ["api.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert),
"worker-client": ("worker.messenger.internal", ["worker.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert),
"stranger": ("stranger.messenger.internal", ["stranger.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert),
"expired": ("lb.messenger.internal", ["lb.messenger.internal"], [], ["client"], 1, True, ca_key, ca_cert),
"other-client": ("lb.messenger.internal", ["lb.messenger.internal"], [], ["client"], 14, False, other_key, other_cert),
"postgres": ("postgres", ["postgres"], [], ["server"], 14, False, ca_key, ca_cert),
"minio": ("minio", ["minio"], [], ["server"], 14, False, ca_key, ca_cert),
"rabbitmq": ("rabbitmq", ["rabbitmq"], [], ["server"], 14, False, ca_key, ca_cert),
"valkey": ("valkey", ["valkey"], [], ["server", "client"], 14, False, ca_key, ca_cert),
"openbao": ("openbao", ["openbao", "localhost"], ["127.0.0.1"], ["server"], 14, False, ca_key, ca_cert),
"health": ("health.messenger.internal", ["health.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert),
"operator": ("operator.messenger.internal", ["operator.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert),
"otel-edge": ("otel-edge", ["otel-edge"], [], ["server"], 14, False, ca_key, ca_cert),
"collector": ("otel-collector", ["otel-collector"], [], ["server", "client"], 14, False, ca_key, ca_cert),
"prometheus": ("prometheus", ["prometheus"], [], ["server"], 14, False, ca_key, ca_cert),
"loki": ("loki", ["loki"], [], ["server"], 14, False, ca_key, ca_cert),
"tempo": ("tempo", ["tempo"], [], ["server"], 14, False, ca_key, ca_cert),
"grafana": ("grafana", ["grafana", "localhost"], ["127.0.0.1"], ["server"], 14, False, ca_key, ca_cert),
}
serials = {}
for stem, spec in leaves.items():
cn, sans, ips, eku, days, expired, issuer_key, issuer_cert = spec
key, cert = certificate(cn, sans, ips, eku, days, issuer_key, issuer_cert, expired)
write_pem(PKI / (stem + ".key"), dump_key(key))
write_pem(PKI / (stem + ".crt"), dump_cert(cert))
serials[stem] = cert.serial_number
minio_dir = PKI / "minio"
minio_dir.mkdir(exist_ok=True)
shutil.copyfile(PKI / "minio.crt", minio_dir / "public.crt")
shutil.copyfile(PKI / "minio.key", minio_dir / "private.key")
(PKI / "serials.json").write_text(json.dumps({key: format(value, "X") for key, value in serials.items()}), encoding="utf-8")
return json.loads((PKI / "serials.json").read_text(encoding="utf-8"))
def password():
return secrets.token_hex(24)
def material_passwords():
OPERATOR.mkdir(parents=True, exist_ok=True)
path = OPERATOR / "passwords.json"
current = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
defaults = {
"postgres_password": password(),
"api_db_password": password(),
"worker_db_password": password(),
"minio_root_user": "lab-root",
"minio_root_password": password(),
"api_s3_key": "api-" + secrets.token_hex(6),
"api_s3_secret": password(),
"worker_s3_key": "worker-" + secrets.token_hex(6),
"worker_s3_secret": password(),
"valkey_password": password(),
"grafana_password": password(),
"s3_admin_key": "admin-" + secrets.token_hex(6),
"s3_admin_secret": password(),
}
changed = False
for key, value in defaults.items():
if key not in current:
current[key] = value
changed = True
if changed:
path.write_text(json.dumps(current), encoding="utf-8")
return current
def colon(serial):
hexed = serial if len(serial) % 2 == 0 else "0" + serial
return ":".join(hexed[i : i + 2] for i in range(0, len(hexed), 2))
def write_configs(pw, serials):
(ROOT / "secrets" / "db").mkdir(parents=True, exist_ok=True)
(ROOT / "secrets" / "db" / "01-roles.sql").write_text(
"CREATE ROLE api_login LOGIN PASSWORD '{api}';\nCREATE ROLE worker_login LOGIN PASSWORD '{worker}';\n".format(
api=pw["api_db_password"], worker=pw["worker_db_password"]
),
encoding="utf-8",
)
(ROOT / "secrets" / "postgres.env").write_text(
"POSTGRES_USER=messenger\nPOSTGRES_PASSWORD={password}\nPOSTGRES_DB=messenger\n".format(password=pw["postgres_password"]),
encoding="utf-8",
)
(ROOT / "secrets" / "minio.env").write_text(
"MINIO_ROOT_USER={user}\nMINIO_ROOT_PASSWORD={password}\n".format(
user=pw["minio_root_user"], password=pw["minio_root_password"]
),
encoding="utf-8",
)
(ROOT / "secrets" / "grafana.env").write_text(
"GF_SECURITY_ADMIN_PASSWORD={password}\n".format(password=pw["grafana_password"]),
encoding="utf-8",
)
(ROOT / "secrets" / "valkey").mkdir(parents=True, exist_ok=True)
(ROOT / "secrets" / "valkey" / "users.acl").write_text(
"\n".join(
[
"user default off",
"user health on nopass +ping",
"user api on >{password} ~login:* +incr +incrby +expire +ttl +ping +get".format(password=pw["valkey_password"]),
"",
]
),
encoding="utf-8",
)
(PKI / "lb-allowed-serials.txt").write_text(serials["lb"] + "\n" + colon(serials["lb"]) + "\n", encoding="utf-8")
seen = {}
for stem in ("api-client", "worker-client"):
raw = serials[stem].upper()
for item in (raw, colon(raw)):
seen[item.lower()] = item
lines = ["map $ssl_client_s_dn $otel_cn_ok {", " default 0;", ' "~*CN=api\\.messenger\\.internal" 1;', ' "~*CN=worker\\.messenger\\.internal" 1;', "}"]
lines.append("map $ssl_client_serial $otel_serial_ok {")
lines.append(" default 0;")
for item in seen.values():
lines.append(' "%s" 1;' % item)
lines.append("}")
nginx = ROOT / "secrets" / "nginx"
nginx.mkdir(parents=True, exist_ok=True)
(nginx / "serials.conf").write_text("\n".join(lines) + "\n", encoding="utf-8")
(ROOT / ".env").write_text(
"\n".join(
[
"SESSION_TTL_SECONDS=3600",
"IDEMPOTENCY_TTL_SECONDS=86400",
"WORKER_CONCURRENCY=2",
"JOB_MAX_ATTEMPTS=5",
"JOB_TIMEOUT_SECONDS=60",
"STAGING_RETENTION_SECONDS=86400",
"",
]
),
encoding="utf-8",
)
for name in ("api-1", "api-2", "worker"):
(ROOT / "run" / "instances" / name).mkdir(parents=True, exist_ok=True)
def compose(*args, check=True, env=None):
command = ["docker", "compose", "--env-file", ".env", *args]
return subprocess.run(command, cwd=ROOT, check=check, env=env)
def bao(*args, token=None):
command = [
"docker",
"compose",
"--env-file",
".env",
"exec",
"-T",
"-e",
"BAO_ADDR=https://127.0.0.1:8200",
"-e",
"BAO_CACERT=/openbao/tls/ca.crt",
"-e",
"BAO_CLIENT_CERT=/openbao/tls/operator.crt",
"-e",
"BAO_CLIENT_KEY=/openbao/tls/operator.key",
]
if token:
command += ["-e", "BAO_TOKEN=" + token]
command += ["openbao", "bao", *args]
return subprocess.run(command, cwd=ROOT, capture_output=True, text=True)
def parse_json(text):
start = text.find("{")
end = text.rfind("}")
if start < 0 or end < start:
return None
return json.loads(text[start : end + 1])
def wait_status():
for _ in range(40):
result = bao("status", "-format=json")
payload = parse_json(result.stdout or "")
if payload is not None and "initialized" in payload:
return payload
time.sleep(2)
raise SystemExit("OpenBao did not start")
def init_openbao(pw):
status = wait_status()
init_path = OPERATOR / "init.json"
if not status.get("initialized"):
result = bao("operator", "init", "-key-shares=3", "-key-threshold=2", "-format=json")
payload = parse_json(result.stdout or "")
if result.returncode != 0 or payload is None or "root_token" not in payload:
raise SystemExit("OpenBao init failed")
init_path.write_text(json.dumps(payload), encoding="utf-8")
status = wait_status()
if status.get("sealed", True):
keys = json.loads(init_path.read_text(encoding="utf-8"))["unseal_keys_b64"]
for key in keys[:2]:
result = bao("operator", "unseal", key)
if result.returncode != 0:
raise SystemExit("OpenBao unseal failed")
root = json.loads(init_path.read_text(encoding="utf-8"))["root_token"]
bao("secrets", "enable", "-path=secret", "kv-v2", token=root)
bao("policy", "write", "api", "/openbao/policies/api.hcl", token=root)
bao("policy", "write", "worker", "/openbao/policies/worker.hcl", token=root)
bao("auth", "enable", "cert", token=root)
for role, cn in (("api", "api.messenger.internal"), ("worker", "worker.messenger.internal")):
result = bao(
"write",
"auth/cert/certs/" + role,
"display_name=" + role,
"policies=" + role,
"certificate=@/openbao/tls/ca.crt",
"allowed_common_names=" + cn,
"token_ttl=20m",
"token_max_ttl=2h",
token=root,
)
if result.returncode != 0:
raise SystemExit("OpenBao cert role " + role + " failed")
writes = {
"secret/api/db": ["username=api_login", "password=" + pw["api_db_password"], "host=postgres", "dbname=messenger"],
"secret/worker/db": ["username=worker_login", "password=" + pw["worker_db_password"], "host=postgres", "dbname=messenger"],
"secret/api/s3": ["access_key=" + pw["api_s3_key"], "secret_key=" + pw["api_s3_secret"]],
"secret/worker/s3": ["access_key=" + pw["worker_s3_key"], "secret_key=" + pw["worker_s3_secret"]],
"secret/api/valkey": ["username=api", "password=" + pw["valkey_password"], "host=valkey"],
}
for path, fields in writes.items():
result = bao("kv", "put", path, *fields, token=root)
if result.returncode != 0:
raise SystemExit("OpenBao kv put failed for " + path.split("/")[1])
def wait_healthy(*services):
for _ in range(50):
result = subprocess.run(
["docker", "compose", "--env-file", ".env", "ps", "--format", "{{.Service}} {{.Health}}", *services],
cwd=ROOT,
capture_output=True,
text=True,
)
rows = [line.strip() for line in result.stdout.splitlines() if line.strip()]
if len(rows) >= len(services) and all(line.endswith(" healthy") for line in rows):
return
time.sleep(3)
raise SystemExit("services did not become healthy: " + " ".join(services))
def write_s3_config(pw):
bucket = "messenger-attachments"
config = {
"identities": [
{
"name": "admin",
"credentials": [{"accessKey": pw["s3_admin_key"], "secretKey": pw["s3_admin_secret"]}],
"actions": ["Admin", "Read", "Write", "List"],
},
{
"name": "api",
"credentials": [{"accessKey": pw["api_s3_key"], "secretKey": pw["api_s3_secret"]}],
"actions": [
"Read:{0}/staging/*".format(bucket),
"Write:{0}/staging/*".format(bucket),
"List:{0}/staging/*".format(bucket),
"Read:{0}/originals/*".format(bucket),
"List:{0}/originals/*".format(bucket),
"Read:{0}/variants/*".format(bucket),
"List:{0}/variants/*".format(bucket),
],
},
{
"name": "worker",
"credentials": [{"accessKey": pw["worker_s3_key"], "secretKey": pw["worker_s3_secret"]}],
"actions": [
"Read:{0}/staging/*".format(bucket),
"Write:{0}/staging/*".format(bucket),
"List:{0}/staging/*".format(bucket),
"Read:{0}/originals/*".format(bucket),
"Write:{0}/originals/*".format(bucket),
"List:{0}/originals/*".format(bucket),
"Read:{0}/variants/*".format(bucket),
"Write:{0}/variants/*".format(bucket),
"List:{0}/variants/*".format(bucket),
],
},
]
}
target = ROOT / "secrets" / "seaweed"
target.mkdir(parents=True, exist_ok=True)
(target / "s3.json").write_text(json.dumps(config), encoding="utf-8")
def configure_bucket(pw):
admin = ROOT / "secrets" / "seaweed" / "admin.json"
admin.write_text(
json.dumps({"access_key": pw["s3_admin_key"], "secret_key": pw["s3_admin_secret"]}),
encoding="utf-8",
)
result = subprocess.run(
[
"docker",
"run",
"--rm",
"--network",
"mtusi-lab7_app",
"-v",
str(PKI / "ca.crt") + ":/certs/ca.crt:ro",
"-v",
str(admin) + ":/certs/admin.json:ro",
"--entrypoint",
"python",
"mtusi-lab7-messenger:1.7.0",
"-c",
(
"import json,boto3; from botocore.config import Config; "
"admin=json.load(open('/certs/admin.json')); "
"c=boto3.client('s3',endpoint_url='https://minio:8333',aws_access_key_id=admin['access_key'],"
"aws_secret_access_key=admin['secret_key'],region_name='us-east-1',verify='/certs/ca.crt',"
"config=Config(signature_version='s3v4',s3={'addressing_style':'path'})); "
"buckets=[b['Name'] for b in c.list_buckets().get('Buckets',[])]; "
"c.create_bucket(Bucket='messenger-attachments') if 'messenger-attachments' not in buckets else None; "
"c.put_object(Bucket='messenger-attachments',Key='canary/marker',Body=b'canary-not-a-secret'); "
"print('s3-bootstrap-ok')"
),
],
capture_output=True,
text=True,
)
admin.unlink(missing_ok=True)
if result.returncode != 0 or "s3-bootstrap-ok" not in (result.stdout or ""):
raise SystemExit("s3 bootstrap failed")
def configure_rabbit():
users = {
"api.messenger.internal": ("^attachment\\.process(\\.v1)?$", "^attachment\\.process$", "^$"),
"worker.messenger.internal": ("^attachment\\.process\\.v1$", "^$", "^attachment\\.process\\.v1$"),
}
subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "await_online_nodes", "1"],
cwd=ROOT,
capture_output=True,
text=True,
)
for user, perms in users.items():
last = None
for _ in range(10):
subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "add_user", user, password()],
cwd=ROOT,
capture_output=True,
text=True,
)
last = subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "set_permissions", "-p", "/", user, *perms],
cwd=ROOT,
capture_output=True,
text=True,
)
if last.returncode == 0:
break
time.sleep(3)
if last is None or last.returncode != 0:
raise SystemExit("rabbit permissions failed for a service user")
expressions = [
'rabbit_exchange:declare({resource, <<"/">>, exchange, <<"attachment.process">>}, direct, true, false, false, [], <<"bootstrap">>).',
'rabbit_binding:add({binding, {resource, <<"/">>, exchange, <<"attachment.process">>}, <<"attachment.process.v1">>, {resource, <<"/">>, queue, <<"attachment.process.v1">>}, []}, <<"bootstrap">>).',
]
for expression in expressions:
topology = subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "eval", expression],
cwd=ROOT,
capture_output=True,
text=True,
)
if topology.returncode != 0:
raise SystemExit("rabbit topology failed")
subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "delete_user", "guest"],
cwd=ROOT,
capture_output=True,
text=True,
)
def wait_ready():
import ssl
ctx = ssl.create_default_context(cafile=str(PKI / "ca.crt"))
for _ in range(40):
try:
with urllib.request.urlopen("https://127.0.0.1:8443/health/ready", context=ctx, timeout=5) as response:
if response.status == 200:
print("ready https://127.0.0.1:8443/health/ready")
return
except Exception:
time.sleep(3)
raise SystemExit("public HTTPS endpoint did not become ready")
def up(pw):
write_s3_config(pw)
compose("up", "-d", "openbao")
init_openbao(pw)
compose("build", "api-1", "postgres", "rabbitmq")
compose("up", "-d", "--build", "postgres", "minio", "rabbitmq", "valkey")
wait_healthy("postgres", "minio", "rabbitmq", "valkey")
configure_bucket(pw)
configure_rabbit()
compose("up", "-d", "--build", "api-1", "api-2", "worker", "nginx", "otel-edge", "otel-collector", "prometheus", "loki", "tempo", "grafana")
wait_ready()
def main():
ensure_cryptography()
serials = build_material()
pw = material_passwords()
write_configs(pw, serials)
print("local material is in secrets/ and .env; private keys were not printed")
if "--up" in sys.argv:
up(pw)
if __name__ == "__main__":
main()
+30
View File
@@ -0,0 +1,30 @@
#!/bin/sh
# Restart user-data stores and the application. OpenBao stays up: restarting it seals the vault.
set -eu
cd "$(dirname "$0")/.."
wait_healthy() {
i=0
while [ "$i" -lt 40 ]; do
ok=1
for svc in "$@"; do
health=$(docker compose --env-file .env ps --format '{{.Service}} {{.Health}}' "$svc" | awk 'NR==1 {print $2}')
if [ "$health" != "healthy" ]; then
ok=0
fi
done
if [ "$ok" -eq 1 ]; then
return 0
fi
i=$((i + 1))
sleep 2
done
echo "timed out waiting for healthy: $*" >&2
return 1
}
docker compose --env-file .env restart -t 8 postgres minio rabbitmq valkey
wait_healthy postgres minio rabbitmq valkey
docker compose --env-file .env restart -t 8 api-1 api-2 worker
wait_healthy api-1 api-2 worker
docker compose --env-file .env restart -t 3 nginx
+23
View File
@@ -0,0 +1,23 @@
#!/bin/sh
# Map the public random alias to the compose service and stop only that replica.
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
found=""
for dir in run/instances/*; do
[ -f "$dir/id" ] || continue
current=$(tr -d '\r\n' < "$dir/id")
if [ "$current" = "$TARGET_INSTANCE_ID" ]; then
found=$(basename "$dir")
break
fi
done
if [ -z "$found" ]; then
echo "unknown instance alias" >&2
exit 1
fi
case "$found" in
api-1|api-2) ;;
*) echo "alias does not belong to an API replica" >&2; exit 1 ;;
esac
docker compose --env-file .env stop -t 2 "$found"
+312
View File
@@ -0,0 +1,312 @@
"""Track 1: least privilege before and after, then revoke, rotate, and keep a queued job.
The worker is stopped only for the durability section and started again before exit.
RabbitMQ identity and the worker S3 secret are restored to a working pair.
"""
import json
import secrets
import subprocess
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
S3_CONFIG = ROOT / "secrets" / "seaweed" / "s3.json"
PASSWORDS = ROOT / "secrets" / "operator" / "passwords.json"
EVIDENCE = ROOT / "evidence"
BUCKET = "messenger-attachments"
def run(command, check=True, input_text=None):
result = subprocess.run(command, cwd=ROOT, input=input_text, capture_output=True, text=True)
if check and result.returncode != 0:
detail = ((result.stderr or result.stdout or "")[:180]).replace("\n", " ")
raise SystemExit("command failed: " + detail)
return result
def psql(sql):
return run(
["docker", "exec", "-i", "mtusi-lab7-postgres-1", "psql", "-U", "messenger", "-d", "messenger", "-t", "-A"],
input_text=sql,
)
def bao_put_worker_s3(access_key, secret_key):
token = json.loads((ROOT / "secrets" / "operator" / "init.json").read_text(encoding="utf-8"))["root_token"]
command = [
"docker",
"compose",
"--env-file",
".env",
"exec",
"-T",
"-e",
"BAO_ADDR=https://127.0.0.1:8200",
"-e",
"BAO_CACERT=/openbao/tls/ca.crt",
"-e",
"BAO_CLIENT_CERT=/openbao/tls/operator.crt",
"-e",
"BAO_CLIENT_KEY=/openbao/tls/operator.key",
"-e",
"BAO_TOKEN=" + token,
"openbao",
"bao",
"kv",
"put",
"secret/worker/s3",
"access_key=" + access_key,
"secret_key=" + secret_key,
]
run(command)
def restart_minio():
run(["docker", "compose", "--env-file", ".env", "restart", "-t", "5", "minio"])
for _ in range(30):
result = run(["docker", "inspect", "--format", "{{.State.Health.Status}}", "mtusi-lab7-minio-1"], check=False)
if (result.stdout or "").strip() == "healthy":
return
time.sleep(2)
raise SystemExit("minio did not become healthy")
def track1():
result = run(
["docker", "compose", "--env-file", ".env", "--profile", "tools", "run", "--rm", "--entrypoint", "python", "probe", "-m", "app.probes.track1"],
check=False,
)
line = ""
for item in (result.stdout or "").splitlines():
if item.startswith("{"):
line = item
if not line:
raise SystemExit("track1 probe produced no report")
return json.loads(line), result.returncode
def widen_s3(config):
for identity in config["identities"]:
if identity["name"] == "worker":
identity["actions"] = [
"Admin",
"Read",
"Write",
"List",
"Read:%s/canary/*" % BUCKET,
"Write:%s/canary/*" % BUCKET,
"List:%s/canary/*" % BUCKET,
]
return config
def worker_secret(config):
for identity in config["identities"]:
if identity["name"] == "worker":
return identity["credentials"][0]["accessKey"], identity["credentials"][0]["secretKey"]
raise SystemExit("worker s3 identity is missing")
def set_worker_secret(config, secret_key):
for identity in config["identities"]:
if identity["name"] == "worker":
identity["credentials"][0]["secretKey"] = secret_key
return config
def old_secret_denied(access_key, secret_key):
admin = ROOT / "secrets" / "seaweed" / "old-secret.json"
admin.write_text(json.dumps({"access_key": access_key, "secret_key": secret_key}), encoding="utf-8")
code = (
"import json,boto3\n"
"from botocore.config import Config\n"
"from botocore.exceptions import ClientError\n"
"cred=json.load(open('/c/cred.json'))\n"
"client=boto3.client('s3',endpoint_url='https://minio:8333',aws_access_key_id=cred['access_key'],"
"aws_secret_access_key=cred['secret_key'],region_name='us-east-1',verify='/c/ca.crt',"
"config=Config(signature_version='s3v4',s3={'addressing_style':'path'}))\n"
"try:\n"
" client.put_object(Bucket='messenger-attachments',Key='staging/old-secret',Body=b'x')\n"
" print('allow')\n"
"except ClientError as exc:\n"
" print('deny:'+str(exc.response.get('Error',{}).get('Code','error')))\n"
"except Exception as exc:\n"
" print('deny:'+type(exc).__name__)\n"
)
try:
result = run(
[
"docker",
"run",
"--rm",
"--network",
"mtusi-lab7_app",
"-v",
str(ROOT / "secrets" / "pki" / "ca.crt") + ":/c/ca.crt:ro",
"-v",
str(admin) + ":/c/cred.json:ro",
"--entrypoint",
"python",
"mtusi-lab7-messenger:1.7.0",
"-c",
code,
]
)
return (result.stdout or "").strip().splitlines()[-1]
finally:
admin.unlink(missing_ok=True)
def restore_rabbit():
run(
["docker", "exec", "mtusi-lab7-rabbitmq-1", "rabbitmqctl", "add_user", "worker.messenger.internal", "unused-password"],
check=False,
)
run(
[
"docker",
"exec",
"mtusi-lab7-rabbitmq-1",
"rabbitmqctl",
"set_permissions",
"-p",
"/",
"worker.messenger.internal",
"^attachment\\.process\\.v1$",
"^$",
"^attachment\\.process\\.v1$",
]
)
def queue_depth():
result = run(["docker", "exec", "mtusi-lab7-rabbitmq-1", "rabbitmqctl", "list_queues", "name", "messages"])
for line in (result.stdout or "").splitlines():
if line.startswith("attachment.process.v1"):
parts = line.split()
return int(parts[-1])
return 0
def wait_queue(minimum, timeout=25):
last = 0
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
last = queue_depth()
if last >= minimum:
return last
time.sleep(1)
return last
def upload_while_worker_stopped():
import sys
sys.path.insert(0, str(ROOT))
from tests.client import Client
client = Client("https://127.0.0.1:8443", str(ROOT / "secrets" / "pki" / "ca.crt"), timeout=15)
alice, _bob, _eve, chat = client.bootstrap()
attachment, _content = client.attachment(alice, chat)
return alice, attachment["id"]
def wait_status(actor, attachment_id):
import sys
sys.path.insert(0, str(ROOT))
from tests.client import Client
client = Client("https://127.0.0.1:8443", str(ROOT / "secrets" / "pki" / "ca.crt"), timeout=15)
deadline = time.monotonic() + 70
last = ""
while time.monotonic() < deadline:
current = client.call("GET", "/api/v1/attachments/" + attachment_id, actor=actor, schema="Attachment")["json"]
last = current["status"]
if last in {"ready", "failed"}:
return last
time.sleep(1)
return last or "timeout"
def main():
EVIDENCE.mkdir(exist_ok=True)
original = S3_CONFIG.read_text(encoding="utf-8")
original_config = json.loads(original)
access_key, old_secret = worker_secret(original_config)
report = {}
committed = False
try:
least, code = track1()
report["least_privilege_exit"] = code
report["least_privilege"] = least
psql("GRANT SELECT ON users TO worker_login;\n")
broad = widen_s3(json.loads(original))
S3_CONFIG.write_text(json.dumps(broad), encoding="utf-8")
restart_minio()
widened, widened_code = track1()
report["broad_exit"] = widened_code
report["broad"] = widened
S3_CONFIG.write_text(original, encoding="utf-8")
psql("REVOKE SELECT ON users FROM worker_login;\n")
restart_minio()
restored, restored_code = track1()
report["restored_exit"] = restored_code
report["restored"] = restored
run(["docker", "compose", "--env-file", ".env", "stop", "-t", "5", "worker"])
actor, attachment_id = upload_while_worker_stopped()
uuid.UUID(attachment_id)
report["queued_before_revoke"] = psql(
"SELECT status FROM attachments WHERE id = '%s';\n" % attachment_id
).stdout.strip()
report["queue_before_revoke"] = wait_queue(1)
run(["docker", "exec", "mtusi-lab7-rabbitmq-1", "rabbitmqctl", "delete_user", "worker.messenger.internal"])
report["queue_after_revoke"] = queue_depth()
new_secret = secrets.token_hex(24)
rotated = set_worker_secret(json.loads(original), new_secret)
S3_CONFIG.write_text(json.dumps(rotated), encoding="utf-8")
passwords = json.loads(PASSWORDS.read_text(encoding="utf-8"))
passwords["worker_s3_secret"] = new_secret
PASSWORDS.write_text(json.dumps(passwords), encoding="utf-8")
bao_put_worker_s3(access_key, new_secret)
committed = True
restart_minio()
report["old_secret"] = old_secret_denied(access_key, old_secret)
restore_rabbit()
run(["docker", "compose", "--env-file", ".env", "start", "worker"])
for _ in range(30):
health = run(["docker", "inspect", "--format", "{{.State.Health.Status}}", "mtusi-lab7-worker-1"], check=False)
if (health.stdout or "").strip() == "healthy":
break
time.sleep(2)
report["after_restore"] = wait_status(actor, attachment_id)
report["attachment_id_prefix"] = attachment_id[:8]
finally:
if not committed:
S3_CONFIG.write_text(original, encoding="utf-8")
psql("REVOKE SELECT ON users FROM worker_login;\n")
restore_rabbit()
run(["docker", "compose", "--env-file", ".env", "start", "worker"], check=False)
(EVIDENCE / "track1-run.json").write_text(json.dumps(report, indent=2), encoding="utf-8")
print(json.dumps({key: report[key] for key in report if key != "least_privilege" and key != "broad" and key != "restored"}))
print("least", report.get("least_privilege"))
print("broad", report.get("broad"))
print("restored", report.get("restored"))
required = (
report.get("least_privilege_exit") == 0
and report.get("broad", {}).get("db_select_password_hash") == "allow"
and str(report.get("broad", {}).get("s3_get_canary", "")).startswith("allow")
and report.get("restored_exit") == 0
and report.get("queued_before_revoke") == "queued"
and report.get("queue_after_revoke", 0) >= 1
and str(report.get("old_secret", "")).startswith("deny")
and report.get("after_restore") == "ready"
)
if not required:
raise SystemExit(1)
if __name__ == "__main__":
main()
+62
View File
@@ -0,0 +1,62 @@
"""Unseal a restarted OpenBao from the operator's local init file. Prints only seal state."""
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
INIT = ROOT / "secrets" / "operator" / "init.json"
def bao(*args):
command = [
"docker",
"compose",
"--env-file",
".env",
"exec",
"-T",
"-e",
"BAO_ADDR=https://127.0.0.1:8200",
"-e",
"BAO_CACERT=/openbao/tls/ca.crt",
"-e",
"BAO_CLIENT_CERT=/openbao/tls/operator.crt",
"-e",
"BAO_CLIENT_KEY=/openbao/tls/operator.key",
"openbao",
"bao",
*args,
]
return subprocess.run(command, cwd=ROOT, capture_output=True, text=True)
def status():
result = bao("status", "-format=json")
text = result.stdout or ""
start = text.find("{")
end = text.rfind("}")
if start < 0 or end < start:
raise SystemExit("OpenBao status is not available")
return json.loads(text[start : end + 1])
def main():
current = status()
if not current.get("initialized"):
raise SystemExit("OpenBao is not initialized")
if not current.get("sealed"):
print("sealed=false")
return
if not INIT.is_file():
raise SystemExit("operator init file is missing")
keys = json.loads(INIT.read_text(encoding="utf-8")).get("unseal_keys_b64") or []
for key in keys[:2]:
result = bao("operator", "unseal", key)
if result.returncode != 0:
raise SystemExit("OpenBao unseal failed")
print("sealed=" + str(bool(status().get("sealed"))).lower())
if __name__ == "__main__":
sys.exit(main())
+325
View File
@@ -0,0 +1,325 @@
"""Local grade-4 proofs: rehash, session expiry, LB leaf rotation, sealed OpenBao, telemetry.
Prints statuses only. Restores certificates, the allowlist and the vault before exiting.
"""
import json
import ssl
import subprocess
import time
import urllib.error
import urllib.request
import uuid
from pathlib import Path
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID
ROOT = Path(__file__).resolve().parents[1]
PKI = ROOT / "secrets" / "pki"
OPERATOR = ROOT / "secrets" / "operator"
EVIDENCE = ROOT / "evidence"
BASE = "https://127.0.0.1:8443"
CA = PKI / "ca.crt"
def run(command, check=True, input_text=None):
result = subprocess.run(
command,
cwd=ROOT,
input=input_text,
capture_output=True,
text=True,
)
if check and result.returncode != 0:
detail = ((result.stderr or result.stdout or "")[:240]).replace("\n", " ")
raise SystemExit("command failed: " + detail)
return result
def bao(*args, token=None):
command = [
"docker",
"compose",
"--env-file",
".env",
"exec",
"-T",
"-e",
"BAO_ADDR=https://127.0.0.1:8200",
"-e",
"BAO_CACERT=/openbao/tls/ca.crt",
"-e",
"BAO_CLIENT_CERT=/openbao/tls/operator.crt",
"-e",
"BAO_CLIENT_KEY=/openbao/tls/operator.key",
]
if token:
command += ["-e", "BAO_TOKEN=" + token]
command += ["openbao", "bao", *args]
return run(command, check=False)
def bao_status():
result = bao("status", "-format=json")
text = result.stdout or ""
start = text.find("{")
end = text.rfind("}")
if start < 0 or end < start:
return None
return json.loads(text[start : end + 1])
def root_token():
return json.loads((OPERATOR / "init.json").read_text(encoding="utf-8"))["root_token"]
def https(path, cert=None, key=None, data=None, method=None, headers=None):
ctx = ssl.create_default_context(cafile=str(CA))
if cert and key:
ctx.load_cert_chain(str(cert), str(key))
request = urllib.request.Request(BASE + path, data=data, headers=headers or {}, method=method)
try:
with urllib.request.urlopen(request, context=ctx, timeout=8) as response:
return response.status, response.read()
except urllib.error.HTTPError as exc:
return exc.code, exc.read()
def psql(sql):
result = run(
["docker", "exec", "-i", "mtusi-lab7-postgres-1", "psql", "-U", "messenger", "-d", "messenger", "-t", "-A"],
input_text=sql,
)
return (result.stdout or "").strip()
def mtls_ready(cert, key):
code = (
"import ssl,urllib.request,urllib.error\n"
"ctx=ssl.create_default_context(cafile='/c/ca.crt')\n"
"ctx.load_cert_chain('/c/client.crt','/c/client.key')\n"
"try:\n"
" r=urllib.request.urlopen('https://api-1:8443/health/ready', context=ctx, timeout=8)\n"
" print(r.status)\n"
"except urllib.error.HTTPError as exc:\n"
" print('http', exc.code)\n"
"except Exception as exc:\n"
" print(type(exc).__name__)\n"
)
result = run(
[
"docker",
"run",
"--rm",
"--network",
"mtusi-lab7_app",
"-v",
str(CA) + ":/c/ca.crt:ro",
"-v",
str(cert) + ":/c/client.crt:ro",
"-v",
str(key) + ":/c/client.key:ro",
"--entrypoint",
"python",
"mtusi-lab7-messenger:1.7.0",
"-c",
code,
]
)
return (result.stdout or "").strip()
def issue_lb_leaf():
ca_key = serialization.load_pem_private_key((PKI / "ca.key").read_bytes(), password=None)
ca_cert = x509.load_pem_x509_certificate((PKI / "ca.crt").read_bytes())
key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
now = __import__("datetime").datetime.now(__import__("datetime").timezone.utc)
cert = (
x509.CertificateBuilder()
.subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "lb.messenger.internal")]))
.issuer_name(ca_cert.subject)
.public_key(key.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(now - __import__("datetime").timedelta(minutes=5))
.not_valid_after(now + __import__("datetime").timedelta(days=7))
.add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True)
.add_extension(x509.SubjectAlternativeName([x509.DNSName("lb.messenger.internal")]), critical=False)
.add_extension(x509.ExtendedKeyUsage([ExtendedKeyUsageOID.CLIENT_AUTH]), critical=False)
.sign(ca_key, hashes.SHA256())
)
leaf_key = PKI / "lb-rotated.key"
leaf_crt = PKI / "lb-rotated.crt"
leaf_key.write_bytes(
key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption())
)
leaf_crt.write_bytes(cert.public_bytes(serialization.Encoding.PEM))
return leaf_crt, leaf_key, format(cert.serial_number, "X")
def reload_nginx():
run(["docker", "exec", "mtusi-lab7-nginx-1", "nginx", "-s", "reload"])
def count_leaks():
patterns = ("Bearer ", "password=", "postgresql://", "aws_secret_access_key", "BEGIN PRIVATE")
counts = {pattern: 0 for pattern in patterns}
for name in ("mtusi-lab7-api-1-1", "mtusi-lab7-api-2-1", "mtusi-lab7-worker-1", "mtusi-lab7-loki-1"):
result = run(["docker", "logs", name, "--since", "20m"], check=False)
text = (result.stdout or "") + (result.stderr or "")
for pattern in patterns:
counts[pattern] += text.count(pattern)
return counts
def rehash_and_expiry():
from argon2 import PasswordHasher
username = "rehash" + uuid.uuid4().hex[:8]
password = "rotate-horse-2"
body = json.dumps({"username": username, "display_name": "Rehash", "password": password}).encode("utf-8")
status, _ = https("/api/v1/users", data=body, method="POST", headers={"Content-Type": "application/json"})
if status != 201:
raise SystemExit("rehash register failed")
old = PasswordHasher(time_cost=1, memory_cost=8192, parallelism=1).hash(password)
psql("UPDATE users SET password_hash = %s WHERE username = %s;\n" % (_quote(old), _quote(username)))
before = psql("SELECT substring(password_hash from 'm=[0-9]+,t=[0-9]+,p=[0-9]+') FROM users WHERE username = %s;\n" % _quote(username))
import base64
basic = base64.b64encode((username + ":" + password).encode("utf-8")).decode("ascii")
login_status, raw = https("/api/v1/auth/sessions", method="POST", headers={"Authorization": "Basic " + basic})
after = psql("SELECT substring(password_hash from 'm=[0-9]+,t=[0-9]+,p=[0-9]+') FROM users WHERE username = %s;\n" % _quote(username))
token = json.loads(raw.decode("utf-8")).get("token") if login_status == 201 else ""
expired = None
if token:
psql(
"UPDATE sessions SET expires_at = NOW() - INTERVAL '1 minute' "
"WHERE user_id = (SELECT id FROM users WHERE username = %s);\n" % _quote(username)
)
expired, _ = https("/api/v1/users/me", headers={"Authorization": "Bearer " + token})
psql("DELETE FROM sessions WHERE user_id IN (SELECT id FROM users WHERE username = %s);\n" % _quote(username))
psql("DELETE FROM users WHERE username = %s;\n" % _quote(username))
return {
"login": login_status,
"params_before": before,
"params_after": after,
"expired_session": expired,
}
def _quote(value):
return "'" + value.replace("'", "''") + "'"
def _until(action, expected, attempts=8):
last = ""
for _ in range(attempts):
last = action()
if last == expected or last.startswith(expected):
return last
time.sleep(1)
return last
def rotate_leaf():
allow = PKI / "lb-allowed-serials.txt"
backup_allow = allow.read_bytes()
backup_crt = (PKI / "lb.crt").read_bytes()
backup_key = (PKI / "lb.key").read_bytes()
leaf_crt, leaf_key, serial = issue_lb_leaf()
try:
allow.write_text(serial + "\n", encoding="ascii")
rotated = _until(lambda: mtls_ready(leaf_crt, leaf_key), "200")
retired = _until(lambda: mtls_ready(PKI / "lb.crt", PKI / "lb.key"), "http 403")
(PKI / "lb.crt").write_bytes(leaf_crt.read_bytes())
(PKI / "lb.key").write_bytes(leaf_key.read_bytes())
reload_nginx()
time.sleep(1)
public, _ = https("/health/ready")
return {"new_serial_ready": rotated, "old_serial": retired, "public_after_reload": public}
finally:
allow.write_bytes(backup_allow)
(PKI / "lb.crt").write_bytes(backup_crt)
(PKI / "lb.key").write_bytes(backup_key)
leaf_crt.unlink(missing_ok=True)
leaf_key.unlink(missing_ok=True)
reload_nginx()
def seal_cycle():
before, _ = https("/health/ready")
open_login, _ = https(
"/api/v1/auth/sessions",
method="POST",
headers={"Authorization": "Basic YmFkOmJhZA=="},
)
sealed = bao("operator", "seal", token=root_token())
if sealed.returncode != 0:
raise SystemExit("OpenBao seal failed")
report = {"ready_before": before, "login_before_seal": open_login, "login_while_sealed": None, "secret_env": []}
try:
time.sleep(20)
deadline = time.monotonic() + 30
denied = None
while time.monotonic() < deadline:
denied, _ = https(
"/api/v1/auth/sessions",
method="POST",
headers={"Authorization": "Basic YmFkOmJhZA=="},
)
if denied == 503:
break
time.sleep(2)
env = run(["docker", "exec", "mtusi-lab7-api-1-1", "printenv"], check=False)
leaked = [
line.split("=", 1)[0]
for line in (env.stdout or "").splitlines()
if line.startswith(("DATABASE_URL=", "S3_SECRET_ACCESS_KEY=", "VALKEY_PASSWORD=", "BAO_TOKEN="))
]
report["login_while_sealed"] = denied
report["secret_env"] = leaked
finally:
run(["python", "scripts/unseal.py"])
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
status, _ = https("/health/ready")
if status == 200:
break
time.sleep(2)
return report
def main():
EVIDENCE.mkdir(exist_ok=True)
report = {
"rehash": rehash_and_expiry(),
"leaf": rotate_leaf(),
"telemetry_leak_counts": count_leaks(),
}
sealed = seal_cycle()
report["sealed"] = sealed if isinstance(sealed, dict) else {"restored": True}
ready, _ = https("/health/ready")
report["ready_after_unseal"] = ready
(EVIDENCE / "verify-local.json").write_text(json.dumps(report, indent=2), encoding="utf-8")
print(json.dumps(report))
ok = (
report["rehash"]["login"] == 201
and report["rehash"]["params_before"].startswith("m=8192")
and report["rehash"]["params_after"] == "m=19456,t=2,p=1"
and report["rehash"]["expired_session"] == 401
and report["leaf"]["new_serial_ready"] == "200"
and report["leaf"]["old_serial"].startswith("http 403")
and report["leaf"]["public_after_reload"] == 200
and report["sealed"]["login_before_seal"] == 401
and report["sealed"]["login_while_sealed"] == 503
and report["sealed"]["secret_env"] == []
and ready == 200
and all(value == 0 for value in report["telemetry_leak_counts"].values())
)
if not ok:
raise SystemExit(1)
if __name__ == "__main__":
main()