Files
mtusi2026lab7/scripts/bootstrap.py
T
vika baafe2a3ae Добавить стенд лабораторной 7 с mTLS, OpenBao и least privilege worker.
Закрывает обязательный минимум и трек 1: ротация leaf и секрета, sealed OpenBao и сохранность queued-задачи.
2026-11-07 20:46:02 +03:00

545 lines
22 KiB
Python

"""Create local CA, secrets and, with --up, initialize the stand. Never prints secret values."""
import datetime
import ipaddress
import json
import os
import secrets
import shutil
import subprocess
import sys
import time
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
PKI = ROOT / "secrets" / "pki"
OPERATOR = ROOT / "secrets" / "operator"
def ensure_cryptography():
try:
import cryptography # noqa: F401
return
except ImportError:
pass
venv = ROOT / ".venv-bootstrap"
py = venv / "Scripts" / "python.exe"
if not py.exists():
subprocess.check_call([sys.executable, "-m", "venv", str(venv)])
subprocess.check_call([str(py), "-m", "pip", "install", "cryptography==44.0.2"])
os.execv(str(py), [str(py), str(Path(__file__).resolve()), *sys.argv[1:]])
def write_pem(path, data):
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
def build_material():
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID
def private():
return rsa.generate_private_key(public_exponent=65537, key_size=2048)
def dump_key(key):
return key.private_bytes(
serialization.Encoding.PEM,
serialization.PrivateFormat.PKCS8,
serialization.NoEncryption(),
)
def dump_cert(cert):
return cert.public_bytes(serialization.Encoding.PEM)
def name(cn):
return x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, cn)])
now = datetime.datetime.now(datetime.timezone.utc)
def certificate(cn, sans, ips, eku, days, issuer_key, issuer_cert, expired=False):
key = private()
if expired:
start = now - datetime.timedelta(days=3)
end = now - datetime.timedelta(days=1)
else:
start = now - datetime.timedelta(minutes=10)
end = now + datetime.timedelta(days=days)
alt = [x509.DNSName(item) for item in sans]
alt += [x509.IPAddress(ipaddress.ip_address(item)) for item in ips]
uses = []
if "client" in eku:
uses.append(ExtendedKeyUsageOID.CLIENT_AUTH)
if "server" in eku:
uses.append(ExtendedKeyUsageOID.SERVER_AUTH)
builder = (
x509.CertificateBuilder()
.subject_name(name(cn))
.issuer_name(issuer_cert.subject if issuer_cert is not None else name(cn))
.public_key(key.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(start)
.not_valid_after(end)
.add_extension(x509.BasicConstraints(ca=issuer_cert is None, path_length=0 if issuer_cert is None else None), critical=True)
)
if issuer_cert is None:
builder = builder.add_extension(
x509.KeyUsage(
digital_signature=True,
key_cert_sign=True,
crl_sign=True,
content_commitment=False,
key_encipherment=False,
data_encipherment=False,
key_agreement=False,
encipher_only=False,
decipher_only=False,
),
critical=True,
)
else:
builder = builder.add_extension(x509.SubjectAlternativeName(alt), critical=False)
builder = builder.add_extension(x509.ExtendedKeyUsage(uses), critical=False)
signed = builder.sign(issuer_key if issuer_cert is not None else key, hashes.SHA256())
return key, signed
PKI.mkdir(parents=True, exist_ok=True)
if not (PKI / "ca.crt").exists():
ca_key, ca_cert = certificate("messenger-lab-ca", [], [], [], 30, None, None)
other_key, other_cert = certificate("foreign-lab-ca", [], [], [], 30, None, None)
write_pem(PKI / "ca.key", dump_key(ca_key))
write_pem(PKI / "ca.crt", dump_cert(ca_cert))
write_pem(PKI / "other-ca.key", dump_key(other_key))
write_pem(PKI / "other-ca.crt", dump_cert(other_cert))
leaves = {
"ingress": ("localhost", ["localhost", "nginx"], ["127.0.0.1"], ["server"], 14, False, ca_key, ca_cert),
"lb": ("lb.messenger.internal", ["lb.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert),
"api-server": ("api.messenger.internal", ["messenger_api", "api-1", "api-2", "api.messenger.internal"], [], ["server"], 14, False, ca_key, ca_cert),
"api-client": ("api.messenger.internal", ["api.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert),
"worker-client": ("worker.messenger.internal", ["worker.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert),
"stranger": ("stranger.messenger.internal", ["stranger.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert),
"expired": ("lb.messenger.internal", ["lb.messenger.internal"], [], ["client"], 1, True, ca_key, ca_cert),
"other-client": ("lb.messenger.internal", ["lb.messenger.internal"], [], ["client"], 14, False, other_key, other_cert),
"postgres": ("postgres", ["postgres"], [], ["server"], 14, False, ca_key, ca_cert),
"minio": ("minio", ["minio"], [], ["server"], 14, False, ca_key, ca_cert),
"rabbitmq": ("rabbitmq", ["rabbitmq"], [], ["server"], 14, False, ca_key, ca_cert),
"valkey": ("valkey", ["valkey"], [], ["server", "client"], 14, False, ca_key, ca_cert),
"openbao": ("openbao", ["openbao", "localhost"], ["127.0.0.1"], ["server"], 14, False, ca_key, ca_cert),
"health": ("health.messenger.internal", ["health.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert),
"operator": ("operator.messenger.internal", ["operator.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert),
"otel-edge": ("otel-edge", ["otel-edge"], [], ["server"], 14, False, ca_key, ca_cert),
"collector": ("otel-collector", ["otel-collector"], [], ["server", "client"], 14, False, ca_key, ca_cert),
"prometheus": ("prometheus", ["prometheus"], [], ["server"], 14, False, ca_key, ca_cert),
"loki": ("loki", ["loki"], [], ["server"], 14, False, ca_key, ca_cert),
"tempo": ("tempo", ["tempo"], [], ["server"], 14, False, ca_key, ca_cert),
"grafana": ("grafana", ["grafana", "localhost"], ["127.0.0.1"], ["server"], 14, False, ca_key, ca_cert),
}
serials = {}
for stem, spec in leaves.items():
cn, sans, ips, eku, days, expired, issuer_key, issuer_cert = spec
key, cert = certificate(cn, sans, ips, eku, days, issuer_key, issuer_cert, expired)
write_pem(PKI / (stem + ".key"), dump_key(key))
write_pem(PKI / (stem + ".crt"), dump_cert(cert))
serials[stem] = cert.serial_number
minio_dir = PKI / "minio"
minio_dir.mkdir(exist_ok=True)
shutil.copyfile(PKI / "minio.crt", minio_dir / "public.crt")
shutil.copyfile(PKI / "minio.key", minio_dir / "private.key")
(PKI / "serials.json").write_text(json.dumps({key: format(value, "X") for key, value in serials.items()}), encoding="utf-8")
return json.loads((PKI / "serials.json").read_text(encoding="utf-8"))
def password():
return secrets.token_hex(24)
def material_passwords():
OPERATOR.mkdir(parents=True, exist_ok=True)
path = OPERATOR / "passwords.json"
current = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
defaults = {
"postgres_password": password(),
"api_db_password": password(),
"worker_db_password": password(),
"minio_root_user": "lab-root",
"minio_root_password": password(),
"api_s3_key": "api-" + secrets.token_hex(6),
"api_s3_secret": password(),
"worker_s3_key": "worker-" + secrets.token_hex(6),
"worker_s3_secret": password(),
"valkey_password": password(),
"grafana_password": password(),
"s3_admin_key": "admin-" + secrets.token_hex(6),
"s3_admin_secret": password(),
}
changed = False
for key, value in defaults.items():
if key not in current:
current[key] = value
changed = True
if changed:
path.write_text(json.dumps(current), encoding="utf-8")
return current
def colon(serial):
hexed = serial if len(serial) % 2 == 0 else "0" + serial
return ":".join(hexed[i : i + 2] for i in range(0, len(hexed), 2))
def write_configs(pw, serials):
(ROOT / "secrets" / "db").mkdir(parents=True, exist_ok=True)
(ROOT / "secrets" / "db" / "01-roles.sql").write_text(
"CREATE ROLE api_login LOGIN PASSWORD '{api}';\nCREATE ROLE worker_login LOGIN PASSWORD '{worker}';\n".format(
api=pw["api_db_password"], worker=pw["worker_db_password"]
),
encoding="utf-8",
)
(ROOT / "secrets" / "postgres.env").write_text(
"POSTGRES_USER=messenger\nPOSTGRES_PASSWORD={password}\nPOSTGRES_DB=messenger\n".format(password=pw["postgres_password"]),
encoding="utf-8",
)
(ROOT / "secrets" / "minio.env").write_text(
"MINIO_ROOT_USER={user}\nMINIO_ROOT_PASSWORD={password}\n".format(
user=pw["minio_root_user"], password=pw["minio_root_password"]
),
encoding="utf-8",
)
(ROOT / "secrets" / "grafana.env").write_text(
"GF_SECURITY_ADMIN_PASSWORD={password}\n".format(password=pw["grafana_password"]),
encoding="utf-8",
)
(ROOT / "secrets" / "valkey").mkdir(parents=True, exist_ok=True)
(ROOT / "secrets" / "valkey" / "users.acl").write_text(
"\n".join(
[
"user default off",
"user health on nopass +ping",
"user api on >{password} ~login:* +incr +incrby +expire +ttl +ping +get".format(password=pw["valkey_password"]),
"",
]
),
encoding="utf-8",
)
(PKI / "lb-allowed-serials.txt").write_text(serials["lb"] + "\n" + colon(serials["lb"]) + "\n", encoding="utf-8")
seen = {}
for stem in ("api-client", "worker-client"):
raw = serials[stem].upper()
for item in (raw, colon(raw)):
seen[item.lower()] = item
lines = ["map $ssl_client_s_dn $otel_cn_ok {", " default 0;", ' "~*CN=api\\.messenger\\.internal" 1;', ' "~*CN=worker\\.messenger\\.internal" 1;', "}"]
lines.append("map $ssl_client_serial $otel_serial_ok {")
lines.append(" default 0;")
for item in seen.values():
lines.append(' "%s" 1;' % item)
lines.append("}")
nginx = ROOT / "secrets" / "nginx"
nginx.mkdir(parents=True, exist_ok=True)
(nginx / "serials.conf").write_text("\n".join(lines) + "\n", encoding="utf-8")
(ROOT / ".env").write_text(
"\n".join(
[
"SESSION_TTL_SECONDS=3600",
"IDEMPOTENCY_TTL_SECONDS=86400",
"WORKER_CONCURRENCY=2",
"JOB_MAX_ATTEMPTS=5",
"JOB_TIMEOUT_SECONDS=60",
"STAGING_RETENTION_SECONDS=86400",
"",
]
),
encoding="utf-8",
)
for name in ("api-1", "api-2", "worker"):
(ROOT / "run" / "instances" / name).mkdir(parents=True, exist_ok=True)
def compose(*args, check=True, env=None):
command = ["docker", "compose", "--env-file", ".env", *args]
return subprocess.run(command, cwd=ROOT, check=check, env=env)
def bao(*args, token=None):
command = [
"docker",
"compose",
"--env-file",
".env",
"exec",
"-T",
"-e",
"BAO_ADDR=https://127.0.0.1:8200",
"-e",
"BAO_CACERT=/openbao/tls/ca.crt",
"-e",
"BAO_CLIENT_CERT=/openbao/tls/operator.crt",
"-e",
"BAO_CLIENT_KEY=/openbao/tls/operator.key",
]
if token:
command += ["-e", "BAO_TOKEN=" + token]
command += ["openbao", "bao", *args]
return subprocess.run(command, cwd=ROOT, capture_output=True, text=True)
def parse_json(text):
start = text.find("{")
end = text.rfind("}")
if start < 0 or end < start:
return None
return json.loads(text[start : end + 1])
def wait_status():
for _ in range(40):
result = bao("status", "-format=json")
payload = parse_json(result.stdout or "")
if payload is not None and "initialized" in payload:
return payload
time.sleep(2)
raise SystemExit("OpenBao did not start")
def init_openbao(pw):
status = wait_status()
init_path = OPERATOR / "init.json"
if not status.get("initialized"):
result = bao("operator", "init", "-key-shares=3", "-key-threshold=2", "-format=json")
payload = parse_json(result.stdout or "")
if result.returncode != 0 or payload is None or "root_token" not in payload:
raise SystemExit("OpenBao init failed")
init_path.write_text(json.dumps(payload), encoding="utf-8")
status = wait_status()
if status.get("sealed", True):
keys = json.loads(init_path.read_text(encoding="utf-8"))["unseal_keys_b64"]
for key in keys[:2]:
result = bao("operator", "unseal", key)
if result.returncode != 0:
raise SystemExit("OpenBao unseal failed")
root = json.loads(init_path.read_text(encoding="utf-8"))["root_token"]
bao("secrets", "enable", "-path=secret", "kv-v2", token=root)
bao("policy", "write", "api", "/openbao/policies/api.hcl", token=root)
bao("policy", "write", "worker", "/openbao/policies/worker.hcl", token=root)
bao("auth", "enable", "cert", token=root)
for role, cn in (("api", "api.messenger.internal"), ("worker", "worker.messenger.internal")):
result = bao(
"write",
"auth/cert/certs/" + role,
"display_name=" + role,
"policies=" + role,
"certificate=@/openbao/tls/ca.crt",
"allowed_common_names=" + cn,
"token_ttl=20m",
"token_max_ttl=2h",
token=root,
)
if result.returncode != 0:
raise SystemExit("OpenBao cert role " + role + " failed")
writes = {
"secret/api/db": ["username=api_login", "password=" + pw["api_db_password"], "host=postgres", "dbname=messenger"],
"secret/worker/db": ["username=worker_login", "password=" + pw["worker_db_password"], "host=postgres", "dbname=messenger"],
"secret/api/s3": ["access_key=" + pw["api_s3_key"], "secret_key=" + pw["api_s3_secret"]],
"secret/worker/s3": ["access_key=" + pw["worker_s3_key"], "secret_key=" + pw["worker_s3_secret"]],
"secret/api/valkey": ["username=api", "password=" + pw["valkey_password"], "host=valkey"],
}
for path, fields in writes.items():
result = bao("kv", "put", path, *fields, token=root)
if result.returncode != 0:
raise SystemExit("OpenBao kv put failed for " + path.split("/")[1])
def wait_healthy(*services):
for _ in range(50):
result = subprocess.run(
["docker", "compose", "--env-file", ".env", "ps", "--format", "{{.Service}} {{.Health}}", *services],
cwd=ROOT,
capture_output=True,
text=True,
)
rows = [line.strip() for line in result.stdout.splitlines() if line.strip()]
if len(rows) >= len(services) and all(line.endswith(" healthy") for line in rows):
return
time.sleep(3)
raise SystemExit("services did not become healthy: " + " ".join(services))
def write_s3_config(pw):
bucket = "messenger-attachments"
config = {
"identities": [
{
"name": "admin",
"credentials": [{"accessKey": pw["s3_admin_key"], "secretKey": pw["s3_admin_secret"]}],
"actions": ["Admin", "Read", "Write", "List"],
},
{
"name": "api",
"credentials": [{"accessKey": pw["api_s3_key"], "secretKey": pw["api_s3_secret"]}],
"actions": [
"Read:{0}/staging/*".format(bucket),
"Write:{0}/staging/*".format(bucket),
"List:{0}/staging/*".format(bucket),
"Read:{0}/originals/*".format(bucket),
"List:{0}/originals/*".format(bucket),
"Read:{0}/variants/*".format(bucket),
"List:{0}/variants/*".format(bucket),
],
},
{
"name": "worker",
"credentials": [{"accessKey": pw["worker_s3_key"], "secretKey": pw["worker_s3_secret"]}],
"actions": [
"Read:{0}/staging/*".format(bucket),
"Write:{0}/staging/*".format(bucket),
"List:{0}/staging/*".format(bucket),
"Read:{0}/originals/*".format(bucket),
"Write:{0}/originals/*".format(bucket),
"List:{0}/originals/*".format(bucket),
"Read:{0}/variants/*".format(bucket),
"Write:{0}/variants/*".format(bucket),
"List:{0}/variants/*".format(bucket),
],
},
]
}
target = ROOT / "secrets" / "seaweed"
target.mkdir(parents=True, exist_ok=True)
(target / "s3.json").write_text(json.dumps(config), encoding="utf-8")
def configure_bucket(pw):
admin = ROOT / "secrets" / "seaweed" / "admin.json"
admin.write_text(
json.dumps({"access_key": pw["s3_admin_key"], "secret_key": pw["s3_admin_secret"]}),
encoding="utf-8",
)
result = subprocess.run(
[
"docker",
"run",
"--rm",
"--network",
"mtusi-lab7_app",
"-v",
str(PKI / "ca.crt") + ":/certs/ca.crt:ro",
"-v",
str(admin) + ":/certs/admin.json:ro",
"--entrypoint",
"python",
"mtusi-lab7-messenger:1.7.0",
"-c",
(
"import json,boto3; from botocore.config import Config; "
"admin=json.load(open('/certs/admin.json')); "
"c=boto3.client('s3',endpoint_url='https://minio:8333',aws_access_key_id=admin['access_key'],"
"aws_secret_access_key=admin['secret_key'],region_name='us-east-1',verify='/certs/ca.crt',"
"config=Config(signature_version='s3v4',s3={'addressing_style':'path'})); "
"buckets=[b['Name'] for b in c.list_buckets().get('Buckets',[])]; "
"c.create_bucket(Bucket='messenger-attachments') if 'messenger-attachments' not in buckets else None; "
"c.put_object(Bucket='messenger-attachments',Key='canary/marker',Body=b'canary-not-a-secret'); "
"print('s3-bootstrap-ok')"
),
],
capture_output=True,
text=True,
)
admin.unlink(missing_ok=True)
if result.returncode != 0 or "s3-bootstrap-ok" not in (result.stdout or ""):
raise SystemExit("s3 bootstrap failed")
def configure_rabbit():
users = {
"api.messenger.internal": ("^attachment\\.process(\\.v1)?$", "^attachment\\.process$", "^$"),
"worker.messenger.internal": ("^attachment\\.process\\.v1$", "^$", "^attachment\\.process\\.v1$"),
}
subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "await_online_nodes", "1"],
cwd=ROOT,
capture_output=True,
text=True,
)
for user, perms in users.items():
last = None
for _ in range(10):
subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "add_user", user, password()],
cwd=ROOT,
capture_output=True,
text=True,
)
last = subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "set_permissions", "-p", "/", user, *perms],
cwd=ROOT,
capture_output=True,
text=True,
)
if last.returncode == 0:
break
time.sleep(3)
if last is None or last.returncode != 0:
raise SystemExit("rabbit permissions failed for a service user")
expressions = [
'rabbit_exchange:declare({resource, <<"/">>, exchange, <<"attachment.process">>}, direct, true, false, false, [], <<"bootstrap">>).',
'rabbit_binding:add({binding, {resource, <<"/">>, exchange, <<"attachment.process">>}, <<"attachment.process.v1">>, {resource, <<"/">>, queue, <<"attachment.process.v1">>}, []}, <<"bootstrap">>).',
]
for expression in expressions:
topology = subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "eval", expression],
cwd=ROOT,
capture_output=True,
text=True,
)
if topology.returncode != 0:
raise SystemExit("rabbit topology failed")
subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "delete_user", "guest"],
cwd=ROOT,
capture_output=True,
text=True,
)
def wait_ready():
import ssl
ctx = ssl.create_default_context(cafile=str(PKI / "ca.crt"))
for _ in range(40):
try:
with urllib.request.urlopen("https://127.0.0.1:8443/health/ready", context=ctx, timeout=5) as response:
if response.status == 200:
print("ready https://127.0.0.1:8443/health/ready")
return
except Exception:
time.sleep(3)
raise SystemExit("public HTTPS endpoint did not become ready")
def up(pw):
write_s3_config(pw)
compose("up", "-d", "openbao")
init_openbao(pw)
compose("build", "api-1", "postgres", "rabbitmq")
compose("up", "-d", "--build", "postgres", "minio", "rabbitmq", "valkey")
wait_healthy("postgres", "minio", "rabbitmq", "valkey")
configure_bucket(pw)
configure_rabbit()
compose("up", "-d", "--build", "api-1", "api-2", "worker", "nginx", "otel-edge", "otel-collector", "prometheus", "loki", "tempo", "grafana")
wait_ready()
def main():
ensure_cryptography()
serials = build_material()
pw = material_passwords()
write_configs(pw, serials)
print("local material is in secrets/ and .env; private keys were not printed")
if "--up" in sys.argv:
up(pw)
if __name__ == "__main__":
main()