"""Create local CA, secrets and, with --up, initialize the stand. Never prints secret values.""" import datetime import ipaddress import json import os import secrets import shutil import subprocess import sys import time import urllib.request from pathlib import Path ROOT = Path(__file__).resolve().parents[1] PKI = ROOT / "secrets" / "pki" OPERATOR = ROOT / "secrets" / "operator" def ensure_cryptography(): try: import cryptography # noqa: F401 return except ImportError: pass venv = ROOT / ".venv-bootstrap" py = venv / "Scripts" / "python.exe" if not py.exists(): subprocess.check_call([sys.executable, "-m", "venv", str(venv)]) subprocess.check_call([str(py), "-m", "pip", "install", "cryptography==44.0.2"]) os.execv(str(py), [str(py), str(Path(__file__).resolve()), *sys.argv[1:]]) def write_pem(path, data): path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(data) def build_material(): from cryptography import x509 from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID def private(): return rsa.generate_private_key(public_exponent=65537, key_size=2048) def dump_key(key): return key.private_bytes( serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption(), ) def dump_cert(cert): return cert.public_bytes(serialization.Encoding.PEM) def name(cn): return x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, cn)]) now = datetime.datetime.now(datetime.timezone.utc) def certificate(cn, sans, ips, eku, days, issuer_key, issuer_cert, expired=False): key = private() if expired: start = now - datetime.timedelta(days=3) end = now - datetime.timedelta(days=1) else: start = now - datetime.timedelta(minutes=10) end = now + datetime.timedelta(days=days) alt = [x509.DNSName(item) for item in sans] alt += [x509.IPAddress(ipaddress.ip_address(item)) for item in ips] uses = [] if "client" in eku: uses.append(ExtendedKeyUsageOID.CLIENT_AUTH) if "server" in eku: uses.append(ExtendedKeyUsageOID.SERVER_AUTH) builder = ( x509.CertificateBuilder() .subject_name(name(cn)) .issuer_name(issuer_cert.subject if issuer_cert is not None else name(cn)) .public_key(key.public_key()) .serial_number(x509.random_serial_number()) .not_valid_before(start) .not_valid_after(end) .add_extension(x509.BasicConstraints(ca=issuer_cert is None, path_length=0 if issuer_cert is None else None), critical=True) ) if issuer_cert is None: builder = builder.add_extension( x509.KeyUsage( digital_signature=True, key_cert_sign=True, crl_sign=True, content_commitment=False, key_encipherment=False, data_encipherment=False, key_agreement=False, encipher_only=False, decipher_only=False, ), critical=True, ) else: builder = builder.add_extension(x509.SubjectAlternativeName(alt), critical=False) builder = builder.add_extension(x509.ExtendedKeyUsage(uses), critical=False) signed = builder.sign(issuer_key if issuer_cert is not None else key, hashes.SHA256()) return key, signed PKI.mkdir(parents=True, exist_ok=True) if not (PKI / "ca.crt").exists(): ca_key, ca_cert = certificate("messenger-lab-ca", [], [], [], 30, None, None) other_key, other_cert = certificate("foreign-lab-ca", [], [], [], 30, None, None) write_pem(PKI / "ca.key", dump_key(ca_key)) write_pem(PKI / "ca.crt", dump_cert(ca_cert)) write_pem(PKI / "other-ca.key", dump_key(other_key)) write_pem(PKI / "other-ca.crt", dump_cert(other_cert)) leaves = { "ingress": ("localhost", ["localhost", "nginx"], ["127.0.0.1"], ["server"], 14, False, ca_key, ca_cert), "lb": ("lb.messenger.internal", ["lb.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert), "api-server": ("api.messenger.internal", ["messenger_api", "api-1", "api-2", "api.messenger.internal"], [], ["server"], 14, False, ca_key, ca_cert), "api-client": ("api.messenger.internal", ["api.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert), "worker-client": ("worker.messenger.internal", ["worker.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert), "stranger": ("stranger.messenger.internal", ["stranger.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert), "expired": ("lb.messenger.internal", ["lb.messenger.internal"], [], ["client"], 1, True, ca_key, ca_cert), "other-client": ("lb.messenger.internal", ["lb.messenger.internal"], [], ["client"], 14, False, other_key, other_cert), "postgres": ("postgres", ["postgres"], [], ["server"], 14, False, ca_key, ca_cert), "minio": ("minio", ["minio"], [], ["server"], 14, False, ca_key, ca_cert), "rabbitmq": ("rabbitmq", ["rabbitmq"], [], ["server"], 14, False, ca_key, ca_cert), "valkey": ("valkey", ["valkey"], [], ["server", "client"], 14, False, ca_key, ca_cert), "openbao": ("openbao", ["openbao", "localhost"], ["127.0.0.1"], ["server"], 14, False, ca_key, ca_cert), "health": ("health.messenger.internal", ["health.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert), "operator": ("operator.messenger.internal", ["operator.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert), "otel-edge": ("otel-edge", ["otel-edge"], [], ["server"], 14, False, ca_key, ca_cert), "collector": ("otel-collector", ["otel-collector"], [], ["server", "client"], 14, False, ca_key, ca_cert), "prometheus": ("prometheus", ["prometheus"], [], ["server"], 14, False, ca_key, ca_cert), "loki": ("loki", ["loki"], [], ["server"], 14, False, ca_key, ca_cert), "tempo": ("tempo", ["tempo"], [], ["server"], 14, False, ca_key, ca_cert), "grafana": ("grafana", ["grafana", "localhost"], ["127.0.0.1"], ["server"], 14, False, ca_key, ca_cert), } serials = {} for stem, spec in leaves.items(): cn, sans, ips, eku, days, expired, issuer_key, issuer_cert = spec key, cert = certificate(cn, sans, ips, eku, days, issuer_key, issuer_cert, expired) write_pem(PKI / (stem + ".key"), dump_key(key)) write_pem(PKI / (stem + ".crt"), dump_cert(cert)) serials[stem] = cert.serial_number minio_dir = PKI / "minio" minio_dir.mkdir(exist_ok=True) shutil.copyfile(PKI / "minio.crt", minio_dir / "public.crt") shutil.copyfile(PKI / "minio.key", minio_dir / "private.key") (PKI / "serials.json").write_text(json.dumps({key: format(value, "X") for key, value in serials.items()}), encoding="utf-8") return json.loads((PKI / "serials.json").read_text(encoding="utf-8")) def password(): return secrets.token_hex(24) def material_passwords(): OPERATOR.mkdir(parents=True, exist_ok=True) path = OPERATOR / "passwords.json" current = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {} defaults = { "postgres_password": password(), "api_db_password": password(), "worker_db_password": password(), "minio_root_user": "lab-root", "minio_root_password": password(), "api_s3_key": "api-" + secrets.token_hex(6), "api_s3_secret": password(), "worker_s3_key": "worker-" + secrets.token_hex(6), "worker_s3_secret": password(), "valkey_password": password(), "grafana_password": password(), "s3_admin_key": "admin-" + secrets.token_hex(6), "s3_admin_secret": password(), } changed = False for key, value in defaults.items(): if key not in current: current[key] = value changed = True if changed: path.write_text(json.dumps(current), encoding="utf-8") return current def colon(serial): hexed = serial if len(serial) % 2 == 0 else "0" + serial return ":".join(hexed[i : i + 2] for i in range(0, len(hexed), 2)) def write_configs(pw, serials): (ROOT / "secrets" / "db").mkdir(parents=True, exist_ok=True) (ROOT / "secrets" / "db" / "01-roles.sql").write_text( "CREATE ROLE api_login LOGIN PASSWORD '{api}';\nCREATE ROLE worker_login LOGIN PASSWORD '{worker}';\n".format( api=pw["api_db_password"], worker=pw["worker_db_password"] ), encoding="utf-8", ) (ROOT / "secrets" / "postgres.env").write_text( "POSTGRES_USER=messenger\nPOSTGRES_PASSWORD={password}\nPOSTGRES_DB=messenger\n".format(password=pw["postgres_password"]), encoding="utf-8", ) (ROOT / "secrets" / "minio.env").write_text( "MINIO_ROOT_USER={user}\nMINIO_ROOT_PASSWORD={password}\n".format( user=pw["minio_root_user"], password=pw["minio_root_password"] ), encoding="utf-8", ) (ROOT / "secrets" / "grafana.env").write_text( "GF_SECURITY_ADMIN_PASSWORD={password}\n".format(password=pw["grafana_password"]), encoding="utf-8", ) (ROOT / "secrets" / "valkey").mkdir(parents=True, exist_ok=True) (ROOT / "secrets" / "valkey" / "users.acl").write_text( "\n".join( [ "user default off", "user health on nopass +ping", "user api on >{password} ~login:* +incr +incrby +expire +ttl +ping +get".format(password=pw["valkey_password"]), "", ] ), encoding="utf-8", ) (PKI / "lb-allowed-serials.txt").write_text(serials["lb"] + "\n" + colon(serials["lb"]) + "\n", encoding="utf-8") seen = {} for stem in ("api-client", "worker-client"): raw = serials[stem].upper() for item in (raw, colon(raw)): seen[item.lower()] = item lines = ["map $ssl_client_s_dn $otel_cn_ok {", " default 0;", ' "~*CN=api\\.messenger\\.internal" 1;', ' "~*CN=worker\\.messenger\\.internal" 1;', "}"] lines.append("map $ssl_client_serial $otel_serial_ok {") lines.append(" default 0;") for item in seen.values(): lines.append(' "%s" 1;' % item) lines.append("}") nginx = ROOT / "secrets" / "nginx" nginx.mkdir(parents=True, exist_ok=True) (nginx / "serials.conf").write_text("\n".join(lines) + "\n", encoding="utf-8") (ROOT / ".env").write_text( "\n".join( [ "SESSION_TTL_SECONDS=3600", "IDEMPOTENCY_TTL_SECONDS=86400", "WORKER_CONCURRENCY=2", "JOB_MAX_ATTEMPTS=5", "JOB_TIMEOUT_SECONDS=60", "STAGING_RETENTION_SECONDS=86400", "", ] ), encoding="utf-8", ) for name in ("api-1", "api-2", "worker"): (ROOT / "run" / "instances" / name).mkdir(parents=True, exist_ok=True) def compose(*args, check=True, env=None): command = ["docker", "compose", "--env-file", ".env", *args] return subprocess.run(command, cwd=ROOT, check=check, env=env) def bao(*args, token=None): command = [ "docker", "compose", "--env-file", ".env", "exec", "-T", "-e", "BAO_ADDR=https://127.0.0.1:8200", "-e", "BAO_CACERT=/openbao/tls/ca.crt", "-e", "BAO_CLIENT_CERT=/openbao/tls/operator.crt", "-e", "BAO_CLIENT_KEY=/openbao/tls/operator.key", ] if token: command += ["-e", "BAO_TOKEN=" + token] command += ["openbao", "bao", *args] return subprocess.run(command, cwd=ROOT, capture_output=True, text=True) def parse_json(text): start = text.find("{") end = text.rfind("}") if start < 0 or end < start: return None return json.loads(text[start : end + 1]) def wait_status(): for _ in range(40): result = bao("status", "-format=json") payload = parse_json(result.stdout or "") if payload is not None and "initialized" in payload: return payload time.sleep(2) raise SystemExit("OpenBao did not start") def init_openbao(pw): status = wait_status() init_path = OPERATOR / "init.json" if not status.get("initialized"): result = bao("operator", "init", "-key-shares=3", "-key-threshold=2", "-format=json") payload = parse_json(result.stdout or "") if result.returncode != 0 or payload is None or "root_token" not in payload: raise SystemExit("OpenBao init failed") init_path.write_text(json.dumps(payload), encoding="utf-8") status = wait_status() if status.get("sealed", True): keys = json.loads(init_path.read_text(encoding="utf-8"))["unseal_keys_b64"] for key in keys[:2]: result = bao("operator", "unseal", key) if result.returncode != 0: raise SystemExit("OpenBao unseal failed") root = json.loads(init_path.read_text(encoding="utf-8"))["root_token"] bao("secrets", "enable", "-path=secret", "kv-v2", token=root) bao("policy", "write", "api", "/openbao/policies/api.hcl", token=root) bao("policy", "write", "worker", "/openbao/policies/worker.hcl", token=root) bao("auth", "enable", "cert", token=root) for role, cn in (("api", "api.messenger.internal"), ("worker", "worker.messenger.internal")): result = bao( "write", "auth/cert/certs/" + role, "display_name=" + role, "policies=" + role, "certificate=@/openbao/tls/ca.crt", "allowed_common_names=" + cn, "token_ttl=20m", "token_max_ttl=2h", token=root, ) if result.returncode != 0: raise SystemExit("OpenBao cert role " + role + " failed") writes = { "secret/api/db": ["username=api_login", "password=" + pw["api_db_password"], "host=postgres", "dbname=messenger"], "secret/worker/db": ["username=worker_login", "password=" + pw["worker_db_password"], "host=postgres", "dbname=messenger"], "secret/api/s3": ["access_key=" + pw["api_s3_key"], "secret_key=" + pw["api_s3_secret"]], "secret/worker/s3": ["access_key=" + pw["worker_s3_key"], "secret_key=" + pw["worker_s3_secret"]], "secret/api/valkey": ["username=api", "password=" + pw["valkey_password"], "host=valkey"], } for path, fields in writes.items(): result = bao("kv", "put", path, *fields, token=root) if result.returncode != 0: raise SystemExit("OpenBao kv put failed for " + path.split("/")[1]) def wait_healthy(*services): for _ in range(50): result = subprocess.run( ["docker", "compose", "--env-file", ".env", "ps", "--format", "{{.Service}} {{.Health}}", *services], cwd=ROOT, capture_output=True, text=True, ) rows = [line.strip() for line in result.stdout.splitlines() if line.strip()] if len(rows) >= len(services) and all(line.endswith(" healthy") for line in rows): return time.sleep(3) raise SystemExit("services did not become healthy: " + " ".join(services)) def write_s3_config(pw): bucket = "messenger-attachments" config = { "identities": [ { "name": "admin", "credentials": [{"accessKey": pw["s3_admin_key"], "secretKey": pw["s3_admin_secret"]}], "actions": ["Admin", "Read", "Write", "List"], }, { "name": "api", "credentials": [{"accessKey": pw["api_s3_key"], "secretKey": pw["api_s3_secret"]}], "actions": [ "Read:{0}/staging/*".format(bucket), "Write:{0}/staging/*".format(bucket), "List:{0}/staging/*".format(bucket), "Read:{0}/originals/*".format(bucket), "List:{0}/originals/*".format(bucket), "Read:{0}/variants/*".format(bucket), "List:{0}/variants/*".format(bucket), ], }, { "name": "worker", "credentials": [{"accessKey": pw["worker_s3_key"], "secretKey": pw["worker_s3_secret"]}], "actions": [ "Read:{0}/staging/*".format(bucket), "Write:{0}/staging/*".format(bucket), "List:{0}/staging/*".format(bucket), "Read:{0}/originals/*".format(bucket), "Write:{0}/originals/*".format(bucket), "List:{0}/originals/*".format(bucket), "Read:{0}/variants/*".format(bucket), "Write:{0}/variants/*".format(bucket), "List:{0}/variants/*".format(bucket), ], }, ] } target = ROOT / "secrets" / "seaweed" target.mkdir(parents=True, exist_ok=True) (target / "s3.json").write_text(json.dumps(config), encoding="utf-8") def configure_bucket(pw): admin = ROOT / "secrets" / "seaweed" / "admin.json" admin.write_text( json.dumps({"access_key": pw["s3_admin_key"], "secret_key": pw["s3_admin_secret"]}), encoding="utf-8", ) result = subprocess.run( [ "docker", "run", "--rm", "--network", "mtusi-lab7_app", "-v", str(PKI / "ca.crt") + ":/certs/ca.crt:ro", "-v", str(admin) + ":/certs/admin.json:ro", "--entrypoint", "python", "mtusi-lab7-messenger:1.7.0", "-c", ( "import json,boto3; from botocore.config import Config; " "admin=json.load(open('/certs/admin.json')); " "c=boto3.client('s3',endpoint_url='https://minio:8333',aws_access_key_id=admin['access_key']," "aws_secret_access_key=admin['secret_key'],region_name='us-east-1',verify='/certs/ca.crt'," "config=Config(signature_version='s3v4',s3={'addressing_style':'path'})); " "buckets=[b['Name'] for b in c.list_buckets().get('Buckets',[])]; " "c.create_bucket(Bucket='messenger-attachments') if 'messenger-attachments' not in buckets else None; " "c.put_object(Bucket='messenger-attachments',Key='canary/marker',Body=b'canary-not-a-secret'); " "print('s3-bootstrap-ok')" ), ], capture_output=True, text=True, ) admin.unlink(missing_ok=True) if result.returncode != 0 or "s3-bootstrap-ok" not in (result.stdout or ""): raise SystemExit("s3 bootstrap failed") def configure_rabbit(): users = { "api.messenger.internal": ("^attachment\\.process(\\.v1)?$", "^attachment\\.process$", "^$"), "worker.messenger.internal": ("^attachment\\.process\\.v1$", "^$", "^attachment\\.process\\.v1$"), } subprocess.run( ["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "await_online_nodes", "1"], cwd=ROOT, capture_output=True, text=True, ) for user, perms in users.items(): last = None for _ in range(10): subprocess.run( ["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "add_user", user, password()], cwd=ROOT, capture_output=True, text=True, ) last = subprocess.run( ["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "set_permissions", "-p", "/", user, *perms], cwd=ROOT, capture_output=True, text=True, ) if last.returncode == 0: break time.sleep(3) if last is None or last.returncode != 0: raise SystemExit("rabbit permissions failed for a service user") expressions = [ 'rabbit_exchange:declare({resource, <<"/">>, exchange, <<"attachment.process">>}, direct, true, false, false, [], <<"bootstrap">>).', 'rabbit_binding:add({binding, {resource, <<"/">>, exchange, <<"attachment.process">>}, <<"attachment.process.v1">>, {resource, <<"/">>, queue, <<"attachment.process.v1">>}, []}, <<"bootstrap">>).', ] for expression in expressions: topology = subprocess.run( ["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "eval", expression], cwd=ROOT, capture_output=True, text=True, ) if topology.returncode != 0: raise SystemExit("rabbit topology failed") subprocess.run( ["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "delete_user", "guest"], cwd=ROOT, capture_output=True, text=True, ) def wait_ready(): import ssl ctx = ssl.create_default_context(cafile=str(PKI / "ca.crt")) for _ in range(40): try: with urllib.request.urlopen("https://127.0.0.1:8443/health/ready", context=ctx, timeout=5) as response: if response.status == 200: print("ready https://127.0.0.1:8443/health/ready") return except Exception: time.sleep(3) raise SystemExit("public HTTPS endpoint did not become ready") def up(pw): write_s3_config(pw) compose("up", "-d", "openbao") init_openbao(pw) compose("build", "api-1", "postgres", "rabbitmq") compose("up", "-d", "--build", "postgres", "minio", "rabbitmq", "valkey") wait_healthy("postgres", "minio", "rabbitmq", "valkey") configure_bucket(pw) configure_rabbit() compose("up", "-d", "--build", "api-1", "api-2", "worker", "nginx", "otel-edge", "otel-collector", "prometheus", "loki", "tempo", "grafana") wait_ready() def main(): ensure_cryptography() serials = build_material() pw = material_passwords() write_configs(pw, serials) print("local material is in secrets/ and .env; private keys were not printed") if "--up" in sys.argv: up(pw) if __name__ == "__main__": main()