#!/usr/bin/env python3 """Explicit operator bootstrap/unseal/start/rotation; no secret values on stdout.""" import argparse import json import secrets import shutil import subprocess import time from securitylib import ( LOCAL, OPERATOR, ROLES, ROOT, compose, issue, openssl, put, refresh_crl, request, root_token, wait_bao, ) def prepare(): LOCAL.mkdir(mode=0o700, exist_ok=True) LOCAL.chmod(0o700) OPERATOR.mkdir(mode=0o700, exist_ok=True) OPERATOR.chmod(0o700) if not (OPERATOR / "ca.key").exists(): (OPERATOR / "newcerts").mkdir() put(OPERATOR / "index.txt", "", 0o600) put(OPERATOR / "serial", "1000\n", 0o600) put(OPERATOR / "crlnumber", "1000\n", 0o600) put( OPERATOR / "ca.cnf", f"""[ca] default_ca=lab [lab] database={OPERATOR}/index.txt new_certs_dir={OPERATOR}/newcerts certificate={LOCAL}/trust/ca.crt private_key={OPERATOR}/ca.key serial={OPERATOR}/serial crlnumber={OPERATOR}/crlnumber default_md=sha256 default_days=2 default_crl_days=7 policy=names unique_subject=no [names] commonName=supplied """, 0o600, ) (LOCAL / "trust").mkdir(exist_ok=True) openssl( "req", "-x509", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:P-256", "-nodes", "-keyout", OPERATOR / "ca.key", "-out", LOCAL / "trust/ca.crt", "-days", "365", "-subj", "/CN=MTUSI Lab7 Local CA", "-addext", "basicConstraints=critical,CA:TRUE", "-addext", "keyUsage=critical,keyCertSign,cRLSign", ) (OPERATOR / "ca.key").chmod(0o600) for role in ROLES: if not (LOCAL / "identities" / role / "service.crt").exists(): issue(role, expired=role == "expired") foreign = LOCAL / "identities/foreign" if not (foreign / "service.crt").exists(): foreign.mkdir(exist_ok=True) openssl( "req", "-x509", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:P-256", "-nodes", "-keyout", foreign / "service.key", "-out", foreign / "service.crt", "-days", "2", "-subj", "/CN=lb", "-addext", "subjectAltName=DNS:lb", ) (foreign / "service.key").chmod(0o644) if not (LOCAL / "trust/ca.crl").exists(): refresh_crl() path = OPERATOR / "credentials.json" if not path.exists(): values = { name: secrets.token_urlsafe(32) for name in [ "postgres", "admin", "grafana", "minio", "api_db", "worker_db", "dispatcher_db", "reconciler_db", "api_s3", "worker_s3", "reconciler_s3", "api_redis", "operator_redis", "worker_broker", "dispatcher_broker", "api_broker", "operator_broker", ] } put(path, json.dumps(values), 0o600) values = json.loads(path.read_text()) if "dispatcher_s3" not in values: values["dispatcher_s3"] = secrets.token_urlsafe(32) put(path, json.dumps(values), 0o600) put(OPERATOR / "postgres-password", values["postgres"]) put( OPERATOR / "init.sql", "\n".join( f"CREATE ROLE messenger_{role} LOGIN PASSWORD '{values[role + '_db']}' NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION;" for role in ["api", "worker", "dispatcher", "reconciler"] ) + "\nREVOKE CREATE ON SCHEMA public FROM PUBLIC;\n", ) put( OPERATOR / "minio.env", f"MINIO_ROOT_USER=operator\nMINIO_ROOT_PASSWORD={values['minio']}\nMINIO_BROWSER=off\n", ) put(OPERATOR / "grafana.env", f"GF_SECURITY_ADMIN_PASSWORD={values['grafana']}\n") common = { "S3_ENDPOINT": "https://minio:9000", "S3_BUCKET": "messenger-attachments", "ADMIN_PASSWORD": values["admin"], } bootstrap = { **common, "DATABASE_URL": f"postgresql://operator:{values['postgres']}@postgres:5432/messenger", "REDIS_URL": f"rediss://operator:{values['operator_redis']}@redis:6379/0", "MINIO_ROOT_USER": "operator", "MINIO_ROOT_PASSWORD": values["minio"], "S3_ACCESS_KEY_ID": "messenger-api", "S3_SECRET_ACCESS_KEY": values["api_s3"], "S3_WORKER_ACCESS_KEY_ID": values.get("worker_s3_access", "messenger-worker"), "S3_WORKER_SECRET_ACCESS_KEY": values["worker_s3"], "S3_RECONCILE_ACCESS_KEY_ID": "messenger-reconciler", "S3_RECONCILE_SECRET_ACCESS_KEY": values["reconciler_s3"], "S3_DISPATCHER_ACCESS_KEY_ID": "messenger-dispatcher", "S3_DISPATCHER_SECRET_ACCESS_KEY": values["dispatcher_s3"], } put(OPERATOR / "bootstrap.env", "\n".join(k + "=" + v for k, v in bootstrap.items()) + "\n") put( LOCAL / "config/redis.acl", f"user default off\nuser operator on >{values['operator_redis']} ~* +@all\nuser api on >{values['api_redis']} ~session:v1:* ~user:v1:* ~rl:login:* +ping +get +set +del +incr +pexpire +pttl +eval +multi +exec +client|setinfo\n", ) put( LOCAL / "config/broker.acl", f"user default off\nuser operator on >{values['operator_broker']} ~* +@all\nuser dispatcher on >{values['dispatcher_broker']} ~attachment-jobs-v1 +ping +xadd +client|setinfo\nuser worker on >{values['worker_broker']} ~attachment-jobs-v1 +ping +xgroup +xreadgroup +xautoclaim +xack +xdel +multi +exec +client|setinfo\nuser api on >{values['api_broker']} ~attachment-jobs-v1 +ping +client|setinfo\n", ) # Grafana embeds public CA/client certificate and its own private key only. identity = LOCAL / "identities/grafana" tlsdata = { "tlsCACert": (LOCAL / "trust/ca.crt").read_text(), "tlsClientCert": (identity / "service.crt").read_text(), "tlsClientKey": (identity / "service.key").read_text(), } sources = [] for name, kind, url in [ ("Prometheus", "prometheus", "https://prometheus:9090"), ("Loki", "loki", "https://loki:3100"), ("Tempo", "tempo", "https://tempo:3200"), ]: extra = { "httpMethod": "GET", "tlsAuth": True, "tlsAuthWithCACert": True, "tlsSkipVerify": False, } if kind == "loki": extra["derivedFields"] = [ { "name": "TraceID", "matcherRegex": '"trace_id":\\s*"([a-f0-9]+)"', "datasourceUid": "tempo", "url": "$${__value.raw}", } ] if kind == "tempo": extra["tracesToLogsV2"] = {"datasourceUid": "loki", "filterByTraceID": True} sources.append( { "name": name, "uid": kind, "type": kind, "url": url, "access": "proxy", "isDefault": kind == "prometheus", "jsonData": extra, "secureJsonData": tlsdata, } ) put( LOCAL / "config/grafana-sources.yaml", json.dumps({"apiVersion": 1, "datasources": sources}, indent=2), ) print("Prepared local CA, identities and service credentials; values not printed.") def unseal(): status = wait_bao() if not status["initialized"]: data = request("sys/init", {"secret_shares": 3, "secret_threshold": 2}) put(OPERATOR / "bao-init.json", json.dumps(data), 0o600) if status["sealed"]: data = json.loads((OPERATOR / "bao-init.json").read_text()) for key in data["keys"][:2]: request("sys/unseal", {"key": key}) print("Operator unseal completed; material remains outside server containers.") def configure_bao(): token = root_token() mounts = request("sys/mounts", token=token) if "secret/" not in mounts: request("sys/mounts/secret", {"type": "kv", "options": {"version": "2"}}, token=token) auth = request("sys/auth", token=token) if "cert/" not in auth: request("sys/auth/cert", {"type": "cert"}, token=token) values = json.loads((OPERATOR / "credentials.json").read_text()) for role in ["api", "worker", "dispatcher", "reconciler"]: policy = ( ROOT / f"infra/openbao/{role}.hcl" ).read_text() + '\npath "auth/token/renew-self" { capabilities = ["update"] }\n' request("sys/policies/acl/" + role, {"policy": policy}, token=token) request( "auth/cert/certs/" + role, { "certificate": (LOCAL / "trust/ca.crt").read_text(), "allowed_common_names": [role], "allowed_dns_sans": [role], "token_policies": [role], "token_ttl": 60, "token_max_ttl": 300, }, token=token, ) env = { "DATABASE_URL": f"postgresql://messenger_{role}:{values[role + '_db']}@postgres:5432/messenger", "REDIS_URL": f"rediss://api:{values['api_redis']}@redis:6379/0" if role == "api" else "rediss://unused:unused@redis:6379/0", "ADMIN_PASSWORD": values["admin"] if role == "api" else secrets.token_urlsafe(32), "S3_ENDPOINT": "https://minio:9000", "S3_BUCKET": "messenger-attachments", } s3role = role prefix = { "api": "S3", "worker": "S3_WORKER", "reconciler": "S3_RECONCILE", "dispatcher": "S3_DISPATCHER", }[s3role] env[prefix + "_ACCESS_KEY_ID"] = values.get(s3role + "_s3_access", "messenger-" + s3role) env[prefix + "_SECRET_ACCESS_KEY"] = values[s3role + "_s3"] if role in ["worker", "dispatcher"]: env["BROKER_URL"] = f"rediss://{role}:{values[role + '_broker']}@broker:6379/0" import urllib.error try: request("secret/data/messenger/" + role, token=token) except urllib.error.HTTPError as error: if error.code != 404: raise request("secret/data/messenger/" + role, {"data": env}, token=token) request("auth/cert/crls/lab7", {"crl": (LOCAL / "trust/ca.crl").read_text()}, token=token) try: request( "sys/audit/file", {"type": "file", "options": {"file_path": "/bao/data/audit.jsonl"}}, token=token, ) except Exception: pass print("Configured restricted service policies, certificate auth and KV secrets.") def sql(statement): # stdin keeps credentials and SQL literals out of process command lines/output. subprocess.run( [ "docker", "compose", "exec", "-T", "postgres", "psql", "-U", "operator", "-d", "messenger", "-p", "15432", "-v", "ON_ERROR_STOP=1", ], input=statement, text=True, cwd=ROOT, check=True, stdout=subprocess.DEVNULL, ) def grants(): sql("""REVOKE ALL ON DATABASE messenger FROM PUBLIC; GRANT CONNECT ON DATABASE messenger TO messenger_api,messenger_worker,messenger_dispatcher,messenger_reconciler; GRANT USAGE ON SCHEMA public TO messenger_api,messenger_worker,messenger_dispatcher,messenger_reconciler; GRANT SELECT,INSERT,UPDATE,DELETE ON users,chats,chat_members,messages,message_requests,attachments,attachment_jobs,job_replays,message_attachments,upload_intents,storage_findings TO messenger_api; GRANT SELECT,UPDATE ON attachments,attachment_jobs TO messenger_worker; GRANT SELECT,UPDATE ON attachments,attachment_jobs,upload_intents TO messenger_dispatcher; GRANT SELECT,INSERT,UPDATE,DELETE ON attachments,upload_intents,storage_findings TO messenger_reconciler; GRANT SELECT ON message_attachments TO messenger_reconciler; """) def bootstrap(): prepare() compose("build", "api") compose( "up", "-d", "volume-init", "postgres", "postgres-tls", "redis", "redis-tls", "broker", "broker-tls", "minio", "openbao", ) unseal() # KV existence is checked per path on reruns. configure_bao() for _ in range(40): try: compose("run", "--rm", "--no-deps", "migration") break except subprocess.CalledProcessError: time.sleep(2) else: raise RuntimeError("Migration unavailable") grants() compose("run", "--rm", "--no-deps", "s3-bootstrap") compose( "up", "-d", "otel-collector", "otel-tls", "prometheus", "prometheus-tls", "loki", "loki-tls", "tempo", "tempo-tls", "grafana", "api", "worker", "dispatcher", "reconciler", ) compose("up", "-d", "--wait", "lb") print("Lab7 ready: https://localhost:8443 ; CA: .local/trust/ca.crt") def rotate_certificate(role): folder = LOCAL / "identities" / role archive = OPERATOR / f"revoked-{role}-{int(time.time())}" shutil.copytree(folder, archive) archive.chmod(0o700) issue(role) openssl("ca", "-config", OPERATOR / "ca.cnf", "-revoke", archive / "service.crt") refresh_crl() request( "auth/cert/crls/lab7", {"crl": (LOCAL / "trust/ca.crl").read_text()}, token=root_token() ) # Close old persistent TLS connections and reload CRL on all enforcing gateways. compose( "restart", "postgres-tls", "redis-tls", "broker-tls", "otel-tls", "loki-tls", "tempo-tls", "prometheus-tls", "lb", ) if role not in ["api", "worker", "dispatcher", "reconciler"]: compose("restart", role) print("Rotated leaf certificate and revoked previous serial; no image rebuild.") def rotate_secret(): record = request("secret/data/messenger/worker", token=root_token())["data"]["data"] oldurl = record["DATABASE_URL"] oldrole = oldurl.split("://", 1)[1].split(":", 1)[0] newrole = "worker_" + secrets.token_hex(4) password = secrets.token_urlsafe(32) sql( f"CREATE ROLE {newrole} LOGIN PASSWORD '{password}' NOSUPERUSER NOCREATEDB NOCREATEROLE; GRANT messenger_worker TO {newrole};" ) record["DATABASE_URL"] = f"postgresql://{newrole}:{password}@postgres:5432/messenger" request("secret/data/messenger/worker", {"data": record}, token=root_token()) time.sleep(15) sql( f"ALTER ROLE {oldrole} NOLOGIN; SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE usename='{oldrole}';" ) put( OPERATOR / "last-worker-rotation.json", json.dumps({"old_url": oldurl, "new_role": newrole}), 0o600, ) print("Rotated worker DB credential, revoked old login and closed old sessions.") def rotate_s3(): token = root_token() record = request("secret/data/messenger/worker", token=token)["data"]["data"] old = {key: record[key] for key in ["S3_WORKER_ACCESS_KEY_ID", "S3_WORKER_SECRET_ACCESS_KEY"]} access = "worker-" + secrets.token_hex(5) password = secrets.token_urlsafe(32) code = """import json,sys from src.core.storage_bootstrap import create_admin_client x=json.load(sys.stdin);admin=create_admin_client() if x['action']=='create': admin.user_add(x['access'],x['password']);admin.policy_set('messenger-worker',user=x['access']) else:admin.user_disable(x['access']) """ def admin(data): subprocess.run( [ "docker", "compose", "run", "--rm", "--no-deps", "-T", "s3-bootstrap", "python", "-c", code, ], cwd=ROOT, text=True, input=json.dumps(data), check=True, stdout=subprocess.DEVNULL, ) admin({"action": "create", "access": access, "password": password}) record.update(S3_WORKER_ACCESS_KEY_ID=access, S3_WORKER_SECRET_ACCESS_KEY=password) request("secret/data/messenger/worker", {"data": record}, token=token) time.sleep(15) admin({"action": "disable", "access": old["S3_WORKER_ACCESS_KEY_ID"]}) values = json.loads((OPERATOR / "credentials.json").read_text()) values.update(worker_s3=password, worker_s3_access=access) put(OPERATOR / "credentials.json", json.dumps(values), 0o600) put(OPERATOR / "last-s3-rotation.json", json.dumps(old), 0o600) prepare() print( "Rotated worker S3 identity; disabled old access key; bootstrap retains current credentials." ) def renew_identities(): # Explicit maintenance operation; CA remains unchanged, data volumes untouched. gateways = [ "postgres-tls", "redis-tls", "broker-tls", "otel-tls", "loki-tls", "tempo-tls", "prometheus-tls", ] compose( "stop", "api", "worker", "dispatcher", "reconciler", "lb", *gateways, "openbao", "minio", "grafana", "otel-collector", "prometheus", ) archive = OPERATOR / ("renewal-" + str(int(time.time()))) archive.mkdir(mode=0o700) for role in ROLES: if role == "expired": continue old = archive / (role + ".crt") shutil.copy2(LOCAL / "identities" / role / "service.crt", old) issue(role) openssl("ca", "-config", OPERATOR / "ca.cnf", "-revoke", old) refresh_crl() prepare() compose("up", "-d", "volume-init") compose("start", "openbao", "minio", "grafana", "otel-collector", "prometheus", *gateways) unseal() configure_bao() compose( "up", "-d", "--no-recreate", "--wait", "api", "worker", "dispatcher", "reconciler", "lb" ) print("Renewed leaf identities with the existing CA; old serials revoked; data preserved.") if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "action", choices=[ "prepare", "bootstrap", "unseal", "rotate-cert", "rotate-secret", "renew-identities", "rotate-s3", ], ) parser.add_argument("--role", default="worker", choices=ROLES) args = parser.parse_args() { "prepare": prepare, "bootstrap": bootstrap, "unseal": unseal, "rotate-cert": lambda: rotate_certificate(args.role), "rotate-secret": rotate_secret, "renew-identities": renew_identities, "rotate-s3": rotate_s3, }[args.action]()