Files
2026-11-20 15:30:33 +03:00

569 lines
19 KiB
Python

#!/usr/bin/env python3
"""Explicit operator bootstrap/unseal/start/rotation; no secret values on stdout."""
import argparse
import json
import secrets
import shutil
import subprocess
import time
from securitylib import (
LOCAL,
OPERATOR,
ROLES,
ROOT,
compose,
issue,
openssl,
put,
refresh_crl,
request,
root_token,
wait_bao,
)
def prepare():
LOCAL.mkdir(mode=0o700, exist_ok=True)
LOCAL.chmod(0o700)
OPERATOR.mkdir(mode=0o700, exist_ok=True)
OPERATOR.chmod(0o700)
if not (OPERATOR / "ca.key").exists():
(OPERATOR / "newcerts").mkdir()
put(OPERATOR / "index.txt", "", 0o600)
put(OPERATOR / "serial", "1000\n", 0o600)
put(OPERATOR / "crlnumber", "1000\n", 0o600)
put(
OPERATOR / "ca.cnf",
f"""[ca]
default_ca=lab
[lab]
database={OPERATOR}/index.txt
new_certs_dir={OPERATOR}/newcerts
certificate={LOCAL}/trust/ca.crt
private_key={OPERATOR}/ca.key
serial={OPERATOR}/serial
crlnumber={OPERATOR}/crlnumber
default_md=sha256
default_days=2
default_crl_days=7
policy=names
unique_subject=no
[names]
commonName=supplied
""",
0o600,
)
(LOCAL / "trust").mkdir(exist_ok=True)
openssl(
"req",
"-x509",
"-newkey",
"ec",
"-pkeyopt",
"ec_paramgen_curve:P-256",
"-nodes",
"-keyout",
OPERATOR / "ca.key",
"-out",
LOCAL / "trust/ca.crt",
"-days",
"365",
"-subj",
"/CN=MTUSI Lab7 Local CA",
"-addext",
"basicConstraints=critical,CA:TRUE",
"-addext",
"keyUsage=critical,keyCertSign,cRLSign",
)
(OPERATOR / "ca.key").chmod(0o600)
for role in ROLES:
if not (LOCAL / "identities" / role / "service.crt").exists():
issue(role, expired=role == "expired")
foreign = LOCAL / "identities/foreign"
if not (foreign / "service.crt").exists():
foreign.mkdir(exist_ok=True)
openssl(
"req",
"-x509",
"-newkey",
"ec",
"-pkeyopt",
"ec_paramgen_curve:P-256",
"-nodes",
"-keyout",
foreign / "service.key",
"-out",
foreign / "service.crt",
"-days",
"2",
"-subj",
"/CN=lb",
"-addext",
"subjectAltName=DNS:lb",
)
(foreign / "service.key").chmod(0o644)
if not (LOCAL / "trust/ca.crl").exists():
refresh_crl()
path = OPERATOR / "credentials.json"
if not path.exists():
values = {
name: secrets.token_urlsafe(32)
for name in [
"postgres",
"admin",
"grafana",
"minio",
"api_db",
"worker_db",
"dispatcher_db",
"reconciler_db",
"api_s3",
"worker_s3",
"reconciler_s3",
"api_redis",
"operator_redis",
"worker_broker",
"dispatcher_broker",
"api_broker",
"operator_broker",
]
}
put(path, json.dumps(values), 0o600)
values = json.loads(path.read_text())
if "dispatcher_s3" not in values:
values["dispatcher_s3"] = secrets.token_urlsafe(32)
put(path, json.dumps(values), 0o600)
put(OPERATOR / "postgres-password", values["postgres"])
put(
OPERATOR / "init.sql",
"\n".join(
f"CREATE ROLE messenger_{role} LOGIN PASSWORD '{values[role + '_db']}' NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION;"
for role in ["api", "worker", "dispatcher", "reconciler"]
)
+ "\nREVOKE CREATE ON SCHEMA public FROM PUBLIC;\n",
)
put(
OPERATOR / "minio.env",
f"MINIO_ROOT_USER=operator\nMINIO_ROOT_PASSWORD={values['minio']}\nMINIO_BROWSER=off\n",
)
put(OPERATOR / "grafana.env", f"GF_SECURITY_ADMIN_PASSWORD={values['grafana']}\n")
common = {
"S3_ENDPOINT": "https://minio:9000",
"S3_BUCKET": "messenger-attachments",
"ADMIN_PASSWORD": values["admin"],
}
bootstrap = {
**common,
"DATABASE_URL": f"postgresql://operator:{values['postgres']}@postgres:5432/messenger",
"REDIS_URL": f"rediss://operator:{values['operator_redis']}@redis:6379/0",
"MINIO_ROOT_USER": "operator",
"MINIO_ROOT_PASSWORD": values["minio"],
"S3_ACCESS_KEY_ID": "messenger-api",
"S3_SECRET_ACCESS_KEY": values["api_s3"],
"S3_WORKER_ACCESS_KEY_ID": values.get("worker_s3_access", "messenger-worker"),
"S3_WORKER_SECRET_ACCESS_KEY": values["worker_s3"],
"S3_RECONCILE_ACCESS_KEY_ID": "messenger-reconciler",
"S3_RECONCILE_SECRET_ACCESS_KEY": values["reconciler_s3"],
"S3_DISPATCHER_ACCESS_KEY_ID": "messenger-dispatcher",
"S3_DISPATCHER_SECRET_ACCESS_KEY": values["dispatcher_s3"],
}
put(OPERATOR / "bootstrap.env", "\n".join(k + "=" + v for k, v in bootstrap.items()) + "\n")
put(
LOCAL / "config/redis.acl",
f"user default off\nuser operator on >{values['operator_redis']} ~* +@all\nuser api on >{values['api_redis']} ~session:v1:* ~user:v1:* ~rl:login:* +ping +get +set +del +incr +pexpire +pttl +eval +multi +exec +client|setinfo\n",
)
put(
LOCAL / "config/broker.acl",
f"user default off\nuser operator on >{values['operator_broker']} ~* +@all\nuser dispatcher on >{values['dispatcher_broker']} ~attachment-jobs-v1 +ping +xadd +client|setinfo\nuser worker on >{values['worker_broker']} ~attachment-jobs-v1 +ping +xgroup +xreadgroup +xautoclaim +xack +xdel +multi +exec +client|setinfo\nuser api on >{values['api_broker']} ~attachment-jobs-v1 +ping +client|setinfo\n",
)
# Grafana embeds public CA/client certificate and its own private key only.
identity = LOCAL / "identities/grafana"
tlsdata = {
"tlsCACert": (LOCAL / "trust/ca.crt").read_text(),
"tlsClientCert": (identity / "service.crt").read_text(),
"tlsClientKey": (identity / "service.key").read_text(),
}
sources = []
for name, kind, url in [
("Prometheus", "prometheus", "https://prometheus:9090"),
("Loki", "loki", "https://loki:3100"),
("Tempo", "tempo", "https://tempo:3200"),
]:
extra = {
"httpMethod": "GET",
"tlsAuth": True,
"tlsAuthWithCACert": True,
"tlsSkipVerify": False,
}
if kind == "loki":
extra["derivedFields"] = [
{
"name": "TraceID",
"matcherRegex": '"trace_id":\\s*"([a-f0-9]+)"',
"datasourceUid": "tempo",
"url": "$${__value.raw}",
}
]
if kind == "tempo":
extra["tracesToLogsV2"] = {"datasourceUid": "loki", "filterByTraceID": True}
sources.append(
{
"name": name,
"uid": kind,
"type": kind,
"url": url,
"access": "proxy",
"isDefault": kind == "prometheus",
"jsonData": extra,
"secureJsonData": tlsdata,
}
)
put(
LOCAL / "config/grafana-sources.yaml",
json.dumps({"apiVersion": 1, "datasources": sources}, indent=2),
)
print("Prepared local CA, identities and service credentials; values not printed.")
def unseal():
status = wait_bao()
if not status["initialized"]:
data = request("sys/init", {"secret_shares": 3, "secret_threshold": 2})
put(OPERATOR / "bao-init.json", json.dumps(data), 0o600)
if status["sealed"]:
data = json.loads((OPERATOR / "bao-init.json").read_text())
for key in data["keys"][:2]:
request("sys/unseal", {"key": key})
print("Operator unseal completed; material remains outside server containers.")
def configure_bao():
token = root_token()
mounts = request("sys/mounts", token=token)
if "secret/" not in mounts:
request("sys/mounts/secret", {"type": "kv", "options": {"version": "2"}}, token=token)
auth = request("sys/auth", token=token)
if "cert/" not in auth:
request("sys/auth/cert", {"type": "cert"}, token=token)
values = json.loads((OPERATOR / "credentials.json").read_text())
for role in ["api", "worker", "dispatcher", "reconciler"]:
policy = (
ROOT / f"infra/openbao/{role}.hcl"
).read_text() + '\npath "auth/token/renew-self" { capabilities = ["update"] }\n'
request("sys/policies/acl/" + role, {"policy": policy}, token=token)
request(
"auth/cert/certs/" + role,
{
"certificate": (LOCAL / "trust/ca.crt").read_text(),
"allowed_common_names": [role],
"allowed_dns_sans": [role],
"token_policies": [role],
"token_ttl": 60,
"token_max_ttl": 300,
},
token=token,
)
env = {
"DATABASE_URL": f"postgresql://messenger_{role}:{values[role + '_db']}@postgres:5432/messenger",
"REDIS_URL": f"rediss://api:{values['api_redis']}@redis:6379/0"
if role == "api"
else "rediss://unused:unused@redis:6379/0",
"ADMIN_PASSWORD": values["admin"] if role == "api" else secrets.token_urlsafe(32),
"S3_ENDPOINT": "https://minio:9000",
"S3_BUCKET": "messenger-attachments",
}
s3role = role
prefix = {
"api": "S3",
"worker": "S3_WORKER",
"reconciler": "S3_RECONCILE",
"dispatcher": "S3_DISPATCHER",
}[s3role]
env[prefix + "_ACCESS_KEY_ID"] = values.get(s3role + "_s3_access", "messenger-" + s3role)
env[prefix + "_SECRET_ACCESS_KEY"] = values[s3role + "_s3"]
if role in ["worker", "dispatcher"]:
env["BROKER_URL"] = f"rediss://{role}:{values[role + '_broker']}@broker:6379/0"
import urllib.error
try:
request("secret/data/messenger/" + role, token=token)
except urllib.error.HTTPError as error:
if error.code != 404:
raise
request("secret/data/messenger/" + role, {"data": env}, token=token)
request("auth/cert/crls/lab7", {"crl": (LOCAL / "trust/ca.crl").read_text()}, token=token)
try:
request(
"sys/audit/file",
{"type": "file", "options": {"file_path": "/bao/data/audit.jsonl"}},
token=token,
)
except Exception:
pass
print("Configured restricted service policies, certificate auth and KV secrets.")
def sql(statement):
# stdin keeps credentials and SQL literals out of process command lines/output.
subprocess.run(
[
"docker",
"compose",
"exec",
"-T",
"postgres",
"psql",
"-U",
"operator",
"-d",
"messenger",
"-p",
"15432",
"-v",
"ON_ERROR_STOP=1",
],
input=statement,
text=True,
cwd=ROOT,
check=True,
stdout=subprocess.DEVNULL,
)
def grants():
sql("""REVOKE ALL ON DATABASE messenger FROM PUBLIC;
GRANT CONNECT ON DATABASE messenger TO messenger_api,messenger_worker,messenger_dispatcher,messenger_reconciler;
GRANT USAGE ON SCHEMA public TO messenger_api,messenger_worker,messenger_dispatcher,messenger_reconciler;
GRANT SELECT,INSERT,UPDATE,DELETE ON users,chats,chat_members,messages,message_requests,attachments,attachment_jobs,job_replays,message_attachments,upload_intents,storage_findings TO messenger_api;
GRANT SELECT,UPDATE ON attachments,attachment_jobs TO messenger_worker;
GRANT SELECT,UPDATE ON attachments,attachment_jobs,upload_intents TO messenger_dispatcher;
GRANT SELECT,INSERT,UPDATE,DELETE ON attachments,upload_intents,storage_findings TO messenger_reconciler;
GRANT SELECT ON message_attachments TO messenger_reconciler;
""")
def bootstrap():
prepare()
compose("build", "api")
compose(
"up",
"-d",
"volume-init",
"postgres",
"postgres-tls",
"redis",
"redis-tls",
"broker",
"broker-tls",
"minio",
"openbao",
)
unseal()
# KV existence is checked per path on reruns.
configure_bao()
for _ in range(40):
try:
compose("run", "--rm", "--no-deps", "migration")
break
except subprocess.CalledProcessError:
time.sleep(2)
else:
raise RuntimeError("Migration unavailable")
grants()
compose("run", "--rm", "--no-deps", "s3-bootstrap")
compose(
"up",
"-d",
"otel-collector",
"otel-tls",
"prometheus",
"prometheus-tls",
"loki",
"loki-tls",
"tempo",
"tempo-tls",
"grafana",
"api",
"worker",
"dispatcher",
"reconciler",
)
compose("up", "-d", "--wait", "lb")
print("Lab7 ready: https://localhost:8443 ; CA: .local/trust/ca.crt")
def rotate_certificate(role):
folder = LOCAL / "identities" / role
archive = OPERATOR / f"revoked-{role}-{int(time.time())}"
shutil.copytree(folder, archive)
archive.chmod(0o700)
issue(role)
openssl("ca", "-config", OPERATOR / "ca.cnf", "-revoke", archive / "service.crt")
refresh_crl()
request(
"auth/cert/crls/lab7", {"crl": (LOCAL / "trust/ca.crl").read_text()}, token=root_token()
)
# Close old persistent TLS connections and reload CRL on all enforcing gateways.
compose(
"restart",
"postgres-tls",
"redis-tls",
"broker-tls",
"otel-tls",
"loki-tls",
"tempo-tls",
"prometheus-tls",
"lb",
)
if role not in ["api", "worker", "dispatcher", "reconciler"]:
compose("restart", role)
print("Rotated leaf certificate and revoked previous serial; no image rebuild.")
def rotate_secret():
record = request("secret/data/messenger/worker", token=root_token())["data"]["data"]
oldurl = record["DATABASE_URL"]
oldrole = oldurl.split("://", 1)[1].split(":", 1)[0]
newrole = "worker_" + secrets.token_hex(4)
password = secrets.token_urlsafe(32)
sql(
f"CREATE ROLE {newrole} LOGIN PASSWORD '{password}' NOSUPERUSER NOCREATEDB NOCREATEROLE; GRANT messenger_worker TO {newrole};"
)
record["DATABASE_URL"] = f"postgresql://{newrole}:{password}@postgres:5432/messenger"
request("secret/data/messenger/worker", {"data": record}, token=root_token())
time.sleep(15)
sql(
f"ALTER ROLE {oldrole} NOLOGIN; SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE usename='{oldrole}';"
)
put(
OPERATOR / "last-worker-rotation.json",
json.dumps({"old_url": oldurl, "new_role": newrole}),
0o600,
)
print("Rotated worker DB credential, revoked old login and closed old sessions.")
def rotate_s3():
token = root_token()
record = request("secret/data/messenger/worker", token=token)["data"]["data"]
old = {key: record[key] for key in ["S3_WORKER_ACCESS_KEY_ID", "S3_WORKER_SECRET_ACCESS_KEY"]}
access = "worker-" + secrets.token_hex(5)
password = secrets.token_urlsafe(32)
code = """import json,sys
from src.core.storage_bootstrap import create_admin_client
x=json.load(sys.stdin);admin=create_admin_client()
if x['action']=='create':
admin.user_add(x['access'],x['password']);admin.policy_set('messenger-worker',user=x['access'])
else:admin.user_disable(x['access'])
"""
def admin(data):
subprocess.run(
[
"docker",
"compose",
"run",
"--rm",
"--no-deps",
"-T",
"s3-bootstrap",
"python",
"-c",
code,
],
cwd=ROOT,
text=True,
input=json.dumps(data),
check=True,
stdout=subprocess.DEVNULL,
)
admin({"action": "create", "access": access, "password": password})
record.update(S3_WORKER_ACCESS_KEY_ID=access, S3_WORKER_SECRET_ACCESS_KEY=password)
request("secret/data/messenger/worker", {"data": record}, token=token)
time.sleep(15)
admin({"action": "disable", "access": old["S3_WORKER_ACCESS_KEY_ID"]})
values = json.loads((OPERATOR / "credentials.json").read_text())
values.update(worker_s3=password, worker_s3_access=access)
put(OPERATOR / "credentials.json", json.dumps(values), 0o600)
put(OPERATOR / "last-s3-rotation.json", json.dumps(old), 0o600)
prepare()
print(
"Rotated worker S3 identity; disabled old access key; bootstrap retains current credentials."
)
def renew_identities():
# Explicit maintenance operation; CA remains unchanged, data volumes untouched.
gateways = [
"postgres-tls",
"redis-tls",
"broker-tls",
"otel-tls",
"loki-tls",
"tempo-tls",
"prometheus-tls",
]
compose(
"stop",
"api",
"worker",
"dispatcher",
"reconciler",
"lb",
*gateways,
"openbao",
"minio",
"grafana",
"otel-collector",
"prometheus",
)
archive = OPERATOR / ("renewal-" + str(int(time.time())))
archive.mkdir(mode=0o700)
for role in ROLES:
if role == "expired":
continue
old = archive / (role + ".crt")
shutil.copy2(LOCAL / "identities" / role / "service.crt", old)
issue(role)
openssl("ca", "-config", OPERATOR / "ca.cnf", "-revoke", old)
refresh_crl()
prepare()
compose("up", "-d", "volume-init")
compose("start", "openbao", "minio", "grafana", "otel-collector", "prometheus", *gateways)
unseal()
configure_bao()
compose(
"up", "-d", "--no-recreate", "--wait", "api", "worker", "dispatcher", "reconciler", "lb"
)
print("Renewed leaf identities with the existing CA; old serials revoked; data preserved.")
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
"action",
choices=[
"prepare",
"bootstrap",
"unseal",
"rotate-cert",
"rotate-secret",
"renew-identities",
"rotate-s3",
],
)
parser.add_argument("--role", default="worker", choices=ROLES)
args = parser.parse_args()
{
"prepare": prepare,
"bootstrap": bootstrap,
"unseal": unseal,
"rotate-cert": lambda: rotate_certificate(args.role),
"rotate-secret": rotate_secret,
"renew-identities": renew_identities,
"rotate-s3": rotate_s3,
}[args.action]()