forked from mtusi2026/mtusi2026lab7
569 lines
19 KiB
Python
569 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Explicit operator bootstrap/unseal/start/rotation; no secret values on stdout."""
|
|
|
|
import argparse
|
|
import json
|
|
import secrets
|
|
import shutil
|
|
import subprocess
|
|
import time
|
|
|
|
from securitylib import (
|
|
LOCAL,
|
|
OPERATOR,
|
|
ROLES,
|
|
ROOT,
|
|
compose,
|
|
issue,
|
|
openssl,
|
|
put,
|
|
refresh_crl,
|
|
request,
|
|
root_token,
|
|
wait_bao,
|
|
)
|
|
|
|
|
|
def prepare():
|
|
LOCAL.mkdir(mode=0o700, exist_ok=True)
|
|
LOCAL.chmod(0o700)
|
|
OPERATOR.mkdir(mode=0o700, exist_ok=True)
|
|
OPERATOR.chmod(0o700)
|
|
if not (OPERATOR / "ca.key").exists():
|
|
(OPERATOR / "newcerts").mkdir()
|
|
put(OPERATOR / "index.txt", "", 0o600)
|
|
put(OPERATOR / "serial", "1000\n", 0o600)
|
|
put(OPERATOR / "crlnumber", "1000\n", 0o600)
|
|
put(
|
|
OPERATOR / "ca.cnf",
|
|
f"""[ca]
|
|
default_ca=lab
|
|
[lab]
|
|
database={OPERATOR}/index.txt
|
|
new_certs_dir={OPERATOR}/newcerts
|
|
certificate={LOCAL}/trust/ca.crt
|
|
private_key={OPERATOR}/ca.key
|
|
serial={OPERATOR}/serial
|
|
crlnumber={OPERATOR}/crlnumber
|
|
default_md=sha256
|
|
default_days=2
|
|
default_crl_days=7
|
|
policy=names
|
|
unique_subject=no
|
|
[names]
|
|
commonName=supplied
|
|
""",
|
|
0o600,
|
|
)
|
|
(LOCAL / "trust").mkdir(exist_ok=True)
|
|
openssl(
|
|
"req",
|
|
"-x509",
|
|
"-newkey",
|
|
"ec",
|
|
"-pkeyopt",
|
|
"ec_paramgen_curve:P-256",
|
|
"-nodes",
|
|
"-keyout",
|
|
OPERATOR / "ca.key",
|
|
"-out",
|
|
LOCAL / "trust/ca.crt",
|
|
"-days",
|
|
"365",
|
|
"-subj",
|
|
"/CN=MTUSI Lab7 Local CA",
|
|
"-addext",
|
|
"basicConstraints=critical,CA:TRUE",
|
|
"-addext",
|
|
"keyUsage=critical,keyCertSign,cRLSign",
|
|
)
|
|
(OPERATOR / "ca.key").chmod(0o600)
|
|
for role in ROLES:
|
|
if not (LOCAL / "identities" / role / "service.crt").exists():
|
|
issue(role, expired=role == "expired")
|
|
foreign = LOCAL / "identities/foreign"
|
|
if not (foreign / "service.crt").exists():
|
|
foreign.mkdir(exist_ok=True)
|
|
openssl(
|
|
"req",
|
|
"-x509",
|
|
"-newkey",
|
|
"ec",
|
|
"-pkeyopt",
|
|
"ec_paramgen_curve:P-256",
|
|
"-nodes",
|
|
"-keyout",
|
|
foreign / "service.key",
|
|
"-out",
|
|
foreign / "service.crt",
|
|
"-days",
|
|
"2",
|
|
"-subj",
|
|
"/CN=lb",
|
|
"-addext",
|
|
"subjectAltName=DNS:lb",
|
|
)
|
|
(foreign / "service.key").chmod(0o644)
|
|
if not (LOCAL / "trust/ca.crl").exists():
|
|
refresh_crl()
|
|
path = OPERATOR / "credentials.json"
|
|
if not path.exists():
|
|
values = {
|
|
name: secrets.token_urlsafe(32)
|
|
for name in [
|
|
"postgres",
|
|
"admin",
|
|
"grafana",
|
|
"minio",
|
|
"api_db",
|
|
"worker_db",
|
|
"dispatcher_db",
|
|
"reconciler_db",
|
|
"api_s3",
|
|
"worker_s3",
|
|
"reconciler_s3",
|
|
"api_redis",
|
|
"operator_redis",
|
|
"worker_broker",
|
|
"dispatcher_broker",
|
|
"api_broker",
|
|
"operator_broker",
|
|
]
|
|
}
|
|
put(path, json.dumps(values), 0o600)
|
|
values = json.loads(path.read_text())
|
|
if "dispatcher_s3" not in values:
|
|
values["dispatcher_s3"] = secrets.token_urlsafe(32)
|
|
put(path, json.dumps(values), 0o600)
|
|
put(OPERATOR / "postgres-password", values["postgres"])
|
|
put(
|
|
OPERATOR / "init.sql",
|
|
"\n".join(
|
|
f"CREATE ROLE messenger_{role} LOGIN PASSWORD '{values[role + '_db']}' NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION;"
|
|
for role in ["api", "worker", "dispatcher", "reconciler"]
|
|
)
|
|
+ "\nREVOKE CREATE ON SCHEMA public FROM PUBLIC;\n",
|
|
)
|
|
put(
|
|
OPERATOR / "minio.env",
|
|
f"MINIO_ROOT_USER=operator\nMINIO_ROOT_PASSWORD={values['minio']}\nMINIO_BROWSER=off\n",
|
|
)
|
|
put(OPERATOR / "grafana.env", f"GF_SECURITY_ADMIN_PASSWORD={values['grafana']}\n")
|
|
common = {
|
|
"S3_ENDPOINT": "https://minio:9000",
|
|
"S3_BUCKET": "messenger-attachments",
|
|
"ADMIN_PASSWORD": values["admin"],
|
|
}
|
|
bootstrap = {
|
|
**common,
|
|
"DATABASE_URL": f"postgresql://operator:{values['postgres']}@postgres:5432/messenger",
|
|
"REDIS_URL": f"rediss://operator:{values['operator_redis']}@redis:6379/0",
|
|
"MINIO_ROOT_USER": "operator",
|
|
"MINIO_ROOT_PASSWORD": values["minio"],
|
|
"S3_ACCESS_KEY_ID": "messenger-api",
|
|
"S3_SECRET_ACCESS_KEY": values["api_s3"],
|
|
"S3_WORKER_ACCESS_KEY_ID": values.get("worker_s3_access", "messenger-worker"),
|
|
"S3_WORKER_SECRET_ACCESS_KEY": values["worker_s3"],
|
|
"S3_RECONCILE_ACCESS_KEY_ID": "messenger-reconciler",
|
|
"S3_RECONCILE_SECRET_ACCESS_KEY": values["reconciler_s3"],
|
|
"S3_DISPATCHER_ACCESS_KEY_ID": "messenger-dispatcher",
|
|
"S3_DISPATCHER_SECRET_ACCESS_KEY": values["dispatcher_s3"],
|
|
}
|
|
put(OPERATOR / "bootstrap.env", "\n".join(k + "=" + v for k, v in bootstrap.items()) + "\n")
|
|
put(
|
|
LOCAL / "config/redis.acl",
|
|
f"user default off\nuser operator on >{values['operator_redis']} ~* +@all\nuser api on >{values['api_redis']} ~session:v1:* ~user:v1:* ~rl:login:* +ping +get +set +del +incr +pexpire +pttl +eval +multi +exec +client|setinfo\n",
|
|
)
|
|
put(
|
|
LOCAL / "config/broker.acl",
|
|
f"user default off\nuser operator on >{values['operator_broker']} ~* +@all\nuser dispatcher on >{values['dispatcher_broker']} ~attachment-jobs-v1 +ping +xadd +client|setinfo\nuser worker on >{values['worker_broker']} ~attachment-jobs-v1 +ping +xgroup +xreadgroup +xautoclaim +xack +xdel +multi +exec +client|setinfo\nuser api on >{values['api_broker']} ~attachment-jobs-v1 +ping +client|setinfo\n",
|
|
)
|
|
# Grafana embeds public CA/client certificate and its own private key only.
|
|
identity = LOCAL / "identities/grafana"
|
|
tlsdata = {
|
|
"tlsCACert": (LOCAL / "trust/ca.crt").read_text(),
|
|
"tlsClientCert": (identity / "service.crt").read_text(),
|
|
"tlsClientKey": (identity / "service.key").read_text(),
|
|
}
|
|
sources = []
|
|
for name, kind, url in [
|
|
("Prometheus", "prometheus", "https://prometheus:9090"),
|
|
("Loki", "loki", "https://loki:3100"),
|
|
("Tempo", "tempo", "https://tempo:3200"),
|
|
]:
|
|
extra = {
|
|
"httpMethod": "GET",
|
|
"tlsAuth": True,
|
|
"tlsAuthWithCACert": True,
|
|
"tlsSkipVerify": False,
|
|
}
|
|
if kind == "loki":
|
|
extra["derivedFields"] = [
|
|
{
|
|
"name": "TraceID",
|
|
"matcherRegex": '"trace_id":\\s*"([a-f0-9]+)"',
|
|
"datasourceUid": "tempo",
|
|
"url": "$${__value.raw}",
|
|
}
|
|
]
|
|
if kind == "tempo":
|
|
extra["tracesToLogsV2"] = {"datasourceUid": "loki", "filterByTraceID": True}
|
|
sources.append(
|
|
{
|
|
"name": name,
|
|
"uid": kind,
|
|
"type": kind,
|
|
"url": url,
|
|
"access": "proxy",
|
|
"isDefault": kind == "prometheus",
|
|
"jsonData": extra,
|
|
"secureJsonData": tlsdata,
|
|
}
|
|
)
|
|
put(
|
|
LOCAL / "config/grafana-sources.yaml",
|
|
json.dumps({"apiVersion": 1, "datasources": sources}, indent=2),
|
|
)
|
|
print("Prepared local CA, identities and service credentials; values not printed.")
|
|
|
|
|
|
def unseal():
|
|
status = wait_bao()
|
|
if not status["initialized"]:
|
|
data = request("sys/init", {"secret_shares": 3, "secret_threshold": 2})
|
|
put(OPERATOR / "bao-init.json", json.dumps(data), 0o600)
|
|
if status["sealed"]:
|
|
data = json.loads((OPERATOR / "bao-init.json").read_text())
|
|
for key in data["keys"][:2]:
|
|
request("sys/unseal", {"key": key})
|
|
print("Operator unseal completed; material remains outside server containers.")
|
|
|
|
|
|
def configure_bao():
|
|
token = root_token()
|
|
mounts = request("sys/mounts", token=token)
|
|
if "secret/" not in mounts:
|
|
request("sys/mounts/secret", {"type": "kv", "options": {"version": "2"}}, token=token)
|
|
auth = request("sys/auth", token=token)
|
|
if "cert/" not in auth:
|
|
request("sys/auth/cert", {"type": "cert"}, token=token)
|
|
values = json.loads((OPERATOR / "credentials.json").read_text())
|
|
for role in ["api", "worker", "dispatcher", "reconciler"]:
|
|
policy = (
|
|
ROOT / f"infra/openbao/{role}.hcl"
|
|
).read_text() + '\npath "auth/token/renew-self" { capabilities = ["update"] }\n'
|
|
request("sys/policies/acl/" + role, {"policy": policy}, token=token)
|
|
request(
|
|
"auth/cert/certs/" + role,
|
|
{
|
|
"certificate": (LOCAL / "trust/ca.crt").read_text(),
|
|
"allowed_common_names": [role],
|
|
"allowed_dns_sans": [role],
|
|
"token_policies": [role],
|
|
"token_ttl": 60,
|
|
"token_max_ttl": 300,
|
|
},
|
|
token=token,
|
|
)
|
|
env = {
|
|
"DATABASE_URL": f"postgresql://messenger_{role}:{values[role + '_db']}@postgres:5432/messenger",
|
|
"REDIS_URL": f"rediss://api:{values['api_redis']}@redis:6379/0"
|
|
if role == "api"
|
|
else "rediss://unused:unused@redis:6379/0",
|
|
"ADMIN_PASSWORD": values["admin"] if role == "api" else secrets.token_urlsafe(32),
|
|
"S3_ENDPOINT": "https://minio:9000",
|
|
"S3_BUCKET": "messenger-attachments",
|
|
}
|
|
s3role = role
|
|
prefix = {
|
|
"api": "S3",
|
|
"worker": "S3_WORKER",
|
|
"reconciler": "S3_RECONCILE",
|
|
"dispatcher": "S3_DISPATCHER",
|
|
}[s3role]
|
|
env[prefix + "_ACCESS_KEY_ID"] = values.get(s3role + "_s3_access", "messenger-" + s3role)
|
|
env[prefix + "_SECRET_ACCESS_KEY"] = values[s3role + "_s3"]
|
|
if role in ["worker", "dispatcher"]:
|
|
env["BROKER_URL"] = f"rediss://{role}:{values[role + '_broker']}@broker:6379/0"
|
|
import urllib.error
|
|
|
|
try:
|
|
request("secret/data/messenger/" + role, token=token)
|
|
except urllib.error.HTTPError as error:
|
|
if error.code != 404:
|
|
raise
|
|
request("secret/data/messenger/" + role, {"data": env}, token=token)
|
|
request("auth/cert/crls/lab7", {"crl": (LOCAL / "trust/ca.crl").read_text()}, token=token)
|
|
try:
|
|
request(
|
|
"sys/audit/file",
|
|
{"type": "file", "options": {"file_path": "/bao/data/audit.jsonl"}},
|
|
token=token,
|
|
)
|
|
except Exception:
|
|
pass
|
|
print("Configured restricted service policies, certificate auth and KV secrets.")
|
|
|
|
|
|
def sql(statement):
|
|
# stdin keeps credentials and SQL literals out of process command lines/output.
|
|
subprocess.run(
|
|
[
|
|
"docker",
|
|
"compose",
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"psql",
|
|
"-U",
|
|
"operator",
|
|
"-d",
|
|
"messenger",
|
|
"-p",
|
|
"15432",
|
|
"-v",
|
|
"ON_ERROR_STOP=1",
|
|
],
|
|
input=statement,
|
|
text=True,
|
|
cwd=ROOT,
|
|
check=True,
|
|
stdout=subprocess.DEVNULL,
|
|
)
|
|
|
|
|
|
def grants():
|
|
sql("""REVOKE ALL ON DATABASE messenger FROM PUBLIC;
|
|
GRANT CONNECT ON DATABASE messenger TO messenger_api,messenger_worker,messenger_dispatcher,messenger_reconciler;
|
|
GRANT USAGE ON SCHEMA public TO messenger_api,messenger_worker,messenger_dispatcher,messenger_reconciler;
|
|
GRANT SELECT,INSERT,UPDATE,DELETE ON users,chats,chat_members,messages,message_requests,attachments,attachment_jobs,job_replays,message_attachments,upload_intents,storage_findings TO messenger_api;
|
|
GRANT SELECT,UPDATE ON attachments,attachment_jobs TO messenger_worker;
|
|
GRANT SELECT,UPDATE ON attachments,attachment_jobs,upload_intents TO messenger_dispatcher;
|
|
GRANT SELECT,INSERT,UPDATE,DELETE ON attachments,upload_intents,storage_findings TO messenger_reconciler;
|
|
GRANT SELECT ON message_attachments TO messenger_reconciler;
|
|
""")
|
|
|
|
|
|
def bootstrap():
|
|
prepare()
|
|
compose("build", "api")
|
|
compose(
|
|
"up",
|
|
"-d",
|
|
"volume-init",
|
|
"postgres",
|
|
"postgres-tls",
|
|
"redis",
|
|
"redis-tls",
|
|
"broker",
|
|
"broker-tls",
|
|
"minio",
|
|
"openbao",
|
|
)
|
|
unseal()
|
|
# KV existence is checked per path on reruns.
|
|
configure_bao()
|
|
for _ in range(40):
|
|
try:
|
|
compose("run", "--rm", "--no-deps", "migration")
|
|
break
|
|
except subprocess.CalledProcessError:
|
|
time.sleep(2)
|
|
else:
|
|
raise RuntimeError("Migration unavailable")
|
|
grants()
|
|
compose("run", "--rm", "--no-deps", "s3-bootstrap")
|
|
compose(
|
|
"up",
|
|
"-d",
|
|
"otel-collector",
|
|
"otel-tls",
|
|
"prometheus",
|
|
"prometheus-tls",
|
|
"loki",
|
|
"loki-tls",
|
|
"tempo",
|
|
"tempo-tls",
|
|
"grafana",
|
|
"api",
|
|
"worker",
|
|
"dispatcher",
|
|
"reconciler",
|
|
)
|
|
compose("up", "-d", "--wait", "lb")
|
|
print("Lab7 ready: https://localhost:8443 ; CA: .local/trust/ca.crt")
|
|
|
|
|
|
def rotate_certificate(role):
|
|
folder = LOCAL / "identities" / role
|
|
archive = OPERATOR / f"revoked-{role}-{int(time.time())}"
|
|
shutil.copytree(folder, archive)
|
|
archive.chmod(0o700)
|
|
issue(role)
|
|
openssl("ca", "-config", OPERATOR / "ca.cnf", "-revoke", archive / "service.crt")
|
|
refresh_crl()
|
|
request(
|
|
"auth/cert/crls/lab7", {"crl": (LOCAL / "trust/ca.crl").read_text()}, token=root_token()
|
|
)
|
|
# Close old persistent TLS connections and reload CRL on all enforcing gateways.
|
|
compose(
|
|
"restart",
|
|
"postgres-tls",
|
|
"redis-tls",
|
|
"broker-tls",
|
|
"otel-tls",
|
|
"loki-tls",
|
|
"tempo-tls",
|
|
"prometheus-tls",
|
|
"lb",
|
|
)
|
|
if role not in ["api", "worker", "dispatcher", "reconciler"]:
|
|
compose("restart", role)
|
|
print("Rotated leaf certificate and revoked previous serial; no image rebuild.")
|
|
|
|
|
|
def rotate_secret():
|
|
record = request("secret/data/messenger/worker", token=root_token())["data"]["data"]
|
|
oldurl = record["DATABASE_URL"]
|
|
oldrole = oldurl.split("://", 1)[1].split(":", 1)[0]
|
|
newrole = "worker_" + secrets.token_hex(4)
|
|
password = secrets.token_urlsafe(32)
|
|
sql(
|
|
f"CREATE ROLE {newrole} LOGIN PASSWORD '{password}' NOSUPERUSER NOCREATEDB NOCREATEROLE; GRANT messenger_worker TO {newrole};"
|
|
)
|
|
record["DATABASE_URL"] = f"postgresql://{newrole}:{password}@postgres:5432/messenger"
|
|
request("secret/data/messenger/worker", {"data": record}, token=root_token())
|
|
time.sleep(15)
|
|
sql(
|
|
f"ALTER ROLE {oldrole} NOLOGIN; SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE usename='{oldrole}';"
|
|
)
|
|
put(
|
|
OPERATOR / "last-worker-rotation.json",
|
|
json.dumps({"old_url": oldurl, "new_role": newrole}),
|
|
0o600,
|
|
)
|
|
print("Rotated worker DB credential, revoked old login and closed old sessions.")
|
|
|
|
|
|
def rotate_s3():
|
|
token = root_token()
|
|
record = request("secret/data/messenger/worker", token=token)["data"]["data"]
|
|
old = {key: record[key] for key in ["S3_WORKER_ACCESS_KEY_ID", "S3_WORKER_SECRET_ACCESS_KEY"]}
|
|
access = "worker-" + secrets.token_hex(5)
|
|
password = secrets.token_urlsafe(32)
|
|
code = """import json,sys
|
|
from src.core.storage_bootstrap import create_admin_client
|
|
x=json.load(sys.stdin);admin=create_admin_client()
|
|
if x['action']=='create':
|
|
admin.user_add(x['access'],x['password']);admin.policy_set('messenger-worker',user=x['access'])
|
|
else:admin.user_disable(x['access'])
|
|
"""
|
|
|
|
def admin(data):
|
|
subprocess.run(
|
|
[
|
|
"docker",
|
|
"compose",
|
|
"run",
|
|
"--rm",
|
|
"--no-deps",
|
|
"-T",
|
|
"s3-bootstrap",
|
|
"python",
|
|
"-c",
|
|
code,
|
|
],
|
|
cwd=ROOT,
|
|
text=True,
|
|
input=json.dumps(data),
|
|
check=True,
|
|
stdout=subprocess.DEVNULL,
|
|
)
|
|
|
|
admin({"action": "create", "access": access, "password": password})
|
|
record.update(S3_WORKER_ACCESS_KEY_ID=access, S3_WORKER_SECRET_ACCESS_KEY=password)
|
|
request("secret/data/messenger/worker", {"data": record}, token=token)
|
|
time.sleep(15)
|
|
admin({"action": "disable", "access": old["S3_WORKER_ACCESS_KEY_ID"]})
|
|
values = json.loads((OPERATOR / "credentials.json").read_text())
|
|
values.update(worker_s3=password, worker_s3_access=access)
|
|
put(OPERATOR / "credentials.json", json.dumps(values), 0o600)
|
|
put(OPERATOR / "last-s3-rotation.json", json.dumps(old), 0o600)
|
|
prepare()
|
|
print(
|
|
"Rotated worker S3 identity; disabled old access key; bootstrap retains current credentials."
|
|
)
|
|
|
|
|
|
def renew_identities():
|
|
# Explicit maintenance operation; CA remains unchanged, data volumes untouched.
|
|
gateways = [
|
|
"postgres-tls",
|
|
"redis-tls",
|
|
"broker-tls",
|
|
"otel-tls",
|
|
"loki-tls",
|
|
"tempo-tls",
|
|
"prometheus-tls",
|
|
]
|
|
compose(
|
|
"stop",
|
|
"api",
|
|
"worker",
|
|
"dispatcher",
|
|
"reconciler",
|
|
"lb",
|
|
*gateways,
|
|
"openbao",
|
|
"minio",
|
|
"grafana",
|
|
"otel-collector",
|
|
"prometheus",
|
|
)
|
|
archive = OPERATOR / ("renewal-" + str(int(time.time())))
|
|
archive.mkdir(mode=0o700)
|
|
for role in ROLES:
|
|
if role == "expired":
|
|
continue
|
|
old = archive / (role + ".crt")
|
|
shutil.copy2(LOCAL / "identities" / role / "service.crt", old)
|
|
issue(role)
|
|
openssl("ca", "-config", OPERATOR / "ca.cnf", "-revoke", old)
|
|
refresh_crl()
|
|
prepare()
|
|
compose("up", "-d", "volume-init")
|
|
compose("start", "openbao", "minio", "grafana", "otel-collector", "prometheus", *gateways)
|
|
unseal()
|
|
configure_bao()
|
|
compose(
|
|
"up", "-d", "--no-recreate", "--wait", "api", "worker", "dispatcher", "reconciler", "lb"
|
|
)
|
|
print("Renewed leaf identities with the existing CA; old serials revoked; data preserved.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
"action",
|
|
choices=[
|
|
"prepare",
|
|
"bootstrap",
|
|
"unseal",
|
|
"rotate-cert",
|
|
"rotate-secret",
|
|
"renew-identities",
|
|
"rotate-s3",
|
|
],
|
|
)
|
|
parser.add_argument("--role", default="worker", choices=ROLES)
|
|
args = parser.parse_args()
|
|
{
|
|
"prepare": prepare,
|
|
"bootstrap": bootstrap,
|
|
"unseal": unseal,
|
|
"rotate-cert": lambda: rotate_certificate(args.role),
|
|
"rotate-secret": rotate_secret,
|
|
"renew-identities": renew_identities,
|
|
"rotate-s3": rotate_s3,
|
|
}[args.action]()
|