forked from mtusi2026/mtusi2026lab7
860 lines
43 KiB
Python
Executable File
860 lines
43 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Reproducible operator bootstrap/renewal/rotation for the local lab7 stand.
|
|
|
|
Private root/unseal state is outside this checkout. No secret values are printed.
|
|
All OpenBao calls verify CA and hostname; app roles authenticate independently.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import hashlib
|
|
import hmac
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import secrets
|
|
import ssl
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
from lab7_environment import BaoClient, Environment, OperationError, ROOT, ROLES, atomic_write
|
|
from lab7_pki import PKI
|
|
|
|
GATEWAY_PAIRS = {
|
|
'seaweedfs': ['s3'],
|
|
'broker': ['broker-admin-tls'],
|
|
'otel-collector': ['collector-tls'],
|
|
'prometheus': ['prometheus-tls'],
|
|
'loki': ['loki-tls'],
|
|
'tempo': ['tempo-tls'],
|
|
'grafana': ['grafana-tls'],
|
|
}
|
|
STORES = ['postgres', 'redis', 'seaweedfs', 'broker']
|
|
TELEMETRY = ['otel-collector', 'prometheus', 'loki', 'tempo', 'grafana']
|
|
|
|
|
|
def password_hash(password: str) -> str:
|
|
salt = secrets.token_bytes(4)
|
|
return base64.b64encode(salt + hashlib.sha256(salt + password.encode()).digest()).decode()
|
|
|
|
|
|
def scram_verifier(password: str, iterations: int = 4096) -> str:
|
|
"""Build a SCRAM-SHA-256 verifier so plaintext never enters SQL/logs."""
|
|
salt = secrets.token_bytes(16)
|
|
salted = hashlib.pbkdf2_hmac('sha256', password.encode(), salt, iterations)
|
|
client_key = hmac.new(salted, b'Client Key', hashlib.sha256).digest()
|
|
stored_key = hashlib.sha256(client_key).digest()
|
|
server_key = hmac.new(salted, b'Server Key', hashlib.sha256).digest()
|
|
encode = lambda value: base64.b64encode(value).decode()
|
|
return 'SCRAM-SHA-256$%d:%s$%s:%s' % (iterations, encode(salt), encode(stored_key), encode(server_key))
|
|
|
|
|
|
def service_policy(role: str) -> str:
|
|
rules = [f'''path "pki/issue/{role}" {{ capabilities = ["create", "update"] }}''',
|
|
'''path "auth/token/lookup-self" { capabilities = ["read"] }
|
|
path "auth/token/renew-self" { capabilities = ["update"] }
|
|
path "auth/token/revoke-self" { capabilities = ["update"] }''']
|
|
if role in ('api', 'worker', 'relay'):
|
|
rules.append(f'''path "secret/data/{role}" {{ capabilities = ["read"] }}''')
|
|
return '\n'.join(rules) + '\n'
|
|
|
|
|
|
def operator_policy() -> str:
|
|
paths = {
|
|
'secret/data/*': ['create', 'read', 'update', 'delete', 'list'],
|
|
'secret/metadata/*': ['read', 'list', 'delete'],
|
|
'pki/*': ['create', 'read', 'update', 'delete', 'list', 'sudo'],
|
|
'auth/cert/certs/*': ['create', 'read', 'update', 'delete', 'list'],
|
|
'auth/cert/crls/*': ['create', 'read', 'update', 'delete', 'list'],
|
|
'sys/policies/acl/lab7-*': ['create', 'read', 'update', 'delete', 'list'],
|
|
'sys/mounts': ['read'], 'sys/mounts/pki': ['create', 'read', 'update', 'sudo'],
|
|
'sys/mounts/secret': ['create', 'read', 'update', 'sudo'],
|
|
'sys/auth': ['read'], 'sys/auth/cert': ['create', 'read', 'update', 'sudo'],
|
|
'sys/audit': ['read', 'sudo'], 'sys/audit/file': ['read', 'sudo'],
|
|
'sys/seal': ['update', 'sudo'], 'auth/token/lookup-self': ['read'],
|
|
'auth/token/renew-self': ['update'], 'auth/token/revoke-self': ['update'],
|
|
}
|
|
return '\n'.join('path ' + json.dumps(path) + ' { capabilities = ' +
|
|
json.dumps(capabilities) + ' }' for path, capabilities in paths.items()) + '\n'
|
|
|
|
|
|
class Controller:
|
|
def __init__(self) -> None:
|
|
self.env = Environment()
|
|
self.pki = PKI(self.env)
|
|
self.credentials_path = self.env.secrets / 'backends/credentials.json'
|
|
self.init_path = self.env.state / 'operator-unseal.json'
|
|
|
|
def credentials(self) -> dict:
|
|
values = self.env.read_json(self.credentials_path)
|
|
if not values:
|
|
values = {
|
|
'postgres': {
|
|
'operator': {'username': 'lab_operator', 'password': secrets.token_hex(32)},
|
|
**{role: {'username': 'messenger_' + role, 'password': secrets.token_hex(32)}
|
|
for role in ('api', 'worker', 'relay')},
|
|
},
|
|
'redis': {role: {'username': role, 'password': secrets.token_hex(32)}
|
|
for role in ('cache', 'operator', 'health')},
|
|
's3': {role: {'access_key': 'lab7' + role + secrets.token_hex(8),
|
|
'secret_key': secrets.token_hex(32)}
|
|
for role in ('operator', 'api', 'worker')},
|
|
'grafana': {'password': secrets.token_hex(32)},
|
|
'broker': {'operator': {'password': secrets.token_hex(32)}},
|
|
}
|
|
self.env.private_json(self.credentials_path, values)
|
|
for family in values.values():
|
|
if isinstance(family, dict):
|
|
for record in family.values():
|
|
if isinstance(record, dict):
|
|
self.env.redactions.update(str(value) for key, value in record.items()
|
|
if key in ('password', 'secret_key', 'access_key'))
|
|
self.env.redactions.add(values['grafana']['password'])
|
|
return values
|
|
|
|
def write_file(self, relative: str, content: str) -> None:
|
|
destination = self.env.secrets / relative
|
|
destination.parent.mkdir(parents=True, exist_ok=True, mode=0o750)
|
|
os.chmod(destination.parent, 0o750)
|
|
atomic_write(destination, content, 0o640)
|
|
|
|
def backend_files(self) -> None:
|
|
credentials = self.credentials()
|
|
operator = credentials['postgres']['operator']
|
|
self.write_file('bootstrap/migrate/postgres-password', operator['password'] + '\n')
|
|
self.write_file('backends/postgres/operator-password', operator['password'] + '\n')
|
|
for role, prefix in (('operator', 's3-admin'), ('api', 's3-app')):
|
|
values = credentials['s3'][role]
|
|
self.write_file('bootstrap/s3/' + prefix + '-access-key', values['access_key'] + '\n')
|
|
self.write_file('bootstrap/s3/' + prefix + '-secret-key', values['secret_key'] + '\n')
|
|
identity_config = json.loads((ROOT / 'infra/s3/s3.json').read_text())
|
|
bucket = self.env.value('S3_BUCKET', 'messenger-attachments')
|
|
for identity in identity_config['identities']:
|
|
values = credentials['s3'][identity['name']]
|
|
identity['credentials'] = [{'accessKey': values['access_key'], 'secretKey': values['secret_key']}]
|
|
identity['actions'] = [action.replace('messenger-attachments', bucket)
|
|
for action in identity['actions']]
|
|
self.write_file('backends/s3/s3.json', json.dumps(identity_config, indent=2) + '\n')
|
|
cache = credentials['redis']['cache']['password']
|
|
operator_password = credentials['redis']['operator']['password']
|
|
health = credentials['redis']['health']['password']
|
|
acl = f'''user default off
|
|
user cache on >{cache} ~chat_list:* ~lock:chat_list:* +get +set +del +ping +eval +evalsha +script|load
|
|
user health on >{health} +ping
|
|
user operator on >{operator_password} ~* &* +@all
|
|
'''
|
|
self.write_file('backends/redis/users.acl', acl)
|
|
self.write_file('backends/redis/health-password', health + '\n')
|
|
self.write_file('backends/redis/operator-password', operator_password + '\n')
|
|
self.write_file('backends/grafana/admin-password', credentials['grafana']['password'] + '\n')
|
|
users = [
|
|
{'name': role, 'password_hash': password_hash(secrets.token_hex(32)),
|
|
'hashing_algorithm': 'rabbit_password_hashing_sha256', 'tags': []}
|
|
for role in ('worker', 'relay')
|
|
]
|
|
users.append({'name': 'operator',
|
|
'password_hash': password_hash(credentials['broker']['operator']['password']),
|
|
'hashing_algorithm': 'rabbit_password_hashing_sha256', 'tags': ['administrator']})
|
|
definitions = {
|
|
'users': users, 'vhosts': [{'name': 'messenger'}],
|
|
'permissions': [
|
|
{'user': 'relay', 'vhost': 'messenger', 'configure': '^$',
|
|
'write': r'^attachments\.jobs$', 'read': '^$'},
|
|
{'user': 'worker', 'vhost': 'messenger', 'configure': '^$',
|
|
'write': '^$', 'read': r'^attachment\.process\.v1$'},
|
|
{'user': 'operator', 'vhost': 'messenger', 'configure': '.*', 'write': '.*', 'read': '.*'},
|
|
],
|
|
'exchanges': [
|
|
{'name': name, 'vhost': 'messenger', 'type': 'direct', 'durable': True,
|
|
'auto_delete': False, 'internal': False, 'arguments': {}}
|
|
for name in ('attachments.jobs', 'attachments.dlx')
|
|
],
|
|
'queues': [
|
|
{'name': 'attachment.process.v1', 'vhost': 'messenger', 'durable': True,
|
|
'auto_delete': False, 'arguments': {
|
|
'x-queue-type': 'quorum', 'x-delivery-limit': 100,
|
|
'x-delayed-retry-type': 'all', 'x-delayed-retry-min': 5000,
|
|
'x-delayed-retry-max': 60000,
|
|
'x-dead-letter-exchange': 'attachments.dlx',
|
|
'x-dead-letter-routing-key': 'attachment.process.dead',
|
|
'x-dead-letter-strategy': 'at-least-once',
|
|
'x-overflow': 'reject-publish', 'x-max-length': 100000,
|
|
'x-consumer-timeout': 300000,
|
|
}},
|
|
{'name': 'attachment.process.dead', 'vhost': 'messenger', 'durable': True,
|
|
'auto_delete': False, 'arguments': {'x-queue-type': 'quorum', 'x-max-length': 100000,
|
|
'x-overflow': 'reject-publish', 'x-message-ttl': 604800000}},
|
|
],
|
|
'bindings': [
|
|
{'source': exchange, 'vhost': 'messenger', 'destination': queue,
|
|
'destination_type': 'queue', 'routing_key': queue, 'arguments': {}}
|
|
for exchange, queue in (('attachments.jobs', 'attachment.process.v1'),
|
|
('attachments.dlx', 'attachment.process.dead'))
|
|
],
|
|
}
|
|
self.write_file('backends/broker/definitions.json', json.dumps(definitions, indent=2) + '\n')
|
|
self.env.group_access(self.env.secrets / 'backends')
|
|
self.env.group_access(self.env.secrets / 'bootstrap')
|
|
|
|
def render_grafana(self) -> None:
|
|
template = (ROOT / 'infra/grafana/provisioning/datasources/datasources.yaml').read_text()
|
|
values = self.env.snapshot('grafana')
|
|
replacements = {
|
|
'__LAB7_CA_CERT__': values['ca.crt'],
|
|
'__LAB7_CLIENT_CERT__': values['tls.crt'],
|
|
'__LAB7_CLIENT_KEY__': values['tls.key'],
|
|
}
|
|
lines = []
|
|
for line in template.splitlines():
|
|
value = replacements.get(line.strip())
|
|
if value is None:
|
|
lines.append(line)
|
|
else:
|
|
indent = line[:len(line) - len(line.lstrip())]
|
|
lines.extend(indent + item for item in value.rstrip().splitlines())
|
|
self.write_file('backends/grafana/datasources/datasources.yaml', '\n'.join(lines) + '\n')
|
|
self.write_file('backends/grafana/dashboards/dashboards.yaml',
|
|
(ROOT / 'infra/grafana/provisioning/dashboards/dashboards.yaml').read_text())
|
|
self.env.group_access(self.env.secrets / 'backends/grafana')
|
|
|
|
def prepare(self) -> dict:
|
|
self.pki.prepare()
|
|
self.backend_files()
|
|
self.pki.recover_transport()
|
|
self.render_grafana()
|
|
metadata = {'project': self.env.project, 'pki_mount': 'pki',
|
|
'api_replicas': int(self.env.value('API_REPLICAS', '2'))}
|
|
self.env.private_json(self.env.secrets / 'metadata.json', metadata)
|
|
return {'prepared': True, 'project': self.env.project}
|
|
|
|
def unseal(self) -> dict:
|
|
deadline = time.monotonic() + 60
|
|
client = BaoClient(self.env)
|
|
while True:
|
|
try:
|
|
status = client.request('GET', 'sys/seal-status')
|
|
break
|
|
except OperationError:
|
|
if time.monotonic() >= deadline:
|
|
raise
|
|
time.sleep(1)
|
|
if not status.get('initialized'):
|
|
response = client.request('POST', 'sys/init', {'secret_shares': 3, 'secret_threshold': 2}, timeout=60)
|
|
material = {'keys': response['keys_base64'], 'root_token': response['root_token']}
|
|
self.env.private_json(self.init_path, material)
|
|
status = client.request('GET', 'sys/seal-status')
|
|
material = self.env.read_json(self.init_path)
|
|
if status.get('sealed'):
|
|
keys = material.get('keys', [])
|
|
if len(keys) < 2:
|
|
raise OperationError('operator unseal state is missing; it must never be recovered from the container')
|
|
for key in keys[:2]:
|
|
self.env.redactions.add(key)
|
|
status = client.request('POST', 'sys/unseal', {'key': key}, timeout=30)
|
|
if status.get('sealed'):
|
|
raise OperationError('operator unseal did not complete')
|
|
while True:
|
|
health = client.request('GET', 'sys/health', allowed=(200, 429, 472, 473, 501, 503))
|
|
if health.get('initialized') and not health.get('sealed') and not health.get('standby'):
|
|
break
|
|
if time.monotonic() >= deadline:
|
|
raise OperationError('unsealed OpenBao did not become the active node')
|
|
time.sleep(0.2)
|
|
return {'unsealed': True, 'project': self.env.project}
|
|
|
|
def seal(self) -> dict:
|
|
client = BaoClient(self.env)
|
|
client.request('POST', 'sys/seal', {}, client.login())
|
|
status = client.request('GET', 'sys/seal-status')
|
|
if not status.get('sealed'):
|
|
raise OperationError('operator seal did not close OpenBao')
|
|
return {'sealed': True, 'project': self.env.project}
|
|
|
|
def configure_auth(self, client: BaoClient, token: str) -> None:
|
|
methods = client.request('GET', 'sys/auth', token=token)
|
|
methods = methods.get('data', methods)
|
|
if 'cert/' not in methods:
|
|
client.request('POST', 'sys/auth/cert', {'type': 'cert'}, token)
|
|
for role in ROLES:
|
|
policy = operator_policy() if role == 'operator' else service_policy(role)
|
|
client.request('PUT', 'sys/policies/acl/lab7-' + role, {'policy': policy}, token)
|
|
client.request('POST', 'auth/cert/certs/' + role, {
|
|
'certificate': self.pki.root_bundle(),
|
|
'allowed_common_names': [role], 'allowed_uri_sans': ['spiffe://mtusi/' + role],
|
|
'token_policies': ['lab7-' + role], 'token_ttl': '60s',
|
|
'token_max_ttl': '300s', 'token_explicit_max_ttl': '300s',
|
|
'token_no_default_policy': True,
|
|
}, token)
|
|
|
|
def configure_kv(self, client: BaoClient, token: str) -> None:
|
|
mounts = client.request('GET', 'sys/mounts', token=token)
|
|
mounts = mounts.get('data', mounts)
|
|
if 'secret/' not in mounts:
|
|
client.request('POST', 'sys/mounts/secret', {'type': 'kv', 'options': {'version': '2'}}, token)
|
|
values = self.credentials()
|
|
for role in ('api', 'worker', 'relay'):
|
|
data = {'POSTGRES_PASSWORD': values['postgres'][role]['password']}
|
|
if role != 'relay':
|
|
data.update({'S3_ACCESS_KEY_ID': values['s3'][role]['access_key'],
|
|
'S3_SECRET_ACCESS_KEY': values['s3'][role]['secret_key']})
|
|
if role == 'api':
|
|
data['REDIS_PASSWORD'] = values['redis']['cache']['password']
|
|
client.request('POST', 'secret/data/' + role, {'data': data}, token)
|
|
|
|
def provision_bao(self) -> None:
|
|
material = self.env.read_json(self.init_path)
|
|
root_token = material.get('root_token')
|
|
if root_token:
|
|
self.env.redactions.add(root_token)
|
|
client = BaoClient(self.env)
|
|
token = root_token
|
|
else:
|
|
client = BaoClient(self.env)
|
|
token = client.login()
|
|
audits = client.request('GET', 'sys/audit', token=token)
|
|
audits = audits.get('data', audits)
|
|
if 'file/' not in audits:
|
|
raise OperationError('mandatory declarative audit device is not active')
|
|
self.configure_auth(client, token)
|
|
self.configure_kv(client, token)
|
|
self.pki.provision(client, token)
|
|
self.pki.sync_crls(client, token)
|
|
if root_token:
|
|
# Verify the separately scoped operator path before destroying initial root.
|
|
operator = BaoClient(self.env)
|
|
operator.login()
|
|
client.request('POST', 'auth/token/revoke-self', {}, root_token)
|
|
denied = client.request('GET', 'auth/token/lookup-self', token=root_token,
|
|
allowed=(403,))
|
|
if not denied.get('errors'):
|
|
raise OperationError('root credential was not rejected after revocation')
|
|
material.pop('root_token', None)
|
|
material['root_token_revoked'] = True
|
|
self.env.private_json(self.init_path, material)
|
|
self.env.private_json(self.env.state / 'bootstrap-state.json', {'bao_provisioned': True})
|
|
|
|
def sql(self, sql: str, user: str = 'lab_operator', *, check: bool = True) -> str:
|
|
result = self.env.compose('exec', '-T', 'postgres', 'psql', '-v', 'ON_ERROR_STOP=1',
|
|
'-U', user, '-d', 'messenger', '-At', data=sql,
|
|
check=check)
|
|
return result.stdout if result.returncode == 0 else ''
|
|
|
|
def reconcile_roles(self) -> None:
|
|
values = self.credentials()['postgres']
|
|
existing = None
|
|
for candidate in ('lab_operator', 'messenger', 'postgres'):
|
|
if self.sql('SELECT current_user;\n', candidate, check=False).strip() == candidate:
|
|
existing = candidate
|
|
break
|
|
if not existing:
|
|
raise OperationError('no local operator database role is available')
|
|
statements = []
|
|
for role, record in values.items():
|
|
name = record['username']
|
|
statements.append(f"DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname='{name}') "
|
|
f'THEN CREATE ROLE "{name}"; END IF; END $$;')
|
|
privilege = 'SUPERUSER CREATEDB CREATEROLE' if role == 'operator' else 'NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT'
|
|
statements.append(f'ALTER ROLE "{name}" WITH LOGIN {privilege} '
|
|
f"PASSWORD '{scram_verifier(record['password'])}' CONNECTION LIMIT 32;")
|
|
self.sql('\n'.join(statements), existing)
|
|
|
|
def grants(self) -> None:
|
|
sql = '''
|
|
REVOKE ALL ON DATABASE messenger FROM PUBLIC;
|
|
REVOKE CREATE ON SCHEMA public FROM PUBLIC;
|
|
REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC;
|
|
REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC;
|
|
GRANT CONNECT ON DATABASE messenger TO messenger_api, messenger_worker, messenger_relay;
|
|
GRANT USAGE ON SCHEMA public TO messenger_api, messenger_worker, messenger_relay;
|
|
REVOKE ALL ON ALL TABLES IN SCHEMA public FROM messenger_api, messenger_worker, messenger_relay;
|
|
REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM messenger_api, messenger_worker, messenger_relay;
|
|
GRANT SELECT, INSERT, UPDATE ON users, chats, chat_members, messages, user_chat_versions,
|
|
attachments, attachment_jobs, outbox_messages, attachment_job_events TO messenger_api;
|
|
GRANT SELECT, INSERT, UPDATE, DELETE ON sessions, message_idempotency, login_limits TO messenger_api;
|
|
GRANT SELECT ON attachment_variants TO messenger_api;
|
|
GRANT SELECT, INSERT ON message_attachments TO messenger_api;
|
|
GRANT DELETE ON attachments TO messenger_api;
|
|
GRANT SELECT, INSERT, UPDATE ON attachments, attachment_jobs, attachment_variants,
|
|
outbox_messages, attachment_job_events TO messenger_worker;
|
|
GRANT DELETE ON attachments TO messenger_worker;
|
|
GRANT SELECT, UPDATE ON attachments, attachment_jobs TO messenger_relay;
|
|
GRANT SELECT, INSERT, UPDATE, DELETE ON outbox_messages TO messenger_relay;
|
|
GRANT SELECT, INSERT ON attachment_job_events TO messenger_relay;
|
|
GRANT USAGE, SELECT ON outbox_messages_id_seq, attachment_job_events_id_seq
|
|
TO messenger_api, messenger_worker, messenger_relay;
|
|
DO $$ DECLARE app_object record; object_type text; BEGIN
|
|
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname='messenger') THEN
|
|
-- The initdb bootstrap principal owns pg_catalog objects that cannot be
|
|
-- reassigned globally. Only the application's public objects move owners.
|
|
FOR app_object IN
|
|
SELECT c.relname, c.relkind FROM pg_class c
|
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
WHERE n.nspname = 'public'
|
|
AND c.relowner = (SELECT oid FROM pg_roles WHERE rolname='messenger')
|
|
AND c.relkind IN ('r', 'p', 'S', 'v', 'm')
|
|
AND (c.relkind <> 'S' OR NOT EXISTS (
|
|
SELECT 1 FROM pg_depend d WHERE d.classid = 'pg_class'::regclass
|
|
AND d.objid = c.oid AND d.deptype IN ('a', 'i')
|
|
))
|
|
LOOP
|
|
object_type := CASE app_object.relkind
|
|
WHEN 'S' THEN 'SEQUENCE' WHEN 'v' THEN 'VIEW'
|
|
WHEN 'm' THEN 'MATERIALIZED VIEW' ELSE 'TABLE' END;
|
|
EXECUTE format('ALTER %s public.%I OWNER TO lab_operator', object_type, app_object.relname);
|
|
END LOOP;
|
|
ALTER DATABASE messenger OWNER TO lab_operator;
|
|
ALTER SCHEMA public OWNER TO lab_operator;
|
|
-- PostgreSQL 18 requires the initdb bootstrap role to remain SUPERUSER.
|
|
-- NOLOGIN plus removal of its password retires the legacy identity without
|
|
-- rebuilding the cluster or changing user data.
|
|
ALTER ROLE messenger NOLOGIN PASSWORD NULL NOCREATEDB NOCREATEROLE NOINHERIT;
|
|
END IF;
|
|
END $$;
|
|
'''
|
|
self.sql(sql)
|
|
|
|
def retire_broker_users(self) -> None:
|
|
for user in ('messenger', 'guest'):
|
|
self.env.compose('exec', '-T', 'broker', 'rabbitmqctl', 'delete_user', user, check=False)
|
|
|
|
def ensure_service_identities(self) -> dict:
|
|
"""Reissue leafs that expired while the stand was stopped.
|
|
|
|
`prepare` only fills gaps and does not inspect leaf expiry. Recovery is
|
|
operator-driven here; the running stand uses `make renewer`.
|
|
"""
|
|
replaced = self.pki.recover_transport()
|
|
if replaced:
|
|
self.env.compose('restart', 'openbao', timeout=60)
|
|
self.unseal()
|
|
renewed = []
|
|
for role in ROLES:
|
|
certificate = self.env.identity(role) / 'tls.crt'
|
|
if not certificate.exists():
|
|
continue
|
|
valid = self.env.run(['openssl', 'x509', '-checkend', '300', '-noout',
|
|
'-in', str(certificate)], check=False).returncode == 0
|
|
if not valid:
|
|
self.pki.issue(role)
|
|
renewed.append(role)
|
|
reloaded, skipped = [], []
|
|
for role in renewed:
|
|
try:
|
|
self.reload_identity(role)
|
|
reloaded.append(role)
|
|
except OperationError:
|
|
# A stopped service will load the reissued leaf when compose starts it.
|
|
skipped.append(role)
|
|
return {'renewed': renewed, 'reloaded': reloaded, 'reload_skipped': skipped}
|
|
|
|
def bootstrap(self) -> dict:
|
|
self.prepare()
|
|
self.env.compose('up', '-d', '--no-build', 'volume-init', timeout=300)
|
|
# volume-init is short-lived; consumers require successful completion.
|
|
self.env.compose('up', '-d', '--no-build', 'openbao', timeout=300)
|
|
self.unseal()
|
|
self.provision_bao()
|
|
# On the first successful bootstrap, transition from offline bootstrap
|
|
# leaves to restricted OpenBao-issued, short-lived service identities.
|
|
marker = self.env.state / 'service-leaves.json'
|
|
if not marker.exists():
|
|
leaves = [self.pki.issue(role) for role in ROLES if role != 'rogue']
|
|
self.env.private_json(marker, {'issued': leaves})
|
|
self.env.compose('restart', 'openbao', timeout=60)
|
|
self.unseal()
|
|
recovered = self.ensure_service_identities()
|
|
self.render_grafana()
|
|
self.env.compose('up', '-d', '--no-build', *STORES, 's3', timeout=300)
|
|
self.wait_stores()
|
|
self.reconcile_roles()
|
|
self.env.compose('run', '--rm', '--no-deps', 'migrate', timeout=180)
|
|
self.grants()
|
|
self.env.compose('run', '--rm', '--no-deps', 's3-init', timeout=180)
|
|
self.retire_broker_users()
|
|
return {'bootstrapped': True, 'project': self.env.project,
|
|
'expired_identities_renewed': recovered.get('renewed', []),
|
|
'expired_identities_reloaded': recovered.get('reloaded', []),
|
|
'root_token_revoked': self.env.read_json(self.init_path).get('root_token_revoked', False)}
|
|
|
|
def wait_stores(self) -> None:
|
|
deadline = time.monotonic() + 120
|
|
for service in ('postgres', 'redis', 'broker', 'seaweedfs'):
|
|
while True:
|
|
result = self.env.compose('ps', '-q', service).stdout.strip()
|
|
if result:
|
|
state = self.env.run(['docker', 'inspect', '--format',
|
|
'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}',
|
|
result]).stdout.strip()
|
|
if state in ('healthy', 'running'):
|
|
break
|
|
if time.monotonic() >= deadline:
|
|
raise OperationError(service + ' did not become ready')
|
|
time.sleep(1)
|
|
|
|
def wait_s3(self, timeout: float = 120) -> None:
|
|
"""Read old volume-backed bytes; never repair an established sentinel."""
|
|
deadline = time.monotonic() + timeout
|
|
marker = self.env.state / 's3-readiness.json'
|
|
sentinel = self.env.read_json(marker)
|
|
if not sentinel:
|
|
sentinel = {'bucket': self.env.value('S3_BUCKET', 'messenger-attachments'),
|
|
'key': 'operator-readiness/' + secrets.token_hex(16),
|
|
'body': secrets.token_hex(32768), 'established': False}
|
|
self.env.private_json(marker, sentinel)
|
|
payload = sentinel['body'].encode()
|
|
credential = self.credentials()['s3']['operator']
|
|
host = 'localhost:' + self.env.value('S3_PORT', '8333')
|
|
bucket_path = '/' + urllib.parse.quote(sentinel['bucket'], safe='')
|
|
object_path = bucket_path + '/' + urllib.parse.quote(sentinel['key'], safe='/~')
|
|
context = ssl.create_default_context(cafile=str(self.env.identity('operator') / 'ca.crt'))
|
|
context.load_cert_chain(str(self.env.identity('operator') / 'tls.crt'),
|
|
str(self.env.identity('operator') / 'tls.key'))
|
|
|
|
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
return None
|
|
|
|
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect(),
|
|
urllib.request.HTTPSHandler(context=context))
|
|
|
|
def request(method: str, path: str, body: bytes = b'') -> bytes:
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise TimeoutError()
|
|
stamp = time.strftime('%Y%m%dT%H%M%SZ', time.gmtime())
|
|
day = stamp[:8]
|
|
digest = hashlib.sha256(body).hexdigest()
|
|
headers = {'host': host, 'x-amz-content-sha256': digest, 'x-amz-date': stamp}
|
|
signed = ';'.join(sorted(headers))
|
|
canonical_headers = ''.join(name + ':' + headers[name] + '\n' for name in sorted(headers))
|
|
canonical = '\n'.join((method, path, '', canonical_headers, signed, digest))
|
|
scope = day + '/us-east-1/s3/aws4_request'
|
|
to_sign = 'AWS4-HMAC-SHA256\n' + stamp + '\n' + scope + '\n' + hashlib.sha256(canonical.encode()).hexdigest()
|
|
signing_key = ('AWS4' + credential['secret_key']).encode()
|
|
for part in (day, 'us-east-1', 's3', 'aws4_request'):
|
|
signing_key = hmac.new(signing_key, part.encode(), hashlib.sha256).digest()
|
|
signature = hmac.new(signing_key, to_sign.encode(), hashlib.sha256).hexdigest()
|
|
headers['Authorization'] = ('AWS4-HMAC-SHA256 Credential=' + credential['access_key'] +
|
|
'/' + scope + ', SignedHeaders=' + signed + ', Signature=' + signature)
|
|
req = urllib.request.Request('https://' + host + path, method=method, headers=headers,
|
|
data=body if method == 'PUT' else None)
|
|
with opener.open(req, timeout=min(5, remaining)) as response:
|
|
chunks = bytearray()
|
|
while len(chunks) <= len(payload):
|
|
if time.monotonic() >= deadline:
|
|
raise TimeoutError()
|
|
chunk = response.read1(min(8192, len(payload) + 1 - len(chunks)))
|
|
if not chunk:
|
|
break
|
|
chunks.extend(chunk)
|
|
return bytes(chunks)
|
|
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
if not sentinel['established']:
|
|
# Only this operator-owned key is written on first initialization.
|
|
request('PUT', object_path, payload)
|
|
if request('GET', object_path) != payload:
|
|
raise OperationError('persistent S3 readiness sentinel bytes changed')
|
|
if not sentinel['established']:
|
|
sentinel['established'] = True
|
|
self.env.private_json(marker, sentinel)
|
|
return
|
|
except urllib.error.HTTPError as exc:
|
|
if exc.code not in (404, 408, 429, 500, 502, 503, 504):
|
|
raise OperationError('S3 readiness request rejected (HTTP ' + str(exc.code) + ')') from None
|
|
except (urllib.error.URLError, TimeoutError, ConnectionError):
|
|
pass
|
|
time.sleep(min(1, max(0, deadline - time.monotonic())))
|
|
raise OperationError('CA-verified persistent S3 body readiness deadline exceeded')
|
|
|
|
def wait_ready(self, timeout: float = 120) -> None:
|
|
deadline = time.monotonic() + timeout
|
|
self.wait_s3(timeout=timeout)
|
|
context = ssl.create_default_context(cafile=str(self.env.identity('operator') / 'ca.crt'))
|
|
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}),
|
|
urllib.request.HTTPSHandler(context=context))
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with opener.open(self.env.base_url + '/health/ready', timeout=5) as response:
|
|
if response.status == 200:
|
|
return
|
|
except (urllib.error.URLError, TimeoutError):
|
|
pass
|
|
time.sleep(1)
|
|
raise OperationError('CA-verified public HTTPS readiness deadline exceeded')
|
|
|
|
def up(self) -> dict:
|
|
result = self.bootstrap()
|
|
self.env.compose('up', '-d', '--no-build', timeout=300)
|
|
# Apply current credentials/trust/CRLs even when owners were reused.
|
|
# Namespace consumers are recreated against the current owner ID.
|
|
gateways = [child for children in GATEWAY_PAIRS.values()
|
|
for child in children if child != 's3']
|
|
self.env.compose('up', '-d', '--no-build', '--no-deps', '--force-recreate',
|
|
*gateways, timeout=180)
|
|
self.env.compose('restart', 'lb', timeout=60)
|
|
self.wait_ready()
|
|
self.reset_grafana_admin()
|
|
return {**result, 'ready': True, 'base_url': self.env.base_url}
|
|
|
|
def wait_container_health(self, container: str, timeout: float = 120) -> None:
|
|
deadline = time.monotonic() + timeout
|
|
while True:
|
|
state = self.env.run(['docker', 'inspect', '--format',
|
|
'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}',
|
|
container], check=False).stdout.strip()
|
|
if state in ('healthy', 'running'):
|
|
return
|
|
if time.monotonic() >= deadline:
|
|
raise OperationError(container + ' did not become healthy after restart: ' + state)
|
|
time.sleep(1)
|
|
|
|
def rolling_restart(self, service: str) -> None:
|
|
containers = [item for item in self.env.compose('ps', '-q', service).stdout.split() if item]
|
|
for container in containers:
|
|
self.env.run(['docker', 'restart', container], timeout=120)
|
|
self.wait_container_health(container)
|
|
|
|
def restart_consumers(self, *, stores: bool = True, apps: bool = True) -> None:
|
|
if stores:
|
|
# Establish/read the sentinel before stopping the old volume owner.
|
|
self.wait_s3()
|
|
self.env.compose('restart', 'openbao', timeout=60)
|
|
self.unseal()
|
|
self.env.compose('restart', *STORES, *TELEMETRY, timeout=180)
|
|
self.wait_stores()
|
|
# A shared-network consumer is tied to a concrete owner container ID.
|
|
# Recreate consumers explicitly even if an owner was replaced.
|
|
consumers = [consumer for values in GATEWAY_PAIRS.values() for consumer in values]
|
|
self.env.compose('up', '-d', '--no-build', '--force-recreate', '--no-deps',
|
|
*consumers, timeout=180)
|
|
self.wait_s3()
|
|
if apps:
|
|
# One replica at a time: the surviving API serves while the peer
|
|
# reloads the new leaf/trust bundle. LB is restarted last.
|
|
for service in ('api', 'worker', 'relay'):
|
|
self.rolling_restart(service)
|
|
self.env.compose('restart', 'lb', timeout=60)
|
|
self.wait_ready()
|
|
|
|
def rotate_secret(self, role: str) -> dict:
|
|
if role not in ('api', 'worker'):
|
|
raise OperationError('the completed secret rotation experiment targets an S3 application identity')
|
|
values = self.credentials()
|
|
old = values['s3'][role]
|
|
self.env.private_json(self.env.state / ('retired-' + role + '-s3.json'), old)
|
|
values['s3'][role] = {'access_key': 'lab7' + role + secrets.token_hex(8),
|
|
'secret_key': secrets.token_hex(32)}
|
|
self.env.private_json(self.credentials_path, values)
|
|
self.backend_files()
|
|
client = BaoClient(self.env)
|
|
self.configure_kv(client, client.login())
|
|
self.env.compose('restart', 's3', timeout=60)
|
|
self.env.compose('restart', role, timeout=60)
|
|
self.wait_ready()
|
|
return {'role': role, 'secret_rotated': True, 'old_key_retired': True,
|
|
'images_rebuilt': False}
|
|
|
|
def rollover(self, phase: str) -> dict:
|
|
if phase == 'prepare':
|
|
result = self.pki.prepare_rollover()
|
|
elif phase == 'issue':
|
|
result = self.pki.switch_issuer()
|
|
else:
|
|
result = self.pki.retire_issuer()
|
|
client = BaoClient(self.env)
|
|
self.configure_auth(client, client.login())
|
|
self.render_grafana()
|
|
# Go reloads leaves on new handshakes: during issue the trust bundle is
|
|
# unchanged (overlap from prepare), so only native owners restart.
|
|
# Prepare distributes trust and retire removes it; both need app reload.
|
|
self.restart_consumers(apps=phase != 'issue')
|
|
return {**result, 'all_consumers_reloaded': True, 'images_rebuilt': False,
|
|
'application_reload': phase != 'issue'}
|
|
|
|
def reset_grafana_admin(self) -> None:
|
|
context = ssl.create_default_context(cafile=str(self.env.identity('operator') / 'ca.crt'))
|
|
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}),
|
|
urllib.request.HTTPSHandler(context=context))
|
|
endpoint = 'https://localhost:' + self.env.value('GRAFANA_PORT', '3000') + '/api/health'
|
|
deadline = time.monotonic() + 60
|
|
while True:
|
|
try:
|
|
with opener.open(endpoint, timeout=5) as response:
|
|
if response.status == 200:
|
|
break
|
|
except (urllib.error.URLError, TimeoutError):
|
|
if time.monotonic() >= deadline:
|
|
raise OperationError('Grafana verified TLS health deadline exceeded')
|
|
time.sleep(1)
|
|
self.env.compose('exec', '-T', 'grafana', 'grafana', 'cli',
|
|
'--homepath', '/usr/share/grafana', 'admin',
|
|
'reset-admin-password', '--password-from-stdin',
|
|
data=self.credentials()['grafana']['password'] + '\n')
|
|
|
|
def redis(self, arguments: list[str]) -> str:
|
|
# Read the operator credential inside its own backend mount, never argv.
|
|
result = self.env.compose('exec', '-T', 'redis', 'sh', '-ec',
|
|
'export REDISCLI_AUTH="$(cat /run/backend/operator-password)"; '
|
|
'exec redis-cli --tls --cacert /run/identity/current/ca.crt '
|
|
'--user operator "$@"', '--', *arguments)
|
|
return result.stdout
|
|
|
|
def jobctl(self, arguments: list[str]) -> str:
|
|
return self.env.compose('run', '--rm', '--no-deps', 'jobctl', *arguments).stdout
|
|
|
|
def reload_identity(self, role: str) -> None:
|
|
if role in ('api', 'operator', 'rogue'):
|
|
# Go reloads complete versions on new handshakes; its manager notices
|
|
# actual current leaf expiry independently of the token renewal.
|
|
return
|
|
if role == 'openbao':
|
|
self.env.compose('restart', 'openbao', timeout=60)
|
|
self.unseal()
|
|
return
|
|
if role == 's3':
|
|
self.env.compose('restart', 'seaweedfs', timeout=60)
|
|
self.env.compose('up', '-d', '--no-build', '--force-recreate', '--no-deps', 's3', timeout=90)
|
|
return
|
|
owner = 'otel-collector' if role == 'collector' else role
|
|
if role == 'grafana':
|
|
self.render_grafana()
|
|
self.env.compose('restart', owner, timeout=90)
|
|
children = GATEWAY_PAIRS.get(owner, [])
|
|
if children:
|
|
self.env.compose('up', '-d', '--no-build', '--force-recreate', '--no-deps', *children, timeout=90)
|
|
|
|
def revoke_identity(self, role: str, serial: str) -> dict:
|
|
result = self.pki.revoke(role, serial)
|
|
# Go reads current CRLs per handshake; native gateways hold startup
|
|
# CRLs and must drain old sessions before revocation returns success.
|
|
gateways = ['lb', *[child for children in GATEWAY_PAIRS.values()
|
|
for child in children if child != 's3']]
|
|
self.env.compose('restart', *gateways, timeout=180)
|
|
return {**result, 'native_gateway_crls_reloaded': True,
|
|
'native_gateway_restarts': True,
|
|
'existing_application_pools_drained': False}
|
|
|
|
def recover_identity(self, role: str) -> dict:
|
|
transport_replaced = self.pki.recover_transport()
|
|
if transport_replaced or role == 'openbao':
|
|
self.env.compose('restart', 'openbao', timeout=60)
|
|
self.unseal()
|
|
issued = self.pki.issue(role)
|
|
self.reload_identity(role)
|
|
if role in ('api', 'worker', 'relay'):
|
|
self.env.compose('up', '-d', '--no-build', '--no-deps', role, timeout=90)
|
|
return {**issued, 'recovered_by_operator': True}
|
|
|
|
def sync_crls(self) -> dict:
|
|
client = BaoClient(self.env)
|
|
token = client.login()
|
|
republished = self.pki.sync_crls(client, token)
|
|
return {'crls_updated': True, 'identities_republished': republished}
|
|
|
|
def audit_summary(self) -> dict:
|
|
result = self.env.compose('exec', '-T', 'openbao', 'cat', '/openbao/audit/audit.log')
|
|
counts = {}
|
|
for line in result.stdout.splitlines():
|
|
try:
|
|
record = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
request = record.get('request', {})
|
|
path = request.get('path', '')
|
|
operation = request.get('operation', '')
|
|
if path.startswith(('pki/issue/', 'pki/revoke', 'auth/token/renew-self', 'auth/cert/login')):
|
|
key = operation + ' ' + path
|
|
counts[key] = counts.get(key, 0) + 1
|
|
return {'audit_events': counts, 'raw_credentials_exported': False}
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
commands = parser.add_subparsers(dest='command', required=True)
|
|
for name in ('prepare', 'bootstrap', 'up', 'unseal', 'seal', 'restart', 'sync-crls', 'audit-summary', 'status'):
|
|
commands.add_parser(name)
|
|
for name in ('issue', 'renew'):
|
|
sub = commands.add_parser(name)
|
|
sub.add_argument('role', choices=ROLES)
|
|
sub.add_argument('--ttl', default='1h')
|
|
if name == 'renew':
|
|
sub.add_argument('--watch', action='store_true')
|
|
sub.add_argument('--interval', type=float, default=1)
|
|
sub = commands.add_parser('revoke')
|
|
sub.add_argument('role', choices=ROLES)
|
|
sub.add_argument('--serial', required=True)
|
|
sub = commands.add_parser('rotate-secret')
|
|
sub.add_argument('role', choices=('api', 'worker'))
|
|
sub = commands.add_parser('ca-rollover')
|
|
sub.add_argument('--phase', choices=('prepare', 'issue', 'retire'), required=True)
|
|
sub = commands.add_parser('recover')
|
|
sub.add_argument('role', choices=ROLES)
|
|
commands.add_parser('psql')
|
|
for name in ('redis', 'jobctl'):
|
|
command = commands.add_parser(name)
|
|
command.add_argument('arguments', nargs=argparse.REMAINDER)
|
|
args = parser.parse_args()
|
|
control = Controller()
|
|
if args.command == 'psql':
|
|
print(control.sql(sys.stdin.read()), end='')
|
|
return
|
|
if args.command == 'redis':
|
|
print(control.redis(args.arguments), end='')
|
|
return
|
|
if args.command == 'jobctl':
|
|
print(control.jobctl(args.arguments), end='')
|
|
return
|
|
if args.command == 'prepare':
|
|
result = control.prepare()
|
|
elif args.command == 'bootstrap':
|
|
result = control.bootstrap()
|
|
elif args.command == 'up':
|
|
result = control.up()
|
|
elif args.command == 'unseal':
|
|
result = control.unseal()
|
|
elif args.command == 'seal':
|
|
result = control.seal()
|
|
elif args.command == 'restart':
|
|
control.restart_consumers()
|
|
result = {'restarted': True, 'volumes_preserved': True}
|
|
elif args.command == 'sync-crls':
|
|
result = control.sync_crls()
|
|
elif args.command == 'issue':
|
|
result = control.pki.issue(args.role, args.ttl)
|
|
control.reload_identity(args.role)
|
|
elif args.command == 'renew':
|
|
if args.watch:
|
|
control.pki.renew_watch(args.role, args.ttl, args.interval, control.reload_identity)
|
|
return
|
|
result = control.pki.issue(args.role, args.ttl, service_auth=True)
|
|
control.reload_identity(args.role)
|
|
elif args.command == 'recover':
|
|
result = control.recover_identity(args.role)
|
|
elif args.command == 'revoke':
|
|
result = control.revoke_identity(args.role, args.serial)
|
|
elif args.command == 'rotate-secret':
|
|
result = control.rotate_secret(args.role)
|
|
elif args.command == 'ca-rollover':
|
|
result = control.rollover(args.phase)
|
|
elif args.command == 'audit-summary':
|
|
result = control.audit_summary()
|
|
else:
|
|
result = {'project': control.env.project, 'base_url': control.env.base_url,
|
|
'pki': control.pki.state}
|
|
print(json.dumps(result, ensure_ascii=False), flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except (OperationError, OSError, TimeoutError, subprocess.TimeoutExpired) as error:
|
|
print('FAIL: ' + str(error), file=sys.stderr)
|
|
raise SystemExit(1)
|