#!/usr/bin/env python3 """Reproducible operator bootstrap/renewal/rotation for the local lab7 stand. Private root/unseal state is outside this checkout. No secret values are printed. All OpenBao calls verify CA and hostname; app roles authenticate independently. """ from __future__ import annotations import argparse import base64 import hashlib import hmac import json import os from pathlib import Path import secrets import ssl import subprocess import sys import time import urllib.error import urllib.parse import urllib.request from lab7_environment import BaoClient, Environment, OperationError, ROOT, ROLES, atomic_write from lab7_pki import PKI GATEWAY_PAIRS = { 'seaweedfs': ['s3'], 'broker': ['broker-admin-tls'], 'otel-collector': ['collector-tls'], 'prometheus': ['prometheus-tls'], 'loki': ['loki-tls'], 'tempo': ['tempo-tls'], 'grafana': ['grafana-tls'], } STORES = ['postgres', 'redis', 'seaweedfs', 'broker'] TELEMETRY = ['otel-collector', 'prometheus', 'loki', 'tempo', 'grafana'] def password_hash(password: str) -> str: salt = secrets.token_bytes(4) return base64.b64encode(salt + hashlib.sha256(salt + password.encode()).digest()).decode() def scram_verifier(password: str, iterations: int = 4096) -> str: """Build a SCRAM-SHA-256 verifier so plaintext never enters SQL/logs.""" salt = secrets.token_bytes(16) salted = hashlib.pbkdf2_hmac('sha256', password.encode(), salt, iterations) client_key = hmac.new(salted, b'Client Key', hashlib.sha256).digest() stored_key = hashlib.sha256(client_key).digest() server_key = hmac.new(salted, b'Server Key', hashlib.sha256).digest() encode = lambda value: base64.b64encode(value).decode() return 'SCRAM-SHA-256$%d:%s$%s:%s' % (iterations, encode(salt), encode(stored_key), encode(server_key)) def service_policy(role: str) -> str: rules = [f'''path "pki/issue/{role}" {{ capabilities = ["create", "update"] }}''', '''path "auth/token/lookup-self" { capabilities = ["read"] } path "auth/token/renew-self" { capabilities = ["update"] } path "auth/token/revoke-self" { capabilities = ["update"] }'''] if role in ('api', 'worker', 'relay'): rules.append(f'''path "secret/data/{role}" {{ capabilities = ["read"] }}''') return '\n'.join(rules) + '\n' def operator_policy() -> str: paths = { 'secret/data/*': ['create', 'read', 'update', 'delete', 'list'], 'secret/metadata/*': ['read', 'list', 'delete'], 'pki/*': ['create', 'read', 'update', 'delete', 'list', 'sudo'], 'auth/cert/certs/*': ['create', 'read', 'update', 'delete', 'list'], 'auth/cert/crls/*': ['create', 'read', 'update', 'delete', 'list'], 'sys/policies/acl/lab7-*': ['create', 'read', 'update', 'delete', 'list'], 'sys/mounts': ['read'], 'sys/mounts/pki': ['create', 'read', 'update', 'sudo'], 'sys/mounts/secret': ['create', 'read', 'update', 'sudo'], 'sys/auth': ['read'], 'sys/auth/cert': ['create', 'read', 'update', 'sudo'], 'sys/audit': ['read', 'sudo'], 'sys/audit/file': ['read', 'sudo'], 'sys/seal': ['update', 'sudo'], 'auth/token/lookup-self': ['read'], 'auth/token/renew-self': ['update'], 'auth/token/revoke-self': ['update'], } return '\n'.join('path ' + json.dumps(path) + ' { capabilities = ' + json.dumps(capabilities) + ' }' for path, capabilities in paths.items()) + '\n' class Controller: def __init__(self) -> None: self.env = Environment() self.pki = PKI(self.env) self.credentials_path = self.env.secrets / 'backends/credentials.json' self.init_path = self.env.state / 'operator-unseal.json' def credentials(self) -> dict: values = self.env.read_json(self.credentials_path) if not values: values = { 'postgres': { 'operator': {'username': 'lab_operator', 'password': secrets.token_hex(32)}, **{role: {'username': 'messenger_' + role, 'password': secrets.token_hex(32)} for role in ('api', 'worker', 'relay')}, }, 'redis': {role: {'username': role, 'password': secrets.token_hex(32)} for role in ('cache', 'operator', 'health')}, 's3': {role: {'access_key': 'lab7' + role + secrets.token_hex(8), 'secret_key': secrets.token_hex(32)} for role in ('operator', 'api', 'worker')}, 'grafana': {'password': secrets.token_hex(32)}, 'broker': {'operator': {'password': secrets.token_hex(32)}}, } self.env.private_json(self.credentials_path, values) for family in values.values(): if isinstance(family, dict): for record in family.values(): if isinstance(record, dict): self.env.redactions.update(str(value) for key, value in record.items() if key in ('password', 'secret_key', 'access_key')) self.env.redactions.add(values['grafana']['password']) return values def write_file(self, relative: str, content: str) -> None: destination = self.env.secrets / relative destination.parent.mkdir(parents=True, exist_ok=True, mode=0o750) os.chmod(destination.parent, 0o750) atomic_write(destination, content, 0o640) def backend_files(self) -> None: credentials = self.credentials() operator = credentials['postgres']['operator'] self.write_file('bootstrap/migrate/postgres-password', operator['password'] + '\n') self.write_file('backends/postgres/operator-password', operator['password'] + '\n') for role, prefix in (('operator', 's3-admin'), ('api', 's3-app')): values = credentials['s3'][role] self.write_file('bootstrap/s3/' + prefix + '-access-key', values['access_key'] + '\n') self.write_file('bootstrap/s3/' + prefix + '-secret-key', values['secret_key'] + '\n') identity_config = json.loads((ROOT / 'infra/s3/s3.json').read_text()) bucket = self.env.value('S3_BUCKET', 'messenger-attachments') for identity in identity_config['identities']: values = credentials['s3'][identity['name']] identity['credentials'] = [{'accessKey': values['access_key'], 'secretKey': values['secret_key']}] identity['actions'] = [action.replace('messenger-attachments', bucket) for action in identity['actions']] self.write_file('backends/s3/s3.json', json.dumps(identity_config, indent=2) + '\n') cache = credentials['redis']['cache']['password'] operator_password = credentials['redis']['operator']['password'] health = credentials['redis']['health']['password'] acl = f'''user default off user cache on >{cache} ~chat_list:* ~lock:chat_list:* +get +set +del +ping +eval +evalsha +script|load user health on >{health} +ping user operator on >{operator_password} ~* &* +@all ''' self.write_file('backends/redis/users.acl', acl) self.write_file('backends/redis/health-password', health + '\n') self.write_file('backends/redis/operator-password', operator_password + '\n') self.write_file('backends/grafana/admin-password', credentials['grafana']['password'] + '\n') users = [ {'name': role, 'password_hash': password_hash(secrets.token_hex(32)), 'hashing_algorithm': 'rabbit_password_hashing_sha256', 'tags': []} for role in ('worker', 'relay') ] users.append({'name': 'operator', 'password_hash': password_hash(credentials['broker']['operator']['password']), 'hashing_algorithm': 'rabbit_password_hashing_sha256', 'tags': ['administrator']}) definitions = { 'users': users, 'vhosts': [{'name': 'messenger'}], 'permissions': [ {'user': 'relay', 'vhost': 'messenger', 'configure': '^$', 'write': r'^attachments\.jobs$', 'read': '^$'}, {'user': 'worker', 'vhost': 'messenger', 'configure': '^$', 'write': '^$', 'read': r'^attachment\.process\.v1$'}, {'user': 'operator', 'vhost': 'messenger', 'configure': '.*', 'write': '.*', 'read': '.*'}, ], 'exchanges': [ {'name': name, 'vhost': 'messenger', 'type': 'direct', 'durable': True, 'auto_delete': False, 'internal': False, 'arguments': {}} for name in ('attachments.jobs', 'attachments.dlx') ], 'queues': [ {'name': 'attachment.process.v1', 'vhost': 'messenger', 'durable': True, 'auto_delete': False, 'arguments': { 'x-queue-type': 'quorum', 'x-delivery-limit': 100, 'x-delayed-retry-type': 'all', 'x-delayed-retry-min': 5000, 'x-delayed-retry-max': 60000, 'x-dead-letter-exchange': 'attachments.dlx', 'x-dead-letter-routing-key': 'attachment.process.dead', 'x-dead-letter-strategy': 'at-least-once', 'x-overflow': 'reject-publish', 'x-max-length': 100000, 'x-consumer-timeout': 300000, }}, {'name': 'attachment.process.dead', 'vhost': 'messenger', 'durable': True, 'auto_delete': False, 'arguments': {'x-queue-type': 'quorum', 'x-max-length': 100000, 'x-overflow': 'reject-publish', 'x-message-ttl': 604800000}}, ], 'bindings': [ {'source': exchange, 'vhost': 'messenger', 'destination': queue, 'destination_type': 'queue', 'routing_key': queue, 'arguments': {}} for exchange, queue in (('attachments.jobs', 'attachment.process.v1'), ('attachments.dlx', 'attachment.process.dead')) ], } self.write_file('backends/broker/definitions.json', json.dumps(definitions, indent=2) + '\n') self.env.group_access(self.env.secrets / 'backends') self.env.group_access(self.env.secrets / 'bootstrap') def render_grafana(self) -> None: template = (ROOT / 'infra/grafana/provisioning/datasources/datasources.yaml').read_text() values = self.env.snapshot('grafana') replacements = { '__LAB7_CA_CERT__': values['ca.crt'], '__LAB7_CLIENT_CERT__': values['tls.crt'], '__LAB7_CLIENT_KEY__': values['tls.key'], } lines = [] for line in template.splitlines(): value = replacements.get(line.strip()) if value is None: lines.append(line) else: indent = line[:len(line) - len(line.lstrip())] lines.extend(indent + item for item in value.rstrip().splitlines()) self.write_file('backends/grafana/datasources/datasources.yaml', '\n'.join(lines) + '\n') self.write_file('backends/grafana/dashboards/dashboards.yaml', (ROOT / 'infra/grafana/provisioning/dashboards/dashboards.yaml').read_text()) self.env.group_access(self.env.secrets / 'backends/grafana') def prepare(self) -> dict: self.pki.prepare() self.backend_files() self.pki.recover_transport() self.render_grafana() metadata = {'project': self.env.project, 'pki_mount': 'pki', 'api_replicas': int(self.env.value('API_REPLICAS', '2'))} self.env.private_json(self.env.secrets / 'metadata.json', metadata) return {'prepared': True, 'project': self.env.project} def unseal(self) -> dict: deadline = time.monotonic() + 60 client = BaoClient(self.env) while True: try: status = client.request('GET', 'sys/seal-status') break except OperationError: if time.monotonic() >= deadline: raise time.sleep(1) if not status.get('initialized'): response = client.request('POST', 'sys/init', {'secret_shares': 3, 'secret_threshold': 2}, timeout=60) material = {'keys': response['keys_base64'], 'root_token': response['root_token']} self.env.private_json(self.init_path, material) status = client.request('GET', 'sys/seal-status') material = self.env.read_json(self.init_path) if status.get('sealed'): keys = material.get('keys', []) if len(keys) < 2: raise OperationError('operator unseal state is missing; it must never be recovered from the container') for key in keys[:2]: self.env.redactions.add(key) status = client.request('POST', 'sys/unseal', {'key': key}, timeout=30) if status.get('sealed'): raise OperationError('operator unseal did not complete') while True: health = client.request('GET', 'sys/health', allowed=(200, 429, 472, 473, 501, 503)) if health.get('initialized') and not health.get('sealed') and not health.get('standby'): break if time.monotonic() >= deadline: raise OperationError('unsealed OpenBao did not become the active node') time.sleep(0.2) return {'unsealed': True, 'project': self.env.project} def seal(self) -> dict: client = BaoClient(self.env) client.request('POST', 'sys/seal', {}, client.login()) status = client.request('GET', 'sys/seal-status') if not status.get('sealed'): raise OperationError('operator seal did not close OpenBao') return {'sealed': True, 'project': self.env.project} def configure_auth(self, client: BaoClient, token: str) -> None: methods = client.request('GET', 'sys/auth', token=token) methods = methods.get('data', methods) if 'cert/' not in methods: client.request('POST', 'sys/auth/cert', {'type': 'cert'}, token) for role in ROLES: policy = operator_policy() if role == 'operator' else service_policy(role) client.request('PUT', 'sys/policies/acl/lab7-' + role, {'policy': policy}, token) client.request('POST', 'auth/cert/certs/' + role, { 'certificate': self.pki.root_bundle(), 'allowed_common_names': [role], 'allowed_uri_sans': ['spiffe://mtusi/' + role], 'token_policies': ['lab7-' + role], 'token_ttl': '60s', 'token_max_ttl': '300s', 'token_explicit_max_ttl': '300s', 'token_no_default_policy': True, }, token) def configure_kv(self, client: BaoClient, token: str) -> None: mounts = client.request('GET', 'sys/mounts', token=token) mounts = mounts.get('data', mounts) if 'secret/' not in mounts: client.request('POST', 'sys/mounts/secret', {'type': 'kv', 'options': {'version': '2'}}, token) values = self.credentials() for role in ('api', 'worker', 'relay'): data = {'POSTGRES_PASSWORD': values['postgres'][role]['password']} if role != 'relay': data.update({'S3_ACCESS_KEY_ID': values['s3'][role]['access_key'], 'S3_SECRET_ACCESS_KEY': values['s3'][role]['secret_key']}) if role == 'api': data['REDIS_PASSWORD'] = values['redis']['cache']['password'] client.request('POST', 'secret/data/' + role, {'data': data}, token) def provision_bao(self) -> None: material = self.env.read_json(self.init_path) root_token = material.get('root_token') if root_token: self.env.redactions.add(root_token) client = BaoClient(self.env) token = root_token else: client = BaoClient(self.env) token = client.login() audits = client.request('GET', 'sys/audit', token=token) audits = audits.get('data', audits) if 'file/' not in audits: raise OperationError('mandatory declarative audit device is not active') self.configure_auth(client, token) self.configure_kv(client, token) self.pki.provision(client, token) self.pki.sync_crls(client, token) if root_token: # Verify the separately scoped operator path before destroying initial root. operator = BaoClient(self.env) operator.login() client.request('POST', 'auth/token/revoke-self', {}, root_token) denied = client.request('GET', 'auth/token/lookup-self', token=root_token, allowed=(403,)) if not denied.get('errors'): raise OperationError('root credential was not rejected after revocation') material.pop('root_token', None) material['root_token_revoked'] = True self.env.private_json(self.init_path, material) self.env.private_json(self.env.state / 'bootstrap-state.json', {'bao_provisioned': True}) def sql(self, sql: str, user: str = 'lab_operator', *, check: bool = True) -> str: result = self.env.compose('exec', '-T', 'postgres', 'psql', '-v', 'ON_ERROR_STOP=1', '-U', user, '-d', 'messenger', '-At', data=sql, check=check) return result.stdout if result.returncode == 0 else '' def reconcile_roles(self) -> None: values = self.credentials()['postgres'] existing = None for candidate in ('lab_operator', 'messenger', 'postgres'): if self.sql('SELECT current_user;\n', candidate, check=False).strip() == candidate: existing = candidate break if not existing: raise OperationError('no local operator database role is available') statements = [] for role, record in values.items(): name = record['username'] statements.append(f"DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname='{name}') " f'THEN CREATE ROLE "{name}"; END IF; END $$;') privilege = 'SUPERUSER CREATEDB CREATEROLE' if role == 'operator' else 'NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT' statements.append(f'ALTER ROLE "{name}" WITH LOGIN {privilege} ' f"PASSWORD '{scram_verifier(record['password'])}' CONNECTION LIMIT 32;") self.sql('\n'.join(statements), existing) def grants(self) -> None: sql = ''' REVOKE ALL ON DATABASE messenger FROM PUBLIC; REVOKE CREATE ON SCHEMA public FROM PUBLIC; REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; GRANT CONNECT ON DATABASE messenger TO messenger_api, messenger_worker, messenger_relay; GRANT USAGE ON SCHEMA public TO messenger_api, messenger_worker, messenger_relay; REVOKE ALL ON ALL TABLES IN SCHEMA public FROM messenger_api, messenger_worker, messenger_relay; REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM messenger_api, messenger_worker, messenger_relay; GRANT SELECT, INSERT, UPDATE ON users, chats, chat_members, messages, user_chat_versions, attachments, attachment_jobs, outbox_messages, attachment_job_events TO messenger_api; GRANT SELECT, INSERT, UPDATE, DELETE ON sessions, message_idempotency, login_limits TO messenger_api; GRANT SELECT ON attachment_variants TO messenger_api; GRANT SELECT, INSERT ON message_attachments TO messenger_api; GRANT DELETE ON attachments TO messenger_api; GRANT SELECT, INSERT, UPDATE ON attachments, attachment_jobs, attachment_variants, outbox_messages, attachment_job_events TO messenger_worker; GRANT DELETE ON attachments TO messenger_worker; GRANT SELECT, UPDATE ON attachments, attachment_jobs TO messenger_relay; GRANT SELECT, INSERT, UPDATE, DELETE ON outbox_messages TO messenger_relay; GRANT SELECT, INSERT ON attachment_job_events TO messenger_relay; GRANT USAGE, SELECT ON outbox_messages_id_seq, attachment_job_events_id_seq TO messenger_api, messenger_worker, messenger_relay; DO $$ DECLARE app_object record; object_type text; BEGIN IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname='messenger') THEN -- The initdb bootstrap principal owns pg_catalog objects that cannot be -- reassigned globally. Only the application's public objects move owners. FOR app_object IN SELECT c.relname, c.relkind FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'public' AND c.relowner = (SELECT oid FROM pg_roles WHERE rolname='messenger') AND c.relkind IN ('r', 'p', 'S', 'v', 'm') AND (c.relkind <> 'S' OR NOT EXISTS ( SELECT 1 FROM pg_depend d WHERE d.classid = 'pg_class'::regclass AND d.objid = c.oid AND d.deptype IN ('a', 'i') )) LOOP object_type := CASE app_object.relkind WHEN 'S' THEN 'SEQUENCE' WHEN 'v' THEN 'VIEW' WHEN 'm' THEN 'MATERIALIZED VIEW' ELSE 'TABLE' END; EXECUTE format('ALTER %s public.%I OWNER TO lab_operator', object_type, app_object.relname); END LOOP; ALTER DATABASE messenger OWNER TO lab_operator; ALTER SCHEMA public OWNER TO lab_operator; -- PostgreSQL 18 requires the initdb bootstrap role to remain SUPERUSER. -- NOLOGIN plus removal of its password retires the legacy identity without -- rebuilding the cluster or changing user data. ALTER ROLE messenger NOLOGIN PASSWORD NULL NOCREATEDB NOCREATEROLE NOINHERIT; END IF; END $$; ''' self.sql(sql) def retire_broker_users(self) -> None: for user in ('messenger', 'guest'): self.env.compose('exec', '-T', 'broker', 'rabbitmqctl', 'delete_user', user, check=False) def ensure_service_identities(self) -> dict: """Reissue leafs that expired while the stand was stopped. `prepare` only fills gaps and does not inspect leaf expiry. Recovery is operator-driven here; the running stand uses `make renewer`. """ replaced = self.pki.recover_transport() if replaced: self.env.compose('restart', 'openbao', timeout=60) self.unseal() renewed = [] for role in ROLES: certificate = self.env.identity(role) / 'tls.crt' if not certificate.exists(): continue valid = self.env.run(['openssl', 'x509', '-checkend', '300', '-noout', '-in', str(certificate)], check=False).returncode == 0 if not valid: self.pki.issue(role) renewed.append(role) reloaded, skipped = [], [] for role in renewed: try: self.reload_identity(role) reloaded.append(role) except OperationError: # A stopped service will load the reissued leaf when compose starts it. skipped.append(role) return {'renewed': renewed, 'reloaded': reloaded, 'reload_skipped': skipped} def bootstrap(self) -> dict: self.prepare() self.env.compose('up', '-d', '--no-build', 'volume-init', timeout=300) # volume-init is short-lived; consumers require successful completion. self.env.compose('up', '-d', '--no-build', 'openbao', timeout=300) self.unseal() self.provision_bao() # On the first successful bootstrap, transition from offline bootstrap # leaves to restricted OpenBao-issued, short-lived service identities. marker = self.env.state / 'service-leaves.json' if not marker.exists(): leaves = [self.pki.issue(role) for role in ROLES if role != 'rogue'] self.env.private_json(marker, {'issued': leaves}) self.env.compose('restart', 'openbao', timeout=60) self.unseal() recovered = self.ensure_service_identities() self.render_grafana() self.env.compose('up', '-d', '--no-build', *STORES, 's3', timeout=300) self.wait_stores() self.reconcile_roles() self.env.compose('run', '--rm', '--no-deps', 'migrate', timeout=180) self.grants() self.env.compose('run', '--rm', '--no-deps', 's3-init', timeout=180) self.retire_broker_users() return {'bootstrapped': True, 'project': self.env.project, 'expired_identities_renewed': recovered.get('renewed', []), 'expired_identities_reloaded': recovered.get('reloaded', []), 'root_token_revoked': self.env.read_json(self.init_path).get('root_token_revoked', False)} def wait_stores(self) -> None: deadline = time.monotonic() + 120 for service in ('postgres', 'redis', 'broker', 'seaweedfs'): while True: result = self.env.compose('ps', '-q', service).stdout.strip() if result: state = self.env.run(['docker', 'inspect', '--format', '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}', result]).stdout.strip() if state in ('healthy', 'running'): break if time.monotonic() >= deadline: raise OperationError(service + ' did not become ready') time.sleep(1) def wait_s3(self, timeout: float = 120) -> None: """Read old volume-backed bytes; never repair an established sentinel.""" deadline = time.monotonic() + timeout marker = self.env.state / 's3-readiness.json' sentinel = self.env.read_json(marker) if not sentinel: sentinel = {'bucket': self.env.value('S3_BUCKET', 'messenger-attachments'), 'key': 'operator-readiness/' + secrets.token_hex(16), 'body': secrets.token_hex(32768), 'established': False} self.env.private_json(marker, sentinel) payload = sentinel['body'].encode() credential = self.credentials()['s3']['operator'] host = 'localhost:' + self.env.value('S3_PORT', '8333') bucket_path = '/' + urllib.parse.quote(sentinel['bucket'], safe='') object_path = bucket_path + '/' + urllib.parse.quote(sentinel['key'], safe='/~') context = ssl.create_default_context(cafile=str(self.env.identity('operator') / 'ca.crt')) context.load_cert_chain(str(self.env.identity('operator') / 'tls.crt'), str(self.env.identity('operator') / 'tls.key')) class NoRedirect(urllib.request.HTTPRedirectHandler): def redirect_request(self, req, fp, code, msg, headers, newurl): return None opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect(), urllib.request.HTTPSHandler(context=context)) def request(method: str, path: str, body: bytes = b'') -> bytes: remaining = deadline - time.monotonic() if remaining <= 0: raise TimeoutError() stamp = time.strftime('%Y%m%dT%H%M%SZ', time.gmtime()) day = stamp[:8] digest = hashlib.sha256(body).hexdigest() headers = {'host': host, 'x-amz-content-sha256': digest, 'x-amz-date': stamp} signed = ';'.join(sorted(headers)) canonical_headers = ''.join(name + ':' + headers[name] + '\n' for name in sorted(headers)) canonical = '\n'.join((method, path, '', canonical_headers, signed, digest)) scope = day + '/us-east-1/s3/aws4_request' to_sign = 'AWS4-HMAC-SHA256\n' + stamp + '\n' + scope + '\n' + hashlib.sha256(canonical.encode()).hexdigest() signing_key = ('AWS4' + credential['secret_key']).encode() for part in (day, 'us-east-1', 's3', 'aws4_request'): signing_key = hmac.new(signing_key, part.encode(), hashlib.sha256).digest() signature = hmac.new(signing_key, to_sign.encode(), hashlib.sha256).hexdigest() headers['Authorization'] = ('AWS4-HMAC-SHA256 Credential=' + credential['access_key'] + '/' + scope + ', SignedHeaders=' + signed + ', Signature=' + signature) req = urllib.request.Request('https://' + host + path, method=method, headers=headers, data=body if method == 'PUT' else None) with opener.open(req, timeout=min(5, remaining)) as response: chunks = bytearray() while len(chunks) <= len(payload): if time.monotonic() >= deadline: raise TimeoutError() chunk = response.read1(min(8192, len(payload) + 1 - len(chunks))) if not chunk: break chunks.extend(chunk) return bytes(chunks) while time.monotonic() < deadline: try: if not sentinel['established']: # Only this operator-owned key is written on first initialization. request('PUT', object_path, payload) if request('GET', object_path) != payload: raise OperationError('persistent S3 readiness sentinel bytes changed') if not sentinel['established']: sentinel['established'] = True self.env.private_json(marker, sentinel) return except urllib.error.HTTPError as exc: if exc.code not in (404, 408, 429, 500, 502, 503, 504): raise OperationError('S3 readiness request rejected (HTTP ' + str(exc.code) + ')') from None except (urllib.error.URLError, TimeoutError, ConnectionError): pass time.sleep(min(1, max(0, deadline - time.monotonic()))) raise OperationError('CA-verified persistent S3 body readiness deadline exceeded') def wait_ready(self, timeout: float = 120) -> None: deadline = time.monotonic() + timeout self.wait_s3(timeout=timeout) context = ssl.create_default_context(cafile=str(self.env.identity('operator') / 'ca.crt')) opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), urllib.request.HTTPSHandler(context=context)) while time.monotonic() < deadline: try: with opener.open(self.env.base_url + '/health/ready', timeout=5) as response: if response.status == 200: return except (urllib.error.URLError, TimeoutError): pass time.sleep(1) raise OperationError('CA-verified public HTTPS readiness deadline exceeded') def up(self) -> dict: result = self.bootstrap() self.env.compose('up', '-d', '--no-build', timeout=300) # Apply current credentials/trust/CRLs even when owners were reused. # Namespace consumers are recreated against the current owner ID. gateways = [child for children in GATEWAY_PAIRS.values() for child in children if child != 's3'] self.env.compose('up', '-d', '--no-build', '--no-deps', '--force-recreate', *gateways, timeout=180) self.env.compose('restart', 'lb', timeout=60) self.wait_ready() self.reset_grafana_admin() return {**result, 'ready': True, 'base_url': self.env.base_url} def wait_container_health(self, container: str, timeout: float = 120) -> None: deadline = time.monotonic() + timeout while True: state = self.env.run(['docker', 'inspect', '--format', '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}', container], check=False).stdout.strip() if state in ('healthy', 'running'): return if time.monotonic() >= deadline: raise OperationError(container + ' did not become healthy after restart: ' + state) time.sleep(1) def rolling_restart(self, service: str) -> None: containers = [item for item in self.env.compose('ps', '-q', service).stdout.split() if item] for container in containers: self.env.run(['docker', 'restart', container], timeout=120) self.wait_container_health(container) def restart_consumers(self, *, stores: bool = True, apps: bool = True) -> None: if stores: # Establish/read the sentinel before stopping the old volume owner. self.wait_s3() self.env.compose('restart', 'openbao', timeout=60) self.unseal() self.env.compose('restart', *STORES, *TELEMETRY, timeout=180) self.wait_stores() # A shared-network consumer is tied to a concrete owner container ID. # Recreate consumers explicitly even if an owner was replaced. consumers = [consumer for values in GATEWAY_PAIRS.values() for consumer in values] self.env.compose('up', '-d', '--no-build', '--force-recreate', '--no-deps', *consumers, timeout=180) self.wait_s3() if apps: # One replica at a time: the surviving API serves while the peer # reloads the new leaf/trust bundle. LB is restarted last. for service in ('api', 'worker', 'relay'): self.rolling_restart(service) self.env.compose('restart', 'lb', timeout=60) self.wait_ready() def rotate_secret(self, role: str) -> dict: if role not in ('api', 'worker'): raise OperationError('the completed secret rotation experiment targets an S3 application identity') values = self.credentials() old = values['s3'][role] self.env.private_json(self.env.state / ('retired-' + role + '-s3.json'), old) values['s3'][role] = {'access_key': 'lab7' + role + secrets.token_hex(8), 'secret_key': secrets.token_hex(32)} self.env.private_json(self.credentials_path, values) self.backend_files() client = BaoClient(self.env) self.configure_kv(client, client.login()) self.env.compose('restart', 's3', timeout=60) self.env.compose('restart', role, timeout=60) self.wait_ready() return {'role': role, 'secret_rotated': True, 'old_key_retired': True, 'images_rebuilt': False} def rollover(self, phase: str) -> dict: if phase == 'prepare': result = self.pki.prepare_rollover() elif phase == 'issue': result = self.pki.switch_issuer() else: result = self.pki.retire_issuer() client = BaoClient(self.env) self.configure_auth(client, client.login()) self.render_grafana() # Go reloads leaves on new handshakes: during issue the trust bundle is # unchanged (overlap from prepare), so only native owners restart. # Prepare distributes trust and retire removes it; both need app reload. self.restart_consumers(apps=phase != 'issue') return {**result, 'all_consumers_reloaded': True, 'images_rebuilt': False, 'application_reload': phase != 'issue'} def reset_grafana_admin(self) -> None: context = ssl.create_default_context(cafile=str(self.env.identity('operator') / 'ca.crt')) opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), urllib.request.HTTPSHandler(context=context)) endpoint = 'https://localhost:' + self.env.value('GRAFANA_PORT', '3000') + '/api/health' deadline = time.monotonic() + 60 while True: try: with opener.open(endpoint, timeout=5) as response: if response.status == 200: break except (urllib.error.URLError, TimeoutError): if time.monotonic() >= deadline: raise OperationError('Grafana verified TLS health deadline exceeded') time.sleep(1) self.env.compose('exec', '-T', 'grafana', 'grafana', 'cli', '--homepath', '/usr/share/grafana', 'admin', 'reset-admin-password', '--password-from-stdin', data=self.credentials()['grafana']['password'] + '\n') def redis(self, arguments: list[str]) -> str: # Read the operator credential inside its own backend mount, never argv. result = self.env.compose('exec', '-T', 'redis', 'sh', '-ec', 'export REDISCLI_AUTH="$(cat /run/backend/operator-password)"; ' 'exec redis-cli --tls --cacert /run/identity/current/ca.crt ' '--user operator "$@"', '--', *arguments) return result.stdout def jobctl(self, arguments: list[str]) -> str: return self.env.compose('run', '--rm', '--no-deps', 'jobctl', *arguments).stdout def reload_identity(self, role: str) -> None: if role in ('api', 'operator', 'rogue'): # Go reloads complete versions on new handshakes; its manager notices # actual current leaf expiry independently of the token renewal. return if role == 'openbao': self.env.compose('restart', 'openbao', timeout=60) self.unseal() return if role == 's3': self.env.compose('restart', 'seaweedfs', timeout=60) self.env.compose('up', '-d', '--no-build', '--force-recreate', '--no-deps', 's3', timeout=90) return owner = 'otel-collector' if role == 'collector' else role if role == 'grafana': self.render_grafana() self.env.compose('restart', owner, timeout=90) children = GATEWAY_PAIRS.get(owner, []) if children: self.env.compose('up', '-d', '--no-build', '--force-recreate', '--no-deps', *children, timeout=90) def revoke_identity(self, role: str, serial: str) -> dict: result = self.pki.revoke(role, serial) # Go reads current CRLs per handshake; native gateways hold startup # CRLs and must drain old sessions before revocation returns success. gateways = ['lb', *[child for children in GATEWAY_PAIRS.values() for child in children if child != 's3']] self.env.compose('restart', *gateways, timeout=180) return {**result, 'native_gateway_crls_reloaded': True, 'native_gateway_restarts': True, 'existing_application_pools_drained': False} def recover_identity(self, role: str) -> dict: transport_replaced = self.pki.recover_transport() if transport_replaced or role == 'openbao': self.env.compose('restart', 'openbao', timeout=60) self.unseal() issued = self.pki.issue(role) self.reload_identity(role) if role in ('api', 'worker', 'relay'): self.env.compose('up', '-d', '--no-build', '--no-deps', role, timeout=90) return {**issued, 'recovered_by_operator': True} def sync_crls(self) -> dict: client = BaoClient(self.env) token = client.login() republished = self.pki.sync_crls(client, token) return {'crls_updated': True, 'identities_republished': republished} def audit_summary(self) -> dict: result = self.env.compose('exec', '-T', 'openbao', 'cat', '/openbao/audit/audit.log') counts = {} for line in result.stdout.splitlines(): try: record = json.loads(line) except json.JSONDecodeError: continue request = record.get('request', {}) path = request.get('path', '') operation = request.get('operation', '') if path.startswith(('pki/issue/', 'pki/revoke', 'auth/token/renew-self', 'auth/cert/login')): key = operation + ' ' + path counts[key] = counts.get(key, 0) + 1 return {'audit_events': counts, 'raw_credentials_exported': False} def main() -> None: parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest='command', required=True) for name in ('prepare', 'bootstrap', 'up', 'unseal', 'seal', 'restart', 'sync-crls', 'audit-summary', 'status'): commands.add_parser(name) for name in ('issue', 'renew'): sub = commands.add_parser(name) sub.add_argument('role', choices=ROLES) sub.add_argument('--ttl', default='1h') if name == 'renew': sub.add_argument('--watch', action='store_true') sub.add_argument('--interval', type=float, default=1) sub = commands.add_parser('revoke') sub.add_argument('role', choices=ROLES) sub.add_argument('--serial', required=True) sub = commands.add_parser('rotate-secret') sub.add_argument('role', choices=('api', 'worker')) sub = commands.add_parser('ca-rollover') sub.add_argument('--phase', choices=('prepare', 'issue', 'retire'), required=True) sub = commands.add_parser('recover') sub.add_argument('role', choices=ROLES) commands.add_parser('psql') for name in ('redis', 'jobctl'): command = commands.add_parser(name) command.add_argument('arguments', nargs=argparse.REMAINDER) args = parser.parse_args() control = Controller() if args.command == 'psql': print(control.sql(sys.stdin.read()), end='') return if args.command == 'redis': print(control.redis(args.arguments), end='') return if args.command == 'jobctl': print(control.jobctl(args.arguments), end='') return if args.command == 'prepare': result = control.prepare() elif args.command == 'bootstrap': result = control.bootstrap() elif args.command == 'up': result = control.up() elif args.command == 'unseal': result = control.unseal() elif args.command == 'seal': result = control.seal() elif args.command == 'restart': control.restart_consumers() result = {'restarted': True, 'volumes_preserved': True} elif args.command == 'sync-crls': result = control.sync_crls() elif args.command == 'issue': result = control.pki.issue(args.role, args.ttl) control.reload_identity(args.role) elif args.command == 'renew': if args.watch: control.pki.renew_watch(args.role, args.ttl, args.interval, control.reload_identity) return result = control.pki.issue(args.role, args.ttl, service_auth=True) control.reload_identity(args.role) elif args.command == 'recover': result = control.recover_identity(args.role) elif args.command == 'revoke': result = control.revoke_identity(args.role, args.serial) elif args.command == 'rotate-secret': result = control.rotate_secret(args.role) elif args.command == 'ca-rollover': result = control.rollover(args.phase) elif args.command == 'audit-summary': result = control.audit_summary() else: result = {'project': control.env.project, 'base_url': control.env.base_url, 'pki': control.pki.state} print(json.dumps(result, ensure_ascii=False), flush=True) if __name__ == '__main__': try: main() except (OperationError, OSError, TimeoutError, subprocess.TimeoutExpired) as error: print('FAIL: ' + str(error), file=sys.stderr) raise SystemExit(1)