Files
mtusi2026lab7/scripts/securityctl.py
T

860 lines
43 KiB
Python
Executable File

#!/usr/bin/env python3
"""Reproducible operator bootstrap/renewal/rotation for the local lab7 stand.
Private root/unseal state is outside this checkout. No secret values are printed.
All OpenBao calls verify CA and hostname; app roles authenticate independently.
"""
from __future__ import annotations
import argparse
import base64
import hashlib
import hmac
import json
import os
from pathlib import Path
import secrets
import ssl
import subprocess
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from lab7_environment import BaoClient, Environment, OperationError, ROOT, ROLES, atomic_write
from lab7_pki import PKI
GATEWAY_PAIRS = {
'seaweedfs': ['s3'],
'broker': ['broker-admin-tls'],
'otel-collector': ['collector-tls'],
'prometheus': ['prometheus-tls'],
'loki': ['loki-tls'],
'tempo': ['tempo-tls'],
'grafana': ['grafana-tls'],
}
STORES = ['postgres', 'redis', 'seaweedfs', 'broker']
TELEMETRY = ['otel-collector', 'prometheus', 'loki', 'tempo', 'grafana']
def password_hash(password: str) -> str:
salt = secrets.token_bytes(4)
return base64.b64encode(salt + hashlib.sha256(salt + password.encode()).digest()).decode()
def scram_verifier(password: str, iterations: int = 4096) -> str:
"""Build a SCRAM-SHA-256 verifier so plaintext never enters SQL/logs."""
salt = secrets.token_bytes(16)
salted = hashlib.pbkdf2_hmac('sha256', password.encode(), salt, iterations)
client_key = hmac.new(salted, b'Client Key', hashlib.sha256).digest()
stored_key = hashlib.sha256(client_key).digest()
server_key = hmac.new(salted, b'Server Key', hashlib.sha256).digest()
encode = lambda value: base64.b64encode(value).decode()
return 'SCRAM-SHA-256$%d:%s$%s:%s' % (iterations, encode(salt), encode(stored_key), encode(server_key))
def service_policy(role: str) -> str:
rules = [f'''path "pki/issue/{role}" {{ capabilities = ["create", "update"] }}''',
'''path "auth/token/lookup-self" { capabilities = ["read"] }
path "auth/token/renew-self" { capabilities = ["update"] }
path "auth/token/revoke-self" { capabilities = ["update"] }''']
if role in ('api', 'worker', 'relay'):
rules.append(f'''path "secret/data/{role}" {{ capabilities = ["read"] }}''')
return '\n'.join(rules) + '\n'
def operator_policy() -> str:
paths = {
'secret/data/*': ['create', 'read', 'update', 'delete', 'list'],
'secret/metadata/*': ['read', 'list', 'delete'],
'pki/*': ['create', 'read', 'update', 'delete', 'list', 'sudo'],
'auth/cert/certs/*': ['create', 'read', 'update', 'delete', 'list'],
'auth/cert/crls/*': ['create', 'read', 'update', 'delete', 'list'],
'sys/policies/acl/lab7-*': ['create', 'read', 'update', 'delete', 'list'],
'sys/mounts': ['read'], 'sys/mounts/pki': ['create', 'read', 'update', 'sudo'],
'sys/mounts/secret': ['create', 'read', 'update', 'sudo'],
'sys/auth': ['read'], 'sys/auth/cert': ['create', 'read', 'update', 'sudo'],
'sys/audit': ['read', 'sudo'], 'sys/audit/file': ['read', 'sudo'],
'sys/seal': ['update', 'sudo'], 'auth/token/lookup-self': ['read'],
'auth/token/renew-self': ['update'], 'auth/token/revoke-self': ['update'],
}
return '\n'.join('path ' + json.dumps(path) + ' { capabilities = ' +
json.dumps(capabilities) + ' }' for path, capabilities in paths.items()) + '\n'
class Controller:
def __init__(self) -> None:
self.env = Environment()
self.pki = PKI(self.env)
self.credentials_path = self.env.secrets / 'backends/credentials.json'
self.init_path = self.env.state / 'operator-unseal.json'
def credentials(self) -> dict:
values = self.env.read_json(self.credentials_path)
if not values:
values = {
'postgres': {
'operator': {'username': 'lab_operator', 'password': secrets.token_hex(32)},
**{role: {'username': 'messenger_' + role, 'password': secrets.token_hex(32)}
for role in ('api', 'worker', 'relay')},
},
'redis': {role: {'username': role, 'password': secrets.token_hex(32)}
for role in ('cache', 'operator', 'health')},
's3': {role: {'access_key': 'lab7' + role + secrets.token_hex(8),
'secret_key': secrets.token_hex(32)}
for role in ('operator', 'api', 'worker')},
'grafana': {'password': secrets.token_hex(32)},
'broker': {'operator': {'password': secrets.token_hex(32)}},
}
self.env.private_json(self.credentials_path, values)
for family in values.values():
if isinstance(family, dict):
for record in family.values():
if isinstance(record, dict):
self.env.redactions.update(str(value) for key, value in record.items()
if key in ('password', 'secret_key', 'access_key'))
self.env.redactions.add(values['grafana']['password'])
return values
def write_file(self, relative: str, content: str) -> None:
destination = self.env.secrets / relative
destination.parent.mkdir(parents=True, exist_ok=True, mode=0o750)
os.chmod(destination.parent, 0o750)
atomic_write(destination, content, 0o640)
def backend_files(self) -> None:
credentials = self.credentials()
operator = credentials['postgres']['operator']
self.write_file('bootstrap/migrate/postgres-password', operator['password'] + '\n')
self.write_file('backends/postgres/operator-password', operator['password'] + '\n')
for role, prefix in (('operator', 's3-admin'), ('api', 's3-app')):
values = credentials['s3'][role]
self.write_file('bootstrap/s3/' + prefix + '-access-key', values['access_key'] + '\n')
self.write_file('bootstrap/s3/' + prefix + '-secret-key', values['secret_key'] + '\n')
identity_config = json.loads((ROOT / 'infra/s3/s3.json').read_text())
bucket = self.env.value('S3_BUCKET', 'messenger-attachments')
for identity in identity_config['identities']:
values = credentials['s3'][identity['name']]
identity['credentials'] = [{'accessKey': values['access_key'], 'secretKey': values['secret_key']}]
identity['actions'] = [action.replace('messenger-attachments', bucket)
for action in identity['actions']]
self.write_file('backends/s3/s3.json', json.dumps(identity_config, indent=2) + '\n')
cache = credentials['redis']['cache']['password']
operator_password = credentials['redis']['operator']['password']
health = credentials['redis']['health']['password']
acl = f'''user default off
user cache on >{cache} ~chat_list:* ~lock:chat_list:* +get +set +del +ping +eval +evalsha +script|load
user health on >{health} +ping
user operator on >{operator_password} ~* &* +@all
'''
self.write_file('backends/redis/users.acl', acl)
self.write_file('backends/redis/health-password', health + '\n')
self.write_file('backends/redis/operator-password', operator_password + '\n')
self.write_file('backends/grafana/admin-password', credentials['grafana']['password'] + '\n')
users = [
{'name': role, 'password_hash': password_hash(secrets.token_hex(32)),
'hashing_algorithm': 'rabbit_password_hashing_sha256', 'tags': []}
for role in ('worker', 'relay')
]
users.append({'name': 'operator',
'password_hash': password_hash(credentials['broker']['operator']['password']),
'hashing_algorithm': 'rabbit_password_hashing_sha256', 'tags': ['administrator']})
definitions = {
'users': users, 'vhosts': [{'name': 'messenger'}],
'permissions': [
{'user': 'relay', 'vhost': 'messenger', 'configure': '^$',
'write': r'^attachments\.jobs$', 'read': '^$'},
{'user': 'worker', 'vhost': 'messenger', 'configure': '^$',
'write': '^$', 'read': r'^attachment\.process\.v1$'},
{'user': 'operator', 'vhost': 'messenger', 'configure': '.*', 'write': '.*', 'read': '.*'},
],
'exchanges': [
{'name': name, 'vhost': 'messenger', 'type': 'direct', 'durable': True,
'auto_delete': False, 'internal': False, 'arguments': {}}
for name in ('attachments.jobs', 'attachments.dlx')
],
'queues': [
{'name': 'attachment.process.v1', 'vhost': 'messenger', 'durable': True,
'auto_delete': False, 'arguments': {
'x-queue-type': 'quorum', 'x-delivery-limit': 100,
'x-delayed-retry-type': 'all', 'x-delayed-retry-min': 5000,
'x-delayed-retry-max': 60000,
'x-dead-letter-exchange': 'attachments.dlx',
'x-dead-letter-routing-key': 'attachment.process.dead',
'x-dead-letter-strategy': 'at-least-once',
'x-overflow': 'reject-publish', 'x-max-length': 100000,
'x-consumer-timeout': 300000,
}},
{'name': 'attachment.process.dead', 'vhost': 'messenger', 'durable': True,
'auto_delete': False, 'arguments': {'x-queue-type': 'quorum', 'x-max-length': 100000,
'x-overflow': 'reject-publish', 'x-message-ttl': 604800000}},
],
'bindings': [
{'source': exchange, 'vhost': 'messenger', 'destination': queue,
'destination_type': 'queue', 'routing_key': queue, 'arguments': {}}
for exchange, queue in (('attachments.jobs', 'attachment.process.v1'),
('attachments.dlx', 'attachment.process.dead'))
],
}
self.write_file('backends/broker/definitions.json', json.dumps(definitions, indent=2) + '\n')
self.env.group_access(self.env.secrets / 'backends')
self.env.group_access(self.env.secrets / 'bootstrap')
def render_grafana(self) -> None:
template = (ROOT / 'infra/grafana/provisioning/datasources/datasources.yaml').read_text()
values = self.env.snapshot('grafana')
replacements = {
'__LAB7_CA_CERT__': values['ca.crt'],
'__LAB7_CLIENT_CERT__': values['tls.crt'],
'__LAB7_CLIENT_KEY__': values['tls.key'],
}
lines = []
for line in template.splitlines():
value = replacements.get(line.strip())
if value is None:
lines.append(line)
else:
indent = line[:len(line) - len(line.lstrip())]
lines.extend(indent + item for item in value.rstrip().splitlines())
self.write_file('backends/grafana/datasources/datasources.yaml', '\n'.join(lines) + '\n')
self.write_file('backends/grafana/dashboards/dashboards.yaml',
(ROOT / 'infra/grafana/provisioning/dashboards/dashboards.yaml').read_text())
self.env.group_access(self.env.secrets / 'backends/grafana')
def prepare(self) -> dict:
self.pki.prepare()
self.backend_files()
self.pki.recover_transport()
self.render_grafana()
metadata = {'project': self.env.project, 'pki_mount': 'pki',
'api_replicas': int(self.env.value('API_REPLICAS', '2'))}
self.env.private_json(self.env.secrets / 'metadata.json', metadata)
return {'prepared': True, 'project': self.env.project}
def unseal(self) -> dict:
deadline = time.monotonic() + 60
client = BaoClient(self.env)
while True:
try:
status = client.request('GET', 'sys/seal-status')
break
except OperationError:
if time.monotonic() >= deadline:
raise
time.sleep(1)
if not status.get('initialized'):
response = client.request('POST', 'sys/init', {'secret_shares': 3, 'secret_threshold': 2}, timeout=60)
material = {'keys': response['keys_base64'], 'root_token': response['root_token']}
self.env.private_json(self.init_path, material)
status = client.request('GET', 'sys/seal-status')
material = self.env.read_json(self.init_path)
if status.get('sealed'):
keys = material.get('keys', [])
if len(keys) < 2:
raise OperationError('operator unseal state is missing; it must never be recovered from the container')
for key in keys[:2]:
self.env.redactions.add(key)
status = client.request('POST', 'sys/unseal', {'key': key}, timeout=30)
if status.get('sealed'):
raise OperationError('operator unseal did not complete')
while True:
health = client.request('GET', 'sys/health', allowed=(200, 429, 472, 473, 501, 503))
if health.get('initialized') and not health.get('sealed') and not health.get('standby'):
break
if time.monotonic() >= deadline:
raise OperationError('unsealed OpenBao did not become the active node')
time.sleep(0.2)
return {'unsealed': True, 'project': self.env.project}
def seal(self) -> dict:
client = BaoClient(self.env)
client.request('POST', 'sys/seal', {}, client.login())
status = client.request('GET', 'sys/seal-status')
if not status.get('sealed'):
raise OperationError('operator seal did not close OpenBao')
return {'sealed': True, 'project': self.env.project}
def configure_auth(self, client: BaoClient, token: str) -> None:
methods = client.request('GET', 'sys/auth', token=token)
methods = methods.get('data', methods)
if 'cert/' not in methods:
client.request('POST', 'sys/auth/cert', {'type': 'cert'}, token)
for role in ROLES:
policy = operator_policy() if role == 'operator' else service_policy(role)
client.request('PUT', 'sys/policies/acl/lab7-' + role, {'policy': policy}, token)
client.request('POST', 'auth/cert/certs/' + role, {
'certificate': self.pki.root_bundle(),
'allowed_common_names': [role], 'allowed_uri_sans': ['spiffe://mtusi/' + role],
'token_policies': ['lab7-' + role], 'token_ttl': '60s',
'token_max_ttl': '300s', 'token_explicit_max_ttl': '300s',
'token_no_default_policy': True,
}, token)
def configure_kv(self, client: BaoClient, token: str) -> None:
mounts = client.request('GET', 'sys/mounts', token=token)
mounts = mounts.get('data', mounts)
if 'secret/' not in mounts:
client.request('POST', 'sys/mounts/secret', {'type': 'kv', 'options': {'version': '2'}}, token)
values = self.credentials()
for role in ('api', 'worker', 'relay'):
data = {'POSTGRES_PASSWORD': values['postgres'][role]['password']}
if role != 'relay':
data.update({'S3_ACCESS_KEY_ID': values['s3'][role]['access_key'],
'S3_SECRET_ACCESS_KEY': values['s3'][role]['secret_key']})
if role == 'api':
data['REDIS_PASSWORD'] = values['redis']['cache']['password']
client.request('POST', 'secret/data/' + role, {'data': data}, token)
def provision_bao(self) -> None:
material = self.env.read_json(self.init_path)
root_token = material.get('root_token')
if root_token:
self.env.redactions.add(root_token)
client = BaoClient(self.env)
token = root_token
else:
client = BaoClient(self.env)
token = client.login()
audits = client.request('GET', 'sys/audit', token=token)
audits = audits.get('data', audits)
if 'file/' not in audits:
raise OperationError('mandatory declarative audit device is not active')
self.configure_auth(client, token)
self.configure_kv(client, token)
self.pki.provision(client, token)
self.pki.sync_crls(client, token)
if root_token:
# Verify the separately scoped operator path before destroying initial root.
operator = BaoClient(self.env)
operator.login()
client.request('POST', 'auth/token/revoke-self', {}, root_token)
denied = client.request('GET', 'auth/token/lookup-self', token=root_token,
allowed=(403,))
if not denied.get('errors'):
raise OperationError('root credential was not rejected after revocation')
material.pop('root_token', None)
material['root_token_revoked'] = True
self.env.private_json(self.init_path, material)
self.env.private_json(self.env.state / 'bootstrap-state.json', {'bao_provisioned': True})
def sql(self, sql: str, user: str = 'lab_operator', *, check: bool = True) -> str:
result = self.env.compose('exec', '-T', 'postgres', 'psql', '-v', 'ON_ERROR_STOP=1',
'-U', user, '-d', 'messenger', '-At', data=sql,
check=check)
return result.stdout if result.returncode == 0 else ''
def reconcile_roles(self) -> None:
values = self.credentials()['postgres']
existing = None
for candidate in ('lab_operator', 'messenger', 'postgres'):
if self.sql('SELECT current_user;\n', candidate, check=False).strip() == candidate:
existing = candidate
break
if not existing:
raise OperationError('no local operator database role is available')
statements = []
for role, record in values.items():
name = record['username']
statements.append(f"DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname='{name}') "
f'THEN CREATE ROLE "{name}"; END IF; END $$;')
privilege = 'SUPERUSER CREATEDB CREATEROLE' if role == 'operator' else 'NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT'
statements.append(f'ALTER ROLE "{name}" WITH LOGIN {privilege} '
f"PASSWORD '{scram_verifier(record['password'])}' CONNECTION LIMIT 32;")
self.sql('\n'.join(statements), existing)
def grants(self) -> None:
sql = '''
REVOKE ALL ON DATABASE messenger FROM PUBLIC;
REVOKE CREATE ON SCHEMA public FROM PUBLIC;
REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC;
REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC;
GRANT CONNECT ON DATABASE messenger TO messenger_api, messenger_worker, messenger_relay;
GRANT USAGE ON SCHEMA public TO messenger_api, messenger_worker, messenger_relay;
REVOKE ALL ON ALL TABLES IN SCHEMA public FROM messenger_api, messenger_worker, messenger_relay;
REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM messenger_api, messenger_worker, messenger_relay;
GRANT SELECT, INSERT, UPDATE ON users, chats, chat_members, messages, user_chat_versions,
attachments, attachment_jobs, outbox_messages, attachment_job_events TO messenger_api;
GRANT SELECT, INSERT, UPDATE, DELETE ON sessions, message_idempotency, login_limits TO messenger_api;
GRANT SELECT ON attachment_variants TO messenger_api;
GRANT SELECT, INSERT ON message_attachments TO messenger_api;
GRANT DELETE ON attachments TO messenger_api;
GRANT SELECT, INSERT, UPDATE ON attachments, attachment_jobs, attachment_variants,
outbox_messages, attachment_job_events TO messenger_worker;
GRANT DELETE ON attachments TO messenger_worker;
GRANT SELECT, UPDATE ON attachments, attachment_jobs TO messenger_relay;
GRANT SELECT, INSERT, UPDATE, DELETE ON outbox_messages TO messenger_relay;
GRANT SELECT, INSERT ON attachment_job_events TO messenger_relay;
GRANT USAGE, SELECT ON outbox_messages_id_seq, attachment_job_events_id_seq
TO messenger_api, messenger_worker, messenger_relay;
DO $$ DECLARE app_object record; object_type text; BEGIN
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname='messenger') THEN
-- The initdb bootstrap principal owns pg_catalog objects that cannot be
-- reassigned globally. Only the application's public objects move owners.
FOR app_object IN
SELECT c.relname, c.relkind FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = 'public'
AND c.relowner = (SELECT oid FROM pg_roles WHERE rolname='messenger')
AND c.relkind IN ('r', 'p', 'S', 'v', 'm')
AND (c.relkind <> 'S' OR NOT EXISTS (
SELECT 1 FROM pg_depend d WHERE d.classid = 'pg_class'::regclass
AND d.objid = c.oid AND d.deptype IN ('a', 'i')
))
LOOP
object_type := CASE app_object.relkind
WHEN 'S' THEN 'SEQUENCE' WHEN 'v' THEN 'VIEW'
WHEN 'm' THEN 'MATERIALIZED VIEW' ELSE 'TABLE' END;
EXECUTE format('ALTER %s public.%I OWNER TO lab_operator', object_type, app_object.relname);
END LOOP;
ALTER DATABASE messenger OWNER TO lab_operator;
ALTER SCHEMA public OWNER TO lab_operator;
-- PostgreSQL 18 requires the initdb bootstrap role to remain SUPERUSER.
-- NOLOGIN plus removal of its password retires the legacy identity without
-- rebuilding the cluster or changing user data.
ALTER ROLE messenger NOLOGIN PASSWORD NULL NOCREATEDB NOCREATEROLE NOINHERIT;
END IF;
END $$;
'''
self.sql(sql)
def retire_broker_users(self) -> None:
for user in ('messenger', 'guest'):
self.env.compose('exec', '-T', 'broker', 'rabbitmqctl', 'delete_user', user, check=False)
def ensure_service_identities(self) -> dict:
"""Reissue leafs that expired while the stand was stopped.
`prepare` only fills gaps and does not inspect leaf expiry. Recovery is
operator-driven here; the running stand uses `make renewer`.
"""
replaced = self.pki.recover_transport()
if replaced:
self.env.compose('restart', 'openbao', timeout=60)
self.unseal()
renewed = []
for role in ROLES:
certificate = self.env.identity(role) / 'tls.crt'
if not certificate.exists():
continue
valid = self.env.run(['openssl', 'x509', '-checkend', '300', '-noout',
'-in', str(certificate)], check=False).returncode == 0
if not valid:
self.pki.issue(role)
renewed.append(role)
reloaded, skipped = [], []
for role in renewed:
try:
self.reload_identity(role)
reloaded.append(role)
except OperationError:
# A stopped service will load the reissued leaf when compose starts it.
skipped.append(role)
return {'renewed': renewed, 'reloaded': reloaded, 'reload_skipped': skipped}
def bootstrap(self) -> dict:
self.prepare()
self.env.compose('up', '-d', '--no-build', 'volume-init', timeout=300)
# volume-init is short-lived; consumers require successful completion.
self.env.compose('up', '-d', '--no-build', 'openbao', timeout=300)
self.unseal()
self.provision_bao()
# On the first successful bootstrap, transition from offline bootstrap
# leaves to restricted OpenBao-issued, short-lived service identities.
marker = self.env.state / 'service-leaves.json'
if not marker.exists():
leaves = [self.pki.issue(role) for role in ROLES if role != 'rogue']
self.env.private_json(marker, {'issued': leaves})
self.env.compose('restart', 'openbao', timeout=60)
self.unseal()
recovered = self.ensure_service_identities()
self.render_grafana()
self.env.compose('up', '-d', '--no-build', *STORES, 's3', timeout=300)
self.wait_stores()
self.reconcile_roles()
self.env.compose('run', '--rm', '--no-deps', 'migrate', timeout=180)
self.grants()
self.env.compose('run', '--rm', '--no-deps', 's3-init', timeout=180)
self.retire_broker_users()
return {'bootstrapped': True, 'project': self.env.project,
'expired_identities_renewed': recovered.get('renewed', []),
'expired_identities_reloaded': recovered.get('reloaded', []),
'root_token_revoked': self.env.read_json(self.init_path).get('root_token_revoked', False)}
def wait_stores(self) -> None:
deadline = time.monotonic() + 120
for service in ('postgres', 'redis', 'broker', 'seaweedfs'):
while True:
result = self.env.compose('ps', '-q', service).stdout.strip()
if result:
state = self.env.run(['docker', 'inspect', '--format',
'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}',
result]).stdout.strip()
if state in ('healthy', 'running'):
break
if time.monotonic() >= deadline:
raise OperationError(service + ' did not become ready')
time.sleep(1)
def wait_s3(self, timeout: float = 120) -> None:
"""Read old volume-backed bytes; never repair an established sentinel."""
deadline = time.monotonic() + timeout
marker = self.env.state / 's3-readiness.json'
sentinel = self.env.read_json(marker)
if not sentinel:
sentinel = {'bucket': self.env.value('S3_BUCKET', 'messenger-attachments'),
'key': 'operator-readiness/' + secrets.token_hex(16),
'body': secrets.token_hex(32768), 'established': False}
self.env.private_json(marker, sentinel)
payload = sentinel['body'].encode()
credential = self.credentials()['s3']['operator']
host = 'localhost:' + self.env.value('S3_PORT', '8333')
bucket_path = '/' + urllib.parse.quote(sentinel['bucket'], safe='')
object_path = bucket_path + '/' + urllib.parse.quote(sentinel['key'], safe='/~')
context = ssl.create_default_context(cafile=str(self.env.identity('operator') / 'ca.crt'))
context.load_cert_chain(str(self.env.identity('operator') / 'tls.crt'),
str(self.env.identity('operator') / 'tls.key'))
class NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
return None
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect(),
urllib.request.HTTPSHandler(context=context))
def request(method: str, path: str, body: bytes = b'') -> bytes:
remaining = deadline - time.monotonic()
if remaining <= 0:
raise TimeoutError()
stamp = time.strftime('%Y%m%dT%H%M%SZ', time.gmtime())
day = stamp[:8]
digest = hashlib.sha256(body).hexdigest()
headers = {'host': host, 'x-amz-content-sha256': digest, 'x-amz-date': stamp}
signed = ';'.join(sorted(headers))
canonical_headers = ''.join(name + ':' + headers[name] + '\n' for name in sorted(headers))
canonical = '\n'.join((method, path, '', canonical_headers, signed, digest))
scope = day + '/us-east-1/s3/aws4_request'
to_sign = 'AWS4-HMAC-SHA256\n' + stamp + '\n' + scope + '\n' + hashlib.sha256(canonical.encode()).hexdigest()
signing_key = ('AWS4' + credential['secret_key']).encode()
for part in (day, 'us-east-1', 's3', 'aws4_request'):
signing_key = hmac.new(signing_key, part.encode(), hashlib.sha256).digest()
signature = hmac.new(signing_key, to_sign.encode(), hashlib.sha256).hexdigest()
headers['Authorization'] = ('AWS4-HMAC-SHA256 Credential=' + credential['access_key'] +
'/' + scope + ', SignedHeaders=' + signed + ', Signature=' + signature)
req = urllib.request.Request('https://' + host + path, method=method, headers=headers,
data=body if method == 'PUT' else None)
with opener.open(req, timeout=min(5, remaining)) as response:
chunks = bytearray()
while len(chunks) <= len(payload):
if time.monotonic() >= deadline:
raise TimeoutError()
chunk = response.read1(min(8192, len(payload) + 1 - len(chunks)))
if not chunk:
break
chunks.extend(chunk)
return bytes(chunks)
while time.monotonic() < deadline:
try:
if not sentinel['established']:
# Only this operator-owned key is written on first initialization.
request('PUT', object_path, payload)
if request('GET', object_path) != payload:
raise OperationError('persistent S3 readiness sentinel bytes changed')
if not sentinel['established']:
sentinel['established'] = True
self.env.private_json(marker, sentinel)
return
except urllib.error.HTTPError as exc:
if exc.code not in (404, 408, 429, 500, 502, 503, 504):
raise OperationError('S3 readiness request rejected (HTTP ' + str(exc.code) + ')') from None
except (urllib.error.URLError, TimeoutError, ConnectionError):
pass
time.sleep(min(1, max(0, deadline - time.monotonic())))
raise OperationError('CA-verified persistent S3 body readiness deadline exceeded')
def wait_ready(self, timeout: float = 120) -> None:
deadline = time.monotonic() + timeout
self.wait_s3(timeout=timeout)
context = ssl.create_default_context(cafile=str(self.env.identity('operator') / 'ca.crt'))
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}),
urllib.request.HTTPSHandler(context=context))
while time.monotonic() < deadline:
try:
with opener.open(self.env.base_url + '/health/ready', timeout=5) as response:
if response.status == 200:
return
except (urllib.error.URLError, TimeoutError):
pass
time.sleep(1)
raise OperationError('CA-verified public HTTPS readiness deadline exceeded')
def up(self) -> dict:
result = self.bootstrap()
self.env.compose('up', '-d', '--no-build', timeout=300)
# Apply current credentials/trust/CRLs even when owners were reused.
# Namespace consumers are recreated against the current owner ID.
gateways = [child for children in GATEWAY_PAIRS.values()
for child in children if child != 's3']
self.env.compose('up', '-d', '--no-build', '--no-deps', '--force-recreate',
*gateways, timeout=180)
self.env.compose('restart', 'lb', timeout=60)
self.wait_ready()
self.reset_grafana_admin()
return {**result, 'ready': True, 'base_url': self.env.base_url}
def wait_container_health(self, container: str, timeout: float = 120) -> None:
deadline = time.monotonic() + timeout
while True:
state = self.env.run(['docker', 'inspect', '--format',
'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}',
container], check=False).stdout.strip()
if state in ('healthy', 'running'):
return
if time.monotonic() >= deadline:
raise OperationError(container + ' did not become healthy after restart: ' + state)
time.sleep(1)
def rolling_restart(self, service: str) -> None:
containers = [item for item in self.env.compose('ps', '-q', service).stdout.split() if item]
for container in containers:
self.env.run(['docker', 'restart', container], timeout=120)
self.wait_container_health(container)
def restart_consumers(self, *, stores: bool = True, apps: bool = True) -> None:
if stores:
# Establish/read the sentinel before stopping the old volume owner.
self.wait_s3()
self.env.compose('restart', 'openbao', timeout=60)
self.unseal()
self.env.compose('restart', *STORES, *TELEMETRY, timeout=180)
self.wait_stores()
# A shared-network consumer is tied to a concrete owner container ID.
# Recreate consumers explicitly even if an owner was replaced.
consumers = [consumer for values in GATEWAY_PAIRS.values() for consumer in values]
self.env.compose('up', '-d', '--no-build', '--force-recreate', '--no-deps',
*consumers, timeout=180)
self.wait_s3()
if apps:
# One replica at a time: the surviving API serves while the peer
# reloads the new leaf/trust bundle. LB is restarted last.
for service in ('api', 'worker', 'relay'):
self.rolling_restart(service)
self.env.compose('restart', 'lb', timeout=60)
self.wait_ready()
def rotate_secret(self, role: str) -> dict:
if role not in ('api', 'worker'):
raise OperationError('the completed secret rotation experiment targets an S3 application identity')
values = self.credentials()
old = values['s3'][role]
self.env.private_json(self.env.state / ('retired-' + role + '-s3.json'), old)
values['s3'][role] = {'access_key': 'lab7' + role + secrets.token_hex(8),
'secret_key': secrets.token_hex(32)}
self.env.private_json(self.credentials_path, values)
self.backend_files()
client = BaoClient(self.env)
self.configure_kv(client, client.login())
self.env.compose('restart', 's3', timeout=60)
self.env.compose('restart', role, timeout=60)
self.wait_ready()
return {'role': role, 'secret_rotated': True, 'old_key_retired': True,
'images_rebuilt': False}
def rollover(self, phase: str) -> dict:
if phase == 'prepare':
result = self.pki.prepare_rollover()
elif phase == 'issue':
result = self.pki.switch_issuer()
else:
result = self.pki.retire_issuer()
client = BaoClient(self.env)
self.configure_auth(client, client.login())
self.render_grafana()
# Go reloads leaves on new handshakes: during issue the trust bundle is
# unchanged (overlap from prepare), so only native owners restart.
# Prepare distributes trust and retire removes it; both need app reload.
self.restart_consumers(apps=phase != 'issue')
return {**result, 'all_consumers_reloaded': True, 'images_rebuilt': False,
'application_reload': phase != 'issue'}
def reset_grafana_admin(self) -> None:
context = ssl.create_default_context(cafile=str(self.env.identity('operator') / 'ca.crt'))
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}),
urllib.request.HTTPSHandler(context=context))
endpoint = 'https://localhost:' + self.env.value('GRAFANA_PORT', '3000') + '/api/health'
deadline = time.monotonic() + 60
while True:
try:
with opener.open(endpoint, timeout=5) as response:
if response.status == 200:
break
except (urllib.error.URLError, TimeoutError):
if time.monotonic() >= deadline:
raise OperationError('Grafana verified TLS health deadline exceeded')
time.sleep(1)
self.env.compose('exec', '-T', 'grafana', 'grafana', 'cli',
'--homepath', '/usr/share/grafana', 'admin',
'reset-admin-password', '--password-from-stdin',
data=self.credentials()['grafana']['password'] + '\n')
def redis(self, arguments: list[str]) -> str:
# Read the operator credential inside its own backend mount, never argv.
result = self.env.compose('exec', '-T', 'redis', 'sh', '-ec',
'export REDISCLI_AUTH="$(cat /run/backend/operator-password)"; '
'exec redis-cli --tls --cacert /run/identity/current/ca.crt '
'--user operator "$@"', '--', *arguments)
return result.stdout
def jobctl(self, arguments: list[str]) -> str:
return self.env.compose('run', '--rm', '--no-deps', 'jobctl', *arguments).stdout
def reload_identity(self, role: str) -> None:
if role in ('api', 'operator', 'rogue'):
# Go reloads complete versions on new handshakes; its manager notices
# actual current leaf expiry independently of the token renewal.
return
if role == 'openbao':
self.env.compose('restart', 'openbao', timeout=60)
self.unseal()
return
if role == 's3':
self.env.compose('restart', 'seaweedfs', timeout=60)
self.env.compose('up', '-d', '--no-build', '--force-recreate', '--no-deps', 's3', timeout=90)
return
owner = 'otel-collector' if role == 'collector' else role
if role == 'grafana':
self.render_grafana()
self.env.compose('restart', owner, timeout=90)
children = GATEWAY_PAIRS.get(owner, [])
if children:
self.env.compose('up', '-d', '--no-build', '--force-recreate', '--no-deps', *children, timeout=90)
def revoke_identity(self, role: str, serial: str) -> dict:
result = self.pki.revoke(role, serial)
# Go reads current CRLs per handshake; native gateways hold startup
# CRLs and must drain old sessions before revocation returns success.
gateways = ['lb', *[child for children in GATEWAY_PAIRS.values()
for child in children if child != 's3']]
self.env.compose('restart', *gateways, timeout=180)
return {**result, 'native_gateway_crls_reloaded': True,
'native_gateway_restarts': True,
'existing_application_pools_drained': False}
def recover_identity(self, role: str) -> dict:
transport_replaced = self.pki.recover_transport()
if transport_replaced or role == 'openbao':
self.env.compose('restart', 'openbao', timeout=60)
self.unseal()
issued = self.pki.issue(role)
self.reload_identity(role)
if role in ('api', 'worker', 'relay'):
self.env.compose('up', '-d', '--no-build', '--no-deps', role, timeout=90)
return {**issued, 'recovered_by_operator': True}
def sync_crls(self) -> dict:
client = BaoClient(self.env)
token = client.login()
republished = self.pki.sync_crls(client, token)
return {'crls_updated': True, 'identities_republished': republished}
def audit_summary(self) -> dict:
result = self.env.compose('exec', '-T', 'openbao', 'cat', '/openbao/audit/audit.log')
counts = {}
for line in result.stdout.splitlines():
try:
record = json.loads(line)
except json.JSONDecodeError:
continue
request = record.get('request', {})
path = request.get('path', '')
operation = request.get('operation', '')
if path.startswith(('pki/issue/', 'pki/revoke', 'auth/token/renew-self', 'auth/cert/login')):
key = operation + ' ' + path
counts[key] = counts.get(key, 0) + 1
return {'audit_events': counts, 'raw_credentials_exported': False}
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
commands = parser.add_subparsers(dest='command', required=True)
for name in ('prepare', 'bootstrap', 'up', 'unseal', 'seal', 'restart', 'sync-crls', 'audit-summary', 'status'):
commands.add_parser(name)
for name in ('issue', 'renew'):
sub = commands.add_parser(name)
sub.add_argument('role', choices=ROLES)
sub.add_argument('--ttl', default='1h')
if name == 'renew':
sub.add_argument('--watch', action='store_true')
sub.add_argument('--interval', type=float, default=1)
sub = commands.add_parser('revoke')
sub.add_argument('role', choices=ROLES)
sub.add_argument('--serial', required=True)
sub = commands.add_parser('rotate-secret')
sub.add_argument('role', choices=('api', 'worker'))
sub = commands.add_parser('ca-rollover')
sub.add_argument('--phase', choices=('prepare', 'issue', 'retire'), required=True)
sub = commands.add_parser('recover')
sub.add_argument('role', choices=ROLES)
commands.add_parser('psql')
for name in ('redis', 'jobctl'):
command = commands.add_parser(name)
command.add_argument('arguments', nargs=argparse.REMAINDER)
args = parser.parse_args()
control = Controller()
if args.command == 'psql':
print(control.sql(sys.stdin.read()), end='')
return
if args.command == 'redis':
print(control.redis(args.arguments), end='')
return
if args.command == 'jobctl':
print(control.jobctl(args.arguments), end='')
return
if args.command == 'prepare':
result = control.prepare()
elif args.command == 'bootstrap':
result = control.bootstrap()
elif args.command == 'up':
result = control.up()
elif args.command == 'unseal':
result = control.unseal()
elif args.command == 'seal':
result = control.seal()
elif args.command == 'restart':
control.restart_consumers()
result = {'restarted': True, 'volumes_preserved': True}
elif args.command == 'sync-crls':
result = control.sync_crls()
elif args.command == 'issue':
result = control.pki.issue(args.role, args.ttl)
control.reload_identity(args.role)
elif args.command == 'renew':
if args.watch:
control.pki.renew_watch(args.role, args.ttl, args.interval, control.reload_identity)
return
result = control.pki.issue(args.role, args.ttl, service_auth=True)
control.reload_identity(args.role)
elif args.command == 'recover':
result = control.recover_identity(args.role)
elif args.command == 'revoke':
result = control.revoke_identity(args.role, args.serial)
elif args.command == 'rotate-secret':
result = control.rotate_secret(args.role)
elif args.command == 'ca-rollover':
result = control.rollover(args.phase)
elif args.command == 'audit-summary':
result = control.audit_summary()
else:
result = {'project': control.env.project, 'base_url': control.env.base_url,
'pki': control.pki.state}
print(json.dumps(result, ensure_ascii=False), flush=True)
if __name__ == '__main__':
try:
main()
except (OperationError, OSError, TimeoutError, subprocess.TimeoutExpired) as error:
print('FAIL: ' + str(error), file=sys.stderr)
raise SystemExit(1)