"""Operator helpers for explicit certificate repair and working-key rotation.""" import argparse import io import json import os import secrets import ssl import time from uuid import uuid4 import urllib3 from minio import Minio from minio.credentials import StaticProvider from minio.error import MinioAdminException, S3Error from minio.minioadmin import MinioAdmin from scripts.bao_operator import Operator from scripts.bootstrap_security import OPERATOR, ROLES, TRUST, write_private from scripts.prepare_dependencies import attach_policy from scripts.stand import compose INSTALL = """import json,os,pathlib,signal,sys,tempfile data=json.load(sys.stdin) base=pathlib.Path('/run/identity') directory=pathlib.Path(tempfile.mkdtemp(prefix='generation-',dir=base)) certificate=data['certificate']+'\\n'+'\\n'.join(data.get('ca_chain',[])) for name,value in [('cert.pem',certificate),('key.pem',data['private_key']),('bundle.pem',certificate+'\\n'+data['private_key'])]: path=directory/name;path.write_text(value);path.chmod(0o600) link=base/'operator-next';link.unlink(missing_ok=True);link.symlink_to(directory.name);link.replace(base/'current') if data.get('reload_lb'):os.kill(1,signal.SIGUSR2) print('Operator installed a scoped short-lived leaf') """ def install(role, data): payload = {**data, "reload_lb": role == "lb"} args = ["--profile", "maintenance"] if role == "migrate" else [] compose( *args, "run", "--rm", "--no-deps", "-T", "--entrypoint", "python", "identity-" + role, "-c", INSTALL, input=json.dumps(payload), ) def issue(operator, role): cfg = ROLES[role] request = { "common_name": role, "alt_names": ",".join(cfg["dns"]), "uri_sans": "spiffe://messenger/" + role, "ttl": "120s", } if cfg.get("ip"): request["ip_sans"] = ",".join(cfg["ip"]) return operator.authorized("POST", "pki/issue/" + role, request)["data"] def repair(operator, role): result = issue(operator, role) install(role, result) return {key: result[key] for key in ["serial_number", "expiration"]} def admin(): raw = json.loads((OPERATOR / "service-secrets.json").read_text()) context = ssl.create_default_context(cafile=TRUST / "ca.pem") context.minimum_version = ssl.TLSVersion.TLSv1_3 context.load_cert_chain( OPERATOR / "client/cert.pem", OPERATOR / "client/key.pem" ) pool = urllib3.PoolManager( ssl_context=context, timeout=urllib3.Timeout(connect=2, read=8), retries=False, ) return ( raw, MinioAdmin( "localhost:" + os.getenv("S3_ADMIN_PORT", "9007"), credentials=StaticProvider( raw["s3_root_user"], raw["s3_root_password"] ), secure=True, http_client=pool, ), pool, ) def rotate_s3(operator): raw, client, pool = admin() try: journal_path = OPERATOR / "s3-rotation.json" resumed = journal_path.exists() and not json.loads( journal_path.read_text() ).get("complete") journal = json.loads(journal_path.read_text()) if resumed else {} values = journal.get( "previous_values", raw["services"]["api"]["values"] ) previous = values["S3_ACCESS_KEY"] old = Minio( "localhost:" + os.getenv("S3_ADMIN_PORT", "9007"), credentials=StaticProvider(previous, values["S3_SECRET_KEY"]), secure=True, http_client=pool, region="us-east-1", ) probe_key = journal.get( "probe_key", "objects/staging/rotation-" + uuid4().hex ) current = journal.get("next_values") or { **values, "S3_ACCESS_KEY": "api-" + secrets.token_hex(12), "S3_SECRET_KEY": secrets.token_urlsafe(32), } if not resumed: old.put_object("messenger", probe_key, io.BytesIO(b"x"), 1) journal = { "previous_values": values, "next_values": current, "probe_key": probe_key, "complete": False, } write_private(journal_path, json.dumps(journal) + "\n") client.user_add(current["S3_ACCESS_KEY"], current["S3_SECRET_KEY"]) attach_policy(client, "messenger-api", current["S3_ACCESS_KEY"]) operator.authorized("POST", "secret/data/api", {"data": current}) # Compare a fingerprint; credential values are not placed in argv/output. import hashlib expected = hashlib.sha256( current["S3_ACCESS_KEY"].encode() ).hexdigest() deadline = time.monotonic() + 45 while time.monotonic() < deadline: observed = compose( "exec", "-T", "identity-api", "python", "-c", "import hashlib,json;print(hashlib.sha256(json.load(open('/run/identity/secrets.json'))['values']['S3_ACCESS_KEY'].encode()).hexdigest())", ) if observed == expected: break time.sleep(1) else: raise TimeoutError("New scoped S3 key not observed") raw["services"]["api"]["values"] = current write_private( OPERATOR / "service-secrets.json", json.dumps(raw, indent=2) + "\n" ) try: client.user_remove(previous) except MinioAdminException as error: if json.loads(error._body).get("Code") != "XMinioAdminNoSuchUser": raise pool.clear() try: old.put_object("messenger", probe_key, io.BytesIO(b"x"), 1) except S3Error as error: assert error.code in {"AccessDenied", "InvalidAccessKeyId"}, ( error.code ) else: raise AssertionError( "Removed working key accepted on a fresh connection" ) fresh = Minio( "localhost:" + os.getenv("S3_ADMIN_PORT", "9007"), credentials=StaticProvider( current["S3_ACCESS_KEY"], current["S3_SECRET_KEY"] ), secure=True, http_client=pool, region="us-east-1", ) fresh.put_object("messenger", probe_key, io.BytesIO(b"x"), 1) root = Minio( "localhost:" + os.getenv("S3_ADMIN_PORT", "9007"), credentials=StaticProvider( raw["s3_root_user"], raw["s3_root_password"] ), secure=True, http_client=pool, region="us-east-1", ) root.remove_object("messenger", probe_key) write_private( journal_path, json.dumps({**journal, "complete": True}) + "\n" ) return { "new_key_observed": True, "old_account_removed": True, "old_key_fresh_connection_denied": True, "new_key_put_succeeded": True, "image_rebuilt": False, } finally: pool.clear() def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("action", choices=["repair-leaf", "rotate-s3"]) parser.add_argument("--role", choices=ROLES, default="lb") args = parser.parse_args() operator = Operator() try: result = ( repair(operator, args.role) if args.action == "repair-leaf" else rotate_s3(operator) ) print(json.dumps(result)) finally: operator.close() if __name__ == "__main__": main()