import hashlib import importlib import json import os import secrets import shutil import subprocess import sys import time from pathlib import Path from uuid import UUID import pki import security_operator as operator ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) def compose(*args, capture=False): result = subprocess.run( ["docker", "compose", *args], check=True, capture_output=True, text=True ) return result.stdout if capture else None def publication_counts(attachments): rows = [] for attachment in attachments: identifier = str(UUID(attachment["id"])) query = ( "SELECT json_build_object('jobs', " "(SELECT count(*) FROM jobs WHERE attachment_id=:'id'), " "'variants', (SELECT count(*) FROM attachment_variants WHERE attachment_id=:'id'), " "'published_version', (SELECT published_version FROM attachments WHERE id=:'id'));" ) result = subprocess.run( [ "docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-v", "id=" + identifier, ], input=query, text=True, capture_output=True, check=True, ) counts = json.loads(result.stdout) if counts != {"jobs": 1, "variants": 3, "published_version": 1}: raise RuntimeError("Unexpected repeated/missing published effects") rows.append({"attachment_id": identifier, **counts}) return rows def experiment(number): client = importlib.import_module("client").Client( "https://localhost:18443", str(ROOT / "secrets/pki/operator/ca.crt") ) alice, _, _, chat = client.bootstrap() saved = ROOT / "secrets/pki/previous-worker" if saved.exists(): for file in saved.iterdir(): file.chmod(0o600) shutil.rmtree(saved) shutil.copytree(ROOT / "secrets/pki/worker", saved) for file in saved.iterdir(): file.chmod(0o600) (saved / "credentials.json").write_text(json.dumps(operator.configuration("worker"))) (saved / "credentials.json").chmod(0o444) for file in saved.iterdir(): file.chmod(0o444) since = str(int(time.time())) started = time.monotonic() operator.call("auth/cert/certs/worker", token=operator.operator_token(), method="DELETE") while "security.authorization.expired" not in compose( "logs", "--since", since, "worker", capture=True ): if time.monotonic() - started > 80: raise RuntimeError("Worker did not stop within authorization deadline") time.sleep(1) stopped = time.monotonic() - started uploads = [client.attachment(alice, chat) for _ in range(3)] queued = [ client.call("GET", "/api/v1/attachments/" + att["id"], actor=alice, schema="Attachment")[ "json" ]["status"] for att, _ in uploads ] if queued != ["queued"] * 3: raise RuntimeError("Tasks did not remain queued during revocation") recovered = time.monotonic() pki.issue("worker") pki.revoke("worker") password = secrets.token_hex(24) (ROOT / "secrets/db_worker").write_text(password) operator.rotate_worker(password) operator.configure("worker") compose("restart", "s3-tls", "broker-tls", "otel-collector-tls") compose("restart", "worker") ready = [client.wait_ready(alice, att, wait_seconds=90) for att, _ in uploads] for (att, content), result in zip(uploads, ready): body = client.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=alice)[ "bytes" ] if ( hashlib.sha256(body).hexdigest() != hashlib.sha256(content).hexdigest() or result["sha256"] != hashlib.sha256(content).hexdigest() ): raise RuntimeError("Restored attachment checksum differs") recovery = time.monotonic() - recovered denial = json.loads( compose("run", "--rm", "--no-deps", "security-audit", "revoked", capture=True) ) if not denial["passed"]: raise RuntimeError("Revoked identity accepted") row = { "run": number, "authorization_stop_seconds": round(stopped, 3), "recovery_seconds": round(recovery, 3), "queued": len(queued), "ready": len(ready), "checksums_preserved": True, "publications": publication_counts(ready), "revoked": denial, } print(json.dumps(row), flush=True) for file in saved.iterdir(): file.chmod(0o600) shutil.rmtree(saved) return row if __name__ == "__main__": os.chdir(ROOT) results = [experiment(number) for number in range(1, 4)] (ROOT / "evidence/security/worker-compromise.json").write_text( json.dumps(results, indent=2) + "\n" )