import { readFile, copyFile, mkdir, open, writeFile } from "node:fs/promises"; import { resolve } from "node:path"; import { randomBytes } from "node:crypto"; import { root, local, compose, run, privateWrite, requireThat, } from "./runtime"; import { decrypt, encrypt } from "./operator-crypto"; import { unlock } from "./openbao-operator"; import { bao } from "./bao-client"; import { issue, revoke } from "./pki"; import { roles } from "./topology"; import { reloadGateway } from "./reload-gateway"; import { provisionS3 } from "./s3-provision"; import { containerIP, containerID } from "./runtime-inspect"; import { tlsProbe, expectDenied } from "../student-tests/transport"; import { HTTP } from "../student-tests/http"; export async function workerCompromise(password: string) { const claim = await open( resolve(local, "operator", "worker-compromise-claim.json"), "wx", 0o600, ); await claim.writeFile('{"state":"RUNNING"}'); await claim.close(); const http = new HTTP(), actor = await http.user("compromise"), peer = await http.user("peer"); const chat = ( await http.call("POST", "/api/v1/chats", 201, { actor: actor.token, body: { title: "Worker recovery experiment", member_ids: [actor.id, peer.id], }, }) ).json; await compose(["stop", "worker"]); const fixture = new Uint8Array( await readFile(resolve(root, "tests/fixtures/sample.png")), ), pending = []; for (let i = 0; i < 3; i++) { const accepted = await http.call( "POST", `/api/v1/chats/${chat.id}/attachments?filename=probe.png`, 202, { actor: actor.token, raw: fixture, headers: { "Content-Type": "image/png" }, }, ); requireThat( accepted.json.status === "queued", "RESEARCH_NOT_DURABLY_QUEUED", ); pending.push({ id: accepted.json.id, sha256: accepted.json.sha256 }); } const beforeLogin = await bao("/auth/cert/login", { role: "worker", body: { name: "worker" }, }), oldToken = beforeLogin.data.auth.client_token; const operator = await unlock(password), rootToken = operator.root_token; const checkpoint = resolve(local, "operator/worker-compromise-state.enc"); await privateWrite( checkpoint, encrypt( { phase: "QUEUED_BEFORE_REVOCATION", pending, peer, oldToken }, password, ), ); await compose(["stop", "worker-agent"]); const retired = resolve(local, "pki", "retired-worker"); await mkdir(retired, { recursive: true, mode: 0o700 }); for (const name of ["cert.pem", "key.pem", "ca.crt"]) await copyFile(resolve(local, "pki/worker", name), resolve(retired, name)); const ca = resolve(local, "operator/ca"); await revoke(ca, resolve(retired, "cert.pem"), password); for (const role of roles) await copyFile( resolve(ca, "ca.crl"), resolve(local, "pki", role, "ca.crl"), ); await bao("/auth/cert/crls/lab", { token: rootToken, body: { crl: await readFile(resolve(ca, "ca.crl"), "utf8") }, }); const accessors = await bao("/auth/token/accessors", { method: "LIST", token: rootToken, }); let tokensRevoked = 0; for (const accessor of accessors.data.data.keys) { const entry = await bao("/auth/token/lookup-accessor", { token: rootToken, body: { accessor }, }); if (entry.data.data.policies?.includes("worker")) { await bao("/auth/token/revoke-accessor", { token: rootToken, body: { accessor }, }); tokensRevoked++; } } requireThat(tokensRevoked > 0, "RESEARCH_NO_WORKER_TOKEN_REVOKED"); await bao("/kv/data/worker", { role: "retired-worker", token: oldToken, expected: [403], }); await bao("/auth/cert/login", { role: "retired-worker", body: { name: "worker" }, expected: [400, 403], }); await privateWrite( checkpoint, encrypt( { phase: "IDENTITIES_REVOKED", pending, peer, oldToken, tokensRevoked, oldTokenDenied: true, }, password, ), ); await compose([ "exec", "-T", "postgres", "install", "-m", "644", "-o", "postgres", "-g", "postgres", "/source-tls/ca.crl", "/run/tls/ca.crl", ]); const reloaded = await compose([ "exec", "-T", "postgres", "psql", "-X", "-t", "-A", "-U", "operator", "-d", "messenger", "-c", "SELECT pg_reload_conf()", ]); requireThat(reloaded.out.trim() === "t", "RESEARCH_PG_CRL_RELOAD_FAILED"); for (const service of ["broker-gateway", "s3-gateway", "collector-gateway"]) await reloadGateway(service); await Bun.sleep(2500); const brokerIP = await containerIP("broker-gateway"); const revokedTLS = await expectDenied( () => tlsProbe( brokerIP, 6379, "broker-gateway", "retired-worker", "*1\r\n$4\r\nPING\r\n", ), "client", ); return finishWorkerRotation(password, { pending, tokensRevoked, revokedTLS, oldTokenDenied: true, verificationScope: "original peer Bearer API readback", verifyRecovered: async (item) => { const metadata = await http.ready(peer.token, item.id); requireThat( metadata.sha256 === item.sha256, "RESEARCH_ORIGINAL_SHA_CHANGED", ); const content = await http.call( "GET", "/api/v1/attachments/" + item.id + "/content", 200, { actor: peer.token }, ); requireThat( Buffer.from(content.bytes).equals(Buffer.from(fixture)), "RESEARCH_ORIGINAL_BYTES_CHANGED", ); }, }); } export async function finishWorkerRotation( password: string, proof: { pending: { id: string; sha256: string }[]; tokensRevoked: number; revokedTLS: string; oldTokenDenied: boolean; verificationScope: string; verifyRecovered: (item: { id: string; sha256: string }) => Promise; }, ) { const { pending, tokensRevoked, revokedTLS } = proof, rootToken = (await unlock(password)).root_token, ca = resolve(local, "operator/ca"); requireThat( pending.length === 3 && tokensRevoked > 0 && proof.oldTokenDenied && revokedTLS === "ERR_SSL_SSLV3_ALERT_CERTIFICATE_REVOKED", "ROTATION_PARTIAL_PROOF_REQUIRED", ); const servicesPath = resolve(local, "operator/services.enc"), secrets = decrypt(await readFile(servicesPath, "utf8"), password), oldS3Secret = secrets.s3WorkerSecret; await privateWrite( resolve(local, "operator/worker-compromise-before-secret.enc"), encrypt({ pending, oldS3Secret, services: secrets }, password), ); secrets.s3WorkerSecret = randomBytes(32).toString("hex"); await privateWrite(servicesPath, encrypt(secrets, password)); await provisionS3(password); const retiredSecret = resolve(local, "operator", "retired-s3.json"); await privateWrite( retiredSecret, JSON.stringify({ Endpoint: "127.0.0.1:19000", Access: secrets.s3WorkerAccess, Secret: oldS3Secret, Bucket: "messenger-attachments", }), ); // Use a currently permitted prefix, so ordinary worker ACL denial cannot masquerade as secret revocation. const workerNamespace = await containerID("worker-gateway"); // New leaf is installed below before the stale-secret request: otherwise a revoked TLS cert could mask this result. const current = await bao("/kv/data/worker", { token: rootToken }); const version = current.data.data.metadata.version; requireThat( Number.isSafeInteger(version) && version >= 1, "INVALID_KV_CAS_VERSION", ); const replacement = { ...current.data.data.data, s3_json: JSON.stringify({ Endpoint: "127.0.0.1:19000", Access: secrets.s3WorkerAccess, Secret: secrets.s3WorkerSecret, Bucket: "messenger-attachments", }), }; await bao("/kv/data/worker", { token: rootToken, body: { options: { cas: version }, data: replacement }, }); await issue(ca, resolve(local, "pki/worker"), "worker", password); await reloadGateway("worker-gateway"); const stale = await run([ "docker", "run", "--pull=never", "--rm", "--user", `${process.getuid?.()}:${process.getgid?.()}`, "--network", "container:" + workerNamespace, "--mount", `type=bind,source=${retiredSecret},target=/run/operator/s3.json,readonly`, "--entrypoint", "/s3-control", "mtuci-vp7-security-app:local", "-action", "prove-stale", "-config", "/run/operator/s3.json", ]); requireThat( stale.out.trim() === "PASS_RETIRED_S3_SECRET_DENIED_ALLOWED_PREFIX_WRITE", "RESEARCH_OLD_SECRET_NOT_DENIED", ); await compose(["up", "-d", "worker-agent"]); let rendered = false; for (let i = 0; i < 60; i++) { try { const s = JSON.parse( await readFile(resolve(local, "rendered/worker/s3.json"), "utf8"), ); rendered = s.Secret === secrets.s3WorkerSecret; if (rendered) break; } catch {} await Bun.sleep(500); } requireThat(rendered, "RESEARCH_ROTATED_SECRET_NOT_RENDERED"); await compose(["up", "-d", "worker"]); const started = Date.now(); for (const item of pending) await proof.verifyRecovered(item); await writeFile( resolve(root, "evidence/security/worker-compromise.json"), JSON.stringify( { track: 1, hypothesis: "Scoped worker compromise cannot read business credentials/data or administer dependencies; revoked identity can be replaced while durable queued jobs survive", method: "three real uploads while worker stopped; native cert CRL, token revocation, S3 secret rotation, leaf issuance, native gateway reload, agent render and worker restart", queued: pending.length, recovered: pending.length, originals_byte_equal: true, tokens_revoked: tokensRevoked, old_certificate_new_connection_denial: revokedTLS, old_token_new_request_denied: proof.oldTokenDenied, recovery_verification_scope: proof.verificationScope, secret_rotated: true, new_leaf_without_image_rebuild: true, recovery_seconds_after_worker_start: (Date.now() - started) / 1000, capacity_benchmark: false, }, null, 2, ) + "\n", ); await privateWrite( resolve(local, "operator", "worker-compromise-claim.json"), '{"state":"COMPLETE"}', ); console.log("PASS_WORKER_COMPROMISE_REVOKE_ROTATE_RECOVER_DURABLE_JOBS"); }