Files
mtusi2026lab7/scripts/rotation.ts

343 lines
10 KiB
TypeScript

import { readFile, copyFile, mkdir, open, writeFile } from "node:fs/promises";
import { resolve } from "node:path";
import { randomBytes } from "node:crypto";
import {
root,
local,
compose,
run,
privateWrite,
requireThat,
} from "./runtime";
import { decrypt, encrypt } from "./operator-crypto";
import { unlock } from "./openbao-operator";
import { bao } from "./bao-client";
import { issue, revoke } from "./pki";
import { roles } from "./topology";
import { reloadGateway } from "./reload-gateway";
import { provisionS3 } from "./s3-provision";
import { containerIP, containerID } from "./runtime-inspect";
import { tlsProbe, expectDenied } from "../student-tests/transport";
import { HTTP } from "../student-tests/http";
export async function workerCompromise(password: string) {
const claim = await open(
resolve(local, "operator", "worker-compromise-claim.json"),
"wx",
0o600,
);
await claim.writeFile('{"state":"RUNNING"}');
await claim.close();
const http = new HTTP(),
actor = await http.user("compromise"),
peer = await http.user("peer");
const chat = (
await http.call("POST", "/api/v1/chats", 201, {
actor: actor.token,
body: {
title: "Worker recovery experiment",
member_ids: [actor.id, peer.id],
},
})
).json;
await compose(["stop", "worker"]);
const fixture = new Uint8Array(
await readFile(resolve(root, "tests/fixtures/sample.png")),
),
pending = [];
for (let i = 0; i < 3; i++) {
const accepted = await http.call(
"POST",
`/api/v1/chats/${chat.id}/attachments?filename=probe.png`,
202,
{
actor: actor.token,
raw: fixture,
headers: { "Content-Type": "image/png" },
},
);
requireThat(
accepted.json.status === "queued",
"RESEARCH_NOT_DURABLY_QUEUED",
);
pending.push({ id: accepted.json.id, sha256: accepted.json.sha256 });
}
const beforeLogin = await bao("/auth/cert/login", {
role: "worker",
body: { name: "worker" },
}),
oldToken = beforeLogin.data.auth.client_token;
const operator = await unlock(password),
rootToken = operator.root_token;
const checkpoint = resolve(local, "operator/worker-compromise-state.enc");
await privateWrite(
checkpoint,
encrypt(
{ phase: "QUEUED_BEFORE_REVOCATION", pending, peer, oldToken },
password,
),
);
await compose(["stop", "worker-agent"]);
const retired = resolve(local, "pki", "retired-worker");
await mkdir(retired, { recursive: true, mode: 0o700 });
for (const name of ["cert.pem", "key.pem", "ca.crt"])
await copyFile(resolve(local, "pki/worker", name), resolve(retired, name));
const ca = resolve(local, "operator/ca");
await revoke(ca, resolve(retired, "cert.pem"), password);
for (const role of roles)
await copyFile(
resolve(ca, "ca.crl"),
resolve(local, "pki", role, "ca.crl"),
);
await bao("/auth/cert/crls/lab", {
token: rootToken,
body: { crl: await readFile(resolve(ca, "ca.crl"), "utf8") },
});
const accessors = await bao("/auth/token/accessors", {
method: "LIST",
token: rootToken,
});
let tokensRevoked = 0;
for (const accessor of accessors.data.data.keys) {
const entry = await bao("/auth/token/lookup-accessor", {
token: rootToken,
body: { accessor },
});
if (entry.data.data.policies?.includes("worker")) {
await bao("/auth/token/revoke-accessor", {
token: rootToken,
body: { accessor },
});
tokensRevoked++;
}
}
requireThat(tokensRevoked > 0, "RESEARCH_NO_WORKER_TOKEN_REVOKED");
await bao("/kv/data/worker", {
role: "retired-worker",
token: oldToken,
expected: [403],
});
await bao("/auth/cert/login", {
role: "retired-worker",
body: { name: "worker" },
expected: [400, 403],
});
await privateWrite(
checkpoint,
encrypt(
{
phase: "IDENTITIES_REVOKED",
pending,
peer,
oldToken,
tokensRevoked,
oldTokenDenied: true,
},
password,
),
);
await compose([
"exec",
"-T",
"postgres",
"install",
"-m",
"644",
"-o",
"postgres",
"-g",
"postgres",
"/source-tls/ca.crl",
"/run/tls/ca.crl",
]);
const reloaded = await compose([
"exec",
"-T",
"postgres",
"psql",
"-X",
"-t",
"-A",
"-U",
"operator",
"-d",
"messenger",
"-c",
"SELECT pg_reload_conf()",
]);
requireThat(reloaded.out.trim() === "t", "RESEARCH_PG_CRL_RELOAD_FAILED");
for (const service of ["broker-gateway", "s3-gateway", "collector-gateway"])
await reloadGateway(service);
await Bun.sleep(2500);
const brokerIP = await containerIP("broker-gateway");
const revokedTLS = await expectDenied(
() =>
tlsProbe(
brokerIP,
6379,
"broker-gateway",
"retired-worker",
"*1\r\n$4\r\nPING\r\n",
),
"client",
);
return finishWorkerRotation(password, {
pending,
tokensRevoked,
revokedTLS,
oldTokenDenied: true,
verificationScope: "original peer Bearer API readback",
verifyRecovered: async (item) => {
const metadata = await http.ready(peer.token, item.id);
requireThat(
metadata.sha256 === item.sha256,
"RESEARCH_ORIGINAL_SHA_CHANGED",
);
const content = await http.call(
"GET",
"/api/v1/attachments/" + item.id + "/content",
200,
{ actor: peer.token },
);
requireThat(
Buffer.from(content.bytes).equals(Buffer.from(fixture)),
"RESEARCH_ORIGINAL_BYTES_CHANGED",
);
},
});
}
export async function finishWorkerRotation(
password: string,
proof: {
pending: { id: string; sha256: string }[];
tokensRevoked: number;
revokedTLS: string;
oldTokenDenied: boolean;
verificationScope: string;
verifyRecovered: (item: { id: string; sha256: string }) => Promise<void>;
},
) {
const { pending, tokensRevoked, revokedTLS } = proof,
rootToken = (await unlock(password)).root_token,
ca = resolve(local, "operator/ca");
requireThat(
pending.length === 3 &&
tokensRevoked > 0 &&
proof.oldTokenDenied &&
revokedTLS === "ERR_SSL_SSLV3_ALERT_CERTIFICATE_REVOKED",
"ROTATION_PARTIAL_PROOF_REQUIRED",
);
const servicesPath = resolve(local, "operator/services.enc"),
secrets = decrypt<any>(await readFile(servicesPath, "utf8"), password),
oldS3Secret = secrets.s3WorkerSecret;
await privateWrite(
resolve(local, "operator/worker-compromise-before-secret.enc"),
encrypt({ pending, oldS3Secret, services: secrets }, password),
);
secrets.s3WorkerSecret = randomBytes(32).toString("hex");
await privateWrite(servicesPath, encrypt(secrets, password));
await provisionS3(password);
const retiredSecret = resolve(local, "operator", "retired-s3.json");
await privateWrite(
retiredSecret,
JSON.stringify({
Endpoint: "127.0.0.1:19000",
Access: secrets.s3WorkerAccess,
Secret: oldS3Secret,
Bucket: "messenger-attachments",
}),
);
// Use a currently permitted prefix, so ordinary worker ACL denial cannot masquerade as secret revocation.
const workerNamespace = await containerID("worker-gateway");
// New leaf is installed below before the stale-secret request: otherwise a revoked TLS cert could mask this result.
const current = await bao("/kv/data/worker", { token: rootToken });
const version = current.data.data.metadata.version;
requireThat(
Number.isSafeInteger(version) && version >= 1,
"INVALID_KV_CAS_VERSION",
);
const replacement = {
...current.data.data.data,
s3_json: JSON.stringify({
Endpoint: "127.0.0.1:19000",
Access: secrets.s3WorkerAccess,
Secret: secrets.s3WorkerSecret,
Bucket: "messenger-attachments",
}),
};
await bao("/kv/data/worker", {
token: rootToken,
body: { options: { cas: version }, data: replacement },
});
await issue(ca, resolve(local, "pki/worker"), "worker", password);
await reloadGateway("worker-gateway");
const stale = await run([
"docker",
"run",
"--pull=never",
"--rm",
"--user",
`${process.getuid?.()}:${process.getgid?.()}`,
"--network",
"container:" + workerNamespace,
"--mount",
`type=bind,source=${retiredSecret},target=/run/operator/s3.json,readonly`,
"--entrypoint",
"/s3-control",
"mtuci-vp7-security-app:local",
"-action",
"prove-stale",
"-config",
"/run/operator/s3.json",
]);
requireThat(
stale.out.trim() === "PASS_RETIRED_S3_SECRET_DENIED_ALLOWED_PREFIX_WRITE",
"RESEARCH_OLD_SECRET_NOT_DENIED",
);
await compose(["up", "-d", "worker-agent"]);
let rendered = false;
for (let i = 0; i < 60; i++) {
try {
const s = JSON.parse(
await readFile(resolve(local, "rendered/worker/s3.json"), "utf8"),
);
rendered = s.Secret === secrets.s3WorkerSecret;
if (rendered) break;
} catch {}
await Bun.sleep(500);
}
requireThat(rendered, "RESEARCH_ROTATED_SECRET_NOT_RENDERED");
await compose(["up", "-d", "worker"]);
const started = Date.now();
for (const item of pending) await proof.verifyRecovered(item);
await writeFile(
resolve(root, "evidence/security/worker-compromise.json"),
JSON.stringify(
{
track: 1,
hypothesis:
"Scoped worker compromise cannot read business credentials/data or administer dependencies; revoked identity can be replaced while durable queued jobs survive",
method:
"three real uploads while worker stopped; native cert CRL, token revocation, S3 secret rotation, leaf issuance, native gateway reload, agent render and worker restart",
queued: pending.length,
recovered: pending.length,
originals_byte_equal: true,
tokens_revoked: tokensRevoked,
old_certificate_new_connection_denial: revokedTLS,
old_token_new_request_denied: proof.oldTokenDenied,
recovery_verification_scope: proof.verificationScope,
secret_rotated: true,
new_leaf_without_image_rebuild: true,
recovery_seconds_after_worker_start: (Date.now() - started) / 1000,
capacity_benchmark: false,
},
null,
2,
) + "\n",
);
await privateWrite(
resolve(local, "operator", "worker-compromise-claim.json"),
'{"state":"COMPLETE"}',
);
console.log("PASS_WORKER_COMPROMISE_REVOKE_ROTATE_RECOVER_DURABLE_JOBS");
}