forked from mtusi2026/mtusi2026lab7
343 lines
10 KiB
TypeScript
343 lines
10 KiB
TypeScript
import { readFile, copyFile, mkdir, open, writeFile } from "node:fs/promises";
|
|
import { resolve } from "node:path";
|
|
import { randomBytes } from "node:crypto";
|
|
import {
|
|
root,
|
|
local,
|
|
compose,
|
|
run,
|
|
privateWrite,
|
|
requireThat,
|
|
} from "./runtime";
|
|
import { decrypt, encrypt } from "./operator-crypto";
|
|
import { unlock } from "./openbao-operator";
|
|
import { bao } from "./bao-client";
|
|
import { issue, revoke } from "./pki";
|
|
import { roles } from "./topology";
|
|
import { reloadGateway } from "./reload-gateway";
|
|
import { provisionS3 } from "./s3-provision";
|
|
import { containerIP, containerID } from "./runtime-inspect";
|
|
import { tlsProbe, expectDenied } from "../student-tests/transport";
|
|
import { HTTP } from "../student-tests/http";
|
|
export async function workerCompromise(password: string) {
|
|
const claim = await open(
|
|
resolve(local, "operator", "worker-compromise-claim.json"),
|
|
"wx",
|
|
0o600,
|
|
);
|
|
await claim.writeFile('{"state":"RUNNING"}');
|
|
await claim.close();
|
|
const http = new HTTP(),
|
|
actor = await http.user("compromise"),
|
|
peer = await http.user("peer");
|
|
const chat = (
|
|
await http.call("POST", "/api/v1/chats", 201, {
|
|
actor: actor.token,
|
|
body: {
|
|
title: "Worker recovery experiment",
|
|
member_ids: [actor.id, peer.id],
|
|
},
|
|
})
|
|
).json;
|
|
await compose(["stop", "worker"]);
|
|
const fixture = new Uint8Array(
|
|
await readFile(resolve(root, "tests/fixtures/sample.png")),
|
|
),
|
|
pending = [];
|
|
for (let i = 0; i < 3; i++) {
|
|
const accepted = await http.call(
|
|
"POST",
|
|
`/api/v1/chats/${chat.id}/attachments?filename=probe.png`,
|
|
202,
|
|
{
|
|
actor: actor.token,
|
|
raw: fixture,
|
|
headers: { "Content-Type": "image/png" },
|
|
},
|
|
);
|
|
requireThat(
|
|
accepted.json.status === "queued",
|
|
"RESEARCH_NOT_DURABLY_QUEUED",
|
|
);
|
|
pending.push({ id: accepted.json.id, sha256: accepted.json.sha256 });
|
|
}
|
|
const beforeLogin = await bao("/auth/cert/login", {
|
|
role: "worker",
|
|
body: { name: "worker" },
|
|
}),
|
|
oldToken = beforeLogin.data.auth.client_token;
|
|
const operator = await unlock(password),
|
|
rootToken = operator.root_token;
|
|
const checkpoint = resolve(local, "operator/worker-compromise-state.enc");
|
|
await privateWrite(
|
|
checkpoint,
|
|
encrypt(
|
|
{ phase: "QUEUED_BEFORE_REVOCATION", pending, peer, oldToken },
|
|
password,
|
|
),
|
|
);
|
|
await compose(["stop", "worker-agent"]);
|
|
const retired = resolve(local, "pki", "retired-worker");
|
|
await mkdir(retired, { recursive: true, mode: 0o700 });
|
|
for (const name of ["cert.pem", "key.pem", "ca.crt"])
|
|
await copyFile(resolve(local, "pki/worker", name), resolve(retired, name));
|
|
const ca = resolve(local, "operator/ca");
|
|
await revoke(ca, resolve(retired, "cert.pem"), password);
|
|
for (const role of roles)
|
|
await copyFile(
|
|
resolve(ca, "ca.crl"),
|
|
resolve(local, "pki", role, "ca.crl"),
|
|
);
|
|
await bao("/auth/cert/crls/lab", {
|
|
token: rootToken,
|
|
body: { crl: await readFile(resolve(ca, "ca.crl"), "utf8") },
|
|
});
|
|
const accessors = await bao("/auth/token/accessors", {
|
|
method: "LIST",
|
|
token: rootToken,
|
|
});
|
|
let tokensRevoked = 0;
|
|
for (const accessor of accessors.data.data.keys) {
|
|
const entry = await bao("/auth/token/lookup-accessor", {
|
|
token: rootToken,
|
|
body: { accessor },
|
|
});
|
|
if (entry.data.data.policies?.includes("worker")) {
|
|
await bao("/auth/token/revoke-accessor", {
|
|
token: rootToken,
|
|
body: { accessor },
|
|
});
|
|
tokensRevoked++;
|
|
}
|
|
}
|
|
requireThat(tokensRevoked > 0, "RESEARCH_NO_WORKER_TOKEN_REVOKED");
|
|
await bao("/kv/data/worker", {
|
|
role: "retired-worker",
|
|
token: oldToken,
|
|
expected: [403],
|
|
});
|
|
await bao("/auth/cert/login", {
|
|
role: "retired-worker",
|
|
body: { name: "worker" },
|
|
expected: [400, 403],
|
|
});
|
|
await privateWrite(
|
|
checkpoint,
|
|
encrypt(
|
|
{
|
|
phase: "IDENTITIES_REVOKED",
|
|
pending,
|
|
peer,
|
|
oldToken,
|
|
tokensRevoked,
|
|
oldTokenDenied: true,
|
|
},
|
|
password,
|
|
),
|
|
);
|
|
await compose([
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"install",
|
|
"-m",
|
|
"644",
|
|
"-o",
|
|
"postgres",
|
|
"-g",
|
|
"postgres",
|
|
"/source-tls/ca.crl",
|
|
"/run/tls/ca.crl",
|
|
]);
|
|
const reloaded = await compose([
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"psql",
|
|
"-X",
|
|
"-t",
|
|
"-A",
|
|
"-U",
|
|
"operator",
|
|
"-d",
|
|
"messenger",
|
|
"-c",
|
|
"SELECT pg_reload_conf()",
|
|
]);
|
|
requireThat(reloaded.out.trim() === "t", "RESEARCH_PG_CRL_RELOAD_FAILED");
|
|
for (const service of ["broker-gateway", "s3-gateway", "collector-gateway"])
|
|
await reloadGateway(service);
|
|
await Bun.sleep(2500);
|
|
const brokerIP = await containerIP("broker-gateway");
|
|
const revokedTLS = await expectDenied(
|
|
() =>
|
|
tlsProbe(
|
|
brokerIP,
|
|
6379,
|
|
"broker-gateway",
|
|
"retired-worker",
|
|
"*1\r\n$4\r\nPING\r\n",
|
|
),
|
|
"client",
|
|
);
|
|
return finishWorkerRotation(password, {
|
|
pending,
|
|
tokensRevoked,
|
|
revokedTLS,
|
|
oldTokenDenied: true,
|
|
verificationScope: "original peer Bearer API readback",
|
|
verifyRecovered: async (item) => {
|
|
const metadata = await http.ready(peer.token, item.id);
|
|
requireThat(
|
|
metadata.sha256 === item.sha256,
|
|
"RESEARCH_ORIGINAL_SHA_CHANGED",
|
|
);
|
|
const content = await http.call(
|
|
"GET",
|
|
"/api/v1/attachments/" + item.id + "/content",
|
|
200,
|
|
{ actor: peer.token },
|
|
);
|
|
requireThat(
|
|
Buffer.from(content.bytes).equals(Buffer.from(fixture)),
|
|
"RESEARCH_ORIGINAL_BYTES_CHANGED",
|
|
);
|
|
},
|
|
});
|
|
}
|
|
export async function finishWorkerRotation(
|
|
password: string,
|
|
proof: {
|
|
pending: { id: string; sha256: string }[];
|
|
tokensRevoked: number;
|
|
revokedTLS: string;
|
|
oldTokenDenied: boolean;
|
|
verificationScope: string;
|
|
verifyRecovered: (item: { id: string; sha256: string }) => Promise<void>;
|
|
},
|
|
) {
|
|
const { pending, tokensRevoked, revokedTLS } = proof,
|
|
rootToken = (await unlock(password)).root_token,
|
|
ca = resolve(local, "operator/ca");
|
|
requireThat(
|
|
pending.length === 3 &&
|
|
tokensRevoked > 0 &&
|
|
proof.oldTokenDenied &&
|
|
revokedTLS === "ERR_SSL_SSLV3_ALERT_CERTIFICATE_REVOKED",
|
|
"ROTATION_PARTIAL_PROOF_REQUIRED",
|
|
);
|
|
const servicesPath = resolve(local, "operator/services.enc"),
|
|
secrets = decrypt<any>(await readFile(servicesPath, "utf8"), password),
|
|
oldS3Secret = secrets.s3WorkerSecret;
|
|
await privateWrite(
|
|
resolve(local, "operator/worker-compromise-before-secret.enc"),
|
|
encrypt({ pending, oldS3Secret, services: secrets }, password),
|
|
);
|
|
secrets.s3WorkerSecret = randomBytes(32).toString("hex");
|
|
await privateWrite(servicesPath, encrypt(secrets, password));
|
|
await provisionS3(password);
|
|
const retiredSecret = resolve(local, "operator", "retired-s3.json");
|
|
await privateWrite(
|
|
retiredSecret,
|
|
JSON.stringify({
|
|
Endpoint: "127.0.0.1:19000",
|
|
Access: secrets.s3WorkerAccess,
|
|
Secret: oldS3Secret,
|
|
Bucket: "messenger-attachments",
|
|
}),
|
|
);
|
|
// Use a currently permitted prefix, so ordinary worker ACL denial cannot masquerade as secret revocation.
|
|
const workerNamespace = await containerID("worker-gateway");
|
|
// New leaf is installed below before the stale-secret request: otherwise a revoked TLS cert could mask this result.
|
|
const current = await bao("/kv/data/worker", { token: rootToken });
|
|
const version = current.data.data.metadata.version;
|
|
requireThat(
|
|
Number.isSafeInteger(version) && version >= 1,
|
|
"INVALID_KV_CAS_VERSION",
|
|
);
|
|
const replacement = {
|
|
...current.data.data.data,
|
|
s3_json: JSON.stringify({
|
|
Endpoint: "127.0.0.1:19000",
|
|
Access: secrets.s3WorkerAccess,
|
|
Secret: secrets.s3WorkerSecret,
|
|
Bucket: "messenger-attachments",
|
|
}),
|
|
};
|
|
await bao("/kv/data/worker", {
|
|
token: rootToken,
|
|
body: { options: { cas: version }, data: replacement },
|
|
});
|
|
await issue(ca, resolve(local, "pki/worker"), "worker", password);
|
|
await reloadGateway("worker-gateway");
|
|
const stale = await run([
|
|
"docker",
|
|
"run",
|
|
"--pull=never",
|
|
"--rm",
|
|
"--user",
|
|
`${process.getuid?.()}:${process.getgid?.()}`,
|
|
"--network",
|
|
"container:" + workerNamespace,
|
|
"--mount",
|
|
`type=bind,source=${retiredSecret},target=/run/operator/s3.json,readonly`,
|
|
"--entrypoint",
|
|
"/s3-control",
|
|
"mtuci-vp7-security-app:local",
|
|
"-action",
|
|
"prove-stale",
|
|
"-config",
|
|
"/run/operator/s3.json",
|
|
]);
|
|
requireThat(
|
|
stale.out.trim() === "PASS_RETIRED_S3_SECRET_DENIED_ALLOWED_PREFIX_WRITE",
|
|
"RESEARCH_OLD_SECRET_NOT_DENIED",
|
|
);
|
|
await compose(["up", "-d", "worker-agent"]);
|
|
let rendered = false;
|
|
for (let i = 0; i < 60; i++) {
|
|
try {
|
|
const s = JSON.parse(
|
|
await readFile(resolve(local, "rendered/worker/s3.json"), "utf8"),
|
|
);
|
|
rendered = s.Secret === secrets.s3WorkerSecret;
|
|
if (rendered) break;
|
|
} catch {}
|
|
await Bun.sleep(500);
|
|
}
|
|
requireThat(rendered, "RESEARCH_ROTATED_SECRET_NOT_RENDERED");
|
|
await compose(["up", "-d", "worker"]);
|
|
const started = Date.now();
|
|
for (const item of pending) await proof.verifyRecovered(item);
|
|
await writeFile(
|
|
resolve(root, "evidence/security/worker-compromise.json"),
|
|
JSON.stringify(
|
|
{
|
|
track: 1,
|
|
hypothesis:
|
|
"Scoped worker compromise cannot read business credentials/data or administer dependencies; revoked identity can be replaced while durable queued jobs survive",
|
|
method:
|
|
"three real uploads while worker stopped; native cert CRL, token revocation, S3 secret rotation, leaf issuance, native gateway reload, agent render and worker restart",
|
|
queued: pending.length,
|
|
recovered: pending.length,
|
|
originals_byte_equal: true,
|
|
tokens_revoked: tokensRevoked,
|
|
old_certificate_new_connection_denial: revokedTLS,
|
|
old_token_new_request_denied: proof.oldTokenDenied,
|
|
recovery_verification_scope: proof.verificationScope,
|
|
secret_rotated: true,
|
|
new_leaf_without_image_rebuild: true,
|
|
recovery_seconds_after_worker_start: (Date.now() - started) / 1000,
|
|
capacity_benchmark: false,
|
|
},
|
|
null,
|
|
2,
|
|
) + "\n",
|
|
);
|
|
await privateWrite(
|
|
resolve(local, "operator", "worker-compromise-claim.json"),
|
|
'{"state":"COMPLETE"}',
|
|
);
|
|
console.log("PASS_WORKER_COMPROMISE_REVOKE_ROTATE_RECOVER_DURABLE_JOBS");
|
|
}
|