forked from mtusi2026/mtusi2026lab7
295 lines
9.9 KiB
TypeScript
295 lines
9.9 KiB
TypeScript
import { readFile, mkdir, rename, writeFile } from "node:fs/promises";
|
|
import { resolve } from "node:path";
|
|
import { randomBytes } from "node:crypto";
|
|
import {
|
|
root,
|
|
local,
|
|
compose,
|
|
run,
|
|
requireThat,
|
|
privateWrite,
|
|
safeFailure,
|
|
} from "./runtime";
|
|
import { bootstrap } from "./bootstrap";
|
|
import { decrypt } from "./operator-crypto";
|
|
import { preflightBuild, verifyBackendHandover } from "./preflight-build";
|
|
import { assertAcceptedGeneration } from "./checkpoint-guards";
|
|
import { provisionDB } from "./db-provision";
|
|
import { provisionS3 } from "./s3-provision";
|
|
import { waitReady } from "../student-tests/wait";
|
|
import { postgresTransport } from "../student-tests/postgres-transport";
|
|
import { auditWorkerPrivileges } from "../student-tests/privileges";
|
|
import { auditMatrix } from "../student-tests/matrix";
|
|
import { certificateLifetime } from "../student-tests/extended";
|
|
import { containerID } from "./runtime-inspect";
|
|
import { freshnessExpiry } from "../student-tests/freshness-runtime";
|
|
import { checkSource } from "./check-source";
|
|
import { prepareProbePKI } from "../student-tests/probe-pki";
|
|
async function atomicTimestamp(path: string) {
|
|
const temporary = path + "." + crypto.randomUUID();
|
|
await privateWrite(temporary, new Date().toISOString() + "\n");
|
|
await rename(temporary, path);
|
|
}
|
|
async function main() {
|
|
let stage = "BOOTSTRAP_LOCAL";
|
|
const sessionKey = resolve(root, ".local-preflight-key");
|
|
if (!(await Bun.file(sessionKey).exists()))
|
|
await writeFile(sessionKey, randomBytes(32).toString("hex"), {
|
|
flag: "wx",
|
|
mode: 0o600,
|
|
});
|
|
const password = await readFile(sessionKey, "utf8");
|
|
requireThat(/^[a-f0-9]{64}$/.test(password), "INVALID_PREFLIGHT_SESSION_KEY");
|
|
let heartbeat: ReturnType<typeof setInterval> | undefined;
|
|
await mkdir(resolve(root, "evidence/preflight"), { recursive: true });
|
|
try {
|
|
if (!Bun.argv.includes("--resume")) await bootstrap(password);
|
|
else
|
|
requireThat(
|
|
JSON.parse(
|
|
await readFile(resolve(local, "bootstrap-claim.json"), "utf8"),
|
|
).state === "COMPLETE",
|
|
"PREFLIGHT_RESUME_UNOWNED",
|
|
);
|
|
stage = "CHECKED_CACHED_BUILD";
|
|
await preflightBuild();
|
|
const sourceBytes = await readFile(
|
|
resolve(root, "evidence/security/preflight-source-handover.json"),
|
|
),
|
|
buildReceipt = JSON.parse(
|
|
await readFile(
|
|
resolve(root, "evidence/security/preflight-build.json"),
|
|
"utf8",
|
|
),
|
|
);
|
|
const acceptedGeneration = {
|
|
backend_checked_at: buildReceipt.backend_checked_at,
|
|
source_handover_sha256: buildReceipt.source_handover_sha256,
|
|
source_files_verified: buildReceipt.source_files_verified,
|
|
binaries: {
|
|
app: buildReceipt.binaries.app,
|
|
worker: buildReceipt.binaries.worker,
|
|
},
|
|
source_handover_text: sourceBytes.toString(),
|
|
build_receipt: buildReceipt,
|
|
};
|
|
assertAcceptedGeneration(acceptedGeneration, sourceBytes, buildReceipt);
|
|
const runLease = await verifyBackendHandover();
|
|
requireThat(
|
|
runLease.sha256 === acceptedGeneration.source_handover_sha256,
|
|
"ACCEPTED_GENERATION_MISMATCH",
|
|
);
|
|
await import("./generate-agent-configs");
|
|
stage = "NATIVE_COMPOSE_PREFLIGHT";
|
|
await compose(["config", "--quiet"]);
|
|
await compose([
|
|
"up",
|
|
"-d",
|
|
"postgres",
|
|
"broker-gateway",
|
|
"broker",
|
|
"s3-gateway",
|
|
"s3",
|
|
"api1-gateway",
|
|
"api2-gateway",
|
|
"worker-gateway",
|
|
"lb",
|
|
]);
|
|
for (let i = 0; i < 60; i++) {
|
|
const r = await compose(
|
|
[
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"pg_isready",
|
|
"-h",
|
|
"127.0.0.1",
|
|
"-U",
|
|
"operator",
|
|
"-d",
|
|
"messenger",
|
|
],
|
|
{ allowFailure: true },
|
|
);
|
|
if (r.code === 0) break;
|
|
requireThat(i < 59, "PREFLIGHT_PG_NOT_READY");
|
|
await Bun.sleep(500);
|
|
}
|
|
stage = "DB_ROLE_PROVISION";
|
|
await provisionDB(password);
|
|
const secrets = decrypt<any>(
|
|
await readFile(resolve(local, "operator/services.enc"), "utf8"),
|
|
password,
|
|
);
|
|
await privateWrite(
|
|
resolve(local, "operator/migration-url"),
|
|
`postgresql://operator:${secrets.dbOperator}@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/pki/ca.crt&sslcert=/pki/cert.pem&sslkey=/pki/key.pem`,
|
|
);
|
|
stage = "NATIVE_MIGRATIONS";
|
|
await compose(["run", "--rm", "--no-deps", "migrate"]);
|
|
await provisionDB(password, true);
|
|
stage = "NATIVE_S3_SCOPED_BOOTSTRAP";
|
|
await provisionS3(password);
|
|
// Explicit operator preflight provider, never represented as OpenBao delivery.
|
|
for (const name of ["api1", "api2", "worker"]) {
|
|
const role = name === "worker" ? "worker" : "api",
|
|
suffix = role === "api" ? "Api" : "Worker";
|
|
await privateWrite(
|
|
resolve(local, "rendered", name, "database_url"),
|
|
`postgresql://${role}@127.0.0.1:15432/messenger?sslmode=disable`,
|
|
);
|
|
await privateWrite(
|
|
resolve(local, "rendered", name, "cache_url"),
|
|
`redis://${role}:${secrets["broker" + suffix]}@127.0.0.1:16379/0`,
|
|
);
|
|
await privateWrite(
|
|
resolve(local, "rendered", name, "s3.json"),
|
|
JSON.stringify({
|
|
Endpoint: "127.0.0.1:19000",
|
|
Access: secrets["s3" + suffix + "Access"],
|
|
Secret: secrets["s3" + suffix + "Secret"],
|
|
Bucket: "messenger-attachments",
|
|
}),
|
|
);
|
|
if (role === "worker")
|
|
await privateWrite(
|
|
resolve(local, "rendered", name, "publisher_url"),
|
|
`redis://publisher:${secrets.brokerPublisher}@127.0.0.1:16379/0`,
|
|
);
|
|
await atomicTimestamp(resolve(local, "rendered", name, "freshness"));
|
|
}
|
|
let writing = false;
|
|
heartbeat = setInterval(() => {
|
|
if (writing) return;
|
|
writing = true;
|
|
Promise.all(
|
|
["api1", "api2", "worker"].map((name) =>
|
|
atomicTimestamp(resolve(local, "rendered", name, "freshness")),
|
|
),
|
|
).finally(() => {
|
|
writing = false;
|
|
});
|
|
}, 1000);
|
|
stage = "API_WORKER_START";
|
|
await compose(["up", "-d", "--no-deps", "api1", "api2", "worker"]);
|
|
await waitReady();
|
|
const env = {
|
|
BASE_URL: "https://localhost:18443",
|
|
CA_FILE: resolve(local, "pki/lb/ca.crt"),
|
|
};
|
|
stage = "PUBLIC_SOURCE_CHECK";
|
|
await checkSource();
|
|
for (const target of ["test", "test-security"]) {
|
|
stage = "PUBLIC_" + target.toUpperCase().replaceAll("-", "_");
|
|
const r = await run(["make", target], { env });
|
|
await writeFile(
|
|
resolve(root, "evidence/preflight/public-" + target + ".txt"),
|
|
r.out + r.err,
|
|
);
|
|
console.log(
|
|
"PASS_PREFLIGHT_" + target.toUpperCase().replaceAll("-", "_"),
|
|
);
|
|
}
|
|
stage = "POSTGRES_TRANSPORT";
|
|
await prepareProbePKI(password);
|
|
await postgresTransport();
|
|
stage = "TLS_CORE_BOUNDARIES";
|
|
await auditMatrix("cached");
|
|
stage = "WORKER_SCOPE";
|
|
await auditWorkerPrivileges(password);
|
|
stage = "CERTIFICATE_LIFETIME";
|
|
await certificateLifetime(password, "cached");
|
|
stage = "FRESHNESS_EXPIRY";
|
|
if (heartbeat) clearInterval(heartbeat);
|
|
heartbeat = undefined;
|
|
await freshnessExpiry();
|
|
for (const name of ["api1", "api2", "worker"])
|
|
await atomicTimestamp(resolve(local, "rendered", name, "freshness"));
|
|
heartbeat = setInterval(() => {
|
|
Promise.all(
|
|
["api1", "api2", "worker"].map((name) =>
|
|
atomicTimestamp(resolve(local, "rendered", name, "freshness")),
|
|
),
|
|
).catch(() => {
|
|
console.error("PREFLIGHT_HEARTBEAT_WRITE_FAILED");
|
|
});
|
|
}, 1000);
|
|
await compose(["up", "-d", "--no-deps", "api1", "api2", "worker"]);
|
|
await waitReady();
|
|
stage = "ACTUAL_STOP";
|
|
const started = Date.now();
|
|
await compose(["stop", "api1", "api2", "worker"]);
|
|
const elapsed = (Date.now() - started) / 1000;
|
|
for (const service of ["api1", "api2", "worker"]) {
|
|
const state = JSON.parse(
|
|
(await run(["docker", "inspect", await containerID(service)])).out,
|
|
)[0];
|
|
requireThat(!state.State.Running, "PREFLIGHT_PROCESS_STOP_FAILED");
|
|
}
|
|
requireThat(elapsed < 15, "PREFLIGHT_PROCESS_STOP_EXCEEDED_SUPERVISOR");
|
|
await runLease.verify();
|
|
await writeFile(
|
|
resolve(root, "evidence/preflight/summary.json"),
|
|
JSON.stringify(
|
|
{
|
|
state: "PASS",
|
|
accepted_generation: acceptedGeneration,
|
|
provider: "operator scoped files + explicit heartbeat (not OpenBao)",
|
|
pulls: false,
|
|
https_api_pipeline: true,
|
|
core_mtls: true,
|
|
least_privilege: true,
|
|
certificate_lifetime: true,
|
|
stop_seconds: elapsed,
|
|
openbao_acceptance: false,
|
|
telemetry_acceptance: false,
|
|
grade5_acceptance: false,
|
|
},
|
|
null,
|
|
2,
|
|
) + "\n",
|
|
);
|
|
console.log("PASS_CACHED_PREFLIGHT_NO_BAO_OR_TELEMETRY_ACCEPTANCE");
|
|
} catch (error) {
|
|
await Promise.allSettled([
|
|
Promise.resolve().then(() =>
|
|
privateWrite(
|
|
resolve(local, "diagnostics/preflight-error.json"),
|
|
JSON.stringify({
|
|
stage,
|
|
message: error instanceof Error ? error.message : "UNKNOWN",
|
|
stack: error instanceof Error ? error.stack : "",
|
|
}),
|
|
),
|
|
),
|
|
Promise.resolve().then(() =>
|
|
writeFile(
|
|
resolve(root, "evidence/preflight/summary.json"),
|
|
JSON.stringify(
|
|
{
|
|
state: "FAILED",
|
|
stage,
|
|
provider: "operator preflight only",
|
|
openbao_acceptance: false,
|
|
telemetry_acceptance: false,
|
|
},
|
|
null,
|
|
2,
|
|
) + "\n",
|
|
),
|
|
),
|
|
]);
|
|
console.error("PREFLIGHT_STAGE_" + stage);
|
|
throw error;
|
|
} finally {
|
|
if (heartbeat) clearInterval(heartbeat);
|
|
}
|
|
}
|
|
if (import.meta.main) {
|
|
try {
|
|
await main();
|
|
} catch (e) {
|
|
safeFailure(e);
|
|
}
|
|
}
|