import { readFile, mkdir, rename, writeFile } from "node:fs/promises"; import { resolve } from "node:path"; import { randomBytes } from "node:crypto"; import { root, local, compose, run, requireThat, privateWrite, safeFailure, } from "./runtime"; import { bootstrap } from "./bootstrap"; import { decrypt } from "./operator-crypto"; import { preflightBuild, verifyBackendHandover } from "./preflight-build"; import { assertAcceptedGeneration } from "./checkpoint-guards"; import { provisionDB } from "./db-provision"; import { provisionS3 } from "./s3-provision"; import { waitReady } from "../student-tests/wait"; import { postgresTransport } from "../student-tests/postgres-transport"; import { auditWorkerPrivileges } from "../student-tests/privileges"; import { auditMatrix } from "../student-tests/matrix"; import { certificateLifetime } from "../student-tests/extended"; import { containerID } from "./runtime-inspect"; import { freshnessExpiry } from "../student-tests/freshness-runtime"; import { checkSource } from "./check-source"; import { prepareProbePKI } from "../student-tests/probe-pki"; async function atomicTimestamp(path: string) { const temporary = path + "." + crypto.randomUUID(); await privateWrite(temporary, new Date().toISOString() + "\n"); await rename(temporary, path); } async function main() { let stage = "BOOTSTRAP_LOCAL"; const sessionKey = resolve(root, ".local-preflight-key"); if (!(await Bun.file(sessionKey).exists())) await writeFile(sessionKey, randomBytes(32).toString("hex"), { flag: "wx", mode: 0o600, }); const password = await readFile(sessionKey, "utf8"); requireThat(/^[a-f0-9]{64}$/.test(password), "INVALID_PREFLIGHT_SESSION_KEY"); let heartbeat: ReturnType | undefined; await mkdir(resolve(root, "evidence/preflight"), { recursive: true }); try { if (!Bun.argv.includes("--resume")) await bootstrap(password); else requireThat( JSON.parse( await readFile(resolve(local, "bootstrap-claim.json"), "utf8"), ).state === "COMPLETE", "PREFLIGHT_RESUME_UNOWNED", ); stage = "CHECKED_CACHED_BUILD"; await preflightBuild(); const sourceBytes = await readFile( resolve(root, "evidence/security/preflight-source-handover.json"), ), buildReceipt = JSON.parse( await readFile( resolve(root, "evidence/security/preflight-build.json"), "utf8", ), ); const acceptedGeneration = { backend_checked_at: buildReceipt.backend_checked_at, source_handover_sha256: buildReceipt.source_handover_sha256, source_files_verified: buildReceipt.source_files_verified, binaries: { app: buildReceipt.binaries.app, worker: buildReceipt.binaries.worker, }, source_handover_text: sourceBytes.toString(), build_receipt: buildReceipt, }; assertAcceptedGeneration(acceptedGeneration, sourceBytes, buildReceipt); const runLease = await verifyBackendHandover(); requireThat( runLease.sha256 === acceptedGeneration.source_handover_sha256, "ACCEPTED_GENERATION_MISMATCH", ); await import("./generate-agent-configs"); stage = "NATIVE_COMPOSE_PREFLIGHT"; await compose(["config", "--quiet"]); await compose([ "up", "-d", "postgres", "broker-gateway", "broker", "s3-gateway", "s3", "api1-gateway", "api2-gateway", "worker-gateway", "lb", ]); for (let i = 0; i < 60; i++) { const r = await compose( [ "exec", "-T", "postgres", "pg_isready", "-h", "127.0.0.1", "-U", "operator", "-d", "messenger", ], { allowFailure: true }, ); if (r.code === 0) break; requireThat(i < 59, "PREFLIGHT_PG_NOT_READY"); await Bun.sleep(500); } stage = "DB_ROLE_PROVISION"; await provisionDB(password); const secrets = decrypt( await readFile(resolve(local, "operator/services.enc"), "utf8"), password, ); await privateWrite( resolve(local, "operator/migration-url"), `postgresql://operator:${secrets.dbOperator}@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/pki/ca.crt&sslcert=/pki/cert.pem&sslkey=/pki/key.pem`, ); stage = "NATIVE_MIGRATIONS"; await compose(["run", "--rm", "--no-deps", "migrate"]); await provisionDB(password, true); stage = "NATIVE_S3_SCOPED_BOOTSTRAP"; await provisionS3(password); // Explicit operator preflight provider, never represented as OpenBao delivery. for (const name of ["api1", "api2", "worker"]) { const role = name === "worker" ? "worker" : "api", suffix = role === "api" ? "Api" : "Worker"; await privateWrite( resolve(local, "rendered", name, "database_url"), `postgresql://${role}@127.0.0.1:15432/messenger?sslmode=disable`, ); await privateWrite( resolve(local, "rendered", name, "cache_url"), `redis://${role}:${secrets["broker" + suffix]}@127.0.0.1:16379/0`, ); await privateWrite( resolve(local, "rendered", name, "s3.json"), JSON.stringify({ Endpoint: "127.0.0.1:19000", Access: secrets["s3" + suffix + "Access"], Secret: secrets["s3" + suffix + "Secret"], Bucket: "messenger-attachments", }), ); if (role === "worker") await privateWrite( resolve(local, "rendered", name, "publisher_url"), `redis://publisher:${secrets.brokerPublisher}@127.0.0.1:16379/0`, ); await atomicTimestamp(resolve(local, "rendered", name, "freshness")); } let writing = false; heartbeat = setInterval(() => { if (writing) return; writing = true; Promise.all( ["api1", "api2", "worker"].map((name) => atomicTimestamp(resolve(local, "rendered", name, "freshness")), ), ).finally(() => { writing = false; }); }, 1000); stage = "API_WORKER_START"; await compose(["up", "-d", "--no-deps", "api1", "api2", "worker"]); await waitReady(); const env = { BASE_URL: "https://localhost:18443", CA_FILE: resolve(local, "pki/lb/ca.crt"), }; stage = "PUBLIC_SOURCE_CHECK"; await checkSource(); for (const target of ["test", "test-security"]) { stage = "PUBLIC_" + target.toUpperCase().replaceAll("-", "_"); const r = await run(["make", target], { env }); await writeFile( resolve(root, "evidence/preflight/public-" + target + ".txt"), r.out + r.err, ); console.log( "PASS_PREFLIGHT_" + target.toUpperCase().replaceAll("-", "_"), ); } stage = "POSTGRES_TRANSPORT"; await prepareProbePKI(password); await postgresTransport(); stage = "TLS_CORE_BOUNDARIES"; await auditMatrix("cached"); stage = "WORKER_SCOPE"; await auditWorkerPrivileges(password); stage = "CERTIFICATE_LIFETIME"; await certificateLifetime(password, "cached"); stage = "FRESHNESS_EXPIRY"; if (heartbeat) clearInterval(heartbeat); heartbeat = undefined; await freshnessExpiry(); for (const name of ["api1", "api2", "worker"]) await atomicTimestamp(resolve(local, "rendered", name, "freshness")); heartbeat = setInterval(() => { Promise.all( ["api1", "api2", "worker"].map((name) => atomicTimestamp(resolve(local, "rendered", name, "freshness")), ), ).catch(() => { console.error("PREFLIGHT_HEARTBEAT_WRITE_FAILED"); }); }, 1000); await compose(["up", "-d", "--no-deps", "api1", "api2", "worker"]); await waitReady(); stage = "ACTUAL_STOP"; const started = Date.now(); await compose(["stop", "api1", "api2", "worker"]); const elapsed = (Date.now() - started) / 1000; for (const service of ["api1", "api2", "worker"]) { const state = JSON.parse( (await run(["docker", "inspect", await containerID(service)])).out, )[0]; requireThat(!state.State.Running, "PREFLIGHT_PROCESS_STOP_FAILED"); } requireThat(elapsed < 15, "PREFLIGHT_PROCESS_STOP_EXCEEDED_SUPERVISOR"); await runLease.verify(); await writeFile( resolve(root, "evidence/preflight/summary.json"), JSON.stringify( { state: "PASS", accepted_generation: acceptedGeneration, provider: "operator scoped files + explicit heartbeat (not OpenBao)", pulls: false, https_api_pipeline: true, core_mtls: true, least_privilege: true, certificate_lifetime: true, stop_seconds: elapsed, openbao_acceptance: false, telemetry_acceptance: false, grade5_acceptance: false, }, null, 2, ) + "\n", ); console.log("PASS_CACHED_PREFLIGHT_NO_BAO_OR_TELEMETRY_ACCEPTANCE"); } catch (error) { await Promise.allSettled([ Promise.resolve().then(() => privateWrite( resolve(local, "diagnostics/preflight-error.json"), JSON.stringify({ stage, message: error instanceof Error ? error.message : "UNKNOWN", stack: error instanceof Error ? error.stack : "", }), ), ), Promise.resolve().then(() => writeFile( resolve(root, "evidence/preflight/summary.json"), JSON.stringify( { state: "FAILED", stage, provider: "operator preflight only", openbao_acceptance: false, telemetry_acceptance: false, }, null, 2, ) + "\n", ), ), ]); console.error("PREFLIGHT_STAGE_" + stage); throw error; } finally { if (heartbeat) clearInterval(heartbeat); } } if (import.meta.main) { try { await main(); } catch (e) { safeFailure(e); } }