Files
mtusi2026lab7/scripts/preflight-run.ts

295 lines
9.9 KiB
TypeScript

import { readFile, mkdir, rename, writeFile } from "node:fs/promises";
import { resolve } from "node:path";
import { randomBytes } from "node:crypto";
import {
root,
local,
compose,
run,
requireThat,
privateWrite,
safeFailure,
} from "./runtime";
import { bootstrap } from "./bootstrap";
import { decrypt } from "./operator-crypto";
import { preflightBuild, verifyBackendHandover } from "./preflight-build";
import { assertAcceptedGeneration } from "./checkpoint-guards";
import { provisionDB } from "./db-provision";
import { provisionS3 } from "./s3-provision";
import { waitReady } from "../student-tests/wait";
import { postgresTransport } from "../student-tests/postgres-transport";
import { auditWorkerPrivileges } from "../student-tests/privileges";
import { auditMatrix } from "../student-tests/matrix";
import { certificateLifetime } from "../student-tests/extended";
import { containerID } from "./runtime-inspect";
import { freshnessExpiry } from "../student-tests/freshness-runtime";
import { checkSource } from "./check-source";
import { prepareProbePKI } from "../student-tests/probe-pki";
async function atomicTimestamp(path: string) {
const temporary = path + "." + crypto.randomUUID();
await privateWrite(temporary, new Date().toISOString() + "\n");
await rename(temporary, path);
}
async function main() {
let stage = "BOOTSTRAP_LOCAL";
const sessionKey = resolve(root, ".local-preflight-key");
if (!(await Bun.file(sessionKey).exists()))
await writeFile(sessionKey, randomBytes(32).toString("hex"), {
flag: "wx",
mode: 0o600,
});
const password = await readFile(sessionKey, "utf8");
requireThat(/^[a-f0-9]{64}$/.test(password), "INVALID_PREFLIGHT_SESSION_KEY");
let heartbeat: ReturnType<typeof setInterval> | undefined;
await mkdir(resolve(root, "evidence/preflight"), { recursive: true });
try {
if (!Bun.argv.includes("--resume")) await bootstrap(password);
else
requireThat(
JSON.parse(
await readFile(resolve(local, "bootstrap-claim.json"), "utf8"),
).state === "COMPLETE",
"PREFLIGHT_RESUME_UNOWNED",
);
stage = "CHECKED_CACHED_BUILD";
await preflightBuild();
const sourceBytes = await readFile(
resolve(root, "evidence/security/preflight-source-handover.json"),
),
buildReceipt = JSON.parse(
await readFile(
resolve(root, "evidence/security/preflight-build.json"),
"utf8",
),
);
const acceptedGeneration = {
backend_checked_at: buildReceipt.backend_checked_at,
source_handover_sha256: buildReceipt.source_handover_sha256,
source_files_verified: buildReceipt.source_files_verified,
binaries: {
app: buildReceipt.binaries.app,
worker: buildReceipt.binaries.worker,
},
source_handover_text: sourceBytes.toString(),
build_receipt: buildReceipt,
};
assertAcceptedGeneration(acceptedGeneration, sourceBytes, buildReceipt);
const runLease = await verifyBackendHandover();
requireThat(
runLease.sha256 === acceptedGeneration.source_handover_sha256,
"ACCEPTED_GENERATION_MISMATCH",
);
await import("./generate-agent-configs");
stage = "NATIVE_COMPOSE_PREFLIGHT";
await compose(["config", "--quiet"]);
await compose([
"up",
"-d",
"postgres",
"broker-gateway",
"broker",
"s3-gateway",
"s3",
"api1-gateway",
"api2-gateway",
"worker-gateway",
"lb",
]);
for (let i = 0; i < 60; i++) {
const r = await compose(
[
"exec",
"-T",
"postgres",
"pg_isready",
"-h",
"127.0.0.1",
"-U",
"operator",
"-d",
"messenger",
],
{ allowFailure: true },
);
if (r.code === 0) break;
requireThat(i < 59, "PREFLIGHT_PG_NOT_READY");
await Bun.sleep(500);
}
stage = "DB_ROLE_PROVISION";
await provisionDB(password);
const secrets = decrypt<any>(
await readFile(resolve(local, "operator/services.enc"), "utf8"),
password,
);
await privateWrite(
resolve(local, "operator/migration-url"),
`postgresql://operator:${secrets.dbOperator}@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/pki/ca.crt&sslcert=/pki/cert.pem&sslkey=/pki/key.pem`,
);
stage = "NATIVE_MIGRATIONS";
await compose(["run", "--rm", "--no-deps", "migrate"]);
await provisionDB(password, true);
stage = "NATIVE_S3_SCOPED_BOOTSTRAP";
await provisionS3(password);
// Explicit operator preflight provider, never represented as OpenBao delivery.
for (const name of ["api1", "api2", "worker"]) {
const role = name === "worker" ? "worker" : "api",
suffix = role === "api" ? "Api" : "Worker";
await privateWrite(
resolve(local, "rendered", name, "database_url"),
`postgresql://${role}@127.0.0.1:15432/messenger?sslmode=disable`,
);
await privateWrite(
resolve(local, "rendered", name, "cache_url"),
`redis://${role}:${secrets["broker" + suffix]}@127.0.0.1:16379/0`,
);
await privateWrite(
resolve(local, "rendered", name, "s3.json"),
JSON.stringify({
Endpoint: "127.0.0.1:19000",
Access: secrets["s3" + suffix + "Access"],
Secret: secrets["s3" + suffix + "Secret"],
Bucket: "messenger-attachments",
}),
);
if (role === "worker")
await privateWrite(
resolve(local, "rendered", name, "publisher_url"),
`redis://publisher:${secrets.brokerPublisher}@127.0.0.1:16379/0`,
);
await atomicTimestamp(resolve(local, "rendered", name, "freshness"));
}
let writing = false;
heartbeat = setInterval(() => {
if (writing) return;
writing = true;
Promise.all(
["api1", "api2", "worker"].map((name) =>
atomicTimestamp(resolve(local, "rendered", name, "freshness")),
),
).finally(() => {
writing = false;
});
}, 1000);
stage = "API_WORKER_START";
await compose(["up", "-d", "--no-deps", "api1", "api2", "worker"]);
await waitReady();
const env = {
BASE_URL: "https://localhost:18443",
CA_FILE: resolve(local, "pki/lb/ca.crt"),
};
stage = "PUBLIC_SOURCE_CHECK";
await checkSource();
for (const target of ["test", "test-security"]) {
stage = "PUBLIC_" + target.toUpperCase().replaceAll("-", "_");
const r = await run(["make", target], { env });
await writeFile(
resolve(root, "evidence/preflight/public-" + target + ".txt"),
r.out + r.err,
);
console.log(
"PASS_PREFLIGHT_" + target.toUpperCase().replaceAll("-", "_"),
);
}
stage = "POSTGRES_TRANSPORT";
await prepareProbePKI(password);
await postgresTransport();
stage = "TLS_CORE_BOUNDARIES";
await auditMatrix("cached");
stage = "WORKER_SCOPE";
await auditWorkerPrivileges(password);
stage = "CERTIFICATE_LIFETIME";
await certificateLifetime(password, "cached");
stage = "FRESHNESS_EXPIRY";
if (heartbeat) clearInterval(heartbeat);
heartbeat = undefined;
await freshnessExpiry();
for (const name of ["api1", "api2", "worker"])
await atomicTimestamp(resolve(local, "rendered", name, "freshness"));
heartbeat = setInterval(() => {
Promise.all(
["api1", "api2", "worker"].map((name) =>
atomicTimestamp(resolve(local, "rendered", name, "freshness")),
),
).catch(() => {
console.error("PREFLIGHT_HEARTBEAT_WRITE_FAILED");
});
}, 1000);
await compose(["up", "-d", "--no-deps", "api1", "api2", "worker"]);
await waitReady();
stage = "ACTUAL_STOP";
const started = Date.now();
await compose(["stop", "api1", "api2", "worker"]);
const elapsed = (Date.now() - started) / 1000;
for (const service of ["api1", "api2", "worker"]) {
const state = JSON.parse(
(await run(["docker", "inspect", await containerID(service)])).out,
)[0];
requireThat(!state.State.Running, "PREFLIGHT_PROCESS_STOP_FAILED");
}
requireThat(elapsed < 15, "PREFLIGHT_PROCESS_STOP_EXCEEDED_SUPERVISOR");
await runLease.verify();
await writeFile(
resolve(root, "evidence/preflight/summary.json"),
JSON.stringify(
{
state: "PASS",
accepted_generation: acceptedGeneration,
provider: "operator scoped files + explicit heartbeat (not OpenBao)",
pulls: false,
https_api_pipeline: true,
core_mtls: true,
least_privilege: true,
certificate_lifetime: true,
stop_seconds: elapsed,
openbao_acceptance: false,
telemetry_acceptance: false,
grade5_acceptance: false,
},
null,
2,
) + "\n",
);
console.log("PASS_CACHED_PREFLIGHT_NO_BAO_OR_TELEMETRY_ACCEPTANCE");
} catch (error) {
await Promise.allSettled([
Promise.resolve().then(() =>
privateWrite(
resolve(local, "diagnostics/preflight-error.json"),
JSON.stringify({
stage,
message: error instanceof Error ? error.message : "UNKNOWN",
stack: error instanceof Error ? error.stack : "",
}),
),
),
Promise.resolve().then(() =>
writeFile(
resolve(root, "evidence/preflight/summary.json"),
JSON.stringify(
{
state: "FAILED",
stage,
provider: "operator preflight only",
openbao_acceptance: false,
telemetry_acceptance: false,
},
null,
2,
) + "\n",
),
),
]);
console.error("PREFLIGHT_STAGE_" + stage);
throw error;
} finally {
if (heartbeat) clearInterval(heartbeat);
}
}
if (import.meta.main) {
try {
await main();
} catch (e) {
safeFailure(e);
}
}