forked from mtusi2026/mtusi2026lab7
202 lines
6.2 KiB
TypeScript
202 lines
6.2 KiB
TypeScript
import { readFile, writeFile } from "node:fs/promises";
|
|
import { resolve } from "node:path";
|
|
import {
|
|
root,
|
|
local,
|
|
compose,
|
|
run,
|
|
requireThat,
|
|
privateWrite,
|
|
safeFailure,
|
|
} from "./runtime";
|
|
import { operatorPassword, decrypt } from "./operator-crypto";
|
|
import { bootstrap } from "./bootstrap";
|
|
import { writeTopology } from "./topology";
|
|
import { initialize, unseal } from "./openbao-operator";
|
|
import { createAcceptCommand } from "./git-proof-cli";
|
|
import { readCanonicalGitProof } from "./git-tracking-source";
|
|
import { bao } from "./bao-client";
|
|
import { provisionDB } from "./db-provision";
|
|
import { provisionS3 } from "./s3-provision";
|
|
import { waitReady } from "../student-tests/wait";
|
|
import { verifyBackendHandover } from "./preflight-build";
|
|
import { prepareGrafana } from "./grafana";
|
|
import {
|
|
createRuntimePreparation,
|
|
type RuntimeMode,
|
|
} from "./runtime-preparation";
|
|
import { verifyCheckedRuntime } from "./checked-runtime";
|
|
import {
|
|
prepareOpenObserve,
|
|
provisionOpenObserve,
|
|
openObserveRequest,
|
|
} from "./openobserve";
|
|
export async function start(
|
|
password: string,
|
|
runtimeMode: RuntimeMode = "canonical",
|
|
) {
|
|
requireThat(
|
|
await Bun.file(resolve(root, "BACKEND-READY.json")).exists(),
|
|
"CHECKED_BACKEND_HANDOVER_REQUIRED",
|
|
);
|
|
const sourceLease = await verifyBackendHandover();
|
|
await prepareOpenObserve(password);
|
|
await prepareGrafana(password);
|
|
await writeTopology();
|
|
await import("./generate-agent-configs");
|
|
await import("./telemetry-dashboard");
|
|
await compose(["config", "--quiet"]);
|
|
const effectiveImages = (await compose(["config", "--images"])).out
|
|
.trim()
|
|
.split("\n");
|
|
for (const image of new Set(
|
|
effectiveImages.filter((image) => image !== "mtuci-vp7-security-app:local"),
|
|
))
|
|
await run(["docker", "image", "inspect", image]);
|
|
let checkedRuntime:
|
|
Awaited<ReturnType<typeof verifyCheckedRuntime>> | undefined;
|
|
await createRuntimePreparation({
|
|
build: async () => {
|
|
await compose(["build", "--pull=false", "api1"]);
|
|
},
|
|
verifyCache: async () => {
|
|
checkedRuntime = await verifyCheckedRuntime();
|
|
},
|
|
})(runtimeMode);
|
|
await sourceLease.verify();
|
|
requireThat(
|
|
(await verifyBackendHandover()).sha256 === sourceLease.sha256,
|
|
"HANDOVER_GENERATION_CHANGED",
|
|
);
|
|
await compose([
|
|
"up",
|
|
"-d",
|
|
"postgres",
|
|
"broker-gateway",
|
|
"broker",
|
|
"s3-gateway",
|
|
"s3",
|
|
"openbao",
|
|
]);
|
|
for (let attempt = 0; attempt < 40; attempt++) {
|
|
const r = await compose(
|
|
[
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"pg_isready",
|
|
"-h",
|
|
"127.0.0.1",
|
|
"-U",
|
|
"operator",
|
|
"-d",
|
|
"messenger",
|
|
],
|
|
{ allowFailure: true },
|
|
);
|
|
if (r.code === 0) break;
|
|
requireThat(attempt < 39, "PG_BOOT_TIMEOUT");
|
|
await Bun.sleep(500);
|
|
}
|
|
let initialized = false,
|
|
responded = false;
|
|
for (let attempt = 0; attempt < 40; attempt++) {
|
|
try {
|
|
const state = await bao("/sys/init");
|
|
initialized = state.data.initialized === true;
|
|
responded = true;
|
|
break;
|
|
} catch {
|
|
await Bun.sleep(500);
|
|
}
|
|
}
|
|
requireThat(responded, "BAO_TLS_STARTUP_TIMEOUT");
|
|
if (initialized) await unseal(password);
|
|
else await initialize(password);
|
|
await provisionDB(password);
|
|
const secrets = decrypt<any>(
|
|
await readFile(resolve(local, "operator/services.enc"), "utf8"),
|
|
password,
|
|
);
|
|
await privateWrite(
|
|
resolve(local, "operator/migration-url"),
|
|
`postgresql://operator:${secrets.dbOperator}@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/pki/ca.crt&sslcert=/pki/cert.pem&sslkey=/pki/key.pem`,
|
|
);
|
|
await compose(["run", "--rm", "--no-deps", "migrate"]);
|
|
await provisionDB(password, true);
|
|
// RustFS may need to finish its first durable drive bootstrap before the operator's bounded SDK request.
|
|
let bootstrapped = false;
|
|
for (let attempt = 0; attempt < 10; attempt++) {
|
|
try {
|
|
await provisionS3(password);
|
|
bootstrapped = true;
|
|
break;
|
|
} catch {
|
|
await Bun.sleep(1000);
|
|
}
|
|
}
|
|
requireThat(bootstrapped, "S3_BOOTSTRAP_TIMEOUT");
|
|
await compose(["up", "-d", "openobserve-gateway", "openobserve"]);
|
|
let storageReady = false;
|
|
const storageDeadline = Date.now() + 60_000;
|
|
while (Date.now() < storageDeadline) {
|
|
try {
|
|
if (
|
|
(await openObserveRequest("/api/default/streams", { password }))
|
|
.status === 200
|
|
) {
|
|
storageReady = true;
|
|
break;
|
|
}
|
|
} catch {}
|
|
await Bun.sleep(500);
|
|
}
|
|
requireThat(storageReady, "OPENOBSERVE_AUTHENTICATED_BOOT_TIMEOUT");
|
|
await provisionOpenObserve(password);
|
|
await compose(["up", "-d"]);
|
|
await waitReady();
|
|
await checkedRuntime?.verify();
|
|
console.log("PASS_FULL_STAND_STARTED_WITH_VERIFIED_HTTPS");
|
|
}
|
|
const usage =
|
|
"bun scripts/lab.ts prepare | start [--checked-cache] | wait | stop | unseal | accept [--checked-cache] [--git-proof PATH --git-proof-sha SHA256] | help";
|
|
if (import.meta.main) {
|
|
try {
|
|
const action = Bun.argv[2] ?? "help";
|
|
if (["help", "--help", "-h"].includes(action)) console.log(usage);
|
|
else if (action === "prepare") {
|
|
await bootstrap(await operatorPassword());
|
|
await import("./generate-agent-configs");
|
|
await import("./telemetry-dashboard");
|
|
} else if (action === "start")
|
|
await start(
|
|
await operatorPassword(),
|
|
Bun.argv.includes("--checked-cache") ? "checked-cache" : "canonical",
|
|
);
|
|
else if (action === "wait") await waitReady();
|
|
else if (action === "accept") {
|
|
const { acceptanceSession } = await import("./acceptance-session");
|
|
await createAcceptCommand({
|
|
verifyProof: readCanonicalGitProof,
|
|
password: operatorPassword,
|
|
accept: (password, proof) =>
|
|
acceptanceSession(
|
|
password,
|
|
Bun.argv.includes("--checked-cache")
|
|
? "checked-cache"
|
|
: "canonical",
|
|
proof,
|
|
),
|
|
})(Bun.argv.slice(3));
|
|
} else if (action === "unseal") {
|
|
await unseal(await operatorPassword());
|
|
console.log("PASS_BAO_UNSEALED");
|
|
} else if (action === "stop") {
|
|
await compose(["--profile", "*", "stop"]);
|
|
console.log("PASS_OWNED_STAND_STOPPED_DATA_RETAINED");
|
|
} else throw new Error("UNKNOWN_LAB_ACTION");
|
|
} catch (e) {
|
|
safeFailure(e);
|
|
}
|
|
}
|