forked from mtusi2026/mtusi2026lab7
327 lines
9.0 KiB
TypeScript
327 lines
9.0 KiB
TypeScript
import { readFile, writeFile, mkdir, rename } from "node:fs/promises";
|
|
import { resolve } from "node:path";
|
|
import {
|
|
root,
|
|
local,
|
|
compose,
|
|
run,
|
|
privateWrite,
|
|
requireThat,
|
|
safeFailure,
|
|
} from "./runtime";
|
|
import { verifyBackendHandover } from "./preflight-build";
|
|
import { waitReady } from "../student-tests/wait";
|
|
import { openObserveRequest } from "./openobserve";
|
|
import { assertOpenObserveSignals } from "../student-tests/openobserve-signals";
|
|
const components = [
|
|
"postgres",
|
|
"broker-gateway",
|
|
"broker",
|
|
"s3-gateway",
|
|
"s3",
|
|
"api1-gateway",
|
|
"api2-gateway",
|
|
"worker-gateway",
|
|
"lb",
|
|
"openobserve-gateway",
|
|
"openobserve",
|
|
"collector-gateway",
|
|
"collector",
|
|
];
|
|
async function main() {
|
|
requireThat(
|
|
root === "/mnt/data/home/bench-saves/mtuci-vp7-security-20261115",
|
|
"OPENOBSERVE_PIPELINE_REMOTE_BOUNDARY_REQUIRED",
|
|
);
|
|
const receipt = resolve(
|
|
root,
|
|
"evidence/openobserve",
|
|
"pipeline-" + crypto.randomUUID() + ".json",
|
|
);
|
|
await mkdir(resolve(receipt, ".."), { recursive: true });
|
|
await writeFile(
|
|
receipt,
|
|
JSON.stringify({ state: "RUNNING", stage: "SOURCE_LEASE" }),
|
|
{ flag: "wx", mode: 0o600 },
|
|
);
|
|
let stage = "SOURCE_LEASE",
|
|
started = false,
|
|
primary: unknown,
|
|
heartbeat: ReturnType<typeof setInterval> | undefined;
|
|
let heartbeatError = false;
|
|
try {
|
|
const lease = await verifyBackendHandover();
|
|
requireThat(
|
|
lease.sha256 ===
|
|
"09fe19253582282e489406600f3ee20c1e52d786778d6ce90f9d9da4146e5609",
|
|
"OPENOBSERVE_PIPELINE_SOURCE_MISMATCH",
|
|
);
|
|
stage = "OWN_STAND_IDLE";
|
|
requireThat(
|
|
(
|
|
await run([
|
|
"docker",
|
|
"ps",
|
|
"--filter",
|
|
"label=com.docker.compose.project=mtuci-vp7-security",
|
|
"--format",
|
|
"{{.ID}}",
|
|
])
|
|
).out.trim() === "",
|
|
"OPENOBSERVE_EXISTING_OWN_STAND_RUNNING",
|
|
);
|
|
stage = "CHECKED112_APP_BINARY";
|
|
const build = JSON.parse(
|
|
await readFile(
|
|
resolve(root, "evidence/preflight/checked112-build.json"),
|
|
"utf8",
|
|
),
|
|
);
|
|
requireThat(
|
|
build.source_handover_sha256 === lease.sha256,
|
|
"OPENOBSERVE_APP_RECEIPT_GENERATION_MISMATCH",
|
|
);
|
|
const image = JSON.parse(
|
|
(
|
|
await run([
|
|
"docker",
|
|
"image",
|
|
"inspect",
|
|
"mtuci-vp7-security-app:local",
|
|
])
|
|
).out,
|
|
)[0];
|
|
const binaries = (
|
|
await run([
|
|
"docker",
|
|
"run",
|
|
"--rm",
|
|
"--pull=never",
|
|
"--network",
|
|
"none",
|
|
"--read-only",
|
|
"--cap-drop",
|
|
"ALL",
|
|
"--entrypoint",
|
|
"/bin/sh",
|
|
image.Id,
|
|
"-c",
|
|
"sha256sum /app /worker",
|
|
])
|
|
).out
|
|
.trim()
|
|
.split("\n");
|
|
requireThat(
|
|
binaries.length === 2 &&
|
|
binaries[0].split(/\s+/)[0] === build.binaries.app &&
|
|
binaries[1].split(/\s+/)[0] === build.binaries.worker,
|
|
"OPENOBSERVE_APP_BINARY_DRIFT",
|
|
);
|
|
stage = "OPERATOR_PROVIDER_FILES";
|
|
const password = await readFile(
|
|
resolve(root, ".local-preflight-key"),
|
|
"utf8",
|
|
);
|
|
requireThat(
|
|
/^[a-f0-9]{64}$/.test(password),
|
|
"INVALID_PREFLIGHT_SESSION_KEY",
|
|
);
|
|
for (const name of ["api1", "api2", "worker"])
|
|
for (const file of [
|
|
"database_url",
|
|
"cache_url",
|
|
"s3.json",
|
|
...(name === "worker" ? ["publisher_url"] : []),
|
|
])
|
|
requireThat(
|
|
(await Bun.file(resolve(local, "rendered", name, file)).stat()).size >
|
|
0,
|
|
"OPENOBSERVE_SCOPED_PROVIDER_FILE_MISSING",
|
|
);
|
|
stage = "NATIVE_STORES_START";
|
|
started = true;
|
|
await compose(["up", "-d", "--no-build", ...components]);
|
|
stage = "OPENOBSERVE_READY";
|
|
let storageReady = false;
|
|
const bootDeadline = Date.now() + 60_000;
|
|
while (Date.now() < bootDeadline) {
|
|
try {
|
|
if (
|
|
(
|
|
await openObserveRequest("/api/default/streams", {
|
|
password,
|
|
account: "query",
|
|
role: "grafana",
|
|
})
|
|
).status === 200
|
|
) {
|
|
storageReady = true;
|
|
break;
|
|
}
|
|
} catch {}
|
|
await Bun.sleep(500);
|
|
}
|
|
requireThat(storageReady, "OPENOBSERVE_PIPELINE_STORAGE_START_TIMEOUT");
|
|
stage = "SIGNALS_BASELINE_RED";
|
|
let baselineRed: string | null = null;
|
|
try {
|
|
await assertOpenObserveSignals(password, Date.now() - 60_000);
|
|
} catch (error) {
|
|
const code = error instanceof Error ? error.message : "";
|
|
requireThat(
|
|
[
|
|
"OPENOBSERVE_TRACE_STREAM_MISSING",
|
|
"OPENOBSERVE_WORKER_TRACE_MISSING",
|
|
"OPENOBSERVE_SIGNAL_QUERY_REJECTED",
|
|
].includes(code),
|
|
"OPENOBSERVE_BASELINE_WRONG_FAILURE",
|
|
);
|
|
baselineRed = code;
|
|
}
|
|
requireThat(
|
|
baselineRed !== null,
|
|
"OPENOBSERVE_SIGNAL_BASELINE_ALREADY_POPULATED",
|
|
);
|
|
const start = Date.now() - 1000;
|
|
const timestamp = async () => {
|
|
for (const name of ["api1", "api2", "worker"]) {
|
|
const path = resolve(local, "rendered", name, "freshness"),
|
|
temporary = path + "." + crypto.randomUUID();
|
|
await privateWrite(temporary, new Date().toISOString() + "\n");
|
|
await rename(temporary, path);
|
|
}
|
|
};
|
|
await timestamp();
|
|
let writing = false;
|
|
heartbeat = setInterval(() => {
|
|
if (writing) return;
|
|
writing = true;
|
|
timestamp()
|
|
.catch(() => {
|
|
heartbeatError = true;
|
|
})
|
|
.finally(() => {
|
|
writing = false;
|
|
});
|
|
}, 1000);
|
|
stage = "CURRENT112_API_WORKER_START";
|
|
await compose([
|
|
"up",
|
|
"-d",
|
|
"--no-build",
|
|
"--no-deps",
|
|
"api1",
|
|
"api2",
|
|
"worker",
|
|
]);
|
|
await waitReady();
|
|
stage = "CURRENT112_PUBLIC_SMOKE";
|
|
const smoke = await run(["make", "test"], {
|
|
env: {
|
|
BASE_URL: "https://localhost:18443",
|
|
CA_FILE: resolve(local, "pki/lb/ca.crt"),
|
|
},
|
|
});
|
|
await writeFile(receipt + ".public-smoke.txt", smoke.out + smoke.err, {
|
|
flag: "wx",
|
|
mode: 0o600,
|
|
});
|
|
stage = "CURRENT112_NATIVE_ALL3_READBACK";
|
|
let signals:
|
|
Awaited<ReturnType<typeof assertOpenObserveSignals>> | undefined;
|
|
let lastCode = "OPENOBSERVE_SIGNAL_NOT_CHECKED";
|
|
const signalDeadline = Date.now() + 90_000;
|
|
while (Date.now() < signalDeadline) {
|
|
try {
|
|
signals = await assertOpenObserveSignals(password, start);
|
|
break;
|
|
} catch (error) {
|
|
lastCode =
|
|
error instanceof Error ? error.message : "OPENOBSERVE_SIGNAL_UNKNOWN";
|
|
}
|
|
await Bun.sleep(1000);
|
|
}
|
|
requireThat(signals, "OPENOBSERVE_ALL3_NOT_READY_" + lastCode);
|
|
requireThat(!heartbeatError, "OPENOBSERVE_OPERATOR_HEARTBEAT_FAILED");
|
|
stage = "CURRENT112_SOURCE_POSTGUARD";
|
|
await lease.verify();
|
|
requireThat(
|
|
(await verifyBackendHandover()).sha256 === lease.sha256,
|
|
"OPENOBSERVE_PIPELINE_GENERATION_CHANGED",
|
|
);
|
|
await writeFile(
|
|
receipt,
|
|
JSON.stringify(
|
|
{
|
|
state: "PASS_SCOPED_NATIVE_ALL3",
|
|
stage,
|
|
source_handover_sha256: lease.sha256,
|
|
binaries: build.binaries,
|
|
image_id: image.Id,
|
|
baseline_red: baselineRed,
|
|
provider:
|
|
"existing scoped operator files + explicit heartbeat; not OpenBao",
|
|
signals,
|
|
native_grafana: false,
|
|
native_bao: false,
|
|
grade5: false,
|
|
},
|
|
null,
|
|
2,
|
|
) + "\n",
|
|
);
|
|
console.log("PASS_CURRENT112_NATIVE_ALL3_NO_BAO_OR_GRAFANA_ACCEPTANCE");
|
|
} catch (error) {
|
|
primary = error;
|
|
await Promise.allSettled([
|
|
writeFile(
|
|
receipt,
|
|
JSON.stringify({
|
|
state: "FAILED",
|
|
stage,
|
|
code:
|
|
error instanceof Error &&
|
|
/^[A-Z][A-Za-z0-9_:-]{0,240}$/.test(error.message)
|
|
? error.message
|
|
: "PRIVATE_DETAILS_ONLY",
|
|
}) + "\n",
|
|
),
|
|
]);
|
|
throw error;
|
|
} finally {
|
|
if (heartbeat) clearInterval(heartbeat);
|
|
if (started) {
|
|
const stop = await Promise.allSettled([
|
|
compose(["stop", "api1", "api2", "worker", ...components], {
|
|
allowFailure: true,
|
|
}),
|
|
]);
|
|
const stopped =
|
|
stop[0].status === "fulfilled" && stop[0].value.code === 0;
|
|
await Promise.allSettled([
|
|
Promise.resolve().then(async () => {
|
|
const saved = JSON.parse(await readFile(receipt, "utf8"));
|
|
saved.stop_exit =
|
|
stop[0].status === "fulfilled" ? stop[0].value.code : null;
|
|
if (!stopped && !primary) saved.state = "FAILED_TEARDOWN";
|
|
await writeFile(receipt, JSON.stringify(saved, null, 2) + "\n");
|
|
}),
|
|
]);
|
|
if (!stopped && !primary)
|
|
throw new Error("OPENOBSERVE_PIPELINE_STOP_FAILED");
|
|
console.log(
|
|
stopped
|
|
? "OWN_ALL3_STAND_STOPPED"
|
|
: "OWN_ALL3_STOP_REQUIRES_INSPECTION",
|
|
);
|
|
}
|
|
}
|
|
}
|
|
if (import.meta.main) {
|
|
try {
|
|
await main();
|
|
} catch (error) {
|
|
safeFailure(error);
|
|
}
|
|
}
|