Files
mtusi2026lab7/scripts/openobserve-pipeline.ts

327 lines
9.0 KiB
TypeScript

import { readFile, writeFile, mkdir, rename } from "node:fs/promises";
import { resolve } from "node:path";
import {
root,
local,
compose,
run,
privateWrite,
requireThat,
safeFailure,
} from "./runtime";
import { verifyBackendHandover } from "./preflight-build";
import { waitReady } from "../student-tests/wait";
import { openObserveRequest } from "./openobserve";
import { assertOpenObserveSignals } from "../student-tests/openobserve-signals";
const components = [
"postgres",
"broker-gateway",
"broker",
"s3-gateway",
"s3",
"api1-gateway",
"api2-gateway",
"worker-gateway",
"lb",
"openobserve-gateway",
"openobserve",
"collector-gateway",
"collector",
];
async function main() {
requireThat(
root === "/mnt/data/home/bench-saves/mtuci-vp7-security-20261115",
"OPENOBSERVE_PIPELINE_REMOTE_BOUNDARY_REQUIRED",
);
const receipt = resolve(
root,
"evidence/openobserve",
"pipeline-" + crypto.randomUUID() + ".json",
);
await mkdir(resolve(receipt, ".."), { recursive: true });
await writeFile(
receipt,
JSON.stringify({ state: "RUNNING", stage: "SOURCE_LEASE" }),
{ flag: "wx", mode: 0o600 },
);
let stage = "SOURCE_LEASE",
started = false,
primary: unknown,
heartbeat: ReturnType<typeof setInterval> | undefined;
let heartbeatError = false;
try {
const lease = await verifyBackendHandover();
requireThat(
lease.sha256 ===
"09fe19253582282e489406600f3ee20c1e52d786778d6ce90f9d9da4146e5609",
"OPENOBSERVE_PIPELINE_SOURCE_MISMATCH",
);
stage = "OWN_STAND_IDLE";
requireThat(
(
await run([
"docker",
"ps",
"--filter",
"label=com.docker.compose.project=mtuci-vp7-security",
"--format",
"{{.ID}}",
])
).out.trim() === "",
"OPENOBSERVE_EXISTING_OWN_STAND_RUNNING",
);
stage = "CHECKED112_APP_BINARY";
const build = JSON.parse(
await readFile(
resolve(root, "evidence/preflight/checked112-build.json"),
"utf8",
),
);
requireThat(
build.source_handover_sha256 === lease.sha256,
"OPENOBSERVE_APP_RECEIPT_GENERATION_MISMATCH",
);
const image = JSON.parse(
(
await run([
"docker",
"image",
"inspect",
"mtuci-vp7-security-app:local",
])
).out,
)[0];
const binaries = (
await run([
"docker",
"run",
"--rm",
"--pull=never",
"--network",
"none",
"--read-only",
"--cap-drop",
"ALL",
"--entrypoint",
"/bin/sh",
image.Id,
"-c",
"sha256sum /app /worker",
])
).out
.trim()
.split("\n");
requireThat(
binaries.length === 2 &&
binaries[0].split(/\s+/)[0] === build.binaries.app &&
binaries[1].split(/\s+/)[0] === build.binaries.worker,
"OPENOBSERVE_APP_BINARY_DRIFT",
);
stage = "OPERATOR_PROVIDER_FILES";
const password = await readFile(
resolve(root, ".local-preflight-key"),
"utf8",
);
requireThat(
/^[a-f0-9]{64}$/.test(password),
"INVALID_PREFLIGHT_SESSION_KEY",
);
for (const name of ["api1", "api2", "worker"])
for (const file of [
"database_url",
"cache_url",
"s3.json",
...(name === "worker" ? ["publisher_url"] : []),
])
requireThat(
(await Bun.file(resolve(local, "rendered", name, file)).stat()).size >
0,
"OPENOBSERVE_SCOPED_PROVIDER_FILE_MISSING",
);
stage = "NATIVE_STORES_START";
started = true;
await compose(["up", "-d", "--no-build", ...components]);
stage = "OPENOBSERVE_READY";
let storageReady = false;
const bootDeadline = Date.now() + 60_000;
while (Date.now() < bootDeadline) {
try {
if (
(
await openObserveRequest("/api/default/streams", {
password,
account: "query",
role: "grafana",
})
).status === 200
) {
storageReady = true;
break;
}
} catch {}
await Bun.sleep(500);
}
requireThat(storageReady, "OPENOBSERVE_PIPELINE_STORAGE_START_TIMEOUT");
stage = "SIGNALS_BASELINE_RED";
let baselineRed: string | null = null;
try {
await assertOpenObserveSignals(password, Date.now() - 60_000);
} catch (error) {
const code = error instanceof Error ? error.message : "";
requireThat(
[
"OPENOBSERVE_TRACE_STREAM_MISSING",
"OPENOBSERVE_WORKER_TRACE_MISSING",
"OPENOBSERVE_SIGNAL_QUERY_REJECTED",
].includes(code),
"OPENOBSERVE_BASELINE_WRONG_FAILURE",
);
baselineRed = code;
}
requireThat(
baselineRed !== null,
"OPENOBSERVE_SIGNAL_BASELINE_ALREADY_POPULATED",
);
const start = Date.now() - 1000;
const timestamp = async () => {
for (const name of ["api1", "api2", "worker"]) {
const path = resolve(local, "rendered", name, "freshness"),
temporary = path + "." + crypto.randomUUID();
await privateWrite(temporary, new Date().toISOString() + "\n");
await rename(temporary, path);
}
};
await timestamp();
let writing = false;
heartbeat = setInterval(() => {
if (writing) return;
writing = true;
timestamp()
.catch(() => {
heartbeatError = true;
})
.finally(() => {
writing = false;
});
}, 1000);
stage = "CURRENT112_API_WORKER_START";
await compose([
"up",
"-d",
"--no-build",
"--no-deps",
"api1",
"api2",
"worker",
]);
await waitReady();
stage = "CURRENT112_PUBLIC_SMOKE";
const smoke = await run(["make", "test"], {
env: {
BASE_URL: "https://localhost:18443",
CA_FILE: resolve(local, "pki/lb/ca.crt"),
},
});
await writeFile(receipt + ".public-smoke.txt", smoke.out + smoke.err, {
flag: "wx",
mode: 0o600,
});
stage = "CURRENT112_NATIVE_ALL3_READBACK";
let signals:
Awaited<ReturnType<typeof assertOpenObserveSignals>> | undefined;
let lastCode = "OPENOBSERVE_SIGNAL_NOT_CHECKED";
const signalDeadline = Date.now() + 90_000;
while (Date.now() < signalDeadline) {
try {
signals = await assertOpenObserveSignals(password, start);
break;
} catch (error) {
lastCode =
error instanceof Error ? error.message : "OPENOBSERVE_SIGNAL_UNKNOWN";
}
await Bun.sleep(1000);
}
requireThat(signals, "OPENOBSERVE_ALL3_NOT_READY_" + lastCode);
requireThat(!heartbeatError, "OPENOBSERVE_OPERATOR_HEARTBEAT_FAILED");
stage = "CURRENT112_SOURCE_POSTGUARD";
await lease.verify();
requireThat(
(await verifyBackendHandover()).sha256 === lease.sha256,
"OPENOBSERVE_PIPELINE_GENERATION_CHANGED",
);
await writeFile(
receipt,
JSON.stringify(
{
state: "PASS_SCOPED_NATIVE_ALL3",
stage,
source_handover_sha256: lease.sha256,
binaries: build.binaries,
image_id: image.Id,
baseline_red: baselineRed,
provider:
"existing scoped operator files + explicit heartbeat; not OpenBao",
signals,
native_grafana: false,
native_bao: false,
grade5: false,
},
null,
2,
) + "\n",
);
console.log("PASS_CURRENT112_NATIVE_ALL3_NO_BAO_OR_GRAFANA_ACCEPTANCE");
} catch (error) {
primary = error;
await Promise.allSettled([
writeFile(
receipt,
JSON.stringify({
state: "FAILED",
stage,
code:
error instanceof Error &&
/^[A-Z][A-Za-z0-9_:-]{0,240}$/.test(error.message)
? error.message
: "PRIVATE_DETAILS_ONLY",
}) + "\n",
),
]);
throw error;
} finally {
if (heartbeat) clearInterval(heartbeat);
if (started) {
const stop = await Promise.allSettled([
compose(["stop", "api1", "api2", "worker", ...components], {
allowFailure: true,
}),
]);
const stopped =
stop[0].status === "fulfilled" && stop[0].value.code === 0;
await Promise.allSettled([
Promise.resolve().then(async () => {
const saved = JSON.parse(await readFile(receipt, "utf8"));
saved.stop_exit =
stop[0].status === "fulfilled" ? stop[0].value.code : null;
if (!stopped && !primary) saved.state = "FAILED_TEARDOWN";
await writeFile(receipt, JSON.stringify(saved, null, 2) + "\n");
}),
]);
if (!stopped && !primary)
throw new Error("OPENOBSERVE_PIPELINE_STOP_FAILED");
console.log(
stopped
? "OWN_ALL3_STAND_STOPPED"
: "OWN_ALL3_STOP_REQUIRES_INSPECTION",
);
}
}
}
if (import.meta.main) {
try {
await main();
} catch (error) {
safeFailure(error);
}
}