Files
mtusi2026lab7/scripts/openobserve-readback.ts

209 lines
6.0 KiB
TypeScript

import { readFile, writeFile, mkdir, stat } from "node:fs/promises";
import { resolve } from "node:path";
import { createHash } from "node:crypto";
import { root, compose, run, requireThat, safeFailure } from "./runtime";
import { verifyBackendHandover } from "./preflight-build";
import { openObserveRequest } from "./openobserve";
import { assertOpenObserveSignals } from "../student-tests/openobserve-signals";
async function main() {
requireThat(
root === "/mnt/data/home/bench-saves/mtuci-vp7-security-20261115",
"OPENOBSERVE_READBACK_REMOTE_BOUNDARY_REQUIRED",
);
const parentPath = resolve(
root,
"evidence/openobserve/pipeline-fc1a31a6-7c89-4e10-98d4-fb127a6e352a.json",
);
const parentBytes = await readFile(parentPath),
parent = JSON.parse(parentBytes.toString()),
parentTime = (await stat(parentPath)).mtimeMs;
requireThat(
parent.state === "FAILED" &&
parent.stage === "CURRENT112_NATIVE_ALL3_READBACK" &&
parent.code ===
"OPENOBSERVE_ALL3_NOT_READY_INVALID_OPENOBSERVE_REQUEST_PATH",
"OPENOBSERVE_READBACK_PARENT_MISMATCH",
);
const receipt = resolve(
root,
"evidence/openobserve",
"persisted-readback-" + crypto.randomUUID() + ".json",
);
await mkdir(resolve(receipt, ".."), { recursive: true });
await writeFile(
receipt,
JSON.stringify({ state: "RUNNING", stage: "SOURCE_LEASE" }),
{ flag: "wx", mode: 0o600 },
);
let stage = "SOURCE_LEASE",
started = false,
primary: unknown;
try {
const lease = await verifyBackendHandover();
requireThat(
lease.sha256 ===
"09fe19253582282e489406600f3ee20c1e52d786778d6ce90f9d9da4146e5609",
"OPENOBSERVE_READBACK_SOURCE_MISMATCH",
);
stage = "OWN_STAND_IDLE";
requireThat(
(
await run([
"docker",
"ps",
"--filter",
"label=com.docker.compose.project=mtuci-vp7-security",
"--format",
"{{.ID}}",
])
).out.trim() === "",
"OPENOBSERVE_EXISTING_OWN_STAND_RUNNING",
);
stage = "STORAGE_ONLY_START";
started = true;
await compose([
"up",
"-d",
"--no-build",
"openobserve-gateway",
"openobserve",
]);
const password = await readFile(
resolve(root, ".local-preflight-key"),
"utf8",
);
requireThat(
/^[a-f0-9]{64}$/.test(password),
"INVALID_PREFLIGHT_SESSION_KEY",
);
stage = "STORAGE_AUTHENTICATED_READY";
let ready = false;
const deadline = Date.now() + 60_000;
while (Date.now() < deadline) {
try {
if (
(
await openObserveRequest("/api/default/streams", {
password,
account: "query",
role: "grafana",
})
).status === 200
) {
ready = true;
break;
}
} catch {}
await Bun.sleep(500);
}
requireThat(ready, "OPENOBSERVE_READBACK_STORAGE_TIMEOUT");
stage = "PERSISTED_ACTUAL112_SIGNALS";
const signals = await assertOpenObserveSignals(
password,
parentTime - 15 * 60_000,
parentTime - 5000,
);
stage = "PERSISTENCE_RESTART";
await compose(["restart", "openobserve"]);
let recovered: typeof signals | undefined;
const recoveryDeadline = Date.now() + 60_000;
while (Date.now() < recoveryDeadline) {
try {
recovered = await assertOpenObserveSignals(
password,
parentTime - 15 * 60_000,
parentTime - 5000,
);
break;
} catch {}
await Bun.sleep(500);
}
requireThat(
recovered && recovered.trace_id === signals.trace_id,
"OPENOBSERVE_PERSISTED_SIGNALS_CHANGED",
);
stage = "SOURCE_POSTGUARD";
await lease.verify();
await writeFile(
receipt,
JSON.stringify(
{
state: "PASS_PERSISTED_ACTUAL112_ALL3_READBACK",
stage,
parent_failure: {
path: parentPath,
sha256: createHash("sha256").update(parentBytes).digest("hex"),
state: parent.state,
stage: parent.stage,
code: parent.code,
},
source_handover_sha256: lease.sha256,
signals,
restart_retained_same_trace: true,
query_time_milliseconds: parentTime - 5000,
producer_repeated: false,
smoke_repeated: false,
native_bao: false,
native_grafana: false,
grade5: false,
},
null,
2,
) + "\n",
);
console.log(
"PASS_PERSISTED_ACTUAL112_ALL3_NATIVE_QUERIES_NO_BAO_OR_GRAFANA",
);
} catch (error) {
primary = error;
await Promise.allSettled([
writeFile(
receipt,
JSON.stringify({
state: "FAILED",
stage,
code:
error instanceof Error &&
/^[A-Z][A-Za-z0-9_:-]{0,240}$/.test(error.message)
? error.message
: "PRIVATE_DETAILS_ONLY",
}) + "\n",
),
]);
throw error;
} finally {
if (started) {
const stop = await Promise.allSettled([
compose(["stop", "openobserve", "openobserve-gateway"], {
allowFailure: true,
}),
]);
const stopped =
stop[0].status === "fulfilled" && stop[0].value.code === 0;
await Promise.allSettled([
Promise.resolve().then(async () => {
const saved = JSON.parse(await readFile(receipt, "utf8"));
saved.stop_exit =
stop[0].status === "fulfilled" ? stop[0].value.code : null;
if (!stopped && !primary) saved.state = "FAILED_TEARDOWN";
await writeFile(receipt, JSON.stringify(saved, null, 2) + "\n");
}),
]);
if (!stopped && !primary)
throw new Error("OPENOBSERVE_READBACK_STOP_FAILED");
console.log(
stopped
? "OWN_READBACK_STORAGE_STOPPED"
: "OWN_READBACK_STOP_REQUIRES_INSPECTION",
);
}
}
}
if (import.meta.main) {
try {
await main();
} catch (error) {
safeFailure(error);
}
}