forked from mtusi2026/mtusi2026lab7
209 lines
6.0 KiB
TypeScript
209 lines
6.0 KiB
TypeScript
import { readFile, writeFile, mkdir, stat } from "node:fs/promises";
|
|
import { resolve } from "node:path";
|
|
import { createHash } from "node:crypto";
|
|
import { root, compose, run, requireThat, safeFailure } from "./runtime";
|
|
import { verifyBackendHandover } from "./preflight-build";
|
|
import { openObserveRequest } from "./openobserve";
|
|
import { assertOpenObserveSignals } from "../student-tests/openobserve-signals";
|
|
async function main() {
|
|
requireThat(
|
|
root === "/mnt/data/home/bench-saves/mtuci-vp7-security-20261115",
|
|
"OPENOBSERVE_READBACK_REMOTE_BOUNDARY_REQUIRED",
|
|
);
|
|
const parentPath = resolve(
|
|
root,
|
|
"evidence/openobserve/pipeline-fc1a31a6-7c89-4e10-98d4-fb127a6e352a.json",
|
|
);
|
|
const parentBytes = await readFile(parentPath),
|
|
parent = JSON.parse(parentBytes.toString()),
|
|
parentTime = (await stat(parentPath)).mtimeMs;
|
|
requireThat(
|
|
parent.state === "FAILED" &&
|
|
parent.stage === "CURRENT112_NATIVE_ALL3_READBACK" &&
|
|
parent.code ===
|
|
"OPENOBSERVE_ALL3_NOT_READY_INVALID_OPENOBSERVE_REQUEST_PATH",
|
|
"OPENOBSERVE_READBACK_PARENT_MISMATCH",
|
|
);
|
|
const receipt = resolve(
|
|
root,
|
|
"evidence/openobserve",
|
|
"persisted-readback-" + crypto.randomUUID() + ".json",
|
|
);
|
|
await mkdir(resolve(receipt, ".."), { recursive: true });
|
|
await writeFile(
|
|
receipt,
|
|
JSON.stringify({ state: "RUNNING", stage: "SOURCE_LEASE" }),
|
|
{ flag: "wx", mode: 0o600 },
|
|
);
|
|
let stage = "SOURCE_LEASE",
|
|
started = false,
|
|
primary: unknown;
|
|
try {
|
|
const lease = await verifyBackendHandover();
|
|
requireThat(
|
|
lease.sha256 ===
|
|
"09fe19253582282e489406600f3ee20c1e52d786778d6ce90f9d9da4146e5609",
|
|
"OPENOBSERVE_READBACK_SOURCE_MISMATCH",
|
|
);
|
|
stage = "OWN_STAND_IDLE";
|
|
requireThat(
|
|
(
|
|
await run([
|
|
"docker",
|
|
"ps",
|
|
"--filter",
|
|
"label=com.docker.compose.project=mtuci-vp7-security",
|
|
"--format",
|
|
"{{.ID}}",
|
|
])
|
|
).out.trim() === "",
|
|
"OPENOBSERVE_EXISTING_OWN_STAND_RUNNING",
|
|
);
|
|
stage = "STORAGE_ONLY_START";
|
|
started = true;
|
|
await compose([
|
|
"up",
|
|
"-d",
|
|
"--no-build",
|
|
"openobserve-gateway",
|
|
"openobserve",
|
|
]);
|
|
const password = await readFile(
|
|
resolve(root, ".local-preflight-key"),
|
|
"utf8",
|
|
);
|
|
requireThat(
|
|
/^[a-f0-9]{64}$/.test(password),
|
|
"INVALID_PREFLIGHT_SESSION_KEY",
|
|
);
|
|
stage = "STORAGE_AUTHENTICATED_READY";
|
|
let ready = false;
|
|
const deadline = Date.now() + 60_000;
|
|
while (Date.now() < deadline) {
|
|
try {
|
|
if (
|
|
(
|
|
await openObserveRequest("/api/default/streams", {
|
|
password,
|
|
account: "query",
|
|
role: "grafana",
|
|
})
|
|
).status === 200
|
|
) {
|
|
ready = true;
|
|
break;
|
|
}
|
|
} catch {}
|
|
await Bun.sleep(500);
|
|
}
|
|
requireThat(ready, "OPENOBSERVE_READBACK_STORAGE_TIMEOUT");
|
|
stage = "PERSISTED_ACTUAL112_SIGNALS";
|
|
const signals = await assertOpenObserveSignals(
|
|
password,
|
|
parentTime - 15 * 60_000,
|
|
parentTime - 5000,
|
|
);
|
|
stage = "PERSISTENCE_RESTART";
|
|
await compose(["restart", "openobserve"]);
|
|
let recovered: typeof signals | undefined;
|
|
const recoveryDeadline = Date.now() + 60_000;
|
|
while (Date.now() < recoveryDeadline) {
|
|
try {
|
|
recovered = await assertOpenObserveSignals(
|
|
password,
|
|
parentTime - 15 * 60_000,
|
|
parentTime - 5000,
|
|
);
|
|
break;
|
|
} catch {}
|
|
await Bun.sleep(500);
|
|
}
|
|
requireThat(
|
|
recovered && recovered.trace_id === signals.trace_id,
|
|
"OPENOBSERVE_PERSISTED_SIGNALS_CHANGED",
|
|
);
|
|
stage = "SOURCE_POSTGUARD";
|
|
await lease.verify();
|
|
await writeFile(
|
|
receipt,
|
|
JSON.stringify(
|
|
{
|
|
state: "PASS_PERSISTED_ACTUAL112_ALL3_READBACK",
|
|
stage,
|
|
parent_failure: {
|
|
path: parentPath,
|
|
sha256: createHash("sha256").update(parentBytes).digest("hex"),
|
|
state: parent.state,
|
|
stage: parent.stage,
|
|
code: parent.code,
|
|
},
|
|
source_handover_sha256: lease.sha256,
|
|
signals,
|
|
restart_retained_same_trace: true,
|
|
query_time_milliseconds: parentTime - 5000,
|
|
producer_repeated: false,
|
|
smoke_repeated: false,
|
|
native_bao: false,
|
|
native_grafana: false,
|
|
grade5: false,
|
|
},
|
|
null,
|
|
2,
|
|
) + "\n",
|
|
);
|
|
console.log(
|
|
"PASS_PERSISTED_ACTUAL112_ALL3_NATIVE_QUERIES_NO_BAO_OR_GRAFANA",
|
|
);
|
|
} catch (error) {
|
|
primary = error;
|
|
await Promise.allSettled([
|
|
writeFile(
|
|
receipt,
|
|
JSON.stringify({
|
|
state: "FAILED",
|
|
stage,
|
|
code:
|
|
error instanceof Error &&
|
|
/^[A-Z][A-Za-z0-9_:-]{0,240}$/.test(error.message)
|
|
? error.message
|
|
: "PRIVATE_DETAILS_ONLY",
|
|
}) + "\n",
|
|
),
|
|
]);
|
|
throw error;
|
|
} finally {
|
|
if (started) {
|
|
const stop = await Promise.allSettled([
|
|
compose(["stop", "openobserve", "openobserve-gateway"], {
|
|
allowFailure: true,
|
|
}),
|
|
]);
|
|
const stopped =
|
|
stop[0].status === "fulfilled" && stop[0].value.code === 0;
|
|
await Promise.allSettled([
|
|
Promise.resolve().then(async () => {
|
|
const saved = JSON.parse(await readFile(receipt, "utf8"));
|
|
saved.stop_exit =
|
|
stop[0].status === "fulfilled" ? stop[0].value.code : null;
|
|
if (!stopped && !primary) saved.state = "FAILED_TEARDOWN";
|
|
await writeFile(receipt, JSON.stringify(saved, null, 2) + "\n");
|
|
}),
|
|
]);
|
|
if (!stopped && !primary)
|
|
throw new Error("OPENOBSERVE_READBACK_STOP_FAILED");
|
|
console.log(
|
|
stopped
|
|
? "OWN_READBACK_STORAGE_STOPPED"
|
|
: "OWN_READBACK_STOP_REQUIRES_INSPECTION",
|
|
);
|
|
}
|
|
}
|
|
}
|
|
if (import.meta.main) {
|
|
try {
|
|
await main();
|
|
} catch (error) {
|
|
safeFailure(error);
|
|
}
|
|
}
|