forked from mtusi2026/mtusi2026lab7
398 lines
12 KiB
TypeScript
398 lines
12 KiB
TypeScript
import { readFile, writeFile, mkdir } from "node:fs/promises";
|
|
import { resolve } from "node:path";
|
|
import { createHash } from "node:crypto";
|
|
import {
|
|
root,
|
|
local,
|
|
run,
|
|
compose,
|
|
privateWrite,
|
|
requireThat,
|
|
safeFailure,
|
|
} from "./runtime";
|
|
import { verifyBackendHandover } from "./preflight-build";
|
|
import { writeTopology } from "./topology";
|
|
import { provisionOpenObserve, openObserveRequest } from "./openobserve";
|
|
import { prepareGrafana, grafanaRequest, metricFrameValues } from "./grafana";
|
|
import { readGrafanaDatasources } from "./grafana-datasources";
|
|
import { runGrafanaIncidents } from "./grafana-incidents";
|
|
import { runGrafanaBrowser } from "./grafana-browser";
|
|
const hash = (bytes: Uint8Array) =>
|
|
createHash("sha256").update(bytes).digest("hex");
|
|
async function main() {
|
|
requireThat(
|
|
root === "/mnt/data/home/bench-saves/mtuci-vp7-security-20261115",
|
|
"GRAFANA_PREFLIGHT_REMOTE_BOUNDARY_REQUIRED",
|
|
);
|
|
const cachePath = Bun.argv[2];
|
|
const withIncidents = Bun.argv.includes("--incidents");
|
|
const withBrowser = Bun.argv.includes("--browser");
|
|
requireThat(
|
|
!(withIncidents && withBrowser),
|
|
"GRAFANA_SEPARATE_INCIDENT_AND_BROWSER_RECEIPTS_REQUIRED",
|
|
);
|
|
requireThat(
|
|
/^evidence\/grafana\/cache-freeze-[a-f0-9-]+\.json$/.test(cachePath ?? ""),
|
|
"GRAFANA_CACHE_RECEIPT_ARGUMENT_REQUIRED",
|
|
);
|
|
const receipt = resolve(
|
|
root,
|
|
"evidence/grafana/preflight-" + crypto.randomUUID() + ".json",
|
|
);
|
|
await mkdir(resolve(receipt, ".."), { recursive: true });
|
|
await writeFile(
|
|
receipt,
|
|
JSON.stringify({ state: "RUNNING", stage: "SOURCE_LEASE" }),
|
|
{ flag: "wx", mode: 0o600 },
|
|
);
|
|
let stage = "SOURCE_LEASE",
|
|
started = false,
|
|
primary: unknown;
|
|
let result: Record<string, unknown> = {};
|
|
try {
|
|
const lease = await verifyBackendHandover();
|
|
requireThat(
|
|
lease.sha256 ===
|
|
"09fe19253582282e489406600f3ee20c1e52d786778d6ce90f9d9da4146e5609",
|
|
"GRAFANA_BACKEND_LEASE_MISMATCH",
|
|
);
|
|
stage = "OWN_STAND_IDLE";
|
|
requireThat(
|
|
(
|
|
await run([
|
|
"docker",
|
|
"ps",
|
|
"--filter",
|
|
"label=com.docker.compose.project=mtuci-vp7-security",
|
|
"--format",
|
|
"{{.ID}}",
|
|
])
|
|
).out.trim() === "",
|
|
"GRAFANA_OWN_STAND_ALREADY_RUNNING",
|
|
);
|
|
stage = "CACHED_ARTIFACT_GUARD";
|
|
const cacheBytes = await readFile(resolve(root, cachePath)),
|
|
cache = JSON.parse(cacheBytes.toString());
|
|
requireThat(
|
|
cache.state === "PASS_OWN_GRAFANA_CACHE_NO_PULL" &&
|
|
cache.image_id ===
|
|
"sha256:1c88285f1dd391a62232c010e6c224846e6a73bdc583b3365be2eba614b7919e" &&
|
|
cache.own_alias === "mtuci-vp7-grafana-cache:13.3.1-1c88285f1dd3",
|
|
"GRAFANA_CACHE_RECEIPT_MISMATCH",
|
|
);
|
|
requireThat(
|
|
JSON.parse(
|
|
(await run(["docker", "image", "inspect", cache.own_alias])).out,
|
|
)[0].Id === cache.image_id,
|
|
"GRAFANA_IMAGE_DRIFT",
|
|
);
|
|
for (const [path, sha] of Object.entries(cache.copied_files))
|
|
requireThat(
|
|
hash(await readFile(resolve(local, "plugins/openobserve", path))) ===
|
|
sha,
|
|
"GRAFANA_PLUGIN_DRIFT",
|
|
);
|
|
stage = "PRIVATE_INITIALIZER_GUARDS";
|
|
const password = await readFile(
|
|
resolve(root, ".local-preflight-key"),
|
|
"utf8",
|
|
);
|
|
requireThat(
|
|
/^[a-f0-9]{64}$/.test(password),
|
|
"INVALID_PREFLIGHT_SESSION_KEY",
|
|
);
|
|
const initializer = await prepareGrafana(password);
|
|
stage = "OWN_CACHED_IMAGE_BINDING";
|
|
let environment = await readFile(resolve(root, ".env"), "utf8");
|
|
const line = "VP7_GRAFANA_IMAGE=" + cache.own_alias;
|
|
requireThat(
|
|
!environment.includes("VP7_GRAFANA_IMAGE=") ||
|
|
environment.split("\n").includes(line),
|
|
"GRAFANA_IMAGE_ENV_CONFLICT",
|
|
);
|
|
if (!environment.split("\n").includes(line))
|
|
await privateWrite(
|
|
resolve(root, ".env"),
|
|
environment + "\n" + line + "\n",
|
|
);
|
|
stage = "CURRENT_COMPOSE_GENERATION";
|
|
await writeTopology();
|
|
await compose(["config", "--quiet"]);
|
|
stage = "RETAINED_STORAGE_START";
|
|
started = true;
|
|
await compose([
|
|
"up",
|
|
"-d",
|
|
"--no-build",
|
|
"openobserve-gateway",
|
|
"openobserve",
|
|
]);
|
|
stage = "QUERY_ACCOUNT_NATIVE_READY";
|
|
let ready = false;
|
|
const deadline = Date.now() + 60000;
|
|
while (Date.now() < deadline) {
|
|
try {
|
|
if (
|
|
(
|
|
await openObserveRequest("/api/default/streams", {
|
|
password,
|
|
account: "query",
|
|
role: "grafana",
|
|
})
|
|
).status === 200
|
|
) {
|
|
ready = true;
|
|
break;
|
|
}
|
|
} catch (error) {
|
|
await Promise.allSettled([
|
|
privateWrite(
|
|
resolve(local, "diagnostics/grafana-storage-last-error.json"),
|
|
JSON.stringify({
|
|
stage,
|
|
message: error instanceof Error ? error.message : "UNKNOWN",
|
|
code: (error as { code?: unknown })?.code,
|
|
stack: error instanceof Error ? error.stack : "",
|
|
}),
|
|
),
|
|
]);
|
|
}
|
|
await Bun.sleep(500);
|
|
}
|
|
requireThat(ready, "GRAFANA_STORAGE_READY_TIMEOUT");
|
|
stage = "SAVED_SERVICE_ACCOUNT_AUTH_GUARDS";
|
|
const accountGuards = await provisionOpenObserve(password);
|
|
stage = "GRAFANA_NATIVE_FIRST_START";
|
|
await compose(["up", "-d", "--no-build", "grafana-gateway", "grafana"]);
|
|
stage = "PUBLISHED_HTTPS_GRAFANA_HEALTH";
|
|
let health: any;
|
|
const grafanaDeadline = Date.now() + 90000;
|
|
while (Date.now() < grafanaDeadline) {
|
|
try {
|
|
health = await grafanaRequest("/api/health");
|
|
if (health.status === 200 && health.body?.database === "ok") break;
|
|
} catch (error) {
|
|
await Promise.allSettled([
|
|
privateWrite(
|
|
resolve(local, "diagnostics/grafana-health-last-error.json"),
|
|
JSON.stringify({
|
|
stage,
|
|
message: error instanceof Error ? error.message : "UNKNOWN",
|
|
code: (error as { code?: unknown })?.code,
|
|
stack: error instanceof Error ? error.stack : "",
|
|
}),
|
|
),
|
|
]);
|
|
}
|
|
await Bun.sleep(500);
|
|
}
|
|
requireThat(
|
|
health?.status === 200 &&
|
|
health.body?.database === "ok" &&
|
|
health.body?.version === "13.3.1",
|
|
"GRAFANA_NATIVE_HEALTH_TIMEOUT",
|
|
);
|
|
stage = "CLIENT_NETWORK_DNS";
|
|
const dns = await compose([
|
|
"exec",
|
|
"-T",
|
|
"grafana",
|
|
"getent",
|
|
"hosts",
|
|
"openobserve-gateway",
|
|
]);
|
|
requireThat(
|
|
/\b\d+\.\d+\.\d+\.\d+\b/.test(dns.out),
|
|
"GRAFANA_OPENOBSERVE_DNS_FAILED",
|
|
);
|
|
stage = "NATIVE_DATASOURCE_PROVISIONING";
|
|
const datasourceUIDs = await readGrafanaDatasources(grafanaRequest);
|
|
stage = "MOUNTED_DASHBOARD_NATIVE_READ";
|
|
const dashboardResponse = await grafanaRequest(
|
|
"/api/dashboards/uid/messenger-security",
|
|
),
|
|
mounted = JSON.parse(
|
|
await readFile(
|
|
resolve(root, "infra/grafana/dashboards/messenger.json"),
|
|
"utf8",
|
|
),
|
|
);
|
|
requireThat(
|
|
dashboardResponse.status === 200 &&
|
|
dashboardResponse.body?.dashboard?.panels?.length === 19,
|
|
"GRAFANA_DASHBOARD_PANEL_COUNT_MISMATCH",
|
|
);
|
|
const history = JSON.parse(
|
|
await readFile(
|
|
resolve(
|
|
root,
|
|
"evidence/openobserve/persisted-readback-8fa17baa-a9f5-419c-bf24-e9ac820fdfbf.json",
|
|
),
|
|
"utf8",
|
|
),
|
|
);
|
|
const point = Math.floor(history.query_time_milliseconds);
|
|
requireThat(
|
|
Number.isSafeInteger(point) && point > 0,
|
|
"GRAFANA_HISTORY_POINT_INVALID",
|
|
);
|
|
stage = withIncidents
|
|
? "REAL_WORKER_DATABASE_INCIDENTS"
|
|
: "HISTORICAL_METRIC_READS";
|
|
const incidents = withIncidents
|
|
? await runGrafanaIncidents(password)
|
|
: undefined;
|
|
const panels = incidents?.live_panels.panels ?? [];
|
|
const browserProof = withBrowser ? await runGrafanaBrowser() : undefined;
|
|
for (const panel of withIncidents || withBrowser ? [] : mounted.panels) {
|
|
stage = "HISTORICAL_METRIC_PANEL_" + panel.id;
|
|
const response = await grafanaRequest("/api/ds/query", {
|
|
body: {
|
|
from: String(point - 15 * 60000),
|
|
to: String(point),
|
|
queries: [
|
|
{
|
|
refId: "A",
|
|
datasource: { type: "prometheus", uid: "prometheus" },
|
|
expr: panel.targets[0].expr,
|
|
instant: true,
|
|
range: false,
|
|
intervalMs: 1000,
|
|
maxDataPoints: 1000,
|
|
},
|
|
],
|
|
},
|
|
});
|
|
requireThat(
|
|
response.status === 200,
|
|
"GRAFANA_METRIC_HTTP_STATUS_" + response.status,
|
|
);
|
|
const values = metricFrameValues(response.body);
|
|
panels.push({
|
|
id: panel.id,
|
|
title: panel.title,
|
|
samples: values.length,
|
|
values,
|
|
});
|
|
}
|
|
stage = "NATIVE_RULE_PROVISIONING";
|
|
const rules = await grafanaRequest("/api/v1/provisioning/alert-rules");
|
|
requireThat(
|
|
rules.status === 200 &&
|
|
Array.isArray(rules.body) &&
|
|
rules.body.length === 2 &&
|
|
rules.body.every((r: any) =>
|
|
["vp7-server-errors", "vp7-old-jobs"].includes(r.uid),
|
|
),
|
|
"GRAFANA_NATIVE_RULES_MISMATCH",
|
|
);
|
|
stage = "SOURCE_AND_ARTIFACT_POSTGUARDS";
|
|
await lease.verify();
|
|
requireThat(
|
|
hash(await readFile(resolve(root, cachePath))) === hash(cacheBytes),
|
|
"GRAFANA_CACHE_LEASE_CHANGED",
|
|
);
|
|
result = {
|
|
state: withBrowser
|
|
? "OBSERVED_GRAFANA_BROWSER_PENDING_FINAL_ACCEPTANCE"
|
|
: withIncidents
|
|
? "PASS_NATIVE_GRAFANA_LIVE_19_METRICS_AND_REAL_ALERT_INCIDENTS"
|
|
: "PASS_NATIVE_GRAFANA_BOOTSTRAP_AND_19_HISTORICAL_METRICS",
|
|
stage,
|
|
backend_handover_sha256: lease.sha256,
|
|
cache_receipt: { path: cachePath, sha256: hash(cacheBytes) },
|
|
initializer,
|
|
accountGuards,
|
|
grafana_version: health.body.version,
|
|
grafana_build_commit: health.body.commit,
|
|
datasource_uids: datasourceUIDs,
|
|
panels,
|
|
rule_uids: rules.body.map((r: any) => r.uid),
|
|
metric_query_time_milliseconds:
|
|
incidents?.live_panels.query_time_milliseconds ?? point,
|
|
retained_trace_history_query_time_milliseconds: point,
|
|
producer_started: withIncidents,
|
|
incidents,
|
|
browser: browserProof,
|
|
smoke_repeated: false,
|
|
native_grafana_bootstrap: true,
|
|
native_grafana_ui: false,
|
|
alert_transitions_accepted: withIncidents,
|
|
response_cap_accepted: false,
|
|
native_bao: false,
|
|
grade5: false,
|
|
};
|
|
await writeFile(receipt, JSON.stringify(result, null, 2) + "\n");
|
|
console.log(
|
|
withBrowser
|
|
? "OBSERVED_BROWSER_NOT_FINAL_ACCEPTANCE"
|
|
: withIncidents
|
|
? "PASS_GRAFANA_LIVE_19_METRICS_REAL_INCIDENTS_NO_UI_OR_CAP_ACCEPTANCE"
|
|
: "PASS_GRAFANA_BOOTSTRAP_19_METRICS_NO_UI_OR_ALERT_ACCEPTANCE",
|
|
);
|
|
} catch (error) {
|
|
primary = error;
|
|
result = {
|
|
state: "FAILED",
|
|
stage,
|
|
code:
|
|
error instanceof Error &&
|
|
/^[A-Z][A-Za-z0-9_:-]{0,240}$/.test(error.message)
|
|
? error.message
|
|
: "PRIVATE_DETAILS_ONLY",
|
|
native_bao: false,
|
|
grade5: false,
|
|
};
|
|
await Promise.allSettled([
|
|
writeFile(receipt, JSON.stringify(result, null, 2) + "\n"),
|
|
]);
|
|
throw error;
|
|
} finally {
|
|
if (started) {
|
|
const stopped = await Promise.allSettled([
|
|
compose(
|
|
withIncidents
|
|
? ["stop"]
|
|
: [
|
|
"stop",
|
|
"grafana",
|
|
"grafana-gateway",
|
|
"openobserve",
|
|
"openobserve-gateway",
|
|
],
|
|
{ allowFailure: true },
|
|
),
|
|
]);
|
|
const code =
|
|
stopped[0].status === "fulfilled" ? stopped[0].value.code : null;
|
|
await Promise.allSettled([
|
|
writeFile(
|
|
receipt,
|
|
JSON.stringify(
|
|
{
|
|
...result,
|
|
stop_exit: code,
|
|
...(code !== 0 && !primary ? { state: "FAILED_TEARDOWN" } : {}),
|
|
},
|
|
null,
|
|
2,
|
|
) + "\n",
|
|
),
|
|
]);
|
|
if (code !== 0 && !primary) throw Error("GRAFANA_PREFLIGHT_STOP_FAILED");
|
|
console.log(
|
|
code === 0
|
|
? "OWN_GRAFANA_PREFLIGHT_STOPPED"
|
|
: "OWN_GRAFANA_STOP_REQUIRES_INSPECTION",
|
|
);
|
|
}
|
|
}
|
|
}
|
|
if (import.meta.main) {
|
|
try {
|
|
await main();
|
|
} catch (error) {
|
|
safeFailure(error);
|
|
}
|
|
}
|