Files
mtusi2026lab7/scripts/grafana-preflight.ts

398 lines
12 KiB
TypeScript

import { readFile, writeFile, mkdir } from "node:fs/promises";
import { resolve } from "node:path";
import { createHash } from "node:crypto";
import {
root,
local,
run,
compose,
privateWrite,
requireThat,
safeFailure,
} from "./runtime";
import { verifyBackendHandover } from "./preflight-build";
import { writeTopology } from "./topology";
import { provisionOpenObserve, openObserveRequest } from "./openobserve";
import { prepareGrafana, grafanaRequest, metricFrameValues } from "./grafana";
import { readGrafanaDatasources } from "./grafana-datasources";
import { runGrafanaIncidents } from "./grafana-incidents";
import { runGrafanaBrowser } from "./grafana-browser";
const hash = (bytes: Uint8Array) =>
createHash("sha256").update(bytes).digest("hex");
async function main() {
requireThat(
root === "/mnt/data/home/bench-saves/mtuci-vp7-security-20261115",
"GRAFANA_PREFLIGHT_REMOTE_BOUNDARY_REQUIRED",
);
const cachePath = Bun.argv[2];
const withIncidents = Bun.argv.includes("--incidents");
const withBrowser = Bun.argv.includes("--browser");
requireThat(
!(withIncidents && withBrowser),
"GRAFANA_SEPARATE_INCIDENT_AND_BROWSER_RECEIPTS_REQUIRED",
);
requireThat(
/^evidence\/grafana\/cache-freeze-[a-f0-9-]+\.json$/.test(cachePath ?? ""),
"GRAFANA_CACHE_RECEIPT_ARGUMENT_REQUIRED",
);
const receipt = resolve(
root,
"evidence/grafana/preflight-" + crypto.randomUUID() + ".json",
);
await mkdir(resolve(receipt, ".."), { recursive: true });
await writeFile(
receipt,
JSON.stringify({ state: "RUNNING", stage: "SOURCE_LEASE" }),
{ flag: "wx", mode: 0o600 },
);
let stage = "SOURCE_LEASE",
started = false,
primary: unknown;
let result: Record<string, unknown> = {};
try {
const lease = await verifyBackendHandover();
requireThat(
lease.sha256 ===
"09fe19253582282e489406600f3ee20c1e52d786778d6ce90f9d9da4146e5609",
"GRAFANA_BACKEND_LEASE_MISMATCH",
);
stage = "OWN_STAND_IDLE";
requireThat(
(
await run([
"docker",
"ps",
"--filter",
"label=com.docker.compose.project=mtuci-vp7-security",
"--format",
"{{.ID}}",
])
).out.trim() === "",
"GRAFANA_OWN_STAND_ALREADY_RUNNING",
);
stage = "CACHED_ARTIFACT_GUARD";
const cacheBytes = await readFile(resolve(root, cachePath)),
cache = JSON.parse(cacheBytes.toString());
requireThat(
cache.state === "PASS_OWN_GRAFANA_CACHE_NO_PULL" &&
cache.image_id ===
"sha256:1c88285f1dd391a62232c010e6c224846e6a73bdc583b3365be2eba614b7919e" &&
cache.own_alias === "mtuci-vp7-grafana-cache:13.3.1-1c88285f1dd3",
"GRAFANA_CACHE_RECEIPT_MISMATCH",
);
requireThat(
JSON.parse(
(await run(["docker", "image", "inspect", cache.own_alias])).out,
)[0].Id === cache.image_id,
"GRAFANA_IMAGE_DRIFT",
);
for (const [path, sha] of Object.entries(cache.copied_files))
requireThat(
hash(await readFile(resolve(local, "plugins/openobserve", path))) ===
sha,
"GRAFANA_PLUGIN_DRIFT",
);
stage = "PRIVATE_INITIALIZER_GUARDS";
const password = await readFile(
resolve(root, ".local-preflight-key"),
"utf8",
);
requireThat(
/^[a-f0-9]{64}$/.test(password),
"INVALID_PREFLIGHT_SESSION_KEY",
);
const initializer = await prepareGrafana(password);
stage = "OWN_CACHED_IMAGE_BINDING";
let environment = await readFile(resolve(root, ".env"), "utf8");
const line = "VP7_GRAFANA_IMAGE=" + cache.own_alias;
requireThat(
!environment.includes("VP7_GRAFANA_IMAGE=") ||
environment.split("\n").includes(line),
"GRAFANA_IMAGE_ENV_CONFLICT",
);
if (!environment.split("\n").includes(line))
await privateWrite(
resolve(root, ".env"),
environment + "\n" + line + "\n",
);
stage = "CURRENT_COMPOSE_GENERATION";
await writeTopology();
await compose(["config", "--quiet"]);
stage = "RETAINED_STORAGE_START";
started = true;
await compose([
"up",
"-d",
"--no-build",
"openobserve-gateway",
"openobserve",
]);
stage = "QUERY_ACCOUNT_NATIVE_READY";
let ready = false;
const deadline = Date.now() + 60000;
while (Date.now() < deadline) {
try {
if (
(
await openObserveRequest("/api/default/streams", {
password,
account: "query",
role: "grafana",
})
).status === 200
) {
ready = true;
break;
}
} catch (error) {
await Promise.allSettled([
privateWrite(
resolve(local, "diagnostics/grafana-storage-last-error.json"),
JSON.stringify({
stage,
message: error instanceof Error ? error.message : "UNKNOWN",
code: (error as { code?: unknown })?.code,
stack: error instanceof Error ? error.stack : "",
}),
),
]);
}
await Bun.sleep(500);
}
requireThat(ready, "GRAFANA_STORAGE_READY_TIMEOUT");
stage = "SAVED_SERVICE_ACCOUNT_AUTH_GUARDS";
const accountGuards = await provisionOpenObserve(password);
stage = "GRAFANA_NATIVE_FIRST_START";
await compose(["up", "-d", "--no-build", "grafana-gateway", "grafana"]);
stage = "PUBLISHED_HTTPS_GRAFANA_HEALTH";
let health: any;
const grafanaDeadline = Date.now() + 90000;
while (Date.now() < grafanaDeadline) {
try {
health = await grafanaRequest("/api/health");
if (health.status === 200 && health.body?.database === "ok") break;
} catch (error) {
await Promise.allSettled([
privateWrite(
resolve(local, "diagnostics/grafana-health-last-error.json"),
JSON.stringify({
stage,
message: error instanceof Error ? error.message : "UNKNOWN",
code: (error as { code?: unknown })?.code,
stack: error instanceof Error ? error.stack : "",
}),
),
]);
}
await Bun.sleep(500);
}
requireThat(
health?.status === 200 &&
health.body?.database === "ok" &&
health.body?.version === "13.3.1",
"GRAFANA_NATIVE_HEALTH_TIMEOUT",
);
stage = "CLIENT_NETWORK_DNS";
const dns = await compose([
"exec",
"-T",
"grafana",
"getent",
"hosts",
"openobserve-gateway",
]);
requireThat(
/\b\d+\.\d+\.\d+\.\d+\b/.test(dns.out),
"GRAFANA_OPENOBSERVE_DNS_FAILED",
);
stage = "NATIVE_DATASOURCE_PROVISIONING";
const datasourceUIDs = await readGrafanaDatasources(grafanaRequest);
stage = "MOUNTED_DASHBOARD_NATIVE_READ";
const dashboardResponse = await grafanaRequest(
"/api/dashboards/uid/messenger-security",
),
mounted = JSON.parse(
await readFile(
resolve(root, "infra/grafana/dashboards/messenger.json"),
"utf8",
),
);
requireThat(
dashboardResponse.status === 200 &&
dashboardResponse.body?.dashboard?.panels?.length === 19,
"GRAFANA_DASHBOARD_PANEL_COUNT_MISMATCH",
);
const history = JSON.parse(
await readFile(
resolve(
root,
"evidence/openobserve/persisted-readback-8fa17baa-a9f5-419c-bf24-e9ac820fdfbf.json",
),
"utf8",
),
);
const point = Math.floor(history.query_time_milliseconds);
requireThat(
Number.isSafeInteger(point) && point > 0,
"GRAFANA_HISTORY_POINT_INVALID",
);
stage = withIncidents
? "REAL_WORKER_DATABASE_INCIDENTS"
: "HISTORICAL_METRIC_READS";
const incidents = withIncidents
? await runGrafanaIncidents(password)
: undefined;
const panels = incidents?.live_panels.panels ?? [];
const browserProof = withBrowser ? await runGrafanaBrowser() : undefined;
for (const panel of withIncidents || withBrowser ? [] : mounted.panels) {
stage = "HISTORICAL_METRIC_PANEL_" + panel.id;
const response = await grafanaRequest("/api/ds/query", {
body: {
from: String(point - 15 * 60000),
to: String(point),
queries: [
{
refId: "A",
datasource: { type: "prometheus", uid: "prometheus" },
expr: panel.targets[0].expr,
instant: true,
range: false,
intervalMs: 1000,
maxDataPoints: 1000,
},
],
},
});
requireThat(
response.status === 200,
"GRAFANA_METRIC_HTTP_STATUS_" + response.status,
);
const values = metricFrameValues(response.body);
panels.push({
id: panel.id,
title: panel.title,
samples: values.length,
values,
});
}
stage = "NATIVE_RULE_PROVISIONING";
const rules = await grafanaRequest("/api/v1/provisioning/alert-rules");
requireThat(
rules.status === 200 &&
Array.isArray(rules.body) &&
rules.body.length === 2 &&
rules.body.every((r: any) =>
["vp7-server-errors", "vp7-old-jobs"].includes(r.uid),
),
"GRAFANA_NATIVE_RULES_MISMATCH",
);
stage = "SOURCE_AND_ARTIFACT_POSTGUARDS";
await lease.verify();
requireThat(
hash(await readFile(resolve(root, cachePath))) === hash(cacheBytes),
"GRAFANA_CACHE_LEASE_CHANGED",
);
result = {
state: withBrowser
? "OBSERVED_GRAFANA_BROWSER_PENDING_FINAL_ACCEPTANCE"
: withIncidents
? "PASS_NATIVE_GRAFANA_LIVE_19_METRICS_AND_REAL_ALERT_INCIDENTS"
: "PASS_NATIVE_GRAFANA_BOOTSTRAP_AND_19_HISTORICAL_METRICS",
stage,
backend_handover_sha256: lease.sha256,
cache_receipt: { path: cachePath, sha256: hash(cacheBytes) },
initializer,
accountGuards,
grafana_version: health.body.version,
grafana_build_commit: health.body.commit,
datasource_uids: datasourceUIDs,
panels,
rule_uids: rules.body.map((r: any) => r.uid),
metric_query_time_milliseconds:
incidents?.live_panels.query_time_milliseconds ?? point,
retained_trace_history_query_time_milliseconds: point,
producer_started: withIncidents,
incidents,
browser: browserProof,
smoke_repeated: false,
native_grafana_bootstrap: true,
native_grafana_ui: false,
alert_transitions_accepted: withIncidents,
response_cap_accepted: false,
native_bao: false,
grade5: false,
};
await writeFile(receipt, JSON.stringify(result, null, 2) + "\n");
console.log(
withBrowser
? "OBSERVED_BROWSER_NOT_FINAL_ACCEPTANCE"
: withIncidents
? "PASS_GRAFANA_LIVE_19_METRICS_REAL_INCIDENTS_NO_UI_OR_CAP_ACCEPTANCE"
: "PASS_GRAFANA_BOOTSTRAP_19_METRICS_NO_UI_OR_ALERT_ACCEPTANCE",
);
} catch (error) {
primary = error;
result = {
state: "FAILED",
stage,
code:
error instanceof Error &&
/^[A-Z][A-Za-z0-9_:-]{0,240}$/.test(error.message)
? error.message
: "PRIVATE_DETAILS_ONLY",
native_bao: false,
grade5: false,
};
await Promise.allSettled([
writeFile(receipt, JSON.stringify(result, null, 2) + "\n"),
]);
throw error;
} finally {
if (started) {
const stopped = await Promise.allSettled([
compose(
withIncidents
? ["stop"]
: [
"stop",
"grafana",
"grafana-gateway",
"openobserve",
"openobserve-gateway",
],
{ allowFailure: true },
),
]);
const code =
stopped[0].status === "fulfilled" ? stopped[0].value.code : null;
await Promise.allSettled([
writeFile(
receipt,
JSON.stringify(
{
...result,
stop_exit: code,
...(code !== 0 && !primary ? { state: "FAILED_TEARDOWN" } : {}),
},
null,
2,
) + "\n",
),
]);
if (code !== 0 && !primary) throw Error("GRAFANA_PREFLIGHT_STOP_FAILED");
console.log(
code === 0
? "OWN_GRAFANA_PREFLIGHT_STOPPED"
: "OWN_GRAFANA_STOP_REQUIRES_INSPECTION",
);
}
}
}
if (import.meta.main) {
try {
await main();
} catch (error) {
safeFailure(error);
}
}