Files
mtusi2026lab7/scripts/grafana-browser.ts

1038 lines
37 KiB
TypeScript

import { chromium, type BrowserContext } from "playwright-core";
import { readFile, writeFile, mkdir, mkdtemp } from "node:fs/promises";
import { resolve } from "node:path";
import { X509Certificate, createHash } from "node:crypto";
import { local, root, requireThat, run } from "./runtime";
import {
grafanaAddress,
metricFrameValues,
isOwnHistoryWrite,
} from "./grafana";
import { createBrowserLifecycle } from "./browser-lifecycle";
export async function runGrafanaBrowser() {
const linksOnly = Bun.argv.includes("--links-only");
const historyOnly = Bun.argv.includes("--history-only");
requireThat(
!(linksOnly && historyOnly),
"GRAFANA_BROWSER_CASE_SELECTION_CONFLICT",
);
const id = crypto.randomUUID(),
privateDir = resolve(local, "diagnostics/grafana-browser-" + id);
await mkdir(privateDir, { mode: 0o700 });
const receipt = resolve(root, "evidence/grafana/browser-" + id + ".json");
await writeFile(
receipt,
JSON.stringify({ state: "RUNNING", stage: "BROWSER_LAUNCH" }),
{ flag: "wx", mode: 0o600 },
);
const endpoint = await grafanaAddress(),
origin = `https://${endpoint.host}:${endpoint.port}`;
const cert = new X509Certificate(
await readFile(resolve(local, "pki/grafana/cert.pem")),
);
requireThat(
Date.parse(cert.validFrom) <= Date.now() &&
Date.now() < Date.parse(cert.validTo) &&
cert.checkIP(endpoint.host) === endpoint.host,
"GRAFANA_BROWSER_CERT_TIME_IDENTITY_INVALID",
);
const spki = createHash("sha256")
.update(cert.publicKey.export({ type: "spki", format: "der" }))
.digest("base64");
let stage = "BROWSER_LAUNCH";
const profile = await mkdtemp(resolve(local, "u-"));
requireThat(
Buffer.byteLength(profile + "/SingletonSocket") < 90,
"GRAFANA_BROWSER_PROFILE_PATH_TOO_LONG",
);
return await createBrowserLifecycle({
acquire: async (): Promise<BrowserContext> => {
try {
return await chromium.launchPersistentContext(profile, {
executablePath: "/usr/bin/chromium",
headless: true,
args: [
"--no-sandbox",
"--disable-gpu",
"--ignore-certificate-errors-spki-list=" + spki,
],
viewport: { width: 1600, height: 1100 },
colorScheme: "dark",
env: { ...process.env, TMPDIR: root },
});
} catch (error) {
await Promise.allSettled([
writeFile(
resolve(privateDir, "launch-failure.json"),
JSON.stringify({ error: String(error), profile }),
{ flag: "wx", mode: 0o600 },
),
writeFile(
receipt,
JSON.stringify({
state: "FAILED",
stage,
code: "GRAFANA_BROWSER_LAUNCH_FAILED",
}) + "\n",
),
]);
throw new Error("GRAFANA_BROWSER_LAUNCH_FAILED");
}
},
run: async (context) => {
stage = "BROWSER_PAGE_SETUP";
const page = await context.newPage();
page.setDefaultTimeout(20000);
const errors: string[] = [],
requests: unknown[] = [],
pending = new Set<Promise<void>>();
const renderedMetrics = new Map<
string,
{ samples: number; values: number[] }
>();
const metricNoData = new Set<string>();
const requestStages = new WeakMap<object, string>();
const historyEvents: { path: string; method: string; status: number }[] =
[];
stage = "BROWSER_LISTENER_SETUP";
page.on("request", (request) => requestStages.set(request, stage));
page.on("pageerror", (e) => errors.push(e.message));
page.on("console", (m) => {
if (["error", "warning"].includes(m.type()))
errors.push(m.type() + ":" + m.text() + ":" + m.location().url);
});
page.on("response", (response) => {
const resourcePath = new URL(response.url()).pathname;
if (
resourcePath.startsWith(
"/apis/userstorage.grafana.app/v0alpha1/namespaces/default/user-storage",
)
)
historyEvents.push({
path: resourcePath,
method: response.request().method(),
status: response.status(),
});
if (response.status() >= 400)
errors.push(
"HTTP_" +
response.status() +
":" +
new URL(response.url()).pathname,
);
if (!response.url().startsWith(origin + "/api/")) return;
const capture = (async () => {
const request = response.request(),
contentType = response.headers()["content-type"] ?? "",
requestStage = requestStages.get(request) ?? stage;
const body = await response.body();
const index = requests.length;
requests.push({
stage: requestStage,
path: new URL(response.url()).pathname,
status: response.status(),
content_type: contentType,
bytes: body.length,
request_body: request.postData(),
});
await writeFile(
resolve(privateDir, `response-${index}.json`),
JSON.stringify({
...(requests[index] as any),
...(body.length <= 1048576
? { body_base64: body.toString("base64") }
: {
body_sha256: createHash("sha256")
.update(body)
.digest("hex"),
}),
}),
{ flag: "wx", mode: 0o600 },
);
if (
requestStage === "UI_19_PANEL_DASHBOARD" &&
new URL(response.url()).pathname === "/api/ds/query" &&
response.status() === 200
) {
const input = JSON.parse(request.postData() ?? "{}"),
output = JSON.parse(body.toString());
for (const query of input.queries ?? [])
if (
query.datasource?.uid === "prometheus" &&
typeof query.expr === "string"
) {
try {
const values = metricFrameValues(output, query.refId);
renderedMetrics.set(query.expr, {
samples: values.length,
values,
});
} catch (error) {
if (
error instanceof Error &&
error.message === "GRAFANA_METRIC_NO_FINITE_SAMPLES"
)
metricNoData.add(query.expr);
else throw error;
}
}
}
})().catch((e) => {
errors.push("CAPTURE:" + String(e));
});
pending.add(capture);
capture.finally(() => pending.delete(capture));
});
try {
if (historyOnly) {
stage = "NATIVE_GRAFANA_TLS_NEGATIVES";
const controls = [];
for (const test of [
{
name: "positive_before",
host: endpoint.host,
ca: resolve(local, "pki/grafana/ca.crt"),
expected: 0,
},
{
name: "wrong_ca",
host: endpoint.host,
ca: resolve(local, "operator/foreign-ca/ca.crt"),
expected: 60,
},
{
name: "wrong_hostname",
host: "vp7-wrong.invalid",
ca: resolve(local, "pki/grafana/ca.crt"),
expected: 60,
},
{
name: "positive_after",
host: endpoint.host,
ca: resolve(local, "pki/grafana/ca.crt"),
expected: 0,
},
]) {
const output = resolve(privateDir, test.name + ".body");
await writeFile(output, "", { flag: "wx", mode: 0o600 });
const result = await run(
[
"curl",
"--silent",
"--show-error",
"--noproxy",
"*",
"--max-time",
"5",
"--cacert",
test.ca,
"--resolve",
`${test.host}:${endpoint.port}:${endpoint.host}`,
"--output",
output,
"--write-out",
"%{http_code}",
`https://${test.host}:${endpoint.port}/api/health`,
],
{ allowFailure: true },
);
controls.push({
name: test.name,
exit: result.code,
status: Number(result.out),
expected: test.expected,
});
requireThat(
result.code === test.expected &&
(test.expected !== 0 || result.out === "200"),
"GRAFANA_TLS_CONTROL_" + test.name.toUpperCase() + "_FAILED",
);
}
await writeFile(
resolve(root, "evidence/grafana/https-identity-" + id + ".json"),
JSON.stringify(
{
state: "PASS_NATIVE_GRAFANA_CA_HOSTNAME_CONTROLS",
controls,
valid_from: cert.validFrom,
valid_to: cert.validTo,
},
null,
2,
) + "\n",
{ flag: "wx", mode: 0o600 },
);
}
stage = "UI_LOGIN";
await page.goto(origin + "/login", { waitUntil: "domcontentloaded" });
const password = await readFile(
resolve(local, "operator/grafana-password"),
"utf8",
);
requireThat(
/^[a-f0-9]{64}$/.test(password),
"GRAFANA_BROWSER_PASSWORD_INVALID",
);
await page.locator('input[name="user"]').fill("operator");
await page.locator('input[name="password"]').fill(password);
await page.getByRole("button", { name: /^Log in$/i }).click();
await page.waitForURL((url) => url.pathname !== "/login", {
timeout: 30000,
});
const user = await page.evaluate(async () => {
const r = await fetch("/api/user");
return { status: r.status, data: await r.json() };
});
requireThat(
user.status === 200 &&
user.data.login === "operator" &&
user.data.isGrafanaAdmin === true,
"GRAFANA_BROWSER_SESSION_MISMATCH",
);
if (historyOnly) {
stage = "NORMAL_UI_PAGE_HISTORY_WRITE";
requireThat(
typeof user.data.uid === "string" &&
/^[a-z0-9]{8,40}$/.test(user.data.uid),
"GRAFANA_HISTORY_USER_UID_INVALID",
);
const collection =
"/apis/userstorage.grafana.app/v0alpha1/namespaces/default/user-storage",
name = "grafana-page-history:" + user.data.uid,
route = collection + "/" + name;
const written = page.waitForResponse(
(response) => {
const request = response.request(),
method = request.method();
try {
return isOwnHistoryWrite(
method,
new URL(response.url()).pathname,
JSON.parse(request.postData() ?? "{}"),
name,
);
} catch {
return false;
}
},
{ timeout: 45000 },
);
await page.goto(
origin +
"/d/messenger-security?from=1794800561224&to=1794801466224",
{ waitUntil: "domcontentloaded" },
);
await page
.getByText("Messenger lab7 security and pipeline", { exact: true })
.first()
.waitFor();
const writeResponse = await written;
requireThat(
[200, 201].includes(writeResponse.status()),
"GRAFANA_NORMAL_HISTORY_WRITE_FAILED",
);
stage = "NORMAL_UI_PAGE_HISTORY_RELOAD";
const reloaded = page.waitForResponse(
(response) =>
new URL(response.url()).pathname === route &&
response.request().method() === "GET",
{ timeout: 20000 },
);
await page.reload({ waitUntil: "domcontentloaded" });
const getResponse = await reloaded;
requireThat(
getResponse.status() === 200,
"GRAFANA_NORMAL_HISTORY_GET_FAILED",
);
const resource = await getResponse.json();
requireThat(
resource.metadata?.name === name,
"GRAFANA_NORMAL_HISTORY_RESOURCE_MISMATCH",
);
await Promise.allSettled([...pending]);
const expected = errors.filter(
(e) =>
e === "HTTP_404:" + route ||
e ===
"error:Failed to load resource: the server responded with a status of 404 (Not Found):" +
origin +
route,
);
const unexpected = errors.filter((e) => !expected.includes(e));
const proof = {
state:
unexpected.length === 0
? "PASS_NORMAL_UI_HISTORY_WRITE_READBACK"
: "FAILED_UNEXPECTED_BROWSER_EVENT",
stage,
route,
events: historyEvents,
write_status: writeResponse.status(),
get_status: getResponse.status(),
raw_diagnostic_events: errors.length,
expected_optional_state_events: expected.length,
unexpected_events: unexpected.length,
script_seeded: false,
producer_started: false,
native_bao: false,
};
await writeFile(receipt, JSON.stringify(proof, null, 2) + "\n");
await writeFile(
resolve(privateDir, "network.json"),
JSON.stringify({ errors, requests, historyEvents }, null, 2),
{ flag: "wx", mode: 0o600 },
);
requireThat(
unexpected.length === 0,
"GRAFANA_UNEXPECTED_BROWSER_EVENTS",
);
return proof;
}
const point = 1794801461224;
const dashboard = await Bun.file(
resolve(root, "infra/grafana/dashboards/messenger.json"),
).json();
if (!linksOnly) {
stage = "UI_19_PANEL_DASHBOARD";
await page.goto(
origin +
"/d/messenger-security?" +
new URLSearchParams({
from: String(point - 15 * 60000),
to: String(point + 5000),
timezone: "utc",
refresh: "",
}),
{ waitUntil: "domcontentloaded" },
);
for (const panel of dashboard.panels) {
await page
.getByText(panel.title, { exact: true })
.first()
.scrollIntoViewIfNeeded();
for (
const deadline = Date.now() + 20000;
Date.now() < deadline &&
!renderedMetrics.has(panel.targets[0].expr);
)
await page.waitForTimeout(100);
requireThat(
renderedMetrics.has(panel.targets[0].expr),
"GRAFANA_BROWSER_PANEL_" +
panel.id +
"_NO_FINITE_RENDERED_SAMPLES",
);
if ([1, 7, 13, 19].includes(panel.id))
await page.screenshot({
path: resolve(
privateDir,
"dashboard-panel-" + panel.id + ".png",
),
});
}
for (
const deadline = Date.now() + 20000;
Date.now() < deadline &&
!dashboard.panels.every((p: any) =>
renderedMetrics.has(p.targets[0].expr),
);
)
await page.waitForTimeout(100);
requireThat(
dashboard.panels.every((p: any) =>
renderedMetrics.has(p.targets[0].expr),
),
"GRAFANA_BROWSER_19_NATIVE_METRIC_CONTENT_MISSING",
);
await page.screenshot({
path: resolve(privateDir, "dashboard.png"),
fullPage: true,
});
await writeFile(
resolve(privateDir, "dashboard-dom.txt"),
await page.locator("body").innerText(),
{ flag: "wx", mode: 0o600 },
);
}
const traceId = "13aa5cd043d490330a867149fc7f4dbf",
history = Math.floor(1794785756292.7983);
const range = {
from: new Date(history - 15 * 60000).toISOString(),
to: new Date(history).toISOString(),
};
const explore = (query: unknown) =>
origin +
"/explore?" +
new URLSearchParams({
schemaVersion: "1",
panes: JSON.stringify({
A: { datasource: "openobserve", queries: [query], range },
}),
});
stage = "UI_NATIVE_TRACE_42";
await page.goto(
explore({
refId: "A",
datasource: { type: "openobserve", uid: "openobserve" },
queryType: "trace_id",
streamType: "traces",
displayMode: "trace",
traceId,
query: traceId,
stream: "messenger",
organization: "default",
sqlMode: false,
}),
{ waitUntil: "domcontentloaded" },
);
await page
.getByText("messenger-api", { exact: false })
.first()
.waitFor({ timeout: 45000 });
await page.screenshot({
path: resolve(privateDir, "trace.png"),
fullPage: true,
});
const traceText = await page.locator("body").innerText();
await writeFile(resolve(privateDir, "trace-dom.txt"), traceText, {
flag: "wx",
mode: 0o600,
});
requireThat(
/42\s*spans?/i.test(traceText),
"GRAFANA_BROWSER_TRACE_SPAN_COUNT_UNVERIFIED",
);
stage = "UI_CORRELATED_LOGS";
await page.goto(
explore({
refId: "A",
datasource: { type: "openobserve", uid: "openobserve" },
queryType: "logs",
streamType: "logs",
displayMode: "logs",
fastMode: true,
query: `SELECT * FROM "messenger" WHERE "trace_id" = '${traceId}'`,
stream: "messenger",
organization: "default",
sqlMode: true,
}),
{ waitUntil: "domcontentloaded" },
);
await page
.getByText("http.request", { exact: false })
.first()
.waitFor({ timeout: 45000 });
await page.screenshot({
path: resolve(privateDir, "logs.png"),
fullPage: true,
});
await writeFile(
resolve(privateDir, "logs-dom.txt"),
await page.locator("body").innerText(),
{ flag: "wx", mode: 0o600 },
);
if (!linksOnly) {
stage = "NATIVE_OMITTED_SIZE_HISTOGRAM";
const histogram = await page.evaluate(
async ({ traceId }) => {
const query = {
sql: `SELECT histogram("_timestamp") AS zo_sql_key, count(*) AS zo_sql_num FROM "messenger" WHERE "trace_id"='${traceId}' GROUP BY zo_sql_key ORDER BY zo_sql_key`,
start_time: 1794785659000000,
end_time: 1794785661000000,
from: 0,
sql_mode: "full",
};
const results = [];
for (const [name, size, expected] of [
["omitted", undefined, 200],
["zero", 0, 200],
["one", 1, 200],
["negative", -1, 400],
["too_large", 1001, 400],
["fraction", 1.5, 422],
["string", "1", 422],
["null", null, 422],
["object", {}, 422],
] as const) {
const response = await fetch(
"/api/datasources/proxy/uid/openobserve/api/default/_search?type=logs",
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
query: size === undefined ? query : { ...query, size },
}),
},
);
const text = await response.text();
let data: any;
try {
data = JSON.parse(text);
} catch {}
results.push({
name,
status: response.status,
content_type: response.headers.get("content-type"),
bytes: new TextEncoder().encode(text).length,
buckets: data?.hits?.length,
count: data?.hits?.reduce(
(n: number, row: any) => n + Number(row.zo_sql_num),
0,
),
expected,
});
}
return { query, results };
},
{ traceId },
);
const histogramPass =
histogram.results.every((r) => r.status === r.expected) &&
histogram.results[0].count === 3 &&
histogram.results[1].count === 3;
await writeFile(
resolve(root, "evidence/grafana/histogram-" + id + ".json"),
JSON.stringify(
{
state: histogramPass
? "PASS_NATIVE_HISTOGRAM_REQUEST_BOUNDARY"
: "FAILED",
...histogram,
producer_started: false,
},
null,
2,
) + "\n",
{ flag: "wx", mode: 0o600 },
);
requireThat(histogramPass, "GRAFANA_NATIVE_HISTOGRAM_FAILED");
}
stage = "UI_LOG_DETAILS_CORRELATION";
await page.getByText("http.request", { exact: false }).first().click();
await writeFile(
resolve(privateDir, "log-details-dom.txt"),
await page.locator("body").innerText(),
{ flag: "wx", mode: 0o600 },
);
await writeFile(
resolve(privateDir, "log-details-elements.json"),
JSON.stringify(
await page.locator("a,button").evaluateAll((elements) =>
elements.map((e) => ({
tag: e.tagName,
text: e.textContent,
aria: e.getAttribute("aria-label"),
title: e.getAttribute("title"),
href: e.getAttribute("href"),
})),
),
),
{ flag: "wx", mode: 0o600 },
);
let capPass = false;
if (!linksOnly) {
stage = "ACTUAL_GRAFANA_RESPONSE_CONSUMERS";
const caps = await page.evaluate(
async ({ history }) => {
const output: {
name: string;
status: number;
content_type: string | null;
bytes: number;
hits?: number;
frames?: number;
error?: string;
boundary_marker: boolean;
}[] = [];
const query = async (
name: string,
backend: boolean,
large: boolean,
) => {
const sql = large
? `SELECT repeat('x',9437184) AS payload FROM "messenger" LIMIT 1`
: 'SELECT _timestamp,event,trace_id FROM "messenger" LIMIT 1';
const body = backend
? {
from: String(history - 15 * 60000),
to: String(history),
queries: [
{
refId: "A",
datasource: {
type: "openobserve",
uid: "openobserve",
},
queryType: "logs",
streamType: "logs",
stream: "messenger",
organization: "default",
sqlMode: true,
query: sql,
size: 1,
displayMode: "logs",
startTimeInMicro: (history - 15 * 60000) * 1000,
endTimeInMicro: history * 1000,
intervalMs: 1000,
maxDataPoints: 1000,
},
],
}
: {
query: {
sql,
start_time: (history - 15 * 60000) * 1000,
end_time: history * 1000,
from: 0,
size: 1,
sql_mode: "full",
},
};
const response = await fetch(
backend
? "/api/ds/query"
: "/api/datasources/proxy/uid/openobserve/api/default/_search?type=logs",
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(body),
},
);
const text = await response.text();
let data: any;
try {
data = JSON.parse(text);
} catch {}
output.push({
name,
status: response.status,
content_type: response.headers.get("content-type"),
bytes: new TextEncoder().encode(text).length,
hits: data?.hits?.length,
frames: data?.results?.A?.frames?.length,
error: data?.results?.A?.error
? String(data.results.A.error).includes(
"response body too large",
)
? "RESPONSE_LIMIT"
: "NATIVE_QUERY_ERROR"
: undefined,
boundary_marker: data?.error === "query_response_boundary",
});
};
await query("browser_proxy_positive", false, false);
await query("plugin_backend_positive", true, false);
await query("browser_proxy_over_8mib", false, true);
await query("plugin_backend_over_8mib", true, true);
return output;
},
{ history },
);
capPass =
caps[0].status === 200 &&
caps[0].hits === 1 &&
caps[1].status === 200 &&
(caps[1].frames ?? 0) > 0 &&
caps[2].status === 502 &&
caps[2].bytes === 35 &&
caps[2].boundary_marker &&
caps[3].status >= 400 &&
caps[3].bytes < 2048 &&
!!caps[3].error;
await writeFile(
resolve(
root,
"evidence/grafana/response-consumers-" + id + ".json",
),
JSON.stringify(
{
state: capPass
? "PASS_ACTUAL_NATIVE_GRAFANA_RESPONSE_CONSUMERS_8MIB"
: "FAILED_ACTUAL_RESPONSE_BOUNDARY",
cap_bytes: 8388608,
results: caps,
producer_started: false,
},
null,
2,
) + "\n",
{ flag: "wx", mode: 0o600 },
);
requireThat(capPass, "GRAFANA_ACTUAL_RESPONSE_BOUNDARY_FAILED");
}
stage = "UI_LOG_TO_TRACE_CLICK";
const traceResponsePromise = page.waitForResponse((response) => {
if (
!response
.url()
.includes(
"/api/datasources/proxy/uid/openobserve/api/default/_search",
)
)
return false;
try {
const q = JSON.parse(response.request().postData() ?? "{}").query;
return (
q?.sql ===
`SELECT * FROM "messenger" WHERE "trace_id" = '${traceId}' ORDER BY "_timestamp" ASC`
);
} catch {
return false;
}
});
await page
.getByRole("link", { name: "View trace " + traceId, exact: true })
.click();
const traceResponse = await traceResponsePromise;
requireThat(
traceResponse.status() === 200,
"GRAFANA_LINK_TRACE_HTTP_FAILED",
);
const traceBody = await traceResponse.json(),
traceRows = traceBody.hits;
requireThat(
Array.isArray(traceRows) &&
traceRows.length === 42 &&
traceRows.every((r: any) => r.trace_id === traceId),
"GRAFANA_LINK_TRACE_ROWS_INVALID",
);
const selected = traceRows.filter(
(r: any) =>
r.service_name === "messenger-worker" &&
r.operation_name === "attachment.process",
);
requireThat(
selected.length === 1 && /^[a-f0-9]{16}$/.test(selected[0].span_id),
"GRAFANA_SELECTED_SPAN_INVALID",
);
const span = selected[0];
const parents = traceRows.filter(
(r: any) =>
r.reference_parent_span_id &&
r.reference_parent_span_id !== "0000000000000000",
);
requireThat(
parents.length === 41 &&
parents.every((r: any) =>
traceRows.some(
(p: any) => p.span_id === r.reference_parent_span_id,
),
),
"GRAFANA_TRACE_PARENT_CHAIN_INVALID",
);
await page
.getByText("42 spans", { exact: true })
.first()
.waitFor({ timeout: 30000 });
await page.screenshot({
path: resolve(privateDir, "log-to-trace.png"),
fullPage: true,
});
await writeFile(
resolve(privateDir, "log-to-trace-dom.txt"),
await page.locator("body").innerText(),
{ flag: "wx", mode: 0o600 },
);
stage = "UI_TRACE_TO_LOGS_SPAN_DETAILS";
await page
.getByText("attachment.process", { exact: false })
.first()
.click();
await writeFile(
resolve(privateDir, "span-details-dom.txt"),
await page.locator("body").innerText(),
{ flag: "wx", mode: 0o600 },
);
await writeFile(
resolve(privateDir, "span-details-elements.json"),
JSON.stringify(
await page.locator("a,button").evaluateAll((elements) =>
elements.map((e) => ({
tag: e.tagName,
text: e.textContent,
aria: e.getAttribute("aria-label"),
href: e.getAttribute("href"),
})),
),
),
{ flag: "wx", mode: 0o600 },
);
const reverseLink = page.getByRole("link", {
name: "View logs for trace " + traceId,
exact: true,
});
const href = await reverseLink.getAttribute("href");
requireThat(!!href, "GRAFANA_REVERSE_LINK_HREF_MISSING");
const target = new URL(href!, origin);
requireThat(
target.origin === origin && target.pathname === "/explore",
"GRAFANA_REVERSE_LINK_TARGET_UNSAFE",
);
const left = JSON.parse(target.searchParams.get("left") ?? "null");
const from = Number(left?.range?.from),
to = Number(left?.range?.to);
requireThat(
Number.isSafeInteger(from) &&
Number.isSafeInteger(to) &&
from < to &&
from === Math.floor(Number(span.start_time) / 1e6) &&
to === Math.floor(Number(span.end_time) / 1e6),
"GRAFANA_SPAN_LINK_WINDOW_MISMATCH",
);
const beforeRows = await page
.getByText("worker.job", { exact: false })
.count();
const reverseResponsePromise = page.waitForResponse((response) => {
if (
!response
.url()
.includes(
"/api/datasources/proxy/uid/openobserve/api/default/_search",
)
)
return false;
try {
const q = JSON.parse(response.request().postData() ?? "{}").query;
return (
q?.sql === left.queries?.[0]?.query &&
q.start_time === from * 1000 &&
q.end_time === to * 1000
);
} catch {
return false;
}
});
await reverseLink.click();
const reverseResponse = await reverseResponsePromise;
requireThat(
reverseResponse.status() === 200,
"GRAFANA_REVERSE_LINK_HTTP_FAILED",
);
const reverseBody = await reverseResponse.json(),
logRows = reverseBody.hits;
requireThat(
Array.isArray(logRows) &&
logRows.length > 0 &&
logRows.every((r: any) => r.trace_id === traceId) &&
logRows.some(
(r: any) =>
r.span_id === span.span_id && r.event === "worker.job",
),
"GRAFANA_REVERSE_LINK_SPAN_CORRELATION_FAILED",
);
await page
.getByText("worker.job", { exact: false })
.nth(beforeRows)
.waitFor({ timeout: 30000 });
await page.screenshot({
path: resolve(privateDir, "trace-to-logs.png"),
fullPage: true,
});
await writeFile(
resolve(privateDir, "trace-to-logs-dom.txt"),
await page.locator("body").innerText(),
{ flag: "wx", mode: 0o600 },
);
await Promise.allSettled([...pending]);
await writeFile(
resolve(privateDir, "network.json"),
JSON.stringify({ errors, requests }, null, 2),
{ flag: "wx", mode: 0o600 },
);
const proof = {
state: linksOnly
? "PASS_NATIVE_LOG_TRACE_LINKS"
: "PASS_NATIVE_UI_19_METRICS_TRACE_LINKS_AND_RESPONSE_BOUNDARY",
stage,
session_login: "operator",
dashboard_titles: linksOnly ? 0 : dashboard.panels.length,
trace_id: traceId,
expected_spans: 42,
metric_window_end_milliseconds: point + 5000,
diagnostics: privateDir,
console_error_count: errors.length,
native_browser_metrics: linksOnly
? []
: dashboard.panels.map((p: any) => ({
id: p.id,
title: p.title,
expr: p.targets[0].expr,
...renderedMetrics.get(p.targets[0].expr),
})),
response_cap: capPass,
links: {
trace_spans: traceRows.length,
parent_relations: parents.length,
selected_span_id: span.span_id,
from_milliseconds: from,
to_milliseconds: to,
sql: left.queries[0].query,
response_status: reverseResponse.status(),
rows: logRows.map((r: any) => ({
trace_id: r.trace_id,
span_id: r.span_id,
event: r.event,
timestamp_microseconds: r._timestamp,
})),
new_rendered_worker_job_row: true,
},
producer_started: false,
native_bao: false,
};
await writeFile(receipt, JSON.stringify(proof, null, 2) + "\n");
return proof;
} catch (error) {
await Promise.allSettled([...pending]);
await Promise.allSettled([
page.screenshot({
path: resolve(privateDir, "failed.png"),
fullPage: true,
mask: [page.locator("input")],
}),
writeFile(
resolve(privateDir, "failure.json"),
JSON.stringify({
stage,
error: String(error),
errors,
requests,
renderedMetrics: [...renderedMetrics],
metricNoData: [...metricNoData],
}),
{ flag: "wx", mode: 0o600 },
),
Promise.resolve().then(async () =>
writeFile(
resolve(privateDir, "failed-dom.txt"),
await page.locator("body").innerText(),
{ flag: "wx", mode: 0o600 },
),
),
writeFile(
receipt,
JSON.stringify({
state: "FAILED",
stage,
code: "GRAFANA_BROWSER_FAILED_" + stage,
producer_started: false,
native_bao: false,
}) + "\n",
),
]);
throw new Error("GRAFANA_BROWSER_FAILED_" + stage);
}
},
close: async (context) => {
await context.close();
},
reportCleanupFailure: async (error) => {
await writeFile(
resolve(privateDir, "close-failure.json"),
JSON.stringify({ stage, error: String(error) }),
{ flag: "wx", mode: 0o600 },
);
},
})();
}