forked from mtusi2026/mtusi2026lab7
1038 lines
37 KiB
TypeScript
1038 lines
37 KiB
TypeScript
import { chromium, type BrowserContext } from "playwright-core";
|
|
import { readFile, writeFile, mkdir, mkdtemp } from "node:fs/promises";
|
|
import { resolve } from "node:path";
|
|
import { X509Certificate, createHash } from "node:crypto";
|
|
import { local, root, requireThat, run } from "./runtime";
|
|
import {
|
|
grafanaAddress,
|
|
metricFrameValues,
|
|
isOwnHistoryWrite,
|
|
} from "./grafana";
|
|
import { createBrowserLifecycle } from "./browser-lifecycle";
|
|
|
|
export async function runGrafanaBrowser() {
|
|
const linksOnly = Bun.argv.includes("--links-only");
|
|
const historyOnly = Bun.argv.includes("--history-only");
|
|
requireThat(
|
|
!(linksOnly && historyOnly),
|
|
"GRAFANA_BROWSER_CASE_SELECTION_CONFLICT",
|
|
);
|
|
const id = crypto.randomUUID(),
|
|
privateDir = resolve(local, "diagnostics/grafana-browser-" + id);
|
|
await mkdir(privateDir, { mode: 0o700 });
|
|
const receipt = resolve(root, "evidence/grafana/browser-" + id + ".json");
|
|
await writeFile(
|
|
receipt,
|
|
JSON.stringify({ state: "RUNNING", stage: "BROWSER_LAUNCH" }),
|
|
{ flag: "wx", mode: 0o600 },
|
|
);
|
|
const endpoint = await grafanaAddress(),
|
|
origin = `https://${endpoint.host}:${endpoint.port}`;
|
|
const cert = new X509Certificate(
|
|
await readFile(resolve(local, "pki/grafana/cert.pem")),
|
|
);
|
|
requireThat(
|
|
Date.parse(cert.validFrom) <= Date.now() &&
|
|
Date.now() < Date.parse(cert.validTo) &&
|
|
cert.checkIP(endpoint.host) === endpoint.host,
|
|
"GRAFANA_BROWSER_CERT_TIME_IDENTITY_INVALID",
|
|
);
|
|
const spki = createHash("sha256")
|
|
.update(cert.publicKey.export({ type: "spki", format: "der" }))
|
|
.digest("base64");
|
|
let stage = "BROWSER_LAUNCH";
|
|
const profile = await mkdtemp(resolve(local, "u-"));
|
|
requireThat(
|
|
Buffer.byteLength(profile + "/SingletonSocket") < 90,
|
|
"GRAFANA_BROWSER_PROFILE_PATH_TOO_LONG",
|
|
);
|
|
return await createBrowserLifecycle({
|
|
acquire: async (): Promise<BrowserContext> => {
|
|
try {
|
|
return await chromium.launchPersistentContext(profile, {
|
|
executablePath: "/usr/bin/chromium",
|
|
headless: true,
|
|
args: [
|
|
"--no-sandbox",
|
|
"--disable-gpu",
|
|
"--ignore-certificate-errors-spki-list=" + spki,
|
|
],
|
|
viewport: { width: 1600, height: 1100 },
|
|
colorScheme: "dark",
|
|
env: { ...process.env, TMPDIR: root },
|
|
});
|
|
} catch (error) {
|
|
await Promise.allSettled([
|
|
writeFile(
|
|
resolve(privateDir, "launch-failure.json"),
|
|
JSON.stringify({ error: String(error), profile }),
|
|
{ flag: "wx", mode: 0o600 },
|
|
),
|
|
writeFile(
|
|
receipt,
|
|
JSON.stringify({
|
|
state: "FAILED",
|
|
stage,
|
|
code: "GRAFANA_BROWSER_LAUNCH_FAILED",
|
|
}) + "\n",
|
|
),
|
|
]);
|
|
throw new Error("GRAFANA_BROWSER_LAUNCH_FAILED");
|
|
}
|
|
},
|
|
run: async (context) => {
|
|
stage = "BROWSER_PAGE_SETUP";
|
|
const page = await context.newPage();
|
|
page.setDefaultTimeout(20000);
|
|
const errors: string[] = [],
|
|
requests: unknown[] = [],
|
|
pending = new Set<Promise<void>>();
|
|
const renderedMetrics = new Map<
|
|
string,
|
|
{ samples: number; values: number[] }
|
|
>();
|
|
const metricNoData = new Set<string>();
|
|
const requestStages = new WeakMap<object, string>();
|
|
const historyEvents: { path: string; method: string; status: number }[] =
|
|
[];
|
|
stage = "BROWSER_LISTENER_SETUP";
|
|
page.on("request", (request) => requestStages.set(request, stage));
|
|
page.on("pageerror", (e) => errors.push(e.message));
|
|
page.on("console", (m) => {
|
|
if (["error", "warning"].includes(m.type()))
|
|
errors.push(m.type() + ":" + m.text() + ":" + m.location().url);
|
|
});
|
|
page.on("response", (response) => {
|
|
const resourcePath = new URL(response.url()).pathname;
|
|
if (
|
|
resourcePath.startsWith(
|
|
"/apis/userstorage.grafana.app/v0alpha1/namespaces/default/user-storage",
|
|
)
|
|
)
|
|
historyEvents.push({
|
|
path: resourcePath,
|
|
method: response.request().method(),
|
|
status: response.status(),
|
|
});
|
|
if (response.status() >= 400)
|
|
errors.push(
|
|
"HTTP_" +
|
|
response.status() +
|
|
":" +
|
|
new URL(response.url()).pathname,
|
|
);
|
|
if (!response.url().startsWith(origin + "/api/")) return;
|
|
const capture = (async () => {
|
|
const request = response.request(),
|
|
contentType = response.headers()["content-type"] ?? "",
|
|
requestStage = requestStages.get(request) ?? stage;
|
|
const body = await response.body();
|
|
const index = requests.length;
|
|
requests.push({
|
|
stage: requestStage,
|
|
path: new URL(response.url()).pathname,
|
|
status: response.status(),
|
|
content_type: contentType,
|
|
bytes: body.length,
|
|
request_body: request.postData(),
|
|
});
|
|
await writeFile(
|
|
resolve(privateDir, `response-${index}.json`),
|
|
JSON.stringify({
|
|
...(requests[index] as any),
|
|
...(body.length <= 1048576
|
|
? { body_base64: body.toString("base64") }
|
|
: {
|
|
body_sha256: createHash("sha256")
|
|
.update(body)
|
|
.digest("hex"),
|
|
}),
|
|
}),
|
|
{ flag: "wx", mode: 0o600 },
|
|
);
|
|
if (
|
|
requestStage === "UI_19_PANEL_DASHBOARD" &&
|
|
new URL(response.url()).pathname === "/api/ds/query" &&
|
|
response.status() === 200
|
|
) {
|
|
const input = JSON.parse(request.postData() ?? "{}"),
|
|
output = JSON.parse(body.toString());
|
|
for (const query of input.queries ?? [])
|
|
if (
|
|
query.datasource?.uid === "prometheus" &&
|
|
typeof query.expr === "string"
|
|
) {
|
|
try {
|
|
const values = metricFrameValues(output, query.refId);
|
|
renderedMetrics.set(query.expr, {
|
|
samples: values.length,
|
|
values,
|
|
});
|
|
} catch (error) {
|
|
if (
|
|
error instanceof Error &&
|
|
error.message === "GRAFANA_METRIC_NO_FINITE_SAMPLES"
|
|
)
|
|
metricNoData.add(query.expr);
|
|
else throw error;
|
|
}
|
|
}
|
|
}
|
|
})().catch((e) => {
|
|
errors.push("CAPTURE:" + String(e));
|
|
});
|
|
pending.add(capture);
|
|
capture.finally(() => pending.delete(capture));
|
|
});
|
|
try {
|
|
if (historyOnly) {
|
|
stage = "NATIVE_GRAFANA_TLS_NEGATIVES";
|
|
const controls = [];
|
|
for (const test of [
|
|
{
|
|
name: "positive_before",
|
|
host: endpoint.host,
|
|
ca: resolve(local, "pki/grafana/ca.crt"),
|
|
expected: 0,
|
|
},
|
|
{
|
|
name: "wrong_ca",
|
|
host: endpoint.host,
|
|
ca: resolve(local, "operator/foreign-ca/ca.crt"),
|
|
expected: 60,
|
|
},
|
|
{
|
|
name: "wrong_hostname",
|
|
host: "vp7-wrong.invalid",
|
|
ca: resolve(local, "pki/grafana/ca.crt"),
|
|
expected: 60,
|
|
},
|
|
{
|
|
name: "positive_after",
|
|
host: endpoint.host,
|
|
ca: resolve(local, "pki/grafana/ca.crt"),
|
|
expected: 0,
|
|
},
|
|
]) {
|
|
const output = resolve(privateDir, test.name + ".body");
|
|
await writeFile(output, "", { flag: "wx", mode: 0o600 });
|
|
const result = await run(
|
|
[
|
|
"curl",
|
|
"--silent",
|
|
"--show-error",
|
|
"--noproxy",
|
|
"*",
|
|
"--max-time",
|
|
"5",
|
|
"--cacert",
|
|
test.ca,
|
|
"--resolve",
|
|
`${test.host}:${endpoint.port}:${endpoint.host}`,
|
|
"--output",
|
|
output,
|
|
"--write-out",
|
|
"%{http_code}",
|
|
`https://${test.host}:${endpoint.port}/api/health`,
|
|
],
|
|
{ allowFailure: true },
|
|
);
|
|
controls.push({
|
|
name: test.name,
|
|
exit: result.code,
|
|
status: Number(result.out),
|
|
expected: test.expected,
|
|
});
|
|
requireThat(
|
|
result.code === test.expected &&
|
|
(test.expected !== 0 || result.out === "200"),
|
|
"GRAFANA_TLS_CONTROL_" + test.name.toUpperCase() + "_FAILED",
|
|
);
|
|
}
|
|
await writeFile(
|
|
resolve(root, "evidence/grafana/https-identity-" + id + ".json"),
|
|
JSON.stringify(
|
|
{
|
|
state: "PASS_NATIVE_GRAFANA_CA_HOSTNAME_CONTROLS",
|
|
controls,
|
|
valid_from: cert.validFrom,
|
|
valid_to: cert.validTo,
|
|
},
|
|
null,
|
|
2,
|
|
) + "\n",
|
|
{ flag: "wx", mode: 0o600 },
|
|
);
|
|
}
|
|
stage = "UI_LOGIN";
|
|
await page.goto(origin + "/login", { waitUntil: "domcontentloaded" });
|
|
const password = await readFile(
|
|
resolve(local, "operator/grafana-password"),
|
|
"utf8",
|
|
);
|
|
requireThat(
|
|
/^[a-f0-9]{64}$/.test(password),
|
|
"GRAFANA_BROWSER_PASSWORD_INVALID",
|
|
);
|
|
await page.locator('input[name="user"]').fill("operator");
|
|
await page.locator('input[name="password"]').fill(password);
|
|
await page.getByRole("button", { name: /^Log in$/i }).click();
|
|
await page.waitForURL((url) => url.pathname !== "/login", {
|
|
timeout: 30000,
|
|
});
|
|
const user = await page.evaluate(async () => {
|
|
const r = await fetch("/api/user");
|
|
return { status: r.status, data: await r.json() };
|
|
});
|
|
requireThat(
|
|
user.status === 200 &&
|
|
user.data.login === "operator" &&
|
|
user.data.isGrafanaAdmin === true,
|
|
"GRAFANA_BROWSER_SESSION_MISMATCH",
|
|
);
|
|
if (historyOnly) {
|
|
stage = "NORMAL_UI_PAGE_HISTORY_WRITE";
|
|
requireThat(
|
|
typeof user.data.uid === "string" &&
|
|
/^[a-z0-9]{8,40}$/.test(user.data.uid),
|
|
"GRAFANA_HISTORY_USER_UID_INVALID",
|
|
);
|
|
const collection =
|
|
"/apis/userstorage.grafana.app/v0alpha1/namespaces/default/user-storage",
|
|
name = "grafana-page-history:" + user.data.uid,
|
|
route = collection + "/" + name;
|
|
const written = page.waitForResponse(
|
|
(response) => {
|
|
const request = response.request(),
|
|
method = request.method();
|
|
try {
|
|
return isOwnHistoryWrite(
|
|
method,
|
|
new URL(response.url()).pathname,
|
|
JSON.parse(request.postData() ?? "{}"),
|
|
name,
|
|
);
|
|
} catch {
|
|
return false;
|
|
}
|
|
},
|
|
{ timeout: 45000 },
|
|
);
|
|
await page.goto(
|
|
origin +
|
|
"/d/messenger-security?from=1794800561224&to=1794801466224",
|
|
{ waitUntil: "domcontentloaded" },
|
|
);
|
|
await page
|
|
.getByText("Messenger lab7 security and pipeline", { exact: true })
|
|
.first()
|
|
.waitFor();
|
|
const writeResponse = await written;
|
|
requireThat(
|
|
[200, 201].includes(writeResponse.status()),
|
|
"GRAFANA_NORMAL_HISTORY_WRITE_FAILED",
|
|
);
|
|
stage = "NORMAL_UI_PAGE_HISTORY_RELOAD";
|
|
const reloaded = page.waitForResponse(
|
|
(response) =>
|
|
new URL(response.url()).pathname === route &&
|
|
response.request().method() === "GET",
|
|
{ timeout: 20000 },
|
|
);
|
|
await page.reload({ waitUntil: "domcontentloaded" });
|
|
const getResponse = await reloaded;
|
|
requireThat(
|
|
getResponse.status() === 200,
|
|
"GRAFANA_NORMAL_HISTORY_GET_FAILED",
|
|
);
|
|
const resource = await getResponse.json();
|
|
requireThat(
|
|
resource.metadata?.name === name,
|
|
"GRAFANA_NORMAL_HISTORY_RESOURCE_MISMATCH",
|
|
);
|
|
await Promise.allSettled([...pending]);
|
|
const expected = errors.filter(
|
|
(e) =>
|
|
e === "HTTP_404:" + route ||
|
|
e ===
|
|
"error:Failed to load resource: the server responded with a status of 404 (Not Found):" +
|
|
origin +
|
|
route,
|
|
);
|
|
const unexpected = errors.filter((e) => !expected.includes(e));
|
|
const proof = {
|
|
state:
|
|
unexpected.length === 0
|
|
? "PASS_NORMAL_UI_HISTORY_WRITE_READBACK"
|
|
: "FAILED_UNEXPECTED_BROWSER_EVENT",
|
|
stage,
|
|
route,
|
|
events: historyEvents,
|
|
write_status: writeResponse.status(),
|
|
get_status: getResponse.status(),
|
|
raw_diagnostic_events: errors.length,
|
|
expected_optional_state_events: expected.length,
|
|
unexpected_events: unexpected.length,
|
|
script_seeded: false,
|
|
producer_started: false,
|
|
native_bao: false,
|
|
};
|
|
await writeFile(receipt, JSON.stringify(proof, null, 2) + "\n");
|
|
await writeFile(
|
|
resolve(privateDir, "network.json"),
|
|
JSON.stringify({ errors, requests, historyEvents }, null, 2),
|
|
{ flag: "wx", mode: 0o600 },
|
|
);
|
|
requireThat(
|
|
unexpected.length === 0,
|
|
"GRAFANA_UNEXPECTED_BROWSER_EVENTS",
|
|
);
|
|
return proof;
|
|
}
|
|
const point = 1794801461224;
|
|
const dashboard = await Bun.file(
|
|
resolve(root, "infra/grafana/dashboards/messenger.json"),
|
|
).json();
|
|
if (!linksOnly) {
|
|
stage = "UI_19_PANEL_DASHBOARD";
|
|
await page.goto(
|
|
origin +
|
|
"/d/messenger-security?" +
|
|
new URLSearchParams({
|
|
from: String(point - 15 * 60000),
|
|
to: String(point + 5000),
|
|
timezone: "utc",
|
|
refresh: "",
|
|
}),
|
|
{ waitUntil: "domcontentloaded" },
|
|
);
|
|
for (const panel of dashboard.panels) {
|
|
await page
|
|
.getByText(panel.title, { exact: true })
|
|
.first()
|
|
.scrollIntoViewIfNeeded();
|
|
for (
|
|
const deadline = Date.now() + 20000;
|
|
Date.now() < deadline &&
|
|
!renderedMetrics.has(panel.targets[0].expr);
|
|
)
|
|
await page.waitForTimeout(100);
|
|
requireThat(
|
|
renderedMetrics.has(panel.targets[0].expr),
|
|
"GRAFANA_BROWSER_PANEL_" +
|
|
panel.id +
|
|
"_NO_FINITE_RENDERED_SAMPLES",
|
|
);
|
|
if ([1, 7, 13, 19].includes(panel.id))
|
|
await page.screenshot({
|
|
path: resolve(
|
|
privateDir,
|
|
"dashboard-panel-" + panel.id + ".png",
|
|
),
|
|
});
|
|
}
|
|
for (
|
|
const deadline = Date.now() + 20000;
|
|
Date.now() < deadline &&
|
|
!dashboard.panels.every((p: any) =>
|
|
renderedMetrics.has(p.targets[0].expr),
|
|
);
|
|
)
|
|
await page.waitForTimeout(100);
|
|
requireThat(
|
|
dashboard.panels.every((p: any) =>
|
|
renderedMetrics.has(p.targets[0].expr),
|
|
),
|
|
"GRAFANA_BROWSER_19_NATIVE_METRIC_CONTENT_MISSING",
|
|
);
|
|
await page.screenshot({
|
|
path: resolve(privateDir, "dashboard.png"),
|
|
fullPage: true,
|
|
});
|
|
await writeFile(
|
|
resolve(privateDir, "dashboard-dom.txt"),
|
|
await page.locator("body").innerText(),
|
|
{ flag: "wx", mode: 0o600 },
|
|
);
|
|
}
|
|
const traceId = "13aa5cd043d490330a867149fc7f4dbf",
|
|
history = Math.floor(1794785756292.7983);
|
|
const range = {
|
|
from: new Date(history - 15 * 60000).toISOString(),
|
|
to: new Date(history).toISOString(),
|
|
};
|
|
const explore = (query: unknown) =>
|
|
origin +
|
|
"/explore?" +
|
|
new URLSearchParams({
|
|
schemaVersion: "1",
|
|
panes: JSON.stringify({
|
|
A: { datasource: "openobserve", queries: [query], range },
|
|
}),
|
|
});
|
|
stage = "UI_NATIVE_TRACE_42";
|
|
await page.goto(
|
|
explore({
|
|
refId: "A",
|
|
datasource: { type: "openobserve", uid: "openobserve" },
|
|
queryType: "trace_id",
|
|
streamType: "traces",
|
|
displayMode: "trace",
|
|
traceId,
|
|
query: traceId,
|
|
stream: "messenger",
|
|
organization: "default",
|
|
sqlMode: false,
|
|
}),
|
|
{ waitUntil: "domcontentloaded" },
|
|
);
|
|
await page
|
|
.getByText("messenger-api", { exact: false })
|
|
.first()
|
|
.waitFor({ timeout: 45000 });
|
|
await page.screenshot({
|
|
path: resolve(privateDir, "trace.png"),
|
|
fullPage: true,
|
|
});
|
|
const traceText = await page.locator("body").innerText();
|
|
await writeFile(resolve(privateDir, "trace-dom.txt"), traceText, {
|
|
flag: "wx",
|
|
mode: 0o600,
|
|
});
|
|
requireThat(
|
|
/42\s*spans?/i.test(traceText),
|
|
"GRAFANA_BROWSER_TRACE_SPAN_COUNT_UNVERIFIED",
|
|
);
|
|
stage = "UI_CORRELATED_LOGS";
|
|
await page.goto(
|
|
explore({
|
|
refId: "A",
|
|
datasource: { type: "openobserve", uid: "openobserve" },
|
|
queryType: "logs",
|
|
streamType: "logs",
|
|
displayMode: "logs",
|
|
fastMode: true,
|
|
query: `SELECT * FROM "messenger" WHERE "trace_id" = '${traceId}'`,
|
|
stream: "messenger",
|
|
organization: "default",
|
|
sqlMode: true,
|
|
}),
|
|
{ waitUntil: "domcontentloaded" },
|
|
);
|
|
await page
|
|
.getByText("http.request", { exact: false })
|
|
.first()
|
|
.waitFor({ timeout: 45000 });
|
|
await page.screenshot({
|
|
path: resolve(privateDir, "logs.png"),
|
|
fullPage: true,
|
|
});
|
|
await writeFile(
|
|
resolve(privateDir, "logs-dom.txt"),
|
|
await page.locator("body").innerText(),
|
|
{ flag: "wx", mode: 0o600 },
|
|
);
|
|
if (!linksOnly) {
|
|
stage = "NATIVE_OMITTED_SIZE_HISTOGRAM";
|
|
const histogram = await page.evaluate(
|
|
async ({ traceId }) => {
|
|
const query = {
|
|
sql: `SELECT histogram("_timestamp") AS zo_sql_key, count(*) AS zo_sql_num FROM "messenger" WHERE "trace_id"='${traceId}' GROUP BY zo_sql_key ORDER BY zo_sql_key`,
|
|
start_time: 1794785659000000,
|
|
end_time: 1794785661000000,
|
|
from: 0,
|
|
sql_mode: "full",
|
|
};
|
|
const results = [];
|
|
for (const [name, size, expected] of [
|
|
["omitted", undefined, 200],
|
|
["zero", 0, 200],
|
|
["one", 1, 200],
|
|
["negative", -1, 400],
|
|
["too_large", 1001, 400],
|
|
["fraction", 1.5, 422],
|
|
["string", "1", 422],
|
|
["null", null, 422],
|
|
["object", {}, 422],
|
|
] as const) {
|
|
const response = await fetch(
|
|
"/api/datasources/proxy/uid/openobserve/api/default/_search?type=logs",
|
|
{
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({
|
|
query: size === undefined ? query : { ...query, size },
|
|
}),
|
|
},
|
|
);
|
|
const text = await response.text();
|
|
let data: any;
|
|
try {
|
|
data = JSON.parse(text);
|
|
} catch {}
|
|
results.push({
|
|
name,
|
|
status: response.status,
|
|
content_type: response.headers.get("content-type"),
|
|
bytes: new TextEncoder().encode(text).length,
|
|
buckets: data?.hits?.length,
|
|
count: data?.hits?.reduce(
|
|
(n: number, row: any) => n + Number(row.zo_sql_num),
|
|
0,
|
|
),
|
|
expected,
|
|
});
|
|
}
|
|
return { query, results };
|
|
},
|
|
{ traceId },
|
|
);
|
|
const histogramPass =
|
|
histogram.results.every((r) => r.status === r.expected) &&
|
|
histogram.results[0].count === 3 &&
|
|
histogram.results[1].count === 3;
|
|
await writeFile(
|
|
resolve(root, "evidence/grafana/histogram-" + id + ".json"),
|
|
JSON.stringify(
|
|
{
|
|
state: histogramPass
|
|
? "PASS_NATIVE_HISTOGRAM_REQUEST_BOUNDARY"
|
|
: "FAILED",
|
|
...histogram,
|
|
producer_started: false,
|
|
},
|
|
null,
|
|
2,
|
|
) + "\n",
|
|
{ flag: "wx", mode: 0o600 },
|
|
);
|
|
requireThat(histogramPass, "GRAFANA_NATIVE_HISTOGRAM_FAILED");
|
|
}
|
|
stage = "UI_LOG_DETAILS_CORRELATION";
|
|
await page.getByText("http.request", { exact: false }).first().click();
|
|
await writeFile(
|
|
resolve(privateDir, "log-details-dom.txt"),
|
|
await page.locator("body").innerText(),
|
|
{ flag: "wx", mode: 0o600 },
|
|
);
|
|
await writeFile(
|
|
resolve(privateDir, "log-details-elements.json"),
|
|
JSON.stringify(
|
|
await page.locator("a,button").evaluateAll((elements) =>
|
|
elements.map((e) => ({
|
|
tag: e.tagName,
|
|
text: e.textContent,
|
|
aria: e.getAttribute("aria-label"),
|
|
title: e.getAttribute("title"),
|
|
href: e.getAttribute("href"),
|
|
})),
|
|
),
|
|
),
|
|
{ flag: "wx", mode: 0o600 },
|
|
);
|
|
let capPass = false;
|
|
if (!linksOnly) {
|
|
stage = "ACTUAL_GRAFANA_RESPONSE_CONSUMERS";
|
|
const caps = await page.evaluate(
|
|
async ({ history }) => {
|
|
const output: {
|
|
name: string;
|
|
status: number;
|
|
content_type: string | null;
|
|
bytes: number;
|
|
hits?: number;
|
|
frames?: number;
|
|
error?: string;
|
|
boundary_marker: boolean;
|
|
}[] = [];
|
|
const query = async (
|
|
name: string,
|
|
backend: boolean,
|
|
large: boolean,
|
|
) => {
|
|
const sql = large
|
|
? `SELECT repeat('x',9437184) AS payload FROM "messenger" LIMIT 1`
|
|
: 'SELECT _timestamp,event,trace_id FROM "messenger" LIMIT 1';
|
|
const body = backend
|
|
? {
|
|
from: String(history - 15 * 60000),
|
|
to: String(history),
|
|
queries: [
|
|
{
|
|
refId: "A",
|
|
datasource: {
|
|
type: "openobserve",
|
|
uid: "openobserve",
|
|
},
|
|
queryType: "logs",
|
|
streamType: "logs",
|
|
stream: "messenger",
|
|
organization: "default",
|
|
sqlMode: true,
|
|
query: sql,
|
|
size: 1,
|
|
displayMode: "logs",
|
|
startTimeInMicro: (history - 15 * 60000) * 1000,
|
|
endTimeInMicro: history * 1000,
|
|
intervalMs: 1000,
|
|
maxDataPoints: 1000,
|
|
},
|
|
],
|
|
}
|
|
: {
|
|
query: {
|
|
sql,
|
|
start_time: (history - 15 * 60000) * 1000,
|
|
end_time: history * 1000,
|
|
from: 0,
|
|
size: 1,
|
|
sql_mode: "full",
|
|
},
|
|
};
|
|
const response = await fetch(
|
|
backend
|
|
? "/api/ds/query"
|
|
: "/api/datasources/proxy/uid/openobserve/api/default/_search?type=logs",
|
|
{
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify(body),
|
|
},
|
|
);
|
|
const text = await response.text();
|
|
let data: any;
|
|
try {
|
|
data = JSON.parse(text);
|
|
} catch {}
|
|
output.push({
|
|
name,
|
|
status: response.status,
|
|
content_type: response.headers.get("content-type"),
|
|
bytes: new TextEncoder().encode(text).length,
|
|
hits: data?.hits?.length,
|
|
frames: data?.results?.A?.frames?.length,
|
|
error: data?.results?.A?.error
|
|
? String(data.results.A.error).includes(
|
|
"response body too large",
|
|
)
|
|
? "RESPONSE_LIMIT"
|
|
: "NATIVE_QUERY_ERROR"
|
|
: undefined,
|
|
boundary_marker: data?.error === "query_response_boundary",
|
|
});
|
|
};
|
|
await query("browser_proxy_positive", false, false);
|
|
await query("plugin_backend_positive", true, false);
|
|
await query("browser_proxy_over_8mib", false, true);
|
|
await query("plugin_backend_over_8mib", true, true);
|
|
return output;
|
|
},
|
|
{ history },
|
|
);
|
|
capPass =
|
|
caps[0].status === 200 &&
|
|
caps[0].hits === 1 &&
|
|
caps[1].status === 200 &&
|
|
(caps[1].frames ?? 0) > 0 &&
|
|
caps[2].status === 502 &&
|
|
caps[2].bytes === 35 &&
|
|
caps[2].boundary_marker &&
|
|
caps[3].status >= 400 &&
|
|
caps[3].bytes < 2048 &&
|
|
!!caps[3].error;
|
|
await writeFile(
|
|
resolve(
|
|
root,
|
|
"evidence/grafana/response-consumers-" + id + ".json",
|
|
),
|
|
JSON.stringify(
|
|
{
|
|
state: capPass
|
|
? "PASS_ACTUAL_NATIVE_GRAFANA_RESPONSE_CONSUMERS_8MIB"
|
|
: "FAILED_ACTUAL_RESPONSE_BOUNDARY",
|
|
cap_bytes: 8388608,
|
|
results: caps,
|
|
producer_started: false,
|
|
},
|
|
null,
|
|
2,
|
|
) + "\n",
|
|
{ flag: "wx", mode: 0o600 },
|
|
);
|
|
requireThat(capPass, "GRAFANA_ACTUAL_RESPONSE_BOUNDARY_FAILED");
|
|
}
|
|
stage = "UI_LOG_TO_TRACE_CLICK";
|
|
const traceResponsePromise = page.waitForResponse((response) => {
|
|
if (
|
|
!response
|
|
.url()
|
|
.includes(
|
|
"/api/datasources/proxy/uid/openobserve/api/default/_search",
|
|
)
|
|
)
|
|
return false;
|
|
try {
|
|
const q = JSON.parse(response.request().postData() ?? "{}").query;
|
|
return (
|
|
q?.sql ===
|
|
`SELECT * FROM "messenger" WHERE "trace_id" = '${traceId}' ORDER BY "_timestamp" ASC`
|
|
);
|
|
} catch {
|
|
return false;
|
|
}
|
|
});
|
|
await page
|
|
.getByRole("link", { name: "View trace " + traceId, exact: true })
|
|
.click();
|
|
const traceResponse = await traceResponsePromise;
|
|
requireThat(
|
|
traceResponse.status() === 200,
|
|
"GRAFANA_LINK_TRACE_HTTP_FAILED",
|
|
);
|
|
const traceBody = await traceResponse.json(),
|
|
traceRows = traceBody.hits;
|
|
requireThat(
|
|
Array.isArray(traceRows) &&
|
|
traceRows.length === 42 &&
|
|
traceRows.every((r: any) => r.trace_id === traceId),
|
|
"GRAFANA_LINK_TRACE_ROWS_INVALID",
|
|
);
|
|
const selected = traceRows.filter(
|
|
(r: any) =>
|
|
r.service_name === "messenger-worker" &&
|
|
r.operation_name === "attachment.process",
|
|
);
|
|
requireThat(
|
|
selected.length === 1 && /^[a-f0-9]{16}$/.test(selected[0].span_id),
|
|
"GRAFANA_SELECTED_SPAN_INVALID",
|
|
);
|
|
const span = selected[0];
|
|
const parents = traceRows.filter(
|
|
(r: any) =>
|
|
r.reference_parent_span_id &&
|
|
r.reference_parent_span_id !== "0000000000000000",
|
|
);
|
|
requireThat(
|
|
parents.length === 41 &&
|
|
parents.every((r: any) =>
|
|
traceRows.some(
|
|
(p: any) => p.span_id === r.reference_parent_span_id,
|
|
),
|
|
),
|
|
"GRAFANA_TRACE_PARENT_CHAIN_INVALID",
|
|
);
|
|
await page
|
|
.getByText("42 spans", { exact: true })
|
|
.first()
|
|
.waitFor({ timeout: 30000 });
|
|
await page.screenshot({
|
|
path: resolve(privateDir, "log-to-trace.png"),
|
|
fullPage: true,
|
|
});
|
|
await writeFile(
|
|
resolve(privateDir, "log-to-trace-dom.txt"),
|
|
await page.locator("body").innerText(),
|
|
{ flag: "wx", mode: 0o600 },
|
|
);
|
|
stage = "UI_TRACE_TO_LOGS_SPAN_DETAILS";
|
|
await page
|
|
.getByText("attachment.process", { exact: false })
|
|
.first()
|
|
.click();
|
|
await writeFile(
|
|
resolve(privateDir, "span-details-dom.txt"),
|
|
await page.locator("body").innerText(),
|
|
{ flag: "wx", mode: 0o600 },
|
|
);
|
|
await writeFile(
|
|
resolve(privateDir, "span-details-elements.json"),
|
|
JSON.stringify(
|
|
await page.locator("a,button").evaluateAll((elements) =>
|
|
elements.map((e) => ({
|
|
tag: e.tagName,
|
|
text: e.textContent,
|
|
aria: e.getAttribute("aria-label"),
|
|
href: e.getAttribute("href"),
|
|
})),
|
|
),
|
|
),
|
|
{ flag: "wx", mode: 0o600 },
|
|
);
|
|
const reverseLink = page.getByRole("link", {
|
|
name: "View logs for trace " + traceId,
|
|
exact: true,
|
|
});
|
|
const href = await reverseLink.getAttribute("href");
|
|
requireThat(!!href, "GRAFANA_REVERSE_LINK_HREF_MISSING");
|
|
const target = new URL(href!, origin);
|
|
requireThat(
|
|
target.origin === origin && target.pathname === "/explore",
|
|
"GRAFANA_REVERSE_LINK_TARGET_UNSAFE",
|
|
);
|
|
const left = JSON.parse(target.searchParams.get("left") ?? "null");
|
|
const from = Number(left?.range?.from),
|
|
to = Number(left?.range?.to);
|
|
requireThat(
|
|
Number.isSafeInteger(from) &&
|
|
Number.isSafeInteger(to) &&
|
|
from < to &&
|
|
from === Math.floor(Number(span.start_time) / 1e6) &&
|
|
to === Math.floor(Number(span.end_time) / 1e6),
|
|
"GRAFANA_SPAN_LINK_WINDOW_MISMATCH",
|
|
);
|
|
const beforeRows = await page
|
|
.getByText("worker.job", { exact: false })
|
|
.count();
|
|
const reverseResponsePromise = page.waitForResponse((response) => {
|
|
if (
|
|
!response
|
|
.url()
|
|
.includes(
|
|
"/api/datasources/proxy/uid/openobserve/api/default/_search",
|
|
)
|
|
)
|
|
return false;
|
|
try {
|
|
const q = JSON.parse(response.request().postData() ?? "{}").query;
|
|
return (
|
|
q?.sql === left.queries?.[0]?.query &&
|
|
q.start_time === from * 1000 &&
|
|
q.end_time === to * 1000
|
|
);
|
|
} catch {
|
|
return false;
|
|
}
|
|
});
|
|
await reverseLink.click();
|
|
const reverseResponse = await reverseResponsePromise;
|
|
requireThat(
|
|
reverseResponse.status() === 200,
|
|
"GRAFANA_REVERSE_LINK_HTTP_FAILED",
|
|
);
|
|
const reverseBody = await reverseResponse.json(),
|
|
logRows = reverseBody.hits;
|
|
requireThat(
|
|
Array.isArray(logRows) &&
|
|
logRows.length > 0 &&
|
|
logRows.every((r: any) => r.trace_id === traceId) &&
|
|
logRows.some(
|
|
(r: any) =>
|
|
r.span_id === span.span_id && r.event === "worker.job",
|
|
),
|
|
"GRAFANA_REVERSE_LINK_SPAN_CORRELATION_FAILED",
|
|
);
|
|
await page
|
|
.getByText("worker.job", { exact: false })
|
|
.nth(beforeRows)
|
|
.waitFor({ timeout: 30000 });
|
|
await page.screenshot({
|
|
path: resolve(privateDir, "trace-to-logs.png"),
|
|
fullPage: true,
|
|
});
|
|
await writeFile(
|
|
resolve(privateDir, "trace-to-logs-dom.txt"),
|
|
await page.locator("body").innerText(),
|
|
{ flag: "wx", mode: 0o600 },
|
|
);
|
|
await Promise.allSettled([...pending]);
|
|
await writeFile(
|
|
resolve(privateDir, "network.json"),
|
|
JSON.stringify({ errors, requests }, null, 2),
|
|
{ flag: "wx", mode: 0o600 },
|
|
);
|
|
const proof = {
|
|
state: linksOnly
|
|
? "PASS_NATIVE_LOG_TRACE_LINKS"
|
|
: "PASS_NATIVE_UI_19_METRICS_TRACE_LINKS_AND_RESPONSE_BOUNDARY",
|
|
stage,
|
|
session_login: "operator",
|
|
dashboard_titles: linksOnly ? 0 : dashboard.panels.length,
|
|
trace_id: traceId,
|
|
expected_spans: 42,
|
|
metric_window_end_milliseconds: point + 5000,
|
|
diagnostics: privateDir,
|
|
console_error_count: errors.length,
|
|
native_browser_metrics: linksOnly
|
|
? []
|
|
: dashboard.panels.map((p: any) => ({
|
|
id: p.id,
|
|
title: p.title,
|
|
expr: p.targets[0].expr,
|
|
...renderedMetrics.get(p.targets[0].expr),
|
|
})),
|
|
response_cap: capPass,
|
|
links: {
|
|
trace_spans: traceRows.length,
|
|
parent_relations: parents.length,
|
|
selected_span_id: span.span_id,
|
|
from_milliseconds: from,
|
|
to_milliseconds: to,
|
|
sql: left.queries[0].query,
|
|
response_status: reverseResponse.status(),
|
|
rows: logRows.map((r: any) => ({
|
|
trace_id: r.trace_id,
|
|
span_id: r.span_id,
|
|
event: r.event,
|
|
timestamp_microseconds: r._timestamp,
|
|
})),
|
|
new_rendered_worker_job_row: true,
|
|
},
|
|
producer_started: false,
|
|
native_bao: false,
|
|
};
|
|
await writeFile(receipt, JSON.stringify(proof, null, 2) + "\n");
|
|
return proof;
|
|
} catch (error) {
|
|
await Promise.allSettled([...pending]);
|
|
await Promise.allSettled([
|
|
page.screenshot({
|
|
path: resolve(privateDir, "failed.png"),
|
|
fullPage: true,
|
|
mask: [page.locator("input")],
|
|
}),
|
|
writeFile(
|
|
resolve(privateDir, "failure.json"),
|
|
JSON.stringify({
|
|
stage,
|
|
error: String(error),
|
|
errors,
|
|
requests,
|
|
renderedMetrics: [...renderedMetrics],
|
|
metricNoData: [...metricNoData],
|
|
}),
|
|
{ flag: "wx", mode: 0o600 },
|
|
),
|
|
Promise.resolve().then(async () =>
|
|
writeFile(
|
|
resolve(privateDir, "failed-dom.txt"),
|
|
await page.locator("body").innerText(),
|
|
{ flag: "wx", mode: 0o600 },
|
|
),
|
|
),
|
|
writeFile(
|
|
receipt,
|
|
JSON.stringify({
|
|
state: "FAILED",
|
|
stage,
|
|
code: "GRAFANA_BROWSER_FAILED_" + stage,
|
|
producer_started: false,
|
|
native_bao: false,
|
|
}) + "\n",
|
|
),
|
|
]);
|
|
throw new Error("GRAFANA_BROWSER_FAILED_" + stage);
|
|
}
|
|
},
|
|
close: async (context) => {
|
|
await context.close();
|
|
},
|
|
reportCleanupFailure: async (error) => {
|
|
await writeFile(
|
|
resolve(privateDir, "close-failure.json"),
|
|
JSON.stringify({ stage, error: String(error) }),
|
|
{ flag: "wx", mode: 0o600 },
|
|
);
|
|
},
|
|
})();
|
|
}
|