Files
mtusi2026lab3/student-tests/research.ts
T

375 lines
11 KiB
TypeScript

import { resolve } from "node:path";
import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
import { assertNoGame } from "../scripts/k3s-model";
const root = resolve(import.meta.dir, "..");
if (process.argv.includes("--check")) {
console.log("Parsed isolated k3s/Pingap research");
process.exit(0);
}
const cluster =
"vp-lab3-r-" + crypto.randomUUID().replaceAll("-", "").slice(0, 8);
const base = "http://127.0.0.1:28303";
let env = {
...process.env,
VP_RUNTIME: "k3s",
VP_CLUSTER: cluster,
PUBLIC_PORT: "28303",
BASE_URL: base,
API_REPLICAS: "2",
API_CPUS: "1",
API_MEMORY: "512Mi",
GO_MEMORY_LIMIT: "400MiB",
PG_MAX_CONNS: "8",
};
const parent = resolve(root, "evidence/k3s/research");
await mkdir(parent, { recursive: true });
const evidence = await mkdtemp(resolve(parent, "run-"));
await writeFile(
resolve(evidence, "run.json"),
JSON.stringify({
cluster,
started_at: new Date().toISOString(),
state: "started",
}) + "\n",
{ flag: "wx" },
);
let stage = "preflight",
failureStage: string | undefined,
created = false,
primary: unknown;
const cleanupErrors: string[] = [];
let rolling: Promise<void> | undefined,
finished = false;
async function command(label: string, args: string[], input?: string) {
stage = label;
const p = Bun.spawn(args, {
cwd: root,
env,
stdin: input === undefined ? "ignore" : "pipe",
stdout: "pipe",
stderr: "pipe",
signal: AbortSignal.timeout(360000),
});
if (input !== undefined) {
p.stdin.write(input);
p.stdin.end();
}
const [out, err] = await Promise.all([
new Response(p.stdout).text(),
new Response(p.stderr).text(),
]);
if ((await p.exited) !== 0) {
await writeFile(resolve(root, ".local/research-error.txt"), out + err, {
mode: 0o600,
});
throw new Error(label + " failed");
}
return out;
}
function stand(args: string[]) {
return command("stand_" + args[0], [
process.execPath,
"--no-env-file",
"scripts/k3s.ts",
...args,
]);
}
function kube(args: string[], input?: string) {
return command(
"kubernetes_" + args[0],
[resolve(root, "scripts/kube.sh"), ...args],
input,
);
}
function sql(query: string) {
return kube(
[
"exec",
"-i",
"deployment/postgres",
"--",
"psql",
"-X",
"-qAt",
"-U",
"messenger",
"-d",
"messenger",
"-v",
"ON_ERROR_STOP=1",
],
query,
);
}
async function guard() {
const p = Bun.spawn(["game-running"], { stdout: "ignore", stderr: "pipe" });
await new Response(p.stderr).text();
assertNoGame(await p.exited);
}
const samples: any[] = [];
let rollingMetric: unknown;
const poolOnly = process.argv.includes("--pool-only");
try {
await guard();
created = true;
await stand(["up"]);
const hardware = JSON.parse(await command("hardware", ["lscpu", "-J"]));
await writeFile(
resolve(evidence, "hardware.json"),
JSON.stringify(
hardware.lscpu.filter((x: any) =>
/Model name|CPU\(s\)|Thread|Core\(s\)/.test(x.field),
),
null,
2,
) + "\n",
);
const cases: any[] = [];
if (!poolOnly)
for (const phase of ["fixed", "growing"])
for (const replicas of [1, 2, 4])
for (const repeat of [1, 2])
cases.push({ phase, replicas, repeat, pool: 8 });
for (const [index, pool] of [8, 1, 1, 8].entries())
cases.push({ phase: "pool", replicas: 1, repeat: index + 1, pool });
let previous = "";
for (const test of cases) {
await guard();
const fixed = test.phase !== "growing",
memory = fixed ? 2048 / test.replicas : 512,
cpu = fixed ? 2 / test.replicas : 1;
env = {
...env,
API_REPLICAS: String(test.replicas),
API_CPUS: String(cpu),
API_MEMORY: memory + "Mi",
GO_MEMORY_LIMIT: Math.floor(memory * 0.75) + "MiB",
PG_MAX_CONNS: String(test.pool),
};
const configuration = [test.replicas, cpu, memory, test.pool].join("-");
if (configuration !== previous) {
await stand(["configure"]);
previous = configuration;
}
// This cluster is exclusively owned by this experiment; no external clients are running.
await sql(
"TRUNCATE idempotency,messages,chat_members,chats,sessions,users;",
);
const before = JSON.parse(await stand(["stats"]));
const cpuValue = (s: string) =>
s.endsWith("m") ? Number(s.slice(0, -1)) / 1000 : Number(s);
const memoryValue = (s: string) =>
s.endsWith("Gi")
? Number(s.slice(0, -2)) * 1024
: Number(s.replace(/Mi$/, ""));
if (
before.pods.length !== test.replicas ||
before.pods.some(
(p: any) =>
cpuValue(p.limits.cpu) !== cpu ||
memoryValue(p.limits.memory) !== memory,
)
)
throw new Error("Actual API count or resource budget differs");
const response = await fetch(base + "/debug/stats", {
signal: AbortSignal.timeout(3000),
});
const actual = (await response.json()) as any;
if (response.status !== 200 || actual.store?.pool_max !== test.pool)
throw new Error("Actual SQL pool differs");
const name = [test.phase, test.replicas, test.pool, test.repeat].join("-"),
output = resolve(evidence, name + ".json");
const load = command("load_" + name, [
"python3",
"tests/load.py",
"--base-url",
base,
"--duration",
"5",
"--concurrency",
"16",
"--output",
output,
]);
const snapshot = (async () => {
await Bun.sleep(2000);
return JSON.parse(await stand(["stats"]));
})();
const waits = (async () => {
await Bun.sleep(1000);
const query =
"SELECT json_build_object('at',clock_timestamp(),'events',coalesce((SELECT json_agg(t) FROM (SELECT wait_event_type,wait_event,count(*) FROM pg_stat_activity WHERE datname=current_database() AND state='active' AND pid<>pg_backend_pid() GROUP BY wait_event_type,wait_event) t),'[]'::json))::jsonb::text;";
const text = await sql(
Array.from(
{ length: 10 },
() => query + "\nSELECT pg_sleep(0.2);",
).join("\n"),
);
return text
.split("\n")
.filter((x) => x.startsWith("{"))
.map((x) => JSON.parse(x));
})();
const results = await Promise.allSettled([load, snapshot, waits]);
for (const r of results) if (r.status === "rejected") throw r.reason;
await guard();
const metric = await Bun.file(output).json();
const snapshotValue = (results[1] as PromiseFulfilledResult<any>).value;
const messageCount = Number(
(await sql("SELECT count(*) FROM messages;")).trim(),
);
if (
!Number.isFinite(metric.successful_rps) ||
metric.successful_rps <= 0 ||
!Number.isFinite(metric.successful_latency_ms?.p95) ||
!Number.isInteger(messageCount) ||
messageCount < 1
)
throw new Error("No valid successful workload or persisted messages");
if (
snapshotValue.pods.length !== before.pods.length ||
snapshotValue.pods.some(
(p: any) => !before.pods.some((b: any) => b.uid === p.uid),
)
)
throw new Error("API pods changed during measurement");
samples.push({
...test,
api_cpu_each: cpu,
api_memory_mb_each: memory,
metrics: metric,
snapshot: snapshotValue,
postgres_wait_samples: (results[2] as PromiseFulfilledResult<any>).value,
messages: messageCount,
});
}
if (!poolOnly) {
env = {
...env,
API_REPLICAS: "2",
API_CPUS: "1",
API_MEMORY: "512Mi",
GO_MEMORY_LIMIT: "384MiB",
PG_MAX_CONNS: "8",
};
await stand(["configure"]);
await sql(
"TRUNCATE idempotency,messages,chat_members,chats,sessions,users;",
);
const before = JSON.parse(
await kube(["get", "pods", "-l", "app=api", "-o", "json"]),
).items.map((p: any) => ({ name: p.metadata.name, uid: p.metadata.uid }));
let loadFailure: unknown;
rolling = command("rolling_load", [
"python3",
"tests/load.py",
"--base-url",
base,
"--duration",
"30",
"--concurrency",
"8",
"--output",
resolve(evidence, "rolling.json"),
])
.then(
() => {},
(error) => {
loadFailure = error;
},
)
.finally(() => {
finished = true;
});
for (let i = 0; i < 60; i++) {
if (Number((await sql("SELECT count(*) FROM messages;")).trim()) > 10)
break;
if (finished) throw new Error("Rolling load ended before starting");
await Bun.sleep(100);
}
const countBefore = Number(
(await sql("SELECT count(*) FROM messages;")).trim(),
);
if (countBefore <= 10) throw new Error("Rolling load did not start");
await kube(["rollout", "restart", "deployment/api"]);
await stand(["ready"]);
const after = JSON.parse(
await kube(["get", "pods", "-l", "app=api", "-o", "json"]),
).items.map((p: any) => ({ name: p.metadata.name, uid: p.metadata.uid }));
const countAfter = Number(
(await sql("SELECT count(*) FROM messages;")).trim(),
);
if (
finished ||
countAfter <= countBefore ||
after.length !== 2 ||
after.some((p: any) => before.some((b: any) => b.uid === p.uid))
)
throw new Error("Rolling replacements not covered by continuing writes");
await writeFile(
resolve(evidence, "replacements.json"),
JSON.stringify(
{
before,
after,
messages_before: countBefore,
messages_after: countAfter,
strategy: { maxSurge: 1, maxUnavailable: 0 },
},
null,
2,
) + "\n",
);
await rolling;
if (loadFailure) throw loadFailure;
await guard();
rollingMetric = await Bun.file(resolve(evidence, "rolling.json")).json();
}
} catch (error) {
primary = error;
failureStage = stage;
} finally {
if (rolling) await rolling;
if (created)
try {
await stand(["destroy"]);
} catch {
cleanupErrors.push("isolated_cluster");
}
}
if (primary || cleanupErrors.length) {
await writeFile(
resolve(evidence, "failure.json"),
JSON.stringify({
stage: failureStage ?? stage,
primary:
primary instanceof Error ? primary.message : primary ? "unknown" : null,
cleanup_errors: cleanupErrors,
}) + "\n",
);
throw new Error("Research failed; inspect this run");
}
await writeFile(
resolve(evidence, "summary.json"),
JSON.stringify(
{
valid: true,
complete: true,
cluster,
runtime: "k3s1.37.1/Pingap0.16.0",
expected_samples: casesLength(),
profile:
"closed-loop POST text; concurrency16;5seconds;two repetitions;fresh data;single hot sender",
samples,
rolling: rollingMetric,
},
null,
2,
) + "\n",
{ flag: "wx" },
);
function casesLength() {
return poolOnly ? 4 : 16;
}
console.log("PASS k3s/Pingap research: " + evidence);