forked from mtusi2026/mtusi2026lab3
375 lines
11 KiB
TypeScript
375 lines
11 KiB
TypeScript
import { resolve } from "node:path";
|
|
import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
|
|
import { assertNoGame } from "../scripts/k3s-model";
|
|
const root = resolve(import.meta.dir, "..");
|
|
if (process.argv.includes("--check")) {
|
|
console.log("Parsed isolated k3s/Pingap research");
|
|
process.exit(0);
|
|
}
|
|
const cluster =
|
|
"vp-lab3-r-" + crypto.randomUUID().replaceAll("-", "").slice(0, 8);
|
|
const base = "http://127.0.0.1:28303";
|
|
let env = {
|
|
...process.env,
|
|
VP_RUNTIME: "k3s",
|
|
VP_CLUSTER: cluster,
|
|
PUBLIC_PORT: "28303",
|
|
BASE_URL: base,
|
|
API_REPLICAS: "2",
|
|
API_CPUS: "1",
|
|
API_MEMORY: "512Mi",
|
|
GO_MEMORY_LIMIT: "400MiB",
|
|
PG_MAX_CONNS: "8",
|
|
};
|
|
const parent = resolve(root, "evidence/k3s/research");
|
|
await mkdir(parent, { recursive: true });
|
|
const evidence = await mkdtemp(resolve(parent, "run-"));
|
|
await writeFile(
|
|
resolve(evidence, "run.json"),
|
|
JSON.stringify({
|
|
cluster,
|
|
started_at: new Date().toISOString(),
|
|
state: "started",
|
|
}) + "\n",
|
|
{ flag: "wx" },
|
|
);
|
|
let stage = "preflight",
|
|
failureStage: string | undefined,
|
|
created = false,
|
|
primary: unknown;
|
|
const cleanupErrors: string[] = [];
|
|
let rolling: Promise<void> | undefined,
|
|
finished = false;
|
|
async function command(label: string, args: string[], input?: string) {
|
|
stage = label;
|
|
const p = Bun.spawn(args, {
|
|
cwd: root,
|
|
env,
|
|
stdin: input === undefined ? "ignore" : "pipe",
|
|
stdout: "pipe",
|
|
stderr: "pipe",
|
|
signal: AbortSignal.timeout(360000),
|
|
});
|
|
if (input !== undefined) {
|
|
p.stdin.write(input);
|
|
p.stdin.end();
|
|
}
|
|
const [out, err] = await Promise.all([
|
|
new Response(p.stdout).text(),
|
|
new Response(p.stderr).text(),
|
|
]);
|
|
if ((await p.exited) !== 0) {
|
|
await writeFile(resolve(root, ".local/research-error.txt"), out + err, {
|
|
mode: 0o600,
|
|
});
|
|
throw new Error(label + " failed");
|
|
}
|
|
return out;
|
|
}
|
|
function stand(args: string[]) {
|
|
return command("stand_" + args[0], [
|
|
process.execPath,
|
|
"--no-env-file",
|
|
"scripts/k3s.ts",
|
|
...args,
|
|
]);
|
|
}
|
|
function kube(args: string[], input?: string) {
|
|
return command(
|
|
"kubernetes_" + args[0],
|
|
[resolve(root, "scripts/kube.sh"), ...args],
|
|
input,
|
|
);
|
|
}
|
|
function sql(query: string) {
|
|
return kube(
|
|
[
|
|
"exec",
|
|
"-i",
|
|
"deployment/postgres",
|
|
"--",
|
|
"psql",
|
|
"-X",
|
|
"-qAt",
|
|
"-U",
|
|
"messenger",
|
|
"-d",
|
|
"messenger",
|
|
"-v",
|
|
"ON_ERROR_STOP=1",
|
|
],
|
|
query,
|
|
);
|
|
}
|
|
async function guard() {
|
|
const p = Bun.spawn(["game-running"], { stdout: "ignore", stderr: "pipe" });
|
|
await new Response(p.stderr).text();
|
|
assertNoGame(await p.exited);
|
|
}
|
|
const samples: any[] = [];
|
|
let rollingMetric: unknown;
|
|
const poolOnly = process.argv.includes("--pool-only");
|
|
try {
|
|
await guard();
|
|
created = true;
|
|
await stand(["up"]);
|
|
const hardware = JSON.parse(await command("hardware", ["lscpu", "-J"]));
|
|
await writeFile(
|
|
resolve(evidence, "hardware.json"),
|
|
JSON.stringify(
|
|
hardware.lscpu.filter((x: any) =>
|
|
/Model name|CPU\(s\)|Thread|Core\(s\)/.test(x.field),
|
|
),
|
|
null,
|
|
2,
|
|
) + "\n",
|
|
);
|
|
const cases: any[] = [];
|
|
if (!poolOnly)
|
|
for (const phase of ["fixed", "growing"])
|
|
for (const replicas of [1, 2, 4])
|
|
for (const repeat of [1, 2])
|
|
cases.push({ phase, replicas, repeat, pool: 8 });
|
|
for (const [index, pool] of [8, 1, 1, 8].entries())
|
|
cases.push({ phase: "pool", replicas: 1, repeat: index + 1, pool });
|
|
let previous = "";
|
|
for (const test of cases) {
|
|
await guard();
|
|
const fixed = test.phase !== "growing",
|
|
memory = fixed ? 2048 / test.replicas : 512,
|
|
cpu = fixed ? 2 / test.replicas : 1;
|
|
env = {
|
|
...env,
|
|
API_REPLICAS: String(test.replicas),
|
|
API_CPUS: String(cpu),
|
|
API_MEMORY: memory + "Mi",
|
|
GO_MEMORY_LIMIT: Math.floor(memory * 0.75) + "MiB",
|
|
PG_MAX_CONNS: String(test.pool),
|
|
};
|
|
const configuration = [test.replicas, cpu, memory, test.pool].join("-");
|
|
if (configuration !== previous) {
|
|
await stand(["configure"]);
|
|
previous = configuration;
|
|
}
|
|
// This cluster is exclusively owned by this experiment; no external clients are running.
|
|
await sql(
|
|
"TRUNCATE idempotency,messages,chat_members,chats,sessions,users;",
|
|
);
|
|
const before = JSON.parse(await stand(["stats"]));
|
|
const cpuValue = (s: string) =>
|
|
s.endsWith("m") ? Number(s.slice(0, -1)) / 1000 : Number(s);
|
|
const memoryValue = (s: string) =>
|
|
s.endsWith("Gi")
|
|
? Number(s.slice(0, -2)) * 1024
|
|
: Number(s.replace(/Mi$/, ""));
|
|
if (
|
|
before.pods.length !== test.replicas ||
|
|
before.pods.some(
|
|
(p: any) =>
|
|
cpuValue(p.limits.cpu) !== cpu ||
|
|
memoryValue(p.limits.memory) !== memory,
|
|
)
|
|
)
|
|
throw new Error("Actual API count or resource budget differs");
|
|
const response = await fetch(base + "/debug/stats", {
|
|
signal: AbortSignal.timeout(3000),
|
|
});
|
|
const actual = (await response.json()) as any;
|
|
if (response.status !== 200 || actual.store?.pool_max !== test.pool)
|
|
throw new Error("Actual SQL pool differs");
|
|
const name = [test.phase, test.replicas, test.pool, test.repeat].join("-"),
|
|
output = resolve(evidence, name + ".json");
|
|
const load = command("load_" + name, [
|
|
"python3",
|
|
"tests/load.py",
|
|
"--base-url",
|
|
base,
|
|
"--duration",
|
|
"5",
|
|
"--concurrency",
|
|
"16",
|
|
"--output",
|
|
output,
|
|
]);
|
|
const snapshot = (async () => {
|
|
await Bun.sleep(2000);
|
|
return JSON.parse(await stand(["stats"]));
|
|
})();
|
|
const waits = (async () => {
|
|
await Bun.sleep(1000);
|
|
const query =
|
|
"SELECT json_build_object('at',clock_timestamp(),'events',coalesce((SELECT json_agg(t) FROM (SELECT wait_event_type,wait_event,count(*) FROM pg_stat_activity WHERE datname=current_database() AND state='active' AND pid<>pg_backend_pid() GROUP BY wait_event_type,wait_event) t),'[]'::json))::jsonb::text;";
|
|
const text = await sql(
|
|
Array.from(
|
|
{ length: 10 },
|
|
() => query + "\nSELECT pg_sleep(0.2);",
|
|
).join("\n"),
|
|
);
|
|
return text
|
|
.split("\n")
|
|
.filter((x) => x.startsWith("{"))
|
|
.map((x) => JSON.parse(x));
|
|
})();
|
|
const results = await Promise.allSettled([load, snapshot, waits]);
|
|
for (const r of results) if (r.status === "rejected") throw r.reason;
|
|
await guard();
|
|
const metric = await Bun.file(output).json();
|
|
const snapshotValue = (results[1] as PromiseFulfilledResult<any>).value;
|
|
const messageCount = Number(
|
|
(await sql("SELECT count(*) FROM messages;")).trim(),
|
|
);
|
|
if (
|
|
!Number.isFinite(metric.successful_rps) ||
|
|
metric.successful_rps <= 0 ||
|
|
!Number.isFinite(metric.successful_latency_ms?.p95) ||
|
|
!Number.isInteger(messageCount) ||
|
|
messageCount < 1
|
|
)
|
|
throw new Error("No valid successful workload or persisted messages");
|
|
if (
|
|
snapshotValue.pods.length !== before.pods.length ||
|
|
snapshotValue.pods.some(
|
|
(p: any) => !before.pods.some((b: any) => b.uid === p.uid),
|
|
)
|
|
)
|
|
throw new Error("API pods changed during measurement");
|
|
samples.push({
|
|
...test,
|
|
api_cpu_each: cpu,
|
|
api_memory_mb_each: memory,
|
|
metrics: metric,
|
|
snapshot: snapshotValue,
|
|
postgres_wait_samples: (results[2] as PromiseFulfilledResult<any>).value,
|
|
messages: messageCount,
|
|
});
|
|
}
|
|
if (!poolOnly) {
|
|
env = {
|
|
...env,
|
|
API_REPLICAS: "2",
|
|
API_CPUS: "1",
|
|
API_MEMORY: "512Mi",
|
|
GO_MEMORY_LIMIT: "384MiB",
|
|
PG_MAX_CONNS: "8",
|
|
};
|
|
await stand(["configure"]);
|
|
await sql(
|
|
"TRUNCATE idempotency,messages,chat_members,chats,sessions,users;",
|
|
);
|
|
const before = JSON.parse(
|
|
await kube(["get", "pods", "-l", "app=api", "-o", "json"]),
|
|
).items.map((p: any) => ({ name: p.metadata.name, uid: p.metadata.uid }));
|
|
let loadFailure: unknown;
|
|
rolling = command("rolling_load", [
|
|
"python3",
|
|
"tests/load.py",
|
|
"--base-url",
|
|
base,
|
|
"--duration",
|
|
"30",
|
|
"--concurrency",
|
|
"8",
|
|
"--output",
|
|
resolve(evidence, "rolling.json"),
|
|
])
|
|
.then(
|
|
() => {},
|
|
(error) => {
|
|
loadFailure = error;
|
|
},
|
|
)
|
|
.finally(() => {
|
|
finished = true;
|
|
});
|
|
for (let i = 0; i < 60; i++) {
|
|
if (Number((await sql("SELECT count(*) FROM messages;")).trim()) > 10)
|
|
break;
|
|
if (finished) throw new Error("Rolling load ended before starting");
|
|
await Bun.sleep(100);
|
|
}
|
|
const countBefore = Number(
|
|
(await sql("SELECT count(*) FROM messages;")).trim(),
|
|
);
|
|
if (countBefore <= 10) throw new Error("Rolling load did not start");
|
|
await kube(["rollout", "restart", "deployment/api"]);
|
|
await stand(["ready"]);
|
|
const after = JSON.parse(
|
|
await kube(["get", "pods", "-l", "app=api", "-o", "json"]),
|
|
).items.map((p: any) => ({ name: p.metadata.name, uid: p.metadata.uid }));
|
|
const countAfter = Number(
|
|
(await sql("SELECT count(*) FROM messages;")).trim(),
|
|
);
|
|
if (
|
|
finished ||
|
|
countAfter <= countBefore ||
|
|
after.length !== 2 ||
|
|
after.some((p: any) => before.some((b: any) => b.uid === p.uid))
|
|
)
|
|
throw new Error("Rolling replacements not covered by continuing writes");
|
|
await writeFile(
|
|
resolve(evidence, "replacements.json"),
|
|
JSON.stringify(
|
|
{
|
|
before,
|
|
after,
|
|
messages_before: countBefore,
|
|
messages_after: countAfter,
|
|
strategy: { maxSurge: 1, maxUnavailable: 0 },
|
|
},
|
|
null,
|
|
2,
|
|
) + "\n",
|
|
);
|
|
await rolling;
|
|
if (loadFailure) throw loadFailure;
|
|
await guard();
|
|
rollingMetric = await Bun.file(resolve(evidence, "rolling.json")).json();
|
|
}
|
|
} catch (error) {
|
|
primary = error;
|
|
failureStage = stage;
|
|
} finally {
|
|
if (rolling) await rolling;
|
|
if (created)
|
|
try {
|
|
await stand(["destroy"]);
|
|
} catch {
|
|
cleanupErrors.push("isolated_cluster");
|
|
}
|
|
}
|
|
if (primary || cleanupErrors.length) {
|
|
await writeFile(
|
|
resolve(evidence, "failure.json"),
|
|
JSON.stringify({
|
|
stage: failureStage ?? stage,
|
|
primary:
|
|
primary instanceof Error ? primary.message : primary ? "unknown" : null,
|
|
cleanup_errors: cleanupErrors,
|
|
}) + "\n",
|
|
);
|
|
throw new Error("Research failed; inspect this run");
|
|
}
|
|
await writeFile(
|
|
resolve(evidence, "summary.json"),
|
|
JSON.stringify(
|
|
{
|
|
valid: true,
|
|
complete: true,
|
|
cluster,
|
|
runtime: "k3s1.37.1/Pingap0.16.0",
|
|
expected_samples: casesLength(),
|
|
profile:
|
|
"closed-loop POST text; concurrency16;5seconds;two repetitions;fresh data;single hot sender",
|
|
samples,
|
|
rolling: rollingMetric,
|
|
},
|
|
null,
|
|
2,
|
|
) + "\n",
|
|
{ flag: "wx" },
|
|
);
|
|
function casesLength() {
|
|
return poolOnly ? 4 : 16;
|
|
}
|
|
console.log("PASS k3s/Pingap research: " + evidence);
|