forked from mtusi2026/mtusi2026lab3
1180 lines
33 KiB
TypeScript
1180 lines
33 KiB
TypeScript
import { mkdir, readFile, readdir, writeFile, rm } from "node:fs/promises";
|
|
import { resolve, relative } from "node:path";
|
|
import {
|
|
settings,
|
|
guardedOperator,
|
|
pinnedConfigID,
|
|
assertScopedArgs,
|
|
assertNoGame,
|
|
assertPartialClaim,
|
|
assertNoRunningContainers,
|
|
withCleanup,
|
|
assertImageBudget,
|
|
} from "./k3s-model";
|
|
|
|
const root = resolve(import.meta.dir, "..");
|
|
const lab = (await Bun.file(resolve(root, "lab.json")).json()).number;
|
|
const cfg = settings(lab, process.env);
|
|
const local = resolve(root, ".local/k3s", cfg.cluster),
|
|
kubeconfig = resolve(local, "kubeconfig"),
|
|
receiptFile = resolve(local, "receipt.json");
|
|
const evidence = resolve(root, "evidence/k3s");
|
|
await mkdir(local, { recursive: true, mode: 0o700 });
|
|
await mkdir(evidence, { recursive: true });
|
|
const action = process.argv[2] ?? "ready";
|
|
const args = process.argv.slice(3);
|
|
let stage = "start";
|
|
async function command(
|
|
label: string,
|
|
argv: string[],
|
|
input?: string,
|
|
timeout = 240000,
|
|
) {
|
|
stage = label;
|
|
const p = Bun.spawn(argv, {
|
|
cwd: root,
|
|
stdin: input === undefined ? "ignore" : "pipe",
|
|
stdout: "pipe",
|
|
stderr: "pipe",
|
|
signal: AbortSignal.timeout(timeout),
|
|
});
|
|
if (input !== undefined) {
|
|
p.stdin.write(input);
|
|
p.stdin.end();
|
|
}
|
|
const [out, err] = await Promise.all([
|
|
new Response(p.stdout).text(),
|
|
new Response(p.stderr).text(),
|
|
]);
|
|
if ((await p.exited) !== 0) {
|
|
await writeFile(resolve(local, "last-error.txt"), out + err, {
|
|
mode: 0o600,
|
|
});
|
|
throw new Error(label + " failed; private diagnostics in .local/k3s");
|
|
}
|
|
return out;
|
|
}
|
|
function k(argv: string[], input?: string) {
|
|
return command(
|
|
"kubernetes_" + argv[0],
|
|
[
|
|
"kubectl",
|
|
"--kubeconfig",
|
|
kubeconfig,
|
|
"--context",
|
|
"k3d-" + cfg.cluster,
|
|
"--namespace",
|
|
cfg.namespace,
|
|
...argv,
|
|
],
|
|
input,
|
|
);
|
|
}
|
|
async function receipt() {
|
|
return Bun.file(receiptFile).json();
|
|
}
|
|
async function save(r: any) {
|
|
await writeFile(receiptFile, JSON.stringify(r) + "\n", { mode: 0o600 });
|
|
}
|
|
async function serverID() {
|
|
return (
|
|
await command("server_identity", [
|
|
"docker",
|
|
"inspect",
|
|
"--format",
|
|
"{{.Id}}",
|
|
"k3d-" + cfg.cluster + "-server-0",
|
|
])
|
|
).trim();
|
|
}
|
|
async function namespace() {
|
|
const out = await k([
|
|
"get",
|
|
"namespace",
|
|
cfg.namespace,
|
|
"--ignore-not-found",
|
|
"-o",
|
|
"json",
|
|
]);
|
|
return out.trim() ? JSON.parse(out) : undefined;
|
|
}
|
|
async function actual() {
|
|
return {
|
|
container: await serverID(),
|
|
namespaceUID: (await namespace())?.metadata?.uid,
|
|
};
|
|
}
|
|
const operator = guardedOperator(cfg, {
|
|
receipt,
|
|
actual,
|
|
dispatch: async () => execute(),
|
|
});
|
|
const sources = {
|
|
postgres:
|
|
"postgres:18.6@sha256:74935e72241653ca55e0414067e6d8763aceb8a810eb51b452253ec3dcfc4336",
|
|
valkey:
|
|
"valkey/valkey:9@sha256:418652cfb58ef879d4978c33553735d7147016032d5aefaa14c828e611eb9dfd",
|
|
pingap:
|
|
"vicanso/pingap:0.16.0@sha256:e5b6f38a946e3105e0d98271be95e4a2dca0441f7962efcfd30371a3b55405e7",
|
|
...(lab === 4
|
|
? {
|
|
s3: "rustfs/rustfs:1.0.0-rc.1@sha256:f53d700fc16809070326f6ab3fb565d48e722364a7bd5a12e47d0a5abe079ce6",
|
|
}
|
|
: {}),
|
|
};
|
|
async function imageSet() {
|
|
const hash = new Bun.CryptoHasher("sha256");
|
|
hash.update(await Bun.file(resolve(root, "Dockerfile")).arrayBuffer());
|
|
async function walk(dir: string) {
|
|
for (const item of (await readdir(dir, { withFileTypes: true })).sort(
|
|
(a, b) => a.name.localeCompare(b.name),
|
|
)) {
|
|
if (item.name.startsWith(".local")) continue;
|
|
const file = resolve(dir, item.name);
|
|
if (item.isDirectory()) await walk(file);
|
|
else if (item.isFile()) {
|
|
hash.update(file.slice(root.length));
|
|
hash.update(await readFile(file));
|
|
}
|
|
}
|
|
}
|
|
await walk(resolve(root, "backend"));
|
|
const version = hash.digest("hex").slice(0, 16),
|
|
api = `vp-coursework/api-lab${lab}:${version}`;
|
|
await command("build_api", ["docker", "build", "-t", api, "."]);
|
|
const images: Record<string, string> = { api };
|
|
const cached = new Set(
|
|
(
|
|
await command("local_image_inventory", [
|
|
"docker",
|
|
"image",
|
|
"ls",
|
|
"--digests",
|
|
"--format",
|
|
"{{.Repository}}@{{.Digest}}",
|
|
])
|
|
)
|
|
.trim()
|
|
.split("\n")
|
|
.map((ref) => ref.replace(/^docker\.io\//, "").replace(/^library\//, "")),
|
|
);
|
|
for (const [name, source] of Object.entries(sources)) {
|
|
const digest = source!.split("@sha256:")[1],
|
|
alias = `vp-coursework/${name}:${digest.slice(0, 16)}`;
|
|
const repository = source!.split("@")[0].replace(/:[^/:]+$/, "");
|
|
if (!cached.has(repository + "@sha256:" + digest))
|
|
await command("pull_pinned_image_" + name, ["docker", "pull", source!]);
|
|
const pinned = JSON.parse(
|
|
await command("verify_image_" + name, [
|
|
"docker",
|
|
"image",
|
|
"inspect",
|
|
source!,
|
|
]),
|
|
);
|
|
const configID = pinnedConfigID(pinned[0], source!);
|
|
await command("alias_image_" + name, ["docker", "tag", source!, alias]);
|
|
if (
|
|
(
|
|
await command("verify_alias_" + name, [
|
|
"docker",
|
|
"image",
|
|
"inspect",
|
|
alias,
|
|
"--format",
|
|
"{{.Id}}",
|
|
])
|
|
).trim() !== configID
|
|
)
|
|
throw new Error("Image alias content differs");
|
|
images[name] = alias;
|
|
}
|
|
return images;
|
|
}
|
|
async function bootstrap() {
|
|
const inventory = (
|
|
await command("cluster_inventory", [
|
|
"docker",
|
|
"ps",
|
|
"-aq",
|
|
"--filter",
|
|
"label=k3d.cluster=" + cfg.cluster,
|
|
])
|
|
).trim();
|
|
let r: any;
|
|
if (await Bun.file(receiptFile).exists()) r = await receipt();
|
|
else {
|
|
if (inventory)
|
|
throw new Error("Unclaimed existing cluster; refusing adoption");
|
|
r = {
|
|
lab,
|
|
cluster: cfg.cluster,
|
|
nonce: crypto.randomUUID(),
|
|
state: "claimed",
|
|
};
|
|
await writeFile(receiptFile, JSON.stringify(r) + "\n", {
|
|
mode: 0o600,
|
|
flag: "wx",
|
|
});
|
|
}
|
|
if (r.lab !== lab || r.cluster !== cfg.cluster)
|
|
throw new Error("Receipt target differs");
|
|
if (!inventory) {
|
|
r = {
|
|
lab,
|
|
cluster: cfg.cluster,
|
|
nonce: r.state === "claimed" ? r.nonce : crypto.randomUUID(),
|
|
state: "claimed",
|
|
};
|
|
await save(r);
|
|
if (
|
|
(
|
|
await command("public_port_guard", [
|
|
"ss",
|
|
"-H",
|
|
"-ltn",
|
|
`sport = :${cfg.port}`,
|
|
])
|
|
).trim()
|
|
)
|
|
throw new Error("Public port is occupied");
|
|
await command("create_cluster", [
|
|
"k3d",
|
|
"cluster",
|
|
"create",
|
|
cfg.cluster,
|
|
"--image",
|
|
"rancher/k3s:v1.37.1-k3s1@sha256:ca7f37d993d82ef0dcdcfecb2e0e2618ea541dbaffc620c8cedebe01a82acd0d",
|
|
"--servers",
|
|
"1",
|
|
"--agents",
|
|
"0",
|
|
"--servers-memory",
|
|
"8g",
|
|
"--api-port",
|
|
"127.0.0.1:0",
|
|
"--port",
|
|
`127.0.0.1:${cfg.port}:80@loadbalancer`,
|
|
"--k3s-arg",
|
|
"--disable=traefik@server:0",
|
|
"--runtime-label",
|
|
`vp.coursework.owner=${r.nonce}@server:0`,
|
|
"--kubeconfig-update-default=false",
|
|
"--kubeconfig-switch-context=false",
|
|
"--timeout",
|
|
"180s",
|
|
]);
|
|
r.images = undefined;
|
|
} else {
|
|
const owner = (
|
|
await command("cluster_owner", [
|
|
"docker",
|
|
"inspect",
|
|
"--format",
|
|
'{{index .Config.Labels "vp.coursework.owner"}}',
|
|
"k3d-" + cfg.cluster + "-server-0",
|
|
])
|
|
).trim();
|
|
if (owner !== r.nonce) throw new Error("Existing server owner differs");
|
|
await command("start_cluster", ["k3d", "cluster", "start", cfg.cluster]);
|
|
}
|
|
r.container = await serverID();
|
|
const config = Bun.YAML.parse(
|
|
await command("private_kubeconfig", [
|
|
"k3d",
|
|
"kubeconfig",
|
|
"get",
|
|
cfg.cluster,
|
|
]),
|
|
) as any;
|
|
const ports = JSON.parse(
|
|
await command("published_api_port", [
|
|
"docker",
|
|
"inspect",
|
|
"--format",
|
|
"{{json .NetworkSettings.Ports}}",
|
|
"k3d-" + cfg.cluster + "-serverlb",
|
|
]),
|
|
);
|
|
const endpoint = ports["6443/tcp"]?.find(
|
|
(p: any) => p.HostIp === "127.0.0.1",
|
|
);
|
|
if (
|
|
!endpoint ||
|
|
!/^\d+$/.test(endpoint.HostPort) ||
|
|
Number(endpoint.HostPort) < 1024 ||
|
|
Number(endpoint.HostPort) > 65535
|
|
)
|
|
throw new Error("Unverified Kubernetes API port");
|
|
if (config.clusters?.length !== 1)
|
|
throw new Error("Unexpected kubeconfig cluster set");
|
|
config.clusters[0].cluster.server = "https://127.0.0.1:" + endpoint.HostPort;
|
|
const http = ports["80/tcp"]?.find((p: any) => p.HostIp === "127.0.0.1");
|
|
if (!http || Number(http.HostPort) !== cfg.port)
|
|
throw new Error("Public endpoint differs from claimed cluster");
|
|
r.publicPort = cfg.port;
|
|
await writeFile(kubeconfig, Bun.YAML.stringify(config), { mode: 0o600 });
|
|
await k(["wait", "--for=condition=Ready", "node", "--all", "--timeout=120s"]);
|
|
let ns = await namespace();
|
|
if (ns && ns.metadata?.annotations?.["vp-mtuci/owner"] !== r.nonce)
|
|
throw new Error("Foreign namespace");
|
|
if (!ns) {
|
|
await k(
|
|
["create", "-f", "-"],
|
|
JSON.stringify({
|
|
apiVersion: "v1",
|
|
kind: "Namespace",
|
|
metadata: {
|
|
name: cfg.namespace,
|
|
annotations: { "vp-mtuci/owner": r.nonce },
|
|
labels: { "app.kubernetes.io/part-of": "vp-coursework" },
|
|
},
|
|
}),
|
|
);
|
|
ns = await namespace();
|
|
}
|
|
r.namespaceUID = ns.metadata.uid;
|
|
r.state = "ready";
|
|
await save(r);
|
|
await command("credentials_bootstrap", [
|
|
process.execPath,
|
|
"--no-env-file",
|
|
resolve(root, "scripts/bootstrap.ts"),
|
|
]);
|
|
const secretFiles = {
|
|
pg_password: "pg_password",
|
|
database_url: "database_url",
|
|
cache_url: "cache_url",
|
|
valkey_conf: "valkey.conf",
|
|
...(lab === 4 ? { s3_credentials: "s3_credentials" } : {}),
|
|
};
|
|
const stringData: Record<string, string> = {};
|
|
for (const [key, file] of Object.entries(secretFiles))
|
|
stringData[key] = (
|
|
await Bun.file(resolve(root, ".local", file)).text()
|
|
).trim();
|
|
await k(
|
|
["apply", "--server-side", "--field-manager=vp-coursework", "-f", "-"],
|
|
JSON.stringify({
|
|
apiVersion: "v1",
|
|
kind: "Secret",
|
|
metadata: { name: "app-config", namespace: cfg.namespace },
|
|
type: "Opaque",
|
|
stringData,
|
|
}),
|
|
);
|
|
if (lab === 4) {
|
|
const data: Record<string, string> = {};
|
|
for (const name of ["s3_access", "s3_secret", "s3_root"])
|
|
data[name] = (
|
|
await Bun.file(resolve(root, ".local", name)).text()
|
|
).trim();
|
|
await k(
|
|
["apply", "--server-side", "--field-manager=vp-coursework", "-f", "-"],
|
|
JSON.stringify({
|
|
apiVersion: "v1",
|
|
kind: "Secret",
|
|
metadata: { name: "storage-root", namespace: cfg.namespace },
|
|
type: "Opaque",
|
|
stringData: data,
|
|
}),
|
|
);
|
|
}
|
|
const images = await imageSet();
|
|
async function imported() {
|
|
return (
|
|
await command("containerd_inventory", [
|
|
"docker",
|
|
"exec",
|
|
"k3d-" + cfg.cluster + "-server-0",
|
|
"ctr",
|
|
"-n",
|
|
"k8s.io",
|
|
"images",
|
|
"list",
|
|
"-q",
|
|
])
|
|
)
|
|
.trim()
|
|
.split("\n");
|
|
}
|
|
let available = await imported();
|
|
if (
|
|
Object.values(images).some(
|
|
(image) => !available.includes("docker.io/" + image),
|
|
)
|
|
) {
|
|
const archive = resolve(local, "images-" + crypto.randomUUID() + ".tar");
|
|
try {
|
|
await command(
|
|
"export_single_platform",
|
|
[
|
|
"docker",
|
|
"image",
|
|
"save",
|
|
"--platform",
|
|
"linux/amd64",
|
|
"--output",
|
|
archive,
|
|
...Object.values(images),
|
|
],
|
|
undefined,
|
|
240000,
|
|
);
|
|
stage = "import_single_platform";
|
|
const p = Bun.spawn(
|
|
[
|
|
"docker",
|
|
"exec",
|
|
"-i",
|
|
"k3d-" + cfg.cluster + "-server-0",
|
|
"ctr",
|
|
"-n",
|
|
"k8s.io",
|
|
"images",
|
|
"import",
|
|
"--platform",
|
|
"linux/amd64",
|
|
"-",
|
|
],
|
|
{
|
|
stdin: Bun.file(archive),
|
|
stdout: "pipe",
|
|
stderr: "pipe",
|
|
signal: AbortSignal.timeout(240000),
|
|
},
|
|
);
|
|
const [out, err] = await Promise.all([
|
|
new Response(p.stdout).text(),
|
|
new Response(p.stderr).text(),
|
|
]);
|
|
await writeFile(resolve(local, "image-import.txt"), out + err, {
|
|
mode: 0o600,
|
|
});
|
|
if ((await p.exited) !== 0)
|
|
throw new Error("Native single-platform image import failed");
|
|
} finally {
|
|
await rm(archive, { force: true });
|
|
}
|
|
available = await imported();
|
|
if (
|
|
Object.values(images).some(
|
|
(image) => !available.includes("docker.io/" + image),
|
|
)
|
|
)
|
|
throw new Error("Imported images are missing from Kubernetes containerd");
|
|
}
|
|
r.images = images;
|
|
await save(r);
|
|
await applyConfiguration(true);
|
|
for (const name of ["postgres", "valkey", ...(lab === 4 ? ["s3"] : [])])
|
|
await readyDeployment(name);
|
|
const sqlReady = await k([
|
|
"exec",
|
|
"deployment/postgres",
|
|
"--",
|
|
"psql",
|
|
"-h",
|
|
"127.0.0.1",
|
|
"-U",
|
|
"messenger",
|
|
"-d",
|
|
"messenger",
|
|
"-X",
|
|
"-qAt",
|
|
"-c",
|
|
"SELECT 1;",
|
|
]);
|
|
if (sqlReady.trim() !== "1")
|
|
throw new Error("PostgreSQL did not execute the readiness query");
|
|
await job("migrate", []);
|
|
if (lab === 4) await job("bootstrap-s3", []);
|
|
await applyConfiguration(false);
|
|
await ready();
|
|
await writeFile(
|
|
resolve(evidence, "runtime.json"),
|
|
JSON.stringify(
|
|
{
|
|
cluster: cfg.cluster,
|
|
namespaceUID: r.namespaceUID,
|
|
k3s: "v1.37.1-k3s1",
|
|
pingap: "0.16.0",
|
|
images,
|
|
port: cfg.port,
|
|
},
|
|
null,
|
|
2,
|
|
) + "\n",
|
|
);
|
|
console.log("Ready: " + cfg.cluster + " via Pingap on localhost:" + cfg.port);
|
|
}
|
|
function integer(
|
|
value: string | undefined,
|
|
fallback: number,
|
|
min: number,
|
|
max: number,
|
|
) {
|
|
const n = Number(value ?? fallback);
|
|
if (!Number.isInteger(n) || n < min || n > max)
|
|
throw new Error("Invalid bounded runtime setting");
|
|
return n;
|
|
}
|
|
async function applyConfiguration(initial: boolean) {
|
|
const r = await receipt(),
|
|
replicas = initial ? 0 : integer(process.env.API_REPLICAS, 2, 1, 4);
|
|
const cpu = process.env.API_CPUS ?? "1";
|
|
if (!Number.isFinite(Number(cpu)) || Number(cpu) < 0.1 || Number(cpu) > 4)
|
|
throw new Error("Invalid CPU limit");
|
|
const memory = process.env.API_MEMORY ?? (lab === 4 ? "2Gi" : "512Mi");
|
|
const match = /^(\d+)(Mi|Gi|m|g)$/.exec(memory);
|
|
if (!match) throw new Error("Invalid memory limit");
|
|
const mib =
|
|
Number(match[1]) * (match[2].toLowerCase().startsWith("g") ? 1024 : 1);
|
|
if (mib < 128 || mib > 2048) throw new Error("Unsafe memory limit");
|
|
const soft =
|
|
process.env.GO_MEMORY_LIMIT ?? (lab === 4 ? "1536MiB" : "400MiB");
|
|
const sm = /^(\d+)MiB$/.exec(soft);
|
|
if (!sm || Number(sm[1]) > mib * 0.8 || Number(sm[1]) < 64)
|
|
throw new Error("Unsafe soft memory limit");
|
|
assertImageBudget(lab, mib, soft);
|
|
const apiPatch = {
|
|
apiVersion: "apps/v1",
|
|
kind: "Deployment",
|
|
metadata: { name: "api" },
|
|
spec: {
|
|
replicas,
|
|
template: {
|
|
spec: {
|
|
containers: [
|
|
{
|
|
name: "api",
|
|
resources: {
|
|
requests: { cpu: "100m", memory: "128Mi" },
|
|
limits: { cpu, memory: mib + "Mi" },
|
|
},
|
|
env: [
|
|
{
|
|
name: "PG_MAX_CONNS",
|
|
value: String(integer(process.env.PG_MAX_CONNS, 8, 1, 64)),
|
|
},
|
|
{
|
|
name: "SESSION_TTL_SECONDS",
|
|
value: String(
|
|
integer(process.env.SESSION_TTL_SECONDS, 3600, 1, 86400),
|
|
),
|
|
},
|
|
{ name: "GOMEMLIMIT", value: soft },
|
|
],
|
|
},
|
|
],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
const render = resolve(local, "render");
|
|
const overlay = {
|
|
apiVersion: "kustomize.config.k8s.io/v1beta1",
|
|
kind: "Kustomization",
|
|
resources: [relative(render, resolve(root, "deploy/k3s"))],
|
|
images: Object.entries(r.images).map(([name, alias]) => ({
|
|
name: name === "api" ? "vp-messenger" : "vp-" + name,
|
|
newName: (alias as string).split(":")[0],
|
|
newTag: (alias as string).split(":")[1],
|
|
})),
|
|
patches: [
|
|
{
|
|
target: { kind: "Deployment", name: "api" },
|
|
patch: Bun.YAML.stringify(apiPatch),
|
|
},
|
|
],
|
|
};
|
|
await mkdir(render, { recursive: true });
|
|
await writeFile(
|
|
resolve(render, "kustomization.yaml"),
|
|
Bun.YAML.stringify(overlay),
|
|
);
|
|
const manifest = await command("render_kustomize", [
|
|
"kubectl",
|
|
"kustomize",
|
|
render,
|
|
"--load-restrictor=LoadRestrictionsNone",
|
|
]);
|
|
// This runner owns the isolated namespace and reclaims replicas after explicit scale hooks.
|
|
await k(
|
|
[
|
|
"apply",
|
|
"--dry-run=server",
|
|
"--server-side",
|
|
"--force-conflicts",
|
|
"--field-manager=vp-coursework",
|
|
"-f",
|
|
"-",
|
|
],
|
|
manifest,
|
|
);
|
|
await k(
|
|
[
|
|
"apply",
|
|
"--server-side",
|
|
"--force-conflicts",
|
|
"--field-manager=vp-coursework",
|
|
"-f",
|
|
"-",
|
|
],
|
|
manifest,
|
|
);
|
|
}
|
|
async function readyDeployment(name: string) {
|
|
await k(["rollout", "status", "deployment/" + name, "--timeout=150s"]);
|
|
}
|
|
async function pods() {
|
|
return JSON.parse(await k(["get", "pods", "-o", "json"])).items;
|
|
}
|
|
async function ready() {
|
|
for (const name of [
|
|
"postgres",
|
|
"valkey",
|
|
"api",
|
|
"pingap",
|
|
...(lab === 4 ? ["s3"] : []),
|
|
])
|
|
await readyDeployment(name);
|
|
const drainDeadline = performance.now() + 40000;
|
|
for (;;) {
|
|
const draining = (await pods()).some(
|
|
(p: any) =>
|
|
p.metadata.deletionTimestamp &&
|
|
["api", "postgres", "valkey", "pingap", "s3"].includes(
|
|
p.metadata.labels?.app,
|
|
),
|
|
);
|
|
if (!draining) break;
|
|
if (performance.now() >= drainDeadline)
|
|
throw new Error("Old workload Pods did not finish terminating");
|
|
await Bun.sleep(250);
|
|
}
|
|
await command(
|
|
"http_discovery_ready",
|
|
[process.execPath, "--no-env-file", resolve(root, "student-tests/wait.ts")],
|
|
undefined,
|
|
120000,
|
|
);
|
|
}
|
|
async function job(mode: string, extra: string[]) {
|
|
const allowed = [
|
|
"migrate",
|
|
"bootstrap-s3",
|
|
"fixture-cleanup",
|
|
"reconcile",
|
|
"integration",
|
|
"image-memory",
|
|
];
|
|
if (!allowed.includes(mode) || (lab === 3 && mode !== "migrate"))
|
|
throw new Error("Unsupported Job");
|
|
if (mode === "integration" && !cfg.cluster.includes("-r-"))
|
|
throw new Error(
|
|
"Destructive integration requires a fresh research cluster",
|
|
);
|
|
if (mode === "fixture-cleanup" && !/^st_[a-f0-9]{16}$/.test(extra[0] ?? ""))
|
|
throw new Error("Invalid fixture prefix");
|
|
async function gameGuard() {
|
|
const p = Bun.spawn(["game-running"], { stdout: "ignore", stderr: "pipe" });
|
|
await new Response(p.stderr).text();
|
|
assertNoGame(await p.exited);
|
|
}
|
|
if (mode === "image-memory") await gameGuard();
|
|
const template = JSON.parse(
|
|
await k(["get", "deployment", "api", "-o", "json"]),
|
|
).spec.template;
|
|
const name = "task-" + mode + "-" + crypto.randomUUID().slice(0, 8);
|
|
const c = template.spec.containers[0];
|
|
delete c.readinessProbe;
|
|
delete c.livenessProbe;
|
|
delete c.startupProbe;
|
|
delete c.lifecycle;
|
|
delete c.ports;
|
|
c.name = "task";
|
|
c.command = [
|
|
mode === "integration"
|
|
? "/integration.test"
|
|
: mode === "image-memory"
|
|
? "/image-memory.test"
|
|
: mode === "fixture-cleanup"
|
|
? "/fixture-cleanup"
|
|
: "/messenger",
|
|
];
|
|
c.args =
|
|
mode === "integration"
|
|
? [
|
|
"-test.v",
|
|
"-test.run",
|
|
"Test(ReconciliationAgainstPostgresAndPrivateS3|S3InventoryAndStatReturnComparableVersionMetadata|FixtureCleanupRejectsForeignMembership)",
|
|
"-test.timeout=150s",
|
|
]
|
|
: mode === "image-memory"
|
|
? [
|
|
"-test.v",
|
|
"-test.run",
|
|
"TestWorstCaseImagesWithinBudget",
|
|
"-test.timeout=120s",
|
|
]
|
|
: mode === "fixture-cleanup"
|
|
? extra
|
|
: ["-" + mode, ...extra];
|
|
if (mode === "integration")
|
|
c.env.push({ name: "RUN_ISOLATED_RECONCILIATION", value: "1" });
|
|
if (mode === "image-memory") {
|
|
c.env.push({ name: "RUN_IMAGE_STRESS", value: "1" });
|
|
c.resources = {
|
|
requests: { cpu: "100m", memory: "128Mi" },
|
|
limits: { cpu: "2", memory: "2Gi" },
|
|
};
|
|
}
|
|
if (mode === "bootstrap-s3") {
|
|
const volume = template.spec.volumes.find(
|
|
(v: any) => v.name === "credentials",
|
|
);
|
|
const items = volume.secret.items;
|
|
delete volume.secret;
|
|
volume.projected = {
|
|
defaultMode: 288,
|
|
sources: [
|
|
{ secret: { name: "app-config", items } },
|
|
{
|
|
secret: {
|
|
name: "storage-root",
|
|
items: [{ key: "s3_root", path: "s3_root" }],
|
|
},
|
|
},
|
|
],
|
|
};
|
|
}
|
|
template.metadata = { labels: { app: name } };
|
|
template.spec.restartPolicy = "Never";
|
|
template.spec.terminationGracePeriodSeconds = 10;
|
|
await k(
|
|
["create", "-f", "-"],
|
|
JSON.stringify({
|
|
apiVersion: "batch/v1",
|
|
kind: "Job",
|
|
metadata: { name },
|
|
spec: {
|
|
backoffLimit: 0,
|
|
activeDeadlineSeconds: 200,
|
|
ttlSecondsAfterFinished: 300,
|
|
template,
|
|
},
|
|
}),
|
|
);
|
|
let success = false;
|
|
return withCleanup({
|
|
perform: async () => {
|
|
for (let attempt = 0; attempt < 210; attempt++) {
|
|
const j = JSON.parse(await k(["get", "job", name, "-o", "json"]));
|
|
if (j.status?.succeeded) {
|
|
success = true;
|
|
break;
|
|
}
|
|
if (j.status?.failed) break;
|
|
await Bun.sleep(1000);
|
|
}
|
|
const output = await k(["logs", "job/" + name]);
|
|
if (!success) {
|
|
await writeFile(resolve(local, "failed-job.txt"), output, {
|
|
mode: 0o600,
|
|
});
|
|
const snapshot = await k([
|
|
"get",
|
|
"pods",
|
|
"-l",
|
|
"job-name=" + name,
|
|
"-o",
|
|
"json",
|
|
]);
|
|
await writeFile(resolve(local, "failed-job-pods.json"), snapshot, {
|
|
mode: 0o600,
|
|
});
|
|
throw new Error("Job " + mode + " failed; private diagnostics saved");
|
|
}
|
|
if (mode === "image-memory") await gameGuard();
|
|
return output;
|
|
},
|
|
cleanup: async () => {
|
|
await k(["delete", "job", name, "--wait=true", "--timeout=30s"]);
|
|
},
|
|
});
|
|
}
|
|
async function execute() {
|
|
if (action === "info") {
|
|
const r = await receipt();
|
|
console.log(
|
|
JSON.stringify({
|
|
cluster: cfg.cluster,
|
|
port: r.publicPort,
|
|
namespace: cfg.namespace,
|
|
images: r.images,
|
|
}),
|
|
);
|
|
} else if (action === "stats") {
|
|
const out = [];
|
|
for (const p of (await pods()).filter(
|
|
(p: any) =>
|
|
p.metadata.labels?.app === "api" && !p.metadata.deletionTimestamp,
|
|
)) {
|
|
const id = p.status.containerStatuses?.[0]?.containerID?.replace(
|
|
/^containerd:\/\//,
|
|
"",
|
|
);
|
|
if (!/^[a-f0-9]{64}$/.test(id ?? ""))
|
|
throw new Error("API container identity unavailable");
|
|
const inspected = JSON.parse(
|
|
await command("container_pid", [
|
|
"docker",
|
|
"exec",
|
|
"k3d-" + cfg.cluster + "-server-0",
|
|
"crictl",
|
|
"inspect",
|
|
"-o",
|
|
"json",
|
|
id,
|
|
]),
|
|
);
|
|
const pid = Number(inspected.info?.pid);
|
|
if (!Number.isInteger(pid) || pid < 1)
|
|
throw new Error("Invalid API process ID");
|
|
const raw = await command("container_cgroup", [
|
|
"docker",
|
|
"exec",
|
|
"k3d-" + cfg.cluster + "-server-0",
|
|
"cat",
|
|
"/proc/" + pid + "/cgroup",
|
|
]);
|
|
const group = raw
|
|
.split("\n")
|
|
.find((x) => x.startsWith("0::"))
|
|
?.slice(3);
|
|
if (
|
|
!group ||
|
|
!group.startsWith("/") ||
|
|
group.includes("..") ||
|
|
!group.includes(id)
|
|
)
|
|
throw new Error("Unverified API cgroup path");
|
|
async function metric(file: string) {
|
|
return command("cgroup_" + file.replace(".", "_"), [
|
|
"docker",
|
|
"exec",
|
|
"k3d-" + cfg.cluster + "-server-0",
|
|
"cat",
|
|
"/sys/fs/cgroup" + group + "/" + file,
|
|
]);
|
|
}
|
|
const [current, peak, cpu] = await Promise.all([
|
|
metric("memory.current"),
|
|
metric("memory.peak"),
|
|
metric("cpu.stat"),
|
|
]);
|
|
const values = Object.fromEntries(
|
|
cpu
|
|
.trim()
|
|
.split("\n")
|
|
.map((x) => x.split(/\s+/)),
|
|
);
|
|
out.push({
|
|
pod: p.metadata.name,
|
|
uid: p.metadata.uid,
|
|
image: p.status.containerStatuses[0].imageID,
|
|
memory_current_bytes: Number(current.trim()),
|
|
memory_peak_bytes: Number(peak.trim()),
|
|
cpu_usage_usec: Number(values.usage_usec),
|
|
limits: p.spec.containers[0].resources.limits,
|
|
});
|
|
}
|
|
console.log(JSON.stringify({ at: new Date().toISOString(), pods: out }));
|
|
} else if (action === "configure") {
|
|
await applyConfiguration(false);
|
|
await ready();
|
|
} else if (action === "ready") await ready();
|
|
else if (action === "kubectl") {
|
|
assertScopedArgs(args);
|
|
const input =
|
|
args.includes("-i") || args.includes("--stdin")
|
|
? await Bun.stdin.text()
|
|
: undefined;
|
|
if (input !== undefined && input.length > 1024 * 1024)
|
|
throw new Error("Operational stdin exceeds limit");
|
|
process.stdout.write(await k(args, input));
|
|
} else if (action === "task")
|
|
process.stdout.write(await job(args[0], args.slice(1)));
|
|
else if (action === "restart") {
|
|
const runID = crypto.randomUUID();
|
|
const selected = (items: any[]) =>
|
|
items
|
|
.filter(
|
|
(p) =>
|
|
!p.metadata.deletionTimestamp &&
|
|
["api", "postgres", "valkey", "pingap", "s3"].includes(
|
|
p.metadata.labels?.app,
|
|
),
|
|
)
|
|
.map((p) => ({
|
|
name: p.metadata.name,
|
|
uid: p.metadata.uid,
|
|
role: p.metadata.labels.app,
|
|
started: p.status.startTime,
|
|
}));
|
|
const before = selected(await pods());
|
|
if (!before.length) throw new Error("No workload Pods to restart");
|
|
await writeFile(
|
|
resolve(evidence, "restart-before.json"),
|
|
JSON.stringify({ run_id: runID, pods: before }, null, 2) + "\n",
|
|
);
|
|
await k(["rollout", "restart", "deployment"]);
|
|
await ready();
|
|
const after = selected(await pods());
|
|
if (!after.length || after.some((p) => before.some((b) => b.uid === p.uid)))
|
|
throw new Error("An original workload Pod was not replaced");
|
|
await writeFile(
|
|
resolve(evidence, "restart-after.json"),
|
|
JSON.stringify({ run_id: runID, pods: after }, null, 2) + "\n",
|
|
);
|
|
} else if (action === "stop-one") {
|
|
const target = process.env.TARGET_INSTANCE_ID;
|
|
if (!target || !/^[a-z0-9-]+$/.test(target))
|
|
throw new Error("Actual target pod name required");
|
|
const items = (await pods()).filter(
|
|
(p: any) =>
|
|
p.metadata.labels?.app === "api" &&
|
|
!p.metadata.deletionTimestamp &&
|
|
p.status.conditions?.some(
|
|
(c: any) => c.type === "Ready" && c.status === "True",
|
|
),
|
|
);
|
|
const victim = items.find((p: any) => p.metadata.name === target);
|
|
if (items.length !== 2 || !victim)
|
|
throw new Error("Target must belong to two ready API pods");
|
|
await writeFile(
|
|
resolve(evidence, "stop-one-before.json"),
|
|
JSON.stringify(
|
|
{
|
|
target,
|
|
uid: victim.metadata.uid,
|
|
pods: items.map((p: any) => p.metadata.name),
|
|
},
|
|
null,
|
|
2,
|
|
),
|
|
);
|
|
const containerID =
|
|
victim.status.containerStatuses?.[0]?.containerID?.replace(
|
|
/^containerd:\/\//,
|
|
"",
|
|
);
|
|
if (!/^[a-f0-9]{64}$/.test(containerID ?? ""))
|
|
throw new Error("Unverified target container");
|
|
for (const p of items)
|
|
await k([
|
|
"patch",
|
|
"pod",
|
|
p.metadata.name,
|
|
"--type=merge",
|
|
"-p",
|
|
JSON.stringify({
|
|
metadata: {
|
|
uid: p.metadata.uid,
|
|
annotations: {
|
|
"controller.kubernetes.io/pod-deletion-cost":
|
|
p === victim ? "-1000000" : "1000000",
|
|
},
|
|
},
|
|
}),
|
|
]);
|
|
await k([
|
|
"scale",
|
|
"deployment/api",
|
|
"--current-replicas=2",
|
|
"--replicas=1",
|
|
]);
|
|
let terminating: any;
|
|
for (let i = 0; i < 15; i++) {
|
|
const current = (await pods()).find(
|
|
(p: any) => p.metadata.uid === victim.metadata.uid,
|
|
);
|
|
if (current?.metadata.deletionTimestamp) {
|
|
terminating = current;
|
|
break;
|
|
}
|
|
await Bun.sleep(100);
|
|
}
|
|
if (!terminating)
|
|
throw new Error("Controller did not select the intended target");
|
|
const currentCID =
|
|
terminating.status.containerStatuses?.[0]?.containerID?.replace(
|
|
/^containerd:\/\//,
|
|
"",
|
|
);
|
|
if (!/^[a-f0-9]{64}$/.test(currentCID ?? ""))
|
|
throw new Error("Terminating process identity unavailable");
|
|
await command(
|
|
"stop_target_process",
|
|
[
|
|
"docker",
|
|
"exec",
|
|
"k3d-" + cfg.cluster + "-server-0",
|
|
"ctr",
|
|
"-n",
|
|
"k8s.io",
|
|
"tasks",
|
|
"kill",
|
|
"--signal",
|
|
"SIGKILL",
|
|
currentCID,
|
|
],
|
|
undefined,
|
|
4000,
|
|
);
|
|
let stopped: any;
|
|
for (let i = 0; i < 15; i++) {
|
|
stopped = JSON.parse(
|
|
await command("verify_stopped_process", [
|
|
"docker",
|
|
"exec",
|
|
"k3d-" + cfg.cluster + "-server-0",
|
|
"crictl",
|
|
"inspect",
|
|
"-o",
|
|
"json",
|
|
currentCID,
|
|
]),
|
|
);
|
|
if (stopped.status?.state === "CONTAINER_EXITED") break;
|
|
await Bun.sleep(100);
|
|
}
|
|
if (stopped.status?.state !== "CONTAINER_EXITED")
|
|
throw new Error("Target process did not stop");
|
|
if (
|
|
stopped.status?.labels?.["io.kubernetes.pod.uid"] !== victim.metadata.uid
|
|
)
|
|
throw new Error("Stopped process Pod UID differs");
|
|
const running = JSON.parse(
|
|
await command("verify_no_running_victim", [
|
|
"docker",
|
|
"exec",
|
|
"k3d-" + cfg.cluster + "-server-0",
|
|
"crictl",
|
|
"ps",
|
|
"--state",
|
|
"Running",
|
|
"-o",
|
|
"json",
|
|
]),
|
|
);
|
|
assertNoRunningContainers(running.containers ?? [], victim.metadata.uid);
|
|
const finalVictim = (await pods()).find(
|
|
(p: any) => p.metadata.uid === victim.metadata.uid,
|
|
);
|
|
if (
|
|
finalVictim?.status.containerStatuses?.some(
|
|
(c: any) => c.containerID !== "containerd://" + currentCID,
|
|
)
|
|
)
|
|
throw new Error("Target process changed during failure injection");
|
|
const remaining = (await pods()).filter(
|
|
(p: any) =>
|
|
p.metadata.labels?.app === "api" && !p.metadata.deletionTimestamp,
|
|
);
|
|
if (
|
|
remaining.length !== 1 ||
|
|
remaining[0].metadata.uid === victim.metadata.uid
|
|
)
|
|
throw new Error("Unexpected API survivor");
|
|
await writeFile(
|
|
resolve(evidence, "stop-one-after.json"),
|
|
JSON.stringify(
|
|
{
|
|
stopped: target,
|
|
stoppedUID: victim.metadata.uid,
|
|
containerID: currentCID,
|
|
state: "CONTAINER_EXITED",
|
|
survivor: remaining[0].metadata.name,
|
|
},
|
|
null,
|
|
2,
|
|
),
|
|
);
|
|
} else if (action === "destroy") {
|
|
await command("destroy_claimed_cluster", [
|
|
"k3d",
|
|
"cluster",
|
|
"delete",
|
|
cfg.cluster,
|
|
]);
|
|
const r = await receipt();
|
|
r.state = "destroyed";
|
|
await save(r);
|
|
} else if (action === "stop")
|
|
await command("stop_claimed_cluster", [
|
|
"k3d",
|
|
"cluster",
|
|
"stop",
|
|
cfg.cluster,
|
|
]);
|
|
else throw new Error("Unknown runtime action");
|
|
}
|
|
async function destroyPartial() {
|
|
const r = await receipt();
|
|
const inventory = (
|
|
await command("partial_inventory", [
|
|
"docker",
|
|
"ps",
|
|
"-aq",
|
|
"--filter",
|
|
"label=k3d.cluster=" + cfg.cluster,
|
|
])
|
|
).trim();
|
|
if (!inventory) {
|
|
r.state = "destroyed";
|
|
await save(r);
|
|
return;
|
|
}
|
|
const inspected = JSON.parse(
|
|
await command("partial_server_identity", [
|
|
"docker",
|
|
"inspect",
|
|
"k3d-" + cfg.cluster + "-server-0",
|
|
]),
|
|
);
|
|
const server = {
|
|
container: inspected[0].Id,
|
|
owner: inspected[0].Config.Labels["vp.coursework.owner"],
|
|
};
|
|
assertPartialClaim(r, server, cfg);
|
|
r.container = server.container;
|
|
await save(r);
|
|
await command("destroy_partial_claimed_cluster", [
|
|
"k3d",
|
|
"cluster",
|
|
"delete",
|
|
cfg.cluster,
|
|
]);
|
|
r.state = "destroyed";
|
|
await save(r);
|
|
}
|
|
try {
|
|
if (action === "check") console.log("Parsed k3s runtime for lab" + lab);
|
|
else if (action === "up") await bootstrap();
|
|
else if (action === "destroy" && (await receipt()).state === "claimed")
|
|
await destroyPartial();
|
|
else await operator.run(action);
|
|
} catch (error) {
|
|
await writeFile(
|
|
resolve(evidence, "last-failure.json"),
|
|
JSON.stringify({
|
|
stage,
|
|
action,
|
|
error: error instanceof Error ? error.message : "unknown",
|
|
}) + "\n",
|
|
);
|
|
throw error;
|
|
}
|