import { mkdir, readFile, readdir, writeFile, rm } from "node:fs/promises"; import { resolve, relative } from "node:path"; import { settings, guardedOperator, pinnedConfigID, assertScopedArgs, assertNoGame, assertPartialClaim, assertNoRunningContainers, withCleanup, assertImageBudget, } from "./k3s-model"; const root = resolve(import.meta.dir, ".."); const lab = (await Bun.file(resolve(root, "lab.json")).json()).number; const cfg = settings(lab, process.env); const local = resolve(root, ".local/k3s", cfg.cluster), kubeconfig = resolve(local, "kubeconfig"), receiptFile = resolve(local, "receipt.json"); const evidence = resolve(root, "evidence/k3s"); await mkdir(local, { recursive: true, mode: 0o700 }); await mkdir(evidence, { recursive: true }); const action = process.argv[2] ?? "ready"; const args = process.argv.slice(3); let stage = "start"; async function command( label: string, argv: string[], input?: string, timeout = 240000, ) { stage = label; const p = Bun.spawn(argv, { cwd: root, stdin: input === undefined ? "ignore" : "pipe", stdout: "pipe", stderr: "pipe", signal: AbortSignal.timeout(timeout), }); if (input !== undefined) { p.stdin.write(input); p.stdin.end(); } const [out, err] = await Promise.all([ new Response(p.stdout).text(), new Response(p.stderr).text(), ]); if ((await p.exited) !== 0) { await writeFile(resolve(local, "last-error.txt"), out + err, { mode: 0o600, }); throw new Error(label + " failed; private diagnostics in .local/k3s"); } return out; } function k(argv: string[], input?: string) { return command( "kubernetes_" + argv[0], [ "kubectl", "--kubeconfig", kubeconfig, "--context", "k3d-" + cfg.cluster, "--namespace", cfg.namespace, ...argv, ], input, ); } async function receipt() { return Bun.file(receiptFile).json(); } async function save(r: any) { await writeFile(receiptFile, JSON.stringify(r) + "\n", { mode: 0o600 }); } async function serverID() { return ( await command("server_identity", [ "docker", "inspect", "--format", "{{.Id}}", "k3d-" + cfg.cluster + "-server-0", ]) ).trim(); } async function namespace() { const out = await k([ "get", "namespace", cfg.namespace, "--ignore-not-found", "-o", "json", ]); return out.trim() ? JSON.parse(out) : undefined; } async function actual() { return { container: await serverID(), namespaceUID: (await namespace())?.metadata?.uid, }; } const operator = guardedOperator(cfg, { receipt, actual, dispatch: async () => execute(), }); const sources = { postgres: "postgres:18.6@sha256:74935e72241653ca55e0414067e6d8763aceb8a810eb51b452253ec3dcfc4336", valkey: "valkey/valkey:9@sha256:418652cfb58ef879d4978c33553735d7147016032d5aefaa14c828e611eb9dfd", pingap: "vicanso/pingap:0.16.0@sha256:e5b6f38a946e3105e0d98271be95e4a2dca0441f7962efcfd30371a3b55405e7", ...(lab === 4 ? { s3: "rustfs/rustfs:1.0.0-rc.1@sha256:f53d700fc16809070326f6ab3fb565d48e722364a7bd5a12e47d0a5abe079ce6", } : {}), }; async function imageSet() { const hash = new Bun.CryptoHasher("sha256"); hash.update(await Bun.file(resolve(root, "Dockerfile")).arrayBuffer()); async function walk(dir: string) { for (const item of (await readdir(dir, { withFileTypes: true })).sort( (a, b) => a.name.localeCompare(b.name), )) { if (item.name.startsWith(".local")) continue; const file = resolve(dir, item.name); if (item.isDirectory()) await walk(file); else if (item.isFile()) { hash.update(file.slice(root.length)); hash.update(await readFile(file)); } } } await walk(resolve(root, "backend")); const version = hash.digest("hex").slice(0, 16), api = `vp-coursework/api-lab${lab}:${version}`; await command("build_api", ["docker", "build", "-t", api, "."]); const images: Record = { api }; const cached = new Set( ( await command("local_image_inventory", [ "docker", "image", "ls", "--digests", "--format", "{{.Repository}}@{{.Digest}}", ]) ) .trim() .split("\n") .map((ref) => ref.replace(/^docker\.io\//, "").replace(/^library\//, "")), ); for (const [name, source] of Object.entries(sources)) { const digest = source!.split("@sha256:")[1], alias = `vp-coursework/${name}:${digest.slice(0, 16)}`; const repository = source!.split("@")[0].replace(/:[^/:]+$/, ""); if (!cached.has(repository + "@sha256:" + digest)) await command("pull_pinned_image_" + name, ["docker", "pull", source!]); const pinned = JSON.parse( await command("verify_image_" + name, [ "docker", "image", "inspect", source!, ]), ); const configID = pinnedConfigID(pinned[0], source!); await command("alias_image_" + name, ["docker", "tag", source!, alias]); if ( ( await command("verify_alias_" + name, [ "docker", "image", "inspect", alias, "--format", "{{.Id}}", ]) ).trim() !== configID ) throw new Error("Image alias content differs"); images[name] = alias; } return images; } async function bootstrap() { const inventory = ( await command("cluster_inventory", [ "docker", "ps", "-aq", "--filter", "label=k3d.cluster=" + cfg.cluster, ]) ).trim(); let r: any; if (await Bun.file(receiptFile).exists()) r = await receipt(); else { if (inventory) throw new Error("Unclaimed existing cluster; refusing adoption"); r = { lab, cluster: cfg.cluster, nonce: crypto.randomUUID(), state: "claimed", }; await writeFile(receiptFile, JSON.stringify(r) + "\n", { mode: 0o600, flag: "wx", }); } if (r.lab !== lab || r.cluster !== cfg.cluster) throw new Error("Receipt target differs"); if (!inventory) { r = { lab, cluster: cfg.cluster, nonce: r.state === "claimed" ? r.nonce : crypto.randomUUID(), state: "claimed", }; await save(r); if ( ( await command("public_port_guard", [ "ss", "-H", "-ltn", `sport = :${cfg.port}`, ]) ).trim() ) throw new Error("Public port is occupied"); await command("create_cluster", [ "k3d", "cluster", "create", cfg.cluster, "--image", "rancher/k3s:v1.37.1-k3s1@sha256:ca7f37d993d82ef0dcdcfecb2e0e2618ea541dbaffc620c8cedebe01a82acd0d", "--servers", "1", "--agents", "0", "--servers-memory", "8g", "--api-port", "127.0.0.1:0", "--port", `127.0.0.1:${cfg.port}:80@loadbalancer`, "--k3s-arg", "--disable=traefik@server:0", "--runtime-label", `vp.coursework.owner=${r.nonce}@server:0`, "--kubeconfig-update-default=false", "--kubeconfig-switch-context=false", "--timeout", "180s", ]); r.images = undefined; } else { const owner = ( await command("cluster_owner", [ "docker", "inspect", "--format", '{{index .Config.Labels "vp.coursework.owner"}}', "k3d-" + cfg.cluster + "-server-0", ]) ).trim(); if (owner !== r.nonce) throw new Error("Existing server owner differs"); await command("start_cluster", ["k3d", "cluster", "start", cfg.cluster]); } r.container = await serverID(); const config = Bun.YAML.parse( await command("private_kubeconfig", [ "k3d", "kubeconfig", "get", cfg.cluster, ]), ) as any; const ports = JSON.parse( await command("published_api_port", [ "docker", "inspect", "--format", "{{json .NetworkSettings.Ports}}", "k3d-" + cfg.cluster + "-serverlb", ]), ); const endpoint = ports["6443/tcp"]?.find( (p: any) => p.HostIp === "127.0.0.1", ); if ( !endpoint || !/^\d+$/.test(endpoint.HostPort) || Number(endpoint.HostPort) < 1024 || Number(endpoint.HostPort) > 65535 ) throw new Error("Unverified Kubernetes API port"); if (config.clusters?.length !== 1) throw new Error("Unexpected kubeconfig cluster set"); config.clusters[0].cluster.server = "https://127.0.0.1:" + endpoint.HostPort; const http = ports["80/tcp"]?.find((p: any) => p.HostIp === "127.0.0.1"); if (!http || Number(http.HostPort) !== cfg.port) throw new Error("Public endpoint differs from claimed cluster"); r.publicPort = cfg.port; await writeFile(kubeconfig, Bun.YAML.stringify(config), { mode: 0o600 }); await k(["wait", "--for=condition=Ready", "node", "--all", "--timeout=120s"]); let ns = await namespace(); if (ns && ns.metadata?.annotations?.["vp-mtuci/owner"] !== r.nonce) throw new Error("Foreign namespace"); if (!ns) { await k( ["create", "-f", "-"], JSON.stringify({ apiVersion: "v1", kind: "Namespace", metadata: { name: cfg.namespace, annotations: { "vp-mtuci/owner": r.nonce }, labels: { "app.kubernetes.io/part-of": "vp-coursework" }, }, }), ); ns = await namespace(); } r.namespaceUID = ns.metadata.uid; r.state = "ready"; await save(r); await command("credentials_bootstrap", [ process.execPath, "--no-env-file", resolve(root, "scripts/bootstrap.ts"), ]); const secretFiles = { pg_password: "pg_password", database_url: "database_url", cache_url: "cache_url", valkey_conf: "valkey.conf", ...(lab === 4 ? { s3_credentials: "s3_credentials" } : {}), }; const stringData: Record = {}; for (const [key, file] of Object.entries(secretFiles)) stringData[key] = ( await Bun.file(resolve(root, ".local", file)).text() ).trim(); await k( ["apply", "--server-side", "--field-manager=vp-coursework", "-f", "-"], JSON.stringify({ apiVersion: "v1", kind: "Secret", metadata: { name: "app-config", namespace: cfg.namespace }, type: "Opaque", stringData, }), ); if (lab === 4) { const data: Record = {}; for (const name of ["s3_access", "s3_secret", "s3_root"]) data[name] = ( await Bun.file(resolve(root, ".local", name)).text() ).trim(); await k( ["apply", "--server-side", "--field-manager=vp-coursework", "-f", "-"], JSON.stringify({ apiVersion: "v1", kind: "Secret", metadata: { name: "storage-root", namespace: cfg.namespace }, type: "Opaque", stringData: data, }), ); } const images = await imageSet(); async function imported() { return ( await command("containerd_inventory", [ "docker", "exec", "k3d-" + cfg.cluster + "-server-0", "ctr", "-n", "k8s.io", "images", "list", "-q", ]) ) .trim() .split("\n"); } let available = await imported(); if ( Object.values(images).some( (image) => !available.includes("docker.io/" + image), ) ) { const archive = resolve(local, "images-" + crypto.randomUUID() + ".tar"); try { await command( "export_single_platform", [ "docker", "image", "save", "--platform", "linux/amd64", "--output", archive, ...Object.values(images), ], undefined, 240000, ); stage = "import_single_platform"; const p = Bun.spawn( [ "docker", "exec", "-i", "k3d-" + cfg.cluster + "-server-0", "ctr", "-n", "k8s.io", "images", "import", "--platform", "linux/amd64", "-", ], { stdin: Bun.file(archive), stdout: "pipe", stderr: "pipe", signal: AbortSignal.timeout(240000), }, ); const [out, err] = await Promise.all([ new Response(p.stdout).text(), new Response(p.stderr).text(), ]); await writeFile(resolve(local, "image-import.txt"), out + err, { mode: 0o600, }); if ((await p.exited) !== 0) throw new Error("Native single-platform image import failed"); } finally { await rm(archive, { force: true }); } available = await imported(); if ( Object.values(images).some( (image) => !available.includes("docker.io/" + image), ) ) throw new Error("Imported images are missing from Kubernetes containerd"); } r.images = images; await save(r); await applyConfiguration(true); for (const name of ["postgres", "valkey", ...(lab === 4 ? ["s3"] : [])]) await readyDeployment(name); const sqlReady = await k([ "exec", "deployment/postgres", "--", "psql", "-h", "127.0.0.1", "-U", "messenger", "-d", "messenger", "-X", "-qAt", "-c", "SELECT 1;", ]); if (sqlReady.trim() !== "1") throw new Error("PostgreSQL did not execute the readiness query"); await job("migrate", []); if (lab === 4) await job("bootstrap-s3", []); await applyConfiguration(false); await ready(); await writeFile( resolve(evidence, "runtime.json"), JSON.stringify( { cluster: cfg.cluster, namespaceUID: r.namespaceUID, k3s: "v1.37.1-k3s1", pingap: "0.16.0", images, port: cfg.port, }, null, 2, ) + "\n", ); console.log("Ready: " + cfg.cluster + " via Pingap on localhost:" + cfg.port); } function integer( value: string | undefined, fallback: number, min: number, max: number, ) { const n = Number(value ?? fallback); if (!Number.isInteger(n) || n < min || n > max) throw new Error("Invalid bounded runtime setting"); return n; } async function applyConfiguration(initial: boolean) { const r = await receipt(), replicas = initial ? 0 : integer(process.env.API_REPLICAS, 2, 1, 4); const cpu = process.env.API_CPUS ?? "1"; if (!Number.isFinite(Number(cpu)) || Number(cpu) < 0.1 || Number(cpu) > 4) throw new Error("Invalid CPU limit"); const memory = process.env.API_MEMORY ?? (lab === 4 ? "2Gi" : "512Mi"); const match = /^(\d+)(Mi|Gi|m|g)$/.exec(memory); if (!match) throw new Error("Invalid memory limit"); const mib = Number(match[1]) * (match[2].toLowerCase().startsWith("g") ? 1024 : 1); if (mib < 128 || mib > 2048) throw new Error("Unsafe memory limit"); const soft = process.env.GO_MEMORY_LIMIT ?? (lab === 4 ? "1536MiB" : "400MiB"); const sm = /^(\d+)MiB$/.exec(soft); if (!sm || Number(sm[1]) > mib * 0.8 || Number(sm[1]) < 64) throw new Error("Unsafe soft memory limit"); assertImageBudget(lab, mib, soft); const apiPatch = { apiVersion: "apps/v1", kind: "Deployment", metadata: { name: "api" }, spec: { replicas, template: { spec: { containers: [ { name: "api", resources: { requests: { cpu: "100m", memory: "128Mi" }, limits: { cpu, memory: mib + "Mi" }, }, env: [ { name: "PG_MAX_CONNS", value: String(integer(process.env.PG_MAX_CONNS, 8, 1, 64)), }, { name: "SESSION_TTL_SECONDS", value: String( integer(process.env.SESSION_TTL_SECONDS, 3600, 1, 86400), ), }, { name: "GOMEMLIMIT", value: soft }, ], }, ], }, }, }, }; const render = resolve(local, "render"); const overlay = { apiVersion: "kustomize.config.k8s.io/v1beta1", kind: "Kustomization", resources: [relative(render, resolve(root, "deploy/k3s"))], images: Object.entries(r.images).map(([name, alias]) => ({ name: name === "api" ? "vp-messenger" : "vp-" + name, newName: (alias as string).split(":")[0], newTag: (alias as string).split(":")[1], })), patches: [ { target: { kind: "Deployment", name: "api" }, patch: Bun.YAML.stringify(apiPatch), }, ], }; await mkdir(render, { recursive: true }); await writeFile( resolve(render, "kustomization.yaml"), Bun.YAML.stringify(overlay), ); const manifest = await command("render_kustomize", [ "kubectl", "kustomize", render, "--load-restrictor=LoadRestrictionsNone", ]); // This runner owns the isolated namespace and reclaims replicas after explicit scale hooks. await k( [ "apply", "--dry-run=server", "--server-side", "--force-conflicts", "--field-manager=vp-coursework", "-f", "-", ], manifest, ); await k( [ "apply", "--server-side", "--force-conflicts", "--field-manager=vp-coursework", "-f", "-", ], manifest, ); } async function readyDeployment(name: string) { await k(["rollout", "status", "deployment/" + name, "--timeout=150s"]); } async function pods() { return JSON.parse(await k(["get", "pods", "-o", "json"])).items; } async function ready() { for (const name of [ "postgres", "valkey", "api", "pingap", ...(lab === 4 ? ["s3"] : []), ]) await readyDeployment(name); const drainDeadline = performance.now() + 40000; for (;;) { const draining = (await pods()).some( (p: any) => p.metadata.deletionTimestamp && ["api", "postgres", "valkey", "pingap", "s3"].includes( p.metadata.labels?.app, ), ); if (!draining) break; if (performance.now() >= drainDeadline) throw new Error("Old workload Pods did not finish terminating"); await Bun.sleep(250); } await command( "http_discovery_ready", [process.execPath, "--no-env-file", resolve(root, "student-tests/wait.ts")], undefined, 120000, ); } async function job(mode: string, extra: string[]) { const allowed = [ "migrate", "bootstrap-s3", "fixture-cleanup", "reconcile", "integration", "image-memory", ]; if (!allowed.includes(mode) || (lab === 3 && mode !== "migrate")) throw new Error("Unsupported Job"); if (mode === "integration" && !cfg.cluster.includes("-r-")) throw new Error( "Destructive integration requires a fresh research cluster", ); if (mode === "fixture-cleanup" && !/^st_[a-f0-9]{16}$/.test(extra[0] ?? "")) throw new Error("Invalid fixture prefix"); async function gameGuard() { const p = Bun.spawn(["game-running"], { stdout: "ignore", stderr: "pipe" }); await new Response(p.stderr).text(); assertNoGame(await p.exited); } if (mode === "image-memory") await gameGuard(); const template = JSON.parse( await k(["get", "deployment", "api", "-o", "json"]), ).spec.template; const name = "task-" + mode + "-" + crypto.randomUUID().slice(0, 8); const c = template.spec.containers[0]; delete c.readinessProbe; delete c.livenessProbe; delete c.startupProbe; delete c.lifecycle; delete c.ports; c.name = "task"; c.command = [ mode === "integration" ? "/integration.test" : mode === "image-memory" ? "/image-memory.test" : mode === "fixture-cleanup" ? "/fixture-cleanup" : "/messenger", ]; c.args = mode === "integration" ? [ "-test.v", "-test.run", "Test(ReconciliationAgainstPostgresAndPrivateS3|S3InventoryAndStatReturnComparableVersionMetadata|FixtureCleanupRejectsForeignMembership)", "-test.timeout=150s", ] : mode === "image-memory" ? [ "-test.v", "-test.run", "TestWorstCaseImagesWithinBudget", "-test.timeout=120s", ] : mode === "fixture-cleanup" ? extra : ["-" + mode, ...extra]; if (mode === "integration") c.env.push({ name: "RUN_ISOLATED_RECONCILIATION", value: "1" }); if (mode === "image-memory") { c.env.push({ name: "RUN_IMAGE_STRESS", value: "1" }); c.resources = { requests: { cpu: "100m", memory: "128Mi" }, limits: { cpu: "2", memory: "2Gi" }, }; } if (mode === "bootstrap-s3") { const volume = template.spec.volumes.find( (v: any) => v.name === "credentials", ); const items = volume.secret.items; delete volume.secret; volume.projected = { defaultMode: 288, sources: [ { secret: { name: "app-config", items } }, { secret: { name: "storage-root", items: [{ key: "s3_root", path: "s3_root" }], }, }, ], }; } template.metadata = { labels: { app: name } }; template.spec.restartPolicy = "Never"; template.spec.terminationGracePeriodSeconds = 10; await k( ["create", "-f", "-"], JSON.stringify({ apiVersion: "batch/v1", kind: "Job", metadata: { name }, spec: { backoffLimit: 0, activeDeadlineSeconds: 200, ttlSecondsAfterFinished: 300, template, }, }), ); let success = false; return withCleanup({ perform: async () => { for (let attempt = 0; attempt < 210; attempt++) { const j = JSON.parse(await k(["get", "job", name, "-o", "json"])); if (j.status?.succeeded) { success = true; break; } if (j.status?.failed) break; await Bun.sleep(1000); } const output = await k(["logs", "job/" + name]); if (!success) { await writeFile(resolve(local, "failed-job.txt"), output, { mode: 0o600, }); const snapshot = await k([ "get", "pods", "-l", "job-name=" + name, "-o", "json", ]); await writeFile(resolve(local, "failed-job-pods.json"), snapshot, { mode: 0o600, }); throw new Error("Job " + mode + " failed; private diagnostics saved"); } if (mode === "image-memory") await gameGuard(); return output; }, cleanup: async () => { await k(["delete", "job", name, "--wait=true", "--timeout=30s"]); }, }); } async function execute() { if (action === "info") { const r = await receipt(); console.log( JSON.stringify({ cluster: cfg.cluster, port: r.publicPort, namespace: cfg.namespace, images: r.images, }), ); } else if (action === "stats") { const out = []; for (const p of (await pods()).filter( (p: any) => p.metadata.labels?.app === "api" && !p.metadata.deletionTimestamp, )) { const id = p.status.containerStatuses?.[0]?.containerID?.replace( /^containerd:\/\//, "", ); if (!/^[a-f0-9]{64}$/.test(id ?? "")) throw new Error("API container identity unavailable"); const inspected = JSON.parse( await command("container_pid", [ "docker", "exec", "k3d-" + cfg.cluster + "-server-0", "crictl", "inspect", "-o", "json", id, ]), ); const pid = Number(inspected.info?.pid); if (!Number.isInteger(pid) || pid < 1) throw new Error("Invalid API process ID"); const raw = await command("container_cgroup", [ "docker", "exec", "k3d-" + cfg.cluster + "-server-0", "cat", "/proc/" + pid + "/cgroup", ]); const group = raw .split("\n") .find((x) => x.startsWith("0::")) ?.slice(3); if ( !group || !group.startsWith("/") || group.includes("..") || !group.includes(id) ) throw new Error("Unverified API cgroup path"); async function metric(file: string) { return command("cgroup_" + file.replace(".", "_"), [ "docker", "exec", "k3d-" + cfg.cluster + "-server-0", "cat", "/sys/fs/cgroup" + group + "/" + file, ]); } const [current, peak, cpu] = await Promise.all([ metric("memory.current"), metric("memory.peak"), metric("cpu.stat"), ]); const values = Object.fromEntries( cpu .trim() .split("\n") .map((x) => x.split(/\s+/)), ); out.push({ pod: p.metadata.name, uid: p.metadata.uid, image: p.status.containerStatuses[0].imageID, memory_current_bytes: Number(current.trim()), memory_peak_bytes: Number(peak.trim()), cpu_usage_usec: Number(values.usage_usec), limits: p.spec.containers[0].resources.limits, }); } console.log(JSON.stringify({ at: new Date().toISOString(), pods: out })); } else if (action === "configure") { await applyConfiguration(false); await ready(); } else if (action === "ready") await ready(); else if (action === "kubectl") { assertScopedArgs(args); const input = args.includes("-i") || args.includes("--stdin") ? await Bun.stdin.text() : undefined; if (input !== undefined && input.length > 1024 * 1024) throw new Error("Operational stdin exceeds limit"); process.stdout.write(await k(args, input)); } else if (action === "task") process.stdout.write(await job(args[0], args.slice(1))); else if (action === "restart") { const runID = crypto.randomUUID(); const selected = (items: any[]) => items .filter( (p) => !p.metadata.deletionTimestamp && ["api", "postgres", "valkey", "pingap", "s3"].includes( p.metadata.labels?.app, ), ) .map((p) => ({ name: p.metadata.name, uid: p.metadata.uid, role: p.metadata.labels.app, started: p.status.startTime, })); const before = selected(await pods()); if (!before.length) throw new Error("No workload Pods to restart"); await writeFile( resolve(evidence, "restart-before.json"), JSON.stringify({ run_id: runID, pods: before }, null, 2) + "\n", ); await k(["rollout", "restart", "deployment"]); await ready(); const after = selected(await pods()); if (!after.length || after.some((p) => before.some((b) => b.uid === p.uid))) throw new Error("An original workload Pod was not replaced"); await writeFile( resolve(evidence, "restart-after.json"), JSON.stringify({ run_id: runID, pods: after }, null, 2) + "\n", ); } else if (action === "stop-one") { const target = process.env.TARGET_INSTANCE_ID; if (!target || !/^[a-z0-9-]+$/.test(target)) throw new Error("Actual target pod name required"); const items = (await pods()).filter( (p: any) => p.metadata.labels?.app === "api" && !p.metadata.deletionTimestamp && p.status.conditions?.some( (c: any) => c.type === "Ready" && c.status === "True", ), ); const victim = items.find((p: any) => p.metadata.name === target); if (items.length !== 2 || !victim) throw new Error("Target must belong to two ready API pods"); await writeFile( resolve(evidence, "stop-one-before.json"), JSON.stringify( { target, uid: victim.metadata.uid, pods: items.map((p: any) => p.metadata.name), }, null, 2, ), ); const containerID = victim.status.containerStatuses?.[0]?.containerID?.replace( /^containerd:\/\//, "", ); if (!/^[a-f0-9]{64}$/.test(containerID ?? "")) throw new Error("Unverified target container"); for (const p of items) await k([ "patch", "pod", p.metadata.name, "--type=merge", "-p", JSON.stringify({ metadata: { uid: p.metadata.uid, annotations: { "controller.kubernetes.io/pod-deletion-cost": p === victim ? "-1000000" : "1000000", }, }, }), ]); await k([ "scale", "deployment/api", "--current-replicas=2", "--replicas=1", ]); let terminating: any; for (let i = 0; i < 15; i++) { const current = (await pods()).find( (p: any) => p.metadata.uid === victim.metadata.uid, ); if (current?.metadata.deletionTimestamp) { terminating = current; break; } await Bun.sleep(100); } if (!terminating) throw new Error("Controller did not select the intended target"); const currentCID = terminating.status.containerStatuses?.[0]?.containerID?.replace( /^containerd:\/\//, "", ); if (!/^[a-f0-9]{64}$/.test(currentCID ?? "")) throw new Error("Terminating process identity unavailable"); await command( "stop_target_process", [ "docker", "exec", "k3d-" + cfg.cluster + "-server-0", "ctr", "-n", "k8s.io", "tasks", "kill", "--signal", "SIGKILL", currentCID, ], undefined, 4000, ); let stopped: any; for (let i = 0; i < 15; i++) { stopped = JSON.parse( await command("verify_stopped_process", [ "docker", "exec", "k3d-" + cfg.cluster + "-server-0", "crictl", "inspect", "-o", "json", currentCID, ]), ); if (stopped.status?.state === "CONTAINER_EXITED") break; await Bun.sleep(100); } if (stopped.status?.state !== "CONTAINER_EXITED") throw new Error("Target process did not stop"); if ( stopped.status?.labels?.["io.kubernetes.pod.uid"] !== victim.metadata.uid ) throw new Error("Stopped process Pod UID differs"); const running = JSON.parse( await command("verify_no_running_victim", [ "docker", "exec", "k3d-" + cfg.cluster + "-server-0", "crictl", "ps", "--state", "Running", "-o", "json", ]), ); assertNoRunningContainers(running.containers ?? [], victim.metadata.uid); const finalVictim = (await pods()).find( (p: any) => p.metadata.uid === victim.metadata.uid, ); if ( finalVictim?.status.containerStatuses?.some( (c: any) => c.containerID !== "containerd://" + currentCID, ) ) throw new Error("Target process changed during failure injection"); const remaining = (await pods()).filter( (p: any) => p.metadata.labels?.app === "api" && !p.metadata.deletionTimestamp, ); if ( remaining.length !== 1 || remaining[0].metadata.uid === victim.metadata.uid ) throw new Error("Unexpected API survivor"); await writeFile( resolve(evidence, "stop-one-after.json"), JSON.stringify( { stopped: target, stoppedUID: victim.metadata.uid, containerID: currentCID, state: "CONTAINER_EXITED", survivor: remaining[0].metadata.name, }, null, 2, ), ); } else if (action === "destroy") { await command("destroy_claimed_cluster", [ "k3d", "cluster", "delete", cfg.cluster, ]); const r = await receipt(); r.state = "destroyed"; await save(r); } else if (action === "stop") await command("stop_claimed_cluster", [ "k3d", "cluster", "stop", cfg.cluster, ]); else throw new Error("Unknown runtime action"); } async function destroyPartial() { const r = await receipt(); const inventory = ( await command("partial_inventory", [ "docker", "ps", "-aq", "--filter", "label=k3d.cluster=" + cfg.cluster, ]) ).trim(); if (!inventory) { r.state = "destroyed"; await save(r); return; } const inspected = JSON.parse( await command("partial_server_identity", [ "docker", "inspect", "k3d-" + cfg.cluster + "-server-0", ]), ); const server = { container: inspected[0].Id, owner: inspected[0].Config.Labels["vp.coursework.owner"], }; assertPartialClaim(r, server, cfg); r.container = server.container; await save(r); await command("destroy_partial_claimed_cluster", [ "k3d", "cluster", "delete", cfg.cluster, ]); r.state = "destroyed"; await save(r); } try { if (action === "check") console.log("Parsed k3s runtime for lab" + lab); else if (action === "up") await bootstrap(); else if (action === "destroy" && (await receipt()).state === "claimed") await destroyPartial(); else await operator.run(action); } catch (error) { await writeFile( resolve(evidence, "last-failure.json"), JSON.stringify({ stage, action, error: error instanceof Error ? error.message : "unknown", }) + "\n", ); throw error; }