Files
mtusi2026lab4/scripts/k3s.ts

1180 lines
33 KiB
TypeScript

import { mkdir, readFile, readdir, writeFile, rm } from "node:fs/promises";
import { resolve, relative } from "node:path";
import {
settings,
guardedOperator,
pinnedConfigID,
assertScopedArgs,
assertNoGame,
assertPartialClaim,
assertNoRunningContainers,
withCleanup,
assertImageBudget,
} from "./k3s-model";
const root = resolve(import.meta.dir, "..");
const lab = (await Bun.file(resolve(root, "lab.json")).json()).number;
const cfg = settings(lab, process.env);
const local = resolve(root, ".local/k3s", cfg.cluster),
kubeconfig = resolve(local, "kubeconfig"),
receiptFile = resolve(local, "receipt.json");
const evidence = resolve(root, "evidence/k3s");
await mkdir(local, { recursive: true, mode: 0o700 });
await mkdir(evidence, { recursive: true });
const action = process.argv[2] ?? "ready";
const args = process.argv.slice(3);
let stage = "start";
async function command(
label: string,
argv: string[],
input?: string,
timeout = 240000,
) {
stage = label;
const p = Bun.spawn(argv, {
cwd: root,
stdin: input === undefined ? "ignore" : "pipe",
stdout: "pipe",
stderr: "pipe",
signal: AbortSignal.timeout(timeout),
});
if (input !== undefined) {
p.stdin.write(input);
p.stdin.end();
}
const [out, err] = await Promise.all([
new Response(p.stdout).text(),
new Response(p.stderr).text(),
]);
if ((await p.exited) !== 0) {
await writeFile(resolve(local, "last-error.txt"), out + err, {
mode: 0o600,
});
throw new Error(label + " failed; private diagnostics in .local/k3s");
}
return out;
}
function k(argv: string[], input?: string) {
return command(
"kubernetes_" + argv[0],
[
"kubectl",
"--kubeconfig",
kubeconfig,
"--context",
"k3d-" + cfg.cluster,
"--namespace",
cfg.namespace,
...argv,
],
input,
);
}
async function receipt() {
return Bun.file(receiptFile).json();
}
async function save(r: any) {
await writeFile(receiptFile, JSON.stringify(r) + "\n", { mode: 0o600 });
}
async function serverID() {
return (
await command("server_identity", [
"docker",
"inspect",
"--format",
"{{.Id}}",
"k3d-" + cfg.cluster + "-server-0",
])
).trim();
}
async function namespace() {
const out = await k([
"get",
"namespace",
cfg.namespace,
"--ignore-not-found",
"-o",
"json",
]);
return out.trim() ? JSON.parse(out) : undefined;
}
async function actual() {
return {
container: await serverID(),
namespaceUID: (await namespace())?.metadata?.uid,
};
}
const operator = guardedOperator(cfg, {
receipt,
actual,
dispatch: async () => execute(),
});
const sources = {
postgres:
"postgres:18.6@sha256:74935e72241653ca55e0414067e6d8763aceb8a810eb51b452253ec3dcfc4336",
valkey:
"valkey/valkey:9@sha256:418652cfb58ef879d4978c33553735d7147016032d5aefaa14c828e611eb9dfd",
pingap:
"vicanso/pingap:0.16.0@sha256:e5b6f38a946e3105e0d98271be95e4a2dca0441f7962efcfd30371a3b55405e7",
...(lab === 4
? {
s3: "rustfs/rustfs:1.0.0-rc.1@sha256:f53d700fc16809070326f6ab3fb565d48e722364a7bd5a12e47d0a5abe079ce6",
}
: {}),
};
async function imageSet() {
const hash = new Bun.CryptoHasher("sha256");
hash.update(await Bun.file(resolve(root, "Dockerfile")).arrayBuffer());
async function walk(dir: string) {
for (const item of (await readdir(dir, { withFileTypes: true })).sort(
(a, b) => a.name.localeCompare(b.name),
)) {
if (item.name.startsWith(".local")) continue;
const file = resolve(dir, item.name);
if (item.isDirectory()) await walk(file);
else if (item.isFile()) {
hash.update(file.slice(root.length));
hash.update(await readFile(file));
}
}
}
await walk(resolve(root, "backend"));
const version = hash.digest("hex").slice(0, 16),
api = `vp-coursework/api-lab${lab}:${version}`;
await command("build_api", ["docker", "build", "-t", api, "."]);
const images: Record<string, string> = { api };
const cached = new Set(
(
await command("local_image_inventory", [
"docker",
"image",
"ls",
"--digests",
"--format",
"{{.Repository}}@{{.Digest}}",
])
)
.trim()
.split("\n")
.map((ref) => ref.replace(/^docker\.io\//, "").replace(/^library\//, "")),
);
for (const [name, source] of Object.entries(sources)) {
const digest = source!.split("@sha256:")[1],
alias = `vp-coursework/${name}:${digest.slice(0, 16)}`;
const repository = source!.split("@")[0].replace(/:[^/:]+$/, "");
if (!cached.has(repository + "@sha256:" + digest))
await command("pull_pinned_image_" + name, ["docker", "pull", source!]);
const pinned = JSON.parse(
await command("verify_image_" + name, [
"docker",
"image",
"inspect",
source!,
]),
);
const configID = pinnedConfigID(pinned[0], source!);
await command("alias_image_" + name, ["docker", "tag", source!, alias]);
if (
(
await command("verify_alias_" + name, [
"docker",
"image",
"inspect",
alias,
"--format",
"{{.Id}}",
])
).trim() !== configID
)
throw new Error("Image alias content differs");
images[name] = alias;
}
return images;
}
async function bootstrap() {
const inventory = (
await command("cluster_inventory", [
"docker",
"ps",
"-aq",
"--filter",
"label=k3d.cluster=" + cfg.cluster,
])
).trim();
let r: any;
if (await Bun.file(receiptFile).exists()) r = await receipt();
else {
if (inventory)
throw new Error("Unclaimed existing cluster; refusing adoption");
r = {
lab,
cluster: cfg.cluster,
nonce: crypto.randomUUID(),
state: "claimed",
};
await writeFile(receiptFile, JSON.stringify(r) + "\n", {
mode: 0o600,
flag: "wx",
});
}
if (r.lab !== lab || r.cluster !== cfg.cluster)
throw new Error("Receipt target differs");
if (!inventory) {
r = {
lab,
cluster: cfg.cluster,
nonce: r.state === "claimed" ? r.nonce : crypto.randomUUID(),
state: "claimed",
};
await save(r);
if (
(
await command("public_port_guard", [
"ss",
"-H",
"-ltn",
`sport = :${cfg.port}`,
])
).trim()
)
throw new Error("Public port is occupied");
await command("create_cluster", [
"k3d",
"cluster",
"create",
cfg.cluster,
"--image",
"rancher/k3s:v1.37.1-k3s1@sha256:ca7f37d993d82ef0dcdcfecb2e0e2618ea541dbaffc620c8cedebe01a82acd0d",
"--servers",
"1",
"--agents",
"0",
"--servers-memory",
"8g",
"--api-port",
"127.0.0.1:0",
"--port",
`127.0.0.1:${cfg.port}:80@loadbalancer`,
"--k3s-arg",
"--disable=traefik@server:0",
"--runtime-label",
`vp.coursework.owner=${r.nonce}@server:0`,
"--kubeconfig-update-default=false",
"--kubeconfig-switch-context=false",
"--timeout",
"180s",
]);
r.images = undefined;
} else {
const owner = (
await command("cluster_owner", [
"docker",
"inspect",
"--format",
'{{index .Config.Labels "vp.coursework.owner"}}',
"k3d-" + cfg.cluster + "-server-0",
])
).trim();
if (owner !== r.nonce) throw new Error("Existing server owner differs");
await command("start_cluster", ["k3d", "cluster", "start", cfg.cluster]);
}
r.container = await serverID();
const config = Bun.YAML.parse(
await command("private_kubeconfig", [
"k3d",
"kubeconfig",
"get",
cfg.cluster,
]),
) as any;
const ports = JSON.parse(
await command("published_api_port", [
"docker",
"inspect",
"--format",
"{{json .NetworkSettings.Ports}}",
"k3d-" + cfg.cluster + "-serverlb",
]),
);
const endpoint = ports["6443/tcp"]?.find(
(p: any) => p.HostIp === "127.0.0.1",
);
if (
!endpoint ||
!/^\d+$/.test(endpoint.HostPort) ||
Number(endpoint.HostPort) < 1024 ||
Number(endpoint.HostPort) > 65535
)
throw new Error("Unverified Kubernetes API port");
if (config.clusters?.length !== 1)
throw new Error("Unexpected kubeconfig cluster set");
config.clusters[0].cluster.server = "https://127.0.0.1:" + endpoint.HostPort;
const http = ports["80/tcp"]?.find((p: any) => p.HostIp === "127.0.0.1");
if (!http || Number(http.HostPort) !== cfg.port)
throw new Error("Public endpoint differs from claimed cluster");
r.publicPort = cfg.port;
await writeFile(kubeconfig, Bun.YAML.stringify(config), { mode: 0o600 });
await k(["wait", "--for=condition=Ready", "node", "--all", "--timeout=120s"]);
let ns = await namespace();
if (ns && ns.metadata?.annotations?.["vp-mtuci/owner"] !== r.nonce)
throw new Error("Foreign namespace");
if (!ns) {
await k(
["create", "-f", "-"],
JSON.stringify({
apiVersion: "v1",
kind: "Namespace",
metadata: {
name: cfg.namespace,
annotations: { "vp-mtuci/owner": r.nonce },
labels: { "app.kubernetes.io/part-of": "vp-coursework" },
},
}),
);
ns = await namespace();
}
r.namespaceUID = ns.metadata.uid;
r.state = "ready";
await save(r);
await command("credentials_bootstrap", [
process.execPath,
"--no-env-file",
resolve(root, "scripts/bootstrap.ts"),
]);
const secretFiles = {
pg_password: "pg_password",
database_url: "database_url",
cache_url: "cache_url",
valkey_conf: "valkey.conf",
...(lab === 4 ? { s3_credentials: "s3_credentials" } : {}),
};
const stringData: Record<string, string> = {};
for (const [key, file] of Object.entries(secretFiles))
stringData[key] = (
await Bun.file(resolve(root, ".local", file)).text()
).trim();
await k(
["apply", "--server-side", "--field-manager=vp-coursework", "-f", "-"],
JSON.stringify({
apiVersion: "v1",
kind: "Secret",
metadata: { name: "app-config", namespace: cfg.namespace },
type: "Opaque",
stringData,
}),
);
if (lab === 4) {
const data: Record<string, string> = {};
for (const name of ["s3_access", "s3_secret", "s3_root"])
data[name] = (
await Bun.file(resolve(root, ".local", name)).text()
).trim();
await k(
["apply", "--server-side", "--field-manager=vp-coursework", "-f", "-"],
JSON.stringify({
apiVersion: "v1",
kind: "Secret",
metadata: { name: "storage-root", namespace: cfg.namespace },
type: "Opaque",
stringData: data,
}),
);
}
const images = await imageSet();
async function imported() {
return (
await command("containerd_inventory", [
"docker",
"exec",
"k3d-" + cfg.cluster + "-server-0",
"ctr",
"-n",
"k8s.io",
"images",
"list",
"-q",
])
)
.trim()
.split("\n");
}
let available = await imported();
if (
Object.values(images).some(
(image) => !available.includes("docker.io/" + image),
)
) {
const archive = resolve(local, "images-" + crypto.randomUUID() + ".tar");
try {
await command(
"export_single_platform",
[
"docker",
"image",
"save",
"--platform",
"linux/amd64",
"--output",
archive,
...Object.values(images),
],
undefined,
240000,
);
stage = "import_single_platform";
const p = Bun.spawn(
[
"docker",
"exec",
"-i",
"k3d-" + cfg.cluster + "-server-0",
"ctr",
"-n",
"k8s.io",
"images",
"import",
"--platform",
"linux/amd64",
"-",
],
{
stdin: Bun.file(archive),
stdout: "pipe",
stderr: "pipe",
signal: AbortSignal.timeout(240000),
},
);
const [out, err] = await Promise.all([
new Response(p.stdout).text(),
new Response(p.stderr).text(),
]);
await writeFile(resolve(local, "image-import.txt"), out + err, {
mode: 0o600,
});
if ((await p.exited) !== 0)
throw new Error("Native single-platform image import failed");
} finally {
await rm(archive, { force: true });
}
available = await imported();
if (
Object.values(images).some(
(image) => !available.includes("docker.io/" + image),
)
)
throw new Error("Imported images are missing from Kubernetes containerd");
}
r.images = images;
await save(r);
await applyConfiguration(true);
for (const name of ["postgres", "valkey", ...(lab === 4 ? ["s3"] : [])])
await readyDeployment(name);
const sqlReady = await k([
"exec",
"deployment/postgres",
"--",
"psql",
"-h",
"127.0.0.1",
"-U",
"messenger",
"-d",
"messenger",
"-X",
"-qAt",
"-c",
"SELECT 1;",
]);
if (sqlReady.trim() !== "1")
throw new Error("PostgreSQL did not execute the readiness query");
await job("migrate", []);
if (lab === 4) await job("bootstrap-s3", []);
await applyConfiguration(false);
await ready();
await writeFile(
resolve(evidence, "runtime.json"),
JSON.stringify(
{
cluster: cfg.cluster,
namespaceUID: r.namespaceUID,
k3s: "v1.37.1-k3s1",
pingap: "0.16.0",
images,
port: cfg.port,
},
null,
2,
) + "\n",
);
console.log("Ready: " + cfg.cluster + " via Pingap on localhost:" + cfg.port);
}
function integer(
value: string | undefined,
fallback: number,
min: number,
max: number,
) {
const n = Number(value ?? fallback);
if (!Number.isInteger(n) || n < min || n > max)
throw new Error("Invalid bounded runtime setting");
return n;
}
async function applyConfiguration(initial: boolean) {
const r = await receipt(),
replicas = initial ? 0 : integer(process.env.API_REPLICAS, 2, 1, 4);
const cpu = process.env.API_CPUS ?? "1";
if (!Number.isFinite(Number(cpu)) || Number(cpu) < 0.1 || Number(cpu) > 4)
throw new Error("Invalid CPU limit");
const memory = process.env.API_MEMORY ?? (lab === 4 ? "2Gi" : "512Mi");
const match = /^(\d+)(Mi|Gi|m|g)$/.exec(memory);
if (!match) throw new Error("Invalid memory limit");
const mib =
Number(match[1]) * (match[2].toLowerCase().startsWith("g") ? 1024 : 1);
if (mib < 128 || mib > 2048) throw new Error("Unsafe memory limit");
const soft =
process.env.GO_MEMORY_LIMIT ?? (lab === 4 ? "1536MiB" : "400MiB");
const sm = /^(\d+)MiB$/.exec(soft);
if (!sm || Number(sm[1]) > mib * 0.8 || Number(sm[1]) < 64)
throw new Error("Unsafe soft memory limit");
assertImageBudget(lab, mib, soft);
const apiPatch = {
apiVersion: "apps/v1",
kind: "Deployment",
metadata: { name: "api" },
spec: {
replicas,
template: {
spec: {
containers: [
{
name: "api",
resources: {
requests: { cpu: "100m", memory: "128Mi" },
limits: { cpu, memory: mib + "Mi" },
},
env: [
{
name: "PG_MAX_CONNS",
value: String(integer(process.env.PG_MAX_CONNS, 8, 1, 64)),
},
{
name: "SESSION_TTL_SECONDS",
value: String(
integer(process.env.SESSION_TTL_SECONDS, 3600, 1, 86400),
),
},
{ name: "GOMEMLIMIT", value: soft },
],
},
],
},
},
},
};
const render = resolve(local, "render");
const overlay = {
apiVersion: "kustomize.config.k8s.io/v1beta1",
kind: "Kustomization",
resources: [relative(render, resolve(root, "deploy/k3s"))],
images: Object.entries(r.images).map(([name, alias]) => ({
name: name === "api" ? "vp-messenger" : "vp-" + name,
newName: (alias as string).split(":")[0],
newTag: (alias as string).split(":")[1],
})),
patches: [
{
target: { kind: "Deployment", name: "api" },
patch: Bun.YAML.stringify(apiPatch),
},
],
};
await mkdir(render, { recursive: true });
await writeFile(
resolve(render, "kustomization.yaml"),
Bun.YAML.stringify(overlay),
);
const manifest = await command("render_kustomize", [
"kubectl",
"kustomize",
render,
"--load-restrictor=LoadRestrictionsNone",
]);
// This runner owns the isolated namespace and reclaims replicas after explicit scale hooks.
await k(
[
"apply",
"--dry-run=server",
"--server-side",
"--force-conflicts",
"--field-manager=vp-coursework",
"-f",
"-",
],
manifest,
);
await k(
[
"apply",
"--server-side",
"--force-conflicts",
"--field-manager=vp-coursework",
"-f",
"-",
],
manifest,
);
}
async function readyDeployment(name: string) {
await k(["rollout", "status", "deployment/" + name, "--timeout=150s"]);
}
async function pods() {
return JSON.parse(await k(["get", "pods", "-o", "json"])).items;
}
async function ready() {
for (const name of [
"postgres",
"valkey",
"api",
"pingap",
...(lab === 4 ? ["s3"] : []),
])
await readyDeployment(name);
const drainDeadline = performance.now() + 40000;
for (;;) {
const draining = (await pods()).some(
(p: any) =>
p.metadata.deletionTimestamp &&
["api", "postgres", "valkey", "pingap", "s3"].includes(
p.metadata.labels?.app,
),
);
if (!draining) break;
if (performance.now() >= drainDeadline)
throw new Error("Old workload Pods did not finish terminating");
await Bun.sleep(250);
}
await command(
"http_discovery_ready",
[process.execPath, "--no-env-file", resolve(root, "student-tests/wait.ts")],
undefined,
120000,
);
}
async function job(mode: string, extra: string[]) {
const allowed = [
"migrate",
"bootstrap-s3",
"fixture-cleanup",
"reconcile",
"integration",
"image-memory",
];
if (!allowed.includes(mode) || (lab === 3 && mode !== "migrate"))
throw new Error("Unsupported Job");
if (mode === "integration" && !cfg.cluster.includes("-r-"))
throw new Error(
"Destructive integration requires a fresh research cluster",
);
if (mode === "fixture-cleanup" && !/^st_[a-f0-9]{16}$/.test(extra[0] ?? ""))
throw new Error("Invalid fixture prefix");
async function gameGuard() {
const p = Bun.spawn(["game-running"], { stdout: "ignore", stderr: "pipe" });
await new Response(p.stderr).text();
assertNoGame(await p.exited);
}
if (mode === "image-memory") await gameGuard();
const template = JSON.parse(
await k(["get", "deployment", "api", "-o", "json"]),
).spec.template;
const name = "task-" + mode + "-" + crypto.randomUUID().slice(0, 8);
const c = template.spec.containers[0];
delete c.readinessProbe;
delete c.livenessProbe;
delete c.startupProbe;
delete c.lifecycle;
delete c.ports;
c.name = "task";
c.command = [
mode === "integration"
? "/integration.test"
: mode === "image-memory"
? "/image-memory.test"
: mode === "fixture-cleanup"
? "/fixture-cleanup"
: "/messenger",
];
c.args =
mode === "integration"
? [
"-test.v",
"-test.run",
"Test(ReconciliationAgainstPostgresAndPrivateS3|S3InventoryAndStatReturnComparableVersionMetadata|FixtureCleanupRejectsForeignMembership)",
"-test.timeout=150s",
]
: mode === "image-memory"
? [
"-test.v",
"-test.run",
"TestWorstCaseImagesWithinBudget",
"-test.timeout=120s",
]
: mode === "fixture-cleanup"
? extra
: ["-" + mode, ...extra];
if (mode === "integration")
c.env.push({ name: "RUN_ISOLATED_RECONCILIATION", value: "1" });
if (mode === "image-memory") {
c.env.push({ name: "RUN_IMAGE_STRESS", value: "1" });
c.resources = {
requests: { cpu: "100m", memory: "128Mi" },
limits: { cpu: "2", memory: "2Gi" },
};
}
if (mode === "bootstrap-s3") {
const volume = template.spec.volumes.find(
(v: any) => v.name === "credentials",
);
const items = volume.secret.items;
delete volume.secret;
volume.projected = {
defaultMode: 288,
sources: [
{ secret: { name: "app-config", items } },
{
secret: {
name: "storage-root",
items: [{ key: "s3_root", path: "s3_root" }],
},
},
],
};
}
template.metadata = { labels: { app: name } };
template.spec.restartPolicy = "Never";
template.spec.terminationGracePeriodSeconds = 10;
await k(
["create", "-f", "-"],
JSON.stringify({
apiVersion: "batch/v1",
kind: "Job",
metadata: { name },
spec: {
backoffLimit: 0,
activeDeadlineSeconds: 200,
ttlSecondsAfterFinished: 300,
template,
},
}),
);
let success = false;
return withCleanup({
perform: async () => {
for (let attempt = 0; attempt < 210; attempt++) {
const j = JSON.parse(await k(["get", "job", name, "-o", "json"]));
if (j.status?.succeeded) {
success = true;
break;
}
if (j.status?.failed) break;
await Bun.sleep(1000);
}
const output = await k(["logs", "job/" + name]);
if (!success) {
await writeFile(resolve(local, "failed-job.txt"), output, {
mode: 0o600,
});
const snapshot = await k([
"get",
"pods",
"-l",
"job-name=" + name,
"-o",
"json",
]);
await writeFile(resolve(local, "failed-job-pods.json"), snapshot, {
mode: 0o600,
});
throw new Error("Job " + mode + " failed; private diagnostics saved");
}
if (mode === "image-memory") await gameGuard();
return output;
},
cleanup: async () => {
await k(["delete", "job", name, "--wait=true", "--timeout=30s"]);
},
});
}
async function execute() {
if (action === "info") {
const r = await receipt();
console.log(
JSON.stringify({
cluster: cfg.cluster,
port: r.publicPort,
namespace: cfg.namespace,
images: r.images,
}),
);
} else if (action === "stats") {
const out = [];
for (const p of (await pods()).filter(
(p: any) =>
p.metadata.labels?.app === "api" && !p.metadata.deletionTimestamp,
)) {
const id = p.status.containerStatuses?.[0]?.containerID?.replace(
/^containerd:\/\//,
"",
);
if (!/^[a-f0-9]{64}$/.test(id ?? ""))
throw new Error("API container identity unavailable");
const inspected = JSON.parse(
await command("container_pid", [
"docker",
"exec",
"k3d-" + cfg.cluster + "-server-0",
"crictl",
"inspect",
"-o",
"json",
id,
]),
);
const pid = Number(inspected.info?.pid);
if (!Number.isInteger(pid) || pid < 1)
throw new Error("Invalid API process ID");
const raw = await command("container_cgroup", [
"docker",
"exec",
"k3d-" + cfg.cluster + "-server-0",
"cat",
"/proc/" + pid + "/cgroup",
]);
const group = raw
.split("\n")
.find((x) => x.startsWith("0::"))
?.slice(3);
if (
!group ||
!group.startsWith("/") ||
group.includes("..") ||
!group.includes(id)
)
throw new Error("Unverified API cgroup path");
async function metric(file: string) {
return command("cgroup_" + file.replace(".", "_"), [
"docker",
"exec",
"k3d-" + cfg.cluster + "-server-0",
"cat",
"/sys/fs/cgroup" + group + "/" + file,
]);
}
const [current, peak, cpu] = await Promise.all([
metric("memory.current"),
metric("memory.peak"),
metric("cpu.stat"),
]);
const values = Object.fromEntries(
cpu
.trim()
.split("\n")
.map((x) => x.split(/\s+/)),
);
out.push({
pod: p.metadata.name,
uid: p.metadata.uid,
image: p.status.containerStatuses[0].imageID,
memory_current_bytes: Number(current.trim()),
memory_peak_bytes: Number(peak.trim()),
cpu_usage_usec: Number(values.usage_usec),
limits: p.spec.containers[0].resources.limits,
});
}
console.log(JSON.stringify({ at: new Date().toISOString(), pods: out }));
} else if (action === "configure") {
await applyConfiguration(false);
await ready();
} else if (action === "ready") await ready();
else if (action === "kubectl") {
assertScopedArgs(args);
const input =
args.includes("-i") || args.includes("--stdin")
? await Bun.stdin.text()
: undefined;
if (input !== undefined && input.length > 1024 * 1024)
throw new Error("Operational stdin exceeds limit");
process.stdout.write(await k(args, input));
} else if (action === "task")
process.stdout.write(await job(args[0], args.slice(1)));
else if (action === "restart") {
const runID = crypto.randomUUID();
const selected = (items: any[]) =>
items
.filter(
(p) =>
!p.metadata.deletionTimestamp &&
["api", "postgres", "valkey", "pingap", "s3"].includes(
p.metadata.labels?.app,
),
)
.map((p) => ({
name: p.metadata.name,
uid: p.metadata.uid,
role: p.metadata.labels.app,
started: p.status.startTime,
}));
const before = selected(await pods());
if (!before.length) throw new Error("No workload Pods to restart");
await writeFile(
resolve(evidence, "restart-before.json"),
JSON.stringify({ run_id: runID, pods: before }, null, 2) + "\n",
);
await k(["rollout", "restart", "deployment"]);
await ready();
const after = selected(await pods());
if (!after.length || after.some((p) => before.some((b) => b.uid === p.uid)))
throw new Error("An original workload Pod was not replaced");
await writeFile(
resolve(evidence, "restart-after.json"),
JSON.stringify({ run_id: runID, pods: after }, null, 2) + "\n",
);
} else if (action === "stop-one") {
const target = process.env.TARGET_INSTANCE_ID;
if (!target || !/^[a-z0-9-]+$/.test(target))
throw new Error("Actual target pod name required");
const items = (await pods()).filter(
(p: any) =>
p.metadata.labels?.app === "api" &&
!p.metadata.deletionTimestamp &&
p.status.conditions?.some(
(c: any) => c.type === "Ready" && c.status === "True",
),
);
const victim = items.find((p: any) => p.metadata.name === target);
if (items.length !== 2 || !victim)
throw new Error("Target must belong to two ready API pods");
await writeFile(
resolve(evidence, "stop-one-before.json"),
JSON.stringify(
{
target,
uid: victim.metadata.uid,
pods: items.map((p: any) => p.metadata.name),
},
null,
2,
),
);
const containerID =
victim.status.containerStatuses?.[0]?.containerID?.replace(
/^containerd:\/\//,
"",
);
if (!/^[a-f0-9]{64}$/.test(containerID ?? ""))
throw new Error("Unverified target container");
for (const p of items)
await k([
"patch",
"pod",
p.metadata.name,
"--type=merge",
"-p",
JSON.stringify({
metadata: {
uid: p.metadata.uid,
annotations: {
"controller.kubernetes.io/pod-deletion-cost":
p === victim ? "-1000000" : "1000000",
},
},
}),
]);
await k([
"scale",
"deployment/api",
"--current-replicas=2",
"--replicas=1",
]);
let terminating: any;
for (let i = 0; i < 15; i++) {
const current = (await pods()).find(
(p: any) => p.metadata.uid === victim.metadata.uid,
);
if (current?.metadata.deletionTimestamp) {
terminating = current;
break;
}
await Bun.sleep(100);
}
if (!terminating)
throw new Error("Controller did not select the intended target");
const currentCID =
terminating.status.containerStatuses?.[0]?.containerID?.replace(
/^containerd:\/\//,
"",
);
if (!/^[a-f0-9]{64}$/.test(currentCID ?? ""))
throw new Error("Terminating process identity unavailable");
await command(
"stop_target_process",
[
"docker",
"exec",
"k3d-" + cfg.cluster + "-server-0",
"ctr",
"-n",
"k8s.io",
"tasks",
"kill",
"--signal",
"SIGKILL",
currentCID,
],
undefined,
4000,
);
let stopped: any;
for (let i = 0; i < 15; i++) {
stopped = JSON.parse(
await command("verify_stopped_process", [
"docker",
"exec",
"k3d-" + cfg.cluster + "-server-0",
"crictl",
"inspect",
"-o",
"json",
currentCID,
]),
);
if (stopped.status?.state === "CONTAINER_EXITED") break;
await Bun.sleep(100);
}
if (stopped.status?.state !== "CONTAINER_EXITED")
throw new Error("Target process did not stop");
if (
stopped.status?.labels?.["io.kubernetes.pod.uid"] !== victim.metadata.uid
)
throw new Error("Stopped process Pod UID differs");
const running = JSON.parse(
await command("verify_no_running_victim", [
"docker",
"exec",
"k3d-" + cfg.cluster + "-server-0",
"crictl",
"ps",
"--state",
"Running",
"-o",
"json",
]),
);
assertNoRunningContainers(running.containers ?? [], victim.metadata.uid);
const finalVictim = (await pods()).find(
(p: any) => p.metadata.uid === victim.metadata.uid,
);
if (
finalVictim?.status.containerStatuses?.some(
(c: any) => c.containerID !== "containerd://" + currentCID,
)
)
throw new Error("Target process changed during failure injection");
const remaining = (await pods()).filter(
(p: any) =>
p.metadata.labels?.app === "api" && !p.metadata.deletionTimestamp,
);
if (
remaining.length !== 1 ||
remaining[0].metadata.uid === victim.metadata.uid
)
throw new Error("Unexpected API survivor");
await writeFile(
resolve(evidence, "stop-one-after.json"),
JSON.stringify(
{
stopped: target,
stoppedUID: victim.metadata.uid,
containerID: currentCID,
state: "CONTAINER_EXITED",
survivor: remaining[0].metadata.name,
},
null,
2,
),
);
} else if (action === "destroy") {
await command("destroy_claimed_cluster", [
"k3d",
"cluster",
"delete",
cfg.cluster,
]);
const r = await receipt();
r.state = "destroyed";
await save(r);
} else if (action === "stop")
await command("stop_claimed_cluster", [
"k3d",
"cluster",
"stop",
cfg.cluster,
]);
else throw new Error("Unknown runtime action");
}
async function destroyPartial() {
const r = await receipt();
const inventory = (
await command("partial_inventory", [
"docker",
"ps",
"-aq",
"--filter",
"label=k3d.cluster=" + cfg.cluster,
])
).trim();
if (!inventory) {
r.state = "destroyed";
await save(r);
return;
}
const inspected = JSON.parse(
await command("partial_server_identity", [
"docker",
"inspect",
"k3d-" + cfg.cluster + "-server-0",
]),
);
const server = {
container: inspected[0].Id,
owner: inspected[0].Config.Labels["vp.coursework.owner"],
};
assertPartialClaim(r, server, cfg);
r.container = server.container;
await save(r);
await command("destroy_partial_claimed_cluster", [
"k3d",
"cluster",
"delete",
cfg.cluster,
]);
r.state = "destroyed";
await save(r);
}
try {
if (action === "check") console.log("Parsed k3s runtime for lab" + lab);
else if (action === "up") await bootstrap();
else if (action === "destroy" && (await receipt()).state === "claimed")
await destroyPartial();
else await operator.run(action);
} catch (error) {
await writeFile(
resolve(evidence, "last-failure.json"),
JSON.stringify({
stage,
action,
error: error instanceof Error ? error.message : "unknown",
}) + "\n",
);
throw error;
}