ЛР 3: балансировка, идемпотентность и проверки на 4

This commit is contained in:
2026-11-01 17:02:42 +03:00
parent 0ab3a341aa
commit 412b1bb09d
64 changed files with 3904 additions and 53 deletions
+35
View File
@@ -0,0 +1,35 @@
"""Run against a separate API configured with SESSION_TTL_SECONDS=2."""
from pathlib import Path
import json
import os
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check
def main():
c = Client(os.environ["EXPIRY_BASE_URL"])
deadline = time.monotonic() + 30
while True:
try:
c.call("GET", "/health/ready", schema="Health")
break
except AssertionError:
check(time.monotonic() < deadline, "TTL API did not become ready")
time.sleep(0.25)
actor = c.user("expiry")
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
time.sleep(2.3)
response = c.call("GET", "/api/v1/users/me", 401, actor=actor)
report = {"session_ttl_seconds": 2, "wait_seconds": 2.3,
"before_expiry_status": 200, "after_expiry_status": response["status"],
"result": "PASS"}
(ROOT / "evidence" / "expiry.json").write_text(json.dumps(report, indent=2) + "\n")
print("PASS: TTL=2s, initial 200, after 2.3s -> 401; normal API TTL unchanged")
if __name__ == "__main__":
main()
+208
View File
@@ -0,0 +1,208 @@
"""Repeatable API failures and sequential replacement under HTTP load."""
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
import json
import math
import os
from pathlib import Path
import sys
import threading
import time
from uuid import uuid4
from ops import ROOT, command, compose, containers, summary
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure
def wait_instances(client, actor, expected, excluded=(), version=None):
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
seen = {}
try:
for _ in range(20):
r = client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
seen[r["headers"]["X-Instance-Id"]] = r["headers"].get("X-App-Version")
if len(seen) >= expected and not any(any(s.startswith(x + "-") for x in excluded) for s in seen):
if version is None or all(v == version for v in seen.values()):
return seen
except Failure:
pass
time.sleep(.2)
raise AssertionError("Expected API replicas were not observed through the load balancer")
def history(client, actor, chat):
from urllib.parse import quote
result, cursor = [], None
while True:
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100"
if cursor:
path += "&cursor=" + quote(cursor, safe="")
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
result.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
return result
def resource_sample():
ids = [r["id"] for r in summary() if r["status"] == "running"]
return [json.loads(line) for line in command(["docker", "stats", "--no-stream", "--format", "{{json .}}", *ids]).splitlines()]
def workload(base_url, label, action, duration=24, concurrency=4):
client = Client(base_url, timeout=3)
alice, bob, _, chat = client.bootstrap()
before_message = client.message(alice, chat, "confirmed before " + label)["json"]
observed_before = wait_instances(client, alice, 2)
started = time.monotonic()
deadline = started + duration
events, confirmations = [], {}
action_result = {}
lock = threading.Lock()
def worker(index):
local = Client(base_url, timeout=3)
key = uuid4().hex
attempt = 0
while time.monotonic() < deadline:
begin = time.monotonic()
try:
if index == 0 and attempt % 4 == 0:
response = local.call("GET", "/api/v1/users/me", actor=alice, schema="User")
kind = "read"
else:
response = local.message(alice, chat, "load-" + key, key=key)
kind = "write"
with lock:
confirmations[response["json"]["id"]] = response["json"]
key = uuid4().hex
event = {"at": time.monotonic() - started, "ok": True, "kind": kind,
"ms": (time.monotonic() - begin) * 1000,
"instance": response["headers"]["X-Instance-Id"]}
except Failure:
event = {"at": time.monotonic() - started, "ok": False,
"ms": (time.monotonic() - begin) * 1000}
# Preserve the key after uncertain POST responses.
with lock:
events.append(event)
attempt += 1
def disrupt():
time.sleep(4)
action_result["started_seconds"] = time.monotonic() - started
action_result["detail"] = action(client, alice)
action_result["finished_seconds"] = time.monotonic() - started
action_result["resources_during_load"] = resource_sample()
with ThreadPoolExecutor(max_workers=concurrency + 1) as pool:
workers = [pool.submit(worker, i) for i in range(concurrency)]
change = pool.submit(disrupt)
for future in workers:
future.result()
change.result()
elapsed = time.monotonic() - started
rows = history(client, bob, chat)
by_id = {r["id"]: r for r in rows}
assert by_id[before_message["id"]] == before_message
assert all(by_id.get(mid) == msg for mid, msg in confirmations.items()), "Confirmed write lost"
assert len(rows) == len(by_id) == len({r["text"] for r in rows}), "Retry created duplicate messages"
success = [e for e in events if e["ok"]]
failure = [e for e in events if not e["ok"]]
finish = action_result["finished_seconds"]
writes_after = [e for e in success if e["at"] >= finish and e["kind"] == "write"]
reads_after = [e for e in success if e["at"] >= finish and e["kind"] == "read"]
assert writes_after and reads_after, "Read/write did not resume"
recovery = min(e["at"] for e in writes_after) - action_result["started_seconds"]
if label.startswith("failure"):
assert recovery <= 10, "Failover exceeded 10 seconds"
hostname = action_result["detail"]["hostname"]
assert all(not e["instance"].startswith(hostname + "-") for e in writes_after[-20:])
latencies = sorted(e["ms"] for e in success)
def percentile(q):
return round(latencies[max(0, math.ceil(len(latencies) * q) - 1)], 3)
buckets = []
for start in range(0, math.ceil(elapsed), 2):
group = [e for e in events if start <= e["at"] < start + 2]
buckets.append({"start_seconds": start, "rps": sum(e["ok"] for e in group) / 2,
"errors": sum(not e["ok"] for e in group)})
report = {"scenario": label, "utc": datetime.now(timezone.utc).isoformat(),
"duration_seconds": round(elapsed, 3), "concurrency": concurrency,
"successful_requests": len(success), "failed_attempts": len(failure),
"error_ratio": round(len(failure) / len(events), 6),
"successful_rps": round(len(success) / elapsed, 3),
"successful_latency_ms": {"p50": percentile(.5), "p95": percentile(.95), "p99": percentile(.99)},
"first_write_after_action_seconds": round(recovery, 3), "action": action_result,
"confirmed_writes_checked": len(confirmations) + 1, "history_messages": len(rows),
"unique_history_messages": len(by_id), "instances_before": list(observed_before),
"series_2_seconds": buckets, "deployment_after": summary(),
"note": "Closed-loop mixed HTTP load, 4 clients; failures are attempts, retries retain keys; latency covers successes only."}
(ROOT / "evidence" / (label + ".json")).write_text(json.dumps(report, indent=2) + "\n")
print(label + ": " + str(report["successful_rps"]) + " RPS; " + str(len(failure)) + " errors", flush=True)
return report
def failure_action(target):
def stop(client, actor):
begin = time.monotonic()
command(["docker", "stop", "--time", "6", target["Id"]], timeout=10)
row = json.loads(command(["docker", "inspect", target["Id"]]))[0]
assert not row["State"]["Running"] and row["State"]["ExitCode"] == 0
elapsed = time.monotonic() - begin
assert elapsed <= 10
return {"container": row["Id"], "hostname": row["Config"]["Hostname"],
"stop_seconds": round(elapsed, 3), "exit_code": row["State"]["ExitCode"]}
return stop
def rolling_action(client, actor):
original = containers()
assert len(original) == 2
updates = []
env = dict(os.environ, APP_VERSION="lab3-v2")
# Both migrations are additive. Check/replay once before replacing processes.
migration = compose("run", "--rm", "--no-deps", "migrate")
for old in original:
address = next(n["IPAddress"] for name, n in old["NetworkSettings"]["Networks"].items() if name.endswith("_edge"))
slot = compose("run", "--rm", "--no-deps", "checks", "python", "scripts/ops.py", "drain", address)
command(["docker", "stop", "--time", "6", old["Id"]])
command(["docker", "rm", old["Id"]])
compose("up", "-d", "--no-deps", "--no-recreate", "--scale", "api=2", "api", env=env)
compose("run", "--rm", "--no-deps", "checks", "python", "scripts/ops.py", "ready", slot)
current = containers()
fresh = [r for r in current if r["Id"] not in {x["Id"] for x in original} and r["Id"] not in {x["new_container"] for x in updates}]
assert len(fresh) == 1
wait_instances(client, actor, 2, excluded=[old["Config"]["Hostname"]])
updates.append({"old_container": old["Id"], "new_container": fresh[0]["Id"], "backend_slot": slot,
"new_image": fresh[0]["Image"]})
wait_instances(client, actor, 2, version="lab3-v2")
return {"updates": updates, "migration": migration,
"method": "drain, stop, remove and recreate each container; same application image, new APP_VERSION configuration"}
def main():
import argparse
p = argparse.ArgumentParser()
p.add_argument("--base-url", default="http://127.0.0.1:8080")
args = p.parse_args()
originals = containers()
assert len(originals) == 2
reports = []
for index, target in enumerate(originals, 1):
try:
reports.append(workload(args.base_url, "failure-api-" + str(index), failure_action(target)))
finally:
command(["docker", "start", target["Id"]])
c = Client(args.base_url)
alice = c.user("restore")
wait_instances(c, alice, 2)
reports.append(workload(args.base_url, "rolling-update", rolling_action, duration=45))
(ROOT / "evidence" / "experiments.json").write_text(json.dumps([
{k: r[k] for k in ("scenario", "successful_rps", "failed_attempts", "first_write_after_action_seconds",
"confirmed_writes_checked", "successful_latency_ms")} for r in reports], indent=2) + "\n")
if __name__ == "__main__":
main()
+76
View File
@@ -0,0 +1,76 @@
"""Seed an isolated synthetic chat, measure the actual keyset SQL, keep raw plans."""
from datetime import datetime, timedelta, timezone
import json
from pathlib import Path
import sys
from urllib.parse import quote
from uuid import UUID
import psycopg
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(ROOT / "tests"))
from app.config import secret
from app.security import Cursors, stamp
from client import Client, check
def walk(node):
yield node
for child in node.get("Plans", []):
yield from walk(child)
def main():
c = Client()
actor = c.user("explain")
chat = c.call("POST", "/api/v1/chats", 201, actor=actor,
body={"title": "EXPLAIN 10000", "member_ids": [actor["id"]]}, schema="Chat")["json"]
base = datetime.now(timezone.utc)
with psycopg.connect(secret("DATABASE_URL"), connect_timeout=2) as conn:
conn.execute("""INSERT INTO messages(id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), %s, %s, 'synthetic-' || i,
%s + i * interval '1 microsecond'
FROM generate_series(1, 10000) AS i""", (UUID(chat["id"]), UUID(actor["id"]), base))
conn.execute("UPDATE chats SET last_message_at=%s WHERE id=%s",
(base + timedelta(microseconds=10000), UUID(chat["id"])))
conn.execute("ANALYZE messages")
report = {"messages_in_chat": 10000, "page_limit": 50,
"index": "messages_chat_created_id_idx", "cases": {}}
text_plans = []
cursors = Cursors(secret("CURSOR_KEY").encode())
with psycopg.connect(secret("DATABASE_URL"), connect_timeout=2) as conn:
for label, offset in (("first", 0), ("middle", 5000), ("late", 9900)):
sql = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id=%s"
params = [UUID(chat["id"])]
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=50"
if offset:
boundary = conn.execute("SELECT created_at, id FROM messages WHERE chat_id=%s AND text=%s",
(UUID(chat["id"]), "synthetic-" + str(offset))).fetchone()
sql += " AND (created_at, id) > (%s, %s)"
params.extend(boundary)
token = cursors.encode("messages:" + chat["id"],
{"created_at": stamp(boundary[0]), "id": str(boundary[1])})
path += "&cursor=" + quote(token, safe="")
sql += " ORDER BY created_at, id LIMIT %s"
params.append(51)
data = conn.execute("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) " + sql, params).fetchone()[0][0]
nodes = list(walk(data["Plan"]))
check(not any(x["Node Type"] == "Seq Scan" for x in nodes), "Unexpected full scan in keyset query")
check(any(x.get("Index Name") == report["index"] for x in nodes), "History index not used")
check(data["Plan"]["Actual Rows"] == 51, "LIMIT was not respected")
page = c.call("GET", path, actor=actor, schema="MessagePage")["json"]
check(len(page["items"]) == 50 and page["next_cursor"], "HTTP page differs from measured SQL")
expected = ["synthetic-" + str(i) for i in range(offset + 1, offset + 51)]
check([x["text"] for x in page["items"]] == expected, "HTTP keyset boundary differs")
report["cases"][label] = {"after_message": offset, "sql": sql, "plan": data}
text_plans.append(label.upper() + "\n" + sql + "\n" + "\n".join(
row[0] for row in conn.execute("EXPLAIN (ANALYZE, BUFFERS) " + sql, params).fetchall()))
(ROOT / "evidence" / "explain.json").write_text(json.dumps(report, indent=2) + "\n")
(ROOT / "evidence" / "explain.txt").write_text("\n\n".join(text_plans) + "\n")
print("PASS: 10000 messages, first/middle/late pages use history index; 51 SQL rows for 50 HTTP items")
if __name__ == "__main__":
main()
+35
View File
@@ -0,0 +1,35 @@
"""Create local synthetic credentials once without printing secret values."""
from pathlib import Path
import secrets
ROOT = Path(__file__).resolve().parents[1]
def main():
directory = ROOT / "secrets"
directory.mkdir(exist_ok=True)
for name in ("postgres_password", "redis_password", "cursor_key"):
path = directory / name
if not path.exists():
path.write_text(secrets.token_hex(32) + "\n", encoding="utf-8", newline="\n")
else:
# Shell command substitution on Linux must not retain a Windows CR.
value = path.read_text(encoding="utf-8").strip()
path.write_text(value + "\n", encoding="utf-8", newline="\n")
pg_password = (directory / "postgres_password").read_text().strip()
redis_password = (directory / "redis_password").read_text().strip()
for name, value in {
"database_url": f"postgresql://messenger:{pg_password}@postgres:5432/messenger",
"cache_url": f"redis://:{redis_password}@redis:6379/0",
}.items():
path = directory / name
if not path.exists():
path.write_text(value + "\n", encoding="utf-8", newline="\n")
env = ROOT / ".env"
if not env.exists():
env.write_bytes((ROOT / ".env.example").read_bytes())
print("PASS: local .env and secret files exist; existing values preserved")
if __name__ == "__main__":
main()
+94
View File
@@ -0,0 +1,94 @@
"""Docker operations scoped to the configured local Compose project."""
import csv
import io
import json
import os
from pathlib import Path
import socket
import subprocess
import time
ROOT = Path(__file__).resolve().parents[1]
def command(args, timeout=120, env=None):
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
encoding="utf-8", errors="replace", timeout=timeout, env=env)
if result.returncode:
# Commands and stderr can contain secrets. Keep detailed errors local.
raise RuntimeError("Local command failed: " + " ".join(args[:3]) + "\n" + result.stderr[-1500:])
return result.stdout.strip()
def compose(*args, timeout=120, env=None):
return command(["docker", "compose", "--env-file", ".env", *args], timeout, env)
def project():
return os.getenv("COMPOSE_PROJECT_NAME") or json.loads(compose("config", "--format", "json"))["name"]
def containers(service="api", running=True):
args = ["docker", "ps"] + ([] if running else ["-a"])
ids = command(args + ["--filter", "label=com.docker.compose.project=" + project(),
"--filter", "label=com.docker.compose.service=" + service,
"--filter", "label=com.docker.compose.oneoff=False", "--format", "{{.ID}}"])
return json.loads(command(["docker", "inspect", *ids.split()])) if ids else []
def summary():
result = []
for service in ("api", "postgres", "redis", "lb"):
for row in containers(service, running=False):
result.append({"service": service, "id": row["Id"], "name": row["Name"].lstrip("/"),
"started_at": row["State"]["StartedAt"], "status": row["State"]["Status"],
"image": row["Image"], "user": row["Config"]["User"],
"memory_bytes": row["HostConfig"]["Memory"], "nano_cpus": row["HostConfig"]["NanoCpus"],
"published_ports": row["HostConfig"]["PortBindings"],
"volumes": [m["Name"] for m in row["Mounts"] if m["Type"] == "volume"]})
return result
def runtime(line):
with socket.create_connection(("lb", 9999), timeout=3) as sock:
sock.sendall((line + "\n").encode())
sock.shutdown(socket.SHUT_WR)
chunks = []
while True:
data = sock.recv(65536)
if not data:
return b"".join(chunks).decode()
chunks.append(data)
def drain(address):
lines = runtime("show servers state messenger").splitlines()
fields = next(line[2:].split() for line in lines if line.startswith("# "))
for line in lines:
values = line.split()
if not values or line.startswith("#") or len(values) != len(fields):
continue
row = dict(zip(fields, values))
if row["srv_addr"] == address:
slot = "messenger/" + row["srv_name"]
runtime("set server " + slot + " state drain")
# Let requests already assigned to this replica finish (HTTP budget 5 s).
deadline = time.monotonic() + 7
while time.monotonic() < deadline:
stats = list(csv.DictReader(io.StringIO(runtime("show stat").lstrip("# "))))
active = [s for s in stats if s["pxname"] == "messenger" and s["svname"] == row["srv_name"]]
if active and int(active[0]["scur"]) == 0:
return slot
time.sleep(.1)
raise RuntimeError("Replica did not drain within 7 seconds")
raise RuntimeError("Backend address was not found in HAProxy")
if __name__ == "__main__":
import sys
if sys.argv[1] == "drain":
print(drain(sys.argv[2]))
elif sys.argv[1] == "ready":
runtime("set server " + sys.argv[2] + " state ready")
elif sys.argv[1] == "summary":
print(json.dumps(summary(), indent=2))
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose --env-file .env restart --timeout 6 api postgres redis lb
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python scripts/stop_one.py
+24
View File
@@ -0,0 +1,24 @@
"""Stop exactly the replica selected by the public failover scenario."""
import json
import os
import time
from ops import ROOT, command, containers
def main():
target = os.environ["TARGET_INSTANCE_ID"]
matches = [row for row in containers() if target.startswith(row["Config"]["Hostname"] + "-")]
if len(matches) != 1:
raise RuntimeError("Target does not identify exactly one running API replica")
row = matches[0]
started = time.monotonic()
command(["docker", "stop", "--time", "6", row["Id"]], timeout=9)
state = json.loads(command(["docker", "inspect", row["Id"]]))[0]["State"]
assert not state["Running"] and state["ExitCode"] == 0, "Graceful shutdown failed"
report = {"target_instance": target, "container": row["Id"], "exit_code": state["ExitCode"],
"stop_seconds": round(time.monotonic() - started, 3), "remains_stopped": True}
(ROOT / "evidence" / "last-stop.json").write_text(json.dumps(report, indent=2) + "\n")
if __name__ == "__main__":
main()
+97
View File
@@ -0,0 +1,97 @@
"""Build and verify the complete local lab3 stack without deleting volumes."""
import argparse
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import subprocess
import sys
from ops import ROOT, command, compose, containers, summary
def saved(name, args, timeout=300):
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
encoding="utf-8", errors="replace", timeout=timeout)
(ROOT / "evidence" / name).write_text(result.stdout + result.stderr, encoding="utf-8")
if result.returncode:
raise RuntimeError("Check failed; see evidence/" + name)
return result.stdout
def checks(name, *args):
return saved(name, ["docker", "compose", "--env-file", ".env", "run", "--rm", "--no-deps", "checks", *args])
def main():
p = argparse.ArgumentParser()
p.add_argument("--skip-build", action="store_true")
p.add_argument("--base-url", default=None)
p.add_argument("--skip-experiments", action="store_true")
args = p.parse_args()
(ROOT / "evidence").mkdir(exist_ok=True)
command([sys.executable, "-X", "utf8", "scripts/init_local.py"])
compose("config", "--quiet")
if not args.skip_build:
print("Building API and check images", flush=True)
saved("build.txt", ["docker", "compose", "--env-file", ".env", "build", "api", "checks"], 900)
saved("startup.txt", ["docker", "compose", "--env-file", ".env", "up", "-d", "--wait", "--wait-timeout", "120"], 240)
config = json.loads(compose("config", "--format", "json"))
base_url = args.base_url or "http://127.0.0.1:" + str(config["services"]["lb"]["ports"][0]["published"])
print("Checking contract, HTTP API and student tests", flush=True)
checks("contract-check.txt", "make", "check")
checks("smoke.txt", "make", "test")
checks("student-tests.txt", "make", "test-student")
before = summary()
checks("persistence.txt", "sh", "-ec", "chmod +x scripts/restart.sh; make test-persistence ACTION_SCRIPT=scripts/restart.sh")
after = summary()
by_id = {r["id"]: r for r in after}
for old in before:
new = by_id[old["id"]]
assert old["started_at"] != new["started_at"]
assert old["volumes"] == new["volumes"]
(ROOT / "evidence" / "restart.json").write_text(json.dumps({"before": before, "after": after}, indent=2) + "\n")
print("Running public failover scenario", flush=True)
try:
checks("failover.txt", "sh", "-ec", "chmod +x scripts/stop-one.sh; make test-failover ACTION_SCRIPT=scripts/stop-one.sh")
finally:
compose("start", "api")
# Wait for both replicas through LB before starting subsequent checks.
sys.path.insert(0, str(ROOT / "tests"))
from client import Client
from experiments import wait_instances
client = Client(base_url)
actor = client.user("verify")
wait_instances(client, actor, 2)
print("Checking short session expiry and indexed history", flush=True)
ttl_name = config["name"] + "-ttl-check"
assert not command(["docker", "ps", "-a", "--filter", "name=^/" + ttl_name + "$", "--format", "{{.ID}}"])
try:
compose("run", "-d", "--no-deps", "--name", ttl_name, "-e", "SESSION_TTL_SECONDS=2", "api")
checks("expiry.txt", "env", "EXPIRY_BASE_URL=http://" + ttl_name + ":8080", "python", "scripts/check_expiry.py")
finally:
command(["docker", "stop", "--time", "6", ttl_name])
command(["docker", "rm", ttl_name])
checks("explain-run.txt", "make", "explain")
checks("smoke-final.txt", "make", "test")
saved("published-smoke.txt", [sys.executable, "-X", "utf8", "tests/smoke.py", "--base-url", base_url])
if not args.skip_experiments:
print("Measuring failure of each replica and sequential replacement under load", flush=True)
saved("experiments.txt", [sys.executable, "-X", "utf8", "scripts/experiments.py", "--base-url", base_url], 360)
deployed = summary()
assert len([r for r in deployed if r["service"] == "api" and r["status"] == "running"]) == 2
assert all(not r["published_ports"] for r in deployed if r["service"] != "lb")
assert all(r["memory_bytes"] and r["nano_cpus"] for r in deployed)
versions = command(["docker", "info", "--format", "{{json .}}"])
info = json.loads(versions)
report = {"result": "PASS", "utc": datetime.now(timezone.utc).isoformat(), "base_url": base_url,
"checks": ["public_smoke", "student_tests", "full_restart", "public_failover", "expiry", "index_10000", "published_smoke"]
+ ([] if args.skip_experiments else ["each_replica_failure", "sequential_replacement_under_load"]),
"docker": {k: info[k] for k in ("ServerVersion", "OperatingSystem", "Architecture", "NCPU", "MemTotal")},
"deployment": deployed}
(ROOT / "evidence" / "verification.json").write_text(json.dumps(report, indent=2) + "\n")
print("PASS: lab3 checks completed; evidence/verification.json", flush=True)
if __name__ == "__main__":
main()