Files
mtusi2026lab3/student-tests/availability_tests.py
T

88 lines
5.0 KiB
Python

"""Replica failures and sequential replacement under a continuous mixed workload."""
import argparse
import json
import time
from pathlib import Path
import sys
from cluster import alias, api_ids, docker, inspect, rolling_replace, wait_api
from compose_helpers import ROOT, compose, sql
from client import Client, check, run
from traffic import traffic, public_report
def verify_messages(chat, confirmations):
rows = json.loads(sql("SELECT coalesce(json_agg(json_build_object('id', id, 'text', text)), '[]') FROM messages WHERE chat_id = '" + chat["id"] + "';"))
saved = {row["id"]: row["text"] for row in rows}
check(all(saved.get(identifier) == text for identifier, text in confirmations), "confirmed message lost or changed")
check(len({row["text"] for row in rows}) == len(rows), "uncertain retry created duplicate text")
return {"confirmed_writes": len(confirmations), "stored_messages": len(rows), "no_duplicate_text": True}
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://127.0.0.1:18080")
args = parser.parse_args()
wait_api(args.base_url, 2)
c = Client(args.base_url)
report = {"verified_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "failures": []}
try:
for position, mode in enumerate(("graceful", "immediate")):
targets = api_ids()
target = targets[position]
target_alias = alias(target)
alice, bob, eve, chat = c.bootstrap()
def stop():
docker("stop", "--time", "8" if mode == "graceful" else "0", target)
return {"container": target, "instance": target_alias, "running_after_action": inspect(target)["State"]["Running"]}
print("Testing " + mode + " stop of " + target[:12], flush=True)
measured = traffic(args.base_url, alice, chat, duration=16, concurrency=8,
action=stop, retry_uncertain=True)
action = measured["actions"][0]
check(not action["result"]["running_after_action"], "selected process was not stopped")
after = [row for row in measured["_samples"] if row[0] >= action["finished_after_seconds"] and row[1] == "write" and row[2] == 201]
check(after, "no confirmed writes after replica stopped")
recovery = after[0][0] - action["started_after_seconds"]
check(recovery <= 10, "writes did not recover in 10 seconds")
check(all(row[4] != target_alias for row in after[-5:]), "stopped replica still observed")
integrity = verify_messages(chat, measured["_confirmed"])
report["failures"].append({**public_report(measured), "mode": mode,
"recovery_seconds": round(recovery, 3), "integrity": integrity})
compose("start", "api")
wait_api(args.base_url, 2)
print("Building replacement image", flush=True)
compose("build", "--build-arg", "BUILD_REVISION=rolling-" + str(time.time_ns()), "api", timeout=180)
alice, bob, eve, chat = c.bootstrap()
print("Testing sequential replacement under mixed traffic", flush=True)
measured = traffic(args.base_url, alice, chat, duration=45, concurrency=8,
action=lambda: rolling_replace(args.base_url, build=False), retry_uncertain=True)
replacements = measured["actions"][0]["result"]
check(len(replacements) == 2, "not every replica was replaced")
check(all(row["old_container"] != row["new_container"] and row["old_image"] != row["new_image"]
for row in replacements), "replacement did not deploy a new image")
check(measured["actions"][0]["finished_after_seconds"] < 43, "replacement outlasted load window")
report["rolling_update"] = {**public_report(measured), "integrity": verify_messages(chat, measured["_confirmed"])}
c.call("GET", "/api/v1/users/me", actor=alice, schema="User")
containers = [inspect(container) for container in api_ids()]
check(all(not info["HostConfig"]["PortBindings"] for info in containers), "API ports published outside LB")
check(all(info["HostConfig"]["NanoCpus"] > 0 and info["HostConfig"]["Memory"] > 0 for info in containers), "missing API resource limit")
report["infrastructure"] = {
"api_ports_published": False,
"api_containers": [{"id": info["Id"], "image": info["Image"],
"cpu_limit": info["HostConfig"]["NanoCpus"] / 1e9,
"memory_limit": info["HostConfig"]["Memory"]} for info in containers],
"migrations": sql("SELECT string_agg(version::text, ',' ORDER BY version) FROM schema_migrations;")
}
(ROOT / "evidence" / "availability.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
print(json.dumps(report, ensure_ascii=False, indent=2))
finally:
compose("start", "api")
if __name__ == "__main__":
run(main)