Files
mtusi2026lab3/test/grade4.py
T
qrtx 5c8bcf0f80 test: cover persistence failover and graceful draining
Add API integration checks, migration compatibility, rolling restart scenarios and a report verification collector.
2026-10-29 11:47:46 +03:00

333 lines
13 KiB
Python

#!/usr/bin/env python3
"""Lab 3 grade 4: measured replica failover or sequential replacement."""
import argparse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import json
from pathlib import Path
import subprocess
import sys
import threading
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check # noqa: E402
from resilience import history # noqa: E402
def compose(*args, timeout=60):
try:
result = subprocess.run(
["docker", "compose", "-f", "compose.yaml", "-f", "configs/two.yaml", *args],
cwd=ROOT, capture_output=True,
text=True, timeout=timeout, check=False,
)
except (OSError, subprocess.TimeoutExpired) as exc:
raise Failure(f"docker compose {args[0]} failed: {type(exc).__name__}") from None
check(result.returncode == 0, f"docker compose {args[0]} failed; inspect Docker logs")
return result.stdout.strip()
def docker(*args, timeout=60):
try:
result = subprocess.run(
["docker", *args], cwd=ROOT, capture_output=True,
text=True, timeout=timeout, check=False,
)
except (OSError, subprocess.TimeoutExpired) as exc:
raise Failure(f"docker {args[0]} failed: {type(exc).__name__}") from None
check(result.returncode == 0, f"docker {args[0]} failed; inspect Docker logs")
return result.stdout.strip()
def inspect(container_id, field):
try:
result = subprocess.run(
["docker", "inspect", "--format", field, container_id],
cwd=ROOT, capture_output=True, text=True, timeout=10, check=False,
)
except (OSError, subprocess.TimeoutExpired) as exc:
raise Failure(f"docker inspect failed: {type(exc).__name__}") from None
check(result.returncode == 0, "docker inspect failed")
return result.stdout.strip()
def containers(service):
ids = compose("ps", "--all", "-q", service).splitlines()
check(bool(ids), f"{service} container is missing")
return ids
def container(service):
ids = containers(service)
check(len(ids) == 1, f"expected one {service} container")
return ids[0]
def instance(container_id):
return inspect(container_id, "{{.Config.Hostname}}")
def healthy(container_id, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if inspect(container_id, "{{.State.Health.Status}}") == "healthy":
return
time.sleep(0.5)
raise Failure(f"API container did not become healthy in {timeout} seconds")
def both_observed(client, actor, expected, timeout=30):
seen = set()
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
response = client.call("GET", "/api/v1/users/me", actor=actor,
schema="User")
seen.add(response["headers"]["X-Instance-Id"])
except Failure:
pass
if seen == expected:
return
time.sleep(0.1)
raise Failure(f"both replicas were not observed; saw {sorted(seen)}")
class Traffic:
def __init__(self, base_url, actor, chat, concurrency):
self.base_url = base_url
self.actor = actor
self.chat = chat
self.concurrency = concurrency
self.stop = threading.Event()
self.lock = threading.Lock()
self.events = []
self.confirmed = set()
self.attempted = set()
self.pool = ThreadPoolExecutor(max_workers=concurrency)
def start(self):
self.futures = [self.pool.submit(self.worker, i)
for i in range(self.concurrency)]
def worker(self, worker_id):
client = Client(self.base_url, timeout=2)
index = 0
pending = None
while not self.stop.is_set():
write = worker_id % 2 == 0
if write and pending is None:
key = uuid.uuid4().hex
pending = (key, f"grade4-{worker_id}-{index}-{key}")
index += 1
with self.lock:
self.attempted.add(pending[1])
try:
if write:
response = client.message(self.actor, self.chat,
pending[1], key=pending[0])
else:
response = client.call("GET", "/api/v1/users/me",
actor=self.actor, schema="User")
event = (time.monotonic(), "write" if write else "read", True,
response["headers"]["X-Instance-Id"])
if write:
with self.lock:
self.confirmed.add(pending[1])
pending = None
except Failure:
event = (time.monotonic(), "write" if write else "read", False, None)
with self.lock:
self.events.append(event)
self.stop.wait(0.05)
def finish(self):
self.stop.set()
try:
for future in self.futures:
future.result(timeout=5)
finally:
self.pool.shutdown(wait=True)
def snapshot(self):
with self.lock:
return list(self.events)
def metrics(events, started, ended, target, action_done, scenario):
window = [event for event in events if started <= event[0] <= ended]
check(bool(window), "no traffic was recorded during the action")
counts = {}
for kind in ("read", "write"):
records = [event for event in window if event[1] == kind]
successes = [event for event in records if event[2]]
check(bool(successes), f"no successful {kind}s during the action")
survivor = next((event[0] for event in successes
if event[0] >= started and event[3] != target), None)
check(survivor is not None, f"no {kind} served by surviving replica")
after_action = next((event[0] for event in successes
if event[0] >= action_done), None)
check(after_action is not None, f"no {kind} succeeded after the action")
counts[kind] = {
"successful": len(successes),
"errors": len(records) - len(successes),
"successful_rps": round(len(successes) / (ended - started), 3),
"first_survivor_seconds": round(survivor - started, 3),
"first_success_after_action_seconds": round(after_action - started, 3),
}
after = [event for event in window if event[0] >= action_done + 1 and event[2]]
check(bool(after), "no successful traffic after the action")
if scenario == "failover":
check(all(event[3] != target for event in after),
"stopped replica still served traffic after the action")
target_times = [event[0] for event in window
if event[2] and event[3] == target]
counts["last_target_response_seconds"] = (
round(max(target_times) - started, 3) if target_times else None
)
for kind in ("read", "write"):
check(counts[kind]["first_success_after_action_seconds"] <= 10,
f"{kind} did not recover within 10 seconds")
return counts
def verify_history(client, actor, chat, original, traffic):
messages = history(client, actor, chat)
counts = Counter(message["text"] for message in messages)
check(any(message == original for message in messages),
"confirmed pre-action message disappeared")
for text in traffic.attempted:
check(counts[text] <= 1, "an uncertain retry created a duplicate")
for text in traffic.confirmed:
check(counts[text] == 1, "confirmed write disappeared")
def resource_limits():
expected = {
"nginx": (500_000_000, 128 * 1024**2),
"migrate": (1_000_000_000, 512 * 1024**2),
"postgres": (1_000_000_000, 512 * 1024**2),
"redis": (500_000_000, 256 * 1024**2),
}
ids = containers("api")
check(len(ids) in (1, 2, 4), "expected 1, 2 or 4 API replicas")
api_limits = (2_000_000_000 // len(ids), 1024 * 1024**2 // len(ids))
observed = {"api": []}
for container_id in ids:
actual = (
int(inspect(container_id, "{{.HostConfig.NanoCpus}}")),
int(inspect(container_id, "{{.HostConfig.Memory}}")),
)
check(actual == api_limits, "API resource limits differ from fixed budget")
observed["api"].append({"instance": instance(container_id),
"cpus": actual[0] / 1e9,
"memory_mib": actual[1] // 1024**2})
for service, limits in expected.items():
container_id = container(service)
actual = (
int(inspect(container_id, "{{.HostConfig.NanoCpus}}")),
int(inspect(container_id, "{{.HostConfig.Memory}}")),
)
check(actual == limits, f"{service} resource limits differ from compose.yaml")
observed[service] = {"cpus": actual[0] / 1e9,
"memory_mib": actual[1] // 1024**2}
return observed
def run(args):
check(2 <= args.concurrency <= 8, "concurrency must be 2..8")
check(3 <= args.observe_seconds <= 60,
"observe-seconds must be 3..60")
result = {"scenario": args.scenario, "resources": resource_limits(),
"actions": []}
if args.scenario == "resources":
return result
targets = containers("api")
check(len(targets) == 2, "failover and rolling scenarios require the two-replica profile")
client = Client(args.base_url, timeout=10)
alice, bob, _, chat = client.bootstrap()
original = client.message(alice, chat, "Before grade 4 actions")["json"]
both_observed(client, alice, {instance(item) for item in targets})
migrate_id = container("migrate")
traffic = Traffic(args.base_url, alice, chat, args.concurrency)
traffic.start()
try:
for before_id in targets:
target = instance(before_id)
healthy(before_id)
before_ids = set(containers("api"))
both_observed(client, alice, {instance(item) for item in before_ids})
time.sleep(1)
started = time.monotonic()
if args.scenario == "failover":
docker("stop", "--time", "30", before_id, timeout=40)
check(inspect(before_id, "{{.State.Status}}") == "exited",
"target API container was not stopped")
else:
docker("stop", "--time", "30", before_id, timeout=40)
docker("rm", before_id)
compose("up", "-d", "--no-build", "--no-deps",
"--scale", "api=2", "api", timeout=90)
replacements = set(containers("api")) - before_ids
check(len(replacements) == 1, "one API container was not replaced")
healthy(next(iter(replacements)))
action_done = time.monotonic()
time.sleep(args.observe_seconds)
ended = time.monotonic()
item = {"target": target,
"action_seconds": round(action_done - started, 3),
"window_seconds": round(ended - started, 3),
"traffic": metrics(traffic.snapshot(), started, ended,
target, action_done, args.scenario)}
if args.scenario == "failover":
docker("start", before_id)
healthy(before_id)
both_observed(client, alice,
{instance(item) for item in containers("api")})
result["actions"].append(item)
check(container("migrate") == migrate_id,
"migration container changed during API replacement")
finally:
traffic.finish()
# Restore both API replicas after a failed test; never remove volumes.
try:
compose("up", "-d", "--no-build", "--no-deps",
"--scale", "api=2", "api")
except Failure:
print("WARNING: restore API replicas manually", file=sys.stderr)
verify_history(client, bob, chat, original, traffic)
result["confirmed_writes"] = len(traffic.confirmed)
result["attempted_unique_writes"] = len(traffic.attempted)
result["migration_container_unchanged"] = True
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--scenario", choices=("resources", "failover", "rolling"),
required=True)
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--concurrency", type=int, default=4)
parser.add_argument("--observe-seconds", type=float, default=10)
parser.add_argument("--output", help="optional JSON result path without tokens")
args = parser.parse_args()
try:
result = run(args)
output = json.dumps(result, ensure_ascii=False, indent=2)
if args.output:
path = Path(args.output)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(output + "\n", encoding="utf-8")
print(output)
print("PASS: grade 4 scenario")
except (Failure, ValueError, OSError) as exc:
print(f"FAIL: {exc}", file=sys.stderr)
raise SystemExit(1) from None
if __name__ == "__main__":
main()