Files
mtusi2026lab4/student-tests/rolling_replacement.py
T

284 lines
6.8 KiB
Python

import json
from pathlib import Path
import subprocess
import sys
import time
from datetime import datetime, timezone
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, arguments, check, run
PROJECT = "mtusi2026lab3"
def timestamp():
return datetime.now(timezone.utc).isoformat()
def command(*args):
return subprocess.check_output(
list(args),
cwd=ROOT,
text=True,
timeout=45,
).strip()
def compose(*args):
return command(
"docker",
"compose",
"-p",
PROJECT,
*args,
)
def inspect(container, template):
return command(
"docker",
"inspect",
"--format",
template,
container,
)
def api_ids():
ids = compose("ps", "-q", "api").split()
return {
inspect(container, "{{.Id}}")
for container in ids
}
def healthy(container):
return inspect(
container,
"{{.State.Running}} {{.State.Health.Status}}",
) == "true healthy"
def instance_id(container):
code = (
"import urllib.request; "
"r=urllib.request.urlopen("
"'http://127.0.0.1:8080/health/live', timeout=2); "
"print(r.headers.get('X-Instance-Id', '')); "
"r.close()"
)
return command(
"docker",
"exec",
container,
"python",
"-c",
code,
)
def main():
args = arguments("Sequential API container replacement")
client = Client(
args.base_url,
args.ca_file,
timeout=2,
)
alice, bob, eve, chat = client.bootstrap()
import uuid
key = uuid.uuid4().hex
text = "Сохранить при замене " + key
original_message = client.message(
alice,
chat,
text,
key=key,
)["json"]
original_ids = sorted(api_ids())
check(len(original_ids) == 2, "expected exactly two API containers")
for container in original_ids:
check(healthy(container), "both API containers must be healthy")
print(
json.dumps(
{
"event": "replacement_started",
"utc": timestamp(),
"original_container_ids": original_ids,
},
ensure_ascii=False,
),
flush=True,
)
replacements = []
for old in original_ids:
current = api_ids()
check(len(current) == 2, "expected two APIs before replacement")
check(old in current, "original target container is missing")
other = next(container for container in current if container != old)
check(healthy(other), "remaining API is not healthy")
other_started = inspect(other, "{{.State.StartedAt}}")
old_instance = instance_id(old)
step_started = time.monotonic()
print(
json.dumps(
{
"event": "replacing",
"utc": timestamp(),
"old_container_id": old,
"old_instance": old_instance,
"remaining_container_id": other,
},
ensure_ascii=False,
),
flush=True,
)
command("docker", "stop", "--time", "10", old)
command("docker", "rm", old)
compose(
"up",
"-d",
"--no-deps",
"--no-recreate",
"--no-build",
"--scale",
"api=2",
"api",
)
deadline = time.monotonic() + 30
replacement = None
while time.monotonic() < deadline:
current = api_ids()
candidates = current - {other}
if (
len(current) == 2
and len(candidates) == 1
and other in current
):
candidate = next(iter(candidates))
if candidate != old and healthy(candidate):
replacement = candidate
break
time.sleep(0.5)
check(replacement is not None, "replacement did not become healthy")
new_instance = instance_id(replacement)
check(new_instance != old_instance, "instance did not change")
seen_through_lb = False
deadline = time.monotonic() + 20
while time.monotonic() < deadline:
try:
response = client.call(
"GET",
"/api/v1/users/me",
actor=alice,
schema="User",
)
except Failure:
time.sleep(0.2)
continue
if response["headers"]["X-Instance-Id"] == new_instance:
seen_through_lb = True
break
time.sleep(0.2)
check(
seen_through_lb,
"new API was not observed through LB with existing session",
)
check(healthy(other), "remaining API became unhealthy")
check(
inspect(other, "{{.State.StartedAt}}") == other_started,
"the other API was restarted unexpectedly",
)
replay = client.message(
alice,
chat,
text,
key=key,
)["json"]
check(
replay == original_message,
"confirmed message or idempotency result changed",
)
result = {
"event": "replacement_ready",
"utc": timestamp(),
"old_container_id": old,
"new_container_id": replacement,
"old_instance": old_instance,
"new_instance": new_instance,
"step_seconds": round(
time.monotonic() - step_started, 3
),
"other_api_not_restarted": True,
"existing_session_works": True,
"replay_preserved": True,
}
replacements.append(result)
print(json.dumps(result, ensure_ascii=False), flush=True)
time.sleep(3)
final_ids = api_ids()
check(len(final_ids) == 2, "expected two APIs after replacement")
check(
not final_ids.intersection(original_ids),
"some original API containers were not replaced",
)
for container in final_ids:
check(healthy(container), "final API is not healthy")
print(
json.dumps(
{
"event": "replacement_finished",
"utc": timestamp(),
"replacements": len(replacements),
"final_container_ids": sorted(final_ids),
},
ensure_ascii=False,
),
flush=True,
)
print(
"PASS: both API containers replaced sequentially; "
"existing session and confirmed message preserved.",
flush=True,
)
if __name__ == "__main__":
run(main)