284 lines
6.8 KiB
Python
284 lines
6.8 KiB
Python
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timezone
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
|
|
from client import Client, Failure, arguments, check, run
|
|
|
|
|
|
PROJECT = "mtusi2026lab3"
|
|
|
|
|
|
def timestamp():
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def command(*args):
|
|
return subprocess.check_output(
|
|
list(args),
|
|
cwd=ROOT,
|
|
text=True,
|
|
timeout=45,
|
|
).strip()
|
|
|
|
|
|
def compose(*args):
|
|
return command(
|
|
"docker",
|
|
"compose",
|
|
"-p",
|
|
PROJECT,
|
|
*args,
|
|
)
|
|
|
|
|
|
def inspect(container, template):
|
|
return command(
|
|
"docker",
|
|
"inspect",
|
|
"--format",
|
|
template,
|
|
container,
|
|
)
|
|
|
|
|
|
def api_ids():
|
|
ids = compose("ps", "-q", "api").split()
|
|
return {
|
|
inspect(container, "{{.Id}}")
|
|
for container in ids
|
|
}
|
|
|
|
|
|
def healthy(container):
|
|
return inspect(
|
|
container,
|
|
"{{.State.Running}} {{.State.Health.Status}}",
|
|
) == "true healthy"
|
|
|
|
|
|
def instance_id(container):
|
|
code = (
|
|
"import urllib.request; "
|
|
"r=urllib.request.urlopen("
|
|
"'http://127.0.0.1:8080/health/live', timeout=2); "
|
|
"print(r.headers.get('X-Instance-Id', '')); "
|
|
"r.close()"
|
|
)
|
|
return command(
|
|
"docker",
|
|
"exec",
|
|
container,
|
|
"python",
|
|
"-c",
|
|
code,
|
|
)
|
|
|
|
|
|
def main():
|
|
args = arguments("Sequential API container replacement")
|
|
|
|
client = Client(
|
|
args.base_url,
|
|
args.ca_file,
|
|
timeout=2,
|
|
)
|
|
alice, bob, eve, chat = client.bootstrap()
|
|
|
|
import uuid
|
|
|
|
key = uuid.uuid4().hex
|
|
text = "Сохранить при замене " + key
|
|
|
|
original_message = client.message(
|
|
alice,
|
|
chat,
|
|
text,
|
|
key=key,
|
|
)["json"]
|
|
|
|
original_ids = sorted(api_ids())
|
|
check(len(original_ids) == 2, "expected exactly two API containers")
|
|
|
|
for container in original_ids:
|
|
check(healthy(container), "both API containers must be healthy")
|
|
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"event": "replacement_started",
|
|
"utc": timestamp(),
|
|
"original_container_ids": original_ids,
|
|
},
|
|
ensure_ascii=False,
|
|
),
|
|
flush=True,
|
|
)
|
|
|
|
replacements = []
|
|
|
|
for old in original_ids:
|
|
current = api_ids()
|
|
check(len(current) == 2, "expected two APIs before replacement")
|
|
check(old in current, "original target container is missing")
|
|
|
|
other = next(container for container in current if container != old)
|
|
check(healthy(other), "remaining API is not healthy")
|
|
|
|
other_started = inspect(other, "{{.State.StartedAt}}")
|
|
old_instance = instance_id(old)
|
|
step_started = time.monotonic()
|
|
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"event": "replacing",
|
|
"utc": timestamp(),
|
|
"old_container_id": old,
|
|
"old_instance": old_instance,
|
|
"remaining_container_id": other,
|
|
},
|
|
ensure_ascii=False,
|
|
),
|
|
flush=True,
|
|
)
|
|
|
|
command("docker", "stop", "--time", "10", old)
|
|
command("docker", "rm", old)
|
|
|
|
compose(
|
|
"up",
|
|
"-d",
|
|
"--no-deps",
|
|
"--no-recreate",
|
|
"--no-build",
|
|
"--scale",
|
|
"api=2",
|
|
"api",
|
|
)
|
|
|
|
deadline = time.monotonic() + 30
|
|
replacement = None
|
|
|
|
while time.monotonic() < deadline:
|
|
current = api_ids()
|
|
candidates = current - {other}
|
|
|
|
if (
|
|
len(current) == 2
|
|
and len(candidates) == 1
|
|
and other in current
|
|
):
|
|
candidate = next(iter(candidates))
|
|
if candidate != old and healthy(candidate):
|
|
replacement = candidate
|
|
break
|
|
|
|
time.sleep(0.5)
|
|
|
|
check(replacement is not None, "replacement did not become healthy")
|
|
|
|
new_instance = instance_id(replacement)
|
|
check(new_instance != old_instance, "instance did not change")
|
|
|
|
seen_through_lb = False
|
|
deadline = time.monotonic() + 20
|
|
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
response = client.call(
|
|
"GET",
|
|
"/api/v1/users/me",
|
|
actor=alice,
|
|
schema="User",
|
|
)
|
|
except Failure:
|
|
time.sleep(0.2)
|
|
continue
|
|
|
|
if response["headers"]["X-Instance-Id"] == new_instance:
|
|
seen_through_lb = True
|
|
break
|
|
|
|
time.sleep(0.2)
|
|
|
|
check(
|
|
seen_through_lb,
|
|
"new API was not observed through LB with existing session",
|
|
)
|
|
check(healthy(other), "remaining API became unhealthy")
|
|
check(
|
|
inspect(other, "{{.State.StartedAt}}") == other_started,
|
|
"the other API was restarted unexpectedly",
|
|
)
|
|
|
|
replay = client.message(
|
|
alice,
|
|
chat,
|
|
text,
|
|
key=key,
|
|
)["json"]
|
|
|
|
check(
|
|
replay == original_message,
|
|
"confirmed message or idempotency result changed",
|
|
)
|
|
|
|
result = {
|
|
"event": "replacement_ready",
|
|
"utc": timestamp(),
|
|
"old_container_id": old,
|
|
"new_container_id": replacement,
|
|
"old_instance": old_instance,
|
|
"new_instance": new_instance,
|
|
"step_seconds": round(
|
|
time.monotonic() - step_started, 3
|
|
),
|
|
"other_api_not_restarted": True,
|
|
"existing_session_works": True,
|
|
"replay_preserved": True,
|
|
}
|
|
|
|
replacements.append(result)
|
|
print(json.dumps(result, ensure_ascii=False), flush=True)
|
|
|
|
time.sleep(3)
|
|
|
|
final_ids = api_ids()
|
|
|
|
check(len(final_ids) == 2, "expected two APIs after replacement")
|
|
check(
|
|
not final_ids.intersection(original_ids),
|
|
"some original API containers were not replaced",
|
|
)
|
|
|
|
for container in final_ids:
|
|
check(healthy(container), "final API is not healthy")
|
|
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"event": "replacement_finished",
|
|
"utc": timestamp(),
|
|
"replacements": len(replacements),
|
|
"final_container_ids": sorted(final_ids),
|
|
},
|
|
ensure_ascii=False,
|
|
),
|
|
flush=True,
|
|
)
|
|
|
|
print(
|
|
"PASS: both API containers replaced sequentially; "
|
|
"existing session and confirmed message preserved.",
|
|
flush=True,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main) |