Files
mtusi2026lab5/student_tests/run_hooks.py
T

173 lines
5.6 KiB
Python

"""Run the same Docker actions as scripts/restart.sh and scripts/stop-one.sh.
The public harness executes those POSIX scripts directly. This Windows host
cannot spawn a .sh file (WinError 193), so this runner calls the same
docker compose / docker stop commands and checks the contract around them.
It prints no tokens.
"""
import subprocess
import sys
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure
CONTAINERS = [
"mtusi-lab5-postgres",
"mtusi-lab5-minio",
"mtusi-lab5-rabbitmq",
"mtusi-lab5-api-1",
"mtusi-lab5-api-2",
"mtusi-lab5-worker",
"mtusi-lab5-nginx",
]
def started():
completed = subprocess.run(
["docker", "inspect", "--format", "{{.Name}} {{.State.StartedAt}}", *CONTAINERS],
text=True,
capture_output=True,
check=True,
)
return completed.stdout.strip()
def compose(*args):
completed = subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, text=True, capture_output=True)
if completed.returncode != 0:
raise SystemExit("compose failed: " + completed.stderr[-400:])
def stop_instance(instance):
found = subprocess.run(
["docker", "ps", "-q", "--filter", "label=lab.instance_id=" + instance],
text=True,
capture_output=True,
check=True,
)
cid = found.stdout.strip()
if not cid:
raise SystemExit("no running replica " + instance)
completed = subprocess.run(["docker", "stop", "-t", "5", cid], text=True, capture_output=True)
if completed.returncode != 0:
raise SystemExit("stop failed: " + completed.stderr[-200:])
def wait_ready(client, actor=None, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
client.call("GET", "/health/ready", schema="Health")
if actor is not None:
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
return
except Failure:
time.sleep(0.5)
raise SystemExit("stack did not become ready")
def failover(client, alice, chat, target):
client.timeout = 2
started_at = time.monotonic()
outcome = {}
def stopper():
try:
stop_instance(target)
outcome["ok"] = True
except SystemExit as exc:
outcome["error"] = str(exc)
import threading
thread = threading.Thread(target=stopper)
thread.start()
successes = []
after = []
failures = 0
first = None
key = uuid.uuid4().hex
text = "Запись во время отказа " + key
while time.monotonic() - started_at < 15:
stopped = not thread.is_alive() and outcome.get("ok")
try:
response = client.message(alice, chat, text, key=key)
if stopped:
if first is None:
first = time.monotonic() - started_at
after.append(response["headers"]["X-Instance-Id"])
successes.append(response["json"]["id"])
key = uuid.uuid4().hex
text = "Запись во время отказа " + key
except Failure:
failures += 1
time.sleep(0.1)
thread.join(timeout=10)
if outcome.get("error"):
raise SystemExit(outcome["error"])
if first is None or first > 10 or len(after) < 5 or target in after[-5:]:
raise SystemExit(
"failover " + target + " failed: recovery=" + str(first) + " tail=" + ",".join(after[-5:])
)
print(
"PASS failover",
target,
"recovery_s",
round(first, 3),
"writes",
len(successes),
"failed_attempts",
failures,
)
def main():
print("BEFORE")
print(started())
client = Client("http://localhost:8080", timeout=10)
alice, bob, _eve, chat = client.bootstrap()
original = client.message(alice, chat, "Сохранить при отказе")["json"]
attachment, content = client.attachment(alice, chat)
attachment = client.wait_ready(alice, attachment)
compose("restart", "postgres", "minio", "rabbitmq", "api-1", "api-2", "worker", "nginx")
print("AFTER")
print(started())
client.timeout = 10
wait_ready(client, alice)
me = client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["json"]
if me["id"] != alice["id"]:
raise SystemExit("session did not survive restart")
history = client.call(
"GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", actor=bob, schema="MessagePage"
)["json"]
if not any(item == original for item in history["items"]):
raise SystemExit("message did not survive restart")
downloaded = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob)
if downloaded["bytes"] != content:
raise SystemExit("original bytes changed across restart")
print("PASS persistence", original["id"], attachment["id"])
client.timeout = 10
observed = set()
for _ in range(40):
observed.add(client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"]["X-Instance-Id"])
if observed != {"api-1", "api-2"}:
raise SystemExit("expected both replicas, saw " + ",".join(sorted(observed)))
for target in ("api-1", "api-2"):
failover(client, alice, chat, target)
compose("start", target)
client.timeout = 10
wait_ready(client)
print("PASS hooks")
if __name__ == "__main__":
try:
main()
finally:
compose("start", "api-1", "api-2")