forked from mtusi2026/mtusi2026lab5
173 lines
5.6 KiB
Python
173 lines
5.6 KiB
Python
"""Run the same Docker actions as scripts/restart.sh and scripts/stop-one.sh.
|
|
|
|
The public harness executes those POSIX scripts directly. This Windows host
|
|
cannot spawn a .sh file (WinError 193), so this runner calls the same
|
|
docker compose / docker stop commands and checks the contract around them.
|
|
It prints no tokens.
|
|
"""
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure
|
|
|
|
CONTAINERS = [
|
|
"mtusi-lab5-postgres",
|
|
"mtusi-lab5-minio",
|
|
"mtusi-lab5-rabbitmq",
|
|
"mtusi-lab5-api-1",
|
|
"mtusi-lab5-api-2",
|
|
"mtusi-lab5-worker",
|
|
"mtusi-lab5-nginx",
|
|
]
|
|
|
|
|
|
def started():
|
|
completed = subprocess.run(
|
|
["docker", "inspect", "--format", "{{.Name}} {{.State.StartedAt}}", *CONTAINERS],
|
|
text=True,
|
|
capture_output=True,
|
|
check=True,
|
|
)
|
|
return completed.stdout.strip()
|
|
|
|
|
|
def compose(*args):
|
|
completed = subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, text=True, capture_output=True)
|
|
if completed.returncode != 0:
|
|
raise SystemExit("compose failed: " + completed.stderr[-400:])
|
|
|
|
|
|
def stop_instance(instance):
|
|
found = subprocess.run(
|
|
["docker", "ps", "-q", "--filter", "label=lab.instance_id=" + instance],
|
|
text=True,
|
|
capture_output=True,
|
|
check=True,
|
|
)
|
|
cid = found.stdout.strip()
|
|
if not cid:
|
|
raise SystemExit("no running replica " + instance)
|
|
completed = subprocess.run(["docker", "stop", "-t", "5", cid], text=True, capture_output=True)
|
|
if completed.returncode != 0:
|
|
raise SystemExit("stop failed: " + completed.stderr[-200:])
|
|
|
|
|
|
def wait_ready(client, actor=None, timeout=90):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
client.call("GET", "/health/ready", schema="Health")
|
|
if actor is not None:
|
|
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
|
return
|
|
except Failure:
|
|
time.sleep(0.5)
|
|
raise SystemExit("stack did not become ready")
|
|
|
|
|
|
def failover(client, alice, chat, target):
|
|
client.timeout = 2
|
|
started_at = time.monotonic()
|
|
outcome = {}
|
|
|
|
def stopper():
|
|
try:
|
|
stop_instance(target)
|
|
outcome["ok"] = True
|
|
except SystemExit as exc:
|
|
outcome["error"] = str(exc)
|
|
|
|
import threading
|
|
|
|
thread = threading.Thread(target=stopper)
|
|
thread.start()
|
|
successes = []
|
|
after = []
|
|
failures = 0
|
|
first = None
|
|
key = uuid.uuid4().hex
|
|
text = "Запись во время отказа " + key
|
|
while time.monotonic() - started_at < 15:
|
|
stopped = not thread.is_alive() and outcome.get("ok")
|
|
try:
|
|
response = client.message(alice, chat, text, key=key)
|
|
if stopped:
|
|
if first is None:
|
|
first = time.monotonic() - started_at
|
|
after.append(response["headers"]["X-Instance-Id"])
|
|
successes.append(response["json"]["id"])
|
|
key = uuid.uuid4().hex
|
|
text = "Запись во время отказа " + key
|
|
except Failure:
|
|
failures += 1
|
|
time.sleep(0.1)
|
|
thread.join(timeout=10)
|
|
if outcome.get("error"):
|
|
raise SystemExit(outcome["error"])
|
|
if first is None or first > 10 or len(after) < 5 or target in after[-5:]:
|
|
raise SystemExit(
|
|
"failover " + target + " failed: recovery=" + str(first) + " tail=" + ",".join(after[-5:])
|
|
)
|
|
print(
|
|
"PASS failover",
|
|
target,
|
|
"recovery_s",
|
|
round(first, 3),
|
|
"writes",
|
|
len(successes),
|
|
"failed_attempts",
|
|
failures,
|
|
)
|
|
|
|
|
|
def main():
|
|
print("BEFORE")
|
|
print(started())
|
|
client = Client("http://localhost:8080", timeout=10)
|
|
alice, bob, _eve, chat = client.bootstrap()
|
|
original = client.message(alice, chat, "Сохранить при отказе")["json"]
|
|
attachment, content = client.attachment(alice, chat)
|
|
attachment = client.wait_ready(alice, attachment)
|
|
compose("restart", "postgres", "minio", "rabbitmq", "api-1", "api-2", "worker", "nginx")
|
|
print("AFTER")
|
|
print(started())
|
|
client.timeout = 10
|
|
wait_ready(client, alice)
|
|
me = client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["json"]
|
|
if me["id"] != alice["id"]:
|
|
raise SystemExit("session did not survive restart")
|
|
history = client.call(
|
|
"GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", actor=bob, schema="MessagePage"
|
|
)["json"]
|
|
if not any(item == original for item in history["items"]):
|
|
raise SystemExit("message did not survive restart")
|
|
downloaded = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob)
|
|
if downloaded["bytes"] != content:
|
|
raise SystemExit("original bytes changed across restart")
|
|
print("PASS persistence", original["id"], attachment["id"])
|
|
|
|
client.timeout = 10
|
|
observed = set()
|
|
for _ in range(40):
|
|
observed.add(client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"]["X-Instance-Id"])
|
|
if observed != {"api-1", "api-2"}:
|
|
raise SystemExit("expected both replicas, saw " + ",".join(sorted(observed)))
|
|
for target in ("api-1", "api-2"):
|
|
failover(client, alice, chat, target)
|
|
compose("start", target)
|
|
client.timeout = 10
|
|
wait_ready(client)
|
|
print("PASS hooks")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
finally:
|
|
compose("start", "api-1", "api-2")
|