forked from mtusi2026/mtusi2026lab3
реализация 3 лабы
This commit is contained in:
+282
-169
@@ -1,4 +1,5 @@
|
||||
"""Полный прогон lab2 в собственном Compose project; основные данные не затрагивает."""
|
||||
"""Проверки уровня 4 в отдельном Compose project с синтетическими данными."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import os
|
||||
@@ -6,195 +7,307 @@ import platform
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from urllib.error import URLError
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import urlopen
|
||||
from uuid import uuid4
|
||||
|
||||
from helpers import Client, ROOT
|
||||
from client import Failure
|
||||
from resilience import history
|
||||
|
||||
|
||||
def docker(*args):
|
||||
return subprocess.run(["docker", *args], check=True, capture_output=True, text=True, timeout=120).stdout.strip()
|
||||
|
||||
|
||||
def inspect(container):
|
||||
return json.loads(docker("inspect", container))[0]
|
||||
|
||||
|
||||
def check_shutdown(container, base_url, port):
|
||||
with socket.create_connection(("127.0.0.1", port), timeout=2) as pending:
|
||||
pending.sendall(
|
||||
b"POST /api/v1/users HTTP/1.1\r\nHost: localhost\r\n"
|
||||
b"Content-Type: application/json\r\nContent-Length: 1000\r\n"
|
||||
b"Expect: 100-continue\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
assert b"100 Continue" in pending.recv(1024), "Сервер не начал читать тело запроса"
|
||||
pending.sendall(b'{"username":')
|
||||
started = time.monotonic()
|
||||
docker("kill", "--signal=TERM", container)
|
||||
first_refusal = None
|
||||
while inspect(container)["State"]["Running"]:
|
||||
assert time.monotonic() - started < 10, "SIGTERM: превышены 10 секунд"
|
||||
try:
|
||||
with urlopen(base_url + "/health/live", timeout=0.3):
|
||||
pass
|
||||
except (URLError, OSError):
|
||||
if first_refusal is None:
|
||||
first_refusal = time.monotonic() - started
|
||||
time.sleep(0.05)
|
||||
elapsed = time.monotonic() - started
|
||||
|
||||
assert elapsed < 10, "SIGTERM: превышены 10 секунд"
|
||||
assert first_refusal is not None and first_refusal < elapsed, "Не показан отказ новых запросов при остановке"
|
||||
state = inspect(container)["State"]
|
||||
assert not state["OOMKilled"] and state["ExitCode"] in (0, 143), state
|
||||
|
||||
result = subprocess.run(["docker", "logs", container], capture_output=True, text=True, check=True, timeout=10)
|
||||
logs = result.stdout + result.stderr
|
||||
assert "Application shutdown complete" in logs, "Нет подтверждения корректного завершения"
|
||||
try:
|
||||
with urlopen(base_url + "/health/live", timeout=0.5):
|
||||
raise AssertionError("Остановленный сервер отвечает")
|
||||
except (URLError, OSError):
|
||||
pass
|
||||
print(f"PASS: SIGTERM with unfinished request: {elapsed:.2f}s; new requests refused after {first_refusal:.2f}s; exit={state['ExitCode']}; OOM=false")
|
||||
return subprocess.check_output(["docker", *args], text=True, timeout=180).strip()
|
||||
|
||||
|
||||
def main():
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
port = sock.getsockname()[1]
|
||||
project = "lab2-check-" + uuid4().hex[:10]
|
||||
env = dict(os.environ, COMPOSE_PROJECT_NAME=project, HOST_PORT=str(port), SESSION_TTL_SECONDS="3600")
|
||||
base_url = f"http://127.0.0.1:{port}"
|
||||
project = "lab3-check-" + uuid4().hex[:10]
|
||||
base = f"http://127.0.0.1:{port}"
|
||||
env = dict(os.environ, COMPOSE_PROJECT_NAME=project, HOST_PORT=str(port),
|
||||
API_REPLICAS="2", SESSION_TTL_SECONDS="3600", BASE_URL=base)
|
||||
result = {"utc": datetime.now(timezone.utc).isoformat(), "project": project,
|
||||
"base_url": base, "host": platform.system() + " " + platform.machine(),
|
||||
"docker": docker("info", "--format", "{{.ServerVersion}} {{.Architecture}} CPUs={{.NCPU}} RAM={{.MemTotal}}"),
|
||||
"commit": docker("compose", "version", "--short"),
|
||||
"profile": {"workers": 4, "pause_seconds": 0.05, "operation": "POST message then GET history, same-key retry after uncertain POST"}}
|
||||
result["compose_version"] = result.pop("commit")
|
||||
result["source_commit"] = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
|
||||
result["working_tree"] = True
|
||||
|
||||
def compose(*args, capture=True, stdin=None):
|
||||
return subprocess.run(["docker", "compose", *args], cwd=ROOT, env=env, input=stdin,
|
||||
text=True, capture_output=capture, check=True, timeout=180)
|
||||
def compose(*args):
|
||||
return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, env=env, text=True, timeout=180).strip()
|
||||
|
||||
def ready():
|
||||
subprocess.run([str(ROOT / "scripts/start.sh")], cwd=ROOT, env=env,
|
||||
check=True, capture_output=True, text=True, timeout=180)
|
||||
print("PASS: start.sh built and started the stack, readiness reached")
|
||||
def script(name):
|
||||
subprocess.run([str(ROOT / "scripts" / name)], cwd=ROOT, env=env, check=True, timeout=240)
|
||||
|
||||
def states():
|
||||
return {service: inspect(compose("ps", "-q", service).stdout.strip()) for service in ("api", "postgres", "valkey")}
|
||||
def inspect_all():
|
||||
ids = compose("ps", "-a", "-q").split()
|
||||
return json.loads(docker("inspect", *ids))
|
||||
|
||||
def scalar(query, params=()):
|
||||
code = """import json,sys
|
||||
from src import db
|
||||
data=json.load(sys.stdin)
|
||||
db.pool.open(wait=True,timeout=15)
|
||||
try:
|
||||
with db.connection() as conn:
|
||||
row=conn.execute(data['query'], data['params']).fetchone()
|
||||
print(json.dumps(next(iter(row.values()))))
|
||||
finally:
|
||||
db.pool.close()
|
||||
def apis():
|
||||
return [i for i in inspect_all() if i["Config"]["Labels"]["com.docker.compose.service"] == "api"]
|
||||
|
||||
def metadata(info):
|
||||
return {"id": info["Id"], "instance": info["Config"]["Hostname"],
|
||||
"service": info["Config"]["Labels"]["com.docker.compose.service"],
|
||||
"image": info["Image"], "started_at": info["State"]["StartedAt"],
|
||||
"running": info["State"]["Running"], "exit_code": info["State"]["ExitCode"],
|
||||
"memory_bytes": info["HostConfig"]["Memory"], "nano_cpus": info["HostConfig"]["NanoCpus"],
|
||||
"published_ports": info["HostConfig"]["PortBindings"],
|
||||
"mounts": [{"type": m["Type"], "destination": m["Destination"], "name": m.get("Name")} for m in info["Mounts"]]}
|
||||
|
||||
def sql(query):
|
||||
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c", query)
|
||||
|
||||
def check_shutdown():
|
||||
info = apis()[0]
|
||||
# Внутреннее подключение через docker exec не публикует порт API на хост.
|
||||
pending_code = """import socket,time
|
||||
s=socket.create_connection(('127.0.0.1',8080),timeout=2)
|
||||
s.sendall(b'POST /api/v1/users HTTP/1.1\\r\\nHost: localhost\\r\\nContent-Type: application/json\\r\\nContent-Length: 1000\\r\\nExpect: 100-continue\\r\\n\\r\\n')
|
||||
assert b'100 Continue' in s.recv(1024)
|
||||
s.sendall(b'{"username":')
|
||||
print('pending',flush=True)
|
||||
time.sleep(30)
|
||||
"""
|
||||
return json.loads(compose("exec", "-T", "api", "python", "-c", code,
|
||||
stdin=json.dumps({"query": query, "params": params})).stdout)
|
||||
|
||||
print("UTC:", datetime.now(timezone.utc).isoformat())
|
||||
print("Host:", platform.system(), platform.machine(), "Python", platform.python_version())
|
||||
print("Docker:", docker("info", "--format", "{{.ServerVersion}} {{.Architecture}} CPUs={{.NCPU}} RAM={{.MemTotal}}"))
|
||||
print("Commit:", subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, text=True).stdout.strip(), "+ working tree")
|
||||
print("Command: ./scripts/check-docker.sh; project:", project)
|
||||
try:
|
||||
ready()
|
||||
before = states()
|
||||
for service, info in before.items():
|
||||
print(service, "image=" + info["Image"], "started=" + info["State"]["StartedAt"],
|
||||
"memory=" + str(info["HostConfig"]["Memory"]), "nano_cpus=" + str(info["HostConfig"]["NanoCpus"]))
|
||||
api = compose("ps", "-q", "api").stdout.strip()
|
||||
assert docker("exec", api, "id", "-u") == "10001"
|
||||
assert before["api"]["Config"]["Cmd"] == ["python", "-m", "src"]
|
||||
for service in ("postgres", "valkey"):
|
||||
assert any(mount["Type"] == "volume" for mount in before[service]["Mounts"])
|
||||
assert all(mount["Destination"].startswith("/run/secrets/") for mount in before["api"]["Mounts"])
|
||||
bad_health = subprocess.run(["docker", "exec", "-e", "APP_PORT=1", api, "python", "-m", "src.healthcheck"], capture_output=True)
|
||||
assert bad_health.returncode == 1
|
||||
print("PASS: non-root UID 10001; Python PID 1; healthcheck; named volumes; no API data volume")
|
||||
|
||||
subprocess.run([str(ROOT / "scripts/check.sh")], cwd=ROOT, env=dict(env, BASE_URL=base_url), check=True, timeout=240)
|
||||
demo = subprocess.run([str(ROOT / "scripts/demo.sh")], cwd=ROOT, env=dict(env, BASE_URL=base_url),
|
||||
check=True, capture_output=True, text=True, timeout=30)
|
||||
assert "Привет из lab2!" in demo.stdout
|
||||
print("PASS: demo.sh registered users, logged in and read two messages")
|
||||
client = Client(base_url)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
marker = "private-" + uuid4().hex
|
||||
client.message(alice, chat, marker)
|
||||
logs = docker("logs", api)
|
||||
assert marker not in logs and alice["token"] not in logs and alice["registration"]["password"] not in logs
|
||||
logs_result = subprocess.run(["docker", "logs", api], capture_output=True, text=True, check=True)
|
||||
assert "request_id=" in logs_result.stderr + logs_result.stdout
|
||||
assert marker not in logs_result.stderr + logs_result.stdout
|
||||
assert alice["token"] not in logs_result.stderr + logs_result.stdout
|
||||
assert alice["registration"]["password"] not in logs_result.stderr + logs_result.stdout
|
||||
assert scalar("SELECT bool_and(password_hash LIKE %s) FROM users", ("$argon2id$%",))
|
||||
print("PASS: Argon2id in PostgreSQL; logs do not contain message text, passwords or session tokens")
|
||||
|
||||
title = "rollback_" + uuid4().hex
|
||||
client.call("POST", "/api/v1/chats", 400, actor=alice,
|
||||
body={"title": title, "member_ids": [alice["id"], str(uuid4())]})
|
||||
assert scalar("SELECT count(*) FROM chats WHERE title=%s", (title,)) == 0
|
||||
count = scalar("SELECT count(*) FROM messages")
|
||||
compose("exec", "-T", "api", "python", "-m", "src.migrate")
|
||||
compose("exec", "-T", "api", "python", "-m", "src.migrate")
|
||||
assert scalar("SELECT count(*) FROM schema_migrations") == 1
|
||||
assert scalar("SELECT count(*) FROM messages") == count
|
||||
print("PASS: failed membership insert rolled back chat; migrations run twice without duplicate data")
|
||||
|
||||
before = states()
|
||||
subprocess.run(["make", "test-persistence", "PYTHON=" + sys.executable,
|
||||
"ACTION_SCRIPT=scripts/restart.sh", "BASE_URL=" + base_url], cwd=ROOT, env=env, check=True, timeout=240)
|
||||
after = states()
|
||||
for service in before:
|
||||
assert before[service]["State"]["StartedAt"] != after[service]["State"]["StartedAt"]
|
||||
print("Restart", service, before[service]["State"]["StartedAt"], "->", after[service]["State"]["StartedAt"])
|
||||
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")
|
||||
print("PASS: API, PostgreSQL and Valkey really restarted; previous token still works")
|
||||
|
||||
for service in ("valkey", "postgres"):
|
||||
try:
|
||||
compose("stop", "--timeout", "10", service)
|
||||
started = time.monotonic()
|
||||
client.call("GET", "/api/v1/users/me", 503, actor=alice)
|
||||
elapsed = time.monotonic() - started
|
||||
assert elapsed < 6, elapsed
|
||||
client.call("GET", "/api/v1/users/me", 401)
|
||||
client.call("GET", "/health/ready", 503)
|
||||
client.call("GET", "/health/live", schema="Health")
|
||||
print(f"PASS: {service} unavailable -> 503 in {elapsed:.2f}s; anonymous -> 401; readiness 503, liveness 200")
|
||||
finally:
|
||||
compose("start", service)
|
||||
subprocess.run([str(ROOT / "scripts/wait-ready.sh")], cwd=ROOT, env=env, check=True, timeout=90)
|
||||
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")
|
||||
print("PASS:", service, "recovered without a new login")
|
||||
|
||||
env["SESSION_TTL_SECONDS"] = "2"
|
||||
compose("up", "-d", "--force-recreate", "api")
|
||||
subprocess.run([str(ROOT / "scripts/wait-ready.sh")], cwd=ROOT, env=env, check=True, timeout=90)
|
||||
short = client.user("short_ttl")
|
||||
client.call("GET", "/api/v1/users/me", actor=short, schema="User")
|
||||
time.sleep(2.2)
|
||||
client.call("GET", "/api/v1/users/me", 401, actor=short)
|
||||
print("PASS: SESSION_TTL_SECONDS=2; valid token expires without logout")
|
||||
api = compose("ps", "-q", "api").stdout.strip()
|
||||
check_shutdown(api, base_url, port)
|
||||
print("PASS: all lab2 Docker checks completed")
|
||||
finally:
|
||||
pending = subprocess.Popen(["docker", "exec", info["Id"], "python", "-u", "-c", pending_code],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
try:
|
||||
subprocess.run([str(ROOT / "scripts/stop.sh")], cwd=ROOT, env=env,
|
||||
check=True, capture_output=True, text=True, timeout=90)
|
||||
volumes = docker("volume", "ls", "--quiet", "--filter", "label=com.docker.compose.project=" + project)
|
||||
if volumes:
|
||||
print("PASS: stop.sh removed containers and preserved named volumes")
|
||||
assert pending.stdout.readline().strip() == "pending"
|
||||
started = time.monotonic()
|
||||
docker("kill", "--signal=TERM", info["Id"])
|
||||
first_refusal = None
|
||||
while True:
|
||||
state = json.loads(docker("inspect", info["Id"]))[0]["State"]
|
||||
if not state["Running"]:
|
||||
break
|
||||
assert time.monotonic() - started < 8
|
||||
probe = subprocess.run(["docker", "exec", info["Id"], "python", "-c",
|
||||
"import socket; s=socket.socket(); s.settimeout(.2); raise SystemExit(s.connect_ex(('127.0.0.1',8080)))"],
|
||||
capture_output=True, timeout=3)
|
||||
if probe.returncode and first_refusal is None:
|
||||
first_refusal = time.monotonic() - started
|
||||
time.sleep(0.1)
|
||||
elapsed = time.monotonic() - started
|
||||
assert first_refusal is not None and first_refusal < elapsed < 8
|
||||
assert state["ExitCode"] == 0 and not state["OOMKilled"]
|
||||
logs = subprocess.run(["docker", "logs", info["Id"]], capture_output=True, text=True, check=True)
|
||||
assert "Application shutdown complete" in logs.stdout + logs.stderr
|
||||
result["sigterm"] = {"instance": info["Config"]["Hostname"], "unfinished_request": True,
|
||||
"shutdown_seconds": round(elapsed, 3), "new_connections_refused_seconds": round(first_refusal, 3),
|
||||
"exit_code": state["ExitCode"], "oom_killed": state["OOMKilled"]}
|
||||
print("PASS: SIGTERM with unfinished request:", json.dumps(result["sigterm"]), flush=True)
|
||||
finally:
|
||||
compose("down", "--volumes", "--timeout", "10")
|
||||
print("Removed only temporary Compose project and its test volumes:", project)
|
||||
pending.communicate(timeout=10)
|
||||
compose("start", "api")
|
||||
script("wait-ready.sh")
|
||||
|
||||
def load_action(label, action, target=None):
|
||||
client = Client(base, timeout=2)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
original = client.message(alice, chat, "До воздействия")["json"]
|
||||
stop = threading.Event()
|
||||
lock = threading.Lock()
|
||||
successes, errors, confirmed = [], [], {}
|
||||
started = time.monotonic()
|
||||
|
||||
def worker(index):
|
||||
c = Client(base, timeout=2)
|
||||
key = uuid4().hex
|
||||
while not stop.is_set():
|
||||
for method in ("POST", "GET"):
|
||||
try:
|
||||
if method == "POST":
|
||||
response = c.message(alice, chat, "load-" + key, key=key)
|
||||
with lock:
|
||||
confirmed[response["json"]["id"]] = response["json"]
|
||||
key = uuid4().hex
|
||||
else:
|
||||
response = c.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100",
|
||||
actor=bob, schema="MessagePage")
|
||||
with lock:
|
||||
successes.append((time.monotonic() - started, method,
|
||||
response["headers"]["X-Instance-Id"], response["elapsed"]))
|
||||
except Failure as exc:
|
||||
with lock:
|
||||
errors.append((time.monotonic() - started, method, str(exc)))
|
||||
stop.wait(0.05)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||
futures = [pool.submit(worker, i) for i in range(4)]
|
||||
time.sleep(1)
|
||||
action_start = time.monotonic() - started
|
||||
try:
|
||||
action_info = action()
|
||||
action_end = time.monotonic() - started
|
||||
# Охватывает обнаружение отказа и устойчивую работу после него.
|
||||
time.sleep(max(3, 15 - (action_end - action_start)))
|
||||
finally:
|
||||
stop.set()
|
||||
for future in futures:
|
||||
future.result()
|
||||
elapsed = time.monotonic() - started
|
||||
items = history(client, bob, chat)
|
||||
by_id = {m["id"]: m for m in items}
|
||||
assert len(items) == len(by_id) == len({m["text"] for m in items}), "Повтор создал дубль"
|
||||
assert by_id.get(original["id"]) == original
|
||||
assert all(by_id.get(mid) == msg for mid, msg in confirmed.items()), "Потеря подтверждённого сообщения"
|
||||
after = [s for s in successes if s[0] >= action_end]
|
||||
assert {s[1] for s in after} == {"POST", "GET"}, "Нет чтений/записей после воздействия"
|
||||
recovery = {method: round(min(s[0] for s in successes if s[0] >= action_start and s[1] == method) - action_start, 3)
|
||||
for method in ("POST", "GET")}
|
||||
assert max(recovery.values()) <= 10, recovery
|
||||
max_gap = {}
|
||||
for method in ("POST", "GET"):
|
||||
moments = sorted(s[0] for s in successes if s[1] == method)
|
||||
max_gap[method] = round(max(b - a for a, b in zip(moments, moments[1:])), 3)
|
||||
assert max(max_gap.values()) <= 10, max_gap
|
||||
if target:
|
||||
assert target not in {s[2] for s in after[-20:]}, "Остановленная реплика ещё отвечает"
|
||||
buckets = []
|
||||
for second in range(int(elapsed) + 1):
|
||||
buckets.append({"second": second, "successes": sum(second <= s[0] < second + 1 for s in successes),
|
||||
"errors": sum(second <= e[0] < second + 1 for e in errors)})
|
||||
latencies = sorted(s[3] * 1000 for s in successes)
|
||||
summary = {"label": label, "duration_seconds": round(elapsed, 3), "successful_requests": len(successes),
|
||||
"successful_writes": len(confirmed), "errors": len(errors), "successful_rps": round(len(successes) / elapsed, 2),
|
||||
"first_success_after_action_seconds": recovery, "maximum_success_gap_seconds": max_gap,
|
||||
"action_start_seconds": round(action_start, 3), "action_end_seconds": round(action_end, 3),
|
||||
"p50_ms": round(latencies[len(latencies) // 2], 3),
|
||||
"p95_ms": round(latencies[int(len(latencies) * .95)], 3),
|
||||
"confirmed_data_preserved": True, "duplicate_messages": 0,
|
||||
"instances": sorted({s[2] for s in successes}), "per_second": buckets,
|
||||
"error_samples": errors[:5], "action": action_info}
|
||||
print(json.dumps({k: v for k, v in summary.items() if k not in ("per_second", "error_samples", "action")}, ensure_ascii=False), flush=True)
|
||||
return summary
|
||||
|
||||
print("Command: ./scripts/check-docker.sh; project:", project, flush=True)
|
||||
print("Docker:", result["docker"], flush=True)
|
||||
try:
|
||||
script("start.sh")
|
||||
initial = inspect_all()
|
||||
result["containers"] = [metadata(info) for info in initial]
|
||||
assert len(apis()) == 2
|
||||
for info in initial:
|
||||
assert info["HostConfig"]["Memory"] > 0 and info["HostConfig"]["NanoCpus"] > 0
|
||||
if info["Config"]["Labels"]["com.docker.compose.service"] != "lb":
|
||||
assert not info["HostConfig"]["PortBindings"]
|
||||
for info in apis():
|
||||
assert docker("exec", info["Id"], "id", "-u") == "10001"
|
||||
assert all(m["Destination"].startswith("/run/secrets/") for m in info["Mounts"])
|
||||
print("PASS: 2 real API containers, CPU/RAM limits, API non-root and no public ports; only LB published", flush=True)
|
||||
subprocess.run([str(ROOT / "scripts/check.sh")], cwd=ROOT, env=env, check=True, timeout=240)
|
||||
result["api_tests"] = "public smoke + 28 student tests passed"
|
||||
subprocess.run([str(ROOT / "scripts/demo.sh")], cwd=ROOT, env=env, check=True, stdout=subprocess.DEVNULL, timeout=30)
|
||||
assert sql("SELECT count(*) FROM schema_migrations") == "2"
|
||||
count = sql("SELECT count(*) FROM messages")
|
||||
for _ in range(2):
|
||||
compose("run", "--rm", "migrate")
|
||||
assert sql("SELECT count(*) FROM schema_migrations") == "2"
|
||||
assert sql("SELECT count(*) FROM messages") == count
|
||||
print("PASS: migrations only in one-shot service; repeated execution preserves data", flush=True)
|
||||
|
||||
replay_client = Client(base)
|
||||
replay_actor, _, _, replay_chat = replay_client.bootstrap()
|
||||
replay_key = uuid4().hex
|
||||
saved_message = replay_client.message(replay_actor, replay_chat, "Повтор после рестарта", key=replay_key)["json"]
|
||||
before = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]}
|
||||
subprocess.run(["make", "test-persistence", "PYTHON=" + sys.executable, "ACTION_SCRIPT=scripts/restart.sh"],
|
||||
cwd=ROOT, env=env, check=True, timeout=180)
|
||||
after = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]}
|
||||
assert before.keys() == after.keys()
|
||||
assert all(before[cid]["started_at"] != after[cid]["started_at"] for cid in before)
|
||||
assert replay_client.message(replay_actor, replay_chat, "Повтор после рестарта", key=replay_key)["json"] == saved_message
|
||||
result["persistence"] = {"before": list(before.values()), "after": list(after.values()), "passed": True,
|
||||
"idempotency_replay_after_restart": True}
|
||||
print("PASS: both API, LB, PostgreSQL, Valkey actually restarted; session and confirmed data preserved", flush=True)
|
||||
|
||||
check_shutdown()
|
||||
|
||||
dependency_client = Client(base)
|
||||
actor = dependency_client.user("deps")
|
||||
result["dependency_readiness"] = []
|
||||
for dependency in ("valkey", "postgres"):
|
||||
try:
|
||||
compose("stop", "--timeout", "8", dependency)
|
||||
probe_id = apis()[0]["Id"]
|
||||
readiness = subprocess.run(["docker", "exec", probe_id, "python", "-m", "src.healthcheck"], capture_output=True, timeout=10)
|
||||
assert readiness.returncode == 1
|
||||
live = docker("exec", probe_id, "python", "-c",
|
||||
"from urllib.request import urlopen; print(urlopen('http://127.0.0.1:8080/health/live').status)")
|
||||
assert live == "200"
|
||||
deadline = time.monotonic() + 8
|
||||
while True:
|
||||
try:
|
||||
urlopen(base + "/health/ready", timeout=2).close()
|
||||
except HTTPError as error:
|
||||
if error.code == 503:
|
||||
break
|
||||
raise
|
||||
except OSError:
|
||||
# Уже отправленный запрос может истечь до исключения backend.
|
||||
pass
|
||||
assert time.monotonic() < deadline
|
||||
time.sleep(0.1)
|
||||
result["dependency_readiness"].append({"dependency": dependency, "api_live": 200, "api_ready": 503, "lb_ready": 503})
|
||||
print("PASS:", dependency, "outage: API live=200, ready=503, LB ready=503", flush=True)
|
||||
finally:
|
||||
compose("start", dependency)
|
||||
script("wait-ready.sh")
|
||||
dependency_client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
|
||||
subprocess.run(["make", "test-failover", "PYTHON=" + sys.executable, "ACTION_SCRIPT=scripts/stop-one.sh"],
|
||||
cwd=ROOT, env=env, check=True, timeout=90)
|
||||
assert sum(i["State"]["Running"] for i in apis()) == 1
|
||||
result["public_failover"] = {"passed": True, "containers_after": [metadata(i) for i in apis()]}
|
||||
compose("start", "api")
|
||||
script("wait-ready.sh")
|
||||
|
||||
result["failovers"] = []
|
||||
for info in apis():
|
||||
target = info["Config"]["Hostname"]
|
||||
|
||||
def stop_target():
|
||||
subprocess.run([str(ROOT / "scripts/stop-one.sh")], cwd=ROOT,
|
||||
env=dict(env, TARGET_INSTANCE_ID=target), check=True, timeout=10)
|
||||
stopped = json.loads(docker("inspect", info["Id"]))[0]
|
||||
assert not stopped["State"]["Running"]
|
||||
assert not stopped["State"]["OOMKilled"] and stopped["State"]["ExitCode"] == 0
|
||||
# Uvicorn пишет в stderr; получаем оба потока без вывода запросов.
|
||||
logs_result = subprocess.run(["docker", "logs", info["Id"]], capture_output=True, text=True, check=True)
|
||||
assert "Application shutdown complete" in logs_result.stdout + logs_result.stderr
|
||||
return metadata(stopped)
|
||||
|
||||
result["failovers"].append(load_action("stop " + target, stop_target, target))
|
||||
compose("start", "api")
|
||||
script("wait-ready.sh")
|
||||
|
||||
before = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]}
|
||||
|
||||
def replace():
|
||||
script("rolling-update.sh")
|
||||
after = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]}
|
||||
old_api = {i["id"] for i in before.values() if i["service"] == "api"}
|
||||
new_api = {i["id"] for i in after.values() if i["service"] == "api"}
|
||||
assert len(new_api) == 2 and not old_api & new_api
|
||||
for cid, info in before.items():
|
||||
if info["service"] != "api":
|
||||
assert after[cid]["started_at"] == info["started_at"]
|
||||
return {"before": list(before.values()), "after": list(after.values())}
|
||||
|
||||
result["rolling_update"] = load_action("sequential API replacement", replace)
|
||||
result["passed"] = True
|
||||
print("PASS: all lab3 grade-4 checks completed", flush=True)
|
||||
finally:
|
||||
output = ROOT / "evidence" / "lab3-checks.json"
|
||||
output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
||||
compose("down", "--volumes", "--timeout", "8")
|
||||
print("Removed only temporary test containers and volumes:", project, flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user