forked from mtusi2026/mtusi2026lab4
317 lines
18 KiB
Python
317 lines
18 KiB
Python
"""Проверки уровня 4 в отдельном Compose project с синтетическими данными."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timezone
|
|
import json
|
|
import os
|
|
import platform
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from urllib.error import HTTPError
|
|
from urllib.request import urlopen
|
|
from uuid import uuid4
|
|
|
|
from helpers import Client, ROOT
|
|
from client import Failure
|
|
from resilience import history
|
|
from attachment_checks import run_checks
|
|
|
|
|
|
def docker(*args):
|
|
return subprocess.check_output(["docker", *args], text=True, timeout=180).strip()
|
|
|
|
|
|
def main():
|
|
with socket.socket() as sock:
|
|
sock.bind(("127.0.0.1", 0))
|
|
port = sock.getsockname()[1]
|
|
project = "lab4-check-" + uuid4().hex[:10]
|
|
base = f"http://127.0.0.1:{port}"
|
|
env = dict(os.environ, COMPOSE_PROJECT_NAME=project, HOST_PORT=str(port),
|
|
API_REPLICAS="2", SESSION_TTL_SECONDS="3600", BASE_URL=base)
|
|
result = {"utc": datetime.now(timezone.utc).isoformat(), "project": project,
|
|
"base_url": base, "host": platform.system() + " " + platform.machine(),
|
|
"docker": docker("info", "--format", "{{.ServerVersion}} {{.Architecture}} CPUs={{.NCPU}} RAM={{.MemTotal}}"),
|
|
"commit": docker("compose", "version", "--short"),
|
|
"profile": {"workers": 4, "pause_seconds": 0.05, "operation": "POST message then GET history, same-key retry after uncertain POST"}}
|
|
result["compose_version"] = result.pop("commit")
|
|
result["source_commit"] = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
|
|
result["working_tree"] = True
|
|
|
|
def compose(*args):
|
|
return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, env=env, text=True, timeout=180).strip()
|
|
|
|
def script(name):
|
|
subprocess.run([str(ROOT / "scripts" / name)], cwd=ROOT, env=env, check=True, timeout=240)
|
|
|
|
def inspect_all():
|
|
ids = compose("ps", "-a", "-q").split()
|
|
return json.loads(docker("inspect", *ids))
|
|
|
|
def apis():
|
|
return [i for i in inspect_all() if i["Config"]["Labels"]["com.docker.compose.service"] == "api"]
|
|
|
|
def metadata(info):
|
|
return {"id": info["Id"], "instance": info["Config"]["Hostname"],
|
|
"service": info["Config"]["Labels"]["com.docker.compose.service"],
|
|
"image": info["Image"], "started_at": info["State"]["StartedAt"],
|
|
"running": info["State"]["Running"], "exit_code": info["State"]["ExitCode"],
|
|
"memory_bytes": info["HostConfig"]["Memory"], "nano_cpus": info["HostConfig"]["NanoCpus"],
|
|
"published_ports": info["HostConfig"]["PortBindings"],
|
|
"mounts": [{"type": m["Type"], "destination": m["Destination"], "name": m.get("Name")} for m in info["Mounts"]]}
|
|
|
|
def sql(query):
|
|
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c", query)
|
|
|
|
def check_shutdown():
|
|
info = apis()[0]
|
|
# Внутреннее подключение через docker exec не публикует порт API на хост.
|
|
pending_code = """import socket,time
|
|
s=socket.create_connection(('127.0.0.1',8080),timeout=2)
|
|
s.sendall(b'POST /api/v1/users HTTP/1.1\\r\\nHost: localhost\\r\\nContent-Type: application/json\\r\\nContent-Length: 1000\\r\\nExpect: 100-continue\\r\\n\\r\\n')
|
|
assert b'100 Continue' in s.recv(1024)
|
|
s.sendall(b'{"username":')
|
|
print('pending',flush=True)
|
|
time.sleep(30)
|
|
"""
|
|
pending = subprocess.Popen(["docker", "exec", info["Id"], "python", "-u", "-c", pending_code],
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
try:
|
|
assert pending.stdout.readline().strip() == "pending"
|
|
started = time.monotonic()
|
|
docker("kill", "--signal=TERM", info["Id"])
|
|
first_refusal = None
|
|
while True:
|
|
state = json.loads(docker("inspect", info["Id"]))[0]["State"]
|
|
if not state["Running"]:
|
|
break
|
|
assert time.monotonic() - started < 8
|
|
probe = subprocess.run(["docker", "exec", info["Id"], "python", "-c",
|
|
"import socket; s=socket.socket(); s.settimeout(.2); raise SystemExit(s.connect_ex(('127.0.0.1',8080)))"],
|
|
capture_output=True, timeout=3)
|
|
if probe.returncode and first_refusal is None:
|
|
first_refusal = time.monotonic() - started
|
|
time.sleep(0.1)
|
|
elapsed = time.monotonic() - started
|
|
assert first_refusal is not None and first_refusal < elapsed < 8
|
|
assert state["ExitCode"] == 0 and not state["OOMKilled"]
|
|
logs = subprocess.run(["docker", "logs", info["Id"]], capture_output=True, text=True, check=True)
|
|
assert "Application shutdown complete" in logs.stdout + logs.stderr
|
|
result["sigterm"] = {"instance": info["Config"]["Hostname"], "unfinished_request": True,
|
|
"shutdown_seconds": round(elapsed, 3), "new_connections_refused_seconds": round(first_refusal, 3),
|
|
"exit_code": state["ExitCode"], "oom_killed": state["OOMKilled"]}
|
|
print("PASS: SIGTERM with unfinished request:", json.dumps(result["sigterm"]), flush=True)
|
|
finally:
|
|
pending.communicate(timeout=10)
|
|
compose("start", "api")
|
|
script("wait-ready.sh")
|
|
|
|
def load_action(label, action, target=None):
|
|
client = Client(base, timeout=2)
|
|
alice, bob, _, chat = client.bootstrap()
|
|
original = client.message(alice, chat, "До воздействия")["json"]
|
|
stop = threading.Event()
|
|
lock = threading.Lock()
|
|
successes, errors, confirmed = [], [], {}
|
|
started = time.monotonic()
|
|
|
|
def worker(index):
|
|
c = Client(base, timeout=2)
|
|
key = uuid4().hex
|
|
while not stop.is_set():
|
|
for method in ("POST", "GET"):
|
|
try:
|
|
if method == "POST":
|
|
response = c.message(alice, chat, "load-" + key, key=key)
|
|
with lock:
|
|
confirmed[response["json"]["id"]] = response["json"]
|
|
key = uuid4().hex
|
|
else:
|
|
response = c.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100",
|
|
actor=bob, schema="MessagePage")
|
|
with lock:
|
|
successes.append((time.monotonic() - started, method,
|
|
response["headers"]["X-Instance-Id"], response["elapsed"]))
|
|
except Failure as exc:
|
|
with lock:
|
|
errors.append((time.monotonic() - started, method, str(exc)))
|
|
stop.wait(0.05)
|
|
|
|
with ThreadPoolExecutor(max_workers=4) as pool:
|
|
futures = [pool.submit(worker, i) for i in range(4)]
|
|
time.sleep(1)
|
|
action_start = time.monotonic() - started
|
|
try:
|
|
action_info = action()
|
|
action_end = time.monotonic() - started
|
|
# Охватывает обнаружение отказа и устойчивую работу после него.
|
|
time.sleep(max(3, 15 - (action_end - action_start)))
|
|
finally:
|
|
stop.set()
|
|
for future in futures:
|
|
future.result()
|
|
elapsed = time.monotonic() - started
|
|
items = history(client, bob, chat)
|
|
by_id = {m["id"]: m for m in items}
|
|
assert len(items) == len(by_id) == len({m["text"] for m in items}), "Повтор создал дубль"
|
|
assert by_id.get(original["id"]) == original
|
|
assert all(by_id.get(mid) == msg for mid, msg in confirmed.items()), "Потеря подтверждённого сообщения"
|
|
after = [s for s in successes if s[0] >= action_end]
|
|
assert {s[1] for s in after} == {"POST", "GET"}, "Нет чтений/записей после воздействия"
|
|
recovery = {method: round(min(s[0] for s in successes if s[0] >= action_start and s[1] == method) - action_start, 3)
|
|
for method in ("POST", "GET")}
|
|
assert max(recovery.values()) <= 10, recovery
|
|
max_gap = {}
|
|
for method in ("POST", "GET"):
|
|
moments = sorted(s[0] for s in successes if s[1] == method)
|
|
max_gap[method] = round(max(b - a for a, b in zip(moments, moments[1:])), 3)
|
|
assert max(max_gap.values()) <= 10, max_gap
|
|
if target:
|
|
assert target not in {s[2] for s in after[-20:]}, "Остановленная реплика ещё отвечает"
|
|
buckets = []
|
|
for second in range(int(elapsed) + 1):
|
|
buckets.append({"second": second, "successes": sum(second <= s[0] < second + 1 for s in successes),
|
|
"errors": sum(second <= e[0] < second + 1 for e in errors)})
|
|
latencies = sorted(s[3] * 1000 for s in successes)
|
|
summary = {"label": label, "duration_seconds": round(elapsed, 3), "successful_requests": len(successes),
|
|
"successful_writes": len(confirmed), "errors": len(errors), "successful_rps": round(len(successes) / elapsed, 2),
|
|
"first_success_after_action_seconds": recovery, "maximum_success_gap_seconds": max_gap,
|
|
"action_start_seconds": round(action_start, 3), "action_end_seconds": round(action_end, 3),
|
|
"p50_ms": round(latencies[len(latencies) // 2], 3),
|
|
"p95_ms": round(latencies[int(len(latencies) * .95)], 3),
|
|
"confirmed_data_preserved": True, "duplicate_messages": 0,
|
|
"instances": sorted({s[2] for s in successes}), "per_second": buckets,
|
|
"error_samples": errors[:5], "action": action_info}
|
|
print(json.dumps({k: v for k, v in summary.items() if k not in ("per_second", "error_samples", "action")}, ensure_ascii=False), flush=True)
|
|
return summary
|
|
|
|
print("Command: ./scripts/check-docker.sh; project:", project, flush=True)
|
|
print("Docker:", result["docker"], flush=True)
|
|
try:
|
|
script("start.sh")
|
|
initial = inspect_all()
|
|
result["containers"] = [metadata(info) for info in initial]
|
|
assert len(apis()) == 2
|
|
for info in initial:
|
|
assert info["HostConfig"]["Memory"] > 0 and info["HostConfig"]["NanoCpus"] > 0
|
|
if info["Config"]["Labels"]["com.docker.compose.service"] != "lb":
|
|
assert not info["HostConfig"]["PortBindings"]
|
|
for info in apis():
|
|
assert docker("exec", info["Id"], "id", "-u") == "10001"
|
|
assert all(m["Destination"].startswith("/run/secrets/") for m in info["Mounts"])
|
|
print("PASS: 2 real API containers, CPU/RAM limits, API non-root and no public ports; only LB published", flush=True)
|
|
subprocess.run([str(ROOT / "scripts/check.sh")], cwd=ROOT, env=env, check=True, timeout=240)
|
|
result["api_tests"] = "public smoke + student unittest suite passed"
|
|
subprocess.run([str(ROOT / "scripts/demo.sh")], cwd=ROOT, env=env, check=True, stdout=subprocess.DEVNULL, timeout=30)
|
|
assert sql("SELECT count(*) FROM schema_migrations") == "3"
|
|
count = sql("SELECT count(*) FROM messages")
|
|
for _ in range(2):
|
|
compose("run", "--rm", "migrate")
|
|
assert sql("SELECT count(*) FROM schema_migrations") == "3"
|
|
assert sql("SELECT count(*) FROM messages") == count
|
|
print("PASS: migrations only in one-shot service; repeated execution preserves data", flush=True)
|
|
result["attachments"] = run_checks(base, env, compose, apis, script, sql)
|
|
|
|
replay_client = Client(base)
|
|
replay_actor, _, _, replay_chat = replay_client.bootstrap()
|
|
replay_key = uuid4().hex
|
|
saved_message = replay_client.message(replay_actor, replay_chat, "Повтор после рестарта", key=replay_key)["json"]
|
|
before = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]}
|
|
subprocess.run(["make", "test-persistence", "PYTHON=" + sys.executable, "ACTION_SCRIPT=scripts/restart.sh"],
|
|
cwd=ROOT, env=env, check=True, timeout=180)
|
|
after = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]}
|
|
assert before.keys() == after.keys()
|
|
assert all(before[cid]["started_at"] != after[cid]["started_at"] for cid in before)
|
|
assert replay_client.message(replay_actor, replay_chat, "Повтор после рестарта", key=replay_key)["json"] == saved_message
|
|
result["persistence"] = {"before": list(before.values()), "after": list(after.values()), "passed": True,
|
|
"idempotency_replay_after_restart": True}
|
|
print("PASS: both API, LB, PostgreSQL, Valkey, S3 and cleaner actually restarted; data and session preserved", flush=True)
|
|
|
|
check_shutdown()
|
|
|
|
dependency_client = Client(base)
|
|
actor = dependency_client.user("deps")
|
|
result["dependency_readiness"] = []
|
|
for dependency in ("valkey", "postgres", "s3"):
|
|
try:
|
|
compose("stop", "--timeout", "8", dependency)
|
|
probe_id = apis()[0]["Id"]
|
|
readiness = subprocess.run(["docker", "exec", probe_id, "python", "-m", "src.healthcheck"], capture_output=True, timeout=10)
|
|
assert readiness.returncode == 1
|
|
live = docker("exec", probe_id, "python", "-c",
|
|
"from urllib.request import urlopen; print(urlopen('http://127.0.0.1:8080/health/live').status)")
|
|
assert live == "200"
|
|
deadline = time.monotonic() + 8
|
|
while True:
|
|
try:
|
|
urlopen(base + "/health/ready", timeout=2).close()
|
|
except HTTPError as error:
|
|
if error.code == 503:
|
|
break
|
|
raise
|
|
except OSError:
|
|
# Уже отправленный запрос может истечь до исключения backend.
|
|
pass
|
|
assert time.monotonic() < deadline
|
|
time.sleep(0.1)
|
|
result["dependency_readiness"].append({"dependency": dependency, "api_live": 200, "api_ready": 503, "lb_ready": 503})
|
|
print("PASS:", dependency, "outage: API live=200, ready=503, LB ready=503", flush=True)
|
|
finally:
|
|
compose("start", dependency)
|
|
script("wait-ready.sh")
|
|
dependency_client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
|
|
|
subprocess.run(["make", "test-failover", "PYTHON=" + sys.executable, "ACTION_SCRIPT=scripts/stop-one.sh"],
|
|
cwd=ROOT, env=env, check=True, timeout=90)
|
|
assert sum(i["State"]["Running"] for i in apis()) == 1
|
|
result["public_failover"] = {"passed": True, "containers_after": [metadata(i) for i in apis()]}
|
|
compose("start", "api")
|
|
script("wait-ready.sh")
|
|
|
|
result["failovers"] = []
|
|
for info in apis():
|
|
target = info["Config"]["Hostname"]
|
|
|
|
def stop_target():
|
|
subprocess.run([str(ROOT / "scripts/stop-one.sh")], cwd=ROOT,
|
|
env=dict(env, TARGET_INSTANCE_ID=target), check=True, timeout=10)
|
|
stopped = json.loads(docker("inspect", info["Id"]))[0]
|
|
assert not stopped["State"]["Running"]
|
|
assert not stopped["State"]["OOMKilled"] and stopped["State"]["ExitCode"] == 0
|
|
# Uvicorn пишет в stderr; получаем оба потока без вывода запросов.
|
|
logs_result = subprocess.run(["docker", "logs", info["Id"]], capture_output=True, text=True, check=True)
|
|
assert "Application shutdown complete" in logs_result.stdout + logs_result.stderr
|
|
return metadata(stopped)
|
|
|
|
result["failovers"].append(load_action("stop " + target, stop_target, target))
|
|
compose("start", "api")
|
|
script("wait-ready.sh")
|
|
|
|
before = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]}
|
|
|
|
def replace():
|
|
script("rolling-update.sh")
|
|
after = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]}
|
|
old_api = {i["id"] for i in before.values() if i["service"] == "api"}
|
|
new_api = {i["id"] for i in after.values() if i["service"] == "api"}
|
|
assert len(new_api) == 2 and not old_api & new_api
|
|
for cid, info in before.items():
|
|
if info["service"] != "api":
|
|
assert after[cid]["started_at"] == info["started_at"]
|
|
return {"before": list(before.values()), "after": list(after.values())}
|
|
|
|
result["rolling_update"] = load_action("sequential API replacement", replace)
|
|
result["passed"] = True
|
|
print("PASS: all lab4 grade-4 checks completed", flush=True)
|
|
finally:
|
|
output = ROOT / "evidence" / "lab4-checks.json"
|
|
output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
|
compose("down", "--volumes", "--timeout", "8")
|
|
print("Removed only temporary test containers and volumes:", project, flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|