реализация 3 лабы

This commit is contained in:
Ванек
2026-10-29 14:51:19 +03:00
parent 74aa224251
commit 64ea89d040
28 changed files with 2411 additions and 243 deletions
+282 -169
View File
@@ -1,4 +1,5 @@
"""Полный прогон lab2 в собственном Compose project; основные данные не затрагивает."""
"""Проверки уровня 4 в отдельном Compose project с синтетическими данными."""
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
import json
import os
@@ -6,195 +7,307 @@ import platform
import socket
import subprocess
import sys
import threading
import time
from urllib.error import URLError
from urllib.error import HTTPError
from urllib.request import urlopen
from uuid import uuid4
from helpers import Client, ROOT
from client import Failure
from resilience import history
def docker(*args):
return subprocess.run(["docker", *args], check=True, capture_output=True, text=True, timeout=120).stdout.strip()
def inspect(container):
return json.loads(docker("inspect", container))[0]
def check_shutdown(container, base_url, port):
with socket.create_connection(("127.0.0.1", port), timeout=2) as pending:
pending.sendall(
b"POST /api/v1/users HTTP/1.1\r\nHost: localhost\r\n"
b"Content-Type: application/json\r\nContent-Length: 1000\r\n"
b"Expect: 100-continue\r\nConnection: close\r\n\r\n"
)
assert b"100 Continue" in pending.recv(1024), "Сервер не начал читать тело запроса"
pending.sendall(b'{"username":')
started = time.monotonic()
docker("kill", "--signal=TERM", container)
first_refusal = None
while inspect(container)["State"]["Running"]:
assert time.monotonic() - started < 10, "SIGTERM: превышены 10 секунд"
try:
with urlopen(base_url + "/health/live", timeout=0.3):
pass
except (URLError, OSError):
if first_refusal is None:
first_refusal = time.monotonic() - started
time.sleep(0.05)
elapsed = time.monotonic() - started
assert elapsed < 10, "SIGTERM: превышены 10 секунд"
assert first_refusal is not None and first_refusal < elapsed, "Не показан отказ новых запросов при остановке"
state = inspect(container)["State"]
assert not state["OOMKilled"] and state["ExitCode"] in (0, 143), state
result = subprocess.run(["docker", "logs", container], capture_output=True, text=True, check=True, timeout=10)
logs = result.stdout + result.stderr
assert "Application shutdown complete" in logs, "Нет подтверждения корректного завершения"
try:
with urlopen(base_url + "/health/live", timeout=0.5):
raise AssertionError("Остановленный сервер отвечает")
except (URLError, OSError):
pass
print(f"PASS: SIGTERM with unfinished request: {elapsed:.2f}s; new requests refused after {first_refusal:.2f}s; exit={state['ExitCode']}; OOM=false")
return subprocess.check_output(["docker", *args], text=True, timeout=180).strip()
def main():
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
port = sock.getsockname()[1]
project = "lab2-check-" + uuid4().hex[:10]
env = dict(os.environ, COMPOSE_PROJECT_NAME=project, HOST_PORT=str(port), SESSION_TTL_SECONDS="3600")
base_url = f"http://127.0.0.1:{port}"
project = "lab3-check-" + uuid4().hex[:10]
base = f"http://127.0.0.1:{port}"
env = dict(os.environ, COMPOSE_PROJECT_NAME=project, HOST_PORT=str(port),
API_REPLICAS="2", SESSION_TTL_SECONDS="3600", BASE_URL=base)
result = {"utc": datetime.now(timezone.utc).isoformat(), "project": project,
"base_url": base, "host": platform.system() + " " + platform.machine(),
"docker": docker("info", "--format", "{{.ServerVersion}} {{.Architecture}} CPUs={{.NCPU}} RAM={{.MemTotal}}"),
"commit": docker("compose", "version", "--short"),
"profile": {"workers": 4, "pause_seconds": 0.05, "operation": "POST message then GET history, same-key retry after uncertain POST"}}
result["compose_version"] = result.pop("commit")
result["source_commit"] = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
result["working_tree"] = True
def compose(*args, capture=True, stdin=None):
return subprocess.run(["docker", "compose", *args], cwd=ROOT, env=env, input=stdin,
text=True, capture_output=capture, check=True, timeout=180)
def compose(*args):
return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, env=env, text=True, timeout=180).strip()
def ready():
subprocess.run([str(ROOT / "scripts/start.sh")], cwd=ROOT, env=env,
check=True, capture_output=True, text=True, timeout=180)
print("PASS: start.sh built and started the stack, readiness reached")
def script(name):
subprocess.run([str(ROOT / "scripts" / name)], cwd=ROOT, env=env, check=True, timeout=240)
def states():
return {service: inspect(compose("ps", "-q", service).stdout.strip()) for service in ("api", "postgres", "valkey")}
def inspect_all():
ids = compose("ps", "-a", "-q").split()
return json.loads(docker("inspect", *ids))
def scalar(query, params=()):
code = """import json,sys
from src import db
data=json.load(sys.stdin)
db.pool.open(wait=True,timeout=15)
try:
with db.connection() as conn:
row=conn.execute(data['query'], data['params']).fetchone()
print(json.dumps(next(iter(row.values()))))
finally:
db.pool.close()
def apis():
return [i for i in inspect_all() if i["Config"]["Labels"]["com.docker.compose.service"] == "api"]
def metadata(info):
return {"id": info["Id"], "instance": info["Config"]["Hostname"],
"service": info["Config"]["Labels"]["com.docker.compose.service"],
"image": info["Image"], "started_at": info["State"]["StartedAt"],
"running": info["State"]["Running"], "exit_code": info["State"]["ExitCode"],
"memory_bytes": info["HostConfig"]["Memory"], "nano_cpus": info["HostConfig"]["NanoCpus"],
"published_ports": info["HostConfig"]["PortBindings"],
"mounts": [{"type": m["Type"], "destination": m["Destination"], "name": m.get("Name")} for m in info["Mounts"]]}
def sql(query):
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c", query)
def check_shutdown():
info = apis()[0]
# Внутреннее подключение через docker exec не публикует порт API на хост.
pending_code = """import socket,time
s=socket.create_connection(('127.0.0.1',8080),timeout=2)
s.sendall(b'POST /api/v1/users HTTP/1.1\\r\\nHost: localhost\\r\\nContent-Type: application/json\\r\\nContent-Length: 1000\\r\\nExpect: 100-continue\\r\\n\\r\\n')
assert b'100 Continue' in s.recv(1024)
s.sendall(b'{"username":')
print('pending',flush=True)
time.sleep(30)
"""
return json.loads(compose("exec", "-T", "api", "python", "-c", code,
stdin=json.dumps({"query": query, "params": params})).stdout)
print("UTC:", datetime.now(timezone.utc).isoformat())
print("Host:", platform.system(), platform.machine(), "Python", platform.python_version())
print("Docker:", docker("info", "--format", "{{.ServerVersion}} {{.Architecture}} CPUs={{.NCPU}} RAM={{.MemTotal}}"))
print("Commit:", subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, text=True).stdout.strip(), "+ working tree")
print("Command: ./scripts/check-docker.sh; project:", project)
try:
ready()
before = states()
for service, info in before.items():
print(service, "image=" + info["Image"], "started=" + info["State"]["StartedAt"],
"memory=" + str(info["HostConfig"]["Memory"]), "nano_cpus=" + str(info["HostConfig"]["NanoCpus"]))
api = compose("ps", "-q", "api").stdout.strip()
assert docker("exec", api, "id", "-u") == "10001"
assert before["api"]["Config"]["Cmd"] == ["python", "-m", "src"]
for service in ("postgres", "valkey"):
assert any(mount["Type"] == "volume" for mount in before[service]["Mounts"])
assert all(mount["Destination"].startswith("/run/secrets/") for mount in before["api"]["Mounts"])
bad_health = subprocess.run(["docker", "exec", "-e", "APP_PORT=1", api, "python", "-m", "src.healthcheck"], capture_output=True)
assert bad_health.returncode == 1
print("PASS: non-root UID 10001; Python PID 1; healthcheck; named volumes; no API data volume")
subprocess.run([str(ROOT / "scripts/check.sh")], cwd=ROOT, env=dict(env, BASE_URL=base_url), check=True, timeout=240)
demo = subprocess.run([str(ROOT / "scripts/demo.sh")], cwd=ROOT, env=dict(env, BASE_URL=base_url),
check=True, capture_output=True, text=True, timeout=30)
assert "Привет из lab2!" in demo.stdout
print("PASS: demo.sh registered users, logged in and read two messages")
client = Client(base_url)
alice, bob, eve, chat = client.bootstrap()
marker = "private-" + uuid4().hex
client.message(alice, chat, marker)
logs = docker("logs", api)
assert marker not in logs and alice["token"] not in logs and alice["registration"]["password"] not in logs
logs_result = subprocess.run(["docker", "logs", api], capture_output=True, text=True, check=True)
assert "request_id=" in logs_result.stderr + logs_result.stdout
assert marker not in logs_result.stderr + logs_result.stdout
assert alice["token"] not in logs_result.stderr + logs_result.stdout
assert alice["registration"]["password"] not in logs_result.stderr + logs_result.stdout
assert scalar("SELECT bool_and(password_hash LIKE %s) FROM users", ("$argon2id$%",))
print("PASS: Argon2id in PostgreSQL; logs do not contain message text, passwords or session tokens")
title = "rollback_" + uuid4().hex
client.call("POST", "/api/v1/chats", 400, actor=alice,
body={"title": title, "member_ids": [alice["id"], str(uuid4())]})
assert scalar("SELECT count(*) FROM chats WHERE title=%s", (title,)) == 0
count = scalar("SELECT count(*) FROM messages")
compose("exec", "-T", "api", "python", "-m", "src.migrate")
compose("exec", "-T", "api", "python", "-m", "src.migrate")
assert scalar("SELECT count(*) FROM schema_migrations") == 1
assert scalar("SELECT count(*) FROM messages") == count
print("PASS: failed membership insert rolled back chat; migrations run twice without duplicate data")
before = states()
subprocess.run(["make", "test-persistence", "PYTHON=" + sys.executable,
"ACTION_SCRIPT=scripts/restart.sh", "BASE_URL=" + base_url], cwd=ROOT, env=env, check=True, timeout=240)
after = states()
for service in before:
assert before[service]["State"]["StartedAt"] != after[service]["State"]["StartedAt"]
print("Restart", service, before[service]["State"]["StartedAt"], "->", after[service]["State"]["StartedAt"])
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")
print("PASS: API, PostgreSQL and Valkey really restarted; previous token still works")
for service in ("valkey", "postgres"):
try:
compose("stop", "--timeout", "10", service)
started = time.monotonic()
client.call("GET", "/api/v1/users/me", 503, actor=alice)
elapsed = time.monotonic() - started
assert elapsed < 6, elapsed
client.call("GET", "/api/v1/users/me", 401)
client.call("GET", "/health/ready", 503)
client.call("GET", "/health/live", schema="Health")
print(f"PASS: {service} unavailable -> 503 in {elapsed:.2f}s; anonymous -> 401; readiness 503, liveness 200")
finally:
compose("start", service)
subprocess.run([str(ROOT / "scripts/wait-ready.sh")], cwd=ROOT, env=env, check=True, timeout=90)
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")
print("PASS:", service, "recovered without a new login")
env["SESSION_TTL_SECONDS"] = "2"
compose("up", "-d", "--force-recreate", "api")
subprocess.run([str(ROOT / "scripts/wait-ready.sh")], cwd=ROOT, env=env, check=True, timeout=90)
short = client.user("short_ttl")
client.call("GET", "/api/v1/users/me", actor=short, schema="User")
time.sleep(2.2)
client.call("GET", "/api/v1/users/me", 401, actor=short)
print("PASS: SESSION_TTL_SECONDS=2; valid token expires without logout")
api = compose("ps", "-q", "api").stdout.strip()
check_shutdown(api, base_url, port)
print("PASS: all lab2 Docker checks completed")
finally:
pending = subprocess.Popen(["docker", "exec", info["Id"], "python", "-u", "-c", pending_code],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
try:
subprocess.run([str(ROOT / "scripts/stop.sh")], cwd=ROOT, env=env,
check=True, capture_output=True, text=True, timeout=90)
volumes = docker("volume", "ls", "--quiet", "--filter", "label=com.docker.compose.project=" + project)
if volumes:
print("PASS: stop.sh removed containers and preserved named volumes")
assert pending.stdout.readline().strip() == "pending"
started = time.monotonic()
docker("kill", "--signal=TERM", info["Id"])
first_refusal = None
while True:
state = json.loads(docker("inspect", info["Id"]))[0]["State"]
if not state["Running"]:
break
assert time.monotonic() - started < 8
probe = subprocess.run(["docker", "exec", info["Id"], "python", "-c",
"import socket; s=socket.socket(); s.settimeout(.2); raise SystemExit(s.connect_ex(('127.0.0.1',8080)))"],
capture_output=True, timeout=3)
if probe.returncode and first_refusal is None:
first_refusal = time.monotonic() - started
time.sleep(0.1)
elapsed = time.monotonic() - started
assert first_refusal is not None and first_refusal < elapsed < 8
assert state["ExitCode"] == 0 and not state["OOMKilled"]
logs = subprocess.run(["docker", "logs", info["Id"]], capture_output=True, text=True, check=True)
assert "Application shutdown complete" in logs.stdout + logs.stderr
result["sigterm"] = {"instance": info["Config"]["Hostname"], "unfinished_request": True,
"shutdown_seconds": round(elapsed, 3), "new_connections_refused_seconds": round(first_refusal, 3),
"exit_code": state["ExitCode"], "oom_killed": state["OOMKilled"]}
print("PASS: SIGTERM with unfinished request:", json.dumps(result["sigterm"]), flush=True)
finally:
compose("down", "--volumes", "--timeout", "10")
print("Removed only temporary Compose project and its test volumes:", project)
pending.communicate(timeout=10)
compose("start", "api")
script("wait-ready.sh")
def load_action(label, action, target=None):
client = Client(base, timeout=2)
alice, bob, _, chat = client.bootstrap()
original = client.message(alice, chat, "До воздействия")["json"]
stop = threading.Event()
lock = threading.Lock()
successes, errors, confirmed = [], [], {}
started = time.monotonic()
def worker(index):
c = Client(base, timeout=2)
key = uuid4().hex
while not stop.is_set():
for method in ("POST", "GET"):
try:
if method == "POST":
response = c.message(alice, chat, "load-" + key, key=key)
with lock:
confirmed[response["json"]["id"]] = response["json"]
key = uuid4().hex
else:
response = c.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100",
actor=bob, schema="MessagePage")
with lock:
successes.append((time.monotonic() - started, method,
response["headers"]["X-Instance-Id"], response["elapsed"]))
except Failure as exc:
with lock:
errors.append((time.monotonic() - started, method, str(exc)))
stop.wait(0.05)
with ThreadPoolExecutor(max_workers=4) as pool:
futures = [pool.submit(worker, i) for i in range(4)]
time.sleep(1)
action_start = time.monotonic() - started
try:
action_info = action()
action_end = time.monotonic() - started
# Охватывает обнаружение отказа и устойчивую работу после него.
time.sleep(max(3, 15 - (action_end - action_start)))
finally:
stop.set()
for future in futures:
future.result()
elapsed = time.monotonic() - started
items = history(client, bob, chat)
by_id = {m["id"]: m for m in items}
assert len(items) == len(by_id) == len({m["text"] for m in items}), "Повтор создал дубль"
assert by_id.get(original["id"]) == original
assert all(by_id.get(mid) == msg for mid, msg in confirmed.items()), "Потеря подтверждённого сообщения"
after = [s for s in successes if s[0] >= action_end]
assert {s[1] for s in after} == {"POST", "GET"}, "Нет чтений/записей после воздействия"
recovery = {method: round(min(s[0] for s in successes if s[0] >= action_start and s[1] == method) - action_start, 3)
for method in ("POST", "GET")}
assert max(recovery.values()) <= 10, recovery
max_gap = {}
for method in ("POST", "GET"):
moments = sorted(s[0] for s in successes if s[1] == method)
max_gap[method] = round(max(b - a for a, b in zip(moments, moments[1:])), 3)
assert max(max_gap.values()) <= 10, max_gap
if target:
assert target not in {s[2] for s in after[-20:]}, "Остановленная реплика ещё отвечает"
buckets = []
for second in range(int(elapsed) + 1):
buckets.append({"second": second, "successes": sum(second <= s[0] < second + 1 for s in successes),
"errors": sum(second <= e[0] < second + 1 for e in errors)})
latencies = sorted(s[3] * 1000 for s in successes)
summary = {"label": label, "duration_seconds": round(elapsed, 3), "successful_requests": len(successes),
"successful_writes": len(confirmed), "errors": len(errors), "successful_rps": round(len(successes) / elapsed, 2),
"first_success_after_action_seconds": recovery, "maximum_success_gap_seconds": max_gap,
"action_start_seconds": round(action_start, 3), "action_end_seconds": round(action_end, 3),
"p50_ms": round(latencies[len(latencies) // 2], 3),
"p95_ms": round(latencies[int(len(latencies) * .95)], 3),
"confirmed_data_preserved": True, "duplicate_messages": 0,
"instances": sorted({s[2] for s in successes}), "per_second": buckets,
"error_samples": errors[:5], "action": action_info}
print(json.dumps({k: v for k, v in summary.items() if k not in ("per_second", "error_samples", "action")}, ensure_ascii=False), flush=True)
return summary
print("Command: ./scripts/check-docker.sh; project:", project, flush=True)
print("Docker:", result["docker"], flush=True)
try:
script("start.sh")
initial = inspect_all()
result["containers"] = [metadata(info) for info in initial]
assert len(apis()) == 2
for info in initial:
assert info["HostConfig"]["Memory"] > 0 and info["HostConfig"]["NanoCpus"] > 0
if info["Config"]["Labels"]["com.docker.compose.service"] != "lb":
assert not info["HostConfig"]["PortBindings"]
for info in apis():
assert docker("exec", info["Id"], "id", "-u") == "10001"
assert all(m["Destination"].startswith("/run/secrets/") for m in info["Mounts"])
print("PASS: 2 real API containers, CPU/RAM limits, API non-root and no public ports; only LB published", flush=True)
subprocess.run([str(ROOT / "scripts/check.sh")], cwd=ROOT, env=env, check=True, timeout=240)
result["api_tests"] = "public smoke + 28 student tests passed"
subprocess.run([str(ROOT / "scripts/demo.sh")], cwd=ROOT, env=env, check=True, stdout=subprocess.DEVNULL, timeout=30)
assert sql("SELECT count(*) FROM schema_migrations") == "2"
count = sql("SELECT count(*) FROM messages")
for _ in range(2):
compose("run", "--rm", "migrate")
assert sql("SELECT count(*) FROM schema_migrations") == "2"
assert sql("SELECT count(*) FROM messages") == count
print("PASS: migrations only in one-shot service; repeated execution preserves data", flush=True)
replay_client = Client(base)
replay_actor, _, _, replay_chat = replay_client.bootstrap()
replay_key = uuid4().hex
saved_message = replay_client.message(replay_actor, replay_chat, "Повтор после рестарта", key=replay_key)["json"]
before = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]}
subprocess.run(["make", "test-persistence", "PYTHON=" + sys.executable, "ACTION_SCRIPT=scripts/restart.sh"],
cwd=ROOT, env=env, check=True, timeout=180)
after = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]}
assert before.keys() == after.keys()
assert all(before[cid]["started_at"] != after[cid]["started_at"] for cid in before)
assert replay_client.message(replay_actor, replay_chat, "Повтор после рестарта", key=replay_key)["json"] == saved_message
result["persistence"] = {"before": list(before.values()), "after": list(after.values()), "passed": True,
"idempotency_replay_after_restart": True}
print("PASS: both API, LB, PostgreSQL, Valkey actually restarted; session and confirmed data preserved", flush=True)
check_shutdown()
dependency_client = Client(base)
actor = dependency_client.user("deps")
result["dependency_readiness"] = []
for dependency in ("valkey", "postgres"):
try:
compose("stop", "--timeout", "8", dependency)
probe_id = apis()[0]["Id"]
readiness = subprocess.run(["docker", "exec", probe_id, "python", "-m", "src.healthcheck"], capture_output=True, timeout=10)
assert readiness.returncode == 1
live = docker("exec", probe_id, "python", "-c",
"from urllib.request import urlopen; print(urlopen('http://127.0.0.1:8080/health/live').status)")
assert live == "200"
deadline = time.monotonic() + 8
while True:
try:
urlopen(base + "/health/ready", timeout=2).close()
except HTTPError as error:
if error.code == 503:
break
raise
except OSError:
# Уже отправленный запрос может истечь до исключения backend.
pass
assert time.monotonic() < deadline
time.sleep(0.1)
result["dependency_readiness"].append({"dependency": dependency, "api_live": 200, "api_ready": 503, "lb_ready": 503})
print("PASS:", dependency, "outage: API live=200, ready=503, LB ready=503", flush=True)
finally:
compose("start", dependency)
script("wait-ready.sh")
dependency_client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
subprocess.run(["make", "test-failover", "PYTHON=" + sys.executable, "ACTION_SCRIPT=scripts/stop-one.sh"],
cwd=ROOT, env=env, check=True, timeout=90)
assert sum(i["State"]["Running"] for i in apis()) == 1
result["public_failover"] = {"passed": True, "containers_after": [metadata(i) for i in apis()]}
compose("start", "api")
script("wait-ready.sh")
result["failovers"] = []
for info in apis():
target = info["Config"]["Hostname"]
def stop_target():
subprocess.run([str(ROOT / "scripts/stop-one.sh")], cwd=ROOT,
env=dict(env, TARGET_INSTANCE_ID=target), check=True, timeout=10)
stopped = json.loads(docker("inspect", info["Id"]))[0]
assert not stopped["State"]["Running"]
assert not stopped["State"]["OOMKilled"] and stopped["State"]["ExitCode"] == 0
# Uvicorn пишет в stderr; получаем оба потока без вывода запросов.
logs_result = subprocess.run(["docker", "logs", info["Id"]], capture_output=True, text=True, check=True)
assert "Application shutdown complete" in logs_result.stdout + logs_result.stderr
return metadata(stopped)
result["failovers"].append(load_action("stop " + target, stop_target, target))
compose("start", "api")
script("wait-ready.sh")
before = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]}
def replace():
script("rolling-update.sh")
after = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]}
old_api = {i["id"] for i in before.values() if i["service"] == "api"}
new_api = {i["id"] for i in after.values() if i["service"] == "api"}
assert len(new_api) == 2 and not old_api & new_api
for cid, info in before.items():
if info["service"] != "api":
assert after[cid]["started_at"] == info["started_at"]
return {"before": list(before.values()), "after": list(after.values())}
result["rolling_update"] = load_action("sequential API replacement", replace)
result["passed"] = True
print("PASS: all lab3 grade-4 checks completed", flush=True)
finally:
output = ROOT / "evidence" / "lab3-checks.json"
output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
compose("down", "--volumes", "--timeout", "8")
print("Removed only temporary test containers and volumes:", project, flush=True)
if __name__ == "__main__":