ЛР3 на 3

Co-authored-by: Югай Артур Рустамович <f1ekserr@yandex.ru>
This commit is contained in:
2026-11-16 22:23:04 +03:00
co-authored by Югай Артур Рустамович
parent 0ab3a341aa
commit 09aece58fe
37 changed files with 2651 additions and 35 deletions
+535
View File
@@ -0,0 +1,535 @@
import argparse
from datetime import datetime, timezone
import hashlib
import json
import os
from pathlib import Path
import platform
import secrets
import shutil
import socket
import subprocess
import sys
import time
from urllib.error import URLError
from urllib.request import ProxyHandler, build_opener
import venv
ROOT = Path(__file__).resolve().parents[1]
LOG = []
LOCAL_HTTP = build_opener(ProxyHandler({}))
def settings():
path = ROOT / ".env"
result = {}
if path.exists():
for line in path.read_text(encoding="utf-8-sig").splitlines():
key, separator, value = line.partition("=")
if separator and not key.strip().startswith("#"):
result[key.strip()] = value.strip()
return result
def environment(args):
result = os.environ.copy()
result.update(PYTHONUTF8="1", PYTHONIOENCODING="utf-8", PYTHONDONTWRITEBYTECODE="1",
COMPOSE_PROJECT_NAME=args.project, HTTP_PORT=str(args.port),
SESSION_TTL_SECONDS=str(args.ttl), LAB3_PROJECT=args.project,
LAB3_PORT=str(args.port), LAB3_TTL=str(args.ttl))
for key in ("NO_PROXY", "no_proxy"):
result[key] = ",".join(filter(None, [result.get(key), "localhost", "127.0.0.1"]))
return result
def say(value):
line = str(value).replace(str(ROOT), ".")
print(line, flush=True)
LOG.append(line)
def run(args, command, quiet=False):
command = [str(item) for item in command]
if not quiet:
say("$ " + subprocess.list2cmdline(command))
with subprocess.Popen(command, cwd=ROOT, env=environment(args), stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace") as process:
lines = []
for line in process.stdout:
lines.append(line)
if not quiet:
say(line.rstrip())
code = process.wait()
if code:
raise RuntimeError(f"{command[0]} exited with code {code}")
return "".join(lines)
def compose(args, *command, quiet=False):
if args.project != "kuznetsov-lab3" and not args.project.startswith("kuznetsov-lab3-"):
raise RuntimeError("This runner only manages Kuznetsov lab3 projects")
return run(args, ["docker", "compose", "--project-name", args.project, "--env-file", ".env", *command], quiet)
def interpreter():
return ROOT / ".venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
def evidence(name, value):
path = ROOT / "evidence" / name
path.parent.mkdir(exist_ok=True)
content = json.dumps(value, ensure_ascii=False, indent=2) if not isinstance(value, str) else value
path.write_text(content.rstrip() + "\n", encoding="utf-8", newline="\n")
def setup(args):
if not (ROOT / ".env").exists():
shutil.copyfile(ROOT / ".env.example", ROOT / ".env")
folder = ROOT / "secrets"
folder.mkdir(exist_ok=True)
password_file = folder / "postgres_password"
if not password_file.exists():
password_file.write_text(secrets.token_hex(32) + "\n", encoding="ascii")
password = password_file.read_text(encoding="ascii").strip()
values = {"database_url": f"postgresql://messenger:{password}@db:5432/messenger",
"cursor_key": secrets.token_hex(32)}
for name, value in values.items():
path = folder / name
if not path.exists():
path.write_text(value + "\n", encoding="ascii")
if os.name != "nt":
path.chmod(0o644)
if os.name != "nt":
folder.chmod(0o700)
password_file.chmod(0o644)
if not interpreter().exists():
venv.EnvBuilder(with_pip=True).create(ROOT / ".venv")
lockfiles = [ROOT / "requirements.lock", ROOT / "requirements-validation.lock"]
fingerprint = hashlib.sha256(b"".join(path.read_bytes() for path in lockfiles)).hexdigest()
marker = ROOT / ".venv/.lab-dependencies"
if not marker.exists() or marker.read_text(encoding="ascii") != fingerprint:
run(args, [interpreter(), "-X", "utf8", "-m", "pip", "install", "--disable-pip-version-check",
"--no-input", "--progress-bar", "off", "--quiet", "-r", lockfiles[0], "-r", lockfiles[1]])
marker.write_text(fingerprint, encoding="ascii")
say("Environment ready; existing settings and secrets preserved")
def base_url(args):
return f"http://127.0.0.1:{args.port}"
def wait_ready(args, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
with LOCAL_HTTP.open(base_url(args) + "/health/ready", timeout=4) as response:
if response.status == 200 and json.load(response) == {"status": "ok"}:
return
except (URLError, OSError, ValueError):
pass
time.sleep(0.3)
raise RuntimeError("API readiness timeout")
def build(args):
compose(args, "config", "--quiet")
compose(args, "build", "api1")
compose(args, "run", "--rm", "--no-deps", "lb", "haproxy", "-c", "-f",
"/usr/local/etc/haproxy/haproxy.cfg")
def start(args):
current = compose(args, "ps", "--all", "--quiet", "lb", quiet=True).strip()
if not current:
with socket.socket() as probe:
try:
probe.bind(("127.0.0.1", args.port))
except OSError:
raise RuntimeError(f"Port {args.port} is busy; stop its container or use --port") from None
compose(args, "up", "-d", "--no-build", "db", "migrate", "api1", "api2", "lb")
wait_ready(args)
wait_replicas(args)
say("Two API replicas ready through LB: " + base_url(args))
def states(args):
result = {}
for service in ("api1", "api2", "db", "lb"):
container = compose(args, "ps", "--all", "--quiet", service, quiet=True).strip()
if not container:
raise RuntimeError("Missing container: " + service)
data = json.loads(run(args, ["docker", "inspect", container], quiet=True))[0]
labels = data["Config"].get("Labels", {})
if labels.get("com.docker.compose.project") != args.project or labels.get("com.docker.compose.service") != service:
raise RuntimeError("Unexpected container ownership")
result[service] = {
"id": data["Id"], "image": data["Image"], "started_at": data["State"]["StartedAt"],
"running": data["State"]["Running"], "user": data["Config"]["User"],
"volumes": [mount["Name"] for mount in data["Mounts"] if mount["Type"] == "volume"],
}
return result
def restart(args):
before = states(args)
if not all(item["running"] for item in before.values()):
raise RuntimeError("All runtime containers must be running before restart")
compose(args, "restart", "--timeout", "10", "db", "api1", "api2", "lb")
after = states(args)
for service in before:
if before[service]["started_at"] == after[service]["started_at"]:
raise RuntimeError("Container was not restarted: " + service)
if before[service]["volumes"] != after[service]["volumes"]:
raise RuntimeError("Volume changed during restart")
evidence("restart.json", {"project": args.project, "before": before, "after": after})
say("PASS: both APIs, PostgreSQL and LB restarted, volume preserved")
def contract_tests(args):
run(args, [interpreter(), "-X", "utf8", "scripts/check_contract.py"])
run(args, [interpreter(), "-X", "utf8", "scripts/validate_openapi.py"])
compose(args, "config", "--quiet")
def api_tests(args):
wait_ready(args)
run(args, [interpreter(), "-X", "utf8", "tests/smoke.py", "--base-url", base_url(args)])
os.environ["BASE_URL"] = base_url(args)
run(args, [interpreter(), "-X", "utf8", "-m", "unittest", "discover", "-s", "student-tests", "-v"])
def client(args):
sys.path.insert(0, str(ROOT / "tests"))
from client import Client
for key in ("NO_PROXY", "no_proxy"):
os.environ[key] = environment(args)[key]
return Client(base_url(args))
def database_stats(args):
sql = """SELECT json_build_object(
'users',(SELECT count(*) FROM users),'chats',(SELECT count(*) FROM chats),
'members',(SELECT count(*) FROM chat_members),'messages',(SELECT count(*) FROM messages),
'sessions',(SELECT count(*) FROM sessions),'migrations',(SELECT count(*) FROM schema_migrations),
'idempotency_keys',(SELECT count(*) FROM idempotency_keys),
'expiry_fingerprint',(SELECT md5(COALESCE(string_agg(user_id::text || created_at::text || expires_at::text,
',' ORDER BY token_hash),'')) FROM sessions),
'passwords_are_argon2id',(SELECT bool_and(password_hash LIKE '$argon2id$%') FROM users),
'session_hashes_only',(SELECT bool_and(token_hash ~ '^[0-9a-f]{64}$') FROM sessions))"""
result = compose(args, "exec", "-T", "db", "psql", "-U", "messenger", "-d", "messenger",
"-At", "-c", sql, quiet=True)
return json.loads(result)
def restart_check(args):
from urllib.parse import urlencode
c = client(args)
alice, bob, eve, chat = c.bootstrap()
path = "/api/v1/chats/" + chat["id"] + "/messages"
replay_key = secrets.token_hex(16)
original = c.message(alice, chat, "first", key=replay_key)["json"]
last = c.message(bob, chat, "second")["json"]
page = c.call("GET", path + "?limit=1", actor=alice, schema="MessagePage")["json"]
revoked = c.user("revoked")
c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=revoked)
hook = ROOT / "scripts" / ("restart.cmd" if os.name == "nt" else "restart.sh")
run(args, [interpreter(), "-X", "utf8", "tests/resilience.py", "--mode", "persistence",
"--action-script", hook, "--base-url", base_url(args)])
continued = c.call("GET", path + "?" + urlencode({"cursor": page["next_cursor"]}),
actor=alice, schema="MessagePage")["json"]
if continued != {"items": [last], "next_cursor": None}:
raise RuntimeError("Old cursor did not survive restart")
c.call("GET", "/api/v1/users/me", 401, actor=revoked)
c.call("GET", "/api/v1/chats/" + chat["id"], 404, actor=eve)
before = database_stats(args)
compose(args, "run", "--rm", "--no-deps", "migrate")
compose(args, "run", "--rm", "--no-deps", "migrate")
after = database_stats(args)
if before != after or after["migrations"] != 2:
raise RuntimeError("Migration replay changed existing data")
if not after["passwords_are_argon2id"] or not after["session_hashes_only"]:
raise RuntimeError("Unexpected credential representation in DB")
repeated = c.message(alice, chat, "first", key=replay_key)["json"]
if repeated != original:
raise RuntimeError("Idempotency result was not preserved")
result = {"idempotency_after_restart": "PASS", "public_persistence": "PASS", "cursor_after_restart": "PASS",
"logout_after_restart": "PASS", "migration_replay": "PASS", "database": after}
evidence("persistence.json", result)
say("PASS: old session/cursor, durable logout and repeatable migrations")
return result
def ttl_check(args):
test = argparse.Namespace(project=args.project + "-ttl", port=args.test_port, ttl=2,
test_port=args.test_port, action="start")
try:
start(test)
c = client(test)
actor = c.user("expiry")
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
before = database_stats(test)["expiry_fingerprint"]
compose(test, "restart", "--timeout", "3", "api1", "api2")
wait_ready(test)
after = database_stats(test)["expiry_fingerprint"]
if before != after:
raise RuntimeError("Session expiry changed during restart")
time.sleep(2.2)
c.call("GET", "/api/v1/users/me", 401, actor=actor)
registration = actor["registration"]
import base64
basic = base64.b64encode((registration["username"] + ":" + registration["password"]).encode()).decode()
c.call("POST", "/api/v1/auth/sessions", 201, headers={"Authorization": "Basic " + basic}, schema="Session")
say("PASS: TTL=2, expired token rejected after API restart; fresh login works")
result = {"ttl_seconds": 2, "expiry_unchanged_on_restart": True, "before_restart": 200,
"after_expiry_and_restart": 401, "new_login": 201}
evidence("expiry.json", result)
return result
finally:
compose(test, "down", "--timeout", "10")
def dependency_check(args):
c = client(args)
actor = c.user("dbcheck")
try:
compose(args, "stop", "--timeout", "10", "db")
c.call("GET", "/health/live", schema="Health")
outage_response(args, "/health/ready")
outage_response(args, "/api/v1/users/me", actor["token"])
finally:
compose(args, "start", "db")
wait_ready(args)
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
say("PASS: DB outage -> live=200, ready/business=503; session works after recovery")
return {"live_without_db": 200, "ready_without_db": 503, "business_without_db": 503, "recovered": 200}
def verify(args):
summary = {"date_utc": datetime.now(timezone.utc).isoformat(), "platform": platform.platform(),
"source_commit": run(args, ["git", "rev-parse", "HEAD"], quiet=True).strip(),
"worktree_dirty_at_start": bool(run(args, ["git", "status", "--porcelain"], quiet=True).strip()),
"project": args.project, "status": "FAIL"}
try:
setup(args)
contract_tests(args)
build(args)
start(args)
api_tests(args)
summary["persistence"] = restart_check(args)
summary["failover"] = failover_check(args)
summary["expiry"] = ttl_check(args)
summary["dependency"] = dependency_check(args)
run(args, [interpreter(), "-X", "utf8", "tests/smoke.py", "--base-url", base_url(args)])
summary["containers"] = states(args)
summary["docker"] = run(args, ["docker", "version", "--format", "{{.Server.Version}}"], quiet=True).strip()
summary["resources"] = json.loads(run(
args, ["docker", "info", "--format", '{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}'], quiet=True))
summary["status"] = "PASS"
say("PASS: full lab3 verification")
finally:
evidence("verification.txt", "\n".join(LOG))
evidence("summary.json", summary)
def instance_map(args):
current = states(args)
result = {}
for service in ("api1", "api2"):
item = current[service]
if not item["running"]:
continue
code = (
"import urllib.request; "
"o=urllib.request.build_opener(urllib.request.ProxyHandler({})); "
"r=o.open('http://127.0.0.1:8080/health/live',timeout=1); "
"print(r.headers['X-Instance-Id'])"
)
alias = run(args, ["docker", "exec", item["id"], "python", "-c", code], quiet=True).strip()
import re
if not re.fullmatch(r"[0-9a-f]{32}", alias):
raise RuntimeError("Invalid API instance alias")
result[service] = {"instance_id": alias, "container_id": item["id"]}
return result
def wait_replicas(args):
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
try:
mapping = instance_map(args)
if len(mapping) == 2:
c = client(args)
seen = {c.call("GET", "/health/ready", schema="Health")["headers"]["X-Instance-Id"]
for _ in range(8)}
if seen == {item["instance_id"] for item in mapping.values()}:
return mapping
except (RuntimeError, OSError, AssertionError):
pass
time.sleep(0.3)
raise RuntimeError("Two healthy API replicas were not observed through LB")
def restore(args):
compose(args, "start", "api1", "api2")
wait_ready(args)
return wait_replicas(args)
def stop_one(args):
target = os.environ.get("TARGET_INSTANCE_ID", "")
before = instance_map(args)
matches = [(name, item) for name, item in before.items() if item["instance_id"] == target]
if len(before) != 2 or len(matches) != 1:
raise RuntimeError("Target must match exactly one of two running API replicas")
service, item = matches[0]
started = time.monotonic()
run(args, ["docker", "stop", "--timeout", "3", item["container_id"]])
after = states(args)
if after[service]["running"] or not after["api1" if service == "api2" else "api2"]["running"]:
raise RuntimeError("Unexpected API state after stop")
elapsed = time.monotonic() - started
if elapsed >= 10:
raise RuntimeError("Stopping the selected API exceeded 10 seconds")
evidence("stop-one.json", {"service": service, "target_instance": target, "before": before,
"after": after, "stop_seconds": round(elapsed, 3)})
say("Stopped only " + service)
def failover_check(args):
wait_replicas(args)
hook = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh")
try:
output = run(args, [interpreter(), "-X", "utf8", "tests/resilience.py", "--mode", "failover",
"--action-script", hook, "--base-url", base_url(args)])
first = json.loads((ROOT / "evidence/stop-one.json").read_text(encoding="utf-8"))
first["metrics"] = next(json.loads(line) for line in output.splitlines()
if line.startswith("{") and "first_recovery_seconds" in line)
if first["after"][first["service"]]["running"]:
raise RuntimeError("Public hook did not leave the target stopped")
finally:
restore(args)
other_service = "api2" if first["service"] == "api1" else "api1"
mapping = instance_map(args)
target = mapping[other_service]["instance_id"]
c = client(args)
c.timeout = 2
alice, bob, eve, chat = c.bootstrap()
for _ in range(10):
replay_key = secrets.token_hex(16)
replay_text = "before-failure-" + replay_key
response = c.message(alice, chat, replay_text, key=replay_key)
if response["headers"]["X-Instance-Id"] == target:
original = response["json"]
break
else:
raise RuntimeError("Second API was not reached through LB")
env = environment(args)
env["TARGET_INSTANCE_ID"] = target
started = time.monotonic()
process = subprocess.Popen([str(hook)], cwd=ROOT, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
from client import Failure
from resilience import history
confirmations, after_instances = [], []
failures = 0
recovered = None
key = secrets.token_hex(16)
text = "during-failure-" + key
try:
while time.monotonic() - started < 15:
code = process.poll()
if code not in (None, 0):
raise RuntimeError("Second stop hook failed")
if code is None and time.monotonic() - started >= 10:
raise RuntimeError("Second stop hook exceeded 10 seconds")
try:
response = c.message(alice, chat, text, key=key)
confirmations.append(response["json"])
if code == 0:
if recovered is None:
recovered = time.monotonic() - started
after_instances.append(response["headers"]["X-Instance-Id"])
key = secrets.token_hex(16)
text = "during-failure-" + key
except Failure:
failures += 1
time.sleep(0.1)
if process.poll() != 0 or recovered is None or recovered > 10:
raise RuntimeError("Second API failure did not recover within 10 seconds")
if len(after_instances) < 5 or target in after_instances[-5:]:
raise RuntimeError("Stopped API still serves requests")
replayed = c.message(alice, chat, replay_text, key=replay_key)
if replayed["json"] != original or replayed["headers"]["X-Instance-Id"] == target:
raise RuntimeError("Message replay through the survivor changed the result")
messages = history(c, bob, chat)
ids = [item["id"] for item in messages]
texts = [item["text"] for item in messages]
if len(ids) != len(set(ids)) or len(texts) != len(set(texts)):
raise RuntimeError("Duplicates appeared during failover")
if any(item not in messages for item in [original, *confirmations]):
raise RuntimeError("A confirmed message was lost")
second = json.loads((ROOT / "evidence/stop-one.json").read_text(encoding="utf-8"))
if second["service"] != other_service:
raise RuntimeError("Wrong second API was stopped")
second["metrics"] = {"successful_writes": len(confirmations), "failed_attempts": failures,
"first_recovery_seconds": round(recovered, 3),
"replay_on_survivor": "PASS"}
result = {"public": first, "second_replica": second, "status": "PASS"}
evidence("failover.json", result)
say("PASS: both physical API replicas stopped separately; no lost messages or duplicates")
return result
finally:
if process.poll() is None:
process.terminate()
process.wait(timeout=5)
restore(args)
def outage_response(args, path, token=None):
from urllib.error import HTTPError
from urllib.request import Request
from client import shape
headers = {"Authorization": "Bearer " + token} if token else {}
try:
response = LOCAL_HTTP.open(Request(base_url(args) + path, headers=headers), timeout=10)
except HTTPError as error:
response = error
with response:
payload = json.load(response)
if response.status != 503 or not response.headers.get("X-Request-Id"):
raise RuntimeError("Unexpected dependency failure response")
shape(payload, "Error")
if payload["error"]["request_id"] != response.headers["X-Request-Id"]:
raise RuntimeError("LB error request ID mismatch")
def main():
saved = settings()
parser = argparse.ArgumentParser()
parser.add_argument("action", choices=["setup", "build", "start", "test", "restart", "stop-one",
"restart-check", "failover-check", "restore", "stop", "verify"])
parser.add_argument("--project", default=os.environ.get("LAB3_PROJECT", saved.get("COMPOSE_PROJECT_NAME", "kuznetsov-lab3")))
parser.add_argument("--port", type=int, default=int(os.environ.get("LAB3_PORT", saved.get("HTTP_PORT", "18103"))))
parser.add_argument("--ttl", type=int, default=int(os.environ.get("LAB3_TTL", saved.get("SESSION_TTL_SECONDS", "3600"))))
parser.add_argument("--test-port", type=int, default=18113)
args = parser.parse_args()
if args.project != "kuznetsov-lab3" and not args.project.startswith("kuznetsov-lab3-"):
parser.error("Use a dedicated Kuznetsov lab3 project")
if not 1 <= args.port <= 65535 or not 1 <= args.test_port <= 65535 or not 1 <= args.ttl <= 86400:
parser.error("Invalid port or session TTL")
if args.action in ("setup", "build", "start", "test", "restart-check", "failover-check"):
setup(args)
actions = {"setup": lambda item: None, "build": build, "start": start,
"test": lambda item: (contract_tests(item), api_tests(item)),
"restart": restart, "restart-check": restart_check, "verify": verify,
"stop-one": stop_one, "restore": restore, "failover-check": failover_check,
"stop": lambda item: compose(item, "stop", "--timeout", "10", "lb", "api1", "api2", "db")}
actions[args.action](args)
if __name__ == "__main__":
try:
main()
except (RuntimeError, OSError, ValueError) as error:
print("FAIL: " + str(error), file=sys.stderr)
raise SystemExit(1)
+3
View File
@@ -0,0 +1,3 @@
@echo off
python "%~dp0lab.py" restart
exit /b %errorlevel%
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
TASK_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
exec python3 "$TASK_DIR/lab.py" restart
+3
View File
@@ -0,0 +1,3 @@
@echo off
python "%~dp0lab.py" stop-one
exit /b %errorlevel%
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?missing TARGET_INSTANCE_ID}"
TASK_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
exec python3 "$TASK_DIR/lab.py" stop-one