#!/usr/bin/env python3 """Проверки уровня 4: гонка, откат, EXPLAIN, TTL, logout и отказ Redis/PostgreSQL. Стенд уже должен быть поднят: scripts/bootstrap и docker compose up. Скрипт останавливает redis и postgres и снова их запускает. Токены и пароли не печатаются и не пишутся в evidence. """ import asyncio import base64 import json import os import socket import subprocess import sys import threading import time import urllib.error import urllib.request import uuid from datetime import datetime, timezone from pathlib import Path ROOT = Path(__file__).resolve().parents[1] EVIDENCE = ROOT / "evidence" sys.path.insert(0, str(ROOT / "src")) BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") class Failure(AssertionError): pass def check(condition, message): if not condition: raise Failure(message) def host_secret(name): path = ROOT / "secrets" / name if not path.is_file(): raise Failure("missing secrets/%s; run scripts/bootstrap first" % name) value = path.read_text(encoding="utf-8").strip() if not value: raise Failure("empty secrets/%s" % name) return value def call(method, path, expected, body=None, token=None, authorization=None, user_id=None, timeout=15, headers=None): extra = headers or {} headers = {"Accept": "application/json"} headers.update(extra) data = None if body is not None: data = json.dumps(body, ensure_ascii=False).encode("utf-8") headers["Content-Type"] = "application/json" if authorization: headers["Authorization"] = authorization elif token: headers["Authorization"] = "Bearer " + token if user_id: headers["X-User-Id"] = user_id request = urllib.request.Request(BASE + path, data=data, headers=headers, method=method) try: response = urllib.request.urlopen(request, timeout=timeout) except urllib.error.HTTPError as error: response = error except (urllib.error.URLError, TimeoutError, OSError) as exc: raise Failure(method + " " + path.split("?")[0] + " failed: " + type(exc).__name__) from None with response: status = response.status payload = response.read() header_id = response.headers.get("X-Request-Id") challenge = response.headers.get("WWW-Authenticate", "") allowed = [expected] if isinstance(expected, int) else list(expected) check(status in allowed, "%s %s: expected %s, got %s" % (method, path.split("?")[0], allowed, status)) check(header_id and header_id.strip(), "missing X-Request-Id") parsed = None if payload: try: parsed = json.loads(payload) except json.JSONDecodeError as exc: raise Failure("response is not JSON") from exc if status >= 400: check(isinstance(parsed, dict) and set(parsed) == {"error"}, "error envelope") error = parsed["error"] check(set(error) == {"code", "message", "request_id"}, "error fields") check(error["request_id"] == header_id, "request_id differs from header") lowered = error["message"].lower() for leaked in ("traceback", "select ", "password", "syntax", "asyncpg", "sqlalchemy"): check(leaked not in lowered, "error message leaked internals") if status == 204: check(not payload, "204 must not contain a body") return {"status": status, "json": parsed, "www": challenge} def basic(username, password): raw = base64.b64encode(("%s:%s" % (username, password)).encode("utf-8")).decode("ascii") return "Basic " + raw def register(label, password=None): username = ("u_%s_%s" % (label, uuid.uuid4().hex[:10])).lower() password = password or ("T3st-" + uuid.uuid4().hex) created = call( "POST", "/api/v1/users", 201, body={"username": username, "display_name": "Студент " + label, "password": password}, )["json"] check("password" not in created, "password leaked") session = call("POST", "/api/v1/auth/sessions", 201, authorization=basic(username, password))["json"] check(session["token_type"] == "Bearer" and session["user"]["id"] == created["id"], "session shape") check(len(session["token"]) >= 32, "token is too short") return {"id": created["id"], "username": username, "password": password, "token": session["token"]} def compose(*args, timeout=120): proc = subprocess.run( ["docker", "compose", *args], cwd=str(ROOT), capture_output=True, text=True, timeout=timeout, encoding="utf-8", errors="replace", ) if proc.returncode != 0: raise Failure("docker compose %s failed with code %s" % (args[0], proc.returncode)) return proc.stdout def psql(sql, timeout=60): proc = subprocess.run( [ "docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-v", "ON_ERROR_STOP=1", "-At", "-c", sql, ], cwd=str(ROOT), capture_output=True, text=True, timeout=timeout, encoding="utf-8", errors="replace", ) if proc.returncode != 0: raise Failure("psql failed") return proc.stdout.strip() def redis_cli(*args): proc = subprocess.run( ["docker", "compose", "exec", "-T", "redis", "redis-cli", *args], cwd=str(ROOT), capture_output=True, text=True, timeout=20, encoding="utf-8", errors="replace", ) if proc.returncode != 0: raise Failure("redis-cli failed") return proc.stdout.strip() def sql_quote(value): text = str(value) if "'" in text or ";" in text or "\n" in text: raise Failure("refusing unsafe SQL literal") return "'" + text + "'" def free_port(): with socket.socket() as sock: sock.bind(("127.0.0.1", 0)) return sock.getsockname()[1] def wait_ready(timeout=90): deadline = time.monotonic() + timeout last = "" while time.monotonic() < deadline: try: call("GET", "/health/ready", 200, timeout=5) return except Failure as exc: last = str(exc) time.sleep(0.5) raise Failure("API did not become ready: " + last) def test_credentials(): user = register("cred") wrong = call( "POST", "/api/v1/auth/sessions", 401, authorization=basic(user["username"], "wrong-password-value"), ) missing = call( "POST", "/api/v1/auth/sessions", 401, authorization=basic("no_such_user_zz", "wrong-password-value"), ) check(wrong["json"]["error"]["code"] == "unauthorized", "wrong password code") check(wrong["json"]["error"]["message"] == missing["json"]["error"]["message"], "credential errors differ") check(wrong["www"].lower().startswith("basic "), "basic challenge") bare = call("GET", "/api/v1/users/me", 401) check(bare["www"].lower().startswith("bearer "), "bearer challenge") only_header = call("GET", "/api/v1/users/me", 401, user_id=user["id"]) check(only_header["json"]["error"]["code"] == "unauthorized", "X-User-Id alone was accepted") other = register("other") me = call("GET", "/api/v1/users/me", 200, token=user["token"], user_id=other["id"])["json"] check(me["id"] == user["id"], "X-User-Id overrode bearer") basic_on_api = call("GET", "/api/v1/users/me", 401, authorization=basic(user["username"], user["password"])) check(basic_on_api["www"].lower().startswith("bearer "), "basic accepted on business API") print("PASS: credentials, same 401, X-User-Id ignored") def test_unicode_password(): password = "Пароль:пример-12" check(12 <= len(password) <= 128 and ":" in password, "fixture password") user = register("uni", password=password) again = call("POST", "/api/v1/auth/sessions", 201, authorization=basic(user["username"], password))["json"] check(again["user"]["id"] == user["id"], "unicode password login failed") print("PASS: unicode password with colon") def test_logout(): user = register("bye") call("DELETE", "/api/v1/auth/sessions/current", 204, token=user["token"]) rejected = call("GET", "/api/v1/users/me", 401, token=user["token"]) check(rejected["www"].lower().startswith("bearer "), "revoked token challenge") again = call("DELETE", "/api/v1/auth/sessions/current", 401, token=user["token"]) check(again["json"]["error"]["code"] == "unauthorized", "second logout") print("PASS: logout revokes the session") def test_injection_and_rollback(): user = register("sql") chat = call( "POST", "/api/v1/chats", 201, token=user["token"], body={"title": "SQL", "member_ids": [user["id"]]}, )["json"] payload = "'; DROP TABLE messages; --" sent = call( "POST", "/api/v1/chats/%s/messages" % chat["id"], 201, token=user["token"], body={"text": payload}, headers={"Idempotency-Key": uuid.uuid4().hex}, )["json"] check(sent["text"] == payload, "text was rewritten") check(psql("SELECT to_regclass('public.messages')") == "messages", "messages table disappeared") listed = call("GET", "/api/v1/chats/%s/messages?limit=10" % chat["id"], 200, token=user["token"])["json"] check(listed["items"][0]["text"] == payload, "stored text differs") before = int(psql("SELECT count(*) FROM chats")) title = "partial-" + uuid.uuid4().hex rejected = call( "POST", "/api/v1/chats", 400, token=user["token"], body={"title": title, "member_ids": [user["id"], str(uuid.uuid4())]}, ) check(rejected["json"]["error"]["code"] == "invalid_request", "unknown member code") after = int(psql("SELECT count(*) FROM chats")) check(before == after, "unknown member committed a chat") check(psql("SELECT count(*) FROM chats WHERE title = %s" % sql_quote(title)) == "0", "partial chat row exists") print("PASS: parameterized text and unknown-member chat is not stored") return user, chat async def failpoint(owner_id, other_id, title): os.environ["DATABASE_URL"] = host_secret("host_database_url") os.environ["REDIS_URL"] = host_secret("host_cache_url") from app.config import load_settings from app.crud.chats import create_chat from app.db import build_engine, build_session_factory from app.runtime import configure from app.sessions import build_redis settings = load_settings() engine = build_engine(settings) redis = build_redis(settings) configure(build_session_factory(engine), redis, settings) try: try: await create_chat( uuid.UUID(owner_id), title, (uuid.UUID(owner_id), uuid.UUID(other_id)), fail_after_insert=True, ) except RuntimeError as exc: check(str(exc) == "rollback-check", "failpoint raised a different error") else: raise Failure("failpoint committed a chat") finally: await redis.aclose() await engine.dispose() def test_failpoint(owner_id, other_id): title = "rollback-" + uuid.uuid4().hex before = int(psql("SELECT count(*) FROM chats")) try: asyncio.run(failpoint(owner_id, other_id, title)) except Failure: raise except Exception as exc: raise Failure("rollback check failed: " + type(exc).__name__) from None after = int(psql("SELECT count(*) FROM chats")) left = psql("SELECT count(*) FROM chats WHERE title = %s" % sql_quote(title)) check(before == after and left == "0", "chat insert survived the rollback") print("PASS: composite chat insert rolls back before commit") return {"chats_before": before, "chats_after": after, "title_rows": int(left)} def test_race(): username = "race_" + uuid.uuid4().hex[:8] password = "Race-password-" + uuid.uuid4().hex[:8] body = {"username": username, "display_name": "Гонка", "password": password} barrier = threading.Barrier(16) statuses = [] lock = threading.Lock() def worker(): try: barrier.wait(20) result = call("POST", "/api/v1/users", [201, 409], body=body, timeout=30) with lock: statuses.append(result["status"]) except Exception as exc: with lock: statuses.append(str(exc)[:120]) threads = [threading.Thread(target=worker) for _ in range(16)] for thread in threads: thread.start() for thread in threads: thread.join() created = statuses.count(201) conflict = statuses.count(409) check(created == 1 and conflict == 15 and len(statuses) == 16, "race statuses %s" % statuses) print("PASS: 16 parallel registrations, one 201 and fifteen 409") return {"created": created, "conflict": conflict} def assert_plan(name, plan): check("ix_messages_history" in plan, name + " plan ignores ix_messages_history") check("Seq Scan" not in plan, name + " plan seq-scans messages") check("Index Scan" in plan or "Index Only Scan" in plan, name + " plan is not an index scan") check("Buffers:" in plan, name + " plan has no buffer stats") def test_explain(user): chat = call( "POST", "/api/v1/chats", 201, token=user["token"], body={"title": "План", "member_ids": [user["id"]]}, )["json"] chat_sql = sql_quote(chat["id"]) user_sql = sql_quote(user["id"]) psql( "INSERT INTO messages (id, chat_id, sender_id, body, created_at) " "SELECT gen_random_uuid(), %s, %s, 'row-' || g, " "TIMESTAMPTZ '2026-01-01 00:00:00+00' + (g || ' microseconds')::interval " "FROM generate_series(1, 10000) AS g" % (chat_sql, user_sql), timeout=60, ) count = int(psql("SELECT count(*) FROM messages WHERE chat_id = %s" % chat_sql)) check(count >= 10000, "expected at least 10000 messages, got %s" % count) psql("ANALYZE messages") midpoint = psql( "SELECT to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US'), id::text " "FROM messages WHERE chat_id = %s ORDER BY created_at, id OFFSET 5000 LIMIT 1" % chat_sql ) created_at, message_id = midpoint.split("|") late = psql( "EXPLAIN (ANALYZE, BUFFERS) SELECT id, body FROM messages " "WHERE chat_id = %s AND (created_at, id) > (%s::timestamptz, %s::uuid) " "ORDER BY created_at, id LIMIT 50" % (chat_sql, sql_quote(created_at + "+00"), sql_quote(message_id)), timeout=60, ) first = psql( "EXPLAIN (ANALYZE, BUFFERS) SELECT id, body FROM messages " "WHERE chat_id = %s ORDER BY created_at, id LIMIT 50" % chat_sql, timeout=60, ) EVIDENCE.mkdir(exist_ok=True) (EVIDENCE / "explain_messages.txt").write_text( "messages=%s\n\n-- first page\n%s\n\n-- page after row 5000\n%s\n" % (count, first, late), encoding="utf-8", ) assert_plan("first", first) assert_plan("late", late) print("PASS: EXPLAIN uses ix_messages_history on %s messages" % count) return {"messages": count, "chat_id": chat["id"]} def test_expiry_and_migrations(): from app.timeutil import parse_time users_before = int(psql("SELECT count(*) FROM users")) port = free_port() env = os.environ.copy() env["PYTHONPATH"] = str(ROOT / "src") env["PYTHONUNBUFFERED"] = "1" env["APP_HOST"] = "127.0.0.1" env["APP_PORT"] = str(port) env["APP_ENV"] = "lab" env["DATABASE_URL"] = host_secret("host_database_url") env["REDIS_URL"] = host_secret("host_cache_url") env["SESSION_TTL_SECONDS"] = "2" env.pop("DATABASE_URL_FILE", None) env.pop("CACHE_URL_FILE", None) proc = subprocess.Popen( [sys.executable, "-m", "app"], cwd=str(ROOT), env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace", ) logs = [] def read_logs(): if proc.stdout is None: return for line in proc.stdout: if "://" not in line and "password" not in line.lower(): logs.append(line) reader = threading.Thread(target=read_logs, daemon=True) reader.start() base = "http://127.0.0.1:%s" % port try: deadline = time.monotonic() + 40 ready = False while time.monotonic() < deadline: if proc.poll() is not None: raise Failure("short-ttl server exited: " + "".join(logs[-8:])) try: urllib.request.urlopen(base + "/health/ready", timeout=2).read() ready = True break except (urllib.error.URLError, TimeoutError, OSError): time.sleep(0.2) check(ready, "short-ttl server did not become ready: " + "".join(logs[-8:])) username = "ttl_" + uuid.uuid4().hex[:8] password = "Ttl-password-" + uuid.uuid4().hex[:8] body = json.dumps( {"username": username, "display_name": "TTL", "password": password}, ensure_ascii=False, ).encode("utf-8") request = urllib.request.Request( base + "/api/v1/users", data=body, headers={"Content-Type": "application/json", "Accept": "application/json"}, method="POST", ) with urllib.request.urlopen(request, timeout=20) as response: created = json.loads(response.read()) login = urllib.request.Request( base + "/api/v1/auth/sessions", data=b"", headers={"Authorization": basic(username, password), "Accept": "application/json"}, method="POST", ) with urllib.request.urlopen(login, timeout=20) as response: session = json.loads(response.read()) expires = parse_time(session["expires_at"]) delta = (expires - datetime.now(timezone.utc)).total_seconds() check(0 < delta <= 3, "SESSION_TTL_SECONDS=2 was not applied") time.sleep(delta + 1.0) probe = urllib.request.Request( base + "/api/v1/users/me", headers={"Authorization": "Bearer " + session["token"], "Accept": "application/json"}, ) try: with urllib.request.urlopen(probe, timeout=10) as response: status = response.status except urllib.error.HTTPError as error: status = error.status check(status == 401, "expired session status %s" % status) finally: if proc.poll() is None: proc.terminate() try: proc.wait(timeout=8) except subprocess.TimeoutExpired: proc.kill() proc.wait(timeout=5) reader.join(timeout=2) users_after = int(psql("SELECT count(*) FROM users")) versions = psql("SELECT count(*) FROM schema_migrations") version_name = psql("SELECT version FROM schema_migrations ORDER BY version") check(users_after == users_before + 1, "migrations duplicated or dropped users") names = psql("SELECT version FROM schema_migrations ORDER BY version") check(versions == "2", "expected two applied migrations, got %s" % versions) check("001_initial.sql" in names and "002_idempotency.sql" in names, "migration set changed: %s" % names) print("PASS: short TTL expires and migrations stay idempotent") return {"users_before": users_before, "users_after": users_after, "migration": version_name} def test_redis_persistence(user): from app.sessions import session_key before = int(redis_cli("PTTL", session_key(user["token"]))) check(before > 300000, "session ttl is unexpectedly short") compose("restart", "redis") wait_ready() after = int(redis_cli("PTTL", session_key(user["token"]))) check(after > 0, "session disappeared after redis restart") check(after <= before + 1000, "redis restart extended the session ttl") me = call("GET", "/api/v1/users/me", 200, token=user["token"])["json"] check(me["id"] == user["id"], "token did not survive redis restart") print("PASS: Redis AOF kept the session and did not refresh TTL") return {"pttl_before_ms": before, "pttl_after_ms": after} def test_dependency_loss(user, chat_id): observed = {} compose("stop", "redis") try: live = call("GET", "/health/live", 200) ready = call("GET", "/health/ready", 503) anonymous = call("GET", "/api/v1/users/me", 401) trusted = call("GET", "/api/v1/users/me", 503, token=user["token"]) check(live["json"]["status"] == "ok", "live failed while redis is down") check(ready["json"]["error"]["code"] == "unavailable", "ready code") check(anonymous["json"]["error"]["code"] == "unauthorized", "anonymous became trusted") check(trusted["json"]["error"]["code"] == "unavailable", "unverified token was accepted") observed["redis_down"] = { "live": live["status"], "ready": ready["status"], "anonymous": anonymous["status"], "bearer": trusted["status"], } finally: compose("start", "redis") wait_ready() me = call("GET", "/api/v1/users/me", 200, token=user["token"])["json"] check(me["id"] == user["id"], "token lost after redis came back") compose("stop", "postgres") try: live = call("GET", "/health/live", 200) ready = call("GET", "/health/ready", 503, timeout=20) anonymous = call("GET", "/api/v1/users/me", 401, timeout=20) denied = call("GET", "/api/v1/chats/" + chat_id, 503, token=user["token"], timeout=20) check(live["json"]["status"] == "ok", "live depends on postgres") check(ready["json"]["error"]["code"] == "unavailable", "ready while postgres is down") check(anonymous["json"]["error"]["code"] == "unauthorized", "anonymous trusted without postgres") check(denied["json"]["error"]["code"] == "unavailable", "request served without postgres") observed["postgres_down"] = { "live": live["status"], "ready": ready["status"], "anonymous": anonymous["status"], "bearer": denied["status"], } finally: compose("start", "postgres") wait_ready(timeout=90) history = call("GET", "/api/v1/chats/%s/messages?limit=5" % chat_id, 200, token=user["token"])["json"] check(history["items"], "messages lost after postgres restart") print("PASS: redis and postgres outages return 503 and do not trust anonymous clients") return observed def main(): os.environ["DATABASE_URL"] = host_secret("host_database_url") os.environ["REDIS_URL"] = host_secret("host_cache_url") wait_ready(timeout=60) test_credentials() test_unicode_password() test_logout() user, chat = test_injection_and_rollback() other = register("pair") rollback = test_failpoint(user["id"], other["id"]) race = test_race() explain = test_explain(user) expiry = test_expiry_and_migrations() pttl = test_redis_persistence(user) faults = test_dependency_loss(user, chat["id"]) remaining = int(psql("SELECT count(*) FROM messages WHERE chat_id = %s" % sql_quote(explain["chat_id"]))) check(remaining == explain["messages"], "bulk history lost after postgres restart") EVIDENCE.mkdir(exist_ok=True) report = { "date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "base_url": BASE, "race": race, "rollback": rollback, "explain": {"messages": explain["messages"]}, "expiry_and_migrations": expiry, "redis_restart": pttl, "dependency_loss": faults, "messages_after_postgres_restart": remaining, } (EVIDENCE / "level4.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") print("PASS: level 4 checks") if __name__ == "__main__": try: main() except Failure as exc: print("FAIL:", exc) sys.exit(1)