forked from mtusi2026/mtusi2026lab3
636 lines
24 KiB
Python
636 lines
24 KiB
Python
#!/usr/bin/env python3
|
|
"""Проверки уровня 4: гонка, откат, EXPLAIN, TTL, logout и отказ Redis/PostgreSQL.
|
|
|
|
Стенд уже должен быть поднят: scripts/bootstrap и docker compose up.
|
|
Скрипт останавливает redis и postgres и снова их запускает.
|
|
Токены и пароли не печатаются и не пишутся в evidence.
|
|
"""
|
|
|
|
import asyncio
|
|
import base64
|
|
import json
|
|
import os
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
EVIDENCE = ROOT / "evidence"
|
|
sys.path.insert(0, str(ROOT / "src"))
|
|
|
|
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
|
|
|
|
|
class Failure(AssertionError):
|
|
pass
|
|
|
|
|
|
def check(condition, message):
|
|
if not condition:
|
|
raise Failure(message)
|
|
|
|
|
|
def host_secret(name):
|
|
path = ROOT / "secrets" / name
|
|
if not path.is_file():
|
|
raise Failure("missing secrets/%s; run scripts/bootstrap first" % name)
|
|
value = path.read_text(encoding="utf-8").strip()
|
|
if not value:
|
|
raise Failure("empty secrets/%s" % name)
|
|
return value
|
|
|
|
|
|
def call(method, path, expected, body=None, token=None, authorization=None, user_id=None, timeout=15, headers=None):
|
|
extra = headers or {}
|
|
headers = {"Accept": "application/json"}
|
|
headers.update(extra)
|
|
data = None
|
|
if body is not None:
|
|
data = json.dumps(body, ensure_ascii=False).encode("utf-8")
|
|
headers["Content-Type"] = "application/json"
|
|
if authorization:
|
|
headers["Authorization"] = authorization
|
|
elif token:
|
|
headers["Authorization"] = "Bearer " + token
|
|
if user_id:
|
|
headers["X-User-Id"] = user_id
|
|
request = urllib.request.Request(BASE + path, data=data, headers=headers, method=method)
|
|
try:
|
|
response = urllib.request.urlopen(request, timeout=timeout)
|
|
except urllib.error.HTTPError as error:
|
|
response = error
|
|
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
|
raise Failure(method + " " + path.split("?")[0] + " failed: " + type(exc).__name__) from None
|
|
with response:
|
|
status = response.status
|
|
payload = response.read()
|
|
header_id = response.headers.get("X-Request-Id")
|
|
challenge = response.headers.get("WWW-Authenticate", "")
|
|
allowed = [expected] if isinstance(expected, int) else list(expected)
|
|
check(status in allowed, "%s %s: expected %s, got %s" % (method, path.split("?")[0], allowed, status))
|
|
check(header_id and header_id.strip(), "missing X-Request-Id")
|
|
parsed = None
|
|
if payload:
|
|
try:
|
|
parsed = json.loads(payload)
|
|
except json.JSONDecodeError as exc:
|
|
raise Failure("response is not JSON") from exc
|
|
if status >= 400:
|
|
check(isinstance(parsed, dict) and set(parsed) == {"error"}, "error envelope")
|
|
error = parsed["error"]
|
|
check(set(error) == {"code", "message", "request_id"}, "error fields")
|
|
check(error["request_id"] == header_id, "request_id differs from header")
|
|
lowered = error["message"].lower()
|
|
for leaked in ("traceback", "select ", "password", "syntax", "asyncpg", "sqlalchemy"):
|
|
check(leaked not in lowered, "error message leaked internals")
|
|
if status == 204:
|
|
check(not payload, "204 must not contain a body")
|
|
return {"status": status, "json": parsed, "www": challenge}
|
|
|
|
|
|
def basic(username, password):
|
|
raw = base64.b64encode(("%s:%s" % (username, password)).encode("utf-8")).decode("ascii")
|
|
return "Basic " + raw
|
|
|
|
|
|
def register(label, password=None):
|
|
username = ("u_%s_%s" % (label, uuid.uuid4().hex[:10])).lower()
|
|
password = password or ("T3st-" + uuid.uuid4().hex)
|
|
created = call(
|
|
"POST",
|
|
"/api/v1/users",
|
|
201,
|
|
body={"username": username, "display_name": "Студент " + label, "password": password},
|
|
)["json"]
|
|
check("password" not in created, "password leaked")
|
|
session = call("POST", "/api/v1/auth/sessions", 201, authorization=basic(username, password))["json"]
|
|
check(session["token_type"] == "Bearer" and session["user"]["id"] == created["id"], "session shape")
|
|
check(len(session["token"]) >= 32, "token is too short")
|
|
return {"id": created["id"], "username": username, "password": password, "token": session["token"]}
|
|
|
|
|
|
def compose(*args, timeout=120):
|
|
proc = subprocess.run(
|
|
["docker", "compose", *args],
|
|
cwd=str(ROOT),
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
)
|
|
if proc.returncode != 0:
|
|
raise Failure("docker compose %s failed with code %s" % (args[0], proc.returncode))
|
|
return proc.stdout
|
|
|
|
|
|
def psql(sql, timeout=60):
|
|
proc = subprocess.run(
|
|
[
|
|
"docker",
|
|
"compose",
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"psql",
|
|
"-U",
|
|
"messenger",
|
|
"-d",
|
|
"messenger",
|
|
"-v",
|
|
"ON_ERROR_STOP=1",
|
|
"-At",
|
|
"-c",
|
|
sql,
|
|
],
|
|
cwd=str(ROOT),
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
)
|
|
if proc.returncode != 0:
|
|
raise Failure("psql failed")
|
|
return proc.stdout.strip()
|
|
|
|
|
|
def redis_cli(*args):
|
|
proc = subprocess.run(
|
|
["docker", "compose", "exec", "-T", "redis", "redis-cli", *args],
|
|
cwd=str(ROOT),
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=20,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
)
|
|
if proc.returncode != 0:
|
|
raise Failure("redis-cli failed")
|
|
return proc.stdout.strip()
|
|
|
|
|
|
def sql_quote(value):
|
|
text = str(value)
|
|
if "'" in text or ";" in text or "\n" in text:
|
|
raise Failure("refusing unsafe SQL literal")
|
|
return "'" + text + "'"
|
|
|
|
|
|
def free_port():
|
|
with socket.socket() as sock:
|
|
sock.bind(("127.0.0.1", 0))
|
|
return sock.getsockname()[1]
|
|
|
|
|
|
def wait_ready(timeout=90):
|
|
deadline = time.monotonic() + timeout
|
|
last = ""
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
call("GET", "/health/ready", 200, timeout=5)
|
|
return
|
|
except Failure as exc:
|
|
last = str(exc)
|
|
time.sleep(0.5)
|
|
raise Failure("API did not become ready: " + last)
|
|
|
|
|
|
def test_credentials():
|
|
user = register("cred")
|
|
wrong = call(
|
|
"POST",
|
|
"/api/v1/auth/sessions",
|
|
401,
|
|
authorization=basic(user["username"], "wrong-password-value"),
|
|
)
|
|
missing = call(
|
|
"POST",
|
|
"/api/v1/auth/sessions",
|
|
401,
|
|
authorization=basic("no_such_user_zz", "wrong-password-value"),
|
|
)
|
|
check(wrong["json"]["error"]["code"] == "unauthorized", "wrong password code")
|
|
check(wrong["json"]["error"]["message"] == missing["json"]["error"]["message"], "credential errors differ")
|
|
check(wrong["www"].lower().startswith("basic "), "basic challenge")
|
|
bare = call("GET", "/api/v1/users/me", 401)
|
|
check(bare["www"].lower().startswith("bearer "), "bearer challenge")
|
|
only_header = call("GET", "/api/v1/users/me", 401, user_id=user["id"])
|
|
check(only_header["json"]["error"]["code"] == "unauthorized", "X-User-Id alone was accepted")
|
|
other = register("other")
|
|
me = call("GET", "/api/v1/users/me", 200, token=user["token"], user_id=other["id"])["json"]
|
|
check(me["id"] == user["id"], "X-User-Id overrode bearer")
|
|
basic_on_api = call("GET", "/api/v1/users/me", 401, authorization=basic(user["username"], user["password"]))
|
|
check(basic_on_api["www"].lower().startswith("bearer "), "basic accepted on business API")
|
|
print("PASS: credentials, same 401, X-User-Id ignored")
|
|
|
|
|
|
def test_unicode_password():
|
|
password = "Пароль:пример-12"
|
|
check(12 <= len(password) <= 128 and ":" in password, "fixture password")
|
|
user = register("uni", password=password)
|
|
again = call("POST", "/api/v1/auth/sessions", 201, authorization=basic(user["username"], password))["json"]
|
|
check(again["user"]["id"] == user["id"], "unicode password login failed")
|
|
print("PASS: unicode password with colon")
|
|
|
|
|
|
def test_logout():
|
|
user = register("bye")
|
|
call("DELETE", "/api/v1/auth/sessions/current", 204, token=user["token"])
|
|
rejected = call("GET", "/api/v1/users/me", 401, token=user["token"])
|
|
check(rejected["www"].lower().startswith("bearer "), "revoked token challenge")
|
|
again = call("DELETE", "/api/v1/auth/sessions/current", 401, token=user["token"])
|
|
check(again["json"]["error"]["code"] == "unauthorized", "second logout")
|
|
print("PASS: logout revokes the session")
|
|
|
|
|
|
def test_injection_and_rollback():
|
|
user = register("sql")
|
|
chat = call(
|
|
"POST",
|
|
"/api/v1/chats",
|
|
201,
|
|
token=user["token"],
|
|
body={"title": "SQL", "member_ids": [user["id"]]},
|
|
)["json"]
|
|
payload = "'; DROP TABLE messages; --"
|
|
sent = call(
|
|
"POST",
|
|
"/api/v1/chats/%s/messages" % chat["id"],
|
|
201,
|
|
token=user["token"],
|
|
body={"text": payload},
|
|
headers={"Idempotency-Key": uuid.uuid4().hex},
|
|
)["json"]
|
|
check(sent["text"] == payload, "text was rewritten")
|
|
check(psql("SELECT to_regclass('public.messages')") == "messages", "messages table disappeared")
|
|
listed = call("GET", "/api/v1/chats/%s/messages?limit=10" % chat["id"], 200, token=user["token"])["json"]
|
|
check(listed["items"][0]["text"] == payload, "stored text differs")
|
|
|
|
before = int(psql("SELECT count(*) FROM chats"))
|
|
title = "partial-" + uuid.uuid4().hex
|
|
rejected = call(
|
|
"POST",
|
|
"/api/v1/chats",
|
|
400,
|
|
token=user["token"],
|
|
body={"title": title, "member_ids": [user["id"], str(uuid.uuid4())]},
|
|
)
|
|
check(rejected["json"]["error"]["code"] == "invalid_request", "unknown member code")
|
|
after = int(psql("SELECT count(*) FROM chats"))
|
|
check(before == after, "unknown member committed a chat")
|
|
check(psql("SELECT count(*) FROM chats WHERE title = %s" % sql_quote(title)) == "0", "partial chat row exists")
|
|
print("PASS: parameterized text and unknown-member chat is not stored")
|
|
return user, chat
|
|
|
|
|
|
async def failpoint(owner_id, other_id, title):
|
|
os.environ["DATABASE_URL"] = host_secret("host_database_url")
|
|
os.environ["REDIS_URL"] = host_secret("host_cache_url")
|
|
from app.config import load_settings
|
|
from app.crud.chats import create_chat
|
|
from app.db import build_engine, build_session_factory
|
|
from app.runtime import configure
|
|
from app.sessions import build_redis
|
|
|
|
settings = load_settings()
|
|
engine = build_engine(settings)
|
|
redis = build_redis(settings)
|
|
configure(build_session_factory(engine), redis, settings)
|
|
try:
|
|
try:
|
|
await create_chat(
|
|
uuid.UUID(owner_id),
|
|
title,
|
|
(uuid.UUID(owner_id), uuid.UUID(other_id)),
|
|
fail_after_insert=True,
|
|
)
|
|
except RuntimeError as exc:
|
|
check(str(exc) == "rollback-check", "failpoint raised a different error")
|
|
else:
|
|
raise Failure("failpoint committed a chat")
|
|
finally:
|
|
await redis.aclose()
|
|
await engine.dispose()
|
|
|
|
|
|
def test_failpoint(owner_id, other_id):
|
|
title = "rollback-" + uuid.uuid4().hex
|
|
before = int(psql("SELECT count(*) FROM chats"))
|
|
try:
|
|
asyncio.run(failpoint(owner_id, other_id, title))
|
|
except Failure:
|
|
raise
|
|
except Exception as exc:
|
|
raise Failure("rollback check failed: " + type(exc).__name__) from None
|
|
after = int(psql("SELECT count(*) FROM chats"))
|
|
left = psql("SELECT count(*) FROM chats WHERE title = %s" % sql_quote(title))
|
|
check(before == after and left == "0", "chat insert survived the rollback")
|
|
print("PASS: composite chat insert rolls back before commit")
|
|
return {"chats_before": before, "chats_after": after, "title_rows": int(left)}
|
|
|
|
|
|
def test_race():
|
|
username = "race_" + uuid.uuid4().hex[:8]
|
|
password = "Race-password-" + uuid.uuid4().hex[:8]
|
|
body = {"username": username, "display_name": "Гонка", "password": password}
|
|
barrier = threading.Barrier(16)
|
|
statuses = []
|
|
lock = threading.Lock()
|
|
|
|
def worker():
|
|
try:
|
|
barrier.wait(20)
|
|
result = call("POST", "/api/v1/users", [201, 409], body=body, timeout=30)
|
|
with lock:
|
|
statuses.append(result["status"])
|
|
except Exception as exc:
|
|
with lock:
|
|
statuses.append(str(exc)[:120])
|
|
|
|
threads = [threading.Thread(target=worker) for _ in range(16)]
|
|
for thread in threads:
|
|
thread.start()
|
|
for thread in threads:
|
|
thread.join()
|
|
created = statuses.count(201)
|
|
conflict = statuses.count(409)
|
|
check(created == 1 and conflict == 15 and len(statuses) == 16, "race statuses %s" % statuses)
|
|
print("PASS: 16 parallel registrations, one 201 and fifteen 409")
|
|
return {"created": created, "conflict": conflict}
|
|
|
|
|
|
def assert_plan(name, plan):
|
|
check("ix_messages_history" in plan, name + " plan ignores ix_messages_history")
|
|
check("Seq Scan" not in plan, name + " plan seq-scans messages")
|
|
check("Index Scan" in plan or "Index Only Scan" in plan, name + " plan is not an index scan")
|
|
check("Buffers:" in plan, name + " plan has no buffer stats")
|
|
|
|
|
|
def test_explain(user):
|
|
chat = call(
|
|
"POST",
|
|
"/api/v1/chats",
|
|
201,
|
|
token=user["token"],
|
|
body={"title": "План", "member_ids": [user["id"]]},
|
|
)["json"]
|
|
chat_sql = sql_quote(chat["id"])
|
|
user_sql = sql_quote(user["id"])
|
|
psql(
|
|
"INSERT INTO messages (id, chat_id, sender_id, body, created_at) "
|
|
"SELECT gen_random_uuid(), %s, %s, 'row-' || g, "
|
|
"TIMESTAMPTZ '2026-01-01 00:00:00+00' + (g || ' microseconds')::interval "
|
|
"FROM generate_series(1, 10000) AS g" % (chat_sql, user_sql),
|
|
timeout=60,
|
|
)
|
|
count = int(psql("SELECT count(*) FROM messages WHERE chat_id = %s" % chat_sql))
|
|
check(count >= 10000, "expected at least 10000 messages, got %s" % count)
|
|
psql("ANALYZE messages")
|
|
midpoint = psql(
|
|
"SELECT to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US'), id::text "
|
|
"FROM messages WHERE chat_id = %s ORDER BY created_at, id OFFSET 5000 LIMIT 1" % chat_sql
|
|
)
|
|
created_at, message_id = midpoint.split("|")
|
|
late = psql(
|
|
"EXPLAIN (ANALYZE, BUFFERS) SELECT id, body FROM messages "
|
|
"WHERE chat_id = %s AND (created_at, id) > (%s::timestamptz, %s::uuid) "
|
|
"ORDER BY created_at, id LIMIT 50"
|
|
% (chat_sql, sql_quote(created_at + "+00"), sql_quote(message_id)),
|
|
timeout=60,
|
|
)
|
|
first = psql(
|
|
"EXPLAIN (ANALYZE, BUFFERS) SELECT id, body FROM messages "
|
|
"WHERE chat_id = %s ORDER BY created_at, id LIMIT 50" % chat_sql,
|
|
timeout=60,
|
|
)
|
|
EVIDENCE.mkdir(exist_ok=True)
|
|
(EVIDENCE / "explain_messages.txt").write_text(
|
|
"messages=%s\n\n-- first page\n%s\n\n-- page after row 5000\n%s\n" % (count, first, late),
|
|
encoding="utf-8",
|
|
)
|
|
assert_plan("first", first)
|
|
assert_plan("late", late)
|
|
print("PASS: EXPLAIN uses ix_messages_history on %s messages" % count)
|
|
return {"messages": count, "chat_id": chat["id"]}
|
|
|
|
|
|
def test_expiry_and_migrations():
|
|
from app.timeutil import parse_time
|
|
|
|
users_before = int(psql("SELECT count(*) FROM users"))
|
|
port = free_port()
|
|
env = os.environ.copy()
|
|
env["PYTHONPATH"] = str(ROOT / "src")
|
|
env["PYTHONUNBUFFERED"] = "1"
|
|
env["APP_HOST"] = "127.0.0.1"
|
|
env["APP_PORT"] = str(port)
|
|
env["APP_ENV"] = "lab"
|
|
env["DATABASE_URL"] = host_secret("host_database_url")
|
|
env["REDIS_URL"] = host_secret("host_cache_url")
|
|
env["SESSION_TTL_SECONDS"] = "2"
|
|
env.pop("DATABASE_URL_FILE", None)
|
|
env.pop("CACHE_URL_FILE", None)
|
|
proc = subprocess.Popen(
|
|
[sys.executable, "-m", "app"],
|
|
cwd=str(ROOT),
|
|
env=env,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
)
|
|
logs = []
|
|
|
|
def read_logs():
|
|
if proc.stdout is None:
|
|
return
|
|
for line in proc.stdout:
|
|
if "://" not in line and "password" not in line.lower():
|
|
logs.append(line)
|
|
|
|
reader = threading.Thread(target=read_logs, daemon=True)
|
|
reader.start()
|
|
base = "http://127.0.0.1:%s" % port
|
|
try:
|
|
deadline = time.monotonic() + 40
|
|
ready = False
|
|
while time.monotonic() < deadline:
|
|
if proc.poll() is not None:
|
|
raise Failure("short-ttl server exited: " + "".join(logs[-8:]))
|
|
try:
|
|
urllib.request.urlopen(base + "/health/ready", timeout=2).read()
|
|
ready = True
|
|
break
|
|
except (urllib.error.URLError, TimeoutError, OSError):
|
|
time.sleep(0.2)
|
|
check(ready, "short-ttl server did not become ready: " + "".join(logs[-8:]))
|
|
username = "ttl_" + uuid.uuid4().hex[:8]
|
|
password = "Ttl-password-" + uuid.uuid4().hex[:8]
|
|
body = json.dumps(
|
|
{"username": username, "display_name": "TTL", "password": password},
|
|
ensure_ascii=False,
|
|
).encode("utf-8")
|
|
request = urllib.request.Request(
|
|
base + "/api/v1/users",
|
|
data=body,
|
|
headers={"Content-Type": "application/json", "Accept": "application/json"},
|
|
method="POST",
|
|
)
|
|
with urllib.request.urlopen(request, timeout=20) as response:
|
|
created = json.loads(response.read())
|
|
login = urllib.request.Request(
|
|
base + "/api/v1/auth/sessions",
|
|
data=b"",
|
|
headers={"Authorization": basic(username, password), "Accept": "application/json"},
|
|
method="POST",
|
|
)
|
|
with urllib.request.urlopen(login, timeout=20) as response:
|
|
session = json.loads(response.read())
|
|
expires = parse_time(session["expires_at"])
|
|
delta = (expires - datetime.now(timezone.utc)).total_seconds()
|
|
check(0 < delta <= 3, "SESSION_TTL_SECONDS=2 was not applied")
|
|
time.sleep(delta + 1.0)
|
|
probe = urllib.request.Request(
|
|
base + "/api/v1/users/me",
|
|
headers={"Authorization": "Bearer " + session["token"], "Accept": "application/json"},
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(probe, timeout=10) as response:
|
|
status = response.status
|
|
except urllib.error.HTTPError as error:
|
|
status = error.status
|
|
check(status == 401, "expired session status %s" % status)
|
|
finally:
|
|
if proc.poll() is None:
|
|
proc.terminate()
|
|
try:
|
|
proc.wait(timeout=8)
|
|
except subprocess.TimeoutExpired:
|
|
proc.kill()
|
|
proc.wait(timeout=5)
|
|
reader.join(timeout=2)
|
|
users_after = int(psql("SELECT count(*) FROM users"))
|
|
versions = psql("SELECT count(*) FROM schema_migrations")
|
|
version_name = psql("SELECT version FROM schema_migrations ORDER BY version")
|
|
check(users_after == users_before + 1, "migrations duplicated or dropped users")
|
|
names = psql("SELECT version FROM schema_migrations ORDER BY version")
|
|
check(versions == "2", "expected two applied migrations, got %s" % versions)
|
|
check("001_initial.sql" in names and "002_idempotency.sql" in names, "migration set changed: %s" % names)
|
|
print("PASS: short TTL expires and migrations stay idempotent")
|
|
return {"users_before": users_before, "users_after": users_after, "migration": version_name}
|
|
|
|
|
|
def test_redis_persistence(user):
|
|
from app.sessions import session_key
|
|
|
|
before = int(redis_cli("PTTL", session_key(user["token"])))
|
|
check(before > 300000, "session ttl is unexpectedly short")
|
|
compose("restart", "redis")
|
|
wait_ready()
|
|
after = int(redis_cli("PTTL", session_key(user["token"])))
|
|
check(after > 0, "session disappeared after redis restart")
|
|
check(after <= before + 1000, "redis restart extended the session ttl")
|
|
me = call("GET", "/api/v1/users/me", 200, token=user["token"])["json"]
|
|
check(me["id"] == user["id"], "token did not survive redis restart")
|
|
print("PASS: Redis AOF kept the session and did not refresh TTL")
|
|
return {"pttl_before_ms": before, "pttl_after_ms": after}
|
|
|
|
|
|
def test_dependency_loss(user, chat_id):
|
|
observed = {}
|
|
compose("stop", "redis")
|
|
try:
|
|
live = call("GET", "/health/live", 200)
|
|
ready = call("GET", "/health/ready", 503)
|
|
anonymous = call("GET", "/api/v1/users/me", 401)
|
|
trusted = call("GET", "/api/v1/users/me", 503, token=user["token"])
|
|
check(live["json"]["status"] == "ok", "live failed while redis is down")
|
|
check(ready["json"]["error"]["code"] == "unavailable", "ready code")
|
|
check(anonymous["json"]["error"]["code"] == "unauthorized", "anonymous became trusted")
|
|
check(trusted["json"]["error"]["code"] == "unavailable", "unverified token was accepted")
|
|
observed["redis_down"] = {
|
|
"live": live["status"],
|
|
"ready": ready["status"],
|
|
"anonymous": anonymous["status"],
|
|
"bearer": trusted["status"],
|
|
}
|
|
finally:
|
|
compose("start", "redis")
|
|
wait_ready()
|
|
me = call("GET", "/api/v1/users/me", 200, token=user["token"])["json"]
|
|
check(me["id"] == user["id"], "token lost after redis came back")
|
|
|
|
compose("stop", "postgres")
|
|
try:
|
|
live = call("GET", "/health/live", 200)
|
|
ready = call("GET", "/health/ready", 503, timeout=20)
|
|
anonymous = call("GET", "/api/v1/users/me", 401, timeout=20)
|
|
denied = call("GET", "/api/v1/chats/" + chat_id, 503, token=user["token"], timeout=20)
|
|
check(live["json"]["status"] == "ok", "live depends on postgres")
|
|
check(ready["json"]["error"]["code"] == "unavailable", "ready while postgres is down")
|
|
check(anonymous["json"]["error"]["code"] == "unauthorized", "anonymous trusted without postgres")
|
|
check(denied["json"]["error"]["code"] == "unavailable", "request served without postgres")
|
|
observed["postgres_down"] = {
|
|
"live": live["status"],
|
|
"ready": ready["status"],
|
|
"anonymous": anonymous["status"],
|
|
"bearer": denied["status"],
|
|
}
|
|
finally:
|
|
compose("start", "postgres")
|
|
wait_ready(timeout=90)
|
|
history = call("GET", "/api/v1/chats/%s/messages?limit=5" % chat_id, 200, token=user["token"])["json"]
|
|
check(history["items"], "messages lost after postgres restart")
|
|
print("PASS: redis and postgres outages return 503 and do not trust anonymous clients")
|
|
return observed
|
|
|
|
|
|
def main():
|
|
os.environ["DATABASE_URL"] = host_secret("host_database_url")
|
|
os.environ["REDIS_URL"] = host_secret("host_cache_url")
|
|
wait_ready(timeout=60)
|
|
test_credentials()
|
|
test_unicode_password()
|
|
test_logout()
|
|
user, chat = test_injection_and_rollback()
|
|
other = register("pair")
|
|
rollback = test_failpoint(user["id"], other["id"])
|
|
race = test_race()
|
|
explain = test_explain(user)
|
|
expiry = test_expiry_and_migrations()
|
|
pttl = test_redis_persistence(user)
|
|
faults = test_dependency_loss(user, chat["id"])
|
|
remaining = int(psql("SELECT count(*) FROM messages WHERE chat_id = %s" % sql_quote(explain["chat_id"])))
|
|
check(remaining == explain["messages"], "bulk history lost after postgres restart")
|
|
EVIDENCE.mkdir(exist_ok=True)
|
|
report = {
|
|
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
"base_url": BASE,
|
|
"race": race,
|
|
"rollback": rollback,
|
|
"explain": {"messages": explain["messages"]},
|
|
"expiry_and_migrations": expiry,
|
|
"redis_restart": pttl,
|
|
"dependency_loss": faults,
|
|
"messages_after_postgres_restart": remaining,
|
|
}
|
|
(EVIDENCE / "level4.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
print("PASS: level 4 checks")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except Failure as exc:
|
|
print("FAIL:", exc)
|
|
sys.exit(1)
|