PostgreSQL как источник истины, непрозрачные сессии в Redis с TTL и AOF, миграции под advisory lock, пул и таймауты, проверки гонки/отката/отказов и отчёт с evidence.
103 lines
3.7 KiB
Python
103 lines
3.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Dependency-failure matrix: Redis and PostgreSQL outages must be bounded.
|
|
|
|
The check uses the local Compose project. It stops one dependency, verifies
|
|
503 (never 200 and never a trusted anonymous request), starts it again, waits
|
|
for readiness and confirms the original bearer session still works after the
|
|
Redis restart (AOF persistence).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import os
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from common import HttpClient, assert_status, error_code
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def compose(*args: str) -> None:
|
|
subprocess.run(
|
|
["docker", "compose", "--env-file", ".env", *args],
|
|
cwd=ROOT,
|
|
check=True,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
|
|
|
|
def wait_ready(client: HttpClient, expected: int, seconds: float = 90) -> None:
|
|
deadline = time.monotonic() + seconds
|
|
last = None
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
status, _, _, _ = client.call("GET", "/health/ready")
|
|
last = status
|
|
if status == expected:
|
|
return
|
|
except OSError:
|
|
last = "connection error"
|
|
time.sleep(0.5)
|
|
raise AssertionError(f"readiness did not become {expected}; last={last}")
|
|
|
|
|
|
def expect_503(client: HttpClient, path: str, token: str | None, label: str) -> None:
|
|
status, _headers, payload, _raw = client.call("GET", path, token=token)
|
|
if status != 503:
|
|
raise AssertionError(f"{label}: expected 503, received {status}")
|
|
if error_code(payload) != "unavailable":
|
|
raise AssertionError(f"{label}: expected unavailable error code")
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--base-url", default="http://localhost:8080")
|
|
args = parser.parse_args()
|
|
client = HttpClient(args.base_url)
|
|
user, registration = client.register()
|
|
session = client.login(registration)
|
|
token = session["token"]
|
|
|
|
results = []
|
|
try:
|
|
compose("stop", "redis")
|
|
expect_503(client, "/health/ready", None, "Redis down readiness")
|
|
expect_503(client, "/api/v1/users/me", token, "Redis down authenticated request")
|
|
results.append("redis down: readiness=503, bearer request=503")
|
|
started = time.monotonic()
|
|
compose("start", "redis")
|
|
wait_ready(client, 200)
|
|
redis_recovery = round(time.monotonic() - started, 2)
|
|
status, _, me, _ = client.call("GET", "/api/v1/users/me", token=token)
|
|
assert_status(status, 200, "session after Redis restart")
|
|
if me["id"] != user["id"]:
|
|
raise AssertionError("session identity changed after Redis restart")
|
|
results.append(f"redis up: readiness=200 in {redis_recovery}s, same token=200 (AOF persistence)")
|
|
|
|
compose("stop", "postgres")
|
|
time.sleep(1)
|
|
expect_503(client, "/health/ready", None, "PostgreSQL down readiness")
|
|
expect_503(client, "/api/v1/users/me", token, "PostgreSQL down authenticated request")
|
|
results.append("postgres down: readiness=503, bearer request=503")
|
|
started = time.monotonic()
|
|
compose("start", "postgres")
|
|
wait_ready(client, 200)
|
|
postgres_recovery = round(time.monotonic() - started, 2)
|
|
status, _, me, _ = client.call("GET", "/api/v1/users/me", token=token)
|
|
assert_status(status, 200, "session after PostgreSQL restart")
|
|
results.append(f"postgres up: readiness=200 in {postgres_recovery}s, same token=200")
|
|
finally:
|
|
compose("start", "redis")
|
|
compose("start", "postgres")
|
|
|
|
for line in results:
|
|
print("OBSERVED:", line)
|
|
print("PASS: dependency loss produced only bounded 503 responses; no anonymous trust")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|