forked from mtusi2026/mtusi2026lab5
146 lines
6.7 KiB
Python
146 lines
6.7 KiB
Python
"""Пауза живой PostgreSQL: прямой readiness, ограниченное SQL I/O и восстановление."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timezone
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from uuid import uuid4
|
|
|
|
from helpers import Client, ROOT
|
|
from client import Failure
|
|
|
|
|
|
def check_postgres_pause(compose, apis, base):
|
|
client = Client(base)
|
|
alice, bob, _, chat = client.bootstrap()
|
|
saved_message = client.message(alice, chat, "До паузы PostgreSQL")["json"]
|
|
before = {info["Id"]:info["State"]["StartedAt"] for info in apis() if info["State"]["Running"]}
|
|
probe_id = sorted(before)[0]
|
|
|
|
def direct_probe(path, expected):
|
|
# HTTP напрямую в API, без HAProxy и его собственных таймаутов/503.
|
|
code = """
|
|
import json,time
|
|
from urllib.error import HTTPError
|
|
from urllib.request import urlopen
|
|
started=time.monotonic()
|
|
try: response=urlopen('http://127.0.0.1:8080/PATH',timeout=5)
|
|
except HTTPError as error: response=error
|
|
with response:
|
|
body=json.load(response)
|
|
elapsed=time.monotonic()-started
|
|
assert response.status==EXPECTED, response.status
|
|
assert response.headers['X-Request-Id'] and response.headers['X-Instance-Id']
|
|
if response.status==503:
|
|
assert body['error']['code']=='unavailable'
|
|
assert body['error']['request_id']==response.headers['X-Request-Id']
|
|
assert elapsed<4.5, elapsed
|
|
print(json.dumps({'status':response.status,'elapsed_seconds':round(elapsed,3)}))
|
|
""".replace("PATH", path).replace("EXPECTED", str(expected))
|
|
completed = subprocess.run(["docker", "exec", probe_id, "python", "-c", code],
|
|
capture_output=True, text=True, timeout=8, check=True)
|
|
return json.loads(completed.stdout)
|
|
|
|
# Соединение получено ДО pause: проверяем именно выполняющийся SQL, а не
|
|
# только timeout получения нового соединения или check callback пула.
|
|
query_code = """
|
|
import json,sys,time
|
|
import psycopg
|
|
from src import db
|
|
db.pool.open(wait=True,timeout=10)
|
|
conn=db.pool.getconn()
|
|
conn.execute('SET statement_timeout=0')
|
|
print('connected',flush=True)
|
|
sys.stdin.readline()
|
|
started=time.monotonic()
|
|
try:
|
|
try: conn.execute('SELECT 1')
|
|
except psycopg.OperationalError:
|
|
elapsed=time.monotonic()-started
|
|
assert elapsed<4.5, elapsed
|
|
assert conn.closed, 'timed out connection was not discarded'
|
|
print(json.dumps({'elapsed_seconds':round(elapsed,3),'connection_closed':conn.closed}),flush=True)
|
|
else: raise AssertionError('paused database answered a query')
|
|
finally:
|
|
db.pool.putconn(conn)
|
|
db.pool.close()
|
|
"""
|
|
query = subprocess.Popen(["docker", "exec", "-i", probe_id, "python", "-u", "-c", query_code],
|
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
result = {}
|
|
try:
|
|
assert query.stdout.readline().strip() == "connected"
|
|
compose("pause", "postgres")
|
|
paused_at = time.monotonic()
|
|
output, error = query.communicate("\n", timeout=10)
|
|
assert query.returncode == 0, "SQL I/O deadline failed: " + error
|
|
result["sql_without_server_timeout"] = json.loads(output)
|
|
result["direct_ready"] = direct_probe("health/ready", 503)
|
|
result["direct_live"] = direct_probe("health/live", 200)
|
|
# Повторные запросы при длительной паузе не должны оставлять занятые потоки.
|
|
with ThreadPoolExecutor(max_workers=10) as executor:
|
|
result["repeated_ready"] = list(executor.map(lambda _: direct_probe("health/ready", 503), range(10)))
|
|
# Сервер остаётся на pause не менее 12 секунд; ограничиваем именно API.
|
|
remaining = 12 - (time.monotonic() - paused_at)
|
|
if remaining > 0:
|
|
time.sleep(remaining)
|
|
result["ready_after_12_seconds"] = direct_probe("health/ready", 503)
|
|
result["live_after_repeated_requests"] = direct_probe("health/live", 200)
|
|
result["paused_seconds"] = round(time.monotonic() - paused_at, 3)
|
|
finally:
|
|
compose("unpause", "postgres")
|
|
if query.poll() is None:
|
|
query.communicate(timeout=10)
|
|
|
|
resumed_at = time.monotonic()
|
|
while True:
|
|
try:
|
|
result["recovered_ready"] = direct_probe("health/ready", 200)
|
|
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")
|
|
messages = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages", actor=bob, schema="MessagePage")["json"]["items"]
|
|
assert saved_message in messages
|
|
break
|
|
except (subprocess.CalledProcessError, Failure):
|
|
assert time.monotonic() - resumed_at < 30, "readiness did not recover after unpause"
|
|
time.sleep(.2)
|
|
assert before == {info["Id"]:info["State"]["StartedAt"] for info in apis() if info["State"]["Running"]}
|
|
result.update({"recovery_seconds":round(time.monotonic()-resumed_at,3), "api_not_restarted":True,
|
|
"existing_session_and_message_preserved":True, "passed":True})
|
|
print("PASS: paused PostgreSQL: SQL and direct readiness bounded; live works; unpause restores API without restart", flush=True)
|
|
return result
|
|
|
|
|
|
def main():
|
|
with socket.socket() as listener:
|
|
listener.bind(("127.0.0.1", 0))
|
|
port = listener.getsockname()[1]
|
|
project = "lab4-pg-pause-" + uuid4().hex[:10]
|
|
base = f"http://127.0.0.1:{port}"
|
|
env = dict(os.environ, COMPOSE_PROJECT_NAME=project, HOST_PORT=str(port), API_REPLICAS="2", BASE_URL=base)
|
|
|
|
def compose(*args):
|
|
return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, env=env, text=True, timeout=180).strip()
|
|
|
|
def apis():
|
|
ids = compose("ps", "-a", "-q", "api").split()
|
|
return json.loads(subprocess.check_output(["docker", "inspect", *ids], text=True))
|
|
|
|
result = {"utc":datetime.now(timezone.utc).isoformat(), "project":project,
|
|
"command":".venv/bin/python student-tests/postgres_pause.py", "passed":False}
|
|
try:
|
|
compose("up", "-d", "--no-build", "lb")
|
|
subprocess.run([str(ROOT / "scripts/wait-ready.sh")], cwd=ROOT, env=env, check=True, timeout=120)
|
|
result.update(check_postgres_pause(compose, apis, base))
|
|
print(json.dumps(result, ensure_ascii=False), flush=True)
|
|
finally:
|
|
Path(ROOT / "evidence/postgres-pause.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
|
compose("down", "--volumes", "--timeout", "8")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|