Исправить таймаут PostgreSQL на стороне API

This commit is contained in:
2026-11-07 16:18:55 +03:00
parent f878147188
commit 5236f44412
12 changed files with 440 additions and 2 deletions
+208
View File
@@ -0,0 +1,208 @@
#!/usr/bin/env python3
"""Regression: pause PostgreSQL, direct API 503, discard connection and recover.
No published replica ports: direct HTTP is sent from inside each API container.
PostgreSQL is always unpaused in finally. Uses existing local synthetic stand.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor
import json
from pathlib import Path
import queue
import subprocess
import sys
import threading
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check
LOG = []
def record(value):
text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)
LOG.append(text)
print(text, flush=True)
def compose(*args, timeout=40):
proc = subprocess.run(['docker', 'compose', *args], cwd=ROOT, capture_output=True,
text=True, encoding='utf-8', errors='replace', timeout=timeout)
if proc.returncode:
raise RuntimeError('docker compose ' + ' '.join(args[:3]) + ': ' + proc.stderr[-1200:])
return proc.stdout.strip()
PROBE = r'''
from concurrent.futures import ThreadPoolExecutor
import http.client, json, sys, time
expected, count = map(int, sys.argv[1:])
def request(path, status):
c = http.client.HTTPConnection('127.0.0.1', 8080, timeout=8)
start = time.monotonic()
try:
c.request('GET', path)
response = c.getresponse()
body = json.loads(response.read())
assert response.status == status, (response.status, body)
assert response.getheader('X-Request-Id')
assert response.getheader('X-Instance-Id')
if status == 503:
assert body['error']['code'] == 'unavailable'
assert body['error']['request_id'] == response.getheader('X-Request-Id')
return round(time.monotonic() - start, 3)
finally:
c.close()
with ThreadPoolExecutor(max_workers=count) as executor:
futures = [executor.submit(request, '/health/ready', expected) for _ in range(count)]
live = request('/health/live', 200)
elapsed = [future.result() for future in futures]
print(json.dumps(dict(status=expected, requests=count, seconds=elapsed, live_seconds=live)))
'''
WORKER = r'''
import json, sys, time
from messenger.config import read_secret
from messenger.db import create_pool, Repository
from messenger.errors import Unavailable
class SyntheticRollback(Exception): pass
pool = create_pool(read_secret('DATABASE_URL'), 1)
pool.open(wait=True, timeout=10)
repo = Repository(pool)
phase = sys.argv[1]
try:
try:
with repo._transaction() as conn:
conn.execute('SELECT 1')
print('ARMED', flush=True)
assert sys.stdin.readline().strip() == 'go'
started = time.monotonic()
if phase == 'query':
conn.execute('SELECT 1')
elif phase == 'rollback':
raise SyntheticRollback()
# commit: leave the block while PostgreSQL is paused.
except (Unavailable, SyntheticRollback) as error:
elapsed = time.monotonic() - started
assert conn.closed, 'timed-out connection was returned open'
assert elapsed < 8, elapsed
expected = SyntheticRollback if phase == 'rollback' else Unavailable
assert isinstance(error, expected), type(error)
print('TIMED_OUT', flush=True)
else:
raise AssertionError('operation unexpectedly succeeded while PostgreSQL paused')
assert sys.stdin.readline().strip() == 'resume'
recovery = time.monotonic()
while True:
try:
repo.ping()
break
except Unavailable:
if time.monotonic() - recovery >= 15:
raise
time.sleep(0.1)
assert pool.get_stats().get('requests_waiting', 0) == 0
print(json.dumps(dict(phase=phase, timeout_seconds=round(elapsed, 3),
connection_closed=conn.closed, pool_max_size=1,
recovered_seconds=round(time.monotonic()-recovery, 3))), flush=True)
finally:
pool.close(timeout=1)
'''
def line(proc, timeout=15):
result = queue.Queue()
threading.Thread(target=lambda: result.put(proc.stdout.readline()), daemon=True).start()
try:
return result.get(timeout=timeout).strip()
except queue.Empty:
raise AssertionError('database test worker did not finish in time') from None
def ready(c):
start = time.monotonic()
while time.monotonic() - start < 30:
try:
c.call('GET', '/health/ready', schema='Health')
return round(time.monotonic() - start, 3)
except Failure:
time.sleep(0.2)
raise AssertionError('API did not recover after PostgreSQL unpause')
def readiness(c, ids):
a, b, eve, chat = c.bootstrap()
for index in range(len(ids)):
compose('exec', '--index', str(index + 1), '-T', 'api', 'python', '-c', PROBE, '200', '1')
for cycle in range(2):
try:
compose('pause', 'postgres')
with ThreadPoolExecutor(max_workers=len(ids)) as executor:
futures = [executor.submit(compose, 'exec', '--index', str(index + 1), '-T', 'api',
'python', '-c', PROBE, '503', '12') for index in range(len(ids))]
for index, future in enumerate(futures):
record(dict(cycle=cycle + 1, api=index + 1, **json.loads(future.result())))
finally:
compose('unpause', 'postgres')
elapsed = ready(c)
# Same token and a real DB write/read after recovery, no API restart.
message = c.message(a, chat, 'PostgreSQL pause regression ' + str(cycle))['json']
page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=b, schema='MessagePage')['json']
check(message in page['items'], 'confirmed write not visible after unpause')
record(dict(cycle=cycle + 1, recovered_seconds=elapsed, existing_session_and_write='PASS'))
def operation(phase):
proc = subprocess.Popen(['docker', 'compose', 'exec', '-T', 'api', 'python', '-u', '-c', WORKER, phase],
cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
text=True, encoding='utf-8', errors='replace')
try:
check(line(proc) == 'ARMED', 'worker could not open a transaction')
try:
compose('pause', 'postgres')
proc.stdin.write('go\n')
proc.stdin.flush()
check(line(proc) == 'TIMED_OUT', 'operation did not time out and close connection: ' + phase)
finally:
compose('unpause', 'postgres')
out, err = proc.communicate('resume\n', timeout=25)
check(proc.returncode == 0, 'worker failed: ' + err[-1200:])
record(json.loads(out))
finally:
if proc.poll() is None:
proc.kill()
proc.communicate(timeout=5)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
parser.add_argument('--output', type=Path)
args = parser.parse_args()
try:
c = Client(args.base_url, timeout=8)
ready(c)
ids = compose('ps', '-q', 'api').splitlines()
check(len(ids) >= 2, 'start at least two API replicas')
readiness(c, ids)
for phase in ('query', 'commit', 'rollback'):
operation(phase)
ready(c)
record('PASS: paused PostgreSQL returns bounded direct 503; query/commit/rollback discard connections; same pools recover.')
return 0
except Exception as error:
record('FAIL: ' + str(error))
return 1
finally:
if args.output:
args.output.write_text('\n'.join(LOG) + '\n', encoding='utf-8')
if __name__ == '__main__':
sys.exit(main())