forked from mtusi2026/mtusi2026lab5
209 lines
7.7 KiB
Python
209 lines
7.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Regression: pause PostgreSQL, direct API 503, discard connection and recover.
|
|
|
|
No published replica ports: direct HTTP is sent from inside each API container.
|
|
PostgreSQL is always unpaused in finally. Uses existing local synthetic stand.
|
|
"""
|
|
import argparse
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import json
|
|
from pathlib import Path
|
|
import queue
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, Failure, check
|
|
|
|
LOG = []
|
|
|
|
|
|
def record(value):
|
|
text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)
|
|
LOG.append(text)
|
|
print(text, flush=True)
|
|
|
|
|
|
def compose(*args, timeout=40):
|
|
proc = subprocess.run(['docker', 'compose', *args], cwd=ROOT, capture_output=True,
|
|
text=True, encoding='utf-8', errors='replace', timeout=timeout)
|
|
if proc.returncode:
|
|
raise RuntimeError('docker compose ' + ' '.join(args[:3]) + ': ' + proc.stderr[-1200:])
|
|
return proc.stdout.strip()
|
|
|
|
|
|
PROBE = r'''
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import http.client, json, sys, time
|
|
|
|
expected, count = map(int, sys.argv[1:])
|
|
def request(path, status):
|
|
c = http.client.HTTPConnection('127.0.0.1', 8080, timeout=8)
|
|
start = time.monotonic()
|
|
try:
|
|
c.request('GET', path)
|
|
response = c.getresponse()
|
|
body = json.loads(response.read())
|
|
assert response.status == status, (response.status, body)
|
|
assert response.getheader('X-Request-Id')
|
|
assert response.getheader('X-Instance-Id')
|
|
if status == 503:
|
|
assert body['error']['code'] == 'unavailable'
|
|
assert body['error']['request_id'] == response.getheader('X-Request-Id')
|
|
return round(time.monotonic() - start, 3)
|
|
finally:
|
|
c.close()
|
|
|
|
with ThreadPoolExecutor(max_workers=count) as executor:
|
|
futures = [executor.submit(request, '/health/ready', expected) for _ in range(count)]
|
|
live = request('/health/live', 200)
|
|
elapsed = [future.result() for future in futures]
|
|
print(json.dumps(dict(status=expected, requests=count, seconds=elapsed, live_seconds=live)))
|
|
'''
|
|
|
|
|
|
WORKER = r'''
|
|
import json, sys, time
|
|
from messenger.config import read_secret
|
|
from messenger.db import create_pool, Repository
|
|
from messenger.errors import Unavailable
|
|
|
|
class SyntheticRollback(Exception): pass
|
|
pool = create_pool(read_secret('DATABASE_URL'), 1)
|
|
pool.open(wait=True, timeout=10)
|
|
repo = Repository(pool)
|
|
phase = sys.argv[1]
|
|
try:
|
|
try:
|
|
with repo._transaction() as conn:
|
|
conn.execute('SELECT 1')
|
|
print('ARMED', flush=True)
|
|
assert sys.stdin.readline().strip() == 'go'
|
|
started = time.monotonic()
|
|
if phase == 'query':
|
|
conn.execute('SELECT 1')
|
|
elif phase == 'rollback':
|
|
raise SyntheticRollback()
|
|
# commit: leave the block while PostgreSQL is paused.
|
|
except (Unavailable, SyntheticRollback) as error:
|
|
elapsed = time.monotonic() - started
|
|
assert conn.closed, 'timed-out connection was returned open'
|
|
assert elapsed < 8, elapsed
|
|
expected = SyntheticRollback if phase == 'rollback' else Unavailable
|
|
assert isinstance(error, expected), type(error)
|
|
print('TIMED_OUT', flush=True)
|
|
else:
|
|
raise AssertionError('operation unexpectedly succeeded while PostgreSQL paused')
|
|
assert sys.stdin.readline().strip() == 'resume'
|
|
recovery = time.monotonic()
|
|
while True:
|
|
try:
|
|
repo.ping()
|
|
break
|
|
except Unavailable:
|
|
if time.monotonic() - recovery >= 15:
|
|
raise
|
|
time.sleep(0.1)
|
|
assert pool.get_stats().get('requests_waiting', 0) == 0
|
|
print(json.dumps(dict(phase=phase, timeout_seconds=round(elapsed, 3),
|
|
connection_closed=conn.closed, pool_max_size=1,
|
|
recovered_seconds=round(time.monotonic()-recovery, 3))), flush=True)
|
|
finally:
|
|
pool.close(timeout=1)
|
|
'''
|
|
|
|
|
|
def line(proc, timeout=15):
|
|
result = queue.Queue()
|
|
threading.Thread(target=lambda: result.put(proc.stdout.readline()), daemon=True).start()
|
|
try:
|
|
return result.get(timeout=timeout).strip()
|
|
except queue.Empty:
|
|
raise AssertionError('database test worker did not finish in time') from None
|
|
|
|
|
|
def ready(c):
|
|
start = time.monotonic()
|
|
while time.monotonic() - start < 30:
|
|
try:
|
|
c.call('GET', '/health/ready', schema='Health')
|
|
return round(time.monotonic() - start, 3)
|
|
except Failure:
|
|
time.sleep(0.2)
|
|
raise AssertionError('API did not recover after PostgreSQL unpause')
|
|
|
|
|
|
def readiness(c, ids):
|
|
a, b, eve, chat = c.bootstrap()
|
|
for index in range(len(ids)):
|
|
compose('exec', '--index', str(index + 1), '-T', 'api', 'python', '-c', PROBE, '200', '1')
|
|
for cycle in range(2):
|
|
try:
|
|
compose('pause', 'postgres')
|
|
with ThreadPoolExecutor(max_workers=len(ids)) as executor:
|
|
futures = [executor.submit(compose, 'exec', '--index', str(index + 1), '-T', 'api',
|
|
'python', '-c', PROBE, '503', '12') for index in range(len(ids))]
|
|
for index, future in enumerate(futures):
|
|
record(dict(cycle=cycle + 1, api=index + 1, **json.loads(future.result())))
|
|
finally:
|
|
compose('unpause', 'postgres')
|
|
elapsed = ready(c)
|
|
# Same token and a real DB write/read after recovery, no API restart.
|
|
message = c.message(a, chat, 'PostgreSQL pause regression ' + str(cycle))['json']
|
|
page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=b, schema='MessagePage')['json']
|
|
check(message in page['items'], 'confirmed write not visible after unpause')
|
|
record(dict(cycle=cycle + 1, recovered_seconds=elapsed, existing_session_and_write='PASS'))
|
|
|
|
|
|
def operation(phase):
|
|
proc = subprocess.Popen(['docker', 'compose', 'exec', '-T', 'api', 'python', '-u', '-c', WORKER, phase],
|
|
cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
text=True, encoding='utf-8', errors='replace')
|
|
try:
|
|
check(line(proc) == 'ARMED', 'worker could not open a transaction')
|
|
try:
|
|
compose('pause', 'postgres')
|
|
proc.stdin.write('go\n')
|
|
proc.stdin.flush()
|
|
check(line(proc) == 'TIMED_OUT', 'operation did not time out and close connection: ' + phase)
|
|
finally:
|
|
compose('unpause', 'postgres')
|
|
out, err = proc.communicate('resume\n', timeout=25)
|
|
check(proc.returncode == 0, 'worker failed: ' + err[-1200:])
|
|
record(json.loads(out))
|
|
finally:
|
|
if proc.poll() is None:
|
|
proc.kill()
|
|
proc.communicate(timeout=5)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
|
|
parser.add_argument('--output', type=Path)
|
|
args = parser.parse_args()
|
|
try:
|
|
c = Client(args.base_url, timeout=8)
|
|
ready(c)
|
|
ids = compose('ps', '-q', 'api').splitlines()
|
|
check(len(ids) >= 2, 'start at least two API replicas')
|
|
readiness(c, ids)
|
|
for phase in ('query', 'commit', 'rollback'):
|
|
operation(phase)
|
|
ready(c)
|
|
record('PASS: paused PostgreSQL returns bounded direct 503; query/commit/rollback discard connections; same pools recover.')
|
|
return 0
|
|
except Exception as error:
|
|
record('FAIL: ' + str(error))
|
|
return 1
|
|
finally:
|
|
if args.output:
|
|
args.output.write_text('\n'.join(LOG) + '\n', encoding='utf-8')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|