#!/usr/bin/env python3 """Regression: pause PostgreSQL, direct API 503, discard connection and recover. No published replica ports: direct HTTP is sent from inside each API container. PostgreSQL is always unpaused in finally. Uses existing local synthetic stand. """ import argparse from concurrent.futures import ThreadPoolExecutor import json from pathlib import Path import queue import subprocess import sys import threading import time ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) from client import Client, Failure, check LOG = [] def record(value): text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False) LOG.append(text) print(text, flush=True) def compose(*args, timeout=40): proc = subprocess.run(['docker', 'compose', *args], cwd=ROOT, capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=timeout) if proc.returncode: raise RuntimeError('docker compose ' + ' '.join(args[:3]) + ': ' + proc.stderr[-1200:]) return proc.stdout.strip() PROBE = r''' from concurrent.futures import ThreadPoolExecutor import http.client, json, sys, time expected, count = map(int, sys.argv[1:]) def request(path, status): c = http.client.HTTPConnection('127.0.0.1', 8080, timeout=8) start = time.monotonic() try: c.request('GET', path) response = c.getresponse() body = json.loads(response.read()) assert response.status == status, (response.status, body) assert response.getheader('X-Request-Id') assert response.getheader('X-Instance-Id') if status == 503: assert body['error']['code'] == 'unavailable' assert body['error']['request_id'] == response.getheader('X-Request-Id') return round(time.monotonic() - start, 3) finally: c.close() with ThreadPoolExecutor(max_workers=count) as executor: futures = [executor.submit(request, '/health/ready', expected) for _ in range(count)] live = request('/health/live', 200) elapsed = [future.result() for future in futures] print(json.dumps(dict(status=expected, requests=count, seconds=elapsed, live_seconds=live))) ''' WORKER = r''' import json, sys, time from messenger.config import read_secret from messenger.db import create_pool, Repository from messenger.errors import Unavailable class SyntheticRollback(Exception): pass pool = create_pool(read_secret('DATABASE_URL'), 1) pool.open(wait=True, timeout=10) repo = Repository(pool) phase = sys.argv[1] try: try: with repo._transaction() as conn: conn.execute('SELECT 1') print('ARMED', flush=True) assert sys.stdin.readline().strip() == 'go' started = time.monotonic() if phase == 'query': conn.execute('SELECT 1') elif phase == 'rollback': raise SyntheticRollback() # commit: leave the block while PostgreSQL is paused. except (Unavailable, SyntheticRollback) as error: elapsed = time.monotonic() - started assert conn.closed, 'timed-out connection was returned open' assert elapsed < 8, elapsed expected = SyntheticRollback if phase == 'rollback' else Unavailable assert isinstance(error, expected), type(error) print('TIMED_OUT', flush=True) else: raise AssertionError('operation unexpectedly succeeded while PostgreSQL paused') assert sys.stdin.readline().strip() == 'resume' recovery = time.monotonic() while True: try: repo.ping() break except Unavailable: if time.monotonic() - recovery >= 15: raise time.sleep(0.1) assert pool.get_stats().get('requests_waiting', 0) == 0 print(json.dumps(dict(phase=phase, timeout_seconds=round(elapsed, 3), connection_closed=conn.closed, pool_max_size=1, recovered_seconds=round(time.monotonic()-recovery, 3))), flush=True) finally: pool.close(timeout=1) ''' def line(proc, timeout=15): result = queue.Queue() threading.Thread(target=lambda: result.put(proc.stdout.readline()), daemon=True).start() try: return result.get(timeout=timeout).strip() except queue.Empty: raise AssertionError('database test worker did not finish in time') from None def ready(c): start = time.monotonic() while time.monotonic() - start < 30: try: c.call('GET', '/health/ready', schema='Health') return round(time.monotonic() - start, 3) except Failure: time.sleep(0.2) raise AssertionError('API did not recover after PostgreSQL unpause') def readiness(c, ids): a, b, eve, chat = c.bootstrap() for index in range(len(ids)): compose('exec', '--index', str(index + 1), '-T', 'api', 'python', '-c', PROBE, '200', '1') for cycle in range(2): try: compose('pause', 'postgres') with ThreadPoolExecutor(max_workers=len(ids)) as executor: futures = [executor.submit(compose, 'exec', '--index', str(index + 1), '-T', 'api', 'python', '-c', PROBE, '503', '12') for index in range(len(ids))] for index, future in enumerate(futures): record(dict(cycle=cycle + 1, api=index + 1, **json.loads(future.result()))) finally: compose('unpause', 'postgres') elapsed = ready(c) # Same token and a real DB write/read after recovery, no API restart. message = c.message(a, chat, 'PostgreSQL pause regression ' + str(cycle))['json'] page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=b, schema='MessagePage')['json'] check(message in page['items'], 'confirmed write not visible after unpause') record(dict(cycle=cycle + 1, recovered_seconds=elapsed, existing_session_and_write='PASS')) def operation(phase): proc = subprocess.Popen(['docker', 'compose', 'exec', '-T', 'api', 'python', '-u', '-c', WORKER, phase], cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8', errors='replace') try: check(line(proc) == 'ARMED', 'worker could not open a transaction') try: compose('pause', 'postgres') proc.stdin.write('go\n') proc.stdin.flush() check(line(proc) == 'TIMED_OUT', 'operation did not time out and close connection: ' + phase) finally: compose('unpause', 'postgres') out, err = proc.communicate('resume\n', timeout=25) check(proc.returncode == 0, 'worker failed: ' + err[-1200:]) record(json.loads(out)) finally: if proc.poll() is None: proc.kill() proc.communicate(timeout=5) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--base-url', default='http://127.0.0.1:8080') parser.add_argument('--output', type=Path) args = parser.parse_args() try: c = Client(args.base_url, timeout=8) ready(c) ids = compose('ps', '-q', 'api').splitlines() check(len(ids) >= 2, 'start at least two API replicas') readiness(c, ids) for phase in ('query', 'commit', 'rollback'): operation(phase) ready(c) record('PASS: paused PostgreSQL returns bounded direct 503; query/commit/rollback discard connections; same pools recover.') return 0 except Exception as error: record('FAIL: ' + str(error)) return 1 finally: if args.output: args.output.write_text('\n'.join(LOG) + '\n', encoding='utf-8') if __name__ == '__main__': sys.exit(main())