"""Real local PostgreSQL + 2 API processes + HAProxy. Optional, not a Docker substitute. python scripts/check_local.py --pg-bin /path/to/bin --haproxy /path/to/haproxy """ import argparse import json import os from pathlib import Path import shlex import signal import subprocess import sys import tempfile import time from urllib.request import urlopen ROOT = Path(__file__).resolve().parents[1] def pg(state, *args): subprocess.run([str(Path(state['bin']) / 'pg_ctl'), '-D', state['data'], *args], check=True, stdout=subprocess.DEVNULL) def wait_url(port): for _ in range(150): try: with urlopen('http://127.0.0.1:%s/health/ready' % port, timeout=1) as response: if response.status == 200: return except OSError: time.sleep(.1) raise RuntimeError('Service did not become ready on port %s' % port) def launch(state, name): temp = Path(state['data']).parent if name == 'lb': command = [state['haproxy'], '-db', '-f', str(temp / 'haproxy.cfg')] env = dict(os.environ) else: command = [sys.executable, '-m', 'messenger'] env = dict(os.environ, APP_HOST='127.0.0.1', APP_PORT=str(state['ports'][name])) with (temp / (name + '.log')).open('a') as log: process = subprocess.Popen(command, env=env, cwd=ROOT, stdout=log, stderr=log, start_new_session=True) state['pids'][name] = process.pid return process def stop(pid): try: os.kill(pid, signal.SIGTERM) except ProcessLookupError: return for _ in range(70): try: # Zombie processes no longer own sockets and await their parent. status = Path('/proc/%s/stat' % pid) if not status.exists() or status.read_text().split()[2] == 'Z': return except FileNotFoundError: return time.sleep(.1) os.kill(pid, signal.SIGKILL) def hook(path, action): state = json.loads(path.read_text()) if action == 'stop': target = os.environ['TARGET_INSTANCE_ID'] for name, port in state['ports'].items(): try: with urlopen('http://127.0.0.1:%s/health/live' % port, timeout=1) as r: matches = r.headers['X-Instance-Id'] == target except OSError: continue if matches: pid = state['pids'][name] start = time.monotonic() stop(pid) state['stopped'] = name path.write_text(json.dumps(state)) with (ROOT / 'evidence/local-stops.txt').open('a') as f: f.write('%s PID=%s instance=%s stopped in %.3fs; remains stopped\n' % (name, pid, target, time.monotonic()-start)) return raise RuntimeError('Target instance not found') import psycopg with psycopg.connect(os.environ['DATABASE_URL']) as conn: before = conn.execute('SELECT pg_postmaster_start_time()').fetchone()[0] old = dict(state['pids']) for pid in old.values(): stop(pid) pg(state, '-m', 'fast', '-w', 'stop') pg(state, '-l', str(path.parent / 'postgres.log'), '-w', 'start') for name in ('api1', 'api2', 'lb'): launch(state, name) path.write_text(json.dumps(state)) wait_url(18080) with psycopg.connect(os.environ['DATABASE_URL']) as conn: after = conn.execute('SELECT pg_postmaster_start_time()').fetchone()[0] assert before != after and all(old[k] != state['pids'][k] for k in old) (ROOT / 'evidence/local-restart.txt').write_text('Actual process PIDs before: %s\nAfter: %s\nPostgreSQL start: %s -> %s\nSame data directory, no reseed or migrations in hook.\n' % (old, state['pids'], before.isoformat(), after.isoformat())) def main(): parser = argparse.ArgumentParser() parser.add_argument('--pg-bin') parser.add_argument('--haproxy') parser.add_argument('--hook', choices=['stop', 'restart']) parser.add_argument('--state', type=Path) args = parser.parse_args() if args.hook: hook(args.state, args.hook) return if not args.pg_bin or not args.haproxy: parser.error('--pg-bin and --haproxy required') binary = Path(args.pg_bin).resolve() with tempfile.TemporaryDirectory(prefix='messenger-lab3-') as tmp: temp = Path(tmp) path = temp / 'state.json' state = {'bin': str(binary), 'haproxy': str(Path(args.haproxy).resolve()), 'data': str(temp / 'data'), 'ports': {'api1': 18081, 'api2': 18082}, 'pids': {}} os.environ.update(DATABASE_URL='postgresql://127.0.0.1:15432/postgres', SESSION_STORE='postgres', SESSION_TTL_SECONDS='3600', IDEMPOTENCY_TTL_SECONDS='86400') os.environ.pop('DATABASE_URL_FILE', None) subprocess.run([str(binary / 'initdb'), '-D', state['data'], '-A', 'trust', '--no-locale', '-E', 'UTF8'], check=True, stdout=subprocess.DEVNULL) with (Path(state['data']) / 'postgresql.conf').open('a') as f: f.write("\nport=15432\nlisten_addresses='127.0.0.1'\nunix_socket_directories=''\ntimezone='UTC'\n") config = (ROOT / 'infra/haproxy.cfg').read_text().replace('bind :8080', 'bind 127.0.0.1:18080') for name, port in state['ports'].items(): config = config.replace(name + ':8080', '127.0.0.1:' + str(port)) # Local fixed loopback addresses have no Docker DNS; all LB behavior is unchanged. config = config.replace(' resolvers docker resolve-prefer ipv4 init-addr libc,none', '') (temp / 'haproxy.cfg').write_text(config) processes = [] def run(command): print('$ ' + ' '.join(command), flush=True) subprocess.run(command, cwd=ROOT, check=True) try: pg(state, '-l', str(temp / 'postgres.log'), '-w', 'start') run([sys.executable, '-m', 'messenger.migrate']) run([sys.executable, '-m', 'messenger.migrate']) run([state['haproxy'], '-c', '-f', str(temp / 'haproxy.cfg')]) for name in ('api1', 'api2', 'lb'): processes.append(launch(state, name)) path.write_text(json.dumps(state)) wait_url(18081); wait_url(18082); wait_url(18080) print('Distinct running processes:', state['pids'], flush=True) (ROOT / 'evidence/local-stops.txt').write_text('Actual local API stops (not Docker):\n') run(['make', 'check', 'validate', 'student-test', 'PYTHON=' + sys.executable]) run(['make', 'test', 'test-idempotency', 'BASE_URL=http://127.0.0.1:18080', 'PYTHON=' + sys.executable]) for action in ('restart', 'stop'): script = temp / (action + '.sh') script.write_text('#!/bin/sh\nexec ' + shlex.join([sys.executable, str(Path(__file__).resolve()), '--hook', action, '--state', str(path)]) + '\n') script.chmod(0o700) sys.path.insert(0, str(ROOT / 'tests')) from client import Client from uuid import uuid4 client = Client('http://127.0.0.1:18080') actor, _, _, durable_chat = client.bootstrap() durable_key = uuid4().hex original = client.message(actor, durable_chat, 'Replay across restart', key=durable_key)['json'] run(['make', 'test-persistence' , 'ACTION_SCRIPT=' + str(temp / 'restart.sh'), 'BASE_URL=http://127.0.0.1:18080', 'PYTHON=' + sys.executable]) assert client.message(actor, durable_chat, 'Replay across restart', key=durable_key)['json'] == original print('PASS: same key + old session after full restart returns original Message', flush=True) run(['make', 'test-idempotency', 'BASE_URL=http://127.0.0.1:18080', 'PYTHON=' + sys.executable]) for attempt in range(2): if attempt: # Shift the round-robin position so the second public scenario # selects the other process, without changing the public test. with urlopen('http://127.0.0.1:18080/health/live'): pass run(['make', 'test-failover', 'ACTION_SCRIPT=' + str(temp / 'stop.sh'), 'BASE_URL=http://127.0.0.1:18080', 'PYTHON=' + sys.executable]) assert client.message(actor, durable_chat, 'Replay across restart', key=durable_key)['json'] == original print('PASS: original idempotent result survives selected replica stop', flush=True) state = json.loads(path.read_text()) processes.append(launch(state, state.pop('stopped'))) path.write_text(json.dumps(state)) wait_url(18081); wait_url(18082) time.sleep(1.2) print((ROOT / 'evidence/local-restart.txt').read_text(), flush=True) print((ROOT / 'evidence/local-stops.txt').read_text(), flush=True) run([state['haproxy'], '-v']) run([str(binary / 'postgres'), '--version']) finally: if path.exists(): state = json.loads(path.read_text()) for pid in state['pids'].values(): stop(pid) for process in processes: process.wait(timeout=10) subprocess.run([str(binary / 'pg_ctl'), '-D', state['data'], '-m', 'fast', '-w', 'stop'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) if __name__ == '__main__': main()