#!/usr/bin/env python3 """Check readiness 200 -> 503 -> closed, liveness and LB withdrawal on SIGTERM.""" import argparse import json from pathlib import Path import subprocess import sys import time import urllib.error import urllib.request ROOT = Path(__file__).resolve().parents[1] PROBE = r''' import json, sys, threading, time, urllib.error, urllib.request url = sys.argv[1] live_url = url.replace('/health/ready', '/health/live') phase = 'before' events = [] def commands(): global phase for line in sys.stdin: phase = line.strip() if phase == 'end': break def status(address): try: with urllib.request.urlopen(address, timeout=1) as response: return str(response.status) except urllib.error.HTTPError as error: return str(error.code) except (urllib.error.URLError, TimeoutError, OSError): return 'connection_error' threading.Thread(target=commands, daemon=True).start() print('READY', flush=True) while phase != 'end': current = phase ready = status(url) live = status(live_url) events.append({'phase': current, 'ready': ready, 'live': live}) time.sleep(0.05) print(json.dumps(events), flush=True) ''' def command(*args, timeout=45): result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, timeout=timeout, check=False) if result.returncode: raise RuntimeError(f'{args[0]} {args[1]} failed (code {result.returncode})') return result.stdout.strip() def both_ready(base_url, expected): seen = set() deadline = time.monotonic() + 45 while time.monotonic() < deadline: try: with urllib.request.urlopen(base_url + '/health/ready', timeout=2) as response: seen.add(response.headers['X-Instance-Id']) except (urllib.error.URLError, TimeoutError, OSError): pass if seen == expected: return time.sleep(0.1) raise RuntimeError(f'both replicas did not recover: {seen}') def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--base-url', default='http://127.0.0.1:8080') args = parser.parse_args() ids = command('docker', 'compose', 'ps', '--all', '-q', 'api').splitlines() if len(ids) != 2: raise RuntimeError('shutdown probe requires two API replicas') expected = {command('docker', 'inspect', '--format', '{{.Config.Hostname}}', item) for item in ids} target = ids[0] networks = json.loads(command('docker', 'inspect', '--format', '{{json .NetworkSettings.Networks}}', target)) network, settings = next(iter(networks.items())) address = settings['IPAddress'] nginx = command('docker', 'compose', 'ps', '-q', 'nginx') both_ready(args.base_url, expected) probe = subprocess.Popen( ['docker', 'run', '--rm', '-i', '--network', network, 'python:3.11-slim-bookworm', 'python', '-u', '-c', PROBE, f'http://{address}:8080/health/ready'], cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) stop = None def phase(value): probe.stdin.write(value + '\n') probe.stdin.flush() try: if probe.stdout.readline().strip() != 'READY': raise RuntimeError('direct readiness probe did not start') time.sleep(1) phase('stopping') started = time.monotonic() stop = subprocess.Popen(['docker', 'stop', '--time', '30', target], cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) withdrawn_at = None withdrawn_while_running = False while stop.poll() is None and time.monotonic() - started < 35: upstreams = command('docker', 'exec', nginx, 'cat', '/etc/nginx/ready-upstreams.conf', timeout=10) if f'server {address}:8080 ' not in upstreams: withdrawn_at = time.monotonic() - started withdrawn_while_running = command('docker', 'inspect', '--format', '{{.State.Running}}', target) == 'true' if withdrawn_while_running: phase('withdrawn') break time.sleep(0.1) stop.communicate(timeout=40) if stop.returncode: raise RuntimeError('docker stop failed') stop_seconds = time.monotonic() - started exit_code = int(command('docker', 'inspect', '--format', '{{.State.ExitCode}}', target)) phase('stopped') time.sleep(1) phase('end') output, _ = probe.communicate(timeout=15) if probe.returncode: raise RuntimeError('direct probe failed') events = json.loads(output.strip()) summary = {name: {value: sum(e['phase'] == name and e['ready'] == value for e in events) for value in sorted({e['ready'] for e in events if e['phase'] == name})} for name in ('before', 'stopping', 'withdrawn', 'stopped')} draining = [e for e in events if e['ready'] == '503'] first_503 = next((i for i, e in enumerate(events) if e['ready'] == '503'), None) if summary['before'].get('200', 0) < 5: raise RuntimeError('readiness was not healthy before SIGTERM') if first_503 is None or any(e['ready'] == '200' for e in events[first_503:]): raise RuntimeError('expected readiness 200 -> 503 with no late 200') # These are separate requests; transport failures can straddle HTTP # close or the client's deadline. Do not confuse them with unhealthy # HTTP responses. Record them separately rather than hiding them. if not draining or not any(e['live'] == '200' for e in draining): raise RuntimeError('liveness must remain 200 while readiness is 503') if any(e['live'] not in {'200', 'connection_error'} for e in draining): raise RuntimeError('liveness returned an unhealthy HTTP status during draining') if not withdrawn_while_running or not any(e['phase'] == 'withdrawn' and e['ready'] == '503' for e in events): raise RuntimeError('LB did not withdraw the replica while its HTTP server was alive') if summary['stopped'].get('200', 0) or summary['stopped'].get('503', 0) or not summary['stopped'].get('connection_error', 0): raise RuntimeError('replica did not close HTTP after draining') if exit_code == 137: raise RuntimeError('container required SIGKILL instead of graceful shutdown') print(json.dumps({'stop_seconds': round(stop_seconds, 3), 'lb_withdrawn_seconds': round(withdrawn_at, 3), 'lb_withdrawn_before_http_close': True, 'readiness_sequence': [200, 503, 'connection_error'], 'readiness_503_samples': len(draining), 'liveness_200_while_draining': sum(e['live'] == '200' for e in draining), 'liveness_connection_errors_while_draining': sum(e['live'] == 'connection_error' for e in draining), 'exit_code': exit_code, 'direct_ready_responses': summary}, indent=2)) print('PASS: readiness 200 -> 503 -> closed; liveness 200; LB withdrawal before HTTP close') finally: if probe.poll() is None: probe.kill() probe.communicate() if stop is not None and stop.poll() is None: stop.communicate(timeout=40) command('docker', 'start', target) both_ready(args.base_url, expected) if __name__ == '__main__': try: main() except (RuntimeError, OSError, subprocess.TimeoutExpired) as exc: print(f'FAIL: {exc}', file=sys.stderr) raise SystemExit(1) from None