Add API integration checks, migration compatibility, rolling restart scenarios and a report verification collector.
171 lines
7.7 KiB
Python
171 lines
7.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Check readiness 200 -> 503 -> closed, liveness and LB withdrawal on SIGTERM."""
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
PROBE = r'''
|
|
import json, sys, threading, time, urllib.error, urllib.request
|
|
url = sys.argv[1]
|
|
live_url = url.replace('/health/ready', '/health/live')
|
|
phase = 'before'
|
|
events = []
|
|
def commands():
|
|
global phase
|
|
for line in sys.stdin:
|
|
phase = line.strip()
|
|
if phase == 'end': break
|
|
def status(address):
|
|
try:
|
|
with urllib.request.urlopen(address, timeout=1) as response:
|
|
return str(response.status)
|
|
except urllib.error.HTTPError as error:
|
|
return str(error.code)
|
|
except (urllib.error.URLError, TimeoutError, OSError):
|
|
return 'connection_error'
|
|
threading.Thread(target=commands, daemon=True).start()
|
|
print('READY', flush=True)
|
|
while phase != 'end':
|
|
current = phase
|
|
ready = status(url)
|
|
live = status(live_url)
|
|
events.append({'phase': current, 'ready': ready, 'live': live})
|
|
time.sleep(0.05)
|
|
print(json.dumps(events), flush=True)
|
|
'''
|
|
|
|
|
|
def command(*args, timeout=45):
|
|
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
|
|
timeout=timeout, check=False)
|
|
if result.returncode:
|
|
raise RuntimeError(f'{args[0]} {args[1]} failed (code {result.returncode})')
|
|
return result.stdout.strip()
|
|
|
|
|
|
def both_ready(base_url, expected):
|
|
seen = set()
|
|
deadline = time.monotonic() + 45
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with urllib.request.urlopen(base_url + '/health/ready', timeout=2) as response:
|
|
seen.add(response.headers['X-Instance-Id'])
|
|
except (urllib.error.URLError, TimeoutError, OSError):
|
|
pass
|
|
if seen == expected:
|
|
return
|
|
time.sleep(0.1)
|
|
raise RuntimeError(f'both replicas did not recover: {seen}')
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
|
|
args = parser.parse_args()
|
|
ids = command('docker', 'compose', 'ps', '--all', '-q', 'api').splitlines()
|
|
if len(ids) != 2:
|
|
raise RuntimeError('shutdown probe requires two API replicas')
|
|
expected = {command('docker', 'inspect', '--format', '{{.Config.Hostname}}', item) for item in ids}
|
|
target = ids[0]
|
|
networks = json.loads(command('docker', 'inspect', '--format', '{{json .NetworkSettings.Networks}}', target))
|
|
network, settings = next(iter(networks.items()))
|
|
address = settings['IPAddress']
|
|
nginx = command('docker', 'compose', 'ps', '-q', 'nginx')
|
|
both_ready(args.base_url, expected)
|
|
probe = subprocess.Popen(
|
|
['docker', 'run', '--rm', '-i', '--network', network,
|
|
'python:3.11-slim-bookworm', 'python', '-u', '-c', PROBE,
|
|
f'http://{address}:8080/health/ready'],
|
|
cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL, text=True)
|
|
stop = None
|
|
|
|
def phase(value):
|
|
probe.stdin.write(value + '\n')
|
|
probe.stdin.flush()
|
|
|
|
try:
|
|
if probe.stdout.readline().strip() != 'READY':
|
|
raise RuntimeError('direct readiness probe did not start')
|
|
time.sleep(1)
|
|
phase('stopping')
|
|
started = time.monotonic()
|
|
stop = subprocess.Popen(['docker', 'stop', '--time', '30', target], cwd=ROOT,
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
withdrawn_at = None
|
|
withdrawn_while_running = False
|
|
while stop.poll() is None and time.monotonic() - started < 35:
|
|
upstreams = command('docker', 'exec', nginx, 'cat', '/etc/nginx/ready-upstreams.conf', timeout=10)
|
|
if f'server {address}:8080 ' not in upstreams:
|
|
withdrawn_at = time.monotonic() - started
|
|
withdrawn_while_running = command('docker', 'inspect', '--format', '{{.State.Running}}', target) == 'true'
|
|
if withdrawn_while_running:
|
|
phase('withdrawn')
|
|
break
|
|
time.sleep(0.1)
|
|
stop.communicate(timeout=40)
|
|
if stop.returncode:
|
|
raise RuntimeError('docker stop failed')
|
|
stop_seconds = time.monotonic() - started
|
|
exit_code = int(command('docker', 'inspect', '--format', '{{.State.ExitCode}}', target))
|
|
phase('stopped')
|
|
time.sleep(1)
|
|
phase('end')
|
|
output, _ = probe.communicate(timeout=15)
|
|
if probe.returncode:
|
|
raise RuntimeError('direct probe failed')
|
|
events = json.loads(output.strip())
|
|
summary = {name: {value: sum(e['phase'] == name and e['ready'] == value for e in events)
|
|
for value in sorted({e['ready'] for e in events if e['phase'] == name})}
|
|
for name in ('before', 'stopping', 'withdrawn', 'stopped')}
|
|
draining = [e for e in events if e['ready'] == '503']
|
|
first_503 = next((i for i, e in enumerate(events) if e['ready'] == '503'), None)
|
|
if summary['before'].get('200', 0) < 5:
|
|
raise RuntimeError('readiness was not healthy before SIGTERM')
|
|
if first_503 is None or any(e['ready'] == '200' for e in events[first_503:]):
|
|
raise RuntimeError('expected readiness 200 -> 503 with no late 200')
|
|
# These are separate requests; transport failures can straddle HTTP
|
|
# close or the client's deadline. Do not confuse them with unhealthy
|
|
# HTTP responses. Record them separately rather than hiding them.
|
|
if not draining or not any(e['live'] == '200' for e in draining):
|
|
raise RuntimeError('liveness must remain 200 while readiness is 503')
|
|
if any(e['live'] not in {'200', 'connection_error'} for e in draining):
|
|
raise RuntimeError('liveness returned an unhealthy HTTP status during draining')
|
|
if not withdrawn_while_running or not any(e['phase'] == 'withdrawn' and e['ready'] == '503' for e in events):
|
|
raise RuntimeError('LB did not withdraw the replica while its HTTP server was alive')
|
|
if summary['stopped'].get('200', 0) or summary['stopped'].get('503', 0) or not summary['stopped'].get('connection_error', 0):
|
|
raise RuntimeError('replica did not close HTTP after draining')
|
|
if exit_code == 137:
|
|
raise RuntimeError('container required SIGKILL instead of graceful shutdown')
|
|
print(json.dumps({'stop_seconds': round(stop_seconds, 3),
|
|
'lb_withdrawn_seconds': round(withdrawn_at, 3),
|
|
'lb_withdrawn_before_http_close': True,
|
|
'readiness_sequence': [200, 503, 'connection_error'],
|
|
'readiness_503_samples': len(draining),
|
|
'liveness_200_while_draining': sum(e['live'] == '200' for e in draining),
|
|
'liveness_connection_errors_while_draining': sum(e['live'] == 'connection_error' for e in draining),
|
|
'exit_code': exit_code, 'direct_ready_responses': summary}, indent=2))
|
|
print('PASS: readiness 200 -> 503 -> closed; liveness 200; LB withdrawal before HTTP close')
|
|
finally:
|
|
if probe.poll() is None:
|
|
probe.kill()
|
|
probe.communicate()
|
|
if stop is not None and stop.poll() is None:
|
|
stop.communicate(timeout=40)
|
|
command('docker', 'start', target)
|
|
both_ready(args.base_url, expected)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except (RuntimeError, OSError, subprocess.TimeoutExpired) as exc:
|
|
print(f'FAIL: {exc}', file=sys.stderr)
|
|
raise SystemExit(1) from None
|