forked from mtusi2026/mtusi2026lab3
test: cover persistence failover and graceful draining
Add API integration checks, migration compatibility, rolling restart scenarios and a report verification collector.
This commit is contained in:
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check readiness 200 -> 503 -> closed, liveness and LB withdrawal on SIGTERM."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PROBE = r'''
|
||||
import json, sys, threading, time, urllib.error, urllib.request
|
||||
url = sys.argv[1]
|
||||
live_url = url.replace('/health/ready', '/health/live')
|
||||
phase = 'before'
|
||||
events = []
|
||||
def commands():
|
||||
global phase
|
||||
for line in sys.stdin:
|
||||
phase = line.strip()
|
||||
if phase == 'end': break
|
||||
def status(address):
|
||||
try:
|
||||
with urllib.request.urlopen(address, timeout=1) as response:
|
||||
return str(response.status)
|
||||
except urllib.error.HTTPError as error:
|
||||
return str(error.code)
|
||||
except (urllib.error.URLError, TimeoutError, OSError):
|
||||
return 'connection_error'
|
||||
threading.Thread(target=commands, daemon=True).start()
|
||||
print('READY', flush=True)
|
||||
while phase != 'end':
|
||||
current = phase
|
||||
ready = status(url)
|
||||
live = status(live_url)
|
||||
events.append({'phase': current, 'ready': ready, 'live': live})
|
||||
time.sleep(0.05)
|
||||
print(json.dumps(events), flush=True)
|
||||
'''
|
||||
|
||||
|
||||
def command(*args, timeout=45):
|
||||
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
|
||||
timeout=timeout, check=False)
|
||||
if result.returncode:
|
||||
raise RuntimeError(f'{args[0]} {args[1]} failed (code {result.returncode})')
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def both_ready(base_url, expected):
|
||||
seen = set()
|
||||
deadline = time.monotonic() + 45
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(base_url + '/health/ready', timeout=2) as response:
|
||||
seen.add(response.headers['X-Instance-Id'])
|
||||
except (urllib.error.URLError, TimeoutError, OSError):
|
||||
pass
|
||||
if seen == expected:
|
||||
return
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError(f'both replicas did not recover: {seen}')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
|
||||
args = parser.parse_args()
|
||||
ids = command('docker', 'compose', 'ps', '--all', '-q', 'api').splitlines()
|
||||
if len(ids) != 2:
|
||||
raise RuntimeError('shutdown probe requires two API replicas')
|
||||
expected = {command('docker', 'inspect', '--format', '{{.Config.Hostname}}', item) for item in ids}
|
||||
target = ids[0]
|
||||
networks = json.loads(command('docker', 'inspect', '--format', '{{json .NetworkSettings.Networks}}', target))
|
||||
network, settings = next(iter(networks.items()))
|
||||
address = settings['IPAddress']
|
||||
nginx = command('docker', 'compose', 'ps', '-q', 'nginx')
|
||||
both_ready(args.base_url, expected)
|
||||
probe = subprocess.Popen(
|
||||
['docker', 'run', '--rm', '-i', '--network', network,
|
||||
'python:3.11-slim-bookworm', 'python', '-u', '-c', PROBE,
|
||||
f'http://{address}:8080/health/ready'],
|
||||
cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL, text=True)
|
||||
stop = None
|
||||
|
||||
def phase(value):
|
||||
probe.stdin.write(value + '\n')
|
||||
probe.stdin.flush()
|
||||
|
||||
try:
|
||||
if probe.stdout.readline().strip() != 'READY':
|
||||
raise RuntimeError('direct readiness probe did not start')
|
||||
time.sleep(1)
|
||||
phase('stopping')
|
||||
started = time.monotonic()
|
||||
stop = subprocess.Popen(['docker', 'stop', '--time', '30', target], cwd=ROOT,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
withdrawn_at = None
|
||||
withdrawn_while_running = False
|
||||
while stop.poll() is None and time.monotonic() - started < 35:
|
||||
upstreams = command('docker', 'exec', nginx, 'cat', '/etc/nginx/ready-upstreams.conf', timeout=10)
|
||||
if f'server {address}:8080 ' not in upstreams:
|
||||
withdrawn_at = time.monotonic() - started
|
||||
withdrawn_while_running = command('docker', 'inspect', '--format', '{{.State.Running}}', target) == 'true'
|
||||
if withdrawn_while_running:
|
||||
phase('withdrawn')
|
||||
break
|
||||
time.sleep(0.1)
|
||||
stop.communicate(timeout=40)
|
||||
if stop.returncode:
|
||||
raise RuntimeError('docker stop failed')
|
||||
stop_seconds = time.monotonic() - started
|
||||
exit_code = int(command('docker', 'inspect', '--format', '{{.State.ExitCode}}', target))
|
||||
phase('stopped')
|
||||
time.sleep(1)
|
||||
phase('end')
|
||||
output, _ = probe.communicate(timeout=15)
|
||||
if probe.returncode:
|
||||
raise RuntimeError('direct probe failed')
|
||||
events = json.loads(output.strip())
|
||||
summary = {name: {value: sum(e['phase'] == name and e['ready'] == value for e in events)
|
||||
for value in sorted({e['ready'] for e in events if e['phase'] == name})}
|
||||
for name in ('before', 'stopping', 'withdrawn', 'stopped')}
|
||||
draining = [e for e in events if e['ready'] == '503']
|
||||
first_503 = next((i for i, e in enumerate(events) if e['ready'] == '503'), None)
|
||||
if summary['before'].get('200', 0) < 5:
|
||||
raise RuntimeError('readiness was not healthy before SIGTERM')
|
||||
if first_503 is None or any(e['ready'] == '200' for e in events[first_503:]):
|
||||
raise RuntimeError('expected readiness 200 -> 503 with no late 200')
|
||||
# These are separate requests; transport failures can straddle HTTP
|
||||
# close or the client's deadline. Do not confuse them with unhealthy
|
||||
# HTTP responses. Record them separately rather than hiding them.
|
||||
if not draining or not any(e['live'] == '200' for e in draining):
|
||||
raise RuntimeError('liveness must remain 200 while readiness is 503')
|
||||
if any(e['live'] not in {'200', 'connection_error'} for e in draining):
|
||||
raise RuntimeError('liveness returned an unhealthy HTTP status during draining')
|
||||
if not withdrawn_while_running or not any(e['phase'] == 'withdrawn' and e['ready'] == '503' for e in events):
|
||||
raise RuntimeError('LB did not withdraw the replica while its HTTP server was alive')
|
||||
if summary['stopped'].get('200', 0) or summary['stopped'].get('503', 0) or not summary['stopped'].get('connection_error', 0):
|
||||
raise RuntimeError('replica did not close HTTP after draining')
|
||||
if exit_code == 137:
|
||||
raise RuntimeError('container required SIGKILL instead of graceful shutdown')
|
||||
print(json.dumps({'stop_seconds': round(stop_seconds, 3),
|
||||
'lb_withdrawn_seconds': round(withdrawn_at, 3),
|
||||
'lb_withdrawn_before_http_close': True,
|
||||
'readiness_sequence': [200, 503, 'connection_error'],
|
||||
'readiness_503_samples': len(draining),
|
||||
'liveness_200_while_draining': sum(e['live'] == '200' for e in draining),
|
||||
'liveness_connection_errors_while_draining': sum(e['live'] == 'connection_error' for e in draining),
|
||||
'exit_code': exit_code, 'direct_ready_responses': summary}, indent=2))
|
||||
print('PASS: readiness 200 -> 503 -> closed; liveness 200; LB withdrawal before HTTP close')
|
||||
finally:
|
||||
if probe.poll() is None:
|
||||
probe.kill()
|
||||
probe.communicate()
|
||||
if stop is not None and stop.poll() is None:
|
||||
stop.communicate(timeout=40)
|
||||
command('docker', 'start', target)
|
||||
both_ready(args.base_url, expected)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except (RuntimeError, OSError, subprocess.TimeoutExpired) as exc:
|
||||
print(f'FAIL: {exc}', file=sys.stderr)
|
||||
raise SystemExit(1) from None
|
||||
Reference in New Issue
Block a user