Files
mtusi2026lab3/scripts/verify_readiness.py
T
2026-10-24 00:43:12 +03:00

110 lines
4.4 KiB
Python

"""Reproduce an alive but unready API, then check exclusion and recovery."""
import json
import sys
import time
import urllib.error
import urllib.request
from uuid import uuid4
from ops import ROOT, compose, docker, restore, save, snapshot, wait_api
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
def wait_for(probe, timeout=30):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if probe():
return
time.sleep(0.5)
raise AssertionError('Readiness transition timed out')
def state(container):
return json.loads(docker('inspect', '--format',
'{"id":{{json .Id}},"instance":{{json .Config.Hostname}},'
'"health":{{json .State.Health.Status}},'
'"ip":{{range .NetworkSettings.Networks}}{{json .IPAddress}}{{end}}}', container))
def backends():
return compose('exec', '-T', 'nginx', 'cat', '/etc/nginx/ready-backends.conf')
def registrations():
instances = {}
client = Client()
for _ in range(24):
response = client.call('POST', '/api/v1/users', 201, body={
'username': 'ready_' + uuid4().hex[:16],
'display_name': 'Readiness test', 'password': 'Readiness-test-password'}, schema='User')
instance = response['headers']['X-Instance-Id']
instances[instance] = instances.get(instance, 0) + 1
return instances
def main():
original = wait_api()
check(len(original) == 2, 'Run with exactly two API replicas')
peer, replaced = original
temporary = None
result = {'before': original}
try:
docker('stop', '--time', '8', replaced['id'])
temporary = compose('run', '-d', '--no-deps', '--use-aliases',
'-e', 'REDIS_URL=redis://127.0.0.1:6399/0', 'api').splitlines()[-1]
wait_for(lambda: state(temporary)['health'] == 'unhealthy')
bad = state(temporary)
result['unready'] = bad
# Call each endpoint inside this exact container, bypassing the LB.
probe = ('import urllib.request, urllib.error; '
'opener = urllib.request.build_opener(urllib.request.ProxyHandler({})); '
'response = opener.open("http://127.0.0.1:8080/health/live"); '
'print(response.status)')
live = docker('exec', temporary, 'python', '-c', probe)
ready_probe = ('import urllib.request, urllib.error\n'
'try:\n'
' print(urllib.request.urlopen("http://127.0.0.1:8080/health/ready").status)\n'
'except urllib.error.HTTPError as error:\n'
' print(error.code)')
ready = docker('exec', temporary, 'python', '-c', ready_probe)
check((live, ready) == ('200', '503'), 'Expected live=200, ready=503')
result['health_http'] = {'live': int(live), 'ready': int(ready)}
result['upstream_when_unready'] = backends()
check(bad['ip'] not in result['upstream_when_unready'],
'Unhealthy backend is still in nginx upstream')
result['registrations_with_unready_peer'] = registrations()
check(result['registrations_with_unready_peer'] == {peer['instance']: 24},
'Registration reached an unready API')
# No ready API must mean an error, not a fallback to the unready one.
docker('stop', '--time', '8', peer['id'])
wait_for(lambda: '127.0.0.1:9 down' in backends(), timeout=10)
try:
urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=5)
raise AssertionError('LB routed to an unready API')
except urllib.error.HTTPError as error:
check(error.code == 502, 'Expected 502 with no ready backend')
result['no_ready_backend_status'] = error.code
docker('rm', '-f', temporary)
temporary = None
restore()
wait_for(lambda: backends().count('max_fails=1') == 2, timeout=10)
result['registrations_after_recovery'] = registrations()
check(set(result['registrations_after_recovery']) == {r['instance'] for r in original},
'Recovered API did not return to load balancing')
result['after'] = snapshot('api')
result['result'] = 'PASS'
save('readiness.json', result)
print(json.dumps(result, indent=2))
finally:
if temporary:
docker('rm', '-f', temporary)
restore()
if __name__ == '__main__':
main()