forked from mtusi2026/mtusi2026lab3
110 lines
4.4 KiB
Python
110 lines
4.4 KiB
Python
"""Reproduce an alive but unready API, then check exclusion and recovery."""
|
|
import json
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from uuid import uuid4
|
|
|
|
from ops import ROOT, compose, docker, restore, save, snapshot, wait_api
|
|
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, check
|
|
|
|
|
|
def wait_for(probe, timeout=30):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
if probe():
|
|
return
|
|
time.sleep(0.5)
|
|
raise AssertionError('Readiness transition timed out')
|
|
|
|
|
|
def state(container):
|
|
return json.loads(docker('inspect', '--format',
|
|
'{"id":{{json .Id}},"instance":{{json .Config.Hostname}},'
|
|
'"health":{{json .State.Health.Status}},'
|
|
'"ip":{{range .NetworkSettings.Networks}}{{json .IPAddress}}{{end}}}', container))
|
|
|
|
|
|
def backends():
|
|
return compose('exec', '-T', 'nginx', 'cat', '/etc/nginx/ready-backends.conf')
|
|
|
|
|
|
def registrations():
|
|
instances = {}
|
|
client = Client()
|
|
for _ in range(24):
|
|
response = client.call('POST', '/api/v1/users', 201, body={
|
|
'username': 'ready_' + uuid4().hex[:16],
|
|
'display_name': 'Readiness test', 'password': 'Readiness-test-password'}, schema='User')
|
|
instance = response['headers']['X-Instance-Id']
|
|
instances[instance] = instances.get(instance, 0) + 1
|
|
return instances
|
|
|
|
|
|
def main():
|
|
original = wait_api()
|
|
check(len(original) == 2, 'Run with exactly two API replicas')
|
|
peer, replaced = original
|
|
temporary = None
|
|
result = {'before': original}
|
|
try:
|
|
docker('stop', '--time', '8', replaced['id'])
|
|
temporary = compose('run', '-d', '--no-deps', '--use-aliases',
|
|
'-e', 'REDIS_URL=redis://127.0.0.1:6399/0', 'api').splitlines()[-1]
|
|
wait_for(lambda: state(temporary)['health'] == 'unhealthy')
|
|
bad = state(temporary)
|
|
result['unready'] = bad
|
|
# Call each endpoint inside this exact container, bypassing the LB.
|
|
probe = ('import urllib.request, urllib.error; '
|
|
'opener = urllib.request.build_opener(urllib.request.ProxyHandler({})); '
|
|
'response = opener.open("http://127.0.0.1:8080/health/live"); '
|
|
'print(response.status)')
|
|
live = docker('exec', temporary, 'python', '-c', probe)
|
|
ready_probe = ('import urllib.request, urllib.error\n'
|
|
'try:\n'
|
|
' print(urllib.request.urlopen("http://127.0.0.1:8080/health/ready").status)\n'
|
|
'except urllib.error.HTTPError as error:\n'
|
|
' print(error.code)')
|
|
ready = docker('exec', temporary, 'python', '-c', ready_probe)
|
|
check((live, ready) == ('200', '503'), 'Expected live=200, ready=503')
|
|
result['health_http'] = {'live': int(live), 'ready': int(ready)}
|
|
result['upstream_when_unready'] = backends()
|
|
check(bad['ip'] not in result['upstream_when_unready'],
|
|
'Unhealthy backend is still in nginx upstream')
|
|
result['registrations_with_unready_peer'] = registrations()
|
|
check(result['registrations_with_unready_peer'] == {peer['instance']: 24},
|
|
'Registration reached an unready API')
|
|
|
|
# No ready API must mean an error, not a fallback to the unready one.
|
|
docker('stop', '--time', '8', peer['id'])
|
|
wait_for(lambda: '127.0.0.1:9 down' in backends(), timeout=10)
|
|
try:
|
|
urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=5)
|
|
raise AssertionError('LB routed to an unready API')
|
|
except urllib.error.HTTPError as error:
|
|
check(error.code == 502, 'Expected 502 with no ready backend')
|
|
result['no_ready_backend_status'] = error.code
|
|
|
|
docker('rm', '-f', temporary)
|
|
temporary = None
|
|
restore()
|
|
wait_for(lambda: backends().count('max_fails=1') == 2, timeout=10)
|
|
result['registrations_after_recovery'] = registrations()
|
|
check(set(result['registrations_after_recovery']) == {r['instance'] for r in original},
|
|
'Recovered API did not return to load balancing')
|
|
result['after'] = snapshot('api')
|
|
result['result'] = 'PASS'
|
|
save('readiness.json', result)
|
|
print(json.dumps(result, indent=2))
|
|
finally:
|
|
if temporary:
|
|
docker('rm', '-f', temporary)
|
|
restore()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|