forked from mtusi2026/mtusi2026lab3
Fix nginx routing
This commit is contained in:
+1
-1
@@ -34,7 +34,7 @@ def wait_api(count=2, timeout=60):
|
||||
while time.monotonic() < deadline:
|
||||
rows = snapshot('api')
|
||||
if len(rows) == count and all(r['health'] == 'healthy' and r['state'] == 'running' for r in rows):
|
||||
time.sleep(3) # Docker DNS validity 2s, plus resolver budget 1s.
|
||||
time.sleep(3) # Allow the nginx readiness poll (2s) and reload.
|
||||
return rows
|
||||
time.sleep(0.5)
|
||||
raise RuntimeError('API replicas did not become healthy')
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Reproduce an alive but unready API, then check exclusion and recovery."""
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from uuid import uuid4
|
||||
|
||||
from ops import ROOT, compose, docker, restore, save, snapshot, wait_api
|
||||
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, check
|
||||
|
||||
|
||||
def wait_for(probe, timeout=30):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if probe():
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise AssertionError('Readiness transition timed out')
|
||||
|
||||
|
||||
def state(container):
|
||||
return json.loads(docker('inspect', '--format',
|
||||
'{"id":{{json .Id}},"instance":{{json .Config.Hostname}},'
|
||||
'"health":{{json .State.Health.Status}},'
|
||||
'"ip":{{range .NetworkSettings.Networks}}{{json .IPAddress}}{{end}}}', container))
|
||||
|
||||
|
||||
def backends():
|
||||
return compose('exec', '-T', 'nginx', 'cat', '/etc/nginx/ready-backends.conf')
|
||||
|
||||
|
||||
def registrations():
|
||||
instances = {}
|
||||
client = Client()
|
||||
for _ in range(24):
|
||||
response = client.call('POST', '/api/v1/users', 201, body={
|
||||
'username': 'ready_' + uuid4().hex[:16],
|
||||
'display_name': 'Readiness test', 'password': 'Readiness-test-password'}, schema='User')
|
||||
instance = response['headers']['X-Instance-Id']
|
||||
instances[instance] = instances.get(instance, 0) + 1
|
||||
return instances
|
||||
|
||||
|
||||
def main():
|
||||
original = wait_api()
|
||||
check(len(original) == 2, 'Run with exactly two API replicas')
|
||||
peer, replaced = original
|
||||
temporary = None
|
||||
result = {'before': original}
|
||||
try:
|
||||
docker('stop', '--time', '8', replaced['id'])
|
||||
temporary = compose('run', '-d', '--no-deps', '--use-aliases',
|
||||
'-e', 'REDIS_URL=redis://127.0.0.1:6399/0', 'api').splitlines()[-1]
|
||||
wait_for(lambda: state(temporary)['health'] == 'unhealthy')
|
||||
bad = state(temporary)
|
||||
result['unready'] = bad
|
||||
# Call each endpoint inside this exact container, bypassing the LB.
|
||||
probe = ('import urllib.request, urllib.error; '
|
||||
'opener = urllib.request.build_opener(urllib.request.ProxyHandler({})); '
|
||||
'response = opener.open("http://127.0.0.1:8080/health/live"); '
|
||||
'print(response.status)')
|
||||
live = docker('exec', temporary, 'python', '-c', probe)
|
||||
ready_probe = ('import urllib.request, urllib.error\n'
|
||||
'try:\n'
|
||||
' print(urllib.request.urlopen("http://127.0.0.1:8080/health/ready").status)\n'
|
||||
'except urllib.error.HTTPError as error:\n'
|
||||
' print(error.code)')
|
||||
ready = docker('exec', temporary, 'python', '-c', ready_probe)
|
||||
check((live, ready) == ('200', '503'), 'Expected live=200, ready=503')
|
||||
result['health_http'] = {'live': int(live), 'ready': int(ready)}
|
||||
result['upstream_when_unready'] = backends()
|
||||
check(bad['ip'] not in result['upstream_when_unready'],
|
||||
'Unhealthy backend is still in nginx upstream')
|
||||
result['registrations_with_unready_peer'] = registrations()
|
||||
check(result['registrations_with_unready_peer'] == {peer['instance']: 24},
|
||||
'Registration reached an unready API')
|
||||
|
||||
# No ready API must mean an error, not a fallback to the unready one.
|
||||
docker('stop', '--time', '8', peer['id'])
|
||||
wait_for(lambda: '127.0.0.1:9 down' in backends(), timeout=10)
|
||||
try:
|
||||
urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=5)
|
||||
raise AssertionError('LB routed to an unready API')
|
||||
except urllib.error.HTTPError as error:
|
||||
check(error.code == 502, 'Expected 502 with no ready backend')
|
||||
result['no_ready_backend_status'] = error.code
|
||||
|
||||
docker('rm', '-f', temporary)
|
||||
temporary = None
|
||||
restore()
|
||||
wait_for(lambda: backends().count('max_fails=1') == 2, timeout=10)
|
||||
result['registrations_after_recovery'] = registrations()
|
||||
check(set(result['registrations_after_recovery']) == {r['instance'] for r in original},
|
||||
'Recovered API did not return to load balancing')
|
||||
result['after'] = snapshot('api')
|
||||
result['result'] = 'PASS'
|
||||
save('readiness.json', result)
|
||||
print(json.dumps(result, indent=2))
|
||||
finally:
|
||||
if temporary:
|
||||
docker('rm', '-f', temporary)
|
||||
restore()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user