Fix nginx routing

This commit is contained in:
2026-10-24 00:43:12 +03:00
parent 3417e22ba0
commit c8fc32e8b1
18 changed files with 442 additions and 221 deletions
+1 -1
View File
@@ -34,7 +34,7 @@ def wait_api(count=2, timeout=60):
while time.monotonic() < deadline:
rows = snapshot('api')
if len(rows) == count and all(r['health'] == 'healthy' and r['state'] == 'running' for r in rows):
time.sleep(3) # Docker DNS validity 2s, plus resolver budget 1s.
time.sleep(3) # Allow the nginx readiness poll (2s) and reload.
return rows
time.sleep(0.5)
raise RuntimeError('API replicas did not become healthy')
+109
View File
@@ -0,0 +1,109 @@
"""Reproduce an alive but unready API, then check exclusion and recovery."""
import json
import sys
import time
import urllib.error
import urllib.request
from uuid import uuid4
from ops import ROOT, compose, docker, restore, save, snapshot, wait_api
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
def wait_for(probe, timeout=30):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if probe():
return
time.sleep(0.5)
raise AssertionError('Readiness transition timed out')
def state(container):
return json.loads(docker('inspect', '--format',
'{"id":{{json .Id}},"instance":{{json .Config.Hostname}},'
'"health":{{json .State.Health.Status}},'
'"ip":{{range .NetworkSettings.Networks}}{{json .IPAddress}}{{end}}}', container))
def backends():
return compose('exec', '-T', 'nginx', 'cat', '/etc/nginx/ready-backends.conf')
def registrations():
instances = {}
client = Client()
for _ in range(24):
response = client.call('POST', '/api/v1/users', 201, body={
'username': 'ready_' + uuid4().hex[:16],
'display_name': 'Readiness test', 'password': 'Readiness-test-password'}, schema='User')
instance = response['headers']['X-Instance-Id']
instances[instance] = instances.get(instance, 0) + 1
return instances
def main():
original = wait_api()
check(len(original) == 2, 'Run with exactly two API replicas')
peer, replaced = original
temporary = None
result = {'before': original}
try:
docker('stop', '--time', '8', replaced['id'])
temporary = compose('run', '-d', '--no-deps', '--use-aliases',
'-e', 'REDIS_URL=redis://127.0.0.1:6399/0', 'api').splitlines()[-1]
wait_for(lambda: state(temporary)['health'] == 'unhealthy')
bad = state(temporary)
result['unready'] = bad
# Call each endpoint inside this exact container, bypassing the LB.
probe = ('import urllib.request, urllib.error; '
'opener = urllib.request.build_opener(urllib.request.ProxyHandler({})); '
'response = opener.open("http://127.0.0.1:8080/health/live"); '
'print(response.status)')
live = docker('exec', temporary, 'python', '-c', probe)
ready_probe = ('import urllib.request, urllib.error\n'
'try:\n'
' print(urllib.request.urlopen("http://127.0.0.1:8080/health/ready").status)\n'
'except urllib.error.HTTPError as error:\n'
' print(error.code)')
ready = docker('exec', temporary, 'python', '-c', ready_probe)
check((live, ready) == ('200', '503'), 'Expected live=200, ready=503')
result['health_http'] = {'live': int(live), 'ready': int(ready)}
result['upstream_when_unready'] = backends()
check(bad['ip'] not in result['upstream_when_unready'],
'Unhealthy backend is still in nginx upstream')
result['registrations_with_unready_peer'] = registrations()
check(result['registrations_with_unready_peer'] == {peer['instance']: 24},
'Registration reached an unready API')
# No ready API must mean an error, not a fallback to the unready one.
docker('stop', '--time', '8', peer['id'])
wait_for(lambda: '127.0.0.1:9 down' in backends(), timeout=10)
try:
urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=5)
raise AssertionError('LB routed to an unready API')
except urllib.error.HTTPError as error:
check(error.code == 502, 'Expected 502 with no ready backend')
result['no_ready_backend_status'] = error.code
docker('rm', '-f', temporary)
temporary = None
restore()
wait_for(lambda: backends().count('max_fails=1') == 2, timeout=10)
result['registrations_after_recovery'] = registrations()
check(set(result['registrations_after_recovery']) == {r['instance'] for r in original},
'Recovered API did not return to load balancing')
result['after'] = snapshot('api')
result['result'] = 'PASS'
save('readiness.json', result)
print(json.dumps(result, indent=2))
finally:
if temporary:
docker('rm', '-f', temporary)
restore()
if __name__ == '__main__':
main()