"""Reproduce an alive but unready API, then check exclusion and recovery.""" import json import sys import time import urllib.error import urllib.request from uuid import uuid4 from ops import ROOT, compose, docker, restore, save, snapshot, wait_api sys.path.insert(0, str(ROOT / 'tests')) from client import Client, check def wait_for(probe, timeout=30): deadline = time.monotonic() + timeout while time.monotonic() < deadline: if probe(): return time.sleep(0.5) raise AssertionError('Readiness transition timed out') def state(container): return json.loads(docker('inspect', '--format', '{"id":{{json .Id}},"instance":{{json .Config.Hostname}},' '"health":{{json .State.Health.Status}},' '"ip":{{range .NetworkSettings.Networks}}{{json .IPAddress}}{{end}}}', container)) def backends(): return compose('exec', '-T', 'nginx', 'cat', '/etc/nginx/ready-backends.conf') def registrations(): instances = {} client = Client() for _ in range(24): response = client.call('POST', '/api/v1/users', 201, body={ 'username': 'ready_' + uuid4().hex[:16], 'display_name': 'Readiness test', 'password': 'Readiness-test-password'}, schema='User') instance = response['headers']['X-Instance-Id'] instances[instance] = instances.get(instance, 0) + 1 return instances def main(): original = wait_api() check(len(original) == 2, 'Run with exactly two API replicas') peer, replaced = original temporary = None result = {'before': original} try: docker('stop', '--time', '8', replaced['id']) temporary = compose('run', '-d', '--no-deps', '--use-aliases', '-e', 'REDIS_URL=redis://127.0.0.1:6399/0', 'api').splitlines()[-1] wait_for(lambda: state(temporary)['health'] == 'unhealthy') bad = state(temporary) result['unready'] = bad # Call each endpoint inside this exact container, bypassing the LB. probe = ('import urllib.request, urllib.error; ' 'opener = urllib.request.build_opener(urllib.request.ProxyHandler({})); ' 'response = opener.open("http://127.0.0.1:8080/health/live"); ' 'print(response.status)') live = docker('exec', temporary, 'python', '-c', probe) ready_probe = ('import urllib.request, urllib.error\n' 'try:\n' ' print(urllib.request.urlopen("http://127.0.0.1:8080/health/ready").status)\n' 'except urllib.error.HTTPError as error:\n' ' print(error.code)') ready = docker('exec', temporary, 'python', '-c', ready_probe) check((live, ready) == ('200', '503'), 'Expected live=200, ready=503') result['health_http'] = {'live': int(live), 'ready': int(ready)} result['upstream_when_unready'] = backends() check(bad['ip'] not in result['upstream_when_unready'], 'Unhealthy backend is still in nginx upstream') result['registrations_with_unready_peer'] = registrations() check(result['registrations_with_unready_peer'] == {peer['instance']: 24}, 'Registration reached an unready API') # No ready API must mean an error, not a fallback to the unready one. docker('stop', '--time', '8', peer['id']) wait_for(lambda: '127.0.0.1:9 down' in backends(), timeout=10) try: urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=5) raise AssertionError('LB routed to an unready API') except urllib.error.HTTPError as error: check(error.code == 502, 'Expected 502 with no ready backend') result['no_ready_backend_status'] = error.code docker('rm', '-f', temporary) temporary = None restore() wait_for(lambda: backends().count('max_fails=1') == 2, timeout=10) result['registrations_after_recovery'] = registrations() check(set(result['registrations_after_recovery']) == {r['instance'] for r in original}, 'Recovered API did not return to load balancing') result['after'] = snapshot('api') result['result'] = 'PASS' save('readiness.json', result) print(json.dumps(result, indent=2)) finally: if temporary: docker('rm', '-f', temporary) restore() if __name__ == '__main__': main()