105 lines
3.4 KiB
Python
105 lines
3.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Reproduce and guard against serving business traffic from unready APIs."""
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, Failure, arguments, check, run
|
|
|
|
ORIGINAL_TABLE = 'message_idempotency'
|
|
TEMPORARY_TABLE = 'message_idempotency_readiness_test'
|
|
|
|
|
|
def command(*args):
|
|
result = subprocess.run(
|
|
args,
|
|
cwd=ROOT,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
check=False)
|
|
if result.returncode != 0:
|
|
output = result.stdout.strip()
|
|
raise Failure('command failed: ' + ' '.join(args[:3])
|
|
+ (': ' + output if output else ''))
|
|
return result.stdout.strip()
|
|
|
|
|
|
def rename_table(source, target):
|
|
sql = 'ALTER TABLE ' + source + ' RENAME TO ' + target + ';'
|
|
command(
|
|
'docker', 'compose', 'exec', '-T', 'postgres', 'sh', '-ec',
|
|
'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" '
|
|
'-d "$POSTGRES_DB" -c \'' + sql + '\'')
|
|
|
|
|
|
def api_container_ids():
|
|
output = command('docker', 'compose', 'ps', '-q', 'api')
|
|
return [line.strip() for line in output.splitlines() if line.strip()]
|
|
|
|
|
|
def health_status(container_id):
|
|
return command(
|
|
'docker', 'inspect',
|
|
'--format={{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}',
|
|
container_id)
|
|
|
|
|
|
def wait_for_all_health(expected, timeout):
|
|
deadline = time.monotonic() + timeout
|
|
last = []
|
|
while time.monotonic() < deadline:
|
|
containers = api_container_ids()
|
|
last = [health_status(container) for container in containers]
|
|
if len(containers) >= 2 and all(value == expected for value in last):
|
|
return containers
|
|
time.sleep(1)
|
|
raise Failure('API health did not become ' + expected
|
|
+ '; last statuses: ' + repr(last))
|
|
|
|
|
|
def configure(parser):
|
|
parser.add_argument(
|
|
'--health-timeout', type=float, default=45.0,
|
|
help='seconds to wait for both Docker health states')
|
|
|
|
|
|
def main():
|
|
args = arguments('Readiness must gate business traffic through the LB', configure)
|
|
client = Client(args.base_url, args.ca_file, timeout=8)
|
|
client.call('GET', '/health/ready', 200, schema='Health')
|
|
actor = client.user('readiness_gate')
|
|
|
|
renamed = False
|
|
try:
|
|
rename_table(ORIGINAL_TABLE, TEMPORARY_TABLE)
|
|
renamed = True
|
|
containers = wait_for_all_health('unhealthy', args.health_timeout)
|
|
|
|
client.call('GET', '/health/ready', 503)
|
|
instances = set()
|
|
for _ in range(8):
|
|
response = client.call(
|
|
'GET', '/api/v1/users/me', 503, actor=actor)
|
|
instances.add(response['headers']['X-Instance-Id'])
|
|
check(instances, 'no refusing API instance was observed')
|
|
finally:
|
|
if renamed:
|
|
rename_table(TEMPORARY_TABLE, ORIGINAL_TABLE)
|
|
|
|
wait_for_all_health('healthy', args.health_timeout)
|
|
# Nginx may retain a passive failure for fail_timeout=2s.
|
|
time.sleep(3)
|
|
client.call('GET', '/health/ready', 200, schema='Health')
|
|
client.call('GET', '/api/v1/users/me', 200, actor=actor, schema='User')
|
|
print('PASS: ' + str(len(containers))
|
|
+ ' unhealthy API containers refused business traffic with 503 '
|
|
+ 'and recovered after schema restoration')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|