Files
mtusi2026lab7/scripts/lb_readiness_check.sh

115 lines
5.5 KiB
Bash
Executable File

#!/bin/sh
# Verify LB exclusion of a live API whose native TLS PostgreSQL path has failed.
set -eu
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT_DIR"
exec "${PYTHON:-python3}" - "$@" <<'PY'
from collections import Counter
import csv
import io
import json
import os
from pathlib import Path
import re
import sys
import tempfile
import time
import uuid
sys.path.insert(0, str(Path.cwd() / 'scripts'))
sys.path.insert(0, str(Path.cwd() / 'tests'))
from lab7_environment import Environment, ROOT
from lab7_http import BASE_URL, CA_FILE, open_url
from client import Client
environment = Environment()
client = Client(BASE_URL, CA_FILE, timeout=5)
output = Path(os.environ.get('OUT', 'evidence/lab5/lb-readiness-detail.txt'))
output.parent.mkdir(parents=True, exist_ok=True)
replicas = environment.compose('ps', '-q', 'api').stdout.split()
if len(replicas) < 2:
raise SystemExit('readiness exclusion needs two healthy Compose API replicas')
networks = json.loads(environment.run(['docker', 'inspect', '--format',
'{{json .NetworkSettings.Networks}}', replicas[0]]).stdout)
if len(networks) != 1:
raise SystemExit('expected exactly one API network for the isolated TLS tunnel')
network = next(iter(networks))
prefix = environment.project + '-readiness-' + uuid.uuid4().hex[:8]
proxy, candidate = prefix + '-tunnel', prefix + '-api'
stats_url = os.environ.get('STATS', 'https://localhost:' + environment.value('LB_STATS_PORT', '8404') + '/stats;csv')
log = []
def stats():
with open_url(stats_url, operator=True) as response:
rows = csv.DictReader(io.StringIO(response.read().decode().replace('# ', '', 1)))
return {row['svname']: row['status'] for row in rows
if row['pxname'].startswith('api') and row['svname'] not in ('BACKEND', 'FRONTEND')}
def local_status():
result = environment.run(['docker', 'exec', candidate, 'wget', '-qO-', '-S', '-T', '3',
'http://127.0.0.1:8084/health/ready'], check=False)
matches = re.findall(r'HTTP/\S+\s+(\d{3})', result.stderr)
return int(matches[-1]) if matches else None
def wait(predicate, description, seconds=60):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
value = predicate()
if value:
return value
time.sleep(0.5)
raise RuntimeError(description)
def observed():
return Counter(client.call('GET', '/health/live', schema='Health')['headers']['X-Instance-Id']
for _ in range(int(os.environ.get('REQUESTS', '40'))))
with tempfile.TemporaryDirectory(prefix='lab7-readiness-') as temporary:
try:
baseline = stats()
log.append('baseline instances: ' + json.dumps(observed()))
# This network-only tunnel forwards encrypted PostgreSQL bytes unchanged.
# It has no credentials, TLS termination, API listener or host-published ports.
environment.run(['docker', 'run', '-d', '--name', proxy, '--network', network,
'--cap-drop', 'ALL', '--security-opt', 'no-new-privileges:true', '--read-only',
'alpine/socat@sha256:fb3862c949af7c79996d0e514ee96128644d840835ed5368eeecb14a869de361',
'tcp-listen:5432,fork,reuseaddr', 'tcp-connect:postgres:5432'])
proxy_networks = json.loads(environment.run(['docker', 'inspect', '--format',
'{{json .NetworkSettings.Networks}}', proxy]).stdout)
proxy_ip = proxy_networks[network]['IPAddress']
override = Path(temporary) / 'compose.json'
# Preserve the real postgres hostname/SAN and Bao-supplied role credentials.
override.write_text(json.dumps({'services': {'api': {'extra_hosts': ['postgres:' + proxy_ip]}}}))
environment.compose('-f', str(override), 'run', '-d', '--no-deps', '--use-aliases',
'--name', candidate, 'api')
wait(lambda: local_status() == 200, 'candidate did not become ready')
slot = wait(lambda: next((name for name, status in stats().items()
if status.startswith('UP') and not baseline.get(name, '').startswith('UP')), None),
'candidate did not enter the LB rotation')
hostname = environment.run(['docker', 'inspect', '--format', '{{.Config.Hostname}}', candidate]).stdout.strip()
log.append('candidate ready: instance=' + hostname + ' slot=' + slot)
environment.run(['docker', 'stop', proxy])
wait(lambda: local_status() == 503, 'candidate did not become not-ready')
state = environment.run(['docker', 'inspect', '--format', '{{.State.Status}}', candidate]).stdout.strip()
if state != 'running':
raise RuntimeError('candidate stopped instead of reporting not-ready')
wait(lambda: stats().get(slot, '').startswith('DOWN'), 'LB did not exclude the not-ready candidate')
after = observed()
if hostname in after:
raise RuntimeError('LB routed a request to the not-ready candidate')
alice, _, _, chat = client.bootstrap()
for index in range(10):
client.message(alice, chat, 'readiness-exclusion-' + str(index))
log += ['candidate alive/readiness503; LB slot DOWN', 'during outage instances: ' + json.dumps(after),
'business writes through remaining replicas: 10']
finally:
environment.run(['docker', 'rm', '-f', candidate, proxy], check=False)
output.write_text('\n'.join(log) + '\n')
print('\n'.join(log))
PY