forked from mtusi2026/mtusi2026lab7
115 lines
5.5 KiB
Bash
Executable File
115 lines
5.5 KiB
Bash
Executable File
#!/bin/sh
|
|
# Verify LB exclusion of a live API whose native TLS PostgreSQL path has failed.
|
|
set -eu
|
|
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
|
cd "$ROOT_DIR"
|
|
exec "${PYTHON:-python3}" - "$@" <<'PY'
|
|
from collections import Counter
|
|
import csv
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import uuid
|
|
|
|
sys.path.insert(0, str(Path.cwd() / 'scripts'))
|
|
sys.path.insert(0, str(Path.cwd() / 'tests'))
|
|
from lab7_environment import Environment, ROOT
|
|
from lab7_http import BASE_URL, CA_FILE, open_url
|
|
from client import Client
|
|
|
|
environment = Environment()
|
|
client = Client(BASE_URL, CA_FILE, timeout=5)
|
|
output = Path(os.environ.get('OUT', 'evidence/lab5/lb-readiness-detail.txt'))
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
replicas = environment.compose('ps', '-q', 'api').stdout.split()
|
|
if len(replicas) < 2:
|
|
raise SystemExit('readiness exclusion needs two healthy Compose API replicas')
|
|
networks = json.loads(environment.run(['docker', 'inspect', '--format',
|
|
'{{json .NetworkSettings.Networks}}', replicas[0]]).stdout)
|
|
if len(networks) != 1:
|
|
raise SystemExit('expected exactly one API network for the isolated TLS tunnel')
|
|
network = next(iter(networks))
|
|
prefix = environment.project + '-readiness-' + uuid.uuid4().hex[:8]
|
|
proxy, candidate = prefix + '-tunnel', prefix + '-api'
|
|
stats_url = os.environ.get('STATS', 'https://localhost:' + environment.value('LB_STATS_PORT', '8404') + '/stats;csv')
|
|
log = []
|
|
|
|
|
|
def stats():
|
|
with open_url(stats_url, operator=True) as response:
|
|
rows = csv.DictReader(io.StringIO(response.read().decode().replace('# ', '', 1)))
|
|
return {row['svname']: row['status'] for row in rows
|
|
if row['pxname'].startswith('api') and row['svname'] not in ('BACKEND', 'FRONTEND')}
|
|
|
|
|
|
def local_status():
|
|
result = environment.run(['docker', 'exec', candidate, 'wget', '-qO-', '-S', '-T', '3',
|
|
'http://127.0.0.1:8084/health/ready'], check=False)
|
|
matches = re.findall(r'HTTP/\S+\s+(\d{3})', result.stderr)
|
|
return int(matches[-1]) if matches else None
|
|
|
|
|
|
def wait(predicate, description, seconds=60):
|
|
deadline = time.monotonic() + seconds
|
|
while time.monotonic() < deadline:
|
|
value = predicate()
|
|
if value:
|
|
return value
|
|
time.sleep(0.5)
|
|
raise RuntimeError(description)
|
|
|
|
|
|
def observed():
|
|
return Counter(client.call('GET', '/health/live', schema='Health')['headers']['X-Instance-Id']
|
|
for _ in range(int(os.environ.get('REQUESTS', '40'))))
|
|
|
|
|
|
with tempfile.TemporaryDirectory(prefix='lab7-readiness-') as temporary:
|
|
try:
|
|
baseline = stats()
|
|
log.append('baseline instances: ' + json.dumps(observed()))
|
|
# This network-only tunnel forwards encrypted PostgreSQL bytes unchanged.
|
|
# It has no credentials, TLS termination, API listener or host-published ports.
|
|
environment.run(['docker', 'run', '-d', '--name', proxy, '--network', network,
|
|
'--cap-drop', 'ALL', '--security-opt', 'no-new-privileges:true', '--read-only',
|
|
'alpine/socat@sha256:fb3862c949af7c79996d0e514ee96128644d840835ed5368eeecb14a869de361',
|
|
'tcp-listen:5432,fork,reuseaddr', 'tcp-connect:postgres:5432'])
|
|
proxy_networks = json.loads(environment.run(['docker', 'inspect', '--format',
|
|
'{{json .NetworkSettings.Networks}}', proxy]).stdout)
|
|
proxy_ip = proxy_networks[network]['IPAddress']
|
|
override = Path(temporary) / 'compose.json'
|
|
# Preserve the real postgres hostname/SAN and Bao-supplied role credentials.
|
|
override.write_text(json.dumps({'services': {'api': {'extra_hosts': ['postgres:' + proxy_ip]}}}))
|
|
environment.compose('-f', str(override), 'run', '-d', '--no-deps', '--use-aliases',
|
|
'--name', candidate, 'api')
|
|
wait(lambda: local_status() == 200, 'candidate did not become ready')
|
|
slot = wait(lambda: next((name for name, status in stats().items()
|
|
if status.startswith('UP') and not baseline.get(name, '').startswith('UP')), None),
|
|
'candidate did not enter the LB rotation')
|
|
hostname = environment.run(['docker', 'inspect', '--format', '{{.Config.Hostname}}', candidate]).stdout.strip()
|
|
log.append('candidate ready: instance=' + hostname + ' slot=' + slot)
|
|
environment.run(['docker', 'stop', proxy])
|
|
wait(lambda: local_status() == 503, 'candidate did not become not-ready')
|
|
state = environment.run(['docker', 'inspect', '--format', '{{.State.Status}}', candidate]).stdout.strip()
|
|
if state != 'running':
|
|
raise RuntimeError('candidate stopped instead of reporting not-ready')
|
|
wait(lambda: stats().get(slot, '').startswith('DOWN'), 'LB did not exclude the not-ready candidate')
|
|
after = observed()
|
|
if hostname in after:
|
|
raise RuntimeError('LB routed a request to the not-ready candidate')
|
|
alice, _, _, chat = client.bootstrap()
|
|
for index in range(10):
|
|
client.message(alice, chat, 'readiness-exclusion-' + str(index))
|
|
log += ['candidate alive/readiness503; LB slot DOWN', 'during outage instances: ' + json.dumps(after),
|
|
'business writes through remaining replicas: 10']
|
|
finally:
|
|
environment.run(['docker', 'rm', '-f', candidate, proxy], check=False)
|
|
output.write_text('\n'.join(log) + '\n')
|
|
print('\n'.join(log))
|
|
PY
|