forked from mtusi2026/mtusi2026lab3
801 lines
38 KiB
Python
Executable File
801 lines
38 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Infrastructure checks for lab 3 that need control over the compose stand.
|
|
|
|
python -X utf8 scripts/lab3_checks.py --output evidence/lab3-checks.txt
|
|
python -X utf8 scripts/lab3_checks.py --only failover experiment rolling
|
|
|
|
The stand must already be running (docker compose up -d --build --wait). Every command is
|
|
printed together with its output. Sections stop, kill and start containers, but never delete
|
|
volumes, never flush Valkey and never print secrets or session tokens. All API traffic goes
|
|
through the load balancer on 127.0.0.1:8080, as a client would send it.
|
|
Standard library only; works on Windows, Linux and macOS (-X utf8 is needed on Windows only
|
|
because the public tests read the contract in the default encoding).
|
|
"""
|
|
import argparse
|
|
import base64
|
|
import concurrent.futures
|
|
import csv
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
BASE = 'http://127.0.0.1:8080'
|
|
STATS = 'http://127.0.0.1:8404/;csv'
|
|
PROJECT = os.environ.get('COMPOSE_PROJECT_NAME', 'mtusi-lab3')
|
|
VALKEY_CLI = 'valkey-cli --no-auth-warning -a "$(sed -n "s/^requirepass //p" /run/secrets/valkey_auth)" '
|
|
PSQL = ['exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-v', 'ON_ERROR_STOP=1', '-At', '-c']
|
|
PY = [sys.executable, '-X', 'utf8']
|
|
|
|
lines = []
|
|
results = []
|
|
|
|
|
|
class Stop(Exception):
|
|
pass
|
|
|
|
|
|
def out(text=''):
|
|
lines.append(text)
|
|
print(text, flush=True)
|
|
|
|
|
|
def run(command, check=True, quiet=False, env=None):
|
|
"""Run a command and echo it with its output; return (output, exit code, seconds)."""
|
|
out('$ ' + ' '.join(command))
|
|
started = time.monotonic()
|
|
done = subprocess.run(command, cwd=str(ROOT), env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
elapsed = time.monotonic() - started
|
|
text = done.stdout.decode('utf-8', 'replace').strip()
|
|
if text and not quiet:
|
|
out(text)
|
|
if check and done.returncode != 0:
|
|
if quiet and text:
|
|
out(text)
|
|
raise Stop('command failed with exit code %d' % done.returncode)
|
|
return text, done.returncode, elapsed
|
|
|
|
|
|
def silent(command):
|
|
"""Run a command without echo; return its stdout."""
|
|
return subprocess.run(command, cwd=str(ROOT), stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True).stdout.strip()
|
|
|
|
|
|
def compose(*args, **kwargs):
|
|
return run(['docker', 'compose'] + list(args), **kwargs)
|
|
|
|
|
|
def psql(sql, quiet=False):
|
|
if quiet:
|
|
return silent(['docker', 'compose'] + PSQL + [sql])
|
|
return compose(*(PSQL + [sql]))[0]
|
|
|
|
|
|
def valkey_cli(command):
|
|
return compose('exec', '-T', 'valkey', 'sh', '-c', VALKEY_CLI + command)[0]
|
|
|
|
|
|
def verdict(name, ok, detail=''):
|
|
results.append((name, ok))
|
|
out('=> %s: %s%s' % ('PASS' if ok else 'FAIL', name, ' (' + detail + ')' if detail else ''))
|
|
out()
|
|
|
|
|
|
def http(method, path, token=None, body=None, headers=None, timeout=10):
|
|
"""(status, parsed JSON or None, seconds, X-Instance-Id); status None when the connection fails."""
|
|
all_headers = dict(headers or {})
|
|
if token:
|
|
all_headers['Authorization'] = 'Bearer ' + token
|
|
data = None
|
|
if body is not None:
|
|
data = json.dumps(body).encode('utf-8')
|
|
all_headers['Content-Type'] = 'application/json'
|
|
request = urllib.request.Request(BASE + path, data=data, headers=all_headers, method=method)
|
|
started = time.monotonic()
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
raw, status, instance = response.read(), response.status, response.headers.get('X-Instance-Id')
|
|
except urllib.error.HTTPError as error:
|
|
with error:
|
|
raw, status, instance = error.read(), error.code, error.headers.get('X-Instance-Id')
|
|
except OSError:
|
|
return None, None, time.monotonic() - started, None
|
|
try:
|
|
payload = json.loads(raw) if raw else None
|
|
except ValueError:
|
|
payload = None
|
|
return status, payload, time.monotonic() - started, instance
|
|
|
|
|
|
def basic(username, password):
|
|
return {'Authorization': 'Basic ' + base64.b64encode((username + ':' + password).encode()).decode()}
|
|
|
|
|
|
def new_user(label='check'):
|
|
"""Register a synthetic user and log in: (user, password, token). The token is never printed."""
|
|
username = label + '_' + uuid.uuid4().hex[:12]
|
|
password = 'Check-' + uuid.uuid4().hex
|
|
status, user, _, _ = http('POST', '/api/v1/users', body={'username': username, 'display_name': 'Check', 'password': password})
|
|
if status != 201:
|
|
raise Stop('registration failed with %s' % status)
|
|
status, session, _, _ = http('POST', '/api/v1/auth/sessions', headers=basic(username, password))
|
|
if status != 201:
|
|
raise Stop('login failed with %s' % status)
|
|
return user, password, session['token']
|
|
|
|
|
|
def post_message(token, chat_id, text, key):
|
|
return http('POST', '/api/v1/chats/%s/messages' % chat_id, token, {'text': text}, {'Idempotency-Key': key}, timeout=5)
|
|
|
|
|
|
def history(token, chat_id):
|
|
items, cursor = [], None
|
|
while True:
|
|
path = '/api/v1/chats/%s/messages?limit=100' % chat_id + ('&cursor=' + cursor if cursor else '')
|
|
status, page, _, _ = http('GET', path, token)
|
|
if status != 200:
|
|
raise Stop('history read failed with %s' % status)
|
|
items += page['items']
|
|
cursor = page['next_cursor']
|
|
if not cursor:
|
|
return items
|
|
|
|
|
|
def wait_ready(seconds=60):
|
|
deadline = time.monotonic() + seconds
|
|
while time.monotonic() < deadline:
|
|
if http('GET', '/health/ready', timeout=3)[0] == 200:
|
|
return True
|
|
time.sleep(0.5)
|
|
return False
|
|
|
|
|
|
def api_replicas(all_states=False):
|
|
"""{short container id: container name} of the api service of this project."""
|
|
command = ['docker', 'ps', '--filter', 'label=com.docker.compose.project=' + PROJECT,
|
|
'--filter', 'label=com.docker.compose.service=api', '--format', '{{.ID}} {{.Names}}']
|
|
if all_states:
|
|
command.insert(2, '-a')
|
|
return dict(line.split(' ', 1) for line in silent(command).splitlines() if line)
|
|
|
|
|
|
def container(service):
|
|
return silent(['docker', 'compose', 'ps', '-a', '-q', service])
|
|
|
|
|
|
def inspect(target, template):
|
|
return silent(['docker', 'inspect', '-f', template, target])
|
|
|
|
|
|
def wait_replicas(count, seconds=90):
|
|
"""Wait until `count` api replicas are healthy and all of them answer through the load balancer."""
|
|
deadline = time.monotonic() + seconds
|
|
while time.monotonic() < deadline:
|
|
replicas = api_replicas()
|
|
healthy = [r for r in replicas if inspect(r, '{{.State.Health.Status}}') == 'healthy']
|
|
if len(healthy) == count:
|
|
seen = {http('GET', '/health/live', timeout=3)[3] for _ in range(4 * count)}
|
|
if set(healthy) <= seen:
|
|
return True
|
|
time.sleep(1)
|
|
return False
|
|
|
|
|
|
def show(label, result):
|
|
status, payload, seconds, instance = result
|
|
code = payload['error']['code'] if isinstance(payload, dict) and 'error' in payload else ''
|
|
out('%-58s -> %s %s in %.2f s, X-Instance-Id %s' % (label, status, code, seconds, instance or '-'))
|
|
return status
|
|
|
|
|
|
class StatsPoller(threading.Thread):
|
|
"""Polls the HAProxy stats CSV every 100 ms and records when each server slot changes state.
|
|
|
|
A slot (api1..api8) is matched to a container once, by its IP address while it is UP.
|
|
A stopped replica first turns DOWN (failed checks); a killed one usually goes to
|
|
MAINT (resolution) first, because Docker DNS drops its address and the slot loses it.
|
|
"""
|
|
|
|
def __init__(self):
|
|
super().__init__(daemon=True)
|
|
self.stop_event = threading.Event()
|
|
self.changes = [] # (monotonic time, slot, new status)
|
|
|
|
@staticmethod
|
|
def snapshot():
|
|
"""{slot: (status, ip)} for the api backend."""
|
|
try:
|
|
with urllib.request.urlopen(STATS, timeout=1) as response:
|
|
text = response.read().decode('utf-8').lstrip('# ')
|
|
except OSError:
|
|
return {}
|
|
return {row['svname']: (row['status'].split(' ')[0], row.get('addr', '').split(':')[0])
|
|
for row in csv.DictReader(io.StringIO(text))
|
|
if row.get('pxname') == 'api' and row.get('svname') not in ('FRONTEND', 'BACKEND')}
|
|
|
|
def slot_of(self, address):
|
|
for slot, (status, ip) in self.snapshot().items():
|
|
if ip == address:
|
|
return slot
|
|
return None
|
|
|
|
def run(self):
|
|
last = {}
|
|
while not self.stop_event.is_set():
|
|
now = time.monotonic()
|
|
for slot, (status, _) in self.snapshot().items():
|
|
if last.get(slot) != status:
|
|
self.changes.append((now, slot, status))
|
|
last[slot] = status
|
|
time.sleep(0.1)
|
|
|
|
def first_change(self, slot, statuses, after):
|
|
for moment, changed, status in self.changes:
|
|
if changed == slot and status in statuses and moment >= after:
|
|
return moment
|
|
return None
|
|
|
|
|
|
class Load:
|
|
"""Closed-loop load through the load balancer: each worker posts a message and reads /users/me.
|
|
|
|
A failed POST is retried with the SAME Idempotency-Key and body, exactly like a careful client,
|
|
so the final history must contain every acknowledged message once and no duplicates.
|
|
"""
|
|
|
|
def __init__(self, workers=6):
|
|
self.user, _, self.token = new_user(label='load')
|
|
self.chat = http('POST', '/api/v1/chats', self.token, {'title': 'Load', 'member_ids': [self.user['id']]})[1]
|
|
self.workers = workers
|
|
self.events = [] # (monotonic time, ok, instance, kind)
|
|
self.acknowledged = []
|
|
self.lock = threading.Lock()
|
|
self.stop_event = threading.Event()
|
|
self.threads = []
|
|
|
|
def worker(self, index):
|
|
while not self.stop_event.is_set():
|
|
key = uuid.uuid4().hex
|
|
text = 'load %d %s' % (index, key)
|
|
while not self.stop_event.is_set():
|
|
status, payload, _, instance = post_message(self.token, self.chat['id'], text, key)
|
|
with self.lock:
|
|
self.events.append((time.monotonic(), status == 201, instance, 'write'))
|
|
if status == 201:
|
|
self.acknowledged.append(payload['id'])
|
|
if status == 201:
|
|
break
|
|
time.sleep(0.05)
|
|
status, _, _, instance = http('GET', '/api/v1/users/me', self.token, timeout=5)
|
|
with self.lock:
|
|
self.events.append((time.monotonic(), status == 200, instance, 'read'))
|
|
|
|
def start(self):
|
|
for index in range(self.workers):
|
|
thread = threading.Thread(target=self.worker, args=(index,), daemon=True)
|
|
thread.start()
|
|
self.threads.append(thread)
|
|
|
|
def stop(self):
|
|
self.stop_event.set()
|
|
for thread in self.threads:
|
|
thread.join(timeout=10)
|
|
|
|
def window(self, start, end):
|
|
with self.lock:
|
|
selected = [e for e in self.events if start <= e[0] < end]
|
|
ok = sum(1 for e in selected if e[1])
|
|
return {'requests': len(selected), 'ok': ok, 'errors': len(selected) - ok,
|
|
'ok_rps': round(ok / (end - start), 1) if end > start else 0.0}
|
|
|
|
def max_gap(self, start, end):
|
|
"""Longest interval without a successful request."""
|
|
with self.lock:
|
|
moments = sorted(e[0] for e in self.events if e[1] and start <= e[0] < end)
|
|
points = [start] + moments + [end]
|
|
return max(b - a for a, b in zip(points, points[1:]))
|
|
|
|
def last_served_by(self, instance, after):
|
|
with self.lock:
|
|
moments = [e[0] for e in self.events if e[2] == instance and e[0] >= after]
|
|
return max(moments) if moments else None
|
|
|
|
def first_served_by(self, instance, after):
|
|
with self.lock:
|
|
moments = [e[0] for e in self.events if e[2] == instance and e[0] >= after]
|
|
return min(moments) if moments else None
|
|
|
|
def verify(self):
|
|
"""(acknowledged, in history, duplicates by text) for the load chat."""
|
|
items = history(self.token, self.chat['id'])
|
|
texts = [m['text'] for m in items]
|
|
ids = {m['id'] for m in items}
|
|
return len(set(self.acknowledged)), len(set(self.acknowledged) & ids), len(texts) - len(set(texts))
|
|
|
|
|
|
# --- sections ---------------------------------------------------------------
|
|
|
|
def section_stack():
|
|
out('== Stack: replicas, pinned images, health, limits, published ports, secrets ==')
|
|
compose('ps', '-a', '--format', 'table {{.Name}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}')
|
|
config = json.loads(compose('config', '--format', 'json', quiet=True)[0])
|
|
services = config['services']
|
|
images = {name: services[name].get('image', '') for name in ('lb', 'postgres', 'valkey')}
|
|
verdict('third-party images are pinned by tag and digest', all('@sha256:' in image for image in images.values()),
|
|
', '.join(images.values()))
|
|
replicas = api_replicas()
|
|
names = sorted(replicas.values())
|
|
out('api replicas: ' + ', '.join('%s (%s)' % (name, cid) for cid, name in sorted(replicas.items(), key=lambda x: x[1])))
|
|
no_container_name = not any('container_name' in service for service in services.values())
|
|
verdict('at least two api replicas of one compose service, no container_name', len(replicas) >= 2 and no_container_name,
|
|
', '.join(names))
|
|
health = {name: inspect(cid, '{{.State.Health.Status}}') for cid, name in replicas.items()}
|
|
for service in ('postgres', 'valkey'):
|
|
health[service] = inspect(container(service), '{{.State.Health.Status}}')
|
|
out('health: ' + ', '.join('%s=%s' % item for item in sorted(health.items())))
|
|
verdict('all api replicas, postgres and valkey are healthy', set(health.values()) == {'healthy'})
|
|
users = {name: silent(['docker', 'exec', cid, 'id', '-u']) for cid, name in replicas.items()}
|
|
verdict('API processes do not run as root', set(users.values()) == {'10001'}, ', '.join('%s uid=%s' % i for i in sorted(users.items())))
|
|
out('%-26s %8s %10s %s' % ('container', 'cpus', 'memory', 'published ports'))
|
|
limited, bypass = True, []
|
|
targets = [(name, cid) for cid, name in replicas.items()] + [(s, container(s)) for s in ('lb', 'postgres', 'valkey')]
|
|
for name, cid in sorted(targets):
|
|
cpus = int(inspect(cid, '{{.HostConfig.NanoCpus}}') or 0) / 1e9
|
|
memory = int(inspect(cid, '{{.HostConfig.Memory}}') or 0) // (1024 * 1024)
|
|
ports = inspect(cid, '{{range $p, $b := .HostConfig.PortBindings}}{{range $b}}{{.HostIp}}:{{.HostPort}}->{{$p}} {{end}}{{end}}')
|
|
out('%-26s %8.2f %8d MB %s' % (name, cpus, memory, ports or '-'))
|
|
limited = limited and cpus > 0 and memory > 0
|
|
if ports and name != 'lb':
|
|
bypass.append(name)
|
|
if ports and any(not part.startswith('127.0.0.1:') for part in ports.split()):
|
|
bypass.append(name + ' (not 127.0.0.1)')
|
|
verdict('every container has CPU and memory limits', limited)
|
|
verdict('only the load balancer publishes ports, and only on 127.0.0.1', not bypass, ', '.join(bypass))
|
|
secrets = {name: sorted(s['source'] for s in service.get('secrets', [])) for name, service in services.items()}
|
|
out('secrets per service: ' + '; '.join('%s=%s' % (k, ','.join(v) or '-') for k, v in sorted(secrets.items())))
|
|
verdict('secrets are given only where needed', secrets.get('lb') == [] and secrets['api'] == ['cache_url', 'database_url']
|
|
and secrets['postgres'] == ['postgres_password'] and secrets['valkey'] == ['valkey_auth'])
|
|
aof = ' '.join(valkey_cli('CONFIG GET ' + name) for name in ('appendonly', 'appendfsync', 'maxmemory-policy'))
|
|
verdict('Valkey persistence is AOF with fsync always, no eviction', all(w in aof.split() for w in ('yes', 'always', 'noeviction')))
|
|
|
|
|
|
def section_migrations():
|
|
out('== Migrations: run once by the migrate service, idempotent re-run ==')
|
|
psql("SELECT version || ' ' || applied_at FROM schema_migrations ORDER BY version")
|
|
psql("SELECT format('%s %s %s', conrelid::regclass, contype, conname) FROM pg_constraint "
|
|
"WHERE conrelid::regclass::text = 'idempotency_keys' ORDER BY 1")
|
|
before = psql("SELECT (SELECT count(*) FROM schema_migrations) || ' ' || (SELECT count(*) FROM users)")
|
|
output = compose('run', '--rm', '--no-deps', 'migrate')[0]
|
|
after = psql("SELECT (SELECT count(*) FROM schema_migrations) || ' ' || (SELECT count(*) FROM users)")
|
|
out('schema_migrations and users rows before/after: %s / %s' % (before, after))
|
|
verdict('re-running migrations applies nothing and duplicates nothing', 'applied_now=0' in output and before == after)
|
|
|
|
|
|
def section_balancing():
|
|
out('== Balancing: one session, many replicas ==')
|
|
_, _, token = new_user(label='balance')
|
|
replicas = api_replicas()
|
|
counts = {}
|
|
for _ in range(40):
|
|
status, _, _, instance = http('GET', '/api/v1/users/me', token)
|
|
counts[(instance, status)] = counts.get((instance, status), 0) + 1
|
|
for (instance, status), count in sorted(counts.items(), key=str):
|
|
out('X-Instance-Id %s (%s): %d responses with status %s' % (instance, replicas.get(instance, 'unknown container'), count, status))
|
|
instances = {instance for instance, _ in counts}
|
|
verdict('40 requests with one token are served by every replica without a new login',
|
|
instances == set(replicas) and {status for _, status in counts} == {200})
|
|
|
|
|
|
def section_idempotency():
|
|
out('== Idempotency: 20 parallel POSTs with one key through the load balancer ==')
|
|
user, _, token = new_user(label='idem')
|
|
chat = http('POST', '/api/v1/chats', token, {'title': 'Idempotency', 'member_ids': [user['id']]})[1]
|
|
key, text = uuid.uuid4().hex, 'idempotent ' + uuid.uuid4().hex
|
|
barrier = threading.Barrier(20)
|
|
|
|
def post(_):
|
|
barrier.wait()
|
|
return post_message(token, chat['id'], text, key)
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=20) as pool:
|
|
responses = list(pool.map(post, range(20)))
|
|
statuses = sorted({r[0] for r in responses}, key=str)
|
|
ids = {r[1]['id'] for r in responses if r[0] == 201}
|
|
instances = {r[3] for r in responses}
|
|
rows = psql("SELECT count(*) FROM messages WHERE chat_id = '%s'" % chat['id'])
|
|
keys = psql("SELECT count(*) FROM idempotency_keys WHERE key = '%s'" % key)
|
|
out('statuses: %s; distinct message ids: %d; replicas that answered: %s' % (statuses, len(ids), ', '.join(sorted(instances))))
|
|
out('messages in the chat: %s; idempotency_keys rows for the key: %s' % (rows, keys))
|
|
conflict = show('same key, another body', post_message(token, chat['id'], text + ' changed', key))
|
|
verdict('one message for 20 concurrent requests with one key on several replicas',
|
|
statuses == [201] and len(ids) == 1 and len(instances) >= 2 and rows == '1' and keys == '1')
|
|
verdict('the same key with another body is rejected with 409', conflict == 409)
|
|
|
|
|
|
def section_rollback():
|
|
out('== Rollback: chat creation fails on the second INSERT ==')
|
|
user, _, token = new_user(label='rollback')
|
|
title = 'rollback-probe-' + uuid.uuid4().hex[:10]
|
|
status = show('POST /api/v1/chats with one unknown member',
|
|
http('POST', '/api/v1/chats', token, {'title': title, 'member_ids': [user['id'], str(uuid.uuid4())]}))
|
|
rows = psql("SELECT count(*) FROM chats WHERE title = '%s'" % title)
|
|
orphans = psql('SELECT count(*) FROM chats c WHERE NOT EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = c.id)')
|
|
out('rows in chats with this title: %s; chats without members: %s' % (rows, orphans))
|
|
verdict('composite chat creation is rolled back as a whole', status == 400 and rows == '0' and orphans == '0')
|
|
|
|
|
|
def section_race():
|
|
out('== Registration race: 20 parallel requests with one username ==')
|
|
username = 'race_' + uuid.uuid4().hex[:12]
|
|
barrier = threading.Barrier(20)
|
|
|
|
def register(index):
|
|
barrier.wait()
|
|
body = {'username': username, 'display_name': 'Race %d' % index, 'password': 'Race-password-%02d' % index}
|
|
return http('POST', '/api/v1/users', body=body)
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=20) as pool:
|
|
responses = list(pool.map(register, range(20)))
|
|
statuses = sorted((r[0] for r in responses), key=str)
|
|
out('statuses: ' + ' '.join(str(s) for s in statuses) + '; replicas: ' + ', '.join(sorted({r[3] for r in responses})))
|
|
rows = psql("SELECT count(*) FROM users WHERE username = '%s'" % username)
|
|
out('rows in users with this username: ' + rows)
|
|
verdict('exactly one registration wins, the other 19 get 409', statuses == [201] + [409] * 19 and rows == '1')
|
|
|
|
|
|
def section_pool():
|
|
replicas = len(api_replicas())
|
|
out('== Connection pools: 60 parallel clients, at most 10 connections per replica (%d replicas) ==' % replicas)
|
|
users = [new_user(label='pool') for _ in range(4)]
|
|
chats = [http('POST', '/api/v1/chats', token, {'title': 'Pool', 'member_ids': [user['id']]})[1] for user, _, token in users]
|
|
count_sql = "SELECT count(*) FROM pg_stat_activity WHERE application_name = 'messenger-api'"
|
|
stop = threading.Event()
|
|
samples = []
|
|
|
|
def sample():
|
|
while not stop.is_set():
|
|
value = psql(count_sql, quiet=True)
|
|
if value.isdigit():
|
|
samples.append(int(value))
|
|
time.sleep(0.2)
|
|
|
|
def client(index):
|
|
_, _, token = users[index % 4]
|
|
path = '/api/v1/chats/%s/messages?limit=20' % chats[index % 4]['id']
|
|
statuses = []
|
|
deadline = time.monotonic() + 5
|
|
while time.monotonic() < deadline:
|
|
statuses.append(http('GET', path, token)[0])
|
|
return statuses
|
|
|
|
sampler = threading.Thread(target=sample)
|
|
sampler.start()
|
|
try:
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=60) as pool:
|
|
statuses = [s for batch in pool.map(client, range(60)) for s in batch]
|
|
finally:
|
|
stop.set()
|
|
sampler.join()
|
|
counts = {status: statuses.count(status) for status in sorted(set(statuses), key=str)}
|
|
out('requests: %d, statuses: %s' % (len(statuses), counts))
|
|
out('API connections in pg_stat_activity, %d samples: max %s' % (len(samples), max(samples) if samples else '-'))
|
|
verdict('pools never open more than 10 connections per replica and all requests succeed',
|
|
bool(samples) and max(samples) <= 10 * replicas and set(counts) == {200})
|
|
|
|
|
|
def section_timeouts():
|
|
out('== Timeouts: chat row locked for 6 s by another session, API answers 503 after lock_timeout=3s ==')
|
|
user, _, token = new_user(label='lock')
|
|
chat = http('POST', '/api/v1/chats', token, {'title': 'Lock', 'member_ids': [user['id']]})[1]
|
|
lock = "BEGIN; SELECT 1 FROM chats WHERE id = '%s' FOR UPDATE; SELECT pg_sleep(6); ROLLBACK;" % chat['id']
|
|
out('$ docker compose ' + ' '.join(PSQL) + ' "' + lock + '" (in the background)')
|
|
locker = subprocess.Popen(['docker', 'compose'] + PSQL + [lock], cwd=str(ROOT), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
time.sleep(1.5)
|
|
result = post_message(token, chat['id'], 'while locked', uuid.uuid4().hex)
|
|
show('POST message while the chat row is locked', result)
|
|
locker.wait(timeout=30)
|
|
after = show('POST message after the lock is released', post_message(token, chat['id'], 'after lock', uuid.uuid4().hex))
|
|
verdict('waiting for a lock is bounded: 503 after about 3 s, then writes work again',
|
|
result[0] == 503 and 2.5 <= result[2] <= 4.5 and after == 201, '%.2f s' % result[2])
|
|
|
|
|
|
def section_logs():
|
|
out('== Logs: useful fields, no message texts, passwords or tokens (all api replicas and lb) ==')
|
|
user, password, token = new_user(label='logs')
|
|
chat = http('POST', '/api/v1/chats', token, {'title': 'Logs', 'member_ids': [user['id']]})[1]
|
|
marker = 'log-marker-' + uuid.uuid4().hex
|
|
post_message(token, chat['id'], marker, uuid.uuid4().hex)
|
|
logs = compose('logs', '--no-log-prefix', 'api', 'lb', quiet=True)[0]
|
|
compose('logs', '--tail', '2', 'api')
|
|
compose('logs', '--no-log-prefix', '--tail', '2', 'lb')
|
|
leaked = [name for name, value in (('message text', marker), ('password', password), ('token', token)) if value in logs]
|
|
leaked += [p.name for p in (ROOT / 'secrets').glob('*_password') if p.read_text().strip() in logs]
|
|
out('searched %d log lines; found: %s' % (logs.count('\n') + 1, ', '.join(leaked) or 'nothing'))
|
|
verdict('logs contain request_id/status/duration and no secrets', 'request_id=' in logs and not leaked)
|
|
|
|
|
|
def section_expiry():
|
|
out('== Session expiry: api replicas recreated with SESSION_TTL_SECONDS=2, then restored ==')
|
|
count = len(api_replicas())
|
|
short = dict(os.environ, SESSION_TTL_SECONDS='2')
|
|
compose('up', '-d', '--no-deps', '--wait', 'api', env=short, quiet=True)
|
|
try:
|
|
if not wait_replicas(count):
|
|
raise Stop('replicas are not ready with the short TTL')
|
|
_, _, token = new_user(label='expiry')
|
|
key = 'session:' + hashlib.sha256(token.encode()).hexdigest()
|
|
first = show('GET /users/me right after login', http('GET', '/api/v1/users/me', token))
|
|
pttl = valkey_cli('PTTL ' + key)
|
|
out('PTTL of the session key in Valkey (ms): ' + pttl)
|
|
time.sleep(3)
|
|
expired = show('GET /users/me 3 s later', http('GET', '/api/v1/users/me', token))
|
|
gone = valkey_cli('EXISTS ' + key)
|
|
out('session key still exists in Valkey: ' + gone)
|
|
verdict('session expires after SESSION_TTL_SECONDS and the key disappears from Valkey',
|
|
first == 200 and 0 < int(pttl) <= 2000 and expired == 401 and gone == '0')
|
|
finally:
|
|
compose('up', '-d', '--no-deps', '--wait', 'api', quiet=True)
|
|
wait_replicas(count)
|
|
|
|
|
|
def outage(service, limit):
|
|
"""Stop one shared store and look at the answers through the load balancer."""
|
|
user, password, token = new_user(label=service)
|
|
compose('stop', service)
|
|
try:
|
|
me = show('GET /users/me with a valid token (first seconds)', http('GET', '/api/v1/users/me', token))
|
|
time.sleep(3) # the load balancer has seen failing readiness checks by now
|
|
me_later = show('GET /users/me with a valid token (after 3 s)', http('GET', '/api/v1/users/me', token))
|
|
anonymous = show('GET /users/me without a token', http('GET', '/api/v1/users/me'))
|
|
ready = show('GET /health/ready', http('GET', '/health/ready'))
|
|
slowest = max(http('GET', '/api/v1/users/me', token)[2] for _ in range(3))
|
|
out('slowest of 3 more authenticated requests: %.2f s' % slowest)
|
|
finally:
|
|
compose('start', service)
|
|
started = time.monotonic()
|
|
recovered = False
|
|
while time.monotonic() - started < 60 and not recovered:
|
|
recovered = http('GET', '/api/v1/users/me', token, timeout=5)[0] == 200
|
|
time.sleep(0.5)
|
|
out('same token after %s is back: %s' % (service, '200 after %.1f s' % (time.monotonic() - started) if recovered else 'not accepted'))
|
|
verdict('without %s: bounded 503, nothing is served as anonymous' % service,
|
|
me == 503 and me_later == 503 and anonymous in (401, 503) and ready == 503 and slowest < limit)
|
|
verdict('the stand recovers by itself when %s is back, the session is still valid' % service, recovered)
|
|
|
|
|
|
def section_valkey_down():
|
|
out('== Valkey is down ==')
|
|
outage('valkey', 3)
|
|
|
|
|
|
def section_postgres_down():
|
|
out('== PostgreSQL is down ==')
|
|
outage('postgres', 5)
|
|
|
|
|
|
def section_failover():
|
|
out('== Public failover test: tests/resilience.py --mode failover, repeated until every replica was the target ==')
|
|
hook = 'scripts/stop-one.cmd' if os.name == 'nt' else 'scripts/stop-one.sh'
|
|
replicas = api_replicas()
|
|
targeted = set()
|
|
for attempt in range(1, 7):
|
|
before = set(api_replicas())
|
|
_, code, seconds = run(PY + ['tests/resilience.py', '--mode', 'failover', '--action-script', hook, '--base-url', BASE], check=False)
|
|
stopped = before - set(api_replicas())
|
|
for cid in stopped:
|
|
out('attempt %d: stopped container %s (%s), State: %s, ExitCode %s' % (
|
|
attempt, cid, replicas.get(cid), inspect(cid, '{{.State.Status}}'), inspect(cid, '{{.State.ExitCode}}')))
|
|
verdict('public failover test passes (attempt %d)' % attempt, code == 0 and len(stopped) == 1, '%.1f s' % seconds)
|
|
targeted |= stopped
|
|
compose('up', '-d', '--wait', quiet=True)
|
|
if not wait_replicas(len(replicas)):
|
|
raise Stop('replicas did not come back')
|
|
if targeted >= set(replicas):
|
|
break
|
|
verdict('every replica was stopped by the hook at least once', targeted >= set(replicas),
|
|
', '.join(sorted(replicas[c] for c in targeted if c in replicas)))
|
|
|
|
|
|
def section_experiment():
|
|
out('== Experiment: stop, then kill, each replica in turn under load; restore before the next one ==')
|
|
replicas = api_replicas()
|
|
count = len(replicas)
|
|
load = Load()
|
|
poller = StatsPoller()
|
|
poller.start()
|
|
load.start()
|
|
rows = []
|
|
try:
|
|
time.sleep(5)
|
|
for mode in ('stop', 'kill'):
|
|
for cid, name in sorted(replicas.items(), key=lambda item: item[1]):
|
|
address = inspect(cid, '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}')
|
|
slot = poller.slot_of(address)
|
|
base_start = time.monotonic() - 5
|
|
baseline = load.window(base_start, time.monotonic())
|
|
t0 = time.monotonic()
|
|
_, _, command_seconds = run(['docker', mode, cid], quiet=True)
|
|
time.sleep(8)
|
|
t1 = time.monotonic()
|
|
during = load.window(t0, t1)
|
|
gap = load.max_gap(t0, t1)
|
|
detected = poller.first_change(slot, ('DOWN', 'MAINT', 'NOLB'), t0)
|
|
last_served = load.last_served_by(cid, t0)
|
|
run(['docker', 'start', cid], quiet=True)
|
|
t2 = time.monotonic()
|
|
back = None
|
|
while time.monotonic() - t2 < 60 and back is None:
|
|
time.sleep(0.2)
|
|
back = load.first_served_by(cid, t2)
|
|
time.sleep(3)
|
|
after = load.window(back or t2, time.monotonic())
|
|
row = {
|
|
'replica': name, 'lb_slot': slot, 'action': 'docker ' + mode, 'command_s': round(command_seconds, 2),
|
|
'baseline_ok_rps': baseline['ok_rps'], 'during_ok_rps': during['ok_rps'],
|
|
'errors_during': during['errors'], 'requests_during': during['requests'],
|
|
'max_gap_s': round(gap, 2),
|
|
'lb_marked_down_s': round(detected - t0, 2) if detected else None,
|
|
'last_served_s': round(last_served - t0, 2) if last_served else None,
|
|
'back_in_rotation_s': round(back - t2, 2) if back else None,
|
|
'after_ok_rps': after['ok_rps'], 'errors_after': after['errors'],
|
|
}
|
|
rows.append(row)
|
|
out(json.dumps(row, ensure_ascii=False))
|
|
if not wait_replicas(count):
|
|
raise Stop('replicas did not come back after %s' % name)
|
|
time.sleep(2)
|
|
finally:
|
|
load.stop()
|
|
poller.stop_event.set()
|
|
acknowledged, present, duplicates = load.verify()
|
|
out()
|
|
out('%-16s %-12s %6s %8s %8s %7s %8s %8s %8s %8s' % (
|
|
'replica', 'action', 'cmd,s', 'rps0', 'rps', 'errors', 'gap,s', 'down,s', 'back,s', 'rps1'))
|
|
for r in rows:
|
|
out('%-16s %-12s %6s %8s %8s %7s %8s %8s %8s %8s' % (
|
|
r['replica'], r['action'], r['command_s'], r['baseline_ok_rps'], r['during_ok_rps'], r['errors_during'],
|
|
r['max_gap_s'], r['lb_marked_down_s'], r['back_in_rotation_s'], r['after_ok_rps']))
|
|
out('acknowledged writes: %d, found in history: %d, duplicates: %d' % (acknowledged, present, duplicates))
|
|
stops = [r for r in rows if r['action'] == 'docker stop']
|
|
kills = [r for r in rows if r['action'] == 'docker kill']
|
|
verdict('graceful stop of each replica: no client errors, LB removes it within 3 s',
|
|
len(stops) == count and all(r['errors_during'] == 0 and r['lb_marked_down_s'] is not None
|
|
and r['lb_marked_down_s'] <= 3 for r in stops))
|
|
verdict('kill of each replica: LB removes it within 3 s, writes go on within 10 s',
|
|
len(kills) == count and all(r['lb_marked_down_s'] is not None and r['lb_marked_down_s'] <= 3
|
|
and r['max_gap_s'] <= 10 for r in kills))
|
|
verdict('every replica comes back into rotation after docker start', all(r['back_in_rotation_s'] is not None for r in rows))
|
|
verdict('no acknowledged write is lost or duplicated (retries reuse the key)', acknowledged == present and duplicates == 0)
|
|
|
|
|
|
def section_rolling():
|
|
out('== Rolling update under load: one replica at a time, new image, migrations once ==')
|
|
replicas = api_replicas()
|
|
count = len(replicas)
|
|
old_image = inspect('mtusi-messenger:lab3', '{{.Id}}')
|
|
version = 'rolling-%d' % int(time.time())
|
|
env = dict(os.environ, APP_VERSION=version)
|
|
compose('build', 'api', env=env, quiet=True)
|
|
new_image = inspect('mtusi-messenger:lab3', '{{.Id}}')
|
|
out('image before %s, after %s, APP_VERSION=%s' % (old_image[7:19], new_image[7:19], version))
|
|
migrate = compose('run', '--rm', '--no-deps', 'migrate', env=env)[0]
|
|
load = Load()
|
|
load.start()
|
|
started = time.monotonic()
|
|
steps = []
|
|
try:
|
|
time.sleep(3)
|
|
for old in sorted(replicas, key=lambda cid: replicas[cid]):
|
|
known = set(api_replicas(all_states=True))
|
|
compose('up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=%d' % (count + 1), 'api', env=env, quiet=True)
|
|
new = [cid for cid in api_replicas() if cid not in known]
|
|
if len(new) != 1 or not wait_replicas(count + 1):
|
|
raise Stop('new replica did not become ready')
|
|
run(['docker', 'stop', old], quiet=True)
|
|
run(['docker', 'rm', old], quiet=True)
|
|
steps.append('%s (%s) -> %s' % (replicas[old], old, new[0]))
|
|
out('replaced ' + steps[-1])
|
|
time.sleep(3)
|
|
finally:
|
|
load.stop()
|
|
stats = load.window(started, time.monotonic())
|
|
acknowledged, present, duplicates = load.verify()
|
|
final = api_replicas()
|
|
images = {cid: inspect(cid, '{{.Image}}') for cid in final}
|
|
versions = {cid: silent(['docker', 'exec', cid, 'printenv', 'APP_VERSION']) for cid in final}
|
|
out('during the update: %s' % json.dumps(stats))
|
|
out('final replicas: ' + ', '.join('%s %s version=%s' % (final[c], c, versions[c]) for c in sorted(final)))
|
|
out('acknowledged writes: %d, found in history: %d, duplicates: %d' % (acknowledged, present, duplicates))
|
|
verdict('migrations are applied once and the change is backward compatible', 'applied_now=0' in migrate)
|
|
verdict('every replica was replaced one by one by the new image under load',
|
|
len(steps) == count and len(final) == count and set(images.values()) == {new_image}
|
|
and set(versions.values()) == {version})
|
|
verdict('no client errors and no lost or duplicated writes during the rolling update',
|
|
stats['errors'] == 0 and acknowledged == present and duplicates == 0,
|
|
'%d requests, %.1f ok RPS' % (stats['requests'], stats['ok_rps']))
|
|
|
|
|
|
def section_persistence():
|
|
out('== Full restart: public tests/resilience.py --mode persistence with the restart hook ==')
|
|
hook = 'scripts/restart.cmd' if os.name == 'nt' else 'scripts/restart.sh'
|
|
services = {name: cid for cid, name in api_replicas().items()}
|
|
services.update({s: container(s) for s in ('postgres', 'valkey', 'lb')})
|
|
before = {name: inspect(cid, '{{.State.StartedAt}}') for name, cid in services.items()}
|
|
volumes = ['%s_pgdata' % PROJECT, '%s_valkeydata' % PROJECT]
|
|
volumes_before = run(['docker', 'volume', 'inspect', '-f', '{{.Name}} {{.CreatedAt}}'] + volumes)[0]
|
|
_, code, seconds = run(PY + ['tests/resilience.py', '--mode', 'persistence', '--action-script', hook, '--base-url', BASE], check=False)
|
|
after = {name: inspect(cid, '{{.State.StartedAt}}') for name, cid in services.items()}
|
|
volumes_after = run(['docker', 'volume', 'inspect', '-f', '{{.Name}} {{.CreatedAt}}'] + volumes)[0]
|
|
for name in sorted(before):
|
|
out('%-18s StartedAt %s -> %s' % (name, before[name][11:23], after[name][11:23]))
|
|
restarted = all(before[name] != after[name] for name in before)
|
|
verdict('persistence test passes after restarting every replica, postgres, valkey and lb with the same volumes',
|
|
code == 0 and restarted and volumes_before == volumes_after, 'hook %s, %.1f s' % (hook, seconds))
|
|
|
|
|
|
SECTIONS = {
|
|
'stack': section_stack,
|
|
'migrations': section_migrations,
|
|
'balancing': section_balancing,
|
|
'idempotency': section_idempotency,
|
|
'rollback': section_rollback,
|
|
'race': section_race,
|
|
'pool': section_pool,
|
|
'timeouts': section_timeouts,
|
|
'logs': section_logs,
|
|
'expiry': section_expiry,
|
|
'valkey-down': section_valkey_down,
|
|
'postgres-down': section_postgres_down,
|
|
'failover': section_failover,
|
|
'experiment': section_experiment,
|
|
'rolling': section_rolling,
|
|
'persistence': section_persistence,
|
|
}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description='Lab 3 infrastructure checks; creates synthetic users and messages')
|
|
parser.add_argument('--only', nargs='+', choices=sorted(SECTIONS), help='run only these sections')
|
|
parser.add_argument('--output', default=None, help='also write the transcript to this file')
|
|
args = parser.parse_args()
|
|
sys.stdout.reconfigure(errors='replace')
|
|
out('# %s UTC, python %s, %s' % (time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()), sys.version.split()[0], sys.platform))
|
|
out('# ' + silent(['docker', 'version', '--format', 'docker {{.Server.Version}}']))
|
|
if not wait_ready(seconds=30):
|
|
out('API is not ready on %s: start the stand with docker compose up -d --build --wait' % BASE)
|
|
return 2
|
|
for name in args.only or SECTIONS:
|
|
try:
|
|
SECTIONS[name]()
|
|
except (Stop, KeyError, TypeError, ValueError) as error:
|
|
verdict('section %s completed' % name, False, str(error) or type(error).__name__)
|
|
if not wait_ready(seconds=60):
|
|
verdict('stand is ready after section %s' % name, False)
|
|
break
|
|
out('== Summary ==')
|
|
for name, ok in results:
|
|
out('%s %s' % ('PASS' if ok else 'FAIL', name))
|
|
failed = [name for name, ok in results if not ok]
|
|
out('RESULT: %s' % ('FAIL' if failed else 'PASS'))
|
|
if args.output:
|
|
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
|
Path(args.output).write_text('\n'.join(lines) + '\n', encoding='utf-8')
|
|
return 1 if failed else 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
sys.exit(main())
|
|
except FileNotFoundError:
|
|
print('docker CLI was not found: install Docker and make sure it is on PATH')
|
|
sys.exit(2)
|