Files
mtusi2026lab3/scripts/lab3_checks.py
2026-10-21 18:25:07 +03:00

801 lines
38 KiB
Python
Executable File

#!/usr/bin/env python3
"""Infrastructure checks for lab 3 that need control over the compose stand.
python -X utf8 scripts/lab3_checks.py --output evidence/lab3-checks.txt
python -X utf8 scripts/lab3_checks.py --only failover experiment rolling
The stand must already be running (docker compose up -d --build --wait). Every command is
printed together with its output. Sections stop, kill and start containers, but never delete
volumes, never flush Valkey and never print secrets or session tokens. All API traffic goes
through the load balancer on 127.0.0.1:8080, as a client would send it.
Standard library only; works on Windows, Linux and macOS (-X utf8 is needed on Windows only
because the public tests read the contract in the default encoding).
"""
import argparse
import base64
import concurrent.futures
import csv
import hashlib
import io
import json
import os
from pathlib import Path
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
import uuid
ROOT = Path(__file__).resolve().parents[1]
BASE = 'http://127.0.0.1:8080'
STATS = 'http://127.0.0.1:8404/;csv'
PROJECT = os.environ.get('COMPOSE_PROJECT_NAME', 'mtusi-lab3')
VALKEY_CLI = 'valkey-cli --no-auth-warning -a "$(sed -n "s/^requirepass //p" /run/secrets/valkey_auth)" '
PSQL = ['exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-v', 'ON_ERROR_STOP=1', '-At', '-c']
PY = [sys.executable, '-X', 'utf8']
lines = []
results = []
class Stop(Exception):
pass
def out(text=''):
lines.append(text)
print(text, flush=True)
def run(command, check=True, quiet=False, env=None):
"""Run a command and echo it with its output; return (output, exit code, seconds)."""
out('$ ' + ' '.join(command))
started = time.monotonic()
done = subprocess.run(command, cwd=str(ROOT), env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
elapsed = time.monotonic() - started
text = done.stdout.decode('utf-8', 'replace').strip()
if text and not quiet:
out(text)
if check and done.returncode != 0:
if quiet and text:
out(text)
raise Stop('command failed with exit code %d' % done.returncode)
return text, done.returncode, elapsed
def silent(command):
"""Run a command without echo; return its stdout."""
return subprocess.run(command, cwd=str(ROOT), stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True).stdout.strip()
def compose(*args, **kwargs):
return run(['docker', 'compose'] + list(args), **kwargs)
def psql(sql, quiet=False):
if quiet:
return silent(['docker', 'compose'] + PSQL + [sql])
return compose(*(PSQL + [sql]))[0]
def valkey_cli(command):
return compose('exec', '-T', 'valkey', 'sh', '-c', VALKEY_CLI + command)[0]
def verdict(name, ok, detail=''):
results.append((name, ok))
out('=> %s: %s%s' % ('PASS' if ok else 'FAIL', name, ' (' + detail + ')' if detail else ''))
out()
def http(method, path, token=None, body=None, headers=None, timeout=10):
"""(status, parsed JSON or None, seconds, X-Instance-Id); status None when the connection fails."""
all_headers = dict(headers or {})
if token:
all_headers['Authorization'] = 'Bearer ' + token
data = None
if body is not None:
data = json.dumps(body).encode('utf-8')
all_headers['Content-Type'] = 'application/json'
request = urllib.request.Request(BASE + path, data=data, headers=all_headers, method=method)
started = time.monotonic()
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
raw, status, instance = response.read(), response.status, response.headers.get('X-Instance-Id')
except urllib.error.HTTPError as error:
with error:
raw, status, instance = error.read(), error.code, error.headers.get('X-Instance-Id')
except OSError:
return None, None, time.monotonic() - started, None
try:
payload = json.loads(raw) if raw else None
except ValueError:
payload = None
return status, payload, time.monotonic() - started, instance
def basic(username, password):
return {'Authorization': 'Basic ' + base64.b64encode((username + ':' + password).encode()).decode()}
def new_user(label='check'):
"""Register a synthetic user and log in: (user, password, token). The token is never printed."""
username = label + '_' + uuid.uuid4().hex[:12]
password = 'Check-' + uuid.uuid4().hex
status, user, _, _ = http('POST', '/api/v1/users', body={'username': username, 'display_name': 'Check', 'password': password})
if status != 201:
raise Stop('registration failed with %s' % status)
status, session, _, _ = http('POST', '/api/v1/auth/sessions', headers=basic(username, password))
if status != 201:
raise Stop('login failed with %s' % status)
return user, password, session['token']
def post_message(token, chat_id, text, key):
return http('POST', '/api/v1/chats/%s/messages' % chat_id, token, {'text': text}, {'Idempotency-Key': key}, timeout=5)
def history(token, chat_id):
items, cursor = [], None
while True:
path = '/api/v1/chats/%s/messages?limit=100' % chat_id + ('&cursor=' + cursor if cursor else '')
status, page, _, _ = http('GET', path, token)
if status != 200:
raise Stop('history read failed with %s' % status)
items += page['items']
cursor = page['next_cursor']
if not cursor:
return items
def wait_ready(seconds=60):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
if http('GET', '/health/ready', timeout=3)[0] == 200:
return True
time.sleep(0.5)
return False
def api_replicas(all_states=False):
"""{short container id: container name} of the api service of this project."""
command = ['docker', 'ps', '--filter', 'label=com.docker.compose.project=' + PROJECT,
'--filter', 'label=com.docker.compose.service=api', '--format', '{{.ID}} {{.Names}}']
if all_states:
command.insert(2, '-a')
return dict(line.split(' ', 1) for line in silent(command).splitlines() if line)
def container(service):
return silent(['docker', 'compose', 'ps', '-a', '-q', service])
def inspect(target, template):
return silent(['docker', 'inspect', '-f', template, target])
def wait_replicas(count, seconds=90):
"""Wait until `count` api replicas are healthy and all of them answer through the load balancer."""
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
replicas = api_replicas()
healthy = [r for r in replicas if inspect(r, '{{.State.Health.Status}}') == 'healthy']
if len(healthy) == count:
seen = {http('GET', '/health/live', timeout=3)[3] for _ in range(4 * count)}
if set(healthy) <= seen:
return True
time.sleep(1)
return False
def show(label, result):
status, payload, seconds, instance = result
code = payload['error']['code'] if isinstance(payload, dict) and 'error' in payload else ''
out('%-58s -> %s %s in %.2f s, X-Instance-Id %s' % (label, status, code, seconds, instance or '-'))
return status
class StatsPoller(threading.Thread):
"""Polls the HAProxy stats CSV every 100 ms and records when each server slot changes state.
A slot (api1..api8) is matched to a container once, by its IP address while it is UP.
A stopped replica first turns DOWN (failed checks); a killed one usually goes to
MAINT (resolution) first, because Docker DNS drops its address and the slot loses it.
"""
def __init__(self):
super().__init__(daemon=True)
self.stop_event = threading.Event()
self.changes = [] # (monotonic time, slot, new status)
@staticmethod
def snapshot():
"""{slot: (status, ip)} for the api backend."""
try:
with urllib.request.urlopen(STATS, timeout=1) as response:
text = response.read().decode('utf-8').lstrip('# ')
except OSError:
return {}
return {row['svname']: (row['status'].split(' ')[0], row.get('addr', '').split(':')[0])
for row in csv.DictReader(io.StringIO(text))
if row.get('pxname') == 'api' and row.get('svname') not in ('FRONTEND', 'BACKEND')}
def slot_of(self, address):
for slot, (status, ip) in self.snapshot().items():
if ip == address:
return slot
return None
def run(self):
last = {}
while not self.stop_event.is_set():
now = time.monotonic()
for slot, (status, _) in self.snapshot().items():
if last.get(slot) != status:
self.changes.append((now, slot, status))
last[slot] = status
time.sleep(0.1)
def first_change(self, slot, statuses, after):
for moment, changed, status in self.changes:
if changed == slot and status in statuses and moment >= after:
return moment
return None
class Load:
"""Closed-loop load through the load balancer: each worker posts a message and reads /users/me.
A failed POST is retried with the SAME Idempotency-Key and body, exactly like a careful client,
so the final history must contain every acknowledged message once and no duplicates.
"""
def __init__(self, workers=6):
self.user, _, self.token = new_user(label='load')
self.chat = http('POST', '/api/v1/chats', self.token, {'title': 'Load', 'member_ids': [self.user['id']]})[1]
self.workers = workers
self.events = [] # (monotonic time, ok, instance, kind)
self.acknowledged = []
self.lock = threading.Lock()
self.stop_event = threading.Event()
self.threads = []
def worker(self, index):
while not self.stop_event.is_set():
key = uuid.uuid4().hex
text = 'load %d %s' % (index, key)
while not self.stop_event.is_set():
status, payload, _, instance = post_message(self.token, self.chat['id'], text, key)
with self.lock:
self.events.append((time.monotonic(), status == 201, instance, 'write'))
if status == 201:
self.acknowledged.append(payload['id'])
if status == 201:
break
time.sleep(0.05)
status, _, _, instance = http('GET', '/api/v1/users/me', self.token, timeout=5)
with self.lock:
self.events.append((time.monotonic(), status == 200, instance, 'read'))
def start(self):
for index in range(self.workers):
thread = threading.Thread(target=self.worker, args=(index,), daemon=True)
thread.start()
self.threads.append(thread)
def stop(self):
self.stop_event.set()
for thread in self.threads:
thread.join(timeout=10)
def window(self, start, end):
with self.lock:
selected = [e for e in self.events if start <= e[0] < end]
ok = sum(1 for e in selected if e[1])
return {'requests': len(selected), 'ok': ok, 'errors': len(selected) - ok,
'ok_rps': round(ok / (end - start), 1) if end > start else 0.0}
def max_gap(self, start, end):
"""Longest interval without a successful request."""
with self.lock:
moments = sorted(e[0] for e in self.events if e[1] and start <= e[0] < end)
points = [start] + moments + [end]
return max(b - a for a, b in zip(points, points[1:]))
def last_served_by(self, instance, after):
with self.lock:
moments = [e[0] for e in self.events if e[2] == instance and e[0] >= after]
return max(moments) if moments else None
def first_served_by(self, instance, after):
with self.lock:
moments = [e[0] for e in self.events if e[2] == instance and e[0] >= after]
return min(moments) if moments else None
def verify(self):
"""(acknowledged, in history, duplicates by text) for the load chat."""
items = history(self.token, self.chat['id'])
texts = [m['text'] for m in items]
ids = {m['id'] for m in items}
return len(set(self.acknowledged)), len(set(self.acknowledged) & ids), len(texts) - len(set(texts))
# --- sections ---------------------------------------------------------------
def section_stack():
out('== Stack: replicas, pinned images, health, limits, published ports, secrets ==')
compose('ps', '-a', '--format', 'table {{.Name}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}')
config = json.loads(compose('config', '--format', 'json', quiet=True)[0])
services = config['services']
images = {name: services[name].get('image', '') for name in ('lb', 'postgres', 'valkey')}
verdict('third-party images are pinned by tag and digest', all('@sha256:' in image for image in images.values()),
', '.join(images.values()))
replicas = api_replicas()
names = sorted(replicas.values())
out('api replicas: ' + ', '.join('%s (%s)' % (name, cid) for cid, name in sorted(replicas.items(), key=lambda x: x[1])))
no_container_name = not any('container_name' in service for service in services.values())
verdict('at least two api replicas of one compose service, no container_name', len(replicas) >= 2 and no_container_name,
', '.join(names))
health = {name: inspect(cid, '{{.State.Health.Status}}') for cid, name in replicas.items()}
for service in ('postgres', 'valkey'):
health[service] = inspect(container(service), '{{.State.Health.Status}}')
out('health: ' + ', '.join('%s=%s' % item for item in sorted(health.items())))
verdict('all api replicas, postgres and valkey are healthy', set(health.values()) == {'healthy'})
users = {name: silent(['docker', 'exec', cid, 'id', '-u']) for cid, name in replicas.items()}
verdict('API processes do not run as root', set(users.values()) == {'10001'}, ', '.join('%s uid=%s' % i for i in sorted(users.items())))
out('%-26s %8s %10s %s' % ('container', 'cpus', 'memory', 'published ports'))
limited, bypass = True, []
targets = [(name, cid) for cid, name in replicas.items()] + [(s, container(s)) for s in ('lb', 'postgres', 'valkey')]
for name, cid in sorted(targets):
cpus = int(inspect(cid, '{{.HostConfig.NanoCpus}}') or 0) / 1e9
memory = int(inspect(cid, '{{.HostConfig.Memory}}') or 0) // (1024 * 1024)
ports = inspect(cid, '{{range $p, $b := .HostConfig.PortBindings}}{{range $b}}{{.HostIp}}:{{.HostPort}}->{{$p}} {{end}}{{end}}')
out('%-26s %8.2f %8d MB %s' % (name, cpus, memory, ports or '-'))
limited = limited and cpus > 0 and memory > 0
if ports and name != 'lb':
bypass.append(name)
if ports and any(not part.startswith('127.0.0.1:') for part in ports.split()):
bypass.append(name + ' (not 127.0.0.1)')
verdict('every container has CPU and memory limits', limited)
verdict('only the load balancer publishes ports, and only on 127.0.0.1', not bypass, ', '.join(bypass))
secrets = {name: sorted(s['source'] for s in service.get('secrets', [])) for name, service in services.items()}
out('secrets per service: ' + '; '.join('%s=%s' % (k, ','.join(v) or '-') for k, v in sorted(secrets.items())))
verdict('secrets are given only where needed', secrets.get('lb') == [] and secrets['api'] == ['cache_url', 'database_url']
and secrets['postgres'] == ['postgres_password'] and secrets['valkey'] == ['valkey_auth'])
aof = ' '.join(valkey_cli('CONFIG GET ' + name) for name in ('appendonly', 'appendfsync', 'maxmemory-policy'))
verdict('Valkey persistence is AOF with fsync always, no eviction', all(w in aof.split() for w in ('yes', 'always', 'noeviction')))
def section_migrations():
out('== Migrations: run once by the migrate service, idempotent re-run ==')
psql("SELECT version || ' ' || applied_at FROM schema_migrations ORDER BY version")
psql("SELECT format('%s %s %s', conrelid::regclass, contype, conname) FROM pg_constraint "
"WHERE conrelid::regclass::text = 'idempotency_keys' ORDER BY 1")
before = psql("SELECT (SELECT count(*) FROM schema_migrations) || ' ' || (SELECT count(*) FROM users)")
output = compose('run', '--rm', '--no-deps', 'migrate')[0]
after = psql("SELECT (SELECT count(*) FROM schema_migrations) || ' ' || (SELECT count(*) FROM users)")
out('schema_migrations and users rows before/after: %s / %s' % (before, after))
verdict('re-running migrations applies nothing and duplicates nothing', 'applied_now=0' in output and before == after)
def section_balancing():
out('== Balancing: one session, many replicas ==')
_, _, token = new_user(label='balance')
replicas = api_replicas()
counts = {}
for _ in range(40):
status, _, _, instance = http('GET', '/api/v1/users/me', token)
counts[(instance, status)] = counts.get((instance, status), 0) + 1
for (instance, status), count in sorted(counts.items(), key=str):
out('X-Instance-Id %s (%s): %d responses with status %s' % (instance, replicas.get(instance, 'unknown container'), count, status))
instances = {instance for instance, _ in counts}
verdict('40 requests with one token are served by every replica without a new login',
instances == set(replicas) and {status for _, status in counts} == {200})
def section_idempotency():
out('== Idempotency: 20 parallel POSTs with one key through the load balancer ==')
user, _, token = new_user(label='idem')
chat = http('POST', '/api/v1/chats', token, {'title': 'Idempotency', 'member_ids': [user['id']]})[1]
key, text = uuid.uuid4().hex, 'idempotent ' + uuid.uuid4().hex
barrier = threading.Barrier(20)
def post(_):
barrier.wait()
return post_message(token, chat['id'], text, key)
with concurrent.futures.ThreadPoolExecutor(max_workers=20) as pool:
responses = list(pool.map(post, range(20)))
statuses = sorted({r[0] for r in responses}, key=str)
ids = {r[1]['id'] for r in responses if r[0] == 201}
instances = {r[3] for r in responses}
rows = psql("SELECT count(*) FROM messages WHERE chat_id = '%s'" % chat['id'])
keys = psql("SELECT count(*) FROM idempotency_keys WHERE key = '%s'" % key)
out('statuses: %s; distinct message ids: %d; replicas that answered: %s' % (statuses, len(ids), ', '.join(sorted(instances))))
out('messages in the chat: %s; idempotency_keys rows for the key: %s' % (rows, keys))
conflict = show('same key, another body', post_message(token, chat['id'], text + ' changed', key))
verdict('one message for 20 concurrent requests with one key on several replicas',
statuses == [201] and len(ids) == 1 and len(instances) >= 2 and rows == '1' and keys == '1')
verdict('the same key with another body is rejected with 409', conflict == 409)
def section_rollback():
out('== Rollback: chat creation fails on the second INSERT ==')
user, _, token = new_user(label='rollback')
title = 'rollback-probe-' + uuid.uuid4().hex[:10]
status = show('POST /api/v1/chats with one unknown member',
http('POST', '/api/v1/chats', token, {'title': title, 'member_ids': [user['id'], str(uuid.uuid4())]}))
rows = psql("SELECT count(*) FROM chats WHERE title = '%s'" % title)
orphans = psql('SELECT count(*) FROM chats c WHERE NOT EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = c.id)')
out('rows in chats with this title: %s; chats without members: %s' % (rows, orphans))
verdict('composite chat creation is rolled back as a whole', status == 400 and rows == '0' and orphans == '0')
def section_race():
out('== Registration race: 20 parallel requests with one username ==')
username = 'race_' + uuid.uuid4().hex[:12]
barrier = threading.Barrier(20)
def register(index):
barrier.wait()
body = {'username': username, 'display_name': 'Race %d' % index, 'password': 'Race-password-%02d' % index}
return http('POST', '/api/v1/users', body=body)
with concurrent.futures.ThreadPoolExecutor(max_workers=20) as pool:
responses = list(pool.map(register, range(20)))
statuses = sorted((r[0] for r in responses), key=str)
out('statuses: ' + ' '.join(str(s) for s in statuses) + '; replicas: ' + ', '.join(sorted({r[3] for r in responses})))
rows = psql("SELECT count(*) FROM users WHERE username = '%s'" % username)
out('rows in users with this username: ' + rows)
verdict('exactly one registration wins, the other 19 get 409', statuses == [201] + [409] * 19 and rows == '1')
def section_pool():
replicas = len(api_replicas())
out('== Connection pools: 60 parallel clients, at most 10 connections per replica (%d replicas) ==' % replicas)
users = [new_user(label='pool') for _ in range(4)]
chats = [http('POST', '/api/v1/chats', token, {'title': 'Pool', 'member_ids': [user['id']]})[1] for user, _, token in users]
count_sql = "SELECT count(*) FROM pg_stat_activity WHERE application_name = 'messenger-api'"
stop = threading.Event()
samples = []
def sample():
while not stop.is_set():
value = psql(count_sql, quiet=True)
if value.isdigit():
samples.append(int(value))
time.sleep(0.2)
def client(index):
_, _, token = users[index % 4]
path = '/api/v1/chats/%s/messages?limit=20' % chats[index % 4]['id']
statuses = []
deadline = time.monotonic() + 5
while time.monotonic() < deadline:
statuses.append(http('GET', path, token)[0])
return statuses
sampler = threading.Thread(target=sample)
sampler.start()
try:
with concurrent.futures.ThreadPoolExecutor(max_workers=60) as pool:
statuses = [s for batch in pool.map(client, range(60)) for s in batch]
finally:
stop.set()
sampler.join()
counts = {status: statuses.count(status) for status in sorted(set(statuses), key=str)}
out('requests: %d, statuses: %s' % (len(statuses), counts))
out('API connections in pg_stat_activity, %d samples: max %s' % (len(samples), max(samples) if samples else '-'))
verdict('pools never open more than 10 connections per replica and all requests succeed',
bool(samples) and max(samples) <= 10 * replicas and set(counts) == {200})
def section_timeouts():
out('== Timeouts: chat row locked for 6 s by another session, API answers 503 after lock_timeout=3s ==')
user, _, token = new_user(label='lock')
chat = http('POST', '/api/v1/chats', token, {'title': 'Lock', 'member_ids': [user['id']]})[1]
lock = "BEGIN; SELECT 1 FROM chats WHERE id = '%s' FOR UPDATE; SELECT pg_sleep(6); ROLLBACK;" % chat['id']
out('$ docker compose ' + ' '.join(PSQL) + ' "' + lock + '" (in the background)')
locker = subprocess.Popen(['docker', 'compose'] + PSQL + [lock], cwd=str(ROOT), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
time.sleep(1.5)
result = post_message(token, chat['id'], 'while locked', uuid.uuid4().hex)
show('POST message while the chat row is locked', result)
locker.wait(timeout=30)
after = show('POST message after the lock is released', post_message(token, chat['id'], 'after lock', uuid.uuid4().hex))
verdict('waiting for a lock is bounded: 503 after about 3 s, then writes work again',
result[0] == 503 and 2.5 <= result[2] <= 4.5 and after == 201, '%.2f s' % result[2])
def section_logs():
out('== Logs: useful fields, no message texts, passwords or tokens (all api replicas and lb) ==')
user, password, token = new_user(label='logs')
chat = http('POST', '/api/v1/chats', token, {'title': 'Logs', 'member_ids': [user['id']]})[1]
marker = 'log-marker-' + uuid.uuid4().hex
post_message(token, chat['id'], marker, uuid.uuid4().hex)
logs = compose('logs', '--no-log-prefix', 'api', 'lb', quiet=True)[0]
compose('logs', '--tail', '2', 'api')
compose('logs', '--no-log-prefix', '--tail', '2', 'lb')
leaked = [name for name, value in (('message text', marker), ('password', password), ('token', token)) if value in logs]
leaked += [p.name for p in (ROOT / 'secrets').glob('*_password') if p.read_text().strip() in logs]
out('searched %d log lines; found: %s' % (logs.count('\n') + 1, ', '.join(leaked) or 'nothing'))
verdict('logs contain request_id/status/duration and no secrets', 'request_id=' in logs and not leaked)
def section_expiry():
out('== Session expiry: api replicas recreated with SESSION_TTL_SECONDS=2, then restored ==')
count = len(api_replicas())
short = dict(os.environ, SESSION_TTL_SECONDS='2')
compose('up', '-d', '--no-deps', '--wait', 'api', env=short, quiet=True)
try:
if not wait_replicas(count):
raise Stop('replicas are not ready with the short TTL')
_, _, token = new_user(label='expiry')
key = 'session:' + hashlib.sha256(token.encode()).hexdigest()
first = show('GET /users/me right after login', http('GET', '/api/v1/users/me', token))
pttl = valkey_cli('PTTL ' + key)
out('PTTL of the session key in Valkey (ms): ' + pttl)
time.sleep(3)
expired = show('GET /users/me 3 s later', http('GET', '/api/v1/users/me', token))
gone = valkey_cli('EXISTS ' + key)
out('session key still exists in Valkey: ' + gone)
verdict('session expires after SESSION_TTL_SECONDS and the key disappears from Valkey',
first == 200 and 0 < int(pttl) <= 2000 and expired == 401 and gone == '0')
finally:
compose('up', '-d', '--no-deps', '--wait', 'api', quiet=True)
wait_replicas(count)
def outage(service, limit):
"""Stop one shared store and look at the answers through the load balancer."""
user, password, token = new_user(label=service)
compose('stop', service)
try:
me = show('GET /users/me with a valid token (first seconds)', http('GET', '/api/v1/users/me', token))
time.sleep(3) # the load balancer has seen failing readiness checks by now
me_later = show('GET /users/me with a valid token (after 3 s)', http('GET', '/api/v1/users/me', token))
anonymous = show('GET /users/me without a token', http('GET', '/api/v1/users/me'))
ready = show('GET /health/ready', http('GET', '/health/ready'))
slowest = max(http('GET', '/api/v1/users/me', token)[2] for _ in range(3))
out('slowest of 3 more authenticated requests: %.2f s' % slowest)
finally:
compose('start', service)
started = time.monotonic()
recovered = False
while time.monotonic() - started < 60 and not recovered:
recovered = http('GET', '/api/v1/users/me', token, timeout=5)[0] == 200
time.sleep(0.5)
out('same token after %s is back: %s' % (service, '200 after %.1f s' % (time.monotonic() - started) if recovered else 'not accepted'))
verdict('without %s: bounded 503, nothing is served as anonymous' % service,
me == 503 and me_later == 503 and anonymous in (401, 503) and ready == 503 and slowest < limit)
verdict('the stand recovers by itself when %s is back, the session is still valid' % service, recovered)
def section_valkey_down():
out('== Valkey is down ==')
outage('valkey', 3)
def section_postgres_down():
out('== PostgreSQL is down ==')
outage('postgres', 5)
def section_failover():
out('== Public failover test: tests/resilience.py --mode failover, repeated until every replica was the target ==')
hook = 'scripts/stop-one.cmd' if os.name == 'nt' else 'scripts/stop-one.sh'
replicas = api_replicas()
targeted = set()
for attempt in range(1, 7):
before = set(api_replicas())
_, code, seconds = run(PY + ['tests/resilience.py', '--mode', 'failover', '--action-script', hook, '--base-url', BASE], check=False)
stopped = before - set(api_replicas())
for cid in stopped:
out('attempt %d: stopped container %s (%s), State: %s, ExitCode %s' % (
attempt, cid, replicas.get(cid), inspect(cid, '{{.State.Status}}'), inspect(cid, '{{.State.ExitCode}}')))
verdict('public failover test passes (attempt %d)' % attempt, code == 0 and len(stopped) == 1, '%.1f s' % seconds)
targeted |= stopped
compose('up', '-d', '--wait', quiet=True)
if not wait_replicas(len(replicas)):
raise Stop('replicas did not come back')
if targeted >= set(replicas):
break
verdict('every replica was stopped by the hook at least once', targeted >= set(replicas),
', '.join(sorted(replicas[c] for c in targeted if c in replicas)))
def section_experiment():
out('== Experiment: stop, then kill, each replica in turn under load; restore before the next one ==')
replicas = api_replicas()
count = len(replicas)
load = Load()
poller = StatsPoller()
poller.start()
load.start()
rows = []
try:
time.sleep(5)
for mode in ('stop', 'kill'):
for cid, name in sorted(replicas.items(), key=lambda item: item[1]):
address = inspect(cid, '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}')
slot = poller.slot_of(address)
base_start = time.monotonic() - 5
baseline = load.window(base_start, time.monotonic())
t0 = time.monotonic()
_, _, command_seconds = run(['docker', mode, cid], quiet=True)
time.sleep(8)
t1 = time.monotonic()
during = load.window(t0, t1)
gap = load.max_gap(t0, t1)
detected = poller.first_change(slot, ('DOWN', 'MAINT', 'NOLB'), t0)
last_served = load.last_served_by(cid, t0)
run(['docker', 'start', cid], quiet=True)
t2 = time.monotonic()
back = None
while time.monotonic() - t2 < 60 and back is None:
time.sleep(0.2)
back = load.first_served_by(cid, t2)
time.sleep(3)
after = load.window(back or t2, time.monotonic())
row = {
'replica': name, 'lb_slot': slot, 'action': 'docker ' + mode, 'command_s': round(command_seconds, 2),
'baseline_ok_rps': baseline['ok_rps'], 'during_ok_rps': during['ok_rps'],
'errors_during': during['errors'], 'requests_during': during['requests'],
'max_gap_s': round(gap, 2),
'lb_marked_down_s': round(detected - t0, 2) if detected else None,
'last_served_s': round(last_served - t0, 2) if last_served else None,
'back_in_rotation_s': round(back - t2, 2) if back else None,
'after_ok_rps': after['ok_rps'], 'errors_after': after['errors'],
}
rows.append(row)
out(json.dumps(row, ensure_ascii=False))
if not wait_replicas(count):
raise Stop('replicas did not come back after %s' % name)
time.sleep(2)
finally:
load.stop()
poller.stop_event.set()
acknowledged, present, duplicates = load.verify()
out()
out('%-16s %-12s %6s %8s %8s %7s %8s %8s %8s %8s' % (
'replica', 'action', 'cmd,s', 'rps0', 'rps', 'errors', 'gap,s', 'down,s', 'back,s', 'rps1'))
for r in rows:
out('%-16s %-12s %6s %8s %8s %7s %8s %8s %8s %8s' % (
r['replica'], r['action'], r['command_s'], r['baseline_ok_rps'], r['during_ok_rps'], r['errors_during'],
r['max_gap_s'], r['lb_marked_down_s'], r['back_in_rotation_s'], r['after_ok_rps']))
out('acknowledged writes: %d, found in history: %d, duplicates: %d' % (acknowledged, present, duplicates))
stops = [r for r in rows if r['action'] == 'docker stop']
kills = [r for r in rows if r['action'] == 'docker kill']
verdict('graceful stop of each replica: no client errors, LB removes it within 3 s',
len(stops) == count and all(r['errors_during'] == 0 and r['lb_marked_down_s'] is not None
and r['lb_marked_down_s'] <= 3 for r in stops))
verdict('kill of each replica: LB removes it within 3 s, writes go on within 10 s',
len(kills) == count and all(r['lb_marked_down_s'] is not None and r['lb_marked_down_s'] <= 3
and r['max_gap_s'] <= 10 for r in kills))
verdict('every replica comes back into rotation after docker start', all(r['back_in_rotation_s'] is not None for r in rows))
verdict('no acknowledged write is lost or duplicated (retries reuse the key)', acknowledged == present and duplicates == 0)
def section_rolling():
out('== Rolling update under load: one replica at a time, new image, migrations once ==')
replicas = api_replicas()
count = len(replicas)
old_image = inspect('mtusi-messenger:lab3', '{{.Id}}')
version = 'rolling-%d' % int(time.time())
env = dict(os.environ, APP_VERSION=version)
compose('build', 'api', env=env, quiet=True)
new_image = inspect('mtusi-messenger:lab3', '{{.Id}}')
out('image before %s, after %s, APP_VERSION=%s' % (old_image[7:19], new_image[7:19], version))
migrate = compose('run', '--rm', '--no-deps', 'migrate', env=env)[0]
load = Load()
load.start()
started = time.monotonic()
steps = []
try:
time.sleep(3)
for old in sorted(replicas, key=lambda cid: replicas[cid]):
known = set(api_replicas(all_states=True))
compose('up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=%d' % (count + 1), 'api', env=env, quiet=True)
new = [cid for cid in api_replicas() if cid not in known]
if len(new) != 1 or not wait_replicas(count + 1):
raise Stop('new replica did not become ready')
run(['docker', 'stop', old], quiet=True)
run(['docker', 'rm', old], quiet=True)
steps.append('%s (%s) -> %s' % (replicas[old], old, new[0]))
out('replaced ' + steps[-1])
time.sleep(3)
finally:
load.stop()
stats = load.window(started, time.monotonic())
acknowledged, present, duplicates = load.verify()
final = api_replicas()
images = {cid: inspect(cid, '{{.Image}}') for cid in final}
versions = {cid: silent(['docker', 'exec', cid, 'printenv', 'APP_VERSION']) for cid in final}
out('during the update: %s' % json.dumps(stats))
out('final replicas: ' + ', '.join('%s %s version=%s' % (final[c], c, versions[c]) for c in sorted(final)))
out('acknowledged writes: %d, found in history: %d, duplicates: %d' % (acknowledged, present, duplicates))
verdict('migrations are applied once and the change is backward compatible', 'applied_now=0' in migrate)
verdict('every replica was replaced one by one by the new image under load',
len(steps) == count and len(final) == count and set(images.values()) == {new_image}
and set(versions.values()) == {version})
verdict('no client errors and no lost or duplicated writes during the rolling update',
stats['errors'] == 0 and acknowledged == present and duplicates == 0,
'%d requests, %.1f ok RPS' % (stats['requests'], stats['ok_rps']))
def section_persistence():
out('== Full restart: public tests/resilience.py --mode persistence with the restart hook ==')
hook = 'scripts/restart.cmd' if os.name == 'nt' else 'scripts/restart.sh'
services = {name: cid for cid, name in api_replicas().items()}
services.update({s: container(s) for s in ('postgres', 'valkey', 'lb')})
before = {name: inspect(cid, '{{.State.StartedAt}}') for name, cid in services.items()}
volumes = ['%s_pgdata' % PROJECT, '%s_valkeydata' % PROJECT]
volumes_before = run(['docker', 'volume', 'inspect', '-f', '{{.Name}} {{.CreatedAt}}'] + volumes)[0]
_, code, seconds = run(PY + ['tests/resilience.py', '--mode', 'persistence', '--action-script', hook, '--base-url', BASE], check=False)
after = {name: inspect(cid, '{{.State.StartedAt}}') for name, cid in services.items()}
volumes_after = run(['docker', 'volume', 'inspect', '-f', '{{.Name}} {{.CreatedAt}}'] + volumes)[0]
for name in sorted(before):
out('%-18s StartedAt %s -> %s' % (name, before[name][11:23], after[name][11:23]))
restarted = all(before[name] != after[name] for name in before)
verdict('persistence test passes after restarting every replica, postgres, valkey and lb with the same volumes',
code == 0 and restarted and volumes_before == volumes_after, 'hook %s, %.1f s' % (hook, seconds))
SECTIONS = {
'stack': section_stack,
'migrations': section_migrations,
'balancing': section_balancing,
'idempotency': section_idempotency,
'rollback': section_rollback,
'race': section_race,
'pool': section_pool,
'timeouts': section_timeouts,
'logs': section_logs,
'expiry': section_expiry,
'valkey-down': section_valkey_down,
'postgres-down': section_postgres_down,
'failover': section_failover,
'experiment': section_experiment,
'rolling': section_rolling,
'persistence': section_persistence,
}
def main():
parser = argparse.ArgumentParser(description='Lab 3 infrastructure checks; creates synthetic users and messages')
parser.add_argument('--only', nargs='+', choices=sorted(SECTIONS), help='run only these sections')
parser.add_argument('--output', default=None, help='also write the transcript to this file')
args = parser.parse_args()
sys.stdout.reconfigure(errors='replace')
out('# %s UTC, python %s, %s' % (time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()), sys.version.split()[0], sys.platform))
out('# ' + silent(['docker', 'version', '--format', 'docker {{.Server.Version}}']))
if not wait_ready(seconds=30):
out('API is not ready on %s: start the stand with docker compose up -d --build --wait' % BASE)
return 2
for name in args.only or SECTIONS:
try:
SECTIONS[name]()
except (Stop, KeyError, TypeError, ValueError) as error:
verdict('section %s completed' % name, False, str(error) or type(error).__name__)
if not wait_ready(seconds=60):
verdict('stand is ready after section %s' % name, False)
break
out('== Summary ==')
for name, ok in results:
out('%s %s' % ('PASS' if ok else 'FAIL', name))
failed = [name for name, ok in results if not ok]
out('RESULT: %s' % ('FAIL' if failed else 'PASS'))
if args.output:
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
Path(args.output).write_text('\n'.join(lines) + '\n', encoding='utf-8')
return 1 if failed else 0
if __name__ == '__main__':
try:
sys.exit(main())
except FileNotFoundError:
print('docker CLI was not found: install Docker and make sure it is on PATH')
sys.exit(2)