#!/usr/bin/env python3 """Infrastructure checks for lab 2 that need control over the compose stand. python scripts/lab2_checks.py --output evidence/lab2-checks.txt python scripts/lab2_checks.py --only valkey-down postgres-down The stand must already be running (docker compose up -d --build). Every command is printed together with its output. Sections stop and start containers, but never delete volumes, never flush Valkey and never print secrets or session tokens. Standard library only; works on Windows, Linux and macOS. """ import argparse import base64 import concurrent.futures import hashlib import json import os from pathlib import Path import subprocess import sys import threading import time import urllib.error import urllib.request import uuid ROOT = Path(__file__).resolve().parents[1] BASE = 'http://127.0.0.1:8080' SHORT_TTL_BASE = 'http://127.0.0.1:8081' VALKEY_CLI = 'valkey-cli --no-auth-warning -a "$(sed -n "s/^requirepass //p" /run/secrets/valkey_auth)" ' lines = [] results = [] class Stop(Exception): pass def out(text=''): lines.append(text) print(text, flush=True) def run(command, check=True, quiet=False, env=None): """Run a command and echo it with its output; return (output, exit code, seconds).""" out('$ ' + ' '.join(command)) started = time.monotonic() done = subprocess.run(command, cwd=str(ROOT), env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) elapsed = time.monotonic() - started text = done.stdout.decode('utf-8', 'replace').strip() if text and not quiet: out(text) if check and done.returncode != 0: if quiet and text: out(text) raise Stop('command failed with exit code %d' % done.returncode) return text, done.returncode, elapsed def compose(*args, **kwargs): return run(['docker', 'compose'] + list(args), **kwargs) PSQL = ['exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-v', 'ON_ERROR_STOP=1', '-At', '-c'] def psql(sql, quiet=False): if quiet: done = subprocess.run(['docker', 'compose'] + PSQL + [sql], cwd=str(ROOT), stdout=subprocess.PIPE, text=True) return done.stdout.strip() return compose(*(PSQL + [sql]))[0] def valkey_cli(command): return compose('exec', '-T', 'valkey', 'sh', '-c', VALKEY_CLI + command)[0] def verdict(name, ok, detail=''): results.append((name, ok)) out('=> %s: %s%s' % ('PASS' if ok else 'FAIL', name, ' (' + detail + ')' if detail else '')) out() def http(method, path, token=None, body=None, headers=None, base=BASE, timeout=10): """Return (status, parsed JSON or None, seconds); status None when the connection fails.""" all_headers = dict(headers or {}) if token: all_headers['Authorization'] = 'Bearer ' + token data = None if body is not None: data = json.dumps(body).encode('utf-8') all_headers['Content-Type'] = 'application/json' request = urllib.request.Request(base + path, data=data, headers=all_headers, method=method) started = time.monotonic() try: with urllib.request.urlopen(request, timeout=timeout) as response: raw = response.read() status = response.status except urllib.error.HTTPError as error: with error: raw = error.read() status = error.code except OSError: return None, None, time.monotonic() - started return status, (json.loads(raw) if raw else None), time.monotonic() - started def basic(username, password): return {'Authorization': 'Basic ' + base64.b64encode((username + ':' + password).encode()).decode()} def new_user(base=BASE, label='check'): """Register a synthetic user and log in: (user, password, token). The token is never printed.""" username = label + '_' + uuid.uuid4().hex[:12] password = 'Check-' + uuid.uuid4().hex status, user, _ = http('POST', '/api/v1/users', body={'username': username, 'display_name': 'Check', 'password': password}, base=base) if status != 201: raise Stop('registration failed with %s' % status) status, session, _ = http('POST', '/api/v1/auth/sessions', headers=basic(username, password), base=base) if status != 201: raise Stop('login failed with %s' % status) return user, password, session['token'] def wait_ready(base=BASE, seconds=60): deadline = time.monotonic() + seconds while time.monotonic() < deadline: if http('GET', '/health/ready', base=base, timeout=3)[0] == 200: return True time.sleep(0.5) return False def container(service): done = subprocess.run(['docker', 'compose', 'ps', '-a', '-q', service], cwd=str(ROOT), stdout=subprocess.PIPE, text=True) return done.stdout.strip() def started_at(service): return subprocess.run(['docker', 'inspect', '-f', '{{.State.StartedAt}}', container(service)], stdout=subprocess.PIPE, text=True).stdout.strip() def show(label, result): status, payload, seconds = result code = payload['error']['code'] if isinstance(payload, dict) and 'error' in payload else '' out('%-58s -> %s %s in %.2f s' % (label, status, code, seconds)) return status def section_stack(): out('== Stack: services, pinned images, health, non-root, Valkey persistence ==') compose('ps', '-a', '--format', 'table {{.Service}}\t{{.Image}}\t{{.Status}}') config = json.loads(compose('config', '--format', 'json', quiet=True)[0]) images = {name: service.get('image', '') for name, service in config['services'].items()} pinned = all('@sha256:' in images[name] for name in ('postgres', 'valkey')) verdict('third-party images are pinned by tag and digest', pinned, ', '.join(images[n] for n in ('postgres', 'valkey'))) health = {s: subprocess.run(['docker', 'inspect', '-f', '{{.State.Health.Status}}', container(s)], stdout=subprocess.PIPE, text=True).stdout.strip() for s in ('api', 'postgres', 'valkey')} out('health: ' + ', '.join('%s=%s' % item for item in health.items())) verdict('api, postgres and valkey are healthy', set(health.values()) == {'healthy'}) identity = compose('exec', '-T', 'api', 'id')[0] verdict('API process does not run as root', 'uid=10001(app)' in identity) aof = valkey_cli('CONFIG GET appendonly') + ' ' + valkey_cli('CONFIG GET appendfsync') + ' ' + valkey_cli('CONFIG GET maxmemory-policy') verdict('Valkey persistence is AOF with fsync always, no eviction', all(word in aof.split() for word in ('yes', 'always', 'noeviction'))) def section_migrations(): out('== Migrations: schema, constraints, idempotent re-run ==') psql("SELECT format('%s %s %s', conrelid::regclass, contype, conname) FROM pg_constraint " "WHERE conrelid::regclass::text IN ('users', 'chats', 'chat_members', 'messages') ORDER BY 1") psql("SELECT indexrelid::regclass FROM pg_index WHERE indrelid::regclass::text IN ('chat_members', 'messages') ORDER BY 1") before = psql('SELECT (SELECT count(*) FROM schema_migrations) || \' \' || (SELECT count(*) FROM users)') output = compose('run', '--rm', '--no-deps', 'migrate')[0] after = psql('SELECT (SELECT count(*) FROM schema_migrations) || \' \' || (SELECT count(*) FROM users)') out('schema_migrations and users rows before/after: %s / %s' % (before, after)) verdict('re-running migrations applies nothing and duplicates nothing', 'applied_now=0' in output and before == after) def section_rollback(): out('== Rollback: chat creation fails on the second INSERT ==') user, _, token = new_user(label='rollback') title = 'rollback-probe-' + uuid.uuid4().hex[:10] status = show('POST /api/v1/chats with one unknown member', http('POST', '/api/v1/chats', token, {'title': title, 'member_ids': [user['id'], str(uuid.uuid4())]})) rows = psql("SELECT count(*) FROM chats WHERE title = '%s'" % title) orphans = psql('SELECT count(*) FROM chats c WHERE NOT EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = c.id)') logged = 'event=chat_rolled_back' in compose('logs', '--no-log-prefix', '--tail', '20', 'api', quiet=True)[0] out('rows in chats with this title: %s; chats without members in the whole table: %s; rollback logged: %s' % (rows, orphans, logged)) verdict('composite chat creation is rolled back as a whole', status == 400 and rows == '0' and orphans == '0' and logged) def section_race(): out('== Registration race: 20 parallel requests with one username ==') username = 'race_' + uuid.uuid4().hex[:12] barrier = threading.Barrier(20) def register(index): barrier.wait() body = {'username': username, 'display_name': 'Race %d' % index, 'password': 'Race-password-%02d' % index} return http('POST', '/api/v1/users', body=body)[0] with concurrent.futures.ThreadPoolExecutor(max_workers=20) as pool: statuses = sorted(pool.map(register, range(20)), key=str) out('statuses: ' + ' '.join(str(s) for s in statuses)) rows = psql("SELECT count(*) FROM users WHERE username = '%s'" % username) out('rows in users with this username: ' + rows) verdict('exactly one registration wins, the other 19 get 409', statuses == [201] + [409] * 19 and rows == '1') def section_pool(): out('== Connection pool: 60 parallel clients, at most DB_POOL_SIZE=10 connections to PostgreSQL ==') users = [new_user(label='pool') for _ in range(4)] chats = [http('POST', '/api/v1/chats', token, {'title': 'Pool', 'member_ids': [user['id']]})[1] for user, _, token in users] count_sql = "SELECT count(*) FROM pg_stat_activity WHERE application_name = 'messenger-api'" stop = threading.Event() samples = [] def sample(): while not stop.is_set(): value = psql(count_sql, quiet=True) if value.isdigit(): samples.append(int(value)) time.sleep(0.2) def client(index): _, _, token = users[index % 4] path = '/api/v1/chats/%s/messages?limit=20' % chats[index % 4]['id'] statuses = [] deadline = time.monotonic() + 5 while time.monotonic() < deadline: statuses.append(http('GET', path, token)[0]) return statuses sampler = threading.Thread(target=sample) sampler.start() try: with concurrent.futures.ThreadPoolExecutor(max_workers=60) as pool: statuses = [status for batch in pool.map(client, range(60)) for status in batch] finally: stop.set() sampler.join() counts = {status: statuses.count(status) for status in sorted(set(statuses), key=str)} out('requests: %d, statuses: %s' % (len(statuses), counts)) out('API connections in pg_stat_activity, %d samples: max %s' % (len(samples), max(samples) if samples else '-')) verdict('the pool never opens more than 10 connections and all requests succeed', bool(samples) and max(samples) <= 10 and set(counts) == {200}) def section_timeouts(): out('== Timeouts: chat row locked for 6 s by another session, API answers 503 after lock_timeout=3s ==') user, _, token = new_user(label='lock') chat = http('POST', '/api/v1/chats', token, {'title': 'Lock', 'member_ids': [user['id']]})[1] path = '/api/v1/chats/%s/messages' % chat['id'] lock = "BEGIN; SELECT 1 FROM chats WHERE id = '%s' FOR UPDATE; SELECT pg_sleep(6); ROLLBACK;" % chat['id'] out('$ docker compose ' + ' '.join(PSQL) + ' "' + lock + '" (in the background)') locker = subprocess.Popen(['docker', 'compose'] + PSQL + [lock], cwd=str(ROOT), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) time.sleep(1.5) status, _, seconds = http('POST', path, token, {'text': 'while locked'}) show('POST message while the chat row is locked', (status, _, seconds)) locker.wait(timeout=30) after = show('POST message after the lock is released', http('POST', path, token, {'text': 'after lock'})) verdict('waiting for a lock is bounded: 503 after about 3 s, then writes work again', status == 503 and 2.5 <= seconds <= 4.5 and after == 201, '%.2f s' % seconds) def section_logs(): out('== Logs: useful fields, no message texts, passwords or tokens ==') user, password, token = new_user(label='logs') chat = http('POST', '/api/v1/chats', token, {'title': 'Logs', 'member_ids': [user['id']]})[1] marker = 'log-marker-' + uuid.uuid4().hex http('POST', '/api/v1/chats/%s/messages' % chat['id'], token, {'text': marker}) logs = compose('logs', '--no-log-prefix', 'api', quiet=True)[0] compose('logs', '--no-log-prefix', '--tail', '3', 'api') leaked = [name for name, value in (('message text', marker), ('password', password), ('token', token)) if value in logs] secrets_found = [p.name for p in (ROOT / 'secrets').glob('*_password') if p.read_text().strip() in logs] out('searched %d log lines; found: %s' % (logs.count('\n') + 1, ', '.join(leaked + secrets_found) or 'nothing')) verdict('logs contain request_id/status/duration and no secrets', 'request_id=' in logs and not leaked and not secrets_found) def section_expiry(): out('== Session expiry: second API instance with SESSION_TTL_SECONDS=2 ==') compose('--profile', 'expiry', 'up', '-d', 'api-short-ttl') try: if not wait_ready(SHORT_TTL_BASE): raise Stop('api-short-ttl is not ready') user, _, token = new_user(SHORT_TTL_BASE, label='expiry') key = 'session:' + hashlib.sha256(token.encode()).hexdigest() first = show('GET /users/me right after login', http('GET', '/api/v1/users/me', token, base=SHORT_TTL_BASE)) pttl = valkey_cli('PTTL ' + key) out('PTTL of the session key in Valkey (ms): ' + pttl) time.sleep(3) expired = show('GET /users/me 3 s later', http('GET', '/api/v1/users/me', token, base=SHORT_TTL_BASE)) gone = valkey_cli('EXISTS ' + key) out('session key still exists in Valkey: ' + gone) verdict('session expires after SESSION_TTL_SECONDS and the key disappears from Valkey', first == 200 and 0 < int(pttl) <= 2000 and expired == 401 and gone == '0') finally: compose('--profile', 'expiry', 'rm', '-sf', 'api-short-ttl', quiet=True) def section_valkey_down(): out('== Valkey is down: bounded 503, no anonymous access, recovery with the same token ==') user, password, token = new_user(label='valkey') compose('stop', 'valkey') try: me = show('GET /users/me with a valid token', http('GET', '/api/v1/users/me', token)) login = show('POST /auth/sessions with valid credentials', http('POST', '/api/v1/auth/sessions', headers=basic(user['username'], password))) anonymous = show('GET /users/me without a token', http('GET', '/api/v1/users/me')) register = show('POST /users (needs only PostgreSQL)', http('POST', '/api/v1/users', body={ 'username': 'novalkey_' + uuid.uuid4().hex[:10], 'display_name': 'No Valkey', 'password': 'No-valkey-password-1'})) ready = show('GET /health/ready', http('GET', '/health/ready')) live = show('GET /health/live', http('GET', '/health/live')) slowest = max(http('GET', '/api/v1/users/me', token)[2] for _ in range(3)) out('slowest of 3 more authenticated requests: %.2f s' % slowest) finally: compose('start', 'valkey') recovered = False deadline = time.monotonic() + 60 while time.monotonic() < deadline and not recovered: recovered = http('GET', '/api/v1/users/me', token, timeout=3)[0] == 200 time.sleep(0.5) out('same token after Valkey is back: %s' % ('200' if recovered else 'not accepted within 60 s')) verdict('without Valkey: 503 for sessions, 401 for anonymous, registration still works', me == 503 and login == 503 and anonymous == 401 and register == 201 and ready == 503 and live == 200 and slowest < 3) verdict('after Valkey restarts the session is still valid (AOF)', recovered) def section_postgres_down(): out('== PostgreSQL is down: bounded 503, recovery without restarting the API ==') user, _, token = new_user(label='postgres') compose('stop', 'postgres') try: me = show('GET /users/me with a valid token', http('GET', '/api/v1/users/me', token)) chat = show('POST /chats', http('POST', '/api/v1/chats', token, {'title': 'No DB', 'member_ids': [user['id']]})) anonymous = show('GET /users/me without a token', http('GET', '/api/v1/users/me')) ready = show('GET /health/ready', http('GET', '/health/ready')) slowest = max(http('GET', '/api/v1/users/me', token)[2] for _ in range(3)) out('slowest of 3 more requests: %.2f s' % slowest) finally: compose('start', 'postgres') recovered = False deadline = time.monotonic() + 60 while time.monotonic() < deadline and not recovered: recovered = http('GET', '/api/v1/users/me', token, timeout=5)[0] == 200 time.sleep(0.5) out('same token after PostgreSQL is back: %s' % ('200' if recovered else 'not accepted within 60 s')) verdict('without PostgreSQL: bounded 503, anonymous still 401', me == 503 and chat == 503 and anonymous == 401 and ready == 503 and slowest < 5) verdict('API recovers by itself when PostgreSQL is back', recovered) def section_persistence(): out('== Full restart: public tests/resilience.py with the restart hook ==') hook = 'scripts/restart.cmd' if os.name == 'nt' else 'scripts/restart.sh' volumes_before = run(['docker', 'volume', 'inspect', '-f', '{{.Name}} {{.CreatedAt}}', 'mtusi-lab2_pgdata', 'mtusi-lab2_valkeydata'])[0] before = {s: started_at(s) for s in ('api', 'postgres', 'valkey')} _, code, seconds = run([sys.executable, 'tests/resilience.py', '--mode', 'persistence', '--action-script', hook, '--base-url', BASE], check=False) after = {s: started_at(s) for s in ('api', 'postgres', 'valkey')} volumes_after = run(['docker', 'volume', 'inspect', '-f', '{{.Name}} {{.CreatedAt}}', 'mtusi-lab2_pgdata', 'mtusi-lab2_valkeydata'])[0] for service in before: out('%-8s StartedAt %s -> %s' % (service, before[service], after[service])) restarted = all(before[s] != after[s] for s in before) verdict('resilience test passes after restarting api, postgres and valkey with the same volumes', code == 0 and restarted and volumes_before == volumes_after, 'hook %s, %.1f s' % (hook, seconds)) def section_sigterm(): out('== SIGTERM: docker compose stop api ==') _, _, seconds = compose('stop', '-t', '10', 'api') exit_code = subprocess.run(['docker', 'inspect', '-f', '{{.State.ExitCode}}', container('api')], stdout=subprocess.PIPE, text=True).stdout.strip() compose('logs', '--no-log-prefix', '--tail', '2', 'api') refused = http('GET', '/health/live', timeout=3)[0] is None compose('start', 'api') ready = wait_ready() out('stop took %.2f s, exit code %s, new request refused: %s, ready again: %s' % (seconds, exit_code, refused, ready)) verdict('SIGTERM stops the API cleanly within 10 s', exit_code == '0' and seconds < 10 and refused and ready) SECTIONS = { 'stack': section_stack, 'migrations': section_migrations, 'rollback': section_rollback, 'race': section_race, 'pool': section_pool, 'timeouts': section_timeouts, 'logs': section_logs, 'expiry': section_expiry, 'valkey-down': section_valkey_down, 'postgres-down': section_postgres_down, 'persistence': section_persistence, 'sigterm': section_sigterm, } def main(): parser = argparse.ArgumentParser(description='Lab 2 infrastructure checks; creates synthetic users and messages') parser.add_argument('--only', nargs='+', choices=sorted(SECTIONS), help='run only these sections') parser.add_argument('--output', default=None, help='also write the transcript to this file') args = parser.parse_args() sys.stdout.reconfigure(errors='replace') out('# %s UTC, python %s, %s' % (time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()), sys.version.split()[0], sys.platform)) out('# ' + run(['docker', 'version', '--format', 'docker {{.Server.Version}}'], quiet=True)[0]) if not wait_ready(seconds=30): out('API is not ready on %s: start the stand with docker compose up -d --build' % BASE) return 2 for name in args.only or SECTIONS: try: SECTIONS[name]() except (Stop, KeyError, TypeError, ValueError) as error: verdict('section %s completed' % name, False, str(error) or type(error).__name__) if not wait_ready(seconds=60): verdict('stand is ready after section %s' % name, False) break out('== Summary ==') for name, ok in results: out('%s %s' % ('PASS' if ok else 'FAIL', name)) failed = [name for name, ok in results if not ok] out('RESULT: %s' % ('FAIL' if failed else 'PASS')) if args.output: Path(args.output).parent.mkdir(parents=True, exist_ok=True) Path(args.output).write_text('\n'.join(lines) + '\n', encoding='utf-8') return 1 if failed else 0 if __name__ == '__main__': try: sys.exit(main()) except FileNotFoundError: print('docker CLI was not found: install Docker and make sure it is on PATH') sys.exit(2)