forked from mtusi2026/mtusi2026lab2
ЛР 2 — БВТ2304 — Черников
PostgreSQL с версионируемыми миграциями, вход по Basic и Bearer-сессии в Valkey (AOF), Argon2id, compose-стенд с секретами в файлах. Собственные тесты в student-tests/, проверки инфраструктуры и EXPLAIN в scripts/, отчёт в REPORT.md, результаты в evidence/, команды для показа в DEMO.md. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Executable
+139
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env python3
|
||||
"""EXPLAIN (ANALYZE, BUFFERS) of the history page query on 10 000+ messages.
|
||||
|
||||
python scripts/explain_history.py --output evidence/explain.txt
|
||||
|
||||
The stand must be running. The script registers two synthetic users through the API,
|
||||
creates a target chat and a noise chat, bulk-inserts messages with SQL, runs ANALYZE and
|
||||
prints the plans of the same query the API runs (src/messenger/db.py, list_messages):
|
||||
first page, a page in the middle, the last page, the generic plan of the parameterized
|
||||
query, and for comparison the middle page without the index (DROP INDEX inside a
|
||||
transaction that is rolled back, so the database is left unchanged).
|
||||
Standard library only.
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE = 'http://127.0.0.1:8080'
|
||||
PSQL = ['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
|
||||
'-X', '-v', 'ON_ERROR_STOP=1']
|
||||
PAGE = 51
|
||||
|
||||
lines = []
|
||||
|
||||
|
||||
def out(text=''):
|
||||
lines.append(text)
|
||||
print(text, flush=True)
|
||||
|
||||
|
||||
def psql(sql, echo=True):
|
||||
"""Run SQL in one psql session; with echo the transcript shows every statement before its result."""
|
||||
args = PSQL + (['--echo-all'] if echo else ['-At']) + ['-f', '-']
|
||||
done = subprocess.run(args, cwd=str(ROOT), input=sql.encode('utf-8'), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
text = done.stdout.decode('utf-8', 'replace').strip()
|
||||
if done.returncode != 0:
|
||||
out(text)
|
||||
raise SystemExit('psql failed')
|
||||
return text
|
||||
|
||||
|
||||
def api(method, path, body=None, headers=None):
|
||||
headers = dict(headers or {})
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode('utf-8')
|
||||
headers['Content-Type'] = 'application/json'
|
||||
request = urllib.request.Request(BASE + path, data=data, headers=headers, method=method)
|
||||
with urllib.request.urlopen(request, timeout=10) as response:
|
||||
return json.loads(response.read())
|
||||
|
||||
|
||||
def user(label):
|
||||
username = label + '_' + uuid.uuid4().hex[:10]
|
||||
password = 'Explain-' + uuid.uuid4().hex
|
||||
created = api('POST', '/api/v1/users', {'username': username, 'display_name': label, 'password': password})
|
||||
credential = base64.b64encode((username + ':' + password).encode()).decode()
|
||||
session = api('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
|
||||
return created['id'], session['token']
|
||||
|
||||
|
||||
def page_sql(chat_id, after=None):
|
||||
"""The same SQL text as Repository.list_messages, with literal values."""
|
||||
sql = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = '%s' " % chat_id
|
||||
if after:
|
||||
sql += "AND (created_at, id) > ('%s', '%s') " % after
|
||||
return sql + 'ORDER BY created_at, id LIMIT %d' % PAGE
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='EXPLAIN of the history query; inserts synthetic messages')
|
||||
parser.add_argument('--messages', type=int, default=10000, help='messages in the target chat (>= 10000)')
|
||||
parser.add_argument('--noise', type=int, default=20000, help='messages in another chat')
|
||||
parser.add_argument('--output', default=None)
|
||||
args = parser.parse_args()
|
||||
sys.stdout.reconfigure(errors='replace')
|
||||
|
||||
out('# %s UTC, %s' % (time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()), psql('SELECT version()', echo=False)))
|
||||
alice, alice_token = user('explain_a')
|
||||
bob, _ = user('explain_b')
|
||||
target = api('POST', '/api/v1/chats', {'title': 'EXPLAIN target', 'member_ids': [alice, bob]},
|
||||
{'Authorization': 'Bearer ' + alice_token})['id']
|
||||
noise = api('POST', '/api/v1/chats', {'title': 'EXPLAIN noise', 'member_ids': [alice]},
|
||||
{'Authorization': 'Bearer ' + alice_token})['id']
|
||||
out('# target chat %s: %d messages; noise chat %s: %d messages' % (target, args.messages, noise, args.noise))
|
||||
|
||||
started = time.monotonic()
|
||||
psql('''
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), '%(target)s', CASE WHEN i %% 2 = 0 THEN '%(alice)s'::uuid ELSE '%(bob)s'::uuid END,
|
||||
'seed message ' || i, clock_timestamp() + i * interval '1 millisecond'
|
||||
FROM generate_series(1, %(messages)d) AS i;
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), '%(noise)s', '%(alice)s', 'noise message ' || i, clock_timestamp() + i * interval '1 millisecond'
|
||||
FROM generate_series(1, %(noise_count)d) AS i;
|
||||
ANALYZE messages;
|
||||
''' % {'target': target, 'noise': noise, 'alice': alice, 'bob': bob, 'messages': args.messages, 'noise_count': args.noise},
|
||||
echo=False)
|
||||
out('# inserted and analyzed in %.1f s' % (time.monotonic() - started))
|
||||
out(psql("SELECT 'messages total: ' || count(*) || ', in target chat: ' || count(*) FILTER (WHERE chat_id = '%s') FROM messages" % target, echo=False))
|
||||
|
||||
def key_at(position):
|
||||
row = psql("SELECT created_at || '|' || id FROM messages WHERE chat_id = '%s' ORDER BY created_at, id OFFSET %d LIMIT 1"
|
||||
% (target, position - 1), echo=False)
|
||||
return tuple(row.split('|'))
|
||||
|
||||
middle, last = key_at(args.messages // 2), key_at(args.messages - 20)
|
||||
explain = 'EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) '
|
||||
out()
|
||||
out('== 1. First page, middle page and last page: index range scan, no sort ==')
|
||||
out(psql('\n'.join(explain + page_sql(target, after) + ';' for after in (None, middle, last))))
|
||||
out()
|
||||
out('== 2. Generic plan of the parameterized query (how a prepared statement with $1..$4 runs) ==')
|
||||
out(psql('''SET plan_cache_mode = force_generic_plan;
|
||||
PREPARE history_page(uuid, timestamptz, uuid, int) AS
|
||||
SELECT id, chat_id, sender_id, text, created_at FROM messages
|
||||
WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT $4;
|
||||
%sEXECUTE history_page('%s', '%s', '%s', %d);''' % (explain, target, middle[0], middle[1], PAGE)))
|
||||
out()
|
||||
out('== 3. For comparison: the same middle page without the index (rolled back) ==')
|
||||
out(psql('''BEGIN;
|
||||
DROP INDEX messages_chat_history_idx;
|
||||
%s%s;
|
||||
ROLLBACK;''' % (explain, page_sql(target, middle))))
|
||||
out(psql("SELECT 'index still exists: ' || (to_regclass('messages_chat_history_idx') IS NOT NULL)", echo=False))
|
||||
if args.output:
|
||||
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.output).write_text('\n'.join(lines) + '\n', encoding='utf-8')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+49
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create local secret files for the compose stand.
|
||||
|
||||
python scripts/init_secrets.py
|
||||
|
||||
Passwords are random and synthetic; they are created only if missing and never printed.
|
||||
The URL/config files are always rebuilt from the password files, so the pair stays consistent.
|
||||
secrets/ is listed in .gitignore and .dockerignore.
|
||||
|
||||
PostgreSQL applies its password only when the data volume is created. If secrets/ is
|
||||
deleted while the pgdata volume is kept, restore the old files or recreate the volume
|
||||
(docker compose down -v deletes ALL lab data).
|
||||
"""
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DIR = ROOT / 'secrets'
|
||||
|
||||
|
||||
def write(name, content):
|
||||
path = DIR / name
|
||||
path.write_text(content, encoding='utf-8', newline='\n')
|
||||
path.chmod(0o644)
|
||||
|
||||
|
||||
def password(name):
|
||||
path = DIR / name
|
||||
if path.is_file() and path.read_text(encoding='utf-8').strip():
|
||||
print('keep secrets/' + name)
|
||||
return path.read_text(encoding='utf-8').strip()
|
||||
value = secrets.token_hex(24)
|
||||
write(name, value + '\n')
|
||||
print('created secrets/' + name)
|
||||
return value
|
||||
|
||||
|
||||
def main():
|
||||
DIR.mkdir(exist_ok=True)
|
||||
postgres = password('postgres_password')
|
||||
valkey = password('valkey_password')
|
||||
write('database_url', 'postgresql://messenger:%s@postgres:5432/messenger\n' % postgres)
|
||||
write('valkey_auth', 'requirepass %s\n' % valkey)
|
||||
write('cache_url', 'valkey://:%s@valkey:6379/0\n' % valkey)
|
||||
print('written secrets/database_url, secrets/valkey_auth, secrets/cache_url')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+435
@@ -0,0 +1,435 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Infrastructure checks for lab 2 that need control over the compose stand.
|
||||
|
||||
python scripts/lab2_checks.py --output evidence/lab2-checks.txt
|
||||
python scripts/lab2_checks.py --only valkey-down postgres-down
|
||||
|
||||
The stand must already be running (docker compose up -d --build). Every command is printed
|
||||
together with its output. Sections stop and start containers, but never delete volumes,
|
||||
never flush Valkey and never print secrets or session tokens.
|
||||
Standard library only; works on Windows, Linux and macOS.
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE = 'http://127.0.0.1:8080'
|
||||
SHORT_TTL_BASE = 'http://127.0.0.1:8081'
|
||||
VALKEY_CLI = 'valkey-cli --no-auth-warning -a "$(sed -n "s/^requirepass //p" /run/secrets/valkey_auth)" '
|
||||
|
||||
lines = []
|
||||
results = []
|
||||
|
||||
|
||||
class Stop(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def out(text=''):
|
||||
lines.append(text)
|
||||
print(text, flush=True)
|
||||
|
||||
|
||||
def run(command, check=True, quiet=False, env=None):
|
||||
"""Run a command and echo it with its output; return (output, exit code, seconds)."""
|
||||
out('$ ' + ' '.join(command))
|
||||
started = time.monotonic()
|
||||
done = subprocess.run(command, cwd=str(ROOT), env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
elapsed = time.monotonic() - started
|
||||
text = done.stdout.decode('utf-8', 'replace').strip()
|
||||
if text and not quiet:
|
||||
out(text)
|
||||
if check and done.returncode != 0:
|
||||
if quiet and text:
|
||||
out(text)
|
||||
raise Stop('command failed with exit code %d' % done.returncode)
|
||||
return text, done.returncode, elapsed
|
||||
|
||||
|
||||
def compose(*args, **kwargs):
|
||||
return run(['docker', 'compose'] + list(args), **kwargs)
|
||||
|
||||
|
||||
PSQL = ['exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-v', 'ON_ERROR_STOP=1', '-At', '-c']
|
||||
|
||||
|
||||
def psql(sql, quiet=False):
|
||||
if quiet:
|
||||
done = subprocess.run(['docker', 'compose'] + PSQL + [sql], cwd=str(ROOT), stdout=subprocess.PIPE, text=True)
|
||||
return done.stdout.strip()
|
||||
return compose(*(PSQL + [sql]))[0]
|
||||
|
||||
|
||||
def valkey_cli(command):
|
||||
return compose('exec', '-T', 'valkey', 'sh', '-c', VALKEY_CLI + command)[0]
|
||||
|
||||
|
||||
def verdict(name, ok, detail=''):
|
||||
results.append((name, ok))
|
||||
out('=> %s: %s%s' % ('PASS' if ok else 'FAIL', name, ' (' + detail + ')' if detail else ''))
|
||||
out()
|
||||
|
||||
|
||||
def http(method, path, token=None, body=None, headers=None, base=BASE, timeout=10):
|
||||
"""Return (status, parsed JSON or None, seconds); status None when the connection fails."""
|
||||
all_headers = dict(headers or {})
|
||||
if token:
|
||||
all_headers['Authorization'] = 'Bearer ' + token
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode('utf-8')
|
||||
all_headers['Content-Type'] = 'application/json'
|
||||
request = urllib.request.Request(base + path, data=data, headers=all_headers, method=method)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
raw = response.read()
|
||||
status = response.status
|
||||
except urllib.error.HTTPError as error:
|
||||
with error:
|
||||
raw = error.read()
|
||||
status = error.code
|
||||
except OSError:
|
||||
return None, None, time.monotonic() - started
|
||||
return status, (json.loads(raw) if raw else None), time.monotonic() - started
|
||||
|
||||
|
||||
def basic(username, password):
|
||||
return {'Authorization': 'Basic ' + base64.b64encode((username + ':' + password).encode()).decode()}
|
||||
|
||||
|
||||
def new_user(base=BASE, label='check'):
|
||||
"""Register a synthetic user and log in: (user, password, token). The token is never printed."""
|
||||
username = label + '_' + uuid.uuid4().hex[:12]
|
||||
password = 'Check-' + uuid.uuid4().hex
|
||||
status, user, _ = http('POST', '/api/v1/users', body={'username': username, 'display_name': 'Check', 'password': password}, base=base)
|
||||
if status != 201:
|
||||
raise Stop('registration failed with %s' % status)
|
||||
status, session, _ = http('POST', '/api/v1/auth/sessions', headers=basic(username, password), base=base)
|
||||
if status != 201:
|
||||
raise Stop('login failed with %s' % status)
|
||||
return user, password, session['token']
|
||||
|
||||
|
||||
def wait_ready(base=BASE, seconds=60):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
if http('GET', '/health/ready', base=base, timeout=3)[0] == 200:
|
||||
return True
|
||||
time.sleep(0.5)
|
||||
return False
|
||||
|
||||
|
||||
def container(service):
|
||||
done = subprocess.run(['docker', 'compose', 'ps', '-a', '-q', service], cwd=str(ROOT), stdout=subprocess.PIPE, text=True)
|
||||
return done.stdout.strip()
|
||||
|
||||
|
||||
def started_at(service):
|
||||
return subprocess.run(['docker', 'inspect', '-f', '{{.State.StartedAt}}', container(service)],
|
||||
stdout=subprocess.PIPE, text=True).stdout.strip()
|
||||
|
||||
|
||||
def show(label, result):
|
||||
status, payload, seconds = result
|
||||
code = payload['error']['code'] if isinstance(payload, dict) and 'error' in payload else ''
|
||||
out('%-58s -> %s %s in %.2f s' % (label, status, code, seconds))
|
||||
return status
|
||||
|
||||
|
||||
def section_stack():
|
||||
out('== Stack: services, pinned images, health, non-root, Valkey persistence ==')
|
||||
compose('ps', '-a', '--format', 'table {{.Service}}\t{{.Image}}\t{{.Status}}')
|
||||
config = json.loads(compose('config', '--format', 'json', quiet=True)[0])
|
||||
images = {name: service.get('image', '') for name, service in config['services'].items()}
|
||||
pinned = all('@sha256:' in images[name] for name in ('postgres', 'valkey'))
|
||||
verdict('third-party images are pinned by tag and digest', pinned, ', '.join(images[n] for n in ('postgres', 'valkey')))
|
||||
health = {s: subprocess.run(['docker', 'inspect', '-f', '{{.State.Health.Status}}', container(s)],
|
||||
stdout=subprocess.PIPE, text=True).stdout.strip() for s in ('api', 'postgres', 'valkey')}
|
||||
out('health: ' + ', '.join('%s=%s' % item for item in health.items()))
|
||||
verdict('api, postgres and valkey are healthy', set(health.values()) == {'healthy'})
|
||||
identity = compose('exec', '-T', 'api', 'id')[0]
|
||||
verdict('API process does not run as root', 'uid=10001(app)' in identity)
|
||||
aof = valkey_cli('CONFIG GET appendonly') + ' ' + valkey_cli('CONFIG GET appendfsync') + ' ' + valkey_cli('CONFIG GET maxmemory-policy')
|
||||
verdict('Valkey persistence is AOF with fsync always, no eviction',
|
||||
all(word in aof.split() for word in ('yes', 'always', 'noeviction')))
|
||||
|
||||
|
||||
def section_migrations():
|
||||
out('== Migrations: schema, constraints, idempotent re-run ==')
|
||||
psql("SELECT format('%s %s %s', conrelid::regclass, contype, conname) FROM pg_constraint "
|
||||
"WHERE conrelid::regclass::text IN ('users', 'chats', 'chat_members', 'messages') ORDER BY 1")
|
||||
psql("SELECT indexrelid::regclass FROM pg_index WHERE indrelid::regclass::text IN ('chat_members', 'messages') ORDER BY 1")
|
||||
before = psql('SELECT (SELECT count(*) FROM schema_migrations) || \' \' || (SELECT count(*) FROM users)')
|
||||
output = compose('run', '--rm', '--no-deps', 'migrate')[0]
|
||||
after = psql('SELECT (SELECT count(*) FROM schema_migrations) || \' \' || (SELECT count(*) FROM users)')
|
||||
out('schema_migrations and users rows before/after: %s / %s' % (before, after))
|
||||
verdict('re-running migrations applies nothing and duplicates nothing',
|
||||
'applied_now=0' in output and before == after)
|
||||
|
||||
|
||||
def section_rollback():
|
||||
out('== Rollback: chat creation fails on the second INSERT ==')
|
||||
user, _, token = new_user(label='rollback')
|
||||
title = 'rollback-probe-' + uuid.uuid4().hex[:10]
|
||||
status = show('POST /api/v1/chats with one unknown member',
|
||||
http('POST', '/api/v1/chats', token, {'title': title, 'member_ids': [user['id'], str(uuid.uuid4())]}))
|
||||
rows = psql("SELECT count(*) FROM chats WHERE title = '%s'" % title)
|
||||
orphans = psql('SELECT count(*) FROM chats c WHERE NOT EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = c.id)')
|
||||
logged = 'event=chat_rolled_back' in compose('logs', '--no-log-prefix', '--tail', '20', 'api', quiet=True)[0]
|
||||
out('rows in chats with this title: %s; chats without members in the whole table: %s; rollback logged: %s' % (rows, orphans, logged))
|
||||
verdict('composite chat creation is rolled back as a whole', status == 400 and rows == '0' and orphans == '0' and logged)
|
||||
|
||||
|
||||
def section_race():
|
||||
out('== Registration race: 20 parallel requests with one username ==')
|
||||
username = 'race_' + uuid.uuid4().hex[:12]
|
||||
barrier = threading.Barrier(20)
|
||||
|
||||
def register(index):
|
||||
barrier.wait()
|
||||
body = {'username': username, 'display_name': 'Race %d' % index, 'password': 'Race-password-%02d' % index}
|
||||
return http('POST', '/api/v1/users', body=body)[0]
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=20) as pool:
|
||||
statuses = sorted(pool.map(register, range(20)), key=str)
|
||||
out('statuses: ' + ' '.join(str(s) for s in statuses))
|
||||
rows = psql("SELECT count(*) FROM users WHERE username = '%s'" % username)
|
||||
out('rows in users with this username: ' + rows)
|
||||
verdict('exactly one registration wins, the other 19 get 409', statuses == [201] + [409] * 19 and rows == '1')
|
||||
|
||||
|
||||
def section_pool():
|
||||
out('== Connection pool: 60 parallel clients, at most DB_POOL_SIZE=10 connections to PostgreSQL ==')
|
||||
users = [new_user(label='pool') for _ in range(4)]
|
||||
chats = [http('POST', '/api/v1/chats', token, {'title': 'Pool', 'member_ids': [user['id']]})[1] for user, _, token in users]
|
||||
count_sql = "SELECT count(*) FROM pg_stat_activity WHERE application_name = 'messenger-api'"
|
||||
stop = threading.Event()
|
||||
samples = []
|
||||
|
||||
def sample():
|
||||
while not stop.is_set():
|
||||
value = psql(count_sql, quiet=True)
|
||||
if value.isdigit():
|
||||
samples.append(int(value))
|
||||
time.sleep(0.2)
|
||||
|
||||
def client(index):
|
||||
_, _, token = users[index % 4]
|
||||
path = '/api/v1/chats/%s/messages?limit=20' % chats[index % 4]['id']
|
||||
statuses = []
|
||||
deadline = time.monotonic() + 5
|
||||
while time.monotonic() < deadline:
|
||||
statuses.append(http('GET', path, token)[0])
|
||||
return statuses
|
||||
|
||||
sampler = threading.Thread(target=sample)
|
||||
sampler.start()
|
||||
try:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=60) as pool:
|
||||
statuses = [status for batch in pool.map(client, range(60)) for status in batch]
|
||||
finally:
|
||||
stop.set()
|
||||
sampler.join()
|
||||
counts = {status: statuses.count(status) for status in sorted(set(statuses), key=str)}
|
||||
out('requests: %d, statuses: %s' % (len(statuses), counts))
|
||||
out('API connections in pg_stat_activity, %d samples: max %s' % (len(samples), max(samples) if samples else '-'))
|
||||
verdict('the pool never opens more than 10 connections and all requests succeed',
|
||||
bool(samples) and max(samples) <= 10 and set(counts) == {200})
|
||||
|
||||
|
||||
def section_timeouts():
|
||||
out('== Timeouts: chat row locked for 6 s by another session, API answers 503 after lock_timeout=3s ==')
|
||||
user, _, token = new_user(label='lock')
|
||||
chat = http('POST', '/api/v1/chats', token, {'title': 'Lock', 'member_ids': [user['id']]})[1]
|
||||
path = '/api/v1/chats/%s/messages' % chat['id']
|
||||
lock = "BEGIN; SELECT 1 FROM chats WHERE id = '%s' FOR UPDATE; SELECT pg_sleep(6); ROLLBACK;" % chat['id']
|
||||
out('$ docker compose ' + ' '.join(PSQL) + ' "' + lock + '" (in the background)')
|
||||
locker = subprocess.Popen(['docker', 'compose'] + PSQL + [lock], cwd=str(ROOT), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
time.sleep(1.5)
|
||||
status, _, seconds = http('POST', path, token, {'text': 'while locked'})
|
||||
show('POST message while the chat row is locked', (status, _, seconds))
|
||||
locker.wait(timeout=30)
|
||||
after = show('POST message after the lock is released', http('POST', path, token, {'text': 'after lock'}))
|
||||
verdict('waiting for a lock is bounded: 503 after about 3 s, then writes work again',
|
||||
status == 503 and 2.5 <= seconds <= 4.5 and after == 201, '%.2f s' % seconds)
|
||||
|
||||
|
||||
def section_logs():
|
||||
out('== Logs: useful fields, no message texts, passwords or tokens ==')
|
||||
user, password, token = new_user(label='logs')
|
||||
chat = http('POST', '/api/v1/chats', token, {'title': 'Logs', 'member_ids': [user['id']]})[1]
|
||||
marker = 'log-marker-' + uuid.uuid4().hex
|
||||
http('POST', '/api/v1/chats/%s/messages' % chat['id'], token, {'text': marker})
|
||||
logs = compose('logs', '--no-log-prefix', 'api', quiet=True)[0]
|
||||
compose('logs', '--no-log-prefix', '--tail', '3', 'api')
|
||||
leaked = [name for name, value in (('message text', marker), ('password', password), ('token', token)) if value in logs]
|
||||
secrets_found = [p.name for p in (ROOT / 'secrets').glob('*_password') if p.read_text().strip() in logs]
|
||||
out('searched %d log lines; found: %s' % (logs.count('\n') + 1, ', '.join(leaked + secrets_found) or 'nothing'))
|
||||
verdict('logs contain request_id/status/duration and no secrets', 'request_id=' in logs and not leaked and not secrets_found)
|
||||
|
||||
|
||||
def section_expiry():
|
||||
out('== Session expiry: second API instance with SESSION_TTL_SECONDS=2 ==')
|
||||
compose('--profile', 'expiry', 'up', '-d', 'api-short-ttl')
|
||||
try:
|
||||
if not wait_ready(SHORT_TTL_BASE):
|
||||
raise Stop('api-short-ttl is not ready')
|
||||
user, _, token = new_user(SHORT_TTL_BASE, label='expiry')
|
||||
key = 'session:' + hashlib.sha256(token.encode()).hexdigest()
|
||||
first = show('GET /users/me right after login', http('GET', '/api/v1/users/me', token, base=SHORT_TTL_BASE))
|
||||
pttl = valkey_cli('PTTL ' + key)
|
||||
out('PTTL of the session key in Valkey (ms): ' + pttl)
|
||||
time.sleep(3)
|
||||
expired = show('GET /users/me 3 s later', http('GET', '/api/v1/users/me', token, base=SHORT_TTL_BASE))
|
||||
gone = valkey_cli('EXISTS ' + key)
|
||||
out('session key still exists in Valkey: ' + gone)
|
||||
verdict('session expires after SESSION_TTL_SECONDS and the key disappears from Valkey',
|
||||
first == 200 and 0 < int(pttl) <= 2000 and expired == 401 and gone == '0')
|
||||
finally:
|
||||
compose('--profile', 'expiry', 'rm', '-sf', 'api-short-ttl', quiet=True)
|
||||
|
||||
|
||||
def section_valkey_down():
|
||||
out('== Valkey is down: bounded 503, no anonymous access, recovery with the same token ==')
|
||||
user, password, token = new_user(label='valkey')
|
||||
compose('stop', 'valkey')
|
||||
try:
|
||||
me = show('GET /users/me with a valid token', http('GET', '/api/v1/users/me', token))
|
||||
login = show('POST /auth/sessions with valid credentials', http('POST', '/api/v1/auth/sessions', headers=basic(user['username'], password)))
|
||||
anonymous = show('GET /users/me without a token', http('GET', '/api/v1/users/me'))
|
||||
register = show('POST /users (needs only PostgreSQL)', http('POST', '/api/v1/users', body={
|
||||
'username': 'novalkey_' + uuid.uuid4().hex[:10], 'display_name': 'No Valkey', 'password': 'No-valkey-password-1'}))
|
||||
ready = show('GET /health/ready', http('GET', '/health/ready'))
|
||||
live = show('GET /health/live', http('GET', '/health/live'))
|
||||
slowest = max(http('GET', '/api/v1/users/me', token)[2] for _ in range(3))
|
||||
out('slowest of 3 more authenticated requests: %.2f s' % slowest)
|
||||
finally:
|
||||
compose('start', 'valkey')
|
||||
recovered = False
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline and not recovered:
|
||||
recovered = http('GET', '/api/v1/users/me', token, timeout=3)[0] == 200
|
||||
time.sleep(0.5)
|
||||
out('same token after Valkey is back: %s' % ('200' if recovered else 'not accepted within 60 s'))
|
||||
verdict('without Valkey: 503 for sessions, 401 for anonymous, registration still works',
|
||||
me == 503 and login == 503 and anonymous == 401 and register == 201 and ready == 503 and live == 200 and slowest < 3)
|
||||
verdict('after Valkey restarts the session is still valid (AOF)', recovered)
|
||||
|
||||
|
||||
def section_postgres_down():
|
||||
out('== PostgreSQL is down: bounded 503, recovery without restarting the API ==')
|
||||
user, _, token = new_user(label='postgres')
|
||||
compose('stop', 'postgres')
|
||||
try:
|
||||
me = show('GET /users/me with a valid token', http('GET', '/api/v1/users/me', token))
|
||||
chat = show('POST /chats', http('POST', '/api/v1/chats', token, {'title': 'No DB', 'member_ids': [user['id']]}))
|
||||
anonymous = show('GET /users/me without a token', http('GET', '/api/v1/users/me'))
|
||||
ready = show('GET /health/ready', http('GET', '/health/ready'))
|
||||
slowest = max(http('GET', '/api/v1/users/me', token)[2] for _ in range(3))
|
||||
out('slowest of 3 more requests: %.2f s' % slowest)
|
||||
finally:
|
||||
compose('start', 'postgres')
|
||||
recovered = False
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline and not recovered:
|
||||
recovered = http('GET', '/api/v1/users/me', token, timeout=5)[0] == 200
|
||||
time.sleep(0.5)
|
||||
out('same token after PostgreSQL is back: %s' % ('200' if recovered else 'not accepted within 60 s'))
|
||||
verdict('without PostgreSQL: bounded 503, anonymous still 401',
|
||||
me == 503 and chat == 503 and anonymous == 401 and ready == 503 and slowest < 5)
|
||||
verdict('API recovers by itself when PostgreSQL is back', recovered)
|
||||
|
||||
|
||||
def section_persistence():
|
||||
out('== Full restart: public tests/resilience.py with the restart hook ==')
|
||||
hook = 'scripts/restart.cmd' if os.name == 'nt' else 'scripts/restart.sh'
|
||||
volumes_before = run(['docker', 'volume', 'inspect', '-f', '{{.Name}} {{.CreatedAt}}', 'mtusi-lab2_pgdata', 'mtusi-lab2_valkeydata'])[0]
|
||||
before = {s: started_at(s) for s in ('api', 'postgres', 'valkey')}
|
||||
_, code, seconds = run([sys.executable, 'tests/resilience.py', '--mode', 'persistence', '--action-script', hook, '--base-url', BASE], check=False)
|
||||
after = {s: started_at(s) for s in ('api', 'postgres', 'valkey')}
|
||||
volumes_after = run(['docker', 'volume', 'inspect', '-f', '{{.Name}} {{.CreatedAt}}', 'mtusi-lab2_pgdata', 'mtusi-lab2_valkeydata'])[0]
|
||||
for service in before:
|
||||
out('%-8s StartedAt %s -> %s' % (service, before[service], after[service]))
|
||||
restarted = all(before[s] != after[s] for s in before)
|
||||
verdict('resilience test passes after restarting api, postgres and valkey with the same volumes',
|
||||
code == 0 and restarted and volumes_before == volumes_after, 'hook %s, %.1f s' % (hook, seconds))
|
||||
|
||||
|
||||
def section_sigterm():
|
||||
out('== SIGTERM: docker compose stop api ==')
|
||||
_, _, seconds = compose('stop', '-t', '10', 'api')
|
||||
exit_code = subprocess.run(['docker', 'inspect', '-f', '{{.State.ExitCode}}', container('api')],
|
||||
stdout=subprocess.PIPE, text=True).stdout.strip()
|
||||
compose('logs', '--no-log-prefix', '--tail', '2', 'api')
|
||||
refused = http('GET', '/health/live', timeout=3)[0] is None
|
||||
compose('start', 'api')
|
||||
ready = wait_ready()
|
||||
out('stop took %.2f s, exit code %s, new request refused: %s, ready again: %s' % (seconds, exit_code, refused, ready))
|
||||
verdict('SIGTERM stops the API cleanly within 10 s', exit_code == '0' and seconds < 10 and refused and ready)
|
||||
|
||||
|
||||
SECTIONS = {
|
||||
'stack': section_stack,
|
||||
'migrations': section_migrations,
|
||||
'rollback': section_rollback,
|
||||
'race': section_race,
|
||||
'pool': section_pool,
|
||||
'timeouts': section_timeouts,
|
||||
'logs': section_logs,
|
||||
'expiry': section_expiry,
|
||||
'valkey-down': section_valkey_down,
|
||||
'postgres-down': section_postgres_down,
|
||||
'persistence': section_persistence,
|
||||
'sigterm': section_sigterm,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Lab 2 infrastructure checks; creates synthetic users and messages')
|
||||
parser.add_argument('--only', nargs='+', choices=sorted(SECTIONS), help='run only these sections')
|
||||
parser.add_argument('--output', default=None, help='also write the transcript to this file')
|
||||
args = parser.parse_args()
|
||||
sys.stdout.reconfigure(errors='replace')
|
||||
out('# %s UTC, python %s, %s' % (time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()), sys.version.split()[0], sys.platform))
|
||||
out('# ' + run(['docker', 'version', '--format', 'docker {{.Server.Version}}'], quiet=True)[0])
|
||||
if not wait_ready(seconds=30):
|
||||
out('API is not ready on %s: start the stand with docker compose up -d --build' % BASE)
|
||||
return 2
|
||||
for name in args.only or SECTIONS:
|
||||
try:
|
||||
SECTIONS[name]()
|
||||
except (Stop, KeyError, TypeError, ValueError) as error:
|
||||
verdict('section %s completed' % name, False, str(error) or type(error).__name__)
|
||||
if not wait_ready(seconds=60):
|
||||
verdict('stand is ready after section %s' % name, False)
|
||||
break
|
||||
out('== Summary ==')
|
||||
for name, ok in results:
|
||||
out('%s %s' % ('PASS' if ok else 'FAIL', name))
|
||||
failed = [name for name, ok in results if not ok]
|
||||
out('RESULT: %s' % ('FAIL' if failed else 'PASS'))
|
||||
if args.output:
|
||||
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.output).write_text('\n'.join(lines) + '\n', encoding='utf-8')
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
sys.exit(main())
|
||||
except FileNotFoundError:
|
||||
print('docker CLI was not found: install Docker and make sure it is on PATH')
|
||||
sys.exit(2)
|
||||
@@ -0,0 +1,9 @@
|
||||
@echo off
|
||||
rem Windows twin of scripts/restart.sh: tests/resilience.py starts the hook directly,
|
||||
rem and Windows cannot execute a .sh file. Same actions, volumes are kept.
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
docker compose stop api || exit /b 1
|
||||
docker compose restart postgres valkey || exit /b 1
|
||||
docker compose start api || exit /b 1
|
||||
exit /b 0
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop api
|
||||
docker compose restart postgres valkey
|
||||
docker compose start api
|
||||
Reference in New Issue
Block a user