Files
mtusi2026lab3/student-tests/test_lab3.py
T
RainakandClaude Opus 5.5 95556845e6 ЛР3 БВТ2304 Запорожец на 4
HAProxy + 2 API-реплики, общие PostgreSQL/Valkey, обязательный Idempotency-Key,
graceful drain по SIGTERM, stop-one/restart/rolling-update, серии отказов и REPORT.md.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 17:36:07 +03:00

280 lines
15 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Собственные проверки lab3 (уровни 3–4). Запуск: sh scripts/test-lab3.sh (внутри сервиса tester).
Обращаются к репликам и НАПРЯМУЮ (по IP из DNS Docker внутри сети edge), чтобы гарантировать, что
конкурентные повторы действительно приходят в разные процессы, а не полагаться на round-robin LB.
Требуют доступ к docker (останавливают postgres, шлют SIGTERM реплике и возвращают всё обратно).
"""
import json
import os
import socket
import subprocess
import sys
import threading
import time
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check # noqa: E402
BASE_URL = os.environ.get('BASE_URL', 'http://lb:8080')
PROJECT = 'mtusi-lab3'
API_FILTER = ['--filter', f'label=com.docker.compose.project={PROJECT}',
'--filter', 'label=com.docker.compose.service=api']
def compose(*args: str, capture: bool = False) -> str:
result = subprocess.run(['docker', 'compose', '--env-file', str(ROOT / '.env'), *args],
cwd=ROOT, check=True, text=True,
stdout=subprocess.PIPE if capture else subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
return (result.stdout or '').strip()
def docker(*args: str) -> str:
return subprocess.run(['docker', *args], check=True, text=True, capture_output=True).stdout.strip()
def sql(query: str) -> str:
return compose('exec', '-T', 'postgres', 'psql', '-U', 'app', '-d', 'messenger', '-tAc', query, capture=True)
def replica_urls() -> list[str]:
"""Все IP сервиса api из DNS Docker -> прямые адреса реплик (порты наружу не опубликованы)."""
ips = sorted({info[4][0] for info in socket.getaddrinfo('api', 8080, socket.AF_INET, socket.SOCK_STREAM)})
check(len(ips) >= 2, f'expected >= 2 api replicas in DNS, got {len(ips)}')
return [f'http://{ip}:8080' for ip in ips]
def wait_ready(c: Client, seconds: float = 60) -> None:
deadline = time.monotonic() + seconds
while True:
try:
c.call('GET', '/health/ready', schema='Health')
return
except Failure:
check(time.monotonic() < deadline, 'API did not become ready')
time.sleep(0.5)
def raw_post(base: str, path: str, actor: dict, raw: bytes, key: str | None):
"""POST с заданными байтами тела (для проверки семантического равенства JSON)."""
headers = {'Content-Type': 'application/json', 'Authorization': 'Bearer ' + actor['token']}
if key is not None:
headers['Idempotency-Key'] = key
return Client(base).call('POST', path, [201, 400, 409], raw=raw, headers=headers, schema='Message')
def concurrent(calls: list) -> list:
"""Запускает функции одновременно (barrier) и возвращает результаты в исходном порядке."""
results = [None] * len(calls)
barrier = threading.Barrier(len(calls))
def run(i):
barrier.wait()
try:
results[i] = calls[i]()
except Exception as exc: # результат проверяет вызывающий тест
results[i] = exc
threads = [threading.Thread(target=run, args=(i,)) for i in range(len(calls))]
for t in threads:
t.start()
for t in threads:
t.join()
return results
def count_messages(chat: dict) -> int:
return int(sql(f"SELECT count(*) FROM messages WHERE chat_id = '{chat['id']}'"))
# ---------------------------------------------------------------- тесты
def test_session_across_replicas(c: Client):
alice = c.user('xrepl') # вход через LB (обслужила какая-то одна реплика)
seen = {c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['headers']['X-Instance-Id']
for _ in range(20)}
check(len(seen) >= 2, f'one session must be served by >= 2 replicas, saw {seen}')
urls = replica_urls()
for url in urls: # каждая реплика напрямую принимает тот же токен
me = Client(url).call('GET', '/api/v1/users/me', actor=alice, schema='User')['json']
check(me['id'] == alice['id'], 'replica resolved another user')
Client(urls[0]).call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice)
for url in urls: # logout на одной реплике виден всем (общий store)
Client(url).call('GET', '/api/v1/users/me', 401, actor=alice)
def test_same_key_concurrent_across_replicas(c: Client):
alice, _bob, _eve, chat = c.bootstrap()
urls, key = replica_urls(), uuid.uuid4().hex
calls = [lambda u=urls[i % len(urls)]: Client(u, timeout=15).message(alice, chat, 'один раз', key=key)
for i in range(24)]
results = concurrent(calls)
failures = [r for r in results if isinstance(r, Exception)]
check(not failures, f'concurrent retry failed: {failures[:1]}')
bodies = {json.dumps(r['json'], sort_keys=True) for r in results}
instances = {r['headers']['X-Instance-Id'] for r in results}
check(len(bodies) == 1, f'{len(bodies)} different messages returned for one key')
check(len(instances) >= 2, 'retries did not hit different replicas')
check(count_messages(chat) == 1, f'{count_messages(chat)} messages stored for one key')
via_lb = c.message(alice, chat, 'один раз', key=key)['json'] # и позже через LB — тот же ответ
check(json.dumps(via_lb, sort_keys=True) in bodies, 'later replay differs')
def test_same_key_concurrent_different_bodies(c: Client):
alice, _bob, _eve, chat = c.bootstrap()
urls, key = replica_urls(), uuid.uuid4().hex
path = f"/api/v1/chats/{chat['id']}/messages"
def send(url, text):
return Client(url, timeout=15).call('POST', path, [201, 409], actor=alice, body={'text': text},
headers={'Idempotency-Key': key}, schema='Message')
calls = [lambda u=urls[i % len(urls)], t=('A' if i % 2 else 'B'): send(u, t) for i in range(20)]
results = concurrent(calls)
check(not any(isinstance(r, Exception) for r in results), 'unexpected status in conflicting retries')
winners = {r['json']['text'] for r in results if r['status'] == 201}
check(len(winners) == 1, f'both bodies accepted for one key: {winners}')
loser_statuses = {r['status'] for r in results if r['status'] != 201}
check(loser_statuses <= {409}, 'different body must be 409')
check(sum(r['status'] == 409 for r in results) == 10, 'every other-body request must get 409')
check(count_messages(chat) == 1, 'conflicting retries stored more than one message')
def test_key_scope_and_format(c: Client):
alice, bob, _eve, chat = c.bootstrap()
other = c.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'Другой', 'member_ids': [alice['id']]},
schema='Chat')['json']
path = f"/api/v1/chats/{chat['id']}/messages"
key = 'scope.' + uuid.uuid4().hex + ':1-x_y' # все допустимые символы
first = c.message(alice, chat, 'область', key=key)['json']
check(c.message(bob, chat, 'область', key=key)['json']['id'] != first['id'], 'key shared between users')
check(c.message(alice, other, 'область', key=key)['json']['id'] != first['id'], 'key shared between chats')
# Семантически тот же JSON другими байтами (пробелы, \u-экранирование) — тот же ответ.
same = raw_post(BASE_URL, path, alice, b'{ "text" : "\\u043e\\u0431\\u043b\\u0430\\u0441\\u0442\\u044c" }', key)
check(same['status'] == 201 and same['json'] == first, 'semantically equal JSON was not a replay')
conflict = raw_post(BASE_URL, path, alice, json.dumps({'text': 'иначе'}).encode(), key)
check(conflict['status'] == 409 and conflict['json']['error']['code'] == 'conflict', 'expected 409 conflict')
check(c.message(alice, chat, 'область', key=key)['json'] == first, '409 must not replace stored result')
for bad in (None, '', 'has space', 'café', 'x' * 129, 'a/b'):
r = raw_post(BASE_URL, path, alice, b'{"text":"bad key"}', bad)
check(r['status'] == 400, f'invalid key {bad!r} accepted')
check(len(c.message(alice, chat, 'max', key='k' * 128)['json']['id']) == 36, '128-char key rejected')
check(count_messages(chat) == 3, 'unexpected number of messages in scope test') # alice x2, bob x1
def test_replay_rechecks_session(c: Client):
alice, _bob, eve, chat = c.bootstrap()
key = uuid.uuid4().hex
c.message(alice, chat, 'до выхода', key=key)
path = f"/api/v1/chats/{chat['id']}/messages"
# Посторонний с тем же ключом не получает чужой результат.
c.call('POST', path, 404, actor=eve, body={'text': 'до выхода'}, headers={'Idempotency-Key': key})
c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice)
c.call('POST', path, 401, actor=alice, body={'text': 'до выхода'}, headers={'Idempotency-Key': key})
def test_transient_failure_not_recorded(c: Client):
alice, _bob, _eve, chat = c.bootstrap()
key = uuid.uuid4().hex
try:
compose('stop', 'postgres')
r = c.call('POST', f"/api/v1/chats/{chat['id']}/messages", 503, actor=alice,
body={'text': 'после сбоя'}, headers={'Idempotency-Key': key})
check(r['json']['error']['code'] == 'unavailable', 'expected unavailable')
check(bool(r['headers'].get('X-Instance-Id')), '503 must come from API (fallback backend), not LB')
c.call('GET', '/health/live', schema='Health')
finally:
compose('start', 'postgres')
wait_ready(c)
first = c.message(alice, chat, 'после сбоя', key=key)['json'] # повтор с тем же ключом создаёт запись
check(c.message(alice, chat, 'после сбоя', key=key)['json'] == first, 'replay after recovery differs')
check(count_messages(chat) == 1, 'transient failure produced extra message')
def test_sigterm_drain(c: Client):
"""SIGTERM: реплика сначала отвечает ready=503, но ещё обслуживает запросы, затем завершается с кодом 0."""
alice = c.user('drain')
names = docker('ps', *API_FILTER, '--format', '{{.Names}}').split()
name = sorted(names)[0]
ip = docker('inspect', '-f', '{{(index .NetworkSettings.Networks "%s_edge").IPAddress}}' % PROJECT, name)
direct = Client(f'http://{ip}:8080', timeout=2)
docker('kill', '-s', 'TERM', name)
signalled = time.monotonic()
try:
time.sleep(0.3)
direct.call('GET', '/health/ready', 503)
direct.call('GET', '/health/live', schema='Health')
r = direct.call('GET', '/api/v1/users/me', actor=alice, schema='User')
check(r['headers'].get('Connection', '').lower() == 'close', 'draining replica must close keep-alive')
time.sleep(max(0.0, signalled + 2.0 - time.monotonic())) # окно обнаружения LB: fall 2 x 0.5-1 с
served = {c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['headers']['X-Instance-Id']
for _ in range(10)} # LB уже не отправляет на неё запросы
deadline = time.monotonic() + 20
while docker('inspect', '-f', '{{.State.Status}}', name) != 'exited':
check(time.monotonic() < deadline, 'replica did not exit after drain')
time.sleep(0.2)
code = docker('inspect', '-f', '{{.State.ExitCode}}', name)
check(code in ('0', '143'), f'unclean exit code {code}')
hostname = docker('inspect', '-f', '{{.Config.Hostname}}', name)
check(not any(i.startswith(hostname) for i in served), 'LB still routes to the draining replica')
finally:
subprocess.run(['sh', str(ROOT / 'scripts/restore.sh')], check=True, capture_output=True)
deadline = time.monotonic() + 30
while len({c.call('GET', '/health/live', schema='Health')['headers']['X-Instance-Id'] for _ in range(10)}) < 2:
check(time.monotonic() < deadline, 'restored replica did not return to LB')
time.sleep(0.5)
def test_infrastructure_rules(_c: Client):
containers = docker('ps', '--filter', f'label=com.docker.compose.project={PROJECT}', '--format', '{{.Names}}').split()
for name in containers:
cfg = json.loads(docker('inspect', name))[0]
service = cfg['Config']['Labels']['com.docker.compose.service']
if service == 'tester':
continue
host = cfg['HostConfig']
check(host['NanoCpus'] > 0 and host['Memory'] > 0, f'{name}: no CPU/RAM limit')
check(name.startswith(f'{PROJECT}-{service}-'), f'{name}: fixed container_name prevents scaling')
bindings = host.get('PortBindings') or {}
if service == 'lb':
check(all(b['HostIp'] == '127.0.0.1' for v in bindings.values() for b in v), 'LB published beyond loopback')
else:
check(not bindings, f'{name} publishes ports in bypass of LB')
if service == 'api':
check(cfg['HostConfig']['ReadonlyRootfs'], 'api root fs must be read-only (no local state)')
mounts = {m['Destination'] for m in cfg['Mounts']}
check(mounts <= {'/run/secrets/database_url', '/run/secrets/cache_url'}, f'api has unexpected mounts {mounts}')
check(sql('SELECT count(*) FROM schema_migrations') == '3', 'unexpected migration count')
with urllib.request.urlopen(os.environ.get('LB_STATS_URL', 'http://lb:8404/stats') + ';csv', timeout=2) as r:
text = r.read().decode()
check(',UP,' in text or ',UP ' in text, 'LB stats show no UP backend')
TESTS = [test_session_across_replicas, test_same_key_concurrent_across_replicas,
test_same_key_concurrent_different_bodies, test_key_scope_and_format, test_replay_rechecks_session,
test_transient_failure_not_recorded, test_sigterm_drain, test_infrastructure_rules]
def main() -> int:
c = Client(BASE_URL)
wait_ready(c)
failed = 0
for test in TESTS:
started = time.monotonic()
try:
test(c)
print(f'PASS {test.__name__} ({time.monotonic() - started:.1f}s)', flush=True)
except (Failure, subprocess.CalledProcessError, OSError) as exc:
failed += 1
print(f'FAIL {test.__name__}: {exc}', flush=True)
print(f'{len(TESTS) - failed}/{len(TESTS)} passed')
return 1 if failed else 0
if __name__ == '__main__':
sys.exit(main())