forked from mtusi2026/mtusi2026lab3
HAProxy + 2 API-реплики, общие PostgreSQL/Valkey, обязательный Idempotency-Key, graceful drain по SIGTERM, stop-one/restart/rolling-update, серии отказов и REPORT.md. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
280 lines
15 KiB
Python
280 lines
15 KiB
Python
#!/usr/bin/env python3
|
||
"""Собственные проверки lab3 (уровни 3–4). Запуск: sh scripts/test-lab3.sh (внутри сервиса tester).
|
||
|
||
Обращаются к репликам и НАПРЯМУЮ (по IP из DNS Docker внутри сети edge), чтобы гарантировать, что
|
||
конкурентные повторы действительно приходят в разные процессы, а не полагаться на round-robin LB.
|
||
Требуют доступ к docker (останавливают postgres, шлют SIGTERM реплике и возвращают всё обратно).
|
||
"""
|
||
import json
|
||
import os
|
||
import socket
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import urllib.request
|
||
import uuid
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(ROOT / 'tests'))
|
||
from client import Client, Failure, check # noqa: E402
|
||
|
||
BASE_URL = os.environ.get('BASE_URL', 'http://lb:8080')
|
||
PROJECT = 'mtusi-lab3'
|
||
API_FILTER = ['--filter', f'label=com.docker.compose.project={PROJECT}',
|
||
'--filter', 'label=com.docker.compose.service=api']
|
||
|
||
|
||
def compose(*args: str, capture: bool = False) -> str:
|
||
result = subprocess.run(['docker', 'compose', '--env-file', str(ROOT / '.env'), *args],
|
||
cwd=ROOT, check=True, text=True,
|
||
stdout=subprocess.PIPE if capture else subprocess.DEVNULL,
|
||
stderr=subprocess.DEVNULL)
|
||
return (result.stdout or '').strip()
|
||
|
||
|
||
def docker(*args: str) -> str:
|
||
return subprocess.run(['docker', *args], check=True, text=True, capture_output=True).stdout.strip()
|
||
|
||
|
||
def sql(query: str) -> str:
|
||
return compose('exec', '-T', 'postgres', 'psql', '-U', 'app', '-d', 'messenger', '-tAc', query, capture=True)
|
||
|
||
|
||
def replica_urls() -> list[str]:
|
||
"""Все IP сервиса api из DNS Docker -> прямые адреса реплик (порты наружу не опубликованы)."""
|
||
ips = sorted({info[4][0] for info in socket.getaddrinfo('api', 8080, socket.AF_INET, socket.SOCK_STREAM)})
|
||
check(len(ips) >= 2, f'expected >= 2 api replicas in DNS, got {len(ips)}')
|
||
return [f'http://{ip}:8080' for ip in ips]
|
||
|
||
|
||
def wait_ready(c: Client, seconds: float = 60) -> None:
|
||
deadline = time.monotonic() + seconds
|
||
while True:
|
||
try:
|
||
c.call('GET', '/health/ready', schema='Health')
|
||
return
|
||
except Failure:
|
||
check(time.monotonic() < deadline, 'API did not become ready')
|
||
time.sleep(0.5)
|
||
|
||
|
||
def raw_post(base: str, path: str, actor: dict, raw: bytes, key: str | None):
|
||
"""POST с заданными байтами тела (для проверки семантического равенства JSON)."""
|
||
headers = {'Content-Type': 'application/json', 'Authorization': 'Bearer ' + actor['token']}
|
||
if key is not None:
|
||
headers['Idempotency-Key'] = key
|
||
return Client(base).call('POST', path, [201, 400, 409], raw=raw, headers=headers, schema='Message')
|
||
|
||
|
||
def concurrent(calls: list) -> list:
|
||
"""Запускает функции одновременно (barrier) и возвращает результаты в исходном порядке."""
|
||
results = [None] * len(calls)
|
||
barrier = threading.Barrier(len(calls))
|
||
|
||
def run(i):
|
||
barrier.wait()
|
||
try:
|
||
results[i] = calls[i]()
|
||
except Exception as exc: # результат проверяет вызывающий тест
|
||
results[i] = exc
|
||
|
||
threads = [threading.Thread(target=run, args=(i,)) for i in range(len(calls))]
|
||
for t in threads:
|
||
t.start()
|
||
for t in threads:
|
||
t.join()
|
||
return results
|
||
|
||
|
||
def count_messages(chat: dict) -> int:
|
||
return int(sql(f"SELECT count(*) FROM messages WHERE chat_id = '{chat['id']}'"))
|
||
|
||
# ---------------------------------------------------------------- тесты
|
||
|
||
|
||
def test_session_across_replicas(c: Client):
|
||
alice = c.user('xrepl') # вход через LB (обслужила какая-то одна реплика)
|
||
seen = {c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['headers']['X-Instance-Id']
|
||
for _ in range(20)}
|
||
check(len(seen) >= 2, f'one session must be served by >= 2 replicas, saw {seen}')
|
||
urls = replica_urls()
|
||
for url in urls: # каждая реплика напрямую принимает тот же токен
|
||
me = Client(url).call('GET', '/api/v1/users/me', actor=alice, schema='User')['json']
|
||
check(me['id'] == alice['id'], 'replica resolved another user')
|
||
Client(urls[0]).call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice)
|
||
for url in urls: # logout на одной реплике виден всем (общий store)
|
||
Client(url).call('GET', '/api/v1/users/me', 401, actor=alice)
|
||
|
||
|
||
def test_same_key_concurrent_across_replicas(c: Client):
|
||
alice, _bob, _eve, chat = c.bootstrap()
|
||
urls, key = replica_urls(), uuid.uuid4().hex
|
||
calls = [lambda u=urls[i % len(urls)]: Client(u, timeout=15).message(alice, chat, 'один раз', key=key)
|
||
for i in range(24)]
|
||
results = concurrent(calls)
|
||
failures = [r for r in results if isinstance(r, Exception)]
|
||
check(not failures, f'concurrent retry failed: {failures[:1]}')
|
||
bodies = {json.dumps(r['json'], sort_keys=True) for r in results}
|
||
instances = {r['headers']['X-Instance-Id'] for r in results}
|
||
check(len(bodies) == 1, f'{len(bodies)} different messages returned for one key')
|
||
check(len(instances) >= 2, 'retries did not hit different replicas')
|
||
check(count_messages(chat) == 1, f'{count_messages(chat)} messages stored for one key')
|
||
via_lb = c.message(alice, chat, 'один раз', key=key)['json'] # и позже через LB — тот же ответ
|
||
check(json.dumps(via_lb, sort_keys=True) in bodies, 'later replay differs')
|
||
|
||
|
||
def test_same_key_concurrent_different_bodies(c: Client):
|
||
alice, _bob, _eve, chat = c.bootstrap()
|
||
urls, key = replica_urls(), uuid.uuid4().hex
|
||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||
|
||
def send(url, text):
|
||
return Client(url, timeout=15).call('POST', path, [201, 409], actor=alice, body={'text': text},
|
||
headers={'Idempotency-Key': key}, schema='Message')
|
||
calls = [lambda u=urls[i % len(urls)], t=('A' if i % 2 else 'B'): send(u, t) for i in range(20)]
|
||
results = concurrent(calls)
|
||
check(not any(isinstance(r, Exception) for r in results), 'unexpected status in conflicting retries')
|
||
winners = {r['json']['text'] for r in results if r['status'] == 201}
|
||
check(len(winners) == 1, f'both bodies accepted for one key: {winners}')
|
||
loser_statuses = {r['status'] for r in results if r['status'] != 201}
|
||
check(loser_statuses <= {409}, 'different body must be 409')
|
||
check(sum(r['status'] == 409 for r in results) == 10, 'every other-body request must get 409')
|
||
check(count_messages(chat) == 1, 'conflicting retries stored more than one message')
|
||
|
||
|
||
def test_key_scope_and_format(c: Client):
|
||
alice, bob, _eve, chat = c.bootstrap()
|
||
other = c.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'Другой', 'member_ids': [alice['id']]},
|
||
schema='Chat')['json']
|
||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||
key = 'scope.' + uuid.uuid4().hex + ':1-x_y' # все допустимые символы
|
||
first = c.message(alice, chat, 'область', key=key)['json']
|
||
check(c.message(bob, chat, 'область', key=key)['json']['id'] != first['id'], 'key shared between users')
|
||
check(c.message(alice, other, 'область', key=key)['json']['id'] != first['id'], 'key shared between chats')
|
||
# Семантически тот же JSON другими байтами (пробелы, \u-экранирование) — тот же ответ.
|
||
same = raw_post(BASE_URL, path, alice, b'{ "text" : "\\u043e\\u0431\\u043b\\u0430\\u0441\\u0442\\u044c" }', key)
|
||
check(same['status'] == 201 and same['json'] == first, 'semantically equal JSON was not a replay')
|
||
conflict = raw_post(BASE_URL, path, alice, json.dumps({'text': 'иначе'}).encode(), key)
|
||
check(conflict['status'] == 409 and conflict['json']['error']['code'] == 'conflict', 'expected 409 conflict')
|
||
check(c.message(alice, chat, 'область', key=key)['json'] == first, '409 must not replace stored result')
|
||
for bad in (None, '', 'has space', 'café', 'x' * 129, 'a/b'):
|
||
r = raw_post(BASE_URL, path, alice, b'{"text":"bad key"}', bad)
|
||
check(r['status'] == 400, f'invalid key {bad!r} accepted')
|
||
check(len(c.message(alice, chat, 'max', key='k' * 128)['json']['id']) == 36, '128-char key rejected')
|
||
check(count_messages(chat) == 3, 'unexpected number of messages in scope test') # alice x2, bob x1
|
||
|
||
|
||
def test_replay_rechecks_session(c: Client):
|
||
alice, _bob, eve, chat = c.bootstrap()
|
||
key = uuid.uuid4().hex
|
||
c.message(alice, chat, 'до выхода', key=key)
|
||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||
# Посторонний с тем же ключом не получает чужой результат.
|
||
c.call('POST', path, 404, actor=eve, body={'text': 'до выхода'}, headers={'Idempotency-Key': key})
|
||
c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice)
|
||
c.call('POST', path, 401, actor=alice, body={'text': 'до выхода'}, headers={'Idempotency-Key': key})
|
||
|
||
|
||
def test_transient_failure_not_recorded(c: Client):
|
||
alice, _bob, _eve, chat = c.bootstrap()
|
||
key = uuid.uuid4().hex
|
||
try:
|
||
compose('stop', 'postgres')
|
||
r = c.call('POST', f"/api/v1/chats/{chat['id']}/messages", 503, actor=alice,
|
||
body={'text': 'после сбоя'}, headers={'Idempotency-Key': key})
|
||
check(r['json']['error']['code'] == 'unavailable', 'expected unavailable')
|
||
check(bool(r['headers'].get('X-Instance-Id')), '503 must come from API (fallback backend), not LB')
|
||
c.call('GET', '/health/live', schema='Health')
|
||
finally:
|
||
compose('start', 'postgres')
|
||
wait_ready(c)
|
||
first = c.message(alice, chat, 'после сбоя', key=key)['json'] # повтор с тем же ключом создаёт запись
|
||
check(c.message(alice, chat, 'после сбоя', key=key)['json'] == first, 'replay after recovery differs')
|
||
check(count_messages(chat) == 1, 'transient failure produced extra message')
|
||
|
||
|
||
def test_sigterm_drain(c: Client):
|
||
"""SIGTERM: реплика сначала отвечает ready=503, но ещё обслуживает запросы, затем завершается с кодом 0."""
|
||
alice = c.user('drain')
|
||
names = docker('ps', *API_FILTER, '--format', '{{.Names}}').split()
|
||
name = sorted(names)[0]
|
||
ip = docker('inspect', '-f', '{{(index .NetworkSettings.Networks "%s_edge").IPAddress}}' % PROJECT, name)
|
||
direct = Client(f'http://{ip}:8080', timeout=2)
|
||
docker('kill', '-s', 'TERM', name)
|
||
signalled = time.monotonic()
|
||
try:
|
||
time.sleep(0.3)
|
||
direct.call('GET', '/health/ready', 503)
|
||
direct.call('GET', '/health/live', schema='Health')
|
||
r = direct.call('GET', '/api/v1/users/me', actor=alice, schema='User')
|
||
check(r['headers'].get('Connection', '').lower() == 'close', 'draining replica must close keep-alive')
|
||
time.sleep(max(0.0, signalled + 2.0 - time.monotonic())) # окно обнаружения LB: fall 2 x 0.5-1 с
|
||
served = {c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['headers']['X-Instance-Id']
|
||
for _ in range(10)} # LB уже не отправляет на неё запросы
|
||
deadline = time.monotonic() + 20
|
||
while docker('inspect', '-f', '{{.State.Status}}', name) != 'exited':
|
||
check(time.monotonic() < deadline, 'replica did not exit after drain')
|
||
time.sleep(0.2)
|
||
code = docker('inspect', '-f', '{{.State.ExitCode}}', name)
|
||
check(code in ('0', '143'), f'unclean exit code {code}')
|
||
hostname = docker('inspect', '-f', '{{.Config.Hostname}}', name)
|
||
check(not any(i.startswith(hostname) for i in served), 'LB still routes to the draining replica')
|
||
finally:
|
||
subprocess.run(['sh', str(ROOT / 'scripts/restore.sh')], check=True, capture_output=True)
|
||
deadline = time.monotonic() + 30
|
||
while len({c.call('GET', '/health/live', schema='Health')['headers']['X-Instance-Id'] for _ in range(10)}) < 2:
|
||
check(time.monotonic() < deadline, 'restored replica did not return to LB')
|
||
time.sleep(0.5)
|
||
|
||
|
||
def test_infrastructure_rules(_c: Client):
|
||
containers = docker('ps', '--filter', f'label=com.docker.compose.project={PROJECT}', '--format', '{{.Names}}').split()
|
||
for name in containers:
|
||
cfg = json.loads(docker('inspect', name))[0]
|
||
service = cfg['Config']['Labels']['com.docker.compose.service']
|
||
if service == 'tester':
|
||
continue
|
||
host = cfg['HostConfig']
|
||
check(host['NanoCpus'] > 0 and host['Memory'] > 0, f'{name}: no CPU/RAM limit')
|
||
check(name.startswith(f'{PROJECT}-{service}-'), f'{name}: fixed container_name prevents scaling')
|
||
bindings = host.get('PortBindings') or {}
|
||
if service == 'lb':
|
||
check(all(b['HostIp'] == '127.0.0.1' for v in bindings.values() for b in v), 'LB published beyond loopback')
|
||
else:
|
||
check(not bindings, f'{name} publishes ports in bypass of LB')
|
||
if service == 'api':
|
||
check(cfg['HostConfig']['ReadonlyRootfs'], 'api root fs must be read-only (no local state)')
|
||
mounts = {m['Destination'] for m in cfg['Mounts']}
|
||
check(mounts <= {'/run/secrets/database_url', '/run/secrets/cache_url'}, f'api has unexpected mounts {mounts}')
|
||
check(sql('SELECT count(*) FROM schema_migrations') == '3', 'unexpected migration count')
|
||
with urllib.request.urlopen(os.environ.get('LB_STATS_URL', 'http://lb:8404/stats') + ';csv', timeout=2) as r:
|
||
text = r.read().decode()
|
||
check(',UP,' in text or ',UP ' in text, 'LB stats show no UP backend')
|
||
|
||
|
||
TESTS = [test_session_across_replicas, test_same_key_concurrent_across_replicas,
|
||
test_same_key_concurrent_different_bodies, test_key_scope_and_format, test_replay_rechecks_session,
|
||
test_transient_failure_not_recorded, test_sigterm_drain, test_infrastructure_rules]
|
||
|
||
|
||
def main() -> int:
|
||
c = Client(BASE_URL)
|
||
wait_ready(c)
|
||
failed = 0
|
||
for test in TESTS:
|
||
started = time.monotonic()
|
||
try:
|
||
test(c)
|
||
print(f'PASS {test.__name__} ({time.monotonic() - started:.1f}s)', flush=True)
|
||
except (Failure, subprocess.CalledProcessError, OSError) as exc:
|
||
failed += 1
|
||
print(f'FAIL {test.__name__}: {exc}', flush=True)
|
||
print(f'{len(TESTS) - failed}/{len(TESTS)} passed')
|
||
return 1 if failed else 0
|
||
|
||
|
||
if __name__ == '__main__':
|
||
sys.exit(main())
|