From f84831d0bb709632e75fe3ee62a457969aaa7791 Mon Sep 17 00:00:00 2001 From: Pavel Grigorev Date: Fri, 13 Nov 2026 16:22:54 +0300 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20=D1=82=D0=B5=D1=81=D1=82=D1=8B=20=D0=B2=D0=BB=D0=BE?= =?UTF-8?q?=D0=B6=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=B8=20=D1=85=D1=80=D0=B0?= =?UTF-8?q?=D0=BD=D0=B8=D0=BB=D0=B8=D1=89=D0=B0,=20=D0=BF=D1=80=D0=BE?= =?UTF-8?q?=D0=B2=D0=B5=D1=80=D0=BA=D1=83=20=D1=81=D0=BE=D0=B3=D0=BB=D0=B0?= =?UTF-8?q?=D1=81=D0=BE=D0=B2=D0=B0=D0=BD=D0=BD=D0=BE=D1=81=D1=82=D0=B8=20?= =?UTF-8?q?=D0=B8=20=D0=B8=D1=81=D1=81=D0=BB=D0=B5=D0=B4=D0=BE=D0=B2=D0=B0?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D1=81=D0=B2=D0=B5=D1=80=D1=89=D0=B8=D0=BA?= =?UTF-8?q?=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/app.yml | 31 +++ research/s3ops.py | 43 ++++ research/track2.py | 282 ++++++++++++++++++++++++++ student-tests/consistency_check.py | 156 ++++++++++++++ student-tests/expiry_check.py | 56 +++++ student-tests/failover_series.py | 71 +++++++ student-tests/failure_check.py | 127 ++++++++++++ student-tests/fixtures/sample.jpg | Bin 0 -> 19881 bytes student-tests/helpers.py | 191 +++++++++++++++++ student-tests/imagegen.py | 66 ++++++ student-tests/rolling_update_check.py | 60 ++++++ student-tests/sigterm_check.py | 79 ++++++++ student-tests/test_api.py | 157 ++++++++++++++ student-tests/test_attachments.py | 192 ++++++++++++++++++ student-tests/test_auth.py | 132 ++++++++++++ student-tests/test_concurrency.py | 68 +++++++ student-tests/test_db.py | 58 ++++++ student-tests/test_idempotency.py | 96 +++++++++ student-tests/test_replicas.py | 41 ++++ student-tests/test_storage.py | 81 ++++++++ student-tests/upload_memory.py | 118 +++++++++++ 21 files changed, 2105 insertions(+) create mode 100644 .github/workflows/app.yml create mode 100644 research/s3ops.py create mode 100644 research/track2.py create mode 100644 student-tests/consistency_check.py create mode 100644 student-tests/expiry_check.py create mode 100644 student-tests/failover_series.py create mode 100644 student-tests/failure_check.py create mode 100644 student-tests/fixtures/sample.jpg create mode 100644 student-tests/helpers.py create mode 100644 student-tests/imagegen.py create mode 100644 student-tests/rolling_update_check.py create mode 100644 student-tests/sigterm_check.py create mode 100644 student-tests/test_api.py create mode 100644 student-tests/test_attachments.py create mode 100644 student-tests/test_auth.py create mode 100644 student-tests/test_concurrency.py create mode 100644 student-tests/test_db.py create mode 100644 student-tests/test_idempotency.py create mode 100644 student-tests/test_replicas.py create mode 100644 student-tests/test_storage.py create mode 100644 student-tests/upload_memory.py diff --git a/.github/workflows/app.yml b/.github/workflows/app.yml new file mode 100644 index 00000000..b9ec4641 --- /dev/null +++ b/.github/workflows/app.yml @@ -0,0 +1,31 @@ +name: Application +on: [push, pull_request] +permissions: + contents: read +jobs: + stack: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: cp .env.example .env + - run: python scripts/init_secrets.py + - run: docker compose config --quiet + - run: docker compose up -d --build --wait + - run: make test BASE_URL=http://127.0.0.1:8080 PYTHON=python + - run: make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080 PYTHON=python + - run: make student-test BASE_URL=http://127.0.0.1:8080 PYTHON=python + - run: python student-tests/expiry_check.py --ttl 2 + - run: python student-tests/failure_check.py + - run: python student-tests/consistency_check.py + - run: make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://127.0.0.1:8080 PYTHON=python + - run: docker compose start api && sleep 15 + - run: python student-tests/failover_series.py + - run: python student-tests/rolling_update_check.py --no-build + - if: failure() + run: docker compose ps -a && docker compose logs --no-color + - if: always() + run: docker compose down -v diff --git a/research/s3ops.py b/research/s3ops.py new file mode 100644 index 00000000..cd2495e2 --- /dev/null +++ b/research/s3ops.py @@ -0,0 +1,43 @@ +"""Операции с S3 для внедрения сбоев в треке 2. Запускается внутри контейнера reconcile (ключи сверщика): + python research/s3ops.py put SIZE < keys # создать объекты указанного размера + python research/s3ops.py delete < keys # удалить объекты + python research/s3ops.py exists < keys # вывести число существующих +""" +from concurrent.futures import ThreadPoolExecutor +import sys + +sys.path.insert(0, 'src') + +from botocore.exceptions import ClientError # noqa: E402 + +from config import setting, settings # noqa: E402 +from s3 import make_client # noqa: E402 + +client = make_client(setting('S3_RECONCILER_ACCESS_KEY_ID'), setting('S3_RECONCILER_SECRET_ACCESS_KEY'), timeout=30) + + +def exists(key): + try: + client.head_object(Bucket=settings.s3_bucket, Key=key) + return True + except ClientError: + return False + + +def main(): + command = sys.argv[1] + keys = [line.strip() for line in sys.stdin if line.strip()] + with ThreadPoolExecutor(max_workers=16) as pool: + if command == 'put': + body = b'x' * int(sys.argv[2]) + list(pool.map(lambda k: client.put_object(Bucket=settings.s3_bucket, Key=k, Body=body), keys)) + print(len(keys)) + elif command == 'delete': + list(pool.map(lambda k: client.delete_object(Bucket=settings.s3_bucket, Key=k), keys)) + print(len(keys)) + elif command == 'exists': + print(sum(pool.map(exists, keys))) + + +if __name__ == '__main__': + main() diff --git a/research/track2.py b/research/track2.py new file mode 100644 index 00000000..3a0877e2 --- /dev/null +++ b/research/track2.py @@ -0,0 +1,282 @@ +"""Трек 2: согласованность PostgreSQL и S3 — исследование сверщика (src/reconcile.py). + +Стенд запущен (docker compose up -d). BASE_URL — адрес LB (по умолчанию http://127.0.0.1:8080). + python research/track2.py scenario # H1/H2: внедрённые сбои → dry-run → apply → повторный apply + python research/track2.py race # H2: apply в цикле под непрерывными загрузками + python research/track2.py cost # H3: время/память обхода при 1k…20k объектов + python research/track2.py cleanup # удалить синтетические данные исследования + python research/track2.py report # evidence/track2/results.md + python research/track2.py all +Синтетические строки живут в отдельном чате пользователя track2_*; cleanup удаляет их и их объекты. +На время запуска периодический сервис reconciler останавливается и затем запускается снова. +""" +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +from pathlib import Path +import statistics +import subprocess +import sys +import threading +import time +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'student-tests')) + +from helpers import call, compose, new_chat, new_user, sql # noqa: E402 +from imagegen import SAMPLE_PNG # noqa: E402 +from test_attachments import download, upload # noqa: E402 + +OUT = ROOT / 'evidence' / 'track2' +STATE = OUT / 'state.json' +GRACE = 30 + + +def save(name, data): + OUT.mkdir(parents=True, exist_ok=True) + (OUT / name).write_text(json.dumps(data, ensure_ascii=False, indent=2, default=str) + '\n', encoding='utf-8') + print('saved evidence/track2/' + name) + + +def load(name): + return json.loads((OUT / name).read_text(encoding='utf-8')) + + +def s3ops(command, keys, *args): + result = subprocess.run(['docker', 'compose', 'run', '--rm', '-T', '-v', str(ROOT / 'research') + ':/app/research:ro', + '--entrypoint', 'python', 'reconcile', 'research/s3ops.py', command, *map(str, args)], + cwd=ROOT, input='\n'.join(keys), capture_output=True, text=True, encoding='utf-8') + if result.returncode: + raise SystemExit(result.stderr[-2000:]) + return int(result.stdout.strip().splitlines()[-1]) + + +def reconcile(*args): + return json.loads(compose('run', '--rm', 'reconcile', *map(str, args))) + + +def research_chat(): + if STATE.exists(): + return json.loads(STATE.read_text(encoding='utf-8')) + user = new_user('track2') + chat = new_chat(user) + state = {'user_id': user['id'], 'chat_id': chat['id'], 'token': None} + OUT.mkdir(parents=True, exist_ok=True) + STATE.write_text(json.dumps(state), encoding='utf-8') + return state + + +def insert_rows(state, count, status='ready', age_seconds=0, size=1024): + """Синтетические строки attachments в чате исследования; возвращает ключи объектов.""" + if count <= 0: + return [] + out = sql("WITH new AS (INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, " + "object_key, status, created_at, updated_at) SELECT id, '%s', '%s', 'synthetic.png', 'image/png', %d, " + "decode(repeat('00', 32), 'hex'), 'attachments/' || id, '%s', now() - make_interval(secs => %d), now() " + "FROM (SELECT gen_random_uuid() AS id FROM generate_series(1, %d)) g RETURNING object_key) " + "SELECT string_agg(object_key, ' ') FROM new" + % (state['chat_id'], state['user_id'], size, status, age_seconds, count)) + return out.split() + + +def reference(state, keys): + """Привязать вложения к сообщению (используемые вложения).""" + message = sql("INSERT INTO messages (chat_id, sender_id, text) VALUES ('%s', '%s', convert_to('track2', 'UTF8')) " + "RETURNING id" % (state['chat_id'], state['user_id'])).splitlines()[0] + sql("INSERT INTO message_attachments (message_id, attachment_id, position) SELECT '%s', a.id, " + "row_number() OVER () FROM attachments a WHERE a.object_key IN (%s)" + % (message, ','.join("'%s'" % k for k in keys))) + + +def orphan_keys(count): + return ['attachments/%s' % uuid.uuid4() for _ in range(count)] + + +def scenario(): + state = research_chat() + real = new_user('track2real') + real_chat = new_chat(real) + real_att = upload(real, real_chat, SAMPLE_PNG)[2] + call('POST', '/api/v1/chats/%s/messages' % real_chat['id'], {'text': '', 'attachment_ids': [real_att['id']]}, + real['token']) + baseline = reconcile('--grace-seconds', GRACE) + + ok = insert_rows(state, 200) + s3ops('put', ok, 1024) + referenced = ok[:20] + reference(state, referenced) + missing = ok[20:30] + referenced[:5] + s3ops('delete', missing) + mismatch = ok[40:50] + s3ops('put', mismatch, 2048) + stale_with_object = insert_rows(state, 25, 'uploading', 3600) + s3ops('put', stale_with_object, 1024) + stale_without_object = insert_rows(state, 25, 'uploading', 3600) + old_orphans = orphan_keys(25) + s3ops('put', old_orphans, 1024) + print('waiting %ds so that injected orphans become older than grace…' % (GRACE + 3)) + time.sleep(GRACE + 3) + fresh_uploads = insert_rows(state, 10, 'uploading', 0) + s3ops('put', fresh_uploads, 1024) + fresh_orphans = orphan_keys(10) + s3ops('put', fresh_orphans, 1024) + + expected = {'orphan': 25, 'fresh_orphan': 10, 'stale_upload': 50, 'fresh_upload': 10, 'missing': 15, + 'size_mismatch': 10} + base = baseline['counts'] + dry = reconcile('--grace-seconds', GRACE) + dry_again = reconcile('--grace-seconds', GRACE) + applied = reconcile('--apply', '--grace-seconds', GRACE) + rerun = reconcile('--apply', '--grace-seconds', GRACE) + after = reconcile('--grace-seconds', GRACE) + detected = {k: dry['counts'][k] - base.get(k, 0) for k in expected} + real_download = download(real, real_att['id']) + checks = { + 'detected_equals_injected': detected == expected, + 'dry_run_changes_nothing': dry_again['counts'] == dry['counts'] and dry_again['objects_listed'] == dry['objects_listed'], + 'apply_actions': applied['actions'], + 'apply_deleted_exactly_old_orphans_and_stale': applied['actions'] == { + 'orphan_objects_deleted': 25, 'stale_rows_deleted': 50, 'stale_objects_deleted': 25, 'skipped': 0}, + 'rerun_is_noop': not any(rerun['actions'][k] for k in ('orphan_objects_deleted', 'stale_rows_deleted', 'stale_objects_deleted')), + 'fresh_orphans_kept': s3ops('exists', fresh_orphans) == 10, + 'fresh_uploads_kept': s3ops('exists', fresh_uploads) == 10 and after['counts']['fresh_upload'] >= 10, + 'old_orphans_gone': s3ops('exists', old_orphans) == 0, + 'stale_objects_gone': s3ops('exists', stale_with_object) == 0, + 'referenced_rows_kept': int(sql("SELECT count(*) FROM attachments WHERE object_key IN (%s)" + % ','.join("'%s'" % k for k in referenced))) == 20, + 'missing_still_reported_not_deleted': after['counts']['missing'] - base.get('missing', 0) == 15, + 'real_attachment_intact': real_download[0] == 200 and hashlib.sha256(real_download[2]).hexdigest() == real_att['sha256'], + } + passed = all(v for k, v in checks.items() if k != 'apply_actions') + save('scenario.json', {'grace_seconds': GRACE, 'injected': expected, 'baseline_counts': base, + 'dry_run': dry, 'apply': applied, 'rerun': rerun, 'after': after, + 'detected': detected, 'checks': checks, 'passed': passed}) + for name, value in checks.items(): + print(' %-48s %s' % (name, value)) + print('PASS' if passed else 'FAIL') + + +def race(seconds=40): + user = new_user('track2race') + chat = new_chat(user) + uploaded, errors, stop = [], [], threading.Event() + + def uploader(): + while not stop.is_set(): + status, _, body = upload(user, chat, SAMPLE_PNG) + (uploaded.append(body['id']) if status == 201 else errors.append(status)) + + preclean = reconcile('--apply', '--grace-seconds', 0)['actions'] + threads = [threading.Thread(target=uploader) for _ in range(4)] + for thread in threads: + thread.start() + runs, started = [], time.monotonic() + while time.monotonic() - started < seconds: + report = reconcile('--apply', '--grace-seconds', GRACE) + runs.append({'counts': report['counts'], 'actions': report['actions'], 'total_s': report['seconds']['total']}) + stop.set() + for thread in threads: + thread.join() + intact = 0 + for attachment_id in uploaded: + status, _, body = download(user, attachment_id) + intact += status == 200 and body == SAMPLE_PNG + result = {'seconds': seconds, 'grace_seconds': GRACE, 'preclean_before_load': preclean, + 'reconcile_runs': len(runs), 'uploads_201': len(uploaded), + 'upload_errors': errors, 'downloaded_intact': intact, + 'deleted_by_reconciler': sum(r['actions']['orphan_objects_deleted'] + r['actions']['stale_rows_deleted'] + for r in runs), + 'fresh_uploads_seen_in_flight': sum(r['counts']['fresh_upload'] for r in runs), 'runs': runs} + result['passed'] = intact == len(uploaded) and result['deleted_by_reconciler'] == 0 + save('race.json', result) + print('race: %d uploads, %d intact, reconciler runs %d, deleted %d, in-flight uploads seen %d → %s' + % (len(uploaded), intact, len(runs), result['deleted_by_reconciler'], result['fresh_uploads_seen_in_flight'], + 'PASS' if result['passed'] else 'FAIL')) + + +def cost(sizes=(1000, 5000, 10000, 20000), repeats=3): + state = research_chat() + rows = [] + for target in sizes: + current = int(sql("SELECT count(*) FROM attachments")) + if target > current: + keys = insert_rows(state, target - current) + started = time.monotonic() + for start in range(0, len(keys), 2000): + s3ops('put', keys[start:start + 2000], 256) + print(' +%d objects in %.1fs' % (len(keys), time.monotonic() - started)) + reports = [reconcile('--grace-seconds', GRACE) for _ in range(repeats)] + row = {'objects': reports[-1]['objects_listed'], 'rows': reports[-1]['rows_scanned'], + 'list_pages': reports[-1]['list_pages'], + 'list_s3_s': statistics.median(r['seconds']['list_s3'] for r in reports), + 'scan_db_s': statistics.median(r['seconds']['scan_db'] for r in reports), + 'total_s': statistics.median(r['seconds']['total'] for r in reports), + 'max_rss_mib': max(r['max_rss_mib'] for r in reports)} + row['us_per_object'] = round(row['total_s'] / row['objects'] * 1e6, 1) + rows.append(row) + print(' %(objects)6d objects: list %(list_s3_s).2fs, db %(scan_db_s).3fs, total %(total_s).2fs, ' + 'rss %(max_rss_mib).0f MiB, %(us_per_object).0f µs/object' % row) + save('cost.json', {'repeats': repeats, 'rows': rows}) + + +def cleanup(): + if not STATE.exists(): + print('nothing to clean') + return + state = json.loads(STATE.read_text(encoding='utf-8')) + keys = sql("SELECT coalesce(string_agg(object_key, ' '), '') FROM attachments WHERE chat_id = '%s'" % state['chat_id']).split() + for start in range(0, len(keys), 2000): + s3ops('delete', keys[start:start + 2000]) + sql("DELETE FROM message_attachments WHERE message_id IN (SELECT id FROM messages WHERE chat_id = '%s')" % state['chat_id']) + sql("DELETE FROM messages WHERE chat_id = '%s'" % state['chat_id']) + sql("DELETE FROM attachments WHERE chat_id = '%s'" % state['chat_id']) + leftovers = reconcile('--apply', '--grace-seconds', 0) + STATE.unlink() + print('cleanup: %d research objects deleted; reconciler afterwards: %s' % (len(keys), leftovers['counts'])) + + +def report(): + s, r, c = load('scenario.json'), load('race.json'), load('cost.json') + lines = ['# Трек 2: результаты', '', '## H1/H2. Внедрённые сбои (grace %d с)' % s['grace_seconds'], '', + '| категория | внедрено | найдено dry-run | действие apply |', '|---|---|---|---|'] + actions = {'orphan': 'удалено объектов: %d' % s['apply']['actions']['orphan_objects_deleted'], + 'stale_upload': 'удалено строк: %d, объектов: %d' % (s['apply']['actions']['stale_rows_deleted'], + s['apply']['actions']['stale_objects_deleted']), + 'fresh_orphan': 'не тронуто', 'fresh_upload': 'не тронуто', 'missing': 'только отчёт', + 'size_mismatch': 'только отчёт'} + for name, count in s['injected'].items(): + lines.append('| %s | %d | %d | %s |' % (name, count, s['detected'][name], actions[name])) + lines += ['', '| проверка | результат |', '|---|---|'] + lines += ['| %s | %s |' % (k, v) for k, v in s['checks'].items() if k != 'apply_actions'] + lines += ['', '## H2. Сверщик под нагрузкой загрузок', '', + '%d с, 4 потока загрузок, apply с grace %d с в цикле: запусков сверщика %d, загрузок 201 — %d, ' + 'скачано без изменений — %d, удалено сверщиком — %d, «свежих» загрузок в полёте увидено — %d.' + % (r['seconds'], r['grace_seconds'], r['reconcile_runs'], r['uploads_201'], r['downloaded_intact'], + r['deleted_by_reconciler'], r['fresh_uploads_seen_in_flight']), + '', '## H3. Стоимость обхода (медиана из %d)' % c['repeats'], '', + '| объектов | строк | страниц листинга | листинг S3, с | скан БД, с | всего, с | мкс/объект | RSS, MiB |', + '|---|---|---|---|---|---|---|---|'] + for row in c['rows']: + lines.append('| %(objects)d | %(rows)d | %(list_pages)d | %(list_s3_s).2f | %(scan_db_s).3f | %(total_s).2f | ' + '%(us_per_object).0f | %(max_rss_mib).0f |' % row) + (OUT / 'results.md').write_text('\n'.join(lines) + '\n', encoding='utf-8') + print('saved evidence/track2/results.md') + + +COMMANDS = {'scenario': scenario, 'race': race, 'cost': cost, 'cleanup': cleanup, 'report': report} + +if __name__ == '__main__': + command = sys.argv[1] if len(sys.argv) > 1 else 'all' + # Периодический сервис reconciler (--apply, grace 900 с) удалил бы состаренные синтетические заготовки + # посреди эксперимента; на время исследования он остановлен, сверщик запускается явно. + paused = 'reconciler' in compose('ps', '--status', 'running', '--services') + if paused: + compose('stop', 'reconciler') + try: + for step in (['scenario', 'race', 'cost', 'report', 'cleanup'] if command == 'all' else [command]): + print('==', step) + COMMANDS[step]() + finally: + if paused: + compose('start', 'reconciler') diff --git a/student-tests/consistency_check.py b/student-tests/consistency_check.py new file mode 100644 index 00000000..a0b46121 --- /dev/null +++ b/student-tests/consistency_check.py @@ -0,0 +1,156 @@ +"""Usage (стенд запущен через docker compose up -d): + python student-tests/consistency_check.py + +Проверяет, что сбои при загрузке не оставляют вечных неучтённых данных: + 1) клиент обрывает загрузку посреди тела → ни строки, ни объекта; + 2) S3 остановлен → upload/download 503, ни строки, ни объекта; метаданные, текст и ready работают; + 3) процесс «падает» после S3 PUT до COMMIT ready (FAULTS=after_s3_put, только в этом тесте) → + остаётся строка uploading + объект; сверщик не трогает их в пределах grace и убирает после; + 4) объект без строки в БД (orphan) → свежий не трогается, старый удаляется; повторный запуск ничего не делает; + 5) периодический сервис reconciler сам удаляет заготовку старше grace (без ручного запуска). +""" +import json +import os +import socket +import time +import urllib.parse +import uuid + +from helpers import BASE_URL, call, compose, new_chat, new_user, sql, wait_ready +from imagegen import SAMPLE_PNG +from test_attachments import download, upload + +failures = [] + + +def check(title, ok, detail=''): + print(' %-62s %s %s' % (title, 'OK' if ok else 'FAIL', detail)) + if not ok: + failures.append(title) + + +def reconcile(*args): + return json.loads(compose('run', '--rm', 'reconcile', *args)) + + +def s3_python(code): + return compose('run', '--rm', '--entrypoint', 'python', 'reconcile', '-c', + 'import sys; sys.path.insert(0, "src")\n' + code) + + +def counts(): + return sql("SELECT count(*) FILTER (WHERE status = 'ready') || '/' || count(*) FILTER (WHERE status = 'uploading') " + "FROM attachments") + + +def object_exists(key): + return s3_python('from config import setting, settings\nfrom s3 import make_client\n' + 'c = make_client(setting("S3_RECONCILER_ACCESS_KEY_ID"), setting("S3_RECONCILER_SECRET_ACCESS_KEY"))\n' + 'keys = [o["Key"] for o in c.list_objects_v2(Bucket=settings.s3_bucket, Prefix=%r).get("Contents", [])]\n' + 'print(len(keys))' % key).strip() == '1' + + +def restart_api(faults=None): + env = dict(os.environ) + env['FAULTS'] = faults or '' + compose('up', '-d', '--no-deps', '--no-build', 'api', env=env) + time.sleep(3) + wait_ready() + time.sleep(3) + + +def interrupted_upload(alice, chat): + target = urllib.parse.urlsplit(BASE_URL) + with socket.create_connection((target.hostname, target.port), timeout=10) as sock: + head = ('POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: x\r\n' + 'Authorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n' + % (chat['id'], alice['token'], 5 * 1024 * 1024)) + sock.sendall(head.encode() + SAMPLE_PNG + bytes(512 * 1024)) + time.sleep(2) + + +def main(): + alice = new_user('consistency') + chat = new_chat(alice) + + print('1. Обрыв загрузки посреди тела') + before = counts() + interrupted_upload(alice, chat) + check('строк не прибавилось (ready/uploading)', counts() == before, '%s -> %s' % (before, counts())) + + print('2. S3 остановлен: загрузка, скачивание, остальной API') + stored = upload(alice, chat, SAMPLE_PNG)[2] + before = counts() + compose('stop', 's3') + try: + started = time.monotonic() + status = upload(alice, chat, SAMPLE_PNG)[0] + check('upload при остановленном S3 → 503', status == 503, '%s за %.1f с' % (status, time.monotonic() - started)) + check('заготовка удалена компенсацией', counts() == before, '%s -> %s' % (before, counts())) + started = time.monotonic() + status = download(alice, stored['id'])[0] + check('download при остановленном S3 → 503', status == 503, '%s за %.1f с' % (status, time.monotonic() - started)) + check('метаданные вложения доступны (они в PostgreSQL)', + call('GET', '/api/v1/attachments/' + stored['id'], token=alice['token'])[0] == 200) + check('текстовое сообщение → 201', call('POST', '/api/v1/chats/%s/messages' % chat['id'], + {'text': 'без S3'}, alice['token'])[0] == 201) + check('/health/ready → 200 (S3 не критичен для текстового API)', call('GET', '/health/ready')[0] == 200) + finally: + compose('up', '-d', '--wait', 's3') + check('после старта S3 upload снова 201', upload(alice, chat, SAMPLE_PNG)[0] == 201) + + print('3. Сбой после S3 PUT до фиксации ready (FAULTS=after_s3_put)') + restart_api('after_s3_put') + try: + status = upload(alice, chat, SAMPLE_PNG, filename='fault-%s.png' % uuid.uuid4().hex[:6])[0] + row = sql("SELECT id || ' ' || object_key FROM attachments WHERE status = 'uploading' " + "AND filename LIKE 'fault-%%' ORDER BY created_at DESC LIMIT 1").split() + check('клиент получил ошибку (не 201)', status != 201, str(status)) + check('осталась строка uploading и объект в S3', bool(row) and object_exists(row[1]), ' '.join(row)) + finally: + restart_api() + fresh = reconcile('--grace-seconds', '900') + check('dry-run, grace 900 с: заготовка свежая, не трогается', fresh['counts']['fresh_upload'] >= 1 + and fresh['counts']['stale_upload'] == 0, str(fresh['counts'])) + applied = reconcile('--apply', '--grace-seconds', '0') + check('apply, grace 0: строка и объект удалены', applied['actions']['stale_rows_deleted'] >= 1 + and applied['actions']['stale_objects_deleted'] >= 1, str(applied['actions'])) + check('объекта больше нет', not object_exists(row[1])) + + print('4. Объект без строки в БД (orphan)') + key = 'attachments/%s' % uuid.uuid4() + s3_python('from config import setting, settings\nfrom s3 import make_client\n' + 'c = make_client(setting("S3_RECONCILER_ACCESS_KEY_ID"), setting("S3_RECONCILER_SECRET_ACCESS_KEY"))\n' + 'c.put_object(Bucket=settings.s3_bucket, Key=%r, Body=b"orphan")' % key) + fresh = reconcile('--grace-seconds', '900') + check('dry-run, grace 900 с: свежая сирота не трогается', fresh['counts']['fresh_orphan'] >= 1 + and fresh['counts']['orphan'] == 0, str(fresh['counts'])) + applied = reconcile('--apply', '--grace-seconds', '0') + check('apply, grace 0: сирота удалена', applied['actions']['orphan_objects_deleted'] >= 1, str(applied['actions'])) + again = reconcile('--apply', '--grace-seconds', '0') + check('повторный запуск ничего не делает', not any(again['actions'][k] for k in + ('orphan_objects_deleted', 'stale_rows_deleted', 'stale_objects_deleted')), str(again['actions'])) + check('рабочие вложения не пострадали (missing = 0)', again['counts']['missing'] == 0, str(again['counts'])) + + print('5. Периодический сервис reconciler убирает зависшую заготовку сам') + stale = str(uuid.uuid4()) + sql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, " + "status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'stale.png', 'image/png', 1, " + "decode(repeat('00', 32), 'hex'), 'attachments/%s', 'uploading', now() - interval '1 hour', " + "now() - interval '1 hour')" % (stale, chat['id'], alice['id'], stale)) + running = 'reconciler' in compose('ps', '--status', 'running', '--services') + check('сервис reconciler запущен (docker compose ps)', running) + if running: + compose('restart', 'reconciler') # проход выполняется сразу после старта, дальше — раз в интервал + deadline = time.monotonic() + 60 + while time.monotonic() < deadline and sql("SELECT count(*) FROM attachments WHERE id = '%s'" % stale) != '0': + time.sleep(1) + check('заготовка старше grace удалена без ручного запуска', + sql("SELECT count(*) FROM attachments WHERE id = '%s'" % stale) == '0') + + print('FAIL: ' + ', '.join(failures) if failures else 'PASS: сбои загрузки не оставляют неучтённых данных') + raise SystemExit(1 if failures else 0) + + +if __name__ == '__main__': + main() diff --git a/student-tests/expiry_check.py b/student-tests/expiry_check.py new file mode 100644 index 00000000..7bde51ec --- /dev/null +++ b/student-tests/expiry_check.py @@ -0,0 +1,56 @@ +"""Usage (стенд уже запущен через docker compose up -d): + python student-tests/expiry_check.py --ttl 2 + +Скрипт пересоздаёт только api с SESSION_TTL_SECONDS=, проверяет истечение сессии, +затем возвращает api с обычным TTL из .env. Volumes и данные не трогаются. +""" +import argparse +import hashlib +import os +import time + +from helpers import call, compose, new_user, wait_ready + + +def restart_api(ttl=None): + env = dict(os.environ) + if ttl is None: + env.pop('SESSION_TTL_SECONDS', None) + else: + env['SESSION_TTL_SECONDS'] = str(ttl) + compose('up', '-d', '--no-deps', '--no-build', 'api', env=env) + wait_ready() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--ttl', type=int, default=2) + args = parser.parse_args() + + print('пересоздаю api с SESSION_TTL_SECONDS=%d' % args.ttl) + restart_api(args.ttl) + try: + alice = new_user('expiry') + status = call('GET', '/api/v1/users/me', token=alice['token'])[0] + print('сразу после входа: /users/me -> %d' % status) + assert status == 200 + key = 'session:' + hashlib.sha256(alice['token'].encode()).hexdigest() + cache_ttl = int(compose('exec', '-T', 'valkey', 'valkey-cli', 'TTL', key)) + print('TTL ключа этой сессии в Valkey: %d с (-2 = ключа нет)' % cache_ttl) + assert cache_ttl <= args.ttl, 'кэш живёт дольше сессии' + + time.sleep(args.ttl + 1.5) + status, headers, _ = call('GET', '/api/v1/users/me', token=alice['token']) + print('через %.1f с: /users/me -> %d, WWW-Authenticate: %s' % (args.ttl + 1.5, status, headers['WWW-Authenticate'])) + assert status == 401 and headers['WWW-Authenticate'] == 'Bearer realm="messenger"' + status = call('DELETE', '/api/v1/auth/sessions/current', token=alice['token'])[0] + print('logout истёкшим токеном -> %d' % status) + assert status == 401 + print('PASS: сессия истекла по SESSION_TTL_SECONDS=%d, кэш не продлил её' % args.ttl) + finally: + print('возвращаю api с TTL из .env') + restart_api() + + +if __name__ == '__main__': + main() diff --git a/student-tests/failover_series.py b/student-tests/failover_series.py new file mode 100644 index 00000000..bbe15e39 --- /dev/null +++ b/student-tests/failover_series.py @@ -0,0 +1,71 @@ +"""Usage (стенд запущен через docker compose up -d, обе реплики в ротации): + python student-tests/failover_series.py --output evidence/failover-series.json + +Для каждой API-реплики по очереди и в двух режимах (docker stop — штатная остановка с выводом из ротации, +docker kill — внезапная смерть процесса) под нагрузкой: останавливает реплику, меряет ошибки, RPS, +время до последнего ответа остановленной реплики, восстанавливает её (docker compose start api) +и ждёт возврата в ротацию LB. Volumes, БД и LB не трогаются. +""" +import argparse +import json +import subprocess +import time + +from helpers import Load, api_containers, compose, new_chat, new_user, wait_in_rotation + +WARMUP, AFTER = 5, 10 + + +def run(container, mode): + alice = new_user('series') + chat = new_chat(alice) + load = Load(alice['token'], chat['id']).start() + time.sleep(WARMUP) + acted = time.monotonic() + command = ['docker', 'stop', '-t', '15', container] if mode == 'stop' else ['docker', 'kill', container] + subprocess.run(command, capture_output=True, check=True) + action_seconds = time.monotonic() - acted + time.sleep(AFTER) + load.stop() + instance = container[:12] + target = [r for r in load.records if r[2] == 201 and r[3] == instance and r[0] >= acted] + others = [r for r in load.records if r[2] == 201 and r[3] != instance and r[0] >= acted] + restored = time.monotonic() + compose('start', 'api') + back = wait_in_rotation(container) + result = { + 'replica': instance, 'mode': mode, 'action_seconds': round(action_seconds, 2), + 'target_served_until_s': round(max(r[1] for r in target) - acted, 2) if target else 0.0, + 'first_success_on_other_s': round(min(r[1] for r in others) - acted, 2) if others else None, + 'before': load.window(acted - WARMUP, acted), + 'during_10s': load.window(acted, acted + AFTER), + 'back_in_rotation_s': round(back - restored, 2), + 'data': load.verify(alice['token']), + } + print('%-12s %-4s errors=%-3d rps %5.1f -> %5.1f target last answer +%.2fs restore %.1fs lost=%d dup=%d' + % (instance, mode, result['during_10s']['errors'], result['before']['rps'], result['during_10s']['rps'], + result['target_served_until_s'], result['back_in_rotation_s'], result['data']['lost'], + result['data']['duplicate_texts'])) + return result + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--output') + parser.add_argument('--modes', default='stop,kill') + args = parser.parse_args() + replicas = api_containers() + assert len(replicas) >= 2, 'need at least two running API replicas' + results = [run(container, mode) for mode in args.modes.split(',') for container in replicas] + bad = [r for r in results if r['data']['lost'] or r['data']['duplicate_texts'] or r['data']['duplicate_ids']] + summary = {'replicas': [c[:12] for c in replicas], 'warmup_s': WARMUP, 'observe_s': AFTER, 'workers': 4, + 'lb_check': 'GET /health/ready every 1s, fall 2, rise 2', 'results': results} + if args.output: + with open(args.output, 'w', encoding='utf-8') as file: + json.dump(summary, file, ensure_ascii=False, indent=2) + print('FAIL: data loss or duplicates' if bad else 'PASS: every replica stopped once per mode; no loss, no duplicates') + raise SystemExit(1 if bad else 0) + + +if __name__ == '__main__': + main() diff --git a/student-tests/failure_check.py b/student-tests/failure_check.py new file mode 100644 index 00000000..a6ffa7ce --- /dev/null +++ b/student-tests/failure_check.py @@ -0,0 +1,127 @@ +"""Usage (стенд уже запущен через docker compose up -d): + python student-tests/failure_check.py + +Останавливает и замораживает Valkey и PostgreSQL (docker compose stop/pause, без удаления volumes), +проверяет поведение API и восстановление. +""" +import time +import uuid + +from helpers import basic, call, compose, new_chat, new_user, wait_ready + +failures = [] + + +def timed(method, path, token=None, body=None, headers=None): + started = time.monotonic() + try: + status = call(method, path, body=body, token=token, headers=headers, timeout=10)[0] + except OSError as error: + status = type(error).__name__ + return status, time.monotonic() - started + + +def expect(title, result, allowed, max_seconds): + status, seconds = result + ok = status in allowed and seconds <= max_seconds + print(' %-48s -> %-4s за %.2f с %s' % (title, status, seconds, 'OK' if ok else 'FAIL')) + if not ok: + failures.append(title) + + +def main(): + alice = new_user('failure') + chat = new_chat(alice) + history = '/api/v1/chats/%s/messages' % chat['id'] + assert call('POST', history, {'text': 'до отказа'}, alice['token'])[0] == 201 + fake = uuid.uuid4().hex * 2 + + print('1. Valkey остановлен: кэш обходится, всё работает через PostgreSQL') + compose('stop', 'valkey') + try: + expect('первый запрос: обнаружение отказа кэша', timed('GET', '/api/v1/users/me', alice['token']), [200], 1.5) + expect('GET /users/me действующим токеном', timed('GET', '/api/v1/users/me', alice['token']), [200], 1) + expect('POST сообщение', timed('POST', history, alice['token'], {'text': 'без кэша'}), [201], 1) + expect('неизвестный токен', timed('GET', '/api/v1/users/me', fake), [401], 1) + expect('без Authorization', timed('GET', '/api/v1/users/me'), [401], 1) + expect('/health/ready (Valkey некритичен)', timed('GET', '/health/ready'), [200], 1) + started = time.monotonic() + other = new_user('nocache') + expect('регистрация и вход без кэша', (200, time.monotonic() - started), [200], 1) + expect('logout без кэша', timed('DELETE', '/api/v1/auth/sessions/current', other['token']), [204], 1.5) + expect('отозванный токен', timed('GET', '/api/v1/users/me', other['token']), [401], 1) + finally: + compose('up', '-d', '--wait', 'valkey') + + print('1б. Valkey на паузе: logout, затем unpause — отозванный токен не оживает из кэша') + revoked = new_user('paused') + call('GET', '/api/v1/users/me', token=revoked['token']) + compose('pause', 'valkey') + try: + expect('logout при паузе Valkey', timed('DELETE', '/api/v1/auth/sessions/current', revoked['token']), [204], 1.5) + finally: + compose('unpause', 'valkey') + time.sleep(6) + expect('через 6 с после unpause: /users/me', timed('GET', '/api/v1/users/me', revoked['token']), [401], 1) + expect('через 6 с после unpause: сообщение', timed('POST', history, revoked['token'], {'text': 'после logout'}), [401], 1) + + print('1в. Valkey на паузе: logout, рестарт API, unpause — отзыв переживает рестарт процесса') + restarted = new_user('restart') + call('GET', '/api/v1/users/me', token=restarted['token']) + compose('pause', 'valkey') + try: + expect('logout при паузе Valkey', timed('DELETE', '/api/v1/auth/sessions/current', restarted['token']), [204], 1.5) + compose('restart', 'api') + wait_ready() + finally: + compose('unpause', 'valkey') + time.sleep(6) + expect('после рестарта API и unpause: /users/me', timed('GET', '/api/v1/users/me', restarted['token']), [401], 1) + expect('после рестарта API и unpause: сообщение', timed('POST', history, restarted['token'], {'text': 'после рестарта'}), [401], 1) + + print('2. PostgreSQL остановлен: ограниченная ошибка 503, анонимный запрос не проходит') + compose('stop', 'postgres') + stopped_at = time.monotonic() + try: + expect('GET история', timed('GET', history, alice['token']), [503], 3) + expect('POST сообщение', timed('POST', history, alice['token'], {'text': 'при отказе БД'}), [503], 3) + expect('неизвестный токен (не 200!)', timed('GET', '/api/v1/users/me', fake), [503], 3) + expect('без Authorization', timed('GET', '/api/v1/users/me'), [401], 1) + expect('вход по Basic верным паролем', timed('POST', '/api/v1/auth/sessions', + headers=basic(alice['username'], alice['password'])), [503], 3) + expect('/health/live', timed('GET', '/health/live'), [200], 1) + expect('/health/ready', timed('GET', '/health/ready'), [503], 3) + finally: + compose('up', '-d', '--wait', 'postgres') + ready_at = wait_ready() + print('3. PostgreSQL запущен: готовность через %.1f с после остановки' % (ready_at - stopped_at)) + status = call('GET', '/api/v1/users/me', token=alice['token'])[0] + texts = [m['text'] for m in call('GET', history + '?limit=100', token=alice['token'])[2]['items']] + print(' прежний токен -> %d; история: %s' % (status, texts)) + if status != 200 or texts != ['до отказа', 'без кэша']: + failures.append('восстановление') + + print('4. PostgreSQL заморожен (pause): запросы получают 503 в пределах общего срока, а не зависают') + compose('pause', 'postgres') + try: + expect('GET история', timed('GET', history, alice['token']), [503], 4.5) + expect('POST сообщение', timed('POST', history, alice['token'], {'text': 'при заморозке'}), [503], 4.5) + expect('неизвестный токен (не 200!)', timed('GET', '/api/v1/users/me', fake), [503], 4.5) + expect('вход по Basic', timed('POST', '/api/v1/auth/sessions', headers=basic(alice['username'], alice['password'])), [503], 4.5) + expect('/health/ready', timed('GET', '/health/ready'), [503], 2) + expect('/health/live', timed('GET', '/health/live'), [200], 1) + finally: + compose('unpause', 'postgres') + unpaused = time.monotonic() + wait_ready() + texts = [m['text'] for m in call('GET', history + '?limit=100', token=alice['token'])[2]['items']] + print(' готовность через %.1f с после unpause; история: %s' % (time.monotonic() - unpaused, texts)) + if texts != ['до отказа', 'без кэша']: + failures.append('после заморозки') + + print('FAIL: ' + ', '.join(failures) if failures else 'PASS: отказы Valkey и PostgreSQL обработаны') + raise SystemExit(1 if failures else 0) + + +if __name__ == '__main__': + main() diff --git a/student-tests/fixtures/sample.jpg b/student-tests/fixtures/sample.jpg new file mode 100644 index 0000000000000000000000000000000000000000..48428645585f37b25e4daefb54550a2048688977 GIT binary patch literal 19881 zcmd?Qc|4R4^gsGAF_MT>qcE~$X+i5)#-QwF?1U`QM`YiZ7?h6^Gxky#SsG-C>|#h6 zOBrP+;WNb8x512=ai6}w-|ODj>;84`{r5gz<_~6`dCqyxbIy6c-{%~R9!vwr40R23 z0S*oT-~fMsg9+d!zy&#U=n#YpeBt8a;)e3^K*2-c$dSW*f&xN9f&zkq$4`o$Ixc+r zgrFeoEbO$H7#t2ieo9hWQe0Y894`L%Bpl#Jq1;e@9v*&iVL@T>|JTPs8z9I7ba7}x zI3xj1K@Nx@$H5l>4qhi0$NwC_|2{Z4A&0=L|K z;E*7f&{<_I?&GG;P)R@G%P}c$d8D+f+E1AMSeL$X-#_*+@5xgzk<&8gWaZ=)u3o!- z1EHdN^OlaTp1y&hxrL>bwar~y7uN^>xIJ|D2nY-c4hanlfATc$-}q2t@Iubrf>?w+3mgG0k3qhsSU|50Y={>(2>7dJMy zwrM-NdvwNMyEp*I|1t}F{a=Rt-|P|u?cxM)%OU7ryEr&Qz+Z^qA+EE^+(KHWP-nm6 zl9yw6gtb%NR<$3Nx?;9|;=cb6-jmW-XJj`1n)W}I{Xa7-_WzS*|EFR9mtErkAA|$E zc@RMW1?>GnY+1R~kv&p}FHjPOYU}zCm4t48pGv!%r=HDSp;VJSK8okyXWnmqvif%1 zIrX;N!xh@qFZ=fMYw02j*-FwRChI*9t#ct+A?B%up%*UR=v6rO zwE2cS@pZzTFb9&CYIUUJto1(fHO1po7JD&}xnZv7<$;-dIC6iTbA^8|IB6i zYbVAcNJa^@t&rtpY5q(&sqs=(O8wT=?Ry3OCxRgOW_%aKrZ0)-eP1dT|IhdE=Jz$9 zaz6Eq2b~#9A&He70GTH2n+HI`d*z97JpF;dENZXR_gU8w(kB$*`;%n=u&67~etZDz z9RLTwgL|-0lO!FHffs1Ep4tN-HmI|pE|HAoI{b>%~8~g47K*6(;3?6a& z?Gyj!&n0~~VEq!EH-%0e0A)I%2>WX?1{TGYkuR;ty4UV4KWqKYt8|y|pfj4KSE5MO@r`|9Xv4UgP#Iu>Wjgl{J zdzR4-0is#t>9CTU!VkXv{K7M>pN*e08ts2aOQ>G^?)E2}z~!AA$q)LNO`PfZ(Se`& z$Rsj`Zj5!ASeJkZTjtkyd9-e|fT!mSVpsm~_eg z{H4{UpWi!8iSN~%%}rxZS^n~LkqB2vqw@yeza5tOEQ{JCafM;A@iqIEcT@s}fm<)R z^-_tiE8b@}28{S6^Vf+|-$P&rz>NzEdXW=JWi6fufK+vk(E(s}HmMxV{1DIHJMqu8 zI@IQx0BcEby6mJ!MOnAnqQXK>d1g3lja6^%SIM(4RClow39>}2`Z^F_nd_pFhv zsNh<}*fKxz&-8U0Tc1pevxkGd z%`w-6UvIWx$AerM#IDMnh-;lgVOLm#RQiAKu;YI%J4{?lVPA7C>k)3&;b~V_{LS2A zm={2NB_GAhrhSxx=gNiJUsbTR$j^Q4|223_MFrKot|mVe{Hy%~4pFZ1YC^La*b<%b zR}Fvt#P9{;zp~3=yKC*AA3u>AH~=bNC*~I2zGQt$&oJY3LS$3@?!)HmSsTnn^AXp} zo!_{AjRJhB5f=Fd(k@~j#^85GXAgk*trA(QRA(;{);J|t4bQIMDiP>q<9v1*Y63Gm z;tQ-$wHwBcz3@wGn??@Bce*E&%arWIij2B(P0+3FlD1bRZhnWNo;+C^ zsfbFSd+t?7It@*yLGy9pq0ddXE>Yi6=Ezzsjf~g+`8ik9g?UP09ushJ)Dh=<1v=pa-R=qfi9fWX=L- z$t(Cq7`-_NNi^(zeYFy{U*hf)7k>9Dab4aJO%K-dQu&pD7?x;EkRzMHGG!OIkKXy0 zm!Aq#+;G+Xw0{63sYq@P`Mx;Z864Aya~xp)DO8T`ZI0E27-V`zs)dxL7a5Dbzw@c( zQrSy<3(sOrc+Q8MF=0PBzT&ERbx&>5mb%5h%@x<`3_t)6-LPZ~)cM}@rbi=^!E*-& zl_;6G$b@yV;m*-2l0|pSu(^wlA$9?JtGh2BxjzC=)ir!9`dC7Api?wAYF#1YM6Wk4 zc-mKccAlEZ8~#NK!YeE|>UAhYN=1c`&X)XyYx+KjGY(s$+SD&+l)Kg&5!~QEXk)(_ zI|DgglqBKzou>{_9zt-v=iX_WO($kRQ!+_3cu%hICSkSf@>?u7Mo4YnydU_HYmpp5 zIRKusYXx6FF?g7U-VWXjF`ioI@4B5Zu9R|0p#TzVop=CDNxpl2_t>HCe<-oW$h~u! zay@NCIr|SH6S$Eh%*Dv)EBaxip5}PWmq@hjpSPE6eNz`+Qra>%%P^AFb%uOn%5-T3)F_a&izqeBl(%vg{9Br^Q(&nsoUpwc zR!K6jE1#27Y+EL%r4-)usESnq3tH~2Cv?7lisJO0&c2G}7ja#oV=kDdN}RZt3_vd) zrj8VomJ%}3u!ih<#y5o(fULJG$sO@}T ziFbn@eC_gu^$MEv1t-^(!AKn>MDl^(Riw=^#C*dV+Hy1fj`!VU{~lvQu&sEn@o?aA z&xX6hVvj+{z+`cAO`2`_g}(6sB8_Ep->zMC^@X4TR(Tgah>6qqsfWEO8vS0rof|(J z&-v&9M>IVpf`vQ)$iw$v1o*V*KG4dLDiP0k9q@F>_IlE-%2npIQ+B~GmkK7Y)E!6b z)z4Zt*V)|t=b5wUbfr^dDJvfFZoG2*V(ul42NYSN&oI6yAYni;M^z*db7okT)S@SUX%LAaa$bOyg zYjr@~F$5FCph>-Vx3pZgqOIMGL*^Yt zCa+l<`(Da~ameMsJRvD5PVEW`&Li`y`U|T!-iBDj+V~5d^MtN8zKjWSBZ>C@sJI!A z?r-D&)UX}&*CdGPz~}z-C^@a#B^5csrpL~vhL-VW6fs3qeka1I7pIIp#lUZ^n0>+j zV~M@?t?V*C!gR@O#oV`#wCkHFP#M*-MVd2ye)rUO)KU>jQ3UL(hU}=i2Ij#gcg{6Q zugG7@9(dV&YPh#l&t5jNi-L1wARFlayL3qF3&Umf(gJ_0F1=OWXfi2P&ApEzrE`llwOY`3gObVM zpCVJuZspqYDW=B9#Ju!bKLFGR^JSrbBzhCKc3t4${jM}N5$JIvOOpZe`Zgw&1z0+qk zOFlZ;^QD_(m6>`!3waP1@`@z_)vhe7<{kj-(r!Z~s{wR-&_Y>rVLtKoVUJ*K7s*5} z8KqN*&&<4lN!&K!^IHpmGZ7=uogL5__WgFk-RQF~_@kq30$2Jmw&D3P%^F!}8=JYRAM1yn~{$ zN~E4)uRFQKiMp{lRU{~lgwlUCFMkr{^n3CP^O3pqM*pk3+W^hTc$~x*xilUi6$hAq z@YQgxPZa3bSoE;aj0QN9m@YB6O8B?Im=M+2Gl$lZI5R{Sb(Ht zG0S;v8gcJ&J#G1AzS`N#a#EvJLcHr}hzOOvim}ZU9&H0Izu(nt@1}JM5L}Go^flk= z$q&H#jFH?JDRa-y*>PY`8MlbMzpUQ zBavSd_%fThkhYL|Terfowof2W^S5>?LyDb97ExJgal{@4N zLcS5Ra`&Rp+JnJm&|j2^t}|+!8jijkaQSN&9-h=JZ9sDKcS2ApjDgq_up`mB+`w$4 z>J9Jg!I&GI2!$)Tm!S>?rFg%ejp>74!et{fyXE`M?gRPv`m;`-HRCMdgIe4xjI`b; zxo>jHKN=8KzX!gb$?`CX=QzS2P;>kM$g7`gXS&a*OlETF8oBVbW{Hu|?AnV?t4)C` zd)rnad*$!_^d5ly$5kH7=Ynq0)E;~X_~@7wM%?v#X1^`9TEBe2r>UQ+AA;5HQRMQ5^w+(iituksKZ#1hpqm~%#7H%Dp$yb9Na`1rs z`T=IRManx&n0NEN(1&fEQBOw33wd};)r6k1SB)1V1?cm1eZdz(P{ys$E{6~hfLflG zF0Wo`+5O~|pHXYI>!J4`f-q)`8KJQxg|%S&x1=few_*W`Hnd{F`=LkJL+zo6j%kOWlx!wkDxN`JEx{6T9n#+l)Nmx~<1Aq}{W5LO? zahum9(J_W{J-kIX&<_yZVY=>v!<;f-WPK4tkhFohT6q#j(7`gnR3o)ZtZd7}xLk&LpSxKXXu}g+$sLHrH zUOfOR(wU!J1wcRg6p6^(8PmG@dLgZ*uqnXUVcF5{Qi+&v&aM;|iC@^E%r7jX;zuI} zQzT4Wo|fj#6v@I<*tL>hgkdJp35^-?i}Eu!6`JLF1Xd}lBoG+nqVT92GBqD<-K+1J zMGB3Z?G-5oU)TSetddR~-^DcH5K~yY1vLz+$UrFUp;D`%I5+n_Vh3}}>{&Ma)sstC z$dic}JiA3j6NR|n*k^;w+IQBYGv!rOz3$yDud^Coio96#^U?m#^p=6T>{XWuJ{zuy zoaIy0DTZYm=dUXn@TNq5w8H@~B0OQU^*{i=*$+O)ulmNcxL$m(m)>qCZ-2cYy?FDe zp;4iWmzwUKi%s2?doGlMs}qt5f^%QJIeH&|-(F!ymf9zET8hZdzMw9#FsD;qX6c)S zz~uub65&f`C&s_zsh}qb`;%n)_S@dEOE%4NE8ahvtmyJaeP*~+HTk|-qtp5H$ImGH zd-HzFC^DwVNGRp3xBAa3Wc?8^7L$55m$PPNZ8i~$30AyzPm3os^ z)NqKEhFu1RR)Xjex5L<-w{v*?DBr$cO@2`FmZd0QU5(ArgRvFo%t|wQy`*Eh`}9Sq zxl>xd@gJt2G_M9Ugfkcd&Mc2R%?1dRqQ)!ez&`QYhRwdQQA_!rPu>1q>g<0tRTIW8 zg226K0GQ=qP&QG@Oa;qlsRG)eP*fL9o#Es(kg0Ti8sJ8B0!XX?x@7)!=8TRk}hNsI32)TUt2R z7pbSov5e8k7Vt2gs8e3S;%`UclLGn*<=dC2EKK{~IxcyyI4@ebxG%J=Tcs1(f;#>B zngo9qz>(Gs$rac;zD*#VTD5S2*;8Q4VGAgAfpSB^>X4%yB{ULkL3K7_u;aDTi%lyQ zrjmHG?)X{D@#_H4ryK)f6sbNmLfxV?OGN}G7S@CNqn7KhQevqx?yvjH6+2Zzax^uz zJgd17>6aLuyT`yH${d?Fkf*2z0MwyxLagSGo~W}iRHS{JJM;hudCOEAEPmAvN8LUa z)pBN{fA9s&1f*ea9Mc1WsZ3If{;4_X={gRzM&`cY+QfP>@f^dB4gYxnC_cZwt>=2~ z*0W|F*pVMtr7H`1-g>6YMUw_Il7gN(E(x5wU_HQ?a_+?DKjRIrukh{YwvFE^p>B%eS|$PCvao_e^3&bO&U& zh-BIc)vxKMwPC=FDm-Xc96#G#^+v?>oE`$c1l_}Jjc4dKF%Y<#W}9)a^9~p)oO-C%wY*mJKo?~xA0xKs~^;y;=x8CT;u;smW%u?6m<6ABAK7cbvXs+GACU} zn99r;k#}RlCb&!IefjPF_z--+o_W^$a&b3G+7a!zv^SXesYg4lfo_!h3#?|g$rZyg z96eDhsNbs?auL5nxBu5My+^@GKkK^<9sJWDgWO<nQvaseXoi?cBG21-6{#o}(h073teM#r}cg!5{Cb-9_fq z)Rm|Of<5ysuvR2tuWk-mqF>@OBDyf2v-3{4b@%z&3;e*4_Xc`62@#_6{MHA6CVZYT zKxS>#<6iDz#>CkNz^sdHQ^G>3<495M(z~X2MY0pdx|Zo-`-;pX42!z1gbzZ8l?&7W zH9s~t-m|RkL(_T>R%^GdBtsV_Ki+;qWi2fY*ESsLZDanVu0#hsU;1q72qvIpVJmG*nqlT@&46q7Q9MH%DE@6x>arNN(o-id4cpW5Z|AmeW_bEv52Am-_%Fn*Q}4y7P$vbm~Oq z1J|?_n2{@pm8FuduPEekL7hpH4$Bc+?8rZH!d{=_MiE{6(YqS+1I(8S%AQWAq+R(F zLn+wLMLNKHevV#O?3KwXW;}@SGE_E=26_ipYhT{GI@}`!_ROFB_Pr)Zy-d_H&U}r1 ztCz>hvi)dIo}yZ0JQ1YKCG|~8lZrzG(z)1hr^rY&T`11NyeCypwhjrc->R@Rc~>&Y zWD=*@w(e0j6$=)F&!rrKOOp!bIs(N&O6)6!R%(%C?Z(UficjT%LrgU1wsk@xfhmq#Uw6;Hmx4JvE=;0+Iq)n2e zPq~Y5sU$8`viVU_U$%hO&L^Vaa{@3=Bqr0tWzEYx~;u?~QE;U_Q|eNdS8bk%CU}a6osjk`ot_C!Odh3(VrY%xbYU7gBc>0a z$0qIvFyONumL`|&xy1Z(>9Zj0!#RYN8s8XFTx!j%9kJ7hcRiiHDiAAf35 zlcEb>iOwAONp|1Sb`>P8)Qd8S9qfLZ z_|&X%iiCvFsiU};IZt=RNdwII&LRt^r=E%7@-%T$kzq=rv&A|Ojn?BDmC8EjsBD^4mdg)TBQH4##JYoqF`D$cVb!nTz5D7{HopBdzDqrvAtl$ePgi(XkeY0%v+Vov|3 zAfq9o4RD~{^bXu@Z7ImCI-XPD_08z(f+(Df=Jdvw*oZ1H4zGx|X+HgXS?f8!svB&n} z_1Y}3-b(35-+P*q=LI7J{v?J){&knnGtUq41>Q%9sR{Ju%*L5;Cd!d+yZIpITPbFx zKw&*n3$ciKQQv?6ts)5c0Imb|e8tAl{!O#p`sSovR z%70c&&ybDw7eG1SvzQ<6mNdv*vKyadgj08=_jSC3KRXpSYjSAEw5{OQ*^#Q|pR#l5 zOppMmoT?u#j59!P;qBoEfFseTTtCVuMKw^`FFRN~ItuTA4U+G}Uu`woG>CHST?{sV z(|G!pX+US2${{}-b%5}n`(l=Jv-5nmI4ILDg1QVZCRI?c=ZIiiCyJ$sp^-gaI`#b) zR-z;(tETPkv(^3V%NzFVYRik(Ds@5#KDvx|A?8^^dE=bZe7=wkGGv`JLn{duGJ4!% zu!vM`1l2WbJo}gWxY`7P*j?CY$_z*>&OJY`sO~ zSN-DwEUT$ET z4}+%mC-~mpL^i>D@v|6+dDz)OPZJoQ%g6*4EL_DPU6=*lj@#K_mDys9TtvIe%k{d| z)59xCt3FNHKSVvc3W0{j%LwsL;lWb1w~x!R%!9-Pkf>KHi%1U>rQp17N-~J9!*icT z_!*wcK6xK~#)OT4=jzQRh?P`0JSrl*d`CoiFDRtGNDQPc`XZZg+G-XFMVg+jX~{hB8=&6qhBLjpRs8iH#>&V` zW3Cb%tSa^)3kE1o0({K8?d6V23bRy4w{IZUuSPIBgx`WKS!i*#^QW=Cg2AFDWlJ>N z&urR@H>-UntP}HG4_VFg-6e7&$~I{8o}y2>)eG~w<5Ef1=i4{8vRyf98&H+PG+kbU zMDOv#^Qgn;~UK~5g#k5T=1cDbpg)#H1)SgNhg zHqoN5C*aJSzW=jmfgH_4m}5P-p6n_W-~G^ENh5X2%taf^k7u4F8>;NU&YMka34xpk z$nEqm3KdxyWz~MloY1jFUVntqm=OC={9}t&*Y1A0s$H<&$DV^RICFpNvR(9au7HmH zY3%O{Ux>8tsBP7{uJ@lfGPyR9SylLX$B?)$7NMbrhSw9FXlO$7lw*+NohVy)mtblm zpJ{Z=32d-}i&%FzOTp4~x7}*Cp$DHo*zn+f2$w`X;FhvkLZCRS1(5DXA|T>%rG*p& zy9e;z<(?6`k&GM8L|SVXcOPNi``h&`zIh8S341S8MHVt~A2KMHc(JOeBk?I5_`>ya$e8OwDe^ zDsgl00+hFqY_}0MnHBOsN$Er|QCVOU{=nGMFOf6PO^k@|cRPm{b~<)FCR)RCw$dA=9z6Rx z!(64!L03T34c;rxg0?sxk*N+th^?)i- zemonrWeEvBq^Rp>R#S56Y=`VfXt{ngWv+v3C6|Md+e)mwZ4t0p+gnuWb#27@0d;n& z)a`*va-2znsN8f)XJkBqzV_4AH2Mwi#+J*6WhBud60P@)Z7EkUhnVaa$lGDkY3g@G zAAOW%ITAN8G$Q%->rURsJ602O-c36T<6A>@!Jpsq61uJ@-$(`^c`Iza1HfKim@8Qh zBw9$`#QDkt;Dot?(zm2eU4Z1Xh1-fTmVeP0F%&wj&b2 zVxsQKXF)kpkvYj&%q4#hUg}?Y+9|CE$(x9i5SU^3fsWc%7qPk&qdgD5q^iV3kqIio zrDbI&umbBNmmcY>JP@{zAnEwGBC0cu@sE$x|2*2|E>3n zN?zY>CwU(L#U(CUBAyk%!r?_Vfw^sUtUMJ4;XN$j4SDNG0vp!K{@3Ypz5E=~d|XR= za1u>CBi7@vT~Ph6?UA6sn%10fnTf+Gx-q(KDpI0;Sd8%jpsiptq{$IHjpY#w&*oq@ zO+P_f97In0-j6~(6|pk zl5?8GiN#D^1(PRdG>?#+2dGoP^bYJ?VUiNJrKY`q!PzK0x&F{VLE`GDu&a1ZR2Iux zcKn0St@75~sGMcmfX-|kcpE$)D%T_xAi5Slu^T-7*35LX=-Mk}KrqqmonJq%Adwp;)6G00f8tH5hqHk03?OP4*x+%z4TmXgm!40{TPxUAz?<>E&0gkx&JI>nBbuYo?^Yk_J<1xJW+f0)K|&9Q@0q9X|=@ z`gOClJRC2%%0kZ?Af!;g4}kyl=Fpp6@T(FTNUSPmA6_1Cb-FCg=zG*D>!y_;wqB6x zQzPE!;|O>NA3}^M@ZU8hD<>Xp-Bl=`njikYMU?Pdq6BbQsQ|VDwrAZ$)s*{O=q#*3 zpY@^udQYT`W}RTQ9Ui3-DdZ6EdG1z9(wk8E>?qi>K>gy7oS8-AT0ccXz;t8)rNdcd zqSbpmIiS-Q#UFV9tfd%QvvL}CsEl9|TW!sLtAy7#d})FaMF7W?Jo2L<(Rr7Yn`2g3 zs+jO1USFt8)nKJVJTVHNM=LprO3~$8JYc zOkkXy7%+>;#S=ahjdR8cMFX9jAQ!8C0OSlwKGR!YUE0NB$e58S5rqUXGdgwTqt`gY ze&uP`&&$hs5`r*HsoJdBVq?ez>I7Ud=S8O( z!O-Ms#qTx7LdZ|AYRB_Rd8Bo$^2+w^_S#zSf0hqFe{_!D#W|v>jDY?ZN?%`@xa{e{ zb{G+iklndEb*p0_VqjrLTBCf7>%{X73s0<_xf(c7T6eze0f(NMwP-{{q$<00`gXC6 z@2w|3kcREu9VSxq+jHSk-NVi)?eokX7P1-)+YFJX$FU^%d_8@_ULJA&Zqc9Cxxk!} zG{?7-yiXs&+O*mhAp9!eml;PMH8}0PDO<(uD73AgJt$AHHc>C(FPYpQa`XpYjQpHJEn&=?vRAK}fHWu& ztvm`6TiXl2hZG1BO5RWvF{1btocDXPvM?Q(7?h)dsj>^E&z2G^jZuDqfz&9%FU*wg z^MIR1_uMz>?(~p;aKiY^#$BNpdqBTqq2+|MS z9C}o9QeNc9Zom}$zgo%CoPuwkML}OdKy@aPogsZ+$RgvH0W@t1$CeuocXgqPmhpIih-j2&OmNIGPr-h{#($xIUjaeGo2R*l2a-B{v;=eLsP8tG5`aw)!mK)Z+C!;cT(@J%w~ znueX{8fAZK9~o6-7^mM)K)ffwCPRKi$!^RloF9#ml2krZBK7bHc1jNZz0~B>oQ*y) zR9>Bu#*7_oweK?K*G>hevs(^;FSs83T(ZVPErkftzyn|hW&6jy{qDJ!ubG!BgT|Mb z*u@~8%oD&lUyW6muUZuRi-%KL+p(ufzxZLn6pb&%O!;1zCLrXjt2`lg0Hj3)Nq*tQ zUlC2qt{fD{uV1;lSSeh-u+4nTkZ$20z5QJ3%=LA11XFnw?lV+sQdkl&^|n(}yrA-3 zR05Y_-jhsOiTT9oC{b`iaY{11a$n7FWndc|OAh(nIBzCAO&h7-{_Ebx=RPNcF>RO~ z!;d~bIoX(yzplNI-`9s~23=~g(##=+_`C^L-H!w7=k0C&&pGUXSr^OST?uNT{hZ6( zfb0ldaSz^8h=rx>uBCY1{6^Y#%r0$$xIr?{b_!-F&YKm=aC4T5 zI)`?XZ?ip!Ir^m>eKtoM_eUjyrz4JGrmwwtdLcdkWvOF6dVOmlc%b-Jl|r<(yvdcE zLXH6uG80NG$;Vz&dZs*p*jXZfV7|_k0=7;3vN@vBeq<(3lw^^m+7q{bS3^ZuNL+s{ zuTo-d0(F4KZe9-7e(Qk1jCCTLz_fJ$VD|^}rOlHHd=YJ(dVe(V$+QwF?F63N{!lH_ zALu)cg5U3S#{H5DkRgI=BvUN@U*vi0vDnzwTnwl!foVirrfvMytP5}h!~VcR-qAn| zX=jcXBu1@?#}nz)HM6H;gB&DF#x5KOH>H|Mw2{9RH1^Ga1&U1DKKEv}Wt__Ldazpq z!d@w?qLzU86wAk%two4CoYY@Rfp^d`iUV62Qotn_GnKi^%4|&*Qcuz3N3rk3OQI(x z7saxXQn?13dgQ&k(iGJqJ~6sNP_E*^DxZ#cTrk0eQb$@|TkLIys zFI=^@jv@Ng*p4sh3wtv!f;p>17NG-7wnM@33r6A`_r66$U_b(eQGNT3anwIje>#U2 zQI(jH!Hp4B3(AfVGiqnK)!={P>8wqLuB}8t)kPcz93WHkaFmAtC!kAH69jaB?&m^~ z`{8h*j7Yn^WyaH%74DTOX(?zTR;K!cRv7%9A7|swf3p5X>``2I%$hKMEA$ySq`k z-NQ@5#G<-*SB0<)N<6BAE-*&|*F-ofvNsEv8lUt7rpd?ry;MoxdRuH-#pfIGDf3SB^g6{%GRDQO=)VUnAL zgRL-JF;mwh^LVG^&L?gXJ7|9;q_-Atef4Ets(VAFkXPuL%wi7#D`I)X-N>CSa3JBV zez%$BeQ8`e2OV>b*;1i|0=9c!<0R;C(vcC zUK<*cKl6DTvi7(D_dN+MphkqVa_c|;CsbN8XfWmG)z{E#R4{KB`tc(hoItF$d=v3_ znX*sZmoa#7GT4tb#Y!c3jUE>1Je3J$<>F&(8MiN;PAx38ux{;bWIjJ#v2)9GqBHuG zlM;tU10B-_stdkf3LlqEX!b$hT>+=LUcb6kynlW5zCNX18nfIsm;mGJ8tpsdfsKGK z4vkaIv8l0+=UFRbkv;R4s<=iW&10^gl|K1Tf;e$Gs%C@tUyeg=Sk-I_r2Yp+HRG(D z(g_pFkxt`qw=GK$u$b`sw|S9&w?pb7qfbBwl-ZwkEfQ`qV+mp6@i*&ov}K-Ka0(j6 zWUol@zv{zGqss3H`}sZ9!e)``9U|jr2RC5Q{3nUie4G(j%=FET6AC52&ZyrAXd_dX zqvDE9pFSwFHpW-drd+B|@VziG`_+{vRMOx!DuX_KXquz(O|+iZH-W9f=Qef{Vz6V8 zoo$$+yFm}a_nC{M{#pdQClLn7}*W0c=l3^{u;Ubfx333}Sea6aMywA(} zR;wZ%-Xn4xm-UgOK@^Q|?sJ?cVKawi&}W``ZtNXL^B&9Iw0=TDh{ad-xHp#Cdj9Vb z?xC8@mh^eQXECR|wVj+O#Ve^1f#znCvvcN0a&`R{{qPktdxNpa_x1TP|B7iso@56M zK!sUPIgLbPw!(1^IV$Z;57F$9A2M?hk9Qml&hIpi8M|s-pFZacJrjx`?FEqi%oRfJ zHOR_}p;&h{T25y_LqmDB=CQ)_4DqmK56uJtNXp2>V&q5cuhZ>DnVVygQ|nRt?%wf% z=ai3?R-A%XWBJ1|wAQ@SDJCvwqeaHgfSs8J4-M1@S4=a;8|c-($^-EJjlZaoqgz4_ z(JjFWpfNT%%4nwOUUIJkc6emyVX%4DX5cLs2Cv8*{1Q>iXs2Vwre3gZZf? zM!-V&m(*F?vF&<{BI^5E#Jgbgk`lg$$KJo?I84pOvwXG|ZvD`;@>lW=6UE>O+sm%E z`203!7y>u98H}j-*uBA%&!$~x;ai}D27;+aa<$Mip06>f!S`|o9Dk}gUp}uhCgOsH ziqOE}EKv2fN$b_ro_q5y9FvRb>OL>*)@5=fwhg|8^8w`rEhWdSH-DCD{rUtzf&2!(x;>*4~>@1eC z`;72+$+d@fBlVt!{qC=i4k%IIS||i z9DMBG(evter{t}g2c zhbBr#25)xM_fAZmH_MOXN(Zu}#g|n#@c&%hj5)&R`!to1^yPXe^DrTf$2jbKJ0+XH zV`T2?uc3?AAB9YGhp{g%j{n_9Y51b~qW-LslHXu(H?m2-M9|;AB#tbFpdrX3zDy4- z@u7(ynjQRk9PNE7)F%m1pE+c(n)DXEO5^Nl_BDQEwT0BpmevSG>5P4QaC%wdjqh;; zByp^Uml~Qu$_Z2a9CWSj>#MVMbeK0rMv#qJmK40!!^@DI%6|fb#BO{Hx7cl z48Y}4nF{8D-P*#J{@`S{m|MoYR~qXA;t_{vFrOZLi@{P+^Fn2F__nQdWkrtJUx~s` zr~V03{TQ-A0`jYbQ2JDXn@7}1TtGC+C6O`MeWuvbw2J~kaACd#5hP{E}GQHV-?JLCMVy?xKvufU`ZCAY0sZv7cWL-Om;s z7*_YSUUQS+I{-eZZVX5kj$cW&_B&H;4FNNbwkjjSj3|>k3??*ea9QyAs^T0b^%mv7-m|^{05Qp-JM#nm zI}9sc)7U?$Ug7RrOFqqaffA~ zMTXr=fz7q1@{w`yfx2(wgwZc+8Js5-kjrWwz>!M7w@8^WF#m7WeCbq9h zyT|b=9bSM%{#3>FGE=_mr@Z={GYUyoi-N2(u<;4WioMZau>j1Wg|rvM5Fd0ulu@?t zKD#}nhV**(X2+=dc;jh_&SSnxZTz7i1Vw!~F&Z7?dIniCc!8+m3!CV0BWpX!recEbmEw+C&T;%-PXP_S?V0B-egK8PO8< zTira*>82#>4ylXfiD#|)u`wLzB+D>_L}Wj=E;4e9!BXv>OLTeYTwmwJegG(ji7dTF8*6ygz-Ed<0vQ8da125xi27Cmf(fh zI_Mw3+U!<&V*qVk{$YLHq<->iPLqQo>GMN~Wod8I$_zjqdvOG(i} zzL5yT_A-~su~OIAQf~h%3u-IaqdwHdEIa&|h2FV^)PED$$qo`B&t`hbB ziWC=YZVT3{^r?kY*4u>l22k?%8mFU`6KS2{#|nxJ6L@9}?0uEH#o~<%x7N^t6~U>8 z`+jPPkgbqNRkQT`GD_WV;<}^WCMfHqNOGz^J+#5@Tjd5;V$A4aF#SG#89mLzYI;<( ziMF?;bF_^d$-E7SRIt?H=XN~v;qdA)*QnYg%A*@CwG)xKO%Cj(0liHlu&jM^c~lPo z-(n&H?J=qm#X=`c03ierOqbvIdZyUydu(Y7=99mNGjFn^EFy#%P*1lq-^q{)m8rat zOYB)u>>S?I^HP28{rCXzI_dQv6SHkY`25E5#LGDjKWyUz(w=#FR)&-bH#eCX>NMEp zI@X{tdu|m!>B+m_WJy)^O>G&P7^ntUk`49QnoqMt^*Jj`rWL%}0#A&ue5+Usr+&4eGZI)CJZ>fr?lN(UEhpm}4(fcjoQV|BJ zS{f7wh0%x-x%b!hJlU)YhSwOliUUi@fQeE#{67T@V8=wm*A;B~|B7BBTumHZR0h1` zbvOyj%UYi000+7pR9!9J==vM3N(CEE2TU#KIifwRXFrbr#>@{7uUslG4A%bVPAO3O zO%u3ffGiC(PyQsw*A+0O4mL3+DgX4K8T0VXYyVivJ)m<4c2z(t>0eck4!k{8UCXk* zIy$sK;E&m9HevI^Cv)0m%Z z{M#{qtEv9l#7|jpU-N2GaI!!T(96*W)mGc+2j~Cw3+Dcr2_xATli58Oa=HEqJ?ZIK z0i<9v4pcXhR1%v$LHf?rAPs^IV+yDZ5yVz}_E7$|Eo%{V^a{$PameYa;)%xa=mPgn zo)wx{i^nZ48%XT`5upfP_Xs#R;Ew!%G5Cs=Vah1lIU8}<{{XUm$7B9W(lfVi3xoG^ zaCdv2&HVoWDkP^Cn$!B(+h0zp+S{&v%5?}tO0R`HU>pyD$El~9q4=c(fW4t;$xKAxac)-V@n0E}RaXFh;u`E?no0s?;U z<+#btI%Dd3dLOMrymU(5T|cY&b$TR@%5x$_0B`^UoPpE09G~;p_NhXiPhxiCAMJnj z`eX1lBB#F;n%;v;q|E_i!i9iRq2krA8+V?3XmZK8+UR@=c&)*>DwpV zQ46R=_Fv_u$7CuIq5s>!;%VVeUaf)9WD)6({{TV99Gc2gQCeyJbnDq`eJ`^+Y0Qty8$%!?9Y8th#~)8# zgEcx0u@(vr2Xn?h!}UFB@y59TDB8mq;Pn1~$a79a$}g_&Ip;pziTCI`bga~!Hr#Y= zbnm9STU)1VYodKl$WG%Th$`-Ej1k+M_89f_9lo_7M%r>P8-T0d*dOM9{dFR$?gu-E zMFWibf%@eC0QKol&HmPS`FP#_$n+ln0OVmhu%`nZ9B#%bvfKf&?#OO zax>nEbF_jmFh_m9oOdRrSaOOsjz)3V`=9nlVg5^%9|IU}rFRT-{qybV z{{ZTuT+vpxpPt%V^6#_WGJTZhMuS1daKKK3{tLH_{PsZt@ZC?^CJ$o}X*G4IsV z(mdcPE5Qm_XA9J~h}EZmEU;DS*CWvWJ;I; z%BWHM!-0$s;T(Q{ok+1bQh<@mPm&}01dr^39cIQ}U} zJo^lOhOyH2mG$fEYx(tzt zAi`yd+Yxd2{{Z^w{c-;QUgP>!Gp!gWJ)2KYURL(gNnh6YJ88`D3seDBn1BF?l6zF> zry+Py8Ib3`dH#5zBF&Va_0B)!YRsxo%klyvkz5a7^NRffgLLRc?5%%Q%{esutqz;) ze;>wof7h4!kVK#&l~J95uJ2UiALr7h!YS%Xj(swX`kWqrDv&3Z?VRE{GIBBkk&ZuF ze$ayq5AM##&p#>Xe-G(LGo=eQmA^-?%b7|}{UCNicVtN!Rtf+N2t&t0KI#6SolJ|= zpa+6N=l=R%>(kzyF9d?zgm%3le-E-4aENd3l5F+Kj)=m88|yVJ9&PK z&g$IPZ%Cc6oygKP+CV=j11twP9SHBoL;f`)kCmH+=N)pRKl>^F06w)KRUDIo2t4#g z?mmb7_opBG=Wic3@cSQ8@BT&=Dn^`lwZGu~$!hn#?%U?Ok99fR%7#4c{{XLX{VH^X z28~Lbya7k#5A>pn)`ciIMon(wzWF5o00w+~WbR26Ylw>`1E-hxgMs|NT9PQhRcAl| z9#b52>4W^gS}3Jtl6KlLdoA2!FVFL9wYzD_k4W|cSqU-xjsWVtznAInQlvl-p>kDl za55P^ze*^i=|&naPr6c%{@A{E{%3y5a%_?~`2kg2Y{H-KFQ?c1Dp*lY+$cP385qd^ z>GVJ6>QPFX)(`o=`%nH$t@c;F<$lOXi5y1U*k`Er zJdeVPD|u6lWY=r<=;FVxmqd}*PE2}60+YZ1Z3olWsQnMMOtB6Q-JlE}Gqiepemrs3 ziYuO)QjPm0{{RdB0KoK1vD*nT>X;b{q;2B^p#6B{dXJ?|8Y$?bp#uZAJa_c>{3xQP z66LC*wzdBNuep84Y#XrHV$R+@2lC_p0ImN3j?~KNiYtZ@il(J) zTd~~(jBNq2k6u2WoDR4g^r@0H3={##!kmBlvO0PWv{6^el}=lAiu>*Uf7OxM zPENy;Fa~%W@t$%$I&<_KaZ)NOup|XlBpiTnIO<0q@Dc6rMHS6A$xX`JB>w<6`hG0! zrrpU>G6&sLwRz7UpP=^t0PEDTBAvKUc;h1(9;eZLzsV@A;<;+VD`)yiNAmoAT77Vn zu#v^Pw(Rlp@w@$z=m)R)6D(=-Wak)Dw0-YIUvKmDD6b|hMx1A4?|uIO!_E1>yVh6q zAl=CXV+U|hR1Y(rgR+my{P9w(Yq%=~1dYvsjj8?L^Zh8IwsBEALojHmKYQAsFi{z9R_j!N3|4JG;KOka$7IWL;nDlOYrQD z*fyW_{d6ReK#F+)P&h@v>5u8sr;!)^BMJu4pkQ$8Q5f)4b zPcQKY1NnZnBvF8>&VT?sra0--2l;-qQA)`q?X+U{Te!wwpXS$VcGHs{k?aJr5@Y!s z0o8kdFVo(oNPr start else 0, + 'p50_ms': p(0.5), 'p95_ms': p(0.95), 'p99_ms': p(0.99), + 'error_kinds': sorted({str(r[2]) for r in rows if r[2] != 201})} + + def verify(self, token): + stored, cursor = [], None + while True: + query = '?limit=100' + ('&cursor=' + urllib.parse.quote(cursor, safe='') if cursor else '') + page = call('GET', self.path + query, token=token)[2] + stored += page['items'] + cursor = page['next_cursor'] + if cursor is None: + break + ids = [m['id'] for m in stored] + texts = [m['text'] for m in stored] + return {'confirmed': len(self.confirmed), 'stored': len(stored), + 'lost': len({m['id'] for m in self.confirmed} - set(ids)), + 'duplicate_ids': len(ids) - len(set(ids)), 'duplicate_texts': len(texts) - len(set(texts)), + 'uncertain_at_stop': self.uncertain} diff --git a/student-tests/imagegen.py b/student-tests/imagegen.py new file mode 100644 index 00000000..8e19ffbe --- /dev/null +++ b/student-tests/imagegen.py @@ -0,0 +1,66 @@ +import os +from pathlib import Path +import struct +import zlib + +ROOT = Path(__file__).resolve().parents[1] +SAMPLE_PNG = (ROOT / 'tests' / 'fixtures' / 'sample.png').read_bytes() +SAMPLE_JPG = (Path(__file__).resolve().parent / 'fixtures' / 'sample.jpg').read_bytes() + + +def chunk(kind, data): + return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff) + + +def png(width, height, rows, color_type=0, level=9): + """rows — итератор байтовых строк пикселей (без байта фильтра).""" + compressor = zlib.compressobj(level) + parts = [] + for row in rows: + parts.append(compressor.compress(b'\x00' + row)) + parts.append(compressor.flush()) + header = struct.pack('>IIBBBBB', width, height, 8, color_type, 0, 0, 0) + return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', b''.join(parts)) + chunk(b'IEND', b'') + + +def bomb(width=10000, height=5000): + """50 Мп чёрного: в распакованном виде 50 МБ, в файле — несколько десятков КБ.""" + row = bytes(width) + return png(width, height, (row for _ in range(height))) + + +def noise_png(target_bytes): + """Шумный RGB PNG (почти не сжимается) размером чуть меньше target_bytes.""" + width = 1024 + height = max(1, (target_bytes - 4096) // (width * 3 + 1)) + return png(width, height, (os.urandom(width * 3) for _ in range(height)), color_type=2, level=0) + + +def broken_crc(data): + """Испортить один байт внутри IDAT, не пересчитывая CRC.""" + position = data.index(b'IDAT') + 100 + return data[:position] + bytes([data[position] ^ 0xff]) + data[position + 1:] + + +def with_idat(data, transform): + """Заменить пиксельные данные PNG (все IDAT) на transform(idat), с правильными CRC.""" + position, chunks = 8, [] + while position < len(data): + length, kind = struct.unpack('>I4s', data[position:position + 8]) + chunks.append((kind, data[position + 8:position + 8 + length])) + position += 12 + length + idat = b''.join(body for kind, body in chunks if kind == b'IDAT') + first = next(index for index, (kind, _) in enumerate(chunks) if kind == b'IDAT') + kept = [(kind, body) for kind, body in chunks if kind != b'IDAT'] + kept.insert(first, (b'IDAT', transform(idat))) + return data[:8] + b''.join(chunk(kind, body) for kind, body in kept) + + +def garbage_idat(data): + """Мусор вместо сжатых пикселей, CRC верные.""" + return with_idat(data, lambda idat: idat[:2] + os.urandom(len(idat) - 2)) + + +def short_idat(data): + """Корректный zlib-поток, но пикселей меньше, чем требует IHDR.""" + return with_idat(data, lambda idat: zlib.compress(zlib.decompress(idat)[:-500])) diff --git a/student-tests/rolling_update_check.py b/student-tests/rolling_update_check.py new file mode 100644 index 00000000..eabfedcc --- /dev/null +++ b/student-tests/rolling_update_check.py @@ -0,0 +1,60 @@ +"""Usage (стенд запущен, обе реплики в ротации): + python student-tests/rolling_update_check.py --output evidence/rolling-update.json + +Под непрерывной нагрузкой (4 потока, повтор с тем же Idempotency-Key) выполняет scripts/rolling_update.py +и проверяет: ошибки клиента, RPS, что все реплики заменены, что подтверждённые сообщения не потеряны и не задвоены. +""" +import argparse +import json +from pathlib import Path +import subprocess +import sys +import time + +from helpers import Load, api_containers, new_chat, new_user + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--output') + parser.add_argument('--no-build', action='store_true') + args = parser.parse_args() + before = api_containers() + alice = new_user('rolling') + chat = new_chat(alice) + load = Load(alice['token'], chat['id']).start() + time.sleep(5) + started = time.monotonic() + command = [sys.executable, str(ROOT / 'scripts' / 'rolling_update.py')] + (['--no-build'] if args.no_build else []) + update = subprocess.run(command, capture_output=True, text=True, encoding='utf-8') + finished = time.monotonic() + time.sleep(5) + load.stop() + print(update.stdout.strip()) + after = api_containers() + result = { + 'replicas_before': [c[:12] for c in before], 'replicas_after': [c[:12] for c in after], + 'update_seconds': round(finished - started, 1), 'update_exit_code': update.returncode, + 'before_5s': load.window(started - 5, started), + 'during_update': load.window(started, finished), + 'after_5s': load.window(finished, finished + 5), + 'data': load.verify(alice['token']), + 'log': update.stdout.strip().splitlines(), + } + replaced = not set(before) & set(after) and len(after) == len(before) + ok = update.returncode == 0 and replaced and not result['data']['lost'] and not result['data']['duplicate_texts'] + print('rolling update %.1fs: errors=%d, rps %.1f -> %.1f -> %.1f, lost=%d, duplicates=%d, all replaced=%s' + % (result['update_seconds'], result['during_update']['errors'], result['before_5s']['rps'], + result['during_update']['rps'], result['after_5s']['rps'], result['data']['lost'], + result['data']['duplicate_texts'], replaced)) + if args.output: + with open(args.output, 'w', encoding='utf-8') as file: + json.dump(result, file, ensure_ascii=False, indent=2) + print('PASS' if ok else 'FAIL: ' + update.stderr.strip()[-500:]) + raise SystemExit(0 if ok else 1) + + +if __name__ == '__main__': + main() diff --git a/student-tests/sigterm_check.py b/student-tests/sigterm_check.py new file mode 100644 index 00000000..5d09b0ee --- /dev/null +++ b/student-tests/sigterm_check.py @@ -0,0 +1,79 @@ +"""Usage (стенд запущен через docker compose up -d): + python student-tests/sigterm_check.py + docker compose start api + +Останавливает одну API-реплику (docker stop = SIGTERM) и всё это время опрашивает API через LB: +реплика сначала выходит из ротации (readiness 503), потом завершается; клиенты не должны получить ошибок. +""" +import argparse +import subprocess +import threading +import time +import urllib.error +import urllib.request +import uuid + +from helpers import BASE_URL, api_containers, call, new_chat, new_user + + +def docker(*args): + return subprocess.run(['docker', *args], capture_output=True, text=True, encoding='utf-8').stdout.strip() + + +def probe_forever(results, stop_event): + while not stop_event.is_set(): + started = time.monotonic() + try: + with urllib.request.urlopen(BASE_URL + '/health/live', timeout=1) as response: + outcome = str(response.status) + except urllib.error.HTTPError as error: + outcome = str(error.code) + except Exception as error: + outcome = type(error).__name__ + results.append((started, outcome)) + time.sleep(0.02) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--container', default=None) + parser.add_argument('--stop-timeout', type=int, default=10, help='секунд до SIGKILL у docker stop') + args = parser.parse_args() + args.container = args.container or api_containers()[0] + + marker = 'PRIVATE-' + uuid.uuid4().hex + alice = new_user('alice') + chat = new_chat(alice) + status = call('POST', '/api/v1/chats/%s/messages' % chat['id'], {'text': marker}, alice['token'])[0] + logs_before = docker('logs', args.container) + print('сообщение с маркером отправлено: HTTP %d; маркер в логах: %s' % (status, marker in logs_before)) + + results, stop_event = [], threading.Event() + prober = threading.Thread(target=probe_forever, args=(results, stop_event)) + prober.start() + time.sleep(1) + + sigterm_at = time.monotonic() + docker('stop', '-t', str(args.stop_timeout), args.container) + stopped_at = time.monotonic() + time.sleep(1) + stop_event.set() + prober.join() + + before = [r for t, r in results if t < sigterm_at] + after = [r for t, r in results if t >= sigterm_at] + exit_code = docker('inspect', '--format', '{{.State.ExitCode}}', args.container) + print('docker stop занял: %.2f с (лимит %d с)' % (stopped_at - sigterm_at, args.stop_timeout)) + print('код выхода контейнера: %s (0 — сам завершился по SIGTERM; 137 — убит SIGKILL)' % exit_code) + print('запросы до SIGTERM: %d, все 200: %s' % (len(before), set(before) == {'200'})) + print('запросы после SIGTERM: %d, результаты: %s' % (len(after), sorted(set(after)))) + print('ответов 5xx за всё время: %d' % sum(r.startswith('5') for _, r in results)) + + print('\nлоги остановки:') + for line in docker('logs', args.container).splitlines(): + if any(word in line for word in ('draining', 'Shutting down', 'stopping', 'shutdown', 'Finished server')): + print(' ' + line) + + +if __name__ == '__main__': + main() diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..be589b60 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,157 @@ +import unittest +import uuid + +from helpers import call, login, new_chat, new_user, read_all_messages + + +class ErrorFormatMixin: + def assertError(self, result, status, code): + actual, headers, body = result + self.assertEqual(actual, status) + self.assertEqual(body['error']['code'], code) + self.assertEqual(body['error']['request_id'], headers['X-Request-Id']) + self.assertNotIn('Traceback', body['error']['message']) + + +class InvalidJsonTest(ErrorFormatMixin, unittest.TestCase): + def test_malformed_json(self): + """Битый JSON -> 400 invalid_request""" + self.assertError(call('POST', '/api/v1/users', raw=b'{"username": "abc",'), 400, 'invalid_request') + + def test_json_not_object(self): + """JSON-массив вместо объекта -> 400""" + self.assertError(call('POST', '/api/v1/users', raw=b'[1, 2, 3]'), 400, 'invalid_request') + + def test_wrong_type(self): + """Число вместо строки -> 400 (без тихого приведения типов)""" + self.assertError(call('POST', '/api/v1/users', {'username': 12345, 'display_name': 'x', 'password': 'Long-enough-pass-1'}), 400, 'invalid_request') + + def test_extra_field(self): + """Лишнее поле в теле -> 400""" + body = {'username': 'u_' + uuid.uuid4().hex[:8], 'display_name': 'x', 'password': 'Long-enough-pass-1', 'is_admin': True} + self.assertError(call('POST', '/api/v1/users', body), 400, 'invalid_request') + + +class DuplicateUsernameTest(ErrorFormatMixin, unittest.TestCase): + def test_duplicate_username(self): + """Повторная регистрация того же username -> 409 conflict""" + user = new_user('dup') + again = call('POST', '/api/v1/users', {'username': user['username'], 'display_name': 'Другой', + 'password': 'Other-password-123'}) + self.assertError(again, 409, 'conflict') + + +class ForeignChatTest(ErrorFormatMixin, unittest.TestCase): + def setUp(self): + self.alice, self.bob, self.eve = new_user('alice'), new_user('bob'), new_user('eve') + self.chat = new_chat(self.alice, self.bob) + self.path = '/api/v1/chats/' + self.chat['id'] + + def test_outsider_cannot_read_chat(self): + """Посторонний читает чат -> 404, как будто чата нет""" + self.assertError(call('GET', self.path, token=self.eve['token']), 404, 'not_found') + missing = call('GET', '/api/v1/chats/' + str(uuid.uuid4()), token=self.eve['token']) + self.assertEqual(missing[2]['error']['message'], call('GET', self.path, token=self.eve['token'])[2]['error']['message']) + + def test_outsider_cannot_write_or_read_history(self): + """Посторонний пишет/читает историю -> 404""" + self.assertError(call('POST', self.path + '/messages', {'text': 'чужое'}, self.eve['token']), 404, 'not_found') + self.assertError(call('GET', self.path + '/messages', token=self.eve['token']), 404, 'not_found') + + def test_outsider_does_not_see_chat_in_list(self): + """Чужой чат не попадает в список чатов""" + status, _, page = call('GET', '/api/v1/chats?limit=100', token=self.eve['token']) + self.assertEqual(status, 200) + self.assertNotIn(self.chat['id'], [c['id'] for c in page['items']]) + + def test_member_has_access(self): + """Участник видит чат (контроль: 404 выше не из-за ошибки в тесте)""" + self.assertEqual(call('GET', self.path, token=self.bob['token'])[0], 200) + + +class EmptyMessageTest(ErrorFormatMixin, unittest.TestCase): + def setUp(self): + self.alice = new_user('alice') + self.path = '/api/v1/chats/%s/messages' % new_chat(self.alice)['id'] + + def test_empty_text(self): + """Пустой текст -> 400""" + self.assertError(call('POST', self.path, {'text': ''}, self.alice['token']), 400, 'invalid_request') + + def test_whitespace_text(self): + """Текст только из пробельных символов -> 400""" + self.assertError(call('POST', self.path, {'text': ' \t\n '}, self.alice['token']), 400, 'invalid_request') + + def test_missing_text(self): + """Тело без поля text -> 400""" + self.assertError(call('POST', self.path, {}, self.alice['token']), 400, 'invalid_request') + + def test_text_length_limit(self): + """4000 символов можно (эмодзи = 1 символ), 4001 — 400""" + self.assertEqual(call('POST', self.path, {'text': '😀' * 4000}, self.alice['token'])[0], 201) + self.assertError(call('POST', self.path, {'text': 'x' * 4001}, self.alice['token']), 400, 'invalid_request') + + +class PaginationTest(ErrorFormatMixin, unittest.TestCase): + def setUp(self): + self.alice = new_user('alice') + self.chat = new_chat(self.alice) + self.sent = [call('POST', '/api/v1/chats/%s/messages' % self.chat['id'], + {'text': 'сообщение %d' % i}, self.alice['token'])[2] for i in range(7)] + + def test_walk_several_pages(self): + """7 сообщений по 3: три страницы, без потерь, повторов и перестановок""" + items, pages = read_all_messages(self.chat['id'], self.alice['token'], limit=3) + self.assertEqual(pages, 3) + self.assertEqual([m['id'] for m in items], [m['id'] for m in self.sent]) + keys = [(m['created_at'], m['id']) for m in items] + self.assertEqual(keys, sorted(keys)) + + def test_cursor_from_other_chat(self): + """cursor от другого чата -> 400""" + _, _, page = call('GET', '/api/v1/chats/%s/messages?limit=1' % self.chat['id'], token=self.alice['token']) + other = new_chat(self.alice, title='Другой') + self.assertError(call('GET', '/api/v1/chats/%s/messages?cursor=%s' % (other['id'], page['next_cursor']), + token=self.alice['token']), 400, 'invalid_request') + + def test_invalid_cursor_and_limit(self): + """Мусорный cursor и limit вне 1..100 -> 400""" + path = '/api/v1/chats/%s/messages' % self.chat['id'] + for query in ('?cursor=not-a-cursor', '?limit=0', '?limit=101', '?limit=abc'): + with self.subTest(query=query): + self.assertError(call('GET', path + query, token=self.alice['token']), 400, 'invalid_request') + + +class IdentityTest(ErrorFormatMixin, unittest.TestCase): + def test_sender_cannot_be_forged(self): + """sender_id в теле запрещён: отправитель только из сессии""" + alice, bob = new_user('alice'), new_user('bob') + chat = new_chat(alice, bob) + forged = call('POST', '/api/v1/chats/%s/messages' % chat['id'], {'text': 'hi', 'sender_id': bob['id']}, alice['token']) + self.assertError(forged, 400, 'invalid_request') + + def test_nul_character_is_preserved(self): + """U+0000 — допустимый code point: сообщение, display_name и title сохраняются и читаются без изменений""" + username, password = 'nul_' + uuid.uuid4().hex[:12], 'Nul-password-123' + status, _, user = call('POST', '/api/v1/users', {'username': username, 'display_name': 'a\u0000lice', 'password': password}) + self.assertEqual((status, user['display_name']), (201, 'a\u0000lice')) + alice = dict(user, token=login(username, password)['token']) + self.assertEqual(call('GET', '/api/v1/users/me', token=alice['token'])[2]['display_name'], 'a\u0000lice') + status, _, chat = call('POST', '/api/v1/chats', {'title': 't\u0000', 'member_ids': [alice['id']]}, alice['token']) + self.assertEqual((status, chat['title']), (201, 't\u0000')) + path = '/api/v1/chats/%s/messages' % chat['id'] + status, _, message = call('POST', path, {'text': 'a\u0000b'}, alice['token']) + self.assertEqual((status, message['text']), (201, 'a\u0000b')) + self.assertEqual(call('GET', path, token=alice['token'])[2]['items'][0]['text'], 'a\u0000b') + self.assertEqual(call('GET', '/api/v1/chats/' + chat['id'], token=alice['token'])[2]['title'], 't\u0000') + + def test_lone_surrogate_is_rejected(self): + """Одиночный суррогат не кодируется в UTF-8 -> 400, а не 500""" + alice = new_user('alice') + chat = new_chat(alice) + self.assertError(call('POST', '/api/v1/chats/%s/messages' % chat['id'], raw=b'{"text": "a\\ud800b"}', + token=alice['token']), 400, 'invalid_request') + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..7b2397bc --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,192 @@ +import hashlib +import http.client +import unittest +import urllib.parse +import uuid + +from helpers import BASE_URL, call, new_chat, new_user +from imagegen import SAMPLE_JPG, SAMPLE_PNG, bomb, broken_crc, garbage_idat, noise_png, short_idat + +MAX = 10 * 1024 * 1024 + + +def upload(user, chat, data, content_type='image/png', filename='photo.png', token=True): + query = '' if filename is None else '?filename=' + urllib.parse.quote(filename, safe='') + return call('POST', '/api/v1/chats/%s/attachments%s' % (chat['id'], query), raw=data, + token=user['token'] if token else None, headers={'Content-Type': content_type}) + + +def download(user, attachment_id): + target = urllib.parse.urlsplit(BASE_URL) + connection = http.client.HTTPConnection(target.hostname, target.port, timeout=30) + connection.request('GET', '/api/v1/attachments/%s/content' % attachment_id, + headers={'Authorization': 'Bearer ' + user['token']}) + response = connection.getresponse() + body = response.read() + connection.close() + return response.status, response.headers, body + + +def send(user, chat, body, key=None): + return call('POST', '/api/v1/chats/%s/messages' % chat['id'], body, user['token'], + headers={'Idempotency-Key': key or uuid.uuid4().hex}) + + +class UploadTest(unittest.TestCase): + def setUp(self): + self.alice, self.bob, self.eve = new_user('alice'), new_user('bob'), new_user('eve') + self.chat = new_chat(self.alice, self.bob) + + def assertCode(self, result, status, code=None): + self.assertEqual(result[0], status, result[2]) + if code: + self.assertEqual(result[2]['error']['code'], code) + + def test_png_and_jpeg_are_stored_and_downloaded_byte_exact(self): + """PNG и JPEG → 201 ready; второй участник скачивает те же байты с верным Content-Type""" + for data, content_type in ((SAMPLE_PNG, 'image/png'), (SAMPLE_JPG, 'image/jpeg')): + with self.subTest(content_type=content_type): + status, _, att = upload(self.alice, self.chat, data, content_type) + self.assertEqual(status, 201) + self.assertEqual((att['status'], att['variants'], att['error_code']), ('ready', [], None)) + self.assertEqual((att['content_type'], att['size_bytes'], att['sha256']), + (content_type, len(data), hashlib.sha256(data).hexdigest())) + self.assertEqual((att['chat_id'], att['owner_id']), (self.chat['id'], self.alice['id'])) + self.assertEqual(call('GET', '/api/v1/attachments/' + att['id'], token=self.bob['token'])[2], att) + status, headers, body = download(self.bob, att['id']) + self.assertEqual((status, headers.get_content_type(), body), (200, content_type, data)) + self.assertEqual(int(headers['Content-Length']), len(data)) + + def test_download_through_every_replica(self): + """Файл скачивается через любую API-реплику (байты не лежат локально у загрузившей)""" + att = upload(self.alice, self.chat, SAMPLE_PNG)[2] + replicas = set() + for _ in range(6): + status, headers, body = download(self.bob, att['id']) + self.assertEqual((status, body), (200, SAMPLE_PNG)) + replicas.add(headers['X-Instance-Id']) + self.assertGreaterEqual(len(replicas), 2) + + def test_filename_is_display_only_and_validated(self): + """Unicode-имя сохраняется как есть; нет имени, пустое, /, \\, NUL, управляющий символ, >255 → 400""" + status, _, att = upload(self.alice, self.chat, SAMPLE_PNG, filename='фото отпуска 😀.png') + self.assertEqual((status, att['filename']), (201, 'фото отпуска 😀.png')) + for name in (None, '', ' ', '../etc/passwd', 'a\\b.png', 'a\x00b.png', 'bell\x07.png', 'x' * 256): + with self.subTest(name=name): + self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG, filename=name), 400, 'invalid_request') + + def test_wrong_declared_type(self): + """Content-Type не image/png|jpeg (octet-stream, multipart, gif) → 415""" + for content_type in ('application/octet-stream', 'multipart/form-data; boundary=x', 'image/gif', 'text/plain'): + with self.subTest(content_type=content_type): + self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG, content_type), 415, 'unsupported_media_type') + + def test_mime_spoofing(self): + """Подмена MIME: PNG как jpeg, JPEG как png, текст и GIF под видом png → 415""" + for data, content_type in ((SAMPLE_PNG, 'image/jpeg'), (SAMPLE_JPG, 'image/png'), + (b'hello, not an image', 'image/png'), (b'GIF89a' + bytes(100), 'image/png'), + (b'', 'image/png')): + with self.subTest(content_type=content_type, head=data[:6]): + self.assertCode(upload(self.alice, self.chat, data, content_type), 415, 'unsupported_media_type') + + def test_corrupted_images(self): + """Обрезанный PNG, битый CRC, мусор или нехватка пикселей в IDAT при верных CRC, обрезанный JPEG → 415""" + for data, content_type in ((SAMPLE_PNG[:5000], 'image/png'), (broken_crc(SAMPLE_PNG), 'image/png'), + (garbage_idat(SAMPLE_PNG), 'image/png'), (short_idat(SAMPLE_PNG), 'image/png'), + (SAMPLE_JPG[:3000], 'image/jpeg')): + with self.subTest(size=len(data), content_type=content_type): + self.assertCode(upload(self.alice, self.chat, data, content_type), 415, 'unsupported_media_type') + + def test_too_large_file(self): + """10 MiB + 1 байт → 413 (и по Content-Length, и chunked без длины); файл ~10 MiB принимается""" + self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG + bytes(MAX + 1 - len(SAMPLE_PNG))), 413, 'payload_too_large') + target = urllib.parse.urlsplit(BASE_URL) + connection = http.client.HTTPConnection(target.hostname, target.port, timeout=60) + chunks = (SAMPLE_PNG if i == 0 else bytes(1024 * 1024) for i in range(12)) + connection.request('POST', '/api/v1/chats/%s/attachments?filename=big.png' % self.chat['id'], body=chunks, + headers={'Authorization': 'Bearer ' + self.alice['token'], 'Content-Type': 'image/png'}, + encode_chunked=True) + self.assertEqual(connection.getresponse().status, 413) + connection.close() + big = noise_png(MAX - 50000) + self.assertLessEqual(len(big), MAX) + status, _, att = upload(self.alice, self.chat, big) + self.assertEqual((status, att['size_bytes']), (201, len(big))) + self.assertEqual(download(self.bob, att['id'])[2], big) + + def test_decompression_bomb(self): + """PNG 10000×5000 (50 Мп) весом в десятки КБ → 413 без распаковки""" + data = bomb() + self.assertLess(len(data), 1024 * 1024) + self.assertCode(upload(self.alice, self.chat, data), 413, 'payload_too_large') + + def test_access_rules(self): + """Без токена 401; посторонний не может загрузить в чужой чат и не видит метаданные/байты (404)""" + self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG, token=False), 401, 'unauthorized') + self.assertCode(upload(self.eve, self.chat, SAMPLE_PNG), 404, 'not_found') + self.assertCode(upload(self.alice, {'id': str(uuid.uuid4())}, SAMPLE_PNG), 404, 'not_found') + att = upload(self.alice, self.chat, SAMPLE_PNG)[2] + self.assertCode(call('GET', '/api/v1/attachments/' + att['id'], token=self.eve['token']), 404, 'not_found') + self.assertEqual(download(self.eve, att['id'])[0], 404) + self.assertCode(call('GET', '/api/v1/attachments/' + str(uuid.uuid4()), token=self.bob['token']), 404) + self.assertCode(call('GET', '/api/v1/attachments/not-a-uuid', token=self.bob['token']), 400) + self.assertCode(call('GET', '/api/v1/attachments/%s/content?variant=thumbnail' % att['id'], + token=self.bob['token']), 400) + + +class MessageAttachmentsTest(unittest.TestCase): + def setUp(self): + self.alice, self.bob = new_user('alice'), new_user('bob') + self.chat = new_chat(self.alice, self.bob) + self.other_chat = new_chat(self.alice, title='Другой') + self.att = upload(self.alice, self.chat, SAMPLE_PNG)[2] + + def history(self): + return call('GET', '/api/v1/chats/%s/messages?limit=100' % self.chat['id'], token=self.bob['token'])[2]['items'] + + def test_message_with_attachment_only(self): + """text="" + одно вложение → 201; attachment_ids видны в истории; обычное сообщение получает []""" + status, _, message = send(self.alice, self.chat, {'text': '', 'attachment_ids': [self.att['id']]}) + self.assertEqual((status, message['attachment_ids'], message['text']), (201, [self.att['id']], '')) + plain = send(self.alice, self.chat, {'text': 'без вложений'})[2] + self.assertEqual(plain['attachment_ids'], []) + self.assertEqual([m['attachment_ids'] for m in self.history()], [[self.att['id']], []]) + + def test_other_member_can_reference_attachment(self): + """Боб прикрепляет вложение, загруженное Алисой в этот же чат""" + status, _, message = send(self.bob, self.chat, {'text': 'смотри', 'attachment_ids': [self.att['id']]}) + self.assertEqual((status, message['attachment_ids']), (201, [self.att['id']])) + + def test_foreign_or_unknown_attachment(self): + """Вложение другого чата или несуществующее → 404, сообщение не создаётся""" + foreign = upload(self.alice, self.other_chat, SAMPLE_PNG)[2] + for ids in ([foreign['id']], [str(uuid.uuid4())], [self.att['id'], foreign['id']]): + with self.subTest(ids=ids): + self.assertEqual(send(self.alice, self.chat, {'text': 'x', 'attachment_ids': ids})[0], 404) + self.assertEqual(self.history(), []) + + def test_invalid_attachment_lists(self): + """Повтор id, 11 вложений, не-UUID, пустой текст без вложений, пробелы без вложений → 400""" + for body in ({'text': 'x', 'attachment_ids': [self.att['id'], self.att['id']]}, + {'text': 'x', 'attachment_ids': [str(uuid.uuid4()) for _ in range(11)]}, + {'text': 'x', 'attachment_ids': ['nope']}, + {'text': ''}, {'text': ' '}, {'text': '', 'attachment_ids': []}): + with self.subTest(body=body): + self.assertEqual(send(self.alice, self.chat, body)[0], 400) + + def test_idempotency_includes_attachments(self): + """Тот же ключ: те же вложения → тот же Message; другие вложения → 409; нет поля = [] для текста""" + key = uuid.uuid4().hex + first = send(self.alice, self.chat, {'text': 'фото', 'attachment_ids': [self.att['id']]}, key) + again = send(self.alice, self.chat, {'attachment_ids': [self.att['id']], 'text': 'фото'}, key) + self.assertEqual((first[0], again[0], first[2]), (201, 201, again[2])) + other = upload(self.alice, self.chat, SAMPLE_JPG, 'image/jpeg', 'b.jpg')[2] + self.assertEqual(send(self.alice, self.chat, {'text': 'фото', 'attachment_ids': [other['id']]}, key)[0], 409) + key = uuid.uuid4().hex + plain = send(self.alice, self.chat, {'text': 'текст'}, key) + same = send(self.alice, self.chat, {'text': 'текст', 'attachment_ids': []}, key) + self.assertEqual(plain[2]['id'], same[2]['id']) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_auth.py b/student-tests/test_auth.py new file mode 100644 index 00000000..3a56ee20 --- /dev/null +++ b/student-tests/test_auth.py @@ -0,0 +1,132 @@ +from datetime import datetime, timezone +import unittest +import uuid + +from helpers import basic, call, login, new_user, register + + +class RegistrationTest(unittest.TestCase): + def body(self, password): + return {'username': 'reg_' + uuid.uuid4().hex[:12], 'display_name': 'Регистрация', 'password': password} + + def test_password_is_not_returned(self): + """Ответ регистрации и /users/me не содержат password/hash""" + status, _, user = call('POST', '/api/v1/users', self.body('Secret-password-1')) + self.assertEqual(status, 201) + self.assertEqual(set(user), {'id', 'username', 'display_name', 'created_at'}) + + def test_password_length_limits(self): + """Пароль 12..128 символов; 11 и 129 -> 400; эмодзи считается одним символом""" + self.assertEqual(call('POST', '/api/v1/users', self.body('x' * 11))[0], 400) + self.assertEqual(call('POST', '/api/v1/users', self.body('x' * 129))[0], 400) + self.assertEqual(call('POST', '/api/v1/users', self.body('x' * 128))[0], 201) + self.assertEqual(call('POST', '/api/v1/users', self.body('🔑' * 12))[0], 201) + + def test_password_required_and_not_blank(self): + """Без password или из одних пробелов -> 400""" + body = self.body('x') + del body['password'] + self.assertEqual(call('POST', '/api/v1/users', body)[0], 400) + self.assertEqual(call('POST', '/api/v1/users', self.body(' ' * 20))[0], 400) + + +class LoginTest(unittest.TestCase): + def setUp(self): + self.user, self.password = register('login') + + def assertBasicChallenge(self, result): + status, headers, body = result + self.assertEqual(status, 401) + self.assertEqual(headers['WWW-Authenticate'], 'Basic realm="messenger", charset="UTF-8"') + self.assertEqual(body['error']['code'], 'unauthorized') + return body['error']['message'] + + def test_login_returns_opaque_session(self): + """Вход: 201, token_type=Bearer, токен случайный и не содержит username/UUID""" + first = login(self.user['username'], self.password) + second = login(self.user['username'], self.password) + self.assertEqual(first['token_type'], 'Bearer') + self.assertEqual(first['user'], self.user) + self.assertGreaterEqual(len(first['token']), 43) + self.assertNotEqual(first['token'], second['token']) + self.assertNotIn(self.user['username'], first['token']) + self.assertNotIn(self.user['id'], first['token']) + expires = datetime.fromisoformat(first['expires_at'].replace('Z', '+00:00')) + self.assertGreater(expires, datetime.now(timezone.utc)) + + def test_wrong_password_and_unknown_user_look_the_same(self): + """Неверный пароль и несуществующий пользователь -> одинаковый 401 с Basic challenge""" + wrong = self.assertBasicChallenge(call('POST', '/api/v1/auth/sessions', + headers=basic(self.user['username'], 'Wrong-password-1'))) + unknown = self.assertBasicChallenge(call('POST', '/api/v1/auth/sessions', + headers=basic('nobody_' + uuid.uuid4().hex[:8], 'Wrong-password-1'))) + self.assertEqual(wrong, unknown) + + def test_malformed_basic(self): + """Нет заголовка, не base64, нет двоеточия, Bearer вместо Basic -> 401 Basic""" + for header in (None, 'Basic', 'Basic !!!not-base64!!!', 'Basic ' + 'bm9jb2xvbg==', 'Bearer abc'): + with self.subTest(header=header): + headers = {'Authorization': header} if header else {} + self.assertBasicChallenge(call('POST', '/api/v1/auth/sessions', headers=headers)) + + def test_password_with_colon_and_unicode(self): + """Двоеточие и не-ASCII в пароле допустимы: делим только по первому ':'""" + user, password = register('colon', password='пароль:с:двоеточием🔑') + self.assertEqual(login(user['username'], password)['user']['id'], user['id']) + + def test_login_rejects_body(self): + """POST /auth/sessions принимает только заголовок, JSON body -> 400""" + status = call('POST', '/api/v1/auth/sessions', {'username': self.user['username']}, + headers=basic(self.user['username'], self.password))[0] + self.assertEqual(status, 400) + + +class BearerTest(unittest.TestCase): + def setUp(self): + self.alice = new_user('alice') + self.eve = new_user('eve') + + def assertBearerChallenge(self, headers): + status, response_headers, _ = call('GET', '/api/v1/users/me', headers=headers) + self.assertEqual(status, 401) + self.assertEqual(response_headers['WWW-Authenticate'], 'Bearer realm="messenger"') + + def test_only_bearer_is_accepted(self): + """Нет заголовка, Basic, X-User-Id, мусорный токен -> 401 Bearer""" + self.assertBearerChallenge({}) + self.assertBearerChallenge(basic(self.alice['username'], self.alice['password'])) + self.assertBearerChallenge({'X-User-Id': self.alice['id']}) + self.assertBearerChallenge({'Authorization': 'Bearer ' + uuid.uuid4().hex}) + self.assertBearerChallenge({'Authorization': 'Bearer'}) + + def test_authentication_before_validation(self): + """Аноним с битым JSON на защищённом endpoint получает 401, а не 400; с сессией — 400""" + self.assertEqual(call('POST', '/api/v1/chats', raw=b'{bad')[0], 401) + self.assertEqual(call('POST', '/api/v1/chats', raw=b'{bad', token=self.alice['token'])[0], 400) + self.assertEqual(call('POST', '/api/v1/users', raw=b'{bad')[0], 400) + + def test_scheme_is_case_insensitive(self): + """Схема аутентификации нечувствительна к регистру (RFC 9110)""" + status, _, me = call('GET', '/api/v1/users/me', headers={'Authorization': 'bearer ' + self.alice['token']}) + self.assertEqual((status, me['id']), (200, self.alice['id'])) + + def test_x_user_id_does_not_override_session(self): + """X-User-Id игнорируется: владелец определяется только токеном""" + status, _, me = call('GET', '/api/v1/users/me', token=self.alice['token'], headers={'X-User-Id': self.eve['id']}) + self.assertEqual((status, me['id']), (200, self.alice['id'])) + + +class LogoutTest(unittest.TestCase): + def test_logout_revokes_only_current_session(self): + """Logout -> 204 без тела; этот токен -> 401, вторая сессия того же пользователя работает""" + alice = new_user('alice') + other = login(alice['username'], alice['password'])['token'] + status, _, body = call('DELETE', '/api/v1/auth/sessions/current', token=alice['token']) + self.assertEqual((status, body), (204, None)) + self.assertEqual(call('GET', '/api/v1/users/me', token=alice['token'])[0], 401) + self.assertEqual(call('DELETE', '/api/v1/auth/sessions/current', token=alice['token'])[0], 401) + self.assertEqual(call('GET', '/api/v1/users/me', token=other)[0], 200) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_concurrency.py b/student-tests/test_concurrency.py new file mode 100644 index 00000000..4d258446 --- /dev/null +++ b/student-tests/test_concurrency.py @@ -0,0 +1,68 @@ +from concurrent.futures import ThreadPoolExecutor +import threading +import unittest +import uuid + +from helpers import basic, call, new_chat, new_user, read_all_messages + +CLIENTS = 10 +REGISTRATIONS = 20 +PER_CLIENT = 10 + + +class ConcurrencyTest(unittest.TestCase): + def test_100_messages_from_10_clients(self): + """100 параллельных сообщений от 10 клиентов -> ровно 100 уникальных в истории""" + users = [new_user('client%d' % i) for i in range(CLIENTS)] + chat = new_chat(users[0], *users[1:], title='Нагрузка') + path = '/api/v1/chats/%s/messages' % chat['id'] + barrier = threading.Barrier(CLIENTS) + + def client(index): + barrier.wait() + results = [] + for n in range(PER_CLIENT): + status, _, message = call('POST', path, {'text': 'client %d msg %d' % (index, n)}, users[index]['token']) + results.append((status, message)) + return results + + with ThreadPoolExecutor(max_workers=CLIENTS) as pool: + responses = [r for batch in pool.map(client, range(CLIENTS)) for r in batch] + + self.assertEqual([s for s, _ in responses], [201] * CLIENTS * PER_CLIENT) + sent_ids = {m['id'] for _, m in responses} + items, pages = read_all_messages(chat['id'], users[0]['token'], limit=30) + read_ids = [m['id'] for m in items] + keys = [(m['created_at'], m['id']) for m in items] + + self.assertEqual(len(sent_ids), 100, 'сервер выдал повторяющиеся id') + self.assertEqual(len(read_ids), 100, 'в истории не ровно 100 сообщений') + self.assertEqual(set(read_ids), sent_ids, 'история не совпадает с подтверждёнными сообщениями') + self.assertEqual(len({m['text'] for m in items}), 100, 'потерялся или задвоился текст') + self.assertEqual(keys, sorted(keys), 'история не упорядочена по (created_at, id)') + print('\n отправлено 201: %d, уникальных id: %d, прочитано: %d за %d страниц(ы)' + % (len(responses), len(sent_ids), len(read_ids), pages)) + + def test_parallel_registration_of_same_username(self): + """20 одновременных регистраций одного username -> ровно один 201, остальные 409; вход только с паролем победителя""" + username = 'race_' + uuid.uuid4().hex[:12] + barrier = threading.Barrier(REGISTRATIONS) + + def register(index): + password = 'Race-password-%02d' % index + barrier.wait() + status = call('POST', '/api/v1/users', {'username': username, 'display_name': 'Гонка', 'password': password})[0] + return status, password + + with ThreadPoolExecutor(max_workers=REGISTRATIONS) as pool: + results = list(pool.map(register, range(REGISTRATIONS))) + statuses = sorted(status for status, _ in results) + self.assertEqual(statuses, [201] + [409] * (REGISTRATIONS - 1)) + winner = [password for status, password in results if status == 201][0] + loser = [password for status, password in results if status == 409][0] + self.assertEqual(call('POST', '/api/v1/auth/sessions', headers=basic(username, winner))[0], 201) + self.assertEqual(call('POST', '/api/v1/auth/sessions', headers=basic(username, loser))[0], 401) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_db.py b/student-tests/test_db.py new file mode 100644 index 00000000..5997fd83 --- /dev/null +++ b/student-tests/test_db.py @@ -0,0 +1,58 @@ +import hashlib +import time +import unittest +import uuid + +from helpers import call, compose, new_chat, new_user, sql + + +class DatabaseTest(unittest.TestCase): + def test_chat_creation_is_rolled_back(self): + """Неизвестный участник: INSERT chats выполнен, INSERT chat_members упал по FK, всё откатано""" + alice = new_user('alice') + chats_before = int(sql('SELECT count(*) FROM chats')) + members_before = int(sql('SELECT count(*) FROM chat_members')) + ghost = str(uuid.uuid4()) + since = sql("SELECT to_char(now() - interval '1 second', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')") + status, _, body = call('POST', '/api/v1/chats', {'title': 'Откат', 'member_ids': [alice['id'], ghost]}, + alice['token']) + self.assertEqual(status, 400, body) + time.sleep(0.5) + log = compose('logs', '--no-log-prefix', '--since', since, 'postgres') + self.assertIn('violates foreign key constraint', log) + self.assertIn(ghost, log) + self.assertEqual(int(sql('SELECT count(*) FROM chats')), chats_before) + self.assertEqual(int(sql('SELECT count(*) FROM chat_members')), members_before) + + def test_secrets_are_not_stored_in_plaintext(self): + """В users лежит argon2id-хеш, в sessions — sha256 токена, не сам токен""" + alice = new_user('alice') + password_hash = sql("SELECT password_hash FROM users WHERE id = '%s'" % alice['id']) + self.assertTrue(password_hash.startswith('$argon2id$'), password_hash[:10]) + self.assertNotIn(alice['password'], password_hash) + digest = hashlib.sha256(alice['token'].encode()).hexdigest() + self.assertEqual(sql("SELECT count(*) FROM sessions WHERE token_hash = '\\x%s'" % digest), '1') + dump = sql("SELECT string_agg(encode(token_hash, 'escape'), '') FROM sessions WHERE user_id = '%s'" % alice['id']) + self.assertNotIn(alice['token'], dump) + + def test_migrations_are_idempotent(self): + """Повторный запуск миграций ничего не меняет и завершается успешно""" + before = sql('SELECT version, applied_at FROM schema_migrations ORDER BY version') + users = sql('SELECT count(*) FROM users') + output = compose('run', '--rm', '--no-deps', 'migrate') + self.assertIn('skip 0001_init: already applied', output) + self.assertEqual(sql('SELECT version, applied_at FROM schema_migrations ORDER BY version'), before) + self.assertEqual(sql('SELECT count(*) FROM users'), users) + + def test_history_query_uses_index(self): + """Запрос страницы истории идёт по индексу (chat_id, created_at, id), без Sort""" + alice = new_user('alice') + chat = new_chat(alice) + plan = sql("SET enable_seqscan = off; SET enable_bitmapscan = off; EXPLAIN SELECT id FROM messages WHERE chat_id = '%s' " + "ORDER BY created_at, id LIMIT 51" % chat['id']) + self.assertIn('messages_chat_history_idx', plan) + self.assertNotIn('Sort', plan) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..4ed9185b --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,96 @@ +from concurrent.futures import ThreadPoolExecutor +import threading +import unittest +import uuid + +from helpers import call, new_chat, new_user, read_all_messages + +PARALLEL = 20 + + +def post(chat, user, key, body=None, raw=None): + return call('POST', '/api/v1/chats/%s/messages' % chat['id'], body, user['token'], raw=raw, + headers={'Idempotency-Key': key}) + + +class IdempotencyTest(unittest.TestCase): + def setUp(self): + self.alice, self.bob = new_user('alice'), new_user('bob') + self.chat = new_chat(self.alice, self.bob) + + def history(self): + return read_all_messages(self.chat['id'], self.alice['token'], limit=100)[0] + + def test_replay_returns_same_message(self): + """Повтор того же запроса с тем же ключом -> тот же 201 и тот же Message, дубля нет""" + key = uuid.uuid4().hex + first = post(self.chat, self.alice, key, {'text': 'один раз'}) + second = post(self.chat, self.alice, key, {'text': 'один раз'}) + self.assertEqual((first[0], second[0]), (201, 201)) + self.assertEqual(first[2], second[2]) + self.assertEqual([m['id'] for m in self.history()], [first[2]['id']]) + + def test_json_formatting_does_not_matter(self): + """Порядок/пробелы в JSON семантически равны: повтор с другим форматированием -> тот же Message""" + key = uuid.uuid4().hex + first = post(self.chat, self.alice, key, raw='{"text":"формат"}'.encode()) + second = post(self.chat, self.alice, key, raw='{ "text" : "формат" }'.encode()) + self.assertEqual((first[0], second[0]), (201, 201)) + self.assertEqual(first[2]['id'], second[2]['id']) + + def test_other_body_with_same_key_conflicts(self): + """Тот же ключ с другим телом -> 409, второе сообщение не создаётся""" + key = uuid.uuid4().hex + self.assertEqual(post(self.chat, self.alice, key, {'text': 'первое'})[0], 201) + status, _, body = post(self.chat, self.alice, key, {'text': 'другое'}) + self.assertEqual((status, body['error']['code']), (409, 'conflict')) + self.assertEqual(len(self.history()), 1) + + def test_missing_or_invalid_key(self): + """Нет ключа, пустой, недопустимые символы, длиннее 128 -> 400""" + for key in (None, '', 'bad key', 'bad/key', 'café', 'x' * 129): + with self.subTest(key=key): + self.assertEqual(post(self.chat, self.alice, key, {'text': 'x'})[0], 400) + self.assertEqual(post(self.chat, self.alice, 'A.b_c:d-1' + 'x' * 119, {'text': 'граница'})[0], 201) + + def test_key_scope_is_user_and_chat(self): + """Одинаковый ключ у разных пользователей и в разных чатах — независимые сообщения""" + key = uuid.uuid4().hex + other_chat = new_chat(self.alice, title='Другой') + a = post(self.chat, self.alice, key, {'text': 'от Алисы'}) + b = post(self.chat, self.bob, key, {'text': 'от Боба'}) + c = post(other_chat, self.alice, key, {'text': 'от Алисы'}) + self.assertEqual([a[0], b[0], c[0]], [201, 201, 201]) + self.assertEqual(len({a[2]['id'], b[2]['id'], c[2]['id']}), 3) + + def test_replay_rechecks_session_and_access(self): + """Повтор снова проверяет сессию и доступ: после logout -> 401, посторонний с тем же ключом -> 404""" + key = uuid.uuid4().hex + self.assertEqual(post(self.chat, self.alice, key, {'text': 'до выхода'})[0], 201) + eve = new_user('eve') + self.assertEqual(post(self.chat, eve, key, {'text': 'до выхода'})[0], 404) + call('DELETE', '/api/v1/auth/sessions/current', token=self.alice['token']) + self.assertEqual(post(self.chat, self.alice, key, {'text': 'до выхода'})[0], 401) + + def test_parallel_retries_through_replicas_create_one_message(self): + """20 одновременных POST с одним ключом через LB (разные реплики) -> ровно одно сообщение, все 201 с одним id""" + key = uuid.uuid4().hex + barrier = threading.Barrier(PARALLEL) + + def send(_): + barrier.wait() + status, headers, body = post(self.chat, self.alice, key, {'text': 'конкурентный повтор'}) + return status, headers['X-Instance-Id'], body + + with ThreadPoolExecutor(max_workers=PARALLEL) as pool: + results = list(pool.map(send, range(PARALLEL))) + self.assertEqual({status for status, _, _ in results}, {201}) + self.assertEqual(len({body['id'] for _, _, body in results}), 1) + self.assertGreaterEqual(len({instance for _, instance, _ in results}), 2, 'запросы не попали на разные реплики') + self.assertEqual(len(self.history()), 1) + print('\n 20 повторов: реплики %s, сообщений в истории: %d' + % (sorted({instance for _, instance, _ in results}), len(self.history()))) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_replicas.py b/student-tests/test_replicas.py new file mode 100644 index 00000000..84149108 --- /dev/null +++ b/student-tests/test_replicas.py @@ -0,0 +1,41 @@ +import unittest +import uuid + +from helpers import call, new_chat, new_user + + +class ReplicasTest(unittest.TestCase): + def test_one_session_is_served_by_several_replicas(self): + """Один токен без повторного входа обслуживается разными репликами (X-Instance-Id), все ответы 200""" + alice = new_user('alice') + results = [call('GET', '/api/v1/users/me', token=alice['token']) for _ in range(10)] + self.assertEqual({status for status, _, _ in results}, {200}) + self.assertGreaterEqual(len({headers['X-Instance-Id'] for _, headers, _ in results}), 2) + + def test_write_on_one_replica_is_read_on_another(self): + """Сообщение, подтверждённое одной репликой, сразу видно при чтении через другую""" + alice = new_user('alice') + chat = new_chat(alice) + path = '/api/v1/chats/%s/messages' % chat['id'] + for i in range(6): + _, written_by, message = call('POST', path, {'text': 'm%d' % i}, alice['token']) + status, read_by, page = call('GET', path + '?limit=100', token=alice['token']) + self.assertEqual(status, 200) + self.assertIn(message['id'], [m['id'] for m in page['items']], + 'запись через %s не видна через %s' % (written_by['X-Instance-Id'], read_by['X-Instance-Id'])) + + def test_every_api_response_has_instance_id(self): + """X-Instance-Id есть и в успешных ответах, и в ошибках API, и в health""" + alice = new_user('alice') + for method, path, token, expected in ( + ('GET', '/health/live', None, 200), ('GET', '/health/ready', None, 200), + ('GET', '/api/v1/users/me', None, 401), ('GET', '/api/v1/users/me', alice['token'], 200), + ('GET', '/api/v1/chats/' + str(uuid.uuid4()), alice['token'], 404), ('GET', '/nope', None, 404)): + with self.subTest(path=path, expected=expected): + status, headers, _ = call(method, path, token=token) + self.assertEqual(status, expected) + self.assertTrue(headers['X-Instance-Id']) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_storage.py b/student-tests/test_storage.py new file mode 100644 index 00000000..c51278c5 --- /dev/null +++ b/student-tests/test_storage.py @@ -0,0 +1,81 @@ +import subprocess +import unittest + +from helpers import compose, new_chat, new_user, sql +from imagegen import SAMPLE_PNG, noise_png +from test_attachments import upload + +LIMITED_CLIENT = ''' +import json +from botocore.exceptions import ClientError +from config import settings +from s3 import client +results = {} +for name, action in ( + ("put_get_delete_own_bucket", lambda: (client.put_object(Bucket=settings.s3_bucket, Key="probe/x", Body=b"x"), + client.get_object(Bucket=settings.s3_bucket, Key="probe/x"), + client.delete_object(Bucket=settings.s3_bucket, Key="probe/x"))), + ("list_bucket_objects", lambda: client.list_objects_v2(Bucket=settings.s3_bucket)), + ("create_bucket", lambda: client.create_bucket(Bucket="intruder")), + ("delete_bucket", lambda: client.delete_bucket(Bucket=settings.s3_bucket)), + ("list_all_buckets", lambda: client.list_buckets()["Buckets"])): + try: + value = action() + results[name] = "allowed" if name != "list_all_buckets" else "allowed:%d" % len(value) + except ClientError as error: + results[name] = error.response["Error"]["Code"] +print(json.dumps(results)) +''' + + +class StorageTest(unittest.TestCase): + def setUp(self): + self.alice = new_user('alice') + self.chat = new_chat(self.alice) + + def object_key(self, attachment_id): + return sql("SELECT object_key FROM attachments WHERE id = '%s'" % attachment_id) + + def test_object_key_is_generated_by_server(self): + """Ключ объекта = attachments/, filename в нём не участвует""" + att = upload(self.alice, self.chat, SAMPLE_PNG, filename='мой-файл.png')[2] + self.assertEqual(self.object_key(att['id']), 'attachments/' + att['id']) + + def test_bucket_is_private_outside_api(self): + """Анонимный GET объекта и листинг bucket напрямую в S3 (из сети backend) → 403""" + att = upload(self.alice, self.chat, SAMPLE_PNG)[2] + for path in ('/messenger-attachments/' + self.object_key(att['id']), '/messenger-attachments'): + with self.subTest(path=path): + result = compose('exec', '-T', 'lb', 'wget', '-S', '-O', '/dev/null', 'http://s3:8333' + path, check=False) + self.assertNotIn('200 OK', result) + output = compose('exec', '-T', 'lb', 'sh', '-c', 'wget -S -O /dev/null http://s3:8333%s 2>&1; true' % path) + self.assertIn('403', output) + + def test_api_credentials_are_limited(self): + """Ключи API: свои объекты читать/писать/удалять можно; листинг, создание и удаление bucket — AccessDenied""" + import json + results = json.loads(compose('exec', '-T', 'api', 'python', '-c', 'import sys; sys.path.insert(0, "src")\n' + LIMITED_CLIENT)) + self.assertEqual(results['put_get_delete_own_bucket'], 'allowed') + for action in ('list_bucket_objects', 'create_bucket', 'delete_bucket'): + self.assertEqual(results[action], 'AccessDenied', results) + self.assertNotEqual(results['list_all_buckets'], 'allowed:1', 'API sees the bucket list') + + def test_bootstrap_is_idempotent(self): + """Повторный s3-init: код 0, bucket уже есть, приватность подтверждена""" + output = compose('run', '--rm', '--no-deps', 's3-init') + self.assertIn('already exists, nothing to do', output) + self.assertIn('is private', output) + + def test_no_original_bytes_in_database_or_api_container(self): + """В attachments единственный bytea — sha256 (байтов файла в БД нет); после загрузки 2 МБ в /tmp реплик API пусто""" + binary = sql("SELECT string_agg(column_name, ',') FROM information_schema.columns " + "WHERE table_name = 'attachments' AND data_type = 'bytea'") + self.assertEqual(binary, 'sha256') + self.assertEqual(upload(self.alice, self.chat, noise_png(2 * 1024 * 1024))[0], 201) + for container in compose('ps', '-q', 'api').split(): + files = subprocess.run(['docker', 'exec', container, 'sh', '-c', 'find /tmp -type f | wc -l'], + capture_output=True, text=True).stdout.strip() + self.assertEqual(files, '0', container[:12]) + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/upload_memory.py b/student-tests/upload_memory.py new file mode 100644 index 00000000..c5a0e8cb --- /dev/null +++ b/student-tests/upload_memory.py @@ -0,0 +1,118 @@ +"""Usage (стенд запущен; на время эксперимента лучше остановить другие стенды): + python student-tests/upload_memory.py --output evidence/upload-memory.json + +N одновременных загрузок PNG ~10 MiB через LB при двух настройках буфера тела запроса +(UPLOAD_MEMORY_BYTES = 1 MiB в RAM, дальше временный файл — рабочая настройка; и 16 MiB — весь файл в RAM). +Перед каждым прогоном реплики перезапускаются (memory.peak cgroup считается с момента старта контейнера); +во время прогона каждые 0,1 с снимаются memory.current и anon/file из memory.stat каждой реплики. +""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import http.client +import json +import os +import subprocess +import threading +import time +import urllib.parse + +from helpers import BASE_URL, api_containers, compose, new_chat, new_user, wait_in_rotation, wait_ready +from imagegen import noise_png + +MIB = 1024 * 1024 +SAMPLER = ('while true; do echo "current $(cat /sys/fs/cgroup/memory.current)"; ' + 'grep -E "^(anon|file) " /sys/fs/cgroup/memory.stat; sleep 0.1; done') + + +class Sampler: + def __init__(self, container): + self.container, self.peaks = container, {'current': 0, 'anon': 0, 'file': 0} + self.process = subprocess.Popen(['docker', 'exec', container, 'sh', '-c', SAMPLER], + stdout=subprocess.PIPE, text=True) + self.thread = threading.Thread(target=self.read, daemon=True) + self.thread.start() + + def read(self): + for line in self.process.stdout: + name, _, value = line.partition(' ') + if name in self.peaks and value.strip().isdigit(): + self.peaks[name] = max(self.peaks[name], int(value)) + + def stop(self): + self.process.terminate() + self.process.wait() + return {name: round(value / MIB, 1) for name, value in self.peaks.items()} + + +def cgroup(container, name): + return int(subprocess.run(['docker', 'exec', container, 'cat', '/sys/fs/cgroup/' + name], + capture_output=True, text=True).stdout.strip()) + + +def post(token, chat_id, data): + target = urllib.parse.urlsplit(BASE_URL) + connection = http.client.HTTPConnection(target.hostname, target.port, timeout=120) + started = time.monotonic() + try: + connection.request('POST', '/api/v1/chats/%s/attachments?filename=big.png' % chat_id, body=data, + headers={'Authorization': 'Bearer ' + token, 'Content-Type': 'image/png'}) + status = connection.getresponse().status + except OSError as error: + status = type(error).__name__ + connection.close() + return status, time.monotonic() - started + + +def restart_api(memory_bytes): + env = dict(os.environ, UPLOAD_MEMORY_BYTES=str(memory_bytes)) + compose('up', '-d', '--no-deps', '--no-build', '--force-recreate', 'api', env=env) + time.sleep(3) + wait_ready() + for container in api_containers(): + wait_in_rotation(container) + + +def run(label, memory_bytes, parallel, data): + restart_api(memory_bytes) + alice = new_user('memory') + chat = new_chat(alice) + replicas = api_containers() + idle = {c[:12]: round(cgroup(c, 'memory.current') / MIB, 1) for c in replicas} + samplers = [Sampler(c) for c in replicas] + started = time.monotonic() + with ThreadPoolExecutor(max_workers=parallel) as pool: + results = list(pool.map(lambda _: post(alice['token'], chat['id'], data), range(parallel))) + wall = time.monotonic() - started + time.sleep(0.5) + sampled = {c[:12]: s.stop() for c, s in zip(replicas, samplers)} + peak = {c[:12]: round(cgroup(c, 'memory.peak') / MIB, 1) for c in replicas} + limit = round(cgroup(replicas[0], 'memory.max') / MIB) + statuses = [r[0] for r in results] + summary = {'label': label, 'upload_memory_bytes': memory_bytes, 'parallel': parallel, + 'file_mib': round(len(data) / MIB, 2), 'ok': statuses.count(201), + 'errors': sorted({str(s) for s in statuses if s != 201}), 'wall_s': round(wall, 2), + 'max_request_s': round(max(r[1] for r in results), 2), 'limit_mib': limit, + 'idle_current_mib': idle, 'peak_since_start_mib': peak, 'sampled_peak_mib': sampled} + worst_anon = max(v['anon'] for v in sampled.values()) + print('%-28s ok=%2d/%d wall=%5.1fs peak(memory.peak)=%s anon max=%.1f MiB' + % (label, summary['ok'], parallel, wall, list(peak.values()), worst_anon)) + return summary + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--output') + parser.add_argument('--parallel', default='10,20') + args = parser.parse_args() + data = noise_png(10 * MIB - 50000) + runs = [run('spool 1 MiB, %s parallel' % n, MIB, int(n), data) for n in args.parallel.split(',')] + runs.append(run('all in RAM, %s parallel' % args.parallel.split(',')[-1], 16 * MIB, + int(args.parallel.split(',')[-1]), data)) + restart_api(MIB) + if args.output: + with open(args.output, 'w', encoding='utf-8') as file: + json.dump({'runs': runs}, file, ensure_ascii=False, indent=2) + + +if __name__ == '__main__': + main()