diff --git a/.github/workflows/app.yml b/.github/workflows/app.yml new file mode 100644 index 00000000..b9ec4641 --- /dev/null +++ b/.github/workflows/app.yml @@ -0,0 +1,31 @@ +name: Application +on: [push, pull_request] +permissions: + contents: read +jobs: + stack: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: cp .env.example .env + - run: python scripts/init_secrets.py + - run: docker compose config --quiet + - run: docker compose up -d --build --wait + - run: make test BASE_URL=http://127.0.0.1:8080 PYTHON=python + - run: make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080 PYTHON=python + - run: make student-test BASE_URL=http://127.0.0.1:8080 PYTHON=python + - run: python student-tests/expiry_check.py --ttl 2 + - run: python student-tests/failure_check.py + - run: python student-tests/consistency_check.py + - run: make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://127.0.0.1:8080 PYTHON=python + - run: docker compose start api && sleep 15 + - run: python student-tests/failover_series.py + - run: python student-tests/rolling_update_check.py --no-build + - if: failure() + run: docker compose ps -a && docker compose logs --no-color + - if: always() + run: docker compose down -v diff --git a/research/s3ops.py b/research/s3ops.py new file mode 100644 index 00000000..cd2495e2 --- /dev/null +++ b/research/s3ops.py @@ -0,0 +1,43 @@ +"""Операции с S3 для внедрения сбоев в треке 2. Запускается внутри контейнера reconcile (ключи сверщика): + python research/s3ops.py put SIZE < keys # создать объекты указанного размера + python research/s3ops.py delete < keys # удалить объекты + python research/s3ops.py exists < keys # вывести число существующих +""" +from concurrent.futures import ThreadPoolExecutor +import sys + +sys.path.insert(0, 'src') + +from botocore.exceptions import ClientError # noqa: E402 + +from config import setting, settings # noqa: E402 +from s3 import make_client # noqa: E402 + +client = make_client(setting('S3_RECONCILER_ACCESS_KEY_ID'), setting('S3_RECONCILER_SECRET_ACCESS_KEY'), timeout=30) + + +def exists(key): + try: + client.head_object(Bucket=settings.s3_bucket, Key=key) + return True + except ClientError: + return False + + +def main(): + command = sys.argv[1] + keys = [line.strip() for line in sys.stdin if line.strip()] + with ThreadPoolExecutor(max_workers=16) as pool: + if command == 'put': + body = b'x' * int(sys.argv[2]) + list(pool.map(lambda k: client.put_object(Bucket=settings.s3_bucket, Key=k, Body=body), keys)) + print(len(keys)) + elif command == 'delete': + list(pool.map(lambda k: client.delete_object(Bucket=settings.s3_bucket, Key=k), keys)) + print(len(keys)) + elif command == 'exists': + print(sum(pool.map(exists, keys))) + + +if __name__ == '__main__': + main() diff --git a/research/track2.py b/research/track2.py new file mode 100644 index 00000000..3a0877e2 --- /dev/null +++ b/research/track2.py @@ -0,0 +1,282 @@ +"""Трек 2: согласованность PostgreSQL и S3 — исследование сверщика (src/reconcile.py). + +Стенд запущен (docker compose up -d). BASE_URL — адрес LB (по умолчанию http://127.0.0.1:8080). + python research/track2.py scenario # H1/H2: внедрённые сбои → dry-run → apply → повторный apply + python research/track2.py race # H2: apply в цикле под непрерывными загрузками + python research/track2.py cost # H3: время/память обхода при 1k…20k объектов + python research/track2.py cleanup # удалить синтетические данные исследования + python research/track2.py report # evidence/track2/results.md + python research/track2.py all +Синтетические строки живут в отдельном чате пользователя track2_*; cleanup удаляет их и их объекты. +На время запуска периодический сервис reconciler останавливается и затем запускается снова. +""" +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +from pathlib import Path +import statistics +import subprocess +import sys +import threading +import time +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'student-tests')) + +from helpers import call, compose, new_chat, new_user, sql # noqa: E402 +from imagegen import SAMPLE_PNG # noqa: E402 +from test_attachments import download, upload # noqa: E402 + +OUT = ROOT / 'evidence' / 'track2' +STATE = OUT / 'state.json' +GRACE = 30 + + +def save(name, data): + OUT.mkdir(parents=True, exist_ok=True) + (OUT / name).write_text(json.dumps(data, ensure_ascii=False, indent=2, default=str) + '\n', encoding='utf-8') + print('saved evidence/track2/' + name) + + +def load(name): + return json.loads((OUT / name).read_text(encoding='utf-8')) + + +def s3ops(command, keys, *args): + result = subprocess.run(['docker', 'compose', 'run', '--rm', '-T', '-v', str(ROOT / 'research') + ':/app/research:ro', + '--entrypoint', 'python', 'reconcile', 'research/s3ops.py', command, *map(str, args)], + cwd=ROOT, input='\n'.join(keys), capture_output=True, text=True, encoding='utf-8') + if result.returncode: + raise SystemExit(result.stderr[-2000:]) + return int(result.stdout.strip().splitlines()[-1]) + + +def reconcile(*args): + return json.loads(compose('run', '--rm', 'reconcile', *map(str, args))) + + +def research_chat(): + if STATE.exists(): + return json.loads(STATE.read_text(encoding='utf-8')) + user = new_user('track2') + chat = new_chat(user) + state = {'user_id': user['id'], 'chat_id': chat['id'], 'token': None} + OUT.mkdir(parents=True, exist_ok=True) + STATE.write_text(json.dumps(state), encoding='utf-8') + return state + + +def insert_rows(state, count, status='ready', age_seconds=0, size=1024): + """Синтетические строки attachments в чате исследования; возвращает ключи объектов.""" + if count <= 0: + return [] + out = sql("WITH new AS (INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, " + "object_key, status, created_at, updated_at) SELECT id, '%s', '%s', 'synthetic.png', 'image/png', %d, " + "decode(repeat('00', 32), 'hex'), 'attachments/' || id, '%s', now() - make_interval(secs => %d), now() " + "FROM (SELECT gen_random_uuid() AS id FROM generate_series(1, %d)) g RETURNING object_key) " + "SELECT string_agg(object_key, ' ') FROM new" + % (state['chat_id'], state['user_id'], size, status, age_seconds, count)) + return out.split() + + +def reference(state, keys): + """Привязать вложения к сообщению (используемые вложения).""" + message = sql("INSERT INTO messages (chat_id, sender_id, text) VALUES ('%s', '%s', convert_to('track2', 'UTF8')) " + "RETURNING id" % (state['chat_id'], state['user_id'])).splitlines()[0] + sql("INSERT INTO message_attachments (message_id, attachment_id, position) SELECT '%s', a.id, " + "row_number() OVER () FROM attachments a WHERE a.object_key IN (%s)" + % (message, ','.join("'%s'" % k for k in keys))) + + +def orphan_keys(count): + return ['attachments/%s' % uuid.uuid4() for _ in range(count)] + + +def scenario(): + state = research_chat() + real = new_user('track2real') + real_chat = new_chat(real) + real_att = upload(real, real_chat, SAMPLE_PNG)[2] + call('POST', '/api/v1/chats/%s/messages' % real_chat['id'], {'text': '', 'attachment_ids': [real_att['id']]}, + real['token']) + baseline = reconcile('--grace-seconds', GRACE) + + ok = insert_rows(state, 200) + s3ops('put', ok, 1024) + referenced = ok[:20] + reference(state, referenced) + missing = ok[20:30] + referenced[:5] + s3ops('delete', missing) + mismatch = ok[40:50] + s3ops('put', mismatch, 2048) + stale_with_object = insert_rows(state, 25, 'uploading', 3600) + s3ops('put', stale_with_object, 1024) + stale_without_object = insert_rows(state, 25, 'uploading', 3600) + old_orphans = orphan_keys(25) + s3ops('put', old_orphans, 1024) + print('waiting %ds so that injected orphans become older than grace…' % (GRACE + 3)) + time.sleep(GRACE + 3) + fresh_uploads = insert_rows(state, 10, 'uploading', 0) + s3ops('put', fresh_uploads, 1024) + fresh_orphans = orphan_keys(10) + s3ops('put', fresh_orphans, 1024) + + expected = {'orphan': 25, 'fresh_orphan': 10, 'stale_upload': 50, 'fresh_upload': 10, 'missing': 15, + 'size_mismatch': 10} + base = baseline['counts'] + dry = reconcile('--grace-seconds', GRACE) + dry_again = reconcile('--grace-seconds', GRACE) + applied = reconcile('--apply', '--grace-seconds', GRACE) + rerun = reconcile('--apply', '--grace-seconds', GRACE) + after = reconcile('--grace-seconds', GRACE) + detected = {k: dry['counts'][k] - base.get(k, 0) for k in expected} + real_download = download(real, real_att['id']) + checks = { + 'detected_equals_injected': detected == expected, + 'dry_run_changes_nothing': dry_again['counts'] == dry['counts'] and dry_again['objects_listed'] == dry['objects_listed'], + 'apply_actions': applied['actions'], + 'apply_deleted_exactly_old_orphans_and_stale': applied['actions'] == { + 'orphan_objects_deleted': 25, 'stale_rows_deleted': 50, 'stale_objects_deleted': 25, 'skipped': 0}, + 'rerun_is_noop': not any(rerun['actions'][k] for k in ('orphan_objects_deleted', 'stale_rows_deleted', 'stale_objects_deleted')), + 'fresh_orphans_kept': s3ops('exists', fresh_orphans) == 10, + 'fresh_uploads_kept': s3ops('exists', fresh_uploads) == 10 and after['counts']['fresh_upload'] >= 10, + 'old_orphans_gone': s3ops('exists', old_orphans) == 0, + 'stale_objects_gone': s3ops('exists', stale_with_object) == 0, + 'referenced_rows_kept': int(sql("SELECT count(*) FROM attachments WHERE object_key IN (%s)" + % ','.join("'%s'" % k for k in referenced))) == 20, + 'missing_still_reported_not_deleted': after['counts']['missing'] - base.get('missing', 0) == 15, + 'real_attachment_intact': real_download[0] == 200 and hashlib.sha256(real_download[2]).hexdigest() == real_att['sha256'], + } + passed = all(v for k, v in checks.items() if k != 'apply_actions') + save('scenario.json', {'grace_seconds': GRACE, 'injected': expected, 'baseline_counts': base, + 'dry_run': dry, 'apply': applied, 'rerun': rerun, 'after': after, + 'detected': detected, 'checks': checks, 'passed': passed}) + for name, value in checks.items(): + print(' %-48s %s' % (name, value)) + print('PASS' if passed else 'FAIL') + + +def race(seconds=40): + user = new_user('track2race') + chat = new_chat(user) + uploaded, errors, stop = [], [], threading.Event() + + def uploader(): + while not stop.is_set(): + status, _, body = upload(user, chat, SAMPLE_PNG) + (uploaded.append(body['id']) if status == 201 else errors.append(status)) + + preclean = reconcile('--apply', '--grace-seconds', 0)['actions'] + threads = [threading.Thread(target=uploader) for _ in range(4)] + for thread in threads: + thread.start() + runs, started = [], time.monotonic() + while time.monotonic() - started < seconds: + report = reconcile('--apply', '--grace-seconds', GRACE) + runs.append({'counts': report['counts'], 'actions': report['actions'], 'total_s': report['seconds']['total']}) + stop.set() + for thread in threads: + thread.join() + intact = 0 + for attachment_id in uploaded: + status, _, body = download(user, attachment_id) + intact += status == 200 and body == SAMPLE_PNG + result = {'seconds': seconds, 'grace_seconds': GRACE, 'preclean_before_load': preclean, + 'reconcile_runs': len(runs), 'uploads_201': len(uploaded), + 'upload_errors': errors, 'downloaded_intact': intact, + 'deleted_by_reconciler': sum(r['actions']['orphan_objects_deleted'] + r['actions']['stale_rows_deleted'] + for r in runs), + 'fresh_uploads_seen_in_flight': sum(r['counts']['fresh_upload'] for r in runs), 'runs': runs} + result['passed'] = intact == len(uploaded) and result['deleted_by_reconciler'] == 0 + save('race.json', result) + print('race: %d uploads, %d intact, reconciler runs %d, deleted %d, in-flight uploads seen %d → %s' + % (len(uploaded), intact, len(runs), result['deleted_by_reconciler'], result['fresh_uploads_seen_in_flight'], + 'PASS' if result['passed'] else 'FAIL')) + + +def cost(sizes=(1000, 5000, 10000, 20000), repeats=3): + state = research_chat() + rows = [] + for target in sizes: + current = int(sql("SELECT count(*) FROM attachments")) + if target > current: + keys = insert_rows(state, target - current) + started = time.monotonic() + for start in range(0, len(keys), 2000): + s3ops('put', keys[start:start + 2000], 256) + print(' +%d objects in %.1fs' % (len(keys), time.monotonic() - started)) + reports = [reconcile('--grace-seconds', GRACE) for _ in range(repeats)] + row = {'objects': reports[-1]['objects_listed'], 'rows': reports[-1]['rows_scanned'], + 'list_pages': reports[-1]['list_pages'], + 'list_s3_s': statistics.median(r['seconds']['list_s3'] for r in reports), + 'scan_db_s': statistics.median(r['seconds']['scan_db'] for r in reports), + 'total_s': statistics.median(r['seconds']['total'] for r in reports), + 'max_rss_mib': max(r['max_rss_mib'] for r in reports)} + row['us_per_object'] = round(row['total_s'] / row['objects'] * 1e6, 1) + rows.append(row) + print(' %(objects)6d objects: list %(list_s3_s).2fs, db %(scan_db_s).3fs, total %(total_s).2fs, ' + 'rss %(max_rss_mib).0f MiB, %(us_per_object).0f µs/object' % row) + save('cost.json', {'repeats': repeats, 'rows': rows}) + + +def cleanup(): + if not STATE.exists(): + print('nothing to clean') + return + state = json.loads(STATE.read_text(encoding='utf-8')) + keys = sql("SELECT coalesce(string_agg(object_key, ' '), '') FROM attachments WHERE chat_id = '%s'" % state['chat_id']).split() + for start in range(0, len(keys), 2000): + s3ops('delete', keys[start:start + 2000]) + sql("DELETE FROM message_attachments WHERE message_id IN (SELECT id FROM messages WHERE chat_id = '%s')" % state['chat_id']) + sql("DELETE FROM messages WHERE chat_id = '%s'" % state['chat_id']) + sql("DELETE FROM attachments WHERE chat_id = '%s'" % state['chat_id']) + leftovers = reconcile('--apply', '--grace-seconds', 0) + STATE.unlink() + print('cleanup: %d research objects deleted; reconciler afterwards: %s' % (len(keys), leftovers['counts'])) + + +def report(): + s, r, c = load('scenario.json'), load('race.json'), load('cost.json') + lines = ['# Трек 2: результаты', '', '## H1/H2. Внедрённые сбои (grace %d с)' % s['grace_seconds'], '', + '| категория | внедрено | найдено dry-run | действие apply |', '|---|---|---|---|'] + actions = {'orphan': 'удалено объектов: %d' % s['apply']['actions']['orphan_objects_deleted'], + 'stale_upload': 'удалено строк: %d, объектов: %d' % (s['apply']['actions']['stale_rows_deleted'], + s['apply']['actions']['stale_objects_deleted']), + 'fresh_orphan': 'не тронуто', 'fresh_upload': 'не тронуто', 'missing': 'только отчёт', + 'size_mismatch': 'только отчёт'} + for name, count in s['injected'].items(): + lines.append('| %s | %d | %d | %s |' % (name, count, s['detected'][name], actions[name])) + lines += ['', '| проверка | результат |', '|---|---|'] + lines += ['| %s | %s |' % (k, v) for k, v in s['checks'].items() if k != 'apply_actions'] + lines += ['', '## H2. Сверщик под нагрузкой загрузок', '', + '%d с, 4 потока загрузок, apply с grace %d с в цикле: запусков сверщика %d, загрузок 201 — %d, ' + 'скачано без изменений — %d, удалено сверщиком — %d, «свежих» загрузок в полёте увидено — %d.' + % (r['seconds'], r['grace_seconds'], r['reconcile_runs'], r['uploads_201'], r['downloaded_intact'], + r['deleted_by_reconciler'], r['fresh_uploads_seen_in_flight']), + '', '## H3. Стоимость обхода (медиана из %d)' % c['repeats'], '', + '| объектов | строк | страниц листинга | листинг S3, с | скан БД, с | всего, с | мкс/объект | RSS, MiB |', + '|---|---|---|---|---|---|---|---|'] + for row in c['rows']: + lines.append('| %(objects)d | %(rows)d | %(list_pages)d | %(list_s3_s).2f | %(scan_db_s).3f | %(total_s).2f | ' + '%(us_per_object).0f | %(max_rss_mib).0f |' % row) + (OUT / 'results.md').write_text('\n'.join(lines) + '\n', encoding='utf-8') + print('saved evidence/track2/results.md') + + +COMMANDS = {'scenario': scenario, 'race': race, 'cost': cost, 'cleanup': cleanup, 'report': report} + +if __name__ == '__main__': + command = sys.argv[1] if len(sys.argv) > 1 else 'all' + # Периодический сервис reconciler (--apply, grace 900 с) удалил бы состаренные синтетические заготовки + # посреди эксперимента; на время исследования он остановлен, сверщик запускается явно. + paused = 'reconciler' in compose('ps', '--status', 'running', '--services') + if paused: + compose('stop', 'reconciler') + try: + for step in (['scenario', 'race', 'cost', 'report', 'cleanup'] if command == 'all' else [command]): + print('==', step) + COMMANDS[step]() + finally: + if paused: + compose('start', 'reconciler') diff --git a/student-tests/consistency_check.py b/student-tests/consistency_check.py new file mode 100644 index 00000000..a0b46121 --- /dev/null +++ b/student-tests/consistency_check.py @@ -0,0 +1,156 @@ +"""Usage (стенд запущен через docker compose up -d): + python student-tests/consistency_check.py + +Проверяет, что сбои при загрузке не оставляют вечных неучтённых данных: + 1) клиент обрывает загрузку посреди тела → ни строки, ни объекта; + 2) S3 остановлен → upload/download 503, ни строки, ни объекта; метаданные, текст и ready работают; + 3) процесс «падает» после S3 PUT до COMMIT ready (FAULTS=after_s3_put, только в этом тесте) → + остаётся строка uploading + объект; сверщик не трогает их в пределах grace и убирает после; + 4) объект без строки в БД (orphan) → свежий не трогается, старый удаляется; повторный запуск ничего не делает; + 5) периодический сервис reconciler сам удаляет заготовку старше grace (без ручного запуска). +""" +import json +import os +import socket +import time +import urllib.parse +import uuid + +from helpers import BASE_URL, call, compose, new_chat, new_user, sql, wait_ready +from imagegen import SAMPLE_PNG +from test_attachments import download, upload + +failures = [] + + +def check(title, ok, detail=''): + print(' %-62s %s %s' % (title, 'OK' if ok else 'FAIL', detail)) + if not ok: + failures.append(title) + + +def reconcile(*args): + return json.loads(compose('run', '--rm', 'reconcile', *args)) + + +def s3_python(code): + return compose('run', '--rm', '--entrypoint', 'python', 'reconcile', '-c', + 'import sys; sys.path.insert(0, "src")\n' + code) + + +def counts(): + return sql("SELECT count(*) FILTER (WHERE status = 'ready') || '/' || count(*) FILTER (WHERE status = 'uploading') " + "FROM attachments") + + +def object_exists(key): + return s3_python('from config import setting, settings\nfrom s3 import make_client\n' + 'c = make_client(setting("S3_RECONCILER_ACCESS_KEY_ID"), setting("S3_RECONCILER_SECRET_ACCESS_KEY"))\n' + 'keys = [o["Key"] for o in c.list_objects_v2(Bucket=settings.s3_bucket, Prefix=%r).get("Contents", [])]\n' + 'print(len(keys))' % key).strip() == '1' + + +def restart_api(faults=None): + env = dict(os.environ) + env['FAULTS'] = faults or '' + compose('up', '-d', '--no-deps', '--no-build', 'api', env=env) + time.sleep(3) + wait_ready() + time.sleep(3) + + +def interrupted_upload(alice, chat): + target = urllib.parse.urlsplit(BASE_URL) + with socket.create_connection((target.hostname, target.port), timeout=10) as sock: + head = ('POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: x\r\n' + 'Authorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n' + % (chat['id'], alice['token'], 5 * 1024 * 1024)) + sock.sendall(head.encode() + SAMPLE_PNG + bytes(512 * 1024)) + time.sleep(2) + + +def main(): + alice = new_user('consistency') + chat = new_chat(alice) + + print('1. Обрыв загрузки посреди тела') + before = counts() + interrupted_upload(alice, chat) + check('строк не прибавилось (ready/uploading)', counts() == before, '%s -> %s' % (before, counts())) + + print('2. S3 остановлен: загрузка, скачивание, остальной API') + stored = upload(alice, chat, SAMPLE_PNG)[2] + before = counts() + compose('stop', 's3') + try: + started = time.monotonic() + status = upload(alice, chat, SAMPLE_PNG)[0] + check('upload при остановленном S3 → 503', status == 503, '%s за %.1f с' % (status, time.monotonic() - started)) + check('заготовка удалена компенсацией', counts() == before, '%s -> %s' % (before, counts())) + started = time.monotonic() + status = download(alice, stored['id'])[0] + check('download при остановленном S3 → 503', status == 503, '%s за %.1f с' % (status, time.monotonic() - started)) + check('метаданные вложения доступны (они в PostgreSQL)', + call('GET', '/api/v1/attachments/' + stored['id'], token=alice['token'])[0] == 200) + check('текстовое сообщение → 201', call('POST', '/api/v1/chats/%s/messages' % chat['id'], + {'text': 'без S3'}, alice['token'])[0] == 201) + check('/health/ready → 200 (S3 не критичен для текстового API)', call('GET', '/health/ready')[0] == 200) + finally: + compose('up', '-d', '--wait', 's3') + check('после старта S3 upload снова 201', upload(alice, chat, SAMPLE_PNG)[0] == 201) + + print('3. Сбой после S3 PUT до фиксации ready (FAULTS=after_s3_put)') + restart_api('after_s3_put') + try: + status = upload(alice, chat, SAMPLE_PNG, filename='fault-%s.png' % uuid.uuid4().hex[:6])[0] + row = sql("SELECT id || ' ' || object_key FROM attachments WHERE status = 'uploading' " + "AND filename LIKE 'fault-%%' ORDER BY created_at DESC LIMIT 1").split() + check('клиент получил ошибку (не 201)', status != 201, str(status)) + check('осталась строка uploading и объект в S3', bool(row) and object_exists(row[1]), ' '.join(row)) + finally: + restart_api() + fresh = reconcile('--grace-seconds', '900') + check('dry-run, grace 900 с: заготовка свежая, не трогается', fresh['counts']['fresh_upload'] >= 1 + and fresh['counts']['stale_upload'] == 0, str(fresh['counts'])) + applied = reconcile('--apply', '--grace-seconds', '0') + check('apply, grace 0: строка и объект удалены', applied['actions']['stale_rows_deleted'] >= 1 + and applied['actions']['stale_objects_deleted'] >= 1, str(applied['actions'])) + check('объекта больше нет', not object_exists(row[1])) + + print('4. Объект без строки в БД (orphan)') + key = 'attachments/%s' % uuid.uuid4() + s3_python('from config import setting, settings\nfrom s3 import make_client\n' + 'c = make_client(setting("S3_RECONCILER_ACCESS_KEY_ID"), setting("S3_RECONCILER_SECRET_ACCESS_KEY"))\n' + 'c.put_object(Bucket=settings.s3_bucket, Key=%r, Body=b"orphan")' % key) + fresh = reconcile('--grace-seconds', '900') + check('dry-run, grace 900 с: свежая сирота не трогается', fresh['counts']['fresh_orphan'] >= 1 + and fresh['counts']['orphan'] == 0, str(fresh['counts'])) + applied = reconcile('--apply', '--grace-seconds', '0') + check('apply, grace 0: сирота удалена', applied['actions']['orphan_objects_deleted'] >= 1, str(applied['actions'])) + again = reconcile('--apply', '--grace-seconds', '0') + check('повторный запуск ничего не делает', not any(again['actions'][k] for k in + ('orphan_objects_deleted', 'stale_rows_deleted', 'stale_objects_deleted')), str(again['actions'])) + check('рабочие вложения не пострадали (missing = 0)', again['counts']['missing'] == 0, str(again['counts'])) + + print('5. Периодический сервис reconciler убирает зависшую заготовку сам') + stale = str(uuid.uuid4()) + sql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, " + "status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'stale.png', 'image/png', 1, " + "decode(repeat('00', 32), 'hex'), 'attachments/%s', 'uploading', now() - interval '1 hour', " + "now() - interval '1 hour')" % (stale, chat['id'], alice['id'], stale)) + running = 'reconciler' in compose('ps', '--status', 'running', '--services') + check('сервис reconciler запущен (docker compose ps)', running) + if running: + compose('restart', 'reconciler') # проход выполняется сразу после старта, дальше — раз в интервал + deadline = time.monotonic() + 60 + while time.monotonic() < deadline and sql("SELECT count(*) FROM attachments WHERE id = '%s'" % stale) != '0': + time.sleep(1) + check('заготовка старше grace удалена без ручного запуска', + sql("SELECT count(*) FROM attachments WHERE id = '%s'" % stale) == '0') + + print('FAIL: ' + ', '.join(failures) if failures else 'PASS: сбои загрузки не оставляют неучтённых данных') + raise SystemExit(1 if failures else 0) + + +if __name__ == '__main__': + main() diff --git a/student-tests/expiry_check.py b/student-tests/expiry_check.py new file mode 100644 index 00000000..7bde51ec --- /dev/null +++ b/student-tests/expiry_check.py @@ -0,0 +1,56 @@ +"""Usage (стенд уже запущен через docker compose up -d): + python student-tests/expiry_check.py --ttl 2 + +Скрипт пересоздаёт только api с SESSION_TTL_SECONDS=, проверяет истечение сессии, +затем возвращает api с обычным TTL из .env. Volumes и данные не трогаются. +""" +import argparse +import hashlib +import os +import time + +from helpers import call, compose, new_user, wait_ready + + +def restart_api(ttl=None): + env = dict(os.environ) + if ttl is None: + env.pop('SESSION_TTL_SECONDS', None) + else: + env['SESSION_TTL_SECONDS'] = str(ttl) + compose('up', '-d', '--no-deps', '--no-build', 'api', env=env) + wait_ready() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--ttl', type=int, default=2) + args = parser.parse_args() + + print('пересоздаю api с SESSION_TTL_SECONDS=%d' % args.ttl) + restart_api(args.ttl) + try: + alice = new_user('expiry') + status = call('GET', '/api/v1/users/me', token=alice['token'])[0] + print('сразу после входа: /users/me -> %d' % status) + assert status == 200 + key = 'session:' + hashlib.sha256(alice['token'].encode()).hexdigest() + cache_ttl = int(compose('exec', '-T', 'valkey', 'valkey-cli', 'TTL', key)) + print('TTL ключа этой сессии в Valkey: %d с (-2 = ключа нет)' % cache_ttl) + assert cache_ttl <= args.ttl, 'кэш живёт дольше сессии' + + time.sleep(args.ttl + 1.5) + status, headers, _ = call('GET', '/api/v1/users/me', token=alice['token']) + print('через %.1f с: /users/me -> %d, WWW-Authenticate: %s' % (args.ttl + 1.5, status, headers['WWW-Authenticate'])) + assert status == 401 and headers['WWW-Authenticate'] == 'Bearer realm="messenger"' + status = call('DELETE', '/api/v1/auth/sessions/current', token=alice['token'])[0] + print('logout истёкшим токеном -> %d' % status) + assert status == 401 + print('PASS: сессия истекла по SESSION_TTL_SECONDS=%d, кэш не продлил её' % args.ttl) + finally: + print('возвращаю api с TTL из .env') + restart_api() + + +if __name__ == '__main__': + main() diff --git a/student-tests/failover_series.py b/student-tests/failover_series.py new file mode 100644 index 00000000..bbe15e39 --- /dev/null +++ b/student-tests/failover_series.py @@ -0,0 +1,71 @@ +"""Usage (стенд запущен через docker compose up -d, обе реплики в ротации): + python student-tests/failover_series.py --output evidence/failover-series.json + +Для каждой API-реплики по очереди и в двух режимах (docker stop — штатная остановка с выводом из ротации, +docker kill — внезапная смерть процесса) под нагрузкой: останавливает реплику, меряет ошибки, RPS, +время до последнего ответа остановленной реплики, восстанавливает её (docker compose start api) +и ждёт возврата в ротацию LB. Volumes, БД и LB не трогаются. +""" +import argparse +import json +import subprocess +import time + +from helpers import Load, api_containers, compose, new_chat, new_user, wait_in_rotation + +WARMUP, AFTER = 5, 10 + + +def run(container, mode): + alice = new_user('series') + chat = new_chat(alice) + load = Load(alice['token'], chat['id']).start() + time.sleep(WARMUP) + acted = time.monotonic() + command = ['docker', 'stop', '-t', '15', container] if mode == 'stop' else ['docker', 'kill', container] + subprocess.run(command, capture_output=True, check=True) + action_seconds = time.monotonic() - acted + time.sleep(AFTER) + load.stop() + instance = container[:12] + target = [r for r in load.records if r[2] == 201 and r[3] == instance and r[0] >= acted] + others = [r for r in load.records if r[2] == 201 and r[3] != instance and r[0] >= acted] + restored = time.monotonic() + compose('start', 'api') + back = wait_in_rotation(container) + result = { + 'replica': instance, 'mode': mode, 'action_seconds': round(action_seconds, 2), + 'target_served_until_s': round(max(r[1] for r in target) - acted, 2) if target else 0.0, + 'first_success_on_other_s': round(min(r[1] for r in others) - acted, 2) if others else None, + 'before': load.window(acted - WARMUP, acted), + 'during_10s': load.window(acted, acted + AFTER), + 'back_in_rotation_s': round(back - restored, 2), + 'data': load.verify(alice['token']), + } + print('%-12s %-4s errors=%-3d rps %5.1f -> %5.1f target last answer +%.2fs restore %.1fs lost=%d dup=%d' + % (instance, mode, result['during_10s']['errors'], result['before']['rps'], result['during_10s']['rps'], + result['target_served_until_s'], result['back_in_rotation_s'], result['data']['lost'], + result['data']['duplicate_texts'])) + return result + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--output') + parser.add_argument('--modes', default='stop,kill') + args = parser.parse_args() + replicas = api_containers() + assert len(replicas) >= 2, 'need at least two running API replicas' + results = [run(container, mode) for mode in args.modes.split(',') for container in replicas] + bad = [r for r in results if r['data']['lost'] or r['data']['duplicate_texts'] or r['data']['duplicate_ids']] + summary = {'replicas': [c[:12] for c in replicas], 'warmup_s': WARMUP, 'observe_s': AFTER, 'workers': 4, + 'lb_check': 'GET /health/ready every 1s, fall 2, rise 2', 'results': results} + if args.output: + with open(args.output, 'w', encoding='utf-8') as file: + json.dump(summary, file, ensure_ascii=False, indent=2) + print('FAIL: data loss or duplicates' if bad else 'PASS: every replica stopped once per mode; no loss, no duplicates') + raise SystemExit(1 if bad else 0) + + +if __name__ == '__main__': + main() diff --git a/student-tests/failure_check.py b/student-tests/failure_check.py new file mode 100644 index 00000000..a6ffa7ce --- /dev/null +++ b/student-tests/failure_check.py @@ -0,0 +1,127 @@ +"""Usage (стенд уже запущен через docker compose up -d): + python student-tests/failure_check.py + +Останавливает и замораживает Valkey и PostgreSQL (docker compose stop/pause, без удаления volumes), +проверяет поведение API и восстановление. +""" +import time +import uuid + +from helpers import basic, call, compose, new_chat, new_user, wait_ready + +failures = [] + + +def timed(method, path, token=None, body=None, headers=None): + started = time.monotonic() + try: + status = call(method, path, body=body, token=token, headers=headers, timeout=10)[0] + except OSError as error: + status = type(error).__name__ + return status, time.monotonic() - started + + +def expect(title, result, allowed, max_seconds): + status, seconds = result + ok = status in allowed and seconds <= max_seconds + print(' %-48s -> %-4s за %.2f с %s' % (title, status, seconds, 'OK' if ok else 'FAIL')) + if not ok: + failures.append(title) + + +def main(): + alice = new_user('failure') + chat = new_chat(alice) + history = '/api/v1/chats/%s/messages' % chat['id'] + assert call('POST', history, {'text': 'до отказа'}, alice['token'])[0] == 201 + fake = uuid.uuid4().hex * 2 + + print('1. Valkey остановлен: кэш обходится, всё работает через PostgreSQL') + compose('stop', 'valkey') + try: + expect('первый запрос: обнаружение отказа кэша', timed('GET', '/api/v1/users/me', alice['token']), [200], 1.5) + expect('GET /users/me действующим токеном', timed('GET', '/api/v1/users/me', alice['token']), [200], 1) + expect('POST сообщение', timed('POST', history, alice['token'], {'text': 'без кэша'}), [201], 1) + expect('неизвестный токен', timed('GET', '/api/v1/users/me', fake), [401], 1) + expect('без Authorization', timed('GET', '/api/v1/users/me'), [401], 1) + expect('/health/ready (Valkey некритичен)', timed('GET', '/health/ready'), [200], 1) + started = time.monotonic() + other = new_user('nocache') + expect('регистрация и вход без кэша', (200, time.monotonic() - started), [200], 1) + expect('logout без кэша', timed('DELETE', '/api/v1/auth/sessions/current', other['token']), [204], 1.5) + expect('отозванный токен', timed('GET', '/api/v1/users/me', other['token']), [401], 1) + finally: + compose('up', '-d', '--wait', 'valkey') + + print('1б. Valkey на паузе: logout, затем unpause — отозванный токен не оживает из кэша') + revoked = new_user('paused') + call('GET', '/api/v1/users/me', token=revoked['token']) + compose('pause', 'valkey') + try: + expect('logout при паузе Valkey', timed('DELETE', '/api/v1/auth/sessions/current', revoked['token']), [204], 1.5) + finally: + compose('unpause', 'valkey') + time.sleep(6) + expect('через 6 с после unpause: /users/me', timed('GET', '/api/v1/users/me', revoked['token']), [401], 1) + expect('через 6 с после unpause: сообщение', timed('POST', history, revoked['token'], {'text': 'после logout'}), [401], 1) + + print('1в. Valkey на паузе: logout, рестарт API, unpause — отзыв переживает рестарт процесса') + restarted = new_user('restart') + call('GET', '/api/v1/users/me', token=restarted['token']) + compose('pause', 'valkey') + try: + expect('logout при паузе Valkey', timed('DELETE', '/api/v1/auth/sessions/current', restarted['token']), [204], 1.5) + compose('restart', 'api') + wait_ready() + finally: + compose('unpause', 'valkey') + time.sleep(6) + expect('после рестарта API и unpause: /users/me', timed('GET', '/api/v1/users/me', restarted['token']), [401], 1) + expect('после рестарта API и unpause: сообщение', timed('POST', history, restarted['token'], {'text': 'после рестарта'}), [401], 1) + + print('2. PostgreSQL остановлен: ограниченная ошибка 503, анонимный запрос не проходит') + compose('stop', 'postgres') + stopped_at = time.monotonic() + try: + expect('GET история', timed('GET', history, alice['token']), [503], 3) + expect('POST сообщение', timed('POST', history, alice['token'], {'text': 'при отказе БД'}), [503], 3) + expect('неизвестный токен (не 200!)', timed('GET', '/api/v1/users/me', fake), [503], 3) + expect('без Authorization', timed('GET', '/api/v1/users/me'), [401], 1) + expect('вход по Basic верным паролем', timed('POST', '/api/v1/auth/sessions', + headers=basic(alice['username'], alice['password'])), [503], 3) + expect('/health/live', timed('GET', '/health/live'), [200], 1) + expect('/health/ready', timed('GET', '/health/ready'), [503], 3) + finally: + compose('up', '-d', '--wait', 'postgres') + ready_at = wait_ready() + print('3. PostgreSQL запущен: готовность через %.1f с после остановки' % (ready_at - stopped_at)) + status = call('GET', '/api/v1/users/me', token=alice['token'])[0] + texts = [m['text'] for m in call('GET', history + '?limit=100', token=alice['token'])[2]['items']] + print(' прежний токен -> %d; история: %s' % (status, texts)) + if status != 200 or texts != ['до отказа', 'без кэша']: + failures.append('восстановление') + + print('4. PostgreSQL заморожен (pause): запросы получают 503 в пределах общего срока, а не зависают') + compose('pause', 'postgres') + try: + expect('GET история', timed('GET', history, alice['token']), [503], 4.5) + expect('POST сообщение', timed('POST', history, alice['token'], {'text': 'при заморозке'}), [503], 4.5) + expect('неизвестный токен (не 200!)', timed('GET', '/api/v1/users/me', fake), [503], 4.5) + expect('вход по Basic', timed('POST', '/api/v1/auth/sessions', headers=basic(alice['username'], alice['password'])), [503], 4.5) + expect('/health/ready', timed('GET', '/health/ready'), [503], 2) + expect('/health/live', timed('GET', '/health/live'), [200], 1) + finally: + compose('unpause', 'postgres') + unpaused = time.monotonic() + wait_ready() + texts = [m['text'] for m in call('GET', history + '?limit=100', token=alice['token'])[2]['items']] + print(' готовность через %.1f с после unpause; история: %s' % (time.monotonic() - unpaused, texts)) + if texts != ['до отказа', 'без кэша']: + failures.append('после заморозки') + + print('FAIL: ' + ', '.join(failures) if failures else 'PASS: отказы Valkey и PostgreSQL обработаны') + raise SystemExit(1 if failures else 0) + + +if __name__ == '__main__': + main() diff --git a/student-tests/fixtures/sample.jpg b/student-tests/fixtures/sample.jpg new file mode 100644 index 00000000..48428645 Binary files /dev/null and b/student-tests/fixtures/sample.jpg differ diff --git a/student-tests/helpers.py b/student-tests/helpers.py new file mode 100644 index 00000000..8c577953 --- /dev/null +++ b/student-tests/helpers.py @@ -0,0 +1,191 @@ +import base64 +import json +import os +from pathlib import Path +import subprocess +import threading +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid + +BASE_URL = os.environ.get('BASE_URL', 'http://127.0.0.1:8080').rstrip('/') +ROOT = Path(__file__).resolve().parents[1] + + +def call(method, path, body=None, token=None, raw=None, headers=None, timeout=10): + all_headers = {} + if token: + all_headers['Authorization'] = 'Bearer ' + token + if body is not None: + raw = json.dumps(body, ensure_ascii=False).encode('utf-8') + if raw is not None: + all_headers['Content-Type'] = 'application/json' + all_headers.update(headers or {}) + if method == 'POST' and path.split('?')[0].endswith('/messages') and 'Idempotency-Key' not in all_headers: + all_headers['Idempotency-Key'] = uuid.uuid4().hex + all_headers = {name: value for name, value in all_headers.items() if value is not None} + request = urllib.request.Request(BASE_URL + path, data=raw, headers=all_headers, method=method) + try: + response = urllib.request.urlopen(request, timeout=timeout) + except urllib.error.HTTPError as error: + response = error + with response: + data = response.read() + is_json = response.headers.get_content_type() == 'application/json' + return response.status, response.headers, json.loads(data) if data and is_json else None + + +def basic(username, password): + return {'Authorization': 'Basic ' + base64.b64encode((username + ':' + password).encode('utf-8')).decode()} + + +def register(label='user', password=None): + username = (label + '_' + uuid.uuid4().hex[:12])[:32] + password = password or 'Pw-' + uuid.uuid4().hex + status, _, user = call('POST', '/api/v1/users', {'username': username, 'display_name': label, 'password': password}) + assert status == 201, status + return user, password + + +def login(username, password): + status, _, session = call('POST', '/api/v1/auth/sessions', headers=basic(username, password)) + assert status == 201, status + return session + + +def new_user(label='user'): + user, password = register(label) + session = login(user['username'], password) + return dict(user, password=password, token=session['token']) + + +def new_chat(owner, *others, title='Тестовый чат'): + members = [owner['id']] + [u['id'] for u in others] + status, _, chat = call('POST', '/api/v1/chats', {'title': title, 'member_ids': members}, owner['token']) + assert status == 201, status + return chat + + +def read_all_messages(chat_id, token, limit): + items, cursor, pages = [], None, 0 + while True: + query = '?limit=%d' % limit + ('&cursor=' + urllib.parse.quote(cursor, safe='') if cursor else '') + status, _, page = call('GET', '/api/v1/chats/%s/messages%s' % (chat_id, query), token=token) + assert status == 200, status + items += page['items'] + pages += 1 + cursor = page['next_cursor'] + if cursor is None: + return items, pages + + +def compose(*args, env=None, check=True): + result = subprocess.run(['docker', 'compose', *args], cwd=ROOT, capture_output=True, text=True, + encoding='utf-8', env=env) + if check and result.returncode != 0: + raise RuntimeError('docker compose %s failed: %s' % (' '.join(args), result.stderr.strip())) + return result.stdout.strip() + + +def sql(query, db='messenger', service='postgres'): + return compose('exec', '-T', service, 'psql', '-U', 'messenger', '-d', db, '-X', '-tA', '-v', 'ON_ERROR_STOP=1', + '-c', query) + + +def wait_ready(seconds=90): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + try: + if call('GET', '/health/ready', timeout=2)[0] == 200: + return time.monotonic() + except OSError: + pass + time.sleep(0.3) + raise AssertionError('API is not ready after %ds' % seconds) + + +def api_containers(): + return compose('ps', '-q', 'api').split() + + +def wait_in_rotation(container, seconds=60): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + try: + if call('GET', '/health/ready', timeout=2)[1].get('X-Instance-Id') == container[:12]: + return time.monotonic() + except OSError: + pass + time.sleep(0.1) + raise AssertionError('container %s did not enter LB rotation' % container[:12]) + + +class Load: + """Отправляет сообщения в несколько потоков; неудачный POST повторяет с тем же Idempotency-Key.""" + + def __init__(self, token, chat_id, workers=4): + self.token, self.path, self.workers = token, '/api/v1/chats/%s/messages' % chat_id, workers + self.records, self.confirmed, self.uncertain = [], [], 0 + self.stopped = threading.Event() + self.threads = [] + + def start(self): + self.threads = [threading.Thread(target=self.worker, args=(n,), daemon=True) for n in range(self.workers)] + for thread in self.threads: + thread.start() + return self + + def stop(self): + self.stopped.set() + for thread in self.threads: + thread.join() + + def worker(self, n): + sent = 0 + while not self.stopped.is_set(): + key, text = uuid.uuid4().hex, 'load %d-%d %s' % (n, sent, uuid.uuid4().hex[:6]) + sent += 1 + while True: + started = time.monotonic() + try: + status, headers, body = call('POST', self.path, {'text': text}, self.token, + headers={'Idempotency-Key': key}, timeout=5) + instance = headers.get('X-Instance-Id') + except OSError as error: + status, instance, body = type(error).__name__, None, None + self.records.append((started, time.monotonic(), status, instance)) + if status == 201: + self.confirmed.append(body) + break + if self.stopped.is_set(): + self.uncertain += 1 + break + time.sleep(0.05) + + def window(self, start, end): + rows = [r for r in self.records if start <= r[0] < end] + ok = [r for r in rows if r[2] == 201] + latencies = sorted((r[1] - r[0]) * 1000 for r in ok) + p = lambda q: round(latencies[min(len(latencies) - 1, int(len(latencies) * q))], 1) if latencies else None + return {'seconds': round(end - start, 2), 'ok': len(ok), 'errors': len(rows) - len(ok), + 'rps': round(len(ok) / (end - start), 1) if end > start else 0, + 'p50_ms': p(0.5), 'p95_ms': p(0.95), 'p99_ms': p(0.99), + 'error_kinds': sorted({str(r[2]) for r in rows if r[2] != 201})} + + def verify(self, token): + stored, cursor = [], None + while True: + query = '?limit=100' + ('&cursor=' + urllib.parse.quote(cursor, safe='') if cursor else '') + page = call('GET', self.path + query, token=token)[2] + stored += page['items'] + cursor = page['next_cursor'] + if cursor is None: + break + ids = [m['id'] for m in stored] + texts = [m['text'] for m in stored] + return {'confirmed': len(self.confirmed), 'stored': len(stored), + 'lost': len({m['id'] for m in self.confirmed} - set(ids)), + 'duplicate_ids': len(ids) - len(set(ids)), 'duplicate_texts': len(texts) - len(set(texts)), + 'uncertain_at_stop': self.uncertain} diff --git a/student-tests/imagegen.py b/student-tests/imagegen.py new file mode 100644 index 00000000..8e19ffbe --- /dev/null +++ b/student-tests/imagegen.py @@ -0,0 +1,66 @@ +import os +from pathlib import Path +import struct +import zlib + +ROOT = Path(__file__).resolve().parents[1] +SAMPLE_PNG = (ROOT / 'tests' / 'fixtures' / 'sample.png').read_bytes() +SAMPLE_JPG = (Path(__file__).resolve().parent / 'fixtures' / 'sample.jpg').read_bytes() + + +def chunk(kind, data): + return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff) + + +def png(width, height, rows, color_type=0, level=9): + """rows — итератор байтовых строк пикселей (без байта фильтра).""" + compressor = zlib.compressobj(level) + parts = [] + for row in rows: + parts.append(compressor.compress(b'\x00' + row)) + parts.append(compressor.flush()) + header = struct.pack('>IIBBBBB', width, height, 8, color_type, 0, 0, 0) + return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', b''.join(parts)) + chunk(b'IEND', b'') + + +def bomb(width=10000, height=5000): + """50 Мп чёрного: в распакованном виде 50 МБ, в файле — несколько десятков КБ.""" + row = bytes(width) + return png(width, height, (row for _ in range(height))) + + +def noise_png(target_bytes): + """Шумный RGB PNG (почти не сжимается) размером чуть меньше target_bytes.""" + width = 1024 + height = max(1, (target_bytes - 4096) // (width * 3 + 1)) + return png(width, height, (os.urandom(width * 3) for _ in range(height)), color_type=2, level=0) + + +def broken_crc(data): + """Испортить один байт внутри IDAT, не пересчитывая CRC.""" + position = data.index(b'IDAT') + 100 + return data[:position] + bytes([data[position] ^ 0xff]) + data[position + 1:] + + +def with_idat(data, transform): + """Заменить пиксельные данные PNG (все IDAT) на transform(idat), с правильными CRC.""" + position, chunks = 8, [] + while position < len(data): + length, kind = struct.unpack('>I4s', data[position:position + 8]) + chunks.append((kind, data[position + 8:position + 8 + length])) + position += 12 + length + idat = b''.join(body for kind, body in chunks if kind == b'IDAT') + first = next(index for index, (kind, _) in enumerate(chunks) if kind == b'IDAT') + kept = [(kind, body) for kind, body in chunks if kind != b'IDAT'] + kept.insert(first, (b'IDAT', transform(idat))) + return data[:8] + b''.join(chunk(kind, body) for kind, body in kept) + + +def garbage_idat(data): + """Мусор вместо сжатых пикселей, CRC верные.""" + return with_idat(data, lambda idat: idat[:2] + os.urandom(len(idat) - 2)) + + +def short_idat(data): + """Корректный zlib-поток, но пикселей меньше, чем требует IHDR.""" + return with_idat(data, lambda idat: zlib.compress(zlib.decompress(idat)[:-500])) diff --git a/student-tests/rolling_update_check.py b/student-tests/rolling_update_check.py new file mode 100644 index 00000000..eabfedcc --- /dev/null +++ b/student-tests/rolling_update_check.py @@ -0,0 +1,60 @@ +"""Usage (стенд запущен, обе реплики в ротации): + python student-tests/rolling_update_check.py --output evidence/rolling-update.json + +Под непрерывной нагрузкой (4 потока, повтор с тем же Idempotency-Key) выполняет scripts/rolling_update.py +и проверяет: ошибки клиента, RPS, что все реплики заменены, что подтверждённые сообщения не потеряны и не задвоены. +""" +import argparse +import json +from pathlib import Path +import subprocess +import sys +import time + +from helpers import Load, api_containers, new_chat, new_user + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--output') + parser.add_argument('--no-build', action='store_true') + args = parser.parse_args() + before = api_containers() + alice = new_user('rolling') + chat = new_chat(alice) + load = Load(alice['token'], chat['id']).start() + time.sleep(5) + started = time.monotonic() + command = [sys.executable, str(ROOT / 'scripts' / 'rolling_update.py')] + (['--no-build'] if args.no_build else []) + update = subprocess.run(command, capture_output=True, text=True, encoding='utf-8') + finished = time.monotonic() + time.sleep(5) + load.stop() + print(update.stdout.strip()) + after = api_containers() + result = { + 'replicas_before': [c[:12] for c in before], 'replicas_after': [c[:12] for c in after], + 'update_seconds': round(finished - started, 1), 'update_exit_code': update.returncode, + 'before_5s': load.window(started - 5, started), + 'during_update': load.window(started, finished), + 'after_5s': load.window(finished, finished + 5), + 'data': load.verify(alice['token']), + 'log': update.stdout.strip().splitlines(), + } + replaced = not set(before) & set(after) and len(after) == len(before) + ok = update.returncode == 0 and replaced and not result['data']['lost'] and not result['data']['duplicate_texts'] + print('rolling update %.1fs: errors=%d, rps %.1f -> %.1f -> %.1f, lost=%d, duplicates=%d, all replaced=%s' + % (result['update_seconds'], result['during_update']['errors'], result['before_5s']['rps'], + result['during_update']['rps'], result['after_5s']['rps'], result['data']['lost'], + result['data']['duplicate_texts'], replaced)) + if args.output: + with open(args.output, 'w', encoding='utf-8') as file: + json.dump(result, file, ensure_ascii=False, indent=2) + print('PASS' if ok else 'FAIL: ' + update.stderr.strip()[-500:]) + raise SystemExit(0 if ok else 1) + + +if __name__ == '__main__': + main() diff --git a/student-tests/sigterm_check.py b/student-tests/sigterm_check.py new file mode 100644 index 00000000..5d09b0ee --- /dev/null +++ b/student-tests/sigterm_check.py @@ -0,0 +1,79 @@ +"""Usage (стенд запущен через docker compose up -d): + python student-tests/sigterm_check.py + docker compose start api + +Останавливает одну API-реплику (docker stop = SIGTERM) и всё это время опрашивает API через LB: +реплика сначала выходит из ротации (readiness 503), потом завершается; клиенты не должны получить ошибок. +""" +import argparse +import subprocess +import threading +import time +import urllib.error +import urllib.request +import uuid + +from helpers import BASE_URL, api_containers, call, new_chat, new_user + + +def docker(*args): + return subprocess.run(['docker', *args], capture_output=True, text=True, encoding='utf-8').stdout.strip() + + +def probe_forever(results, stop_event): + while not stop_event.is_set(): + started = time.monotonic() + try: + with urllib.request.urlopen(BASE_URL + '/health/live', timeout=1) as response: + outcome = str(response.status) + except urllib.error.HTTPError as error: + outcome = str(error.code) + except Exception as error: + outcome = type(error).__name__ + results.append((started, outcome)) + time.sleep(0.02) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--container', default=None) + parser.add_argument('--stop-timeout', type=int, default=10, help='секунд до SIGKILL у docker stop') + args = parser.parse_args() + args.container = args.container or api_containers()[0] + + marker = 'PRIVATE-' + uuid.uuid4().hex + alice = new_user('alice') + chat = new_chat(alice) + status = call('POST', '/api/v1/chats/%s/messages' % chat['id'], {'text': marker}, alice['token'])[0] + logs_before = docker('logs', args.container) + print('сообщение с маркером отправлено: HTTP %d; маркер в логах: %s' % (status, marker in logs_before)) + + results, stop_event = [], threading.Event() + prober = threading.Thread(target=probe_forever, args=(results, stop_event)) + prober.start() + time.sleep(1) + + sigterm_at = time.monotonic() + docker('stop', '-t', str(args.stop_timeout), args.container) + stopped_at = time.monotonic() + time.sleep(1) + stop_event.set() + prober.join() + + before = [r for t, r in results if t < sigterm_at] + after = [r for t, r in results if t >= sigterm_at] + exit_code = docker('inspect', '--format', '{{.State.ExitCode}}', args.container) + print('docker stop занял: %.2f с (лимит %d с)' % (stopped_at - sigterm_at, args.stop_timeout)) + print('код выхода контейнера: %s (0 — сам завершился по SIGTERM; 137 — убит SIGKILL)' % exit_code) + print('запросы до SIGTERM: %d, все 200: %s' % (len(before), set(before) == {'200'})) + print('запросы после SIGTERM: %d, результаты: %s' % (len(after), sorted(set(after)))) + print('ответов 5xx за всё время: %d' % sum(r.startswith('5') for _, r in results)) + + print('\nлоги остановки:') + for line in docker('logs', args.container).splitlines(): + if any(word in line for word in ('draining', 'Shutting down', 'stopping', 'shutdown', 'Finished server')): + print(' ' + line) + + +if __name__ == '__main__': + main() diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..be589b60 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,157 @@ +import unittest +import uuid + +from helpers import call, login, new_chat, new_user, read_all_messages + + +class ErrorFormatMixin: + def assertError(self, result, status, code): + actual, headers, body = result + self.assertEqual(actual, status) + self.assertEqual(body['error']['code'], code) + self.assertEqual(body['error']['request_id'], headers['X-Request-Id']) + self.assertNotIn('Traceback', body['error']['message']) + + +class InvalidJsonTest(ErrorFormatMixin, unittest.TestCase): + def test_malformed_json(self): + """Битый JSON -> 400 invalid_request""" + self.assertError(call('POST', '/api/v1/users', raw=b'{"username": "abc",'), 400, 'invalid_request') + + def test_json_not_object(self): + """JSON-массив вместо объекта -> 400""" + self.assertError(call('POST', '/api/v1/users', raw=b'[1, 2, 3]'), 400, 'invalid_request') + + def test_wrong_type(self): + """Число вместо строки -> 400 (без тихого приведения типов)""" + self.assertError(call('POST', '/api/v1/users', {'username': 12345, 'display_name': 'x', 'password': 'Long-enough-pass-1'}), 400, 'invalid_request') + + def test_extra_field(self): + """Лишнее поле в теле -> 400""" + body = {'username': 'u_' + uuid.uuid4().hex[:8], 'display_name': 'x', 'password': 'Long-enough-pass-1', 'is_admin': True} + self.assertError(call('POST', '/api/v1/users', body), 400, 'invalid_request') + + +class DuplicateUsernameTest(ErrorFormatMixin, unittest.TestCase): + def test_duplicate_username(self): + """Повторная регистрация того же username -> 409 conflict""" + user = new_user('dup') + again = call('POST', '/api/v1/users', {'username': user['username'], 'display_name': 'Другой', + 'password': 'Other-password-123'}) + self.assertError(again, 409, 'conflict') + + +class ForeignChatTest(ErrorFormatMixin, unittest.TestCase): + def setUp(self): + self.alice, self.bob, self.eve = new_user('alice'), new_user('bob'), new_user('eve') + self.chat = new_chat(self.alice, self.bob) + self.path = '/api/v1/chats/' + self.chat['id'] + + def test_outsider_cannot_read_chat(self): + """Посторонний читает чат -> 404, как будто чата нет""" + self.assertError(call('GET', self.path, token=self.eve['token']), 404, 'not_found') + missing = call('GET', '/api/v1/chats/' + str(uuid.uuid4()), token=self.eve['token']) + self.assertEqual(missing[2]['error']['message'], call('GET', self.path, token=self.eve['token'])[2]['error']['message']) + + def test_outsider_cannot_write_or_read_history(self): + """Посторонний пишет/читает историю -> 404""" + self.assertError(call('POST', self.path + '/messages', {'text': 'чужое'}, self.eve['token']), 404, 'not_found') + self.assertError(call('GET', self.path + '/messages', token=self.eve['token']), 404, 'not_found') + + def test_outsider_does_not_see_chat_in_list(self): + """Чужой чат не попадает в список чатов""" + status, _, page = call('GET', '/api/v1/chats?limit=100', token=self.eve['token']) + self.assertEqual(status, 200) + self.assertNotIn(self.chat['id'], [c['id'] for c in page['items']]) + + def test_member_has_access(self): + """Участник видит чат (контроль: 404 выше не из-за ошибки в тесте)""" + self.assertEqual(call('GET', self.path, token=self.bob['token'])[0], 200) + + +class EmptyMessageTest(ErrorFormatMixin, unittest.TestCase): + def setUp(self): + self.alice = new_user('alice') + self.path = '/api/v1/chats/%s/messages' % new_chat(self.alice)['id'] + + def test_empty_text(self): + """Пустой текст -> 400""" + self.assertError(call('POST', self.path, {'text': ''}, self.alice['token']), 400, 'invalid_request') + + def test_whitespace_text(self): + """Текст только из пробельных символов -> 400""" + self.assertError(call('POST', self.path, {'text': ' \t\n '}, self.alice['token']), 400, 'invalid_request') + + def test_missing_text(self): + """Тело без поля text -> 400""" + self.assertError(call('POST', self.path, {}, self.alice['token']), 400, 'invalid_request') + + def test_text_length_limit(self): + """4000 символов можно (эмодзи = 1 символ), 4001 — 400""" + self.assertEqual(call('POST', self.path, {'text': '😀' * 4000}, self.alice['token'])[0], 201) + self.assertError(call('POST', self.path, {'text': 'x' * 4001}, self.alice['token']), 400, 'invalid_request') + + +class PaginationTest(ErrorFormatMixin, unittest.TestCase): + def setUp(self): + self.alice = new_user('alice') + self.chat = new_chat(self.alice) + self.sent = [call('POST', '/api/v1/chats/%s/messages' % self.chat['id'], + {'text': 'сообщение %d' % i}, self.alice['token'])[2] for i in range(7)] + + def test_walk_several_pages(self): + """7 сообщений по 3: три страницы, без потерь, повторов и перестановок""" + items, pages = read_all_messages(self.chat['id'], self.alice['token'], limit=3) + self.assertEqual(pages, 3) + self.assertEqual([m['id'] for m in items], [m['id'] for m in self.sent]) + keys = [(m['created_at'], m['id']) for m in items] + self.assertEqual(keys, sorted(keys)) + + def test_cursor_from_other_chat(self): + """cursor от другого чата -> 400""" + _, _, page = call('GET', '/api/v1/chats/%s/messages?limit=1' % self.chat['id'], token=self.alice['token']) + other = new_chat(self.alice, title='Другой') + self.assertError(call('GET', '/api/v1/chats/%s/messages?cursor=%s' % (other['id'], page['next_cursor']), + token=self.alice['token']), 400, 'invalid_request') + + def test_invalid_cursor_and_limit(self): + """Мусорный cursor и limit вне 1..100 -> 400""" + path = '/api/v1/chats/%s/messages' % self.chat['id'] + for query in ('?cursor=not-a-cursor', '?limit=0', '?limit=101', '?limit=abc'): + with self.subTest(query=query): + self.assertError(call('GET', path + query, token=self.alice['token']), 400, 'invalid_request') + + +class IdentityTest(ErrorFormatMixin, unittest.TestCase): + def test_sender_cannot_be_forged(self): + """sender_id в теле запрещён: отправитель только из сессии""" + alice, bob = new_user('alice'), new_user('bob') + chat = new_chat(alice, bob) + forged = call('POST', '/api/v1/chats/%s/messages' % chat['id'], {'text': 'hi', 'sender_id': bob['id']}, alice['token']) + self.assertError(forged, 400, 'invalid_request') + + def test_nul_character_is_preserved(self): + """U+0000 — допустимый code point: сообщение, display_name и title сохраняются и читаются без изменений""" + username, password = 'nul_' + uuid.uuid4().hex[:12], 'Nul-password-123' + status, _, user = call('POST', '/api/v1/users', {'username': username, 'display_name': 'a\u0000lice', 'password': password}) + self.assertEqual((status, user['display_name']), (201, 'a\u0000lice')) + alice = dict(user, token=login(username, password)['token']) + self.assertEqual(call('GET', '/api/v1/users/me', token=alice['token'])[2]['display_name'], 'a\u0000lice') + status, _, chat = call('POST', '/api/v1/chats', {'title': 't\u0000', 'member_ids': [alice['id']]}, alice['token']) + self.assertEqual((status, chat['title']), (201, 't\u0000')) + path = '/api/v1/chats/%s/messages' % chat['id'] + status, _, message = call('POST', path, {'text': 'a\u0000b'}, alice['token']) + self.assertEqual((status, message['text']), (201, 'a\u0000b')) + self.assertEqual(call('GET', path, token=alice['token'])[2]['items'][0]['text'], 'a\u0000b') + self.assertEqual(call('GET', '/api/v1/chats/' + chat['id'], token=alice['token'])[2]['title'], 't\u0000') + + def test_lone_surrogate_is_rejected(self): + """Одиночный суррогат не кодируется в UTF-8 -> 400, а не 500""" + alice = new_user('alice') + chat = new_chat(alice) + self.assertError(call('POST', '/api/v1/chats/%s/messages' % chat['id'], raw=b'{"text": "a\\ud800b"}', + token=alice['token']), 400, 'invalid_request') + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..7b2397bc --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,192 @@ +import hashlib +import http.client +import unittest +import urllib.parse +import uuid + +from helpers import BASE_URL, call, new_chat, new_user +from imagegen import SAMPLE_JPG, SAMPLE_PNG, bomb, broken_crc, garbage_idat, noise_png, short_idat + +MAX = 10 * 1024 * 1024 + + +def upload(user, chat, data, content_type='image/png', filename='photo.png', token=True): + query = '' if filename is None else '?filename=' + urllib.parse.quote(filename, safe='') + return call('POST', '/api/v1/chats/%s/attachments%s' % (chat['id'], query), raw=data, + token=user['token'] if token else None, headers={'Content-Type': content_type}) + + +def download(user, attachment_id): + target = urllib.parse.urlsplit(BASE_URL) + connection = http.client.HTTPConnection(target.hostname, target.port, timeout=30) + connection.request('GET', '/api/v1/attachments/%s/content' % attachment_id, + headers={'Authorization': 'Bearer ' + user['token']}) + response = connection.getresponse() + body = response.read() + connection.close() + return response.status, response.headers, body + + +def send(user, chat, body, key=None): + return call('POST', '/api/v1/chats/%s/messages' % chat['id'], body, user['token'], + headers={'Idempotency-Key': key or uuid.uuid4().hex}) + + +class UploadTest(unittest.TestCase): + def setUp(self): + self.alice, self.bob, self.eve = new_user('alice'), new_user('bob'), new_user('eve') + self.chat = new_chat(self.alice, self.bob) + + def assertCode(self, result, status, code=None): + self.assertEqual(result[0], status, result[2]) + if code: + self.assertEqual(result[2]['error']['code'], code) + + def test_png_and_jpeg_are_stored_and_downloaded_byte_exact(self): + """PNG и JPEG → 201 ready; второй участник скачивает те же байты с верным Content-Type""" + for data, content_type in ((SAMPLE_PNG, 'image/png'), (SAMPLE_JPG, 'image/jpeg')): + with self.subTest(content_type=content_type): + status, _, att = upload(self.alice, self.chat, data, content_type) + self.assertEqual(status, 201) + self.assertEqual((att['status'], att['variants'], att['error_code']), ('ready', [], None)) + self.assertEqual((att['content_type'], att['size_bytes'], att['sha256']), + (content_type, len(data), hashlib.sha256(data).hexdigest())) + self.assertEqual((att['chat_id'], att['owner_id']), (self.chat['id'], self.alice['id'])) + self.assertEqual(call('GET', '/api/v1/attachments/' + att['id'], token=self.bob['token'])[2], att) + status, headers, body = download(self.bob, att['id']) + self.assertEqual((status, headers.get_content_type(), body), (200, content_type, data)) + self.assertEqual(int(headers['Content-Length']), len(data)) + + def test_download_through_every_replica(self): + """Файл скачивается через любую API-реплику (байты не лежат локально у загрузившей)""" + att = upload(self.alice, self.chat, SAMPLE_PNG)[2] + replicas = set() + for _ in range(6): + status, headers, body = download(self.bob, att['id']) + self.assertEqual((status, body), (200, SAMPLE_PNG)) + replicas.add(headers['X-Instance-Id']) + self.assertGreaterEqual(len(replicas), 2) + + def test_filename_is_display_only_and_validated(self): + """Unicode-имя сохраняется как есть; нет имени, пустое, /, \\, NUL, управляющий символ, >255 → 400""" + status, _, att = upload(self.alice, self.chat, SAMPLE_PNG, filename='фото отпуска 😀.png') + self.assertEqual((status, att['filename']), (201, 'фото отпуска 😀.png')) + for name in (None, '', ' ', '../etc/passwd', 'a\\b.png', 'a\x00b.png', 'bell\x07.png', 'x' * 256): + with self.subTest(name=name): + self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG, filename=name), 400, 'invalid_request') + + def test_wrong_declared_type(self): + """Content-Type не image/png|jpeg (octet-stream, multipart, gif) → 415""" + for content_type in ('application/octet-stream', 'multipart/form-data; boundary=x', 'image/gif', 'text/plain'): + with self.subTest(content_type=content_type): + self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG, content_type), 415, 'unsupported_media_type') + + def test_mime_spoofing(self): + """Подмена MIME: PNG как jpeg, JPEG как png, текст и GIF под видом png → 415""" + for data, content_type in ((SAMPLE_PNG, 'image/jpeg'), (SAMPLE_JPG, 'image/png'), + (b'hello, not an image', 'image/png'), (b'GIF89a' + bytes(100), 'image/png'), + (b'', 'image/png')): + with self.subTest(content_type=content_type, head=data[:6]): + self.assertCode(upload(self.alice, self.chat, data, content_type), 415, 'unsupported_media_type') + + def test_corrupted_images(self): + """Обрезанный PNG, битый CRC, мусор или нехватка пикселей в IDAT при верных CRC, обрезанный JPEG → 415""" + for data, content_type in ((SAMPLE_PNG[:5000], 'image/png'), (broken_crc(SAMPLE_PNG), 'image/png'), + (garbage_idat(SAMPLE_PNG), 'image/png'), (short_idat(SAMPLE_PNG), 'image/png'), + (SAMPLE_JPG[:3000], 'image/jpeg')): + with self.subTest(size=len(data), content_type=content_type): + self.assertCode(upload(self.alice, self.chat, data, content_type), 415, 'unsupported_media_type') + + def test_too_large_file(self): + """10 MiB + 1 байт → 413 (и по Content-Length, и chunked без длины); файл ~10 MiB принимается""" + self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG + bytes(MAX + 1 - len(SAMPLE_PNG))), 413, 'payload_too_large') + target = urllib.parse.urlsplit(BASE_URL) + connection = http.client.HTTPConnection(target.hostname, target.port, timeout=60) + chunks = (SAMPLE_PNG if i == 0 else bytes(1024 * 1024) for i in range(12)) + connection.request('POST', '/api/v1/chats/%s/attachments?filename=big.png' % self.chat['id'], body=chunks, + headers={'Authorization': 'Bearer ' + self.alice['token'], 'Content-Type': 'image/png'}, + encode_chunked=True) + self.assertEqual(connection.getresponse().status, 413) + connection.close() + big = noise_png(MAX - 50000) + self.assertLessEqual(len(big), MAX) + status, _, att = upload(self.alice, self.chat, big) + self.assertEqual((status, att['size_bytes']), (201, len(big))) + self.assertEqual(download(self.bob, att['id'])[2], big) + + def test_decompression_bomb(self): + """PNG 10000×5000 (50 Мп) весом в десятки КБ → 413 без распаковки""" + data = bomb() + self.assertLess(len(data), 1024 * 1024) + self.assertCode(upload(self.alice, self.chat, data), 413, 'payload_too_large') + + def test_access_rules(self): + """Без токена 401; посторонний не может загрузить в чужой чат и не видит метаданные/байты (404)""" + self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG, token=False), 401, 'unauthorized') + self.assertCode(upload(self.eve, self.chat, SAMPLE_PNG), 404, 'not_found') + self.assertCode(upload(self.alice, {'id': str(uuid.uuid4())}, SAMPLE_PNG), 404, 'not_found') + att = upload(self.alice, self.chat, SAMPLE_PNG)[2] + self.assertCode(call('GET', '/api/v1/attachments/' + att['id'], token=self.eve['token']), 404, 'not_found') + self.assertEqual(download(self.eve, att['id'])[0], 404) + self.assertCode(call('GET', '/api/v1/attachments/' + str(uuid.uuid4()), token=self.bob['token']), 404) + self.assertCode(call('GET', '/api/v1/attachments/not-a-uuid', token=self.bob['token']), 400) + self.assertCode(call('GET', '/api/v1/attachments/%s/content?variant=thumbnail' % att['id'], + token=self.bob['token']), 400) + + +class MessageAttachmentsTest(unittest.TestCase): + def setUp(self): + self.alice, self.bob = new_user('alice'), new_user('bob') + self.chat = new_chat(self.alice, self.bob) + self.other_chat = new_chat(self.alice, title='Другой') + self.att = upload(self.alice, self.chat, SAMPLE_PNG)[2] + + def history(self): + return call('GET', '/api/v1/chats/%s/messages?limit=100' % self.chat['id'], token=self.bob['token'])[2]['items'] + + def test_message_with_attachment_only(self): + """text="" + одно вложение → 201; attachment_ids видны в истории; обычное сообщение получает []""" + status, _, message = send(self.alice, self.chat, {'text': '', 'attachment_ids': [self.att['id']]}) + self.assertEqual((status, message['attachment_ids'], message['text']), (201, [self.att['id']], '')) + plain = send(self.alice, self.chat, {'text': 'без вложений'})[2] + self.assertEqual(plain['attachment_ids'], []) + self.assertEqual([m['attachment_ids'] for m in self.history()], [[self.att['id']], []]) + + def test_other_member_can_reference_attachment(self): + """Боб прикрепляет вложение, загруженное Алисой в этот же чат""" + status, _, message = send(self.bob, self.chat, {'text': 'смотри', 'attachment_ids': [self.att['id']]}) + self.assertEqual((status, message['attachment_ids']), (201, [self.att['id']])) + + def test_foreign_or_unknown_attachment(self): + """Вложение другого чата или несуществующее → 404, сообщение не создаётся""" + foreign = upload(self.alice, self.other_chat, SAMPLE_PNG)[2] + for ids in ([foreign['id']], [str(uuid.uuid4())], [self.att['id'], foreign['id']]): + with self.subTest(ids=ids): + self.assertEqual(send(self.alice, self.chat, {'text': 'x', 'attachment_ids': ids})[0], 404) + self.assertEqual(self.history(), []) + + def test_invalid_attachment_lists(self): + """Повтор id, 11 вложений, не-UUID, пустой текст без вложений, пробелы без вложений → 400""" + for body in ({'text': 'x', 'attachment_ids': [self.att['id'], self.att['id']]}, + {'text': 'x', 'attachment_ids': [str(uuid.uuid4()) for _ in range(11)]}, + {'text': 'x', 'attachment_ids': ['nope']}, + {'text': ''}, {'text': ' '}, {'text': '', 'attachment_ids': []}): + with self.subTest(body=body): + self.assertEqual(send(self.alice, self.chat, body)[0], 400) + + def test_idempotency_includes_attachments(self): + """Тот же ключ: те же вложения → тот же Message; другие вложения → 409; нет поля = [] для текста""" + key = uuid.uuid4().hex + first = send(self.alice, self.chat, {'text': 'фото', 'attachment_ids': [self.att['id']]}, key) + again = send(self.alice, self.chat, {'attachment_ids': [self.att['id']], 'text': 'фото'}, key) + self.assertEqual((first[0], again[0], first[2]), (201, 201, again[2])) + other = upload(self.alice, self.chat, SAMPLE_JPG, 'image/jpeg', 'b.jpg')[2] + self.assertEqual(send(self.alice, self.chat, {'text': 'фото', 'attachment_ids': [other['id']]}, key)[0], 409) + key = uuid.uuid4().hex + plain = send(self.alice, self.chat, {'text': 'текст'}, key) + same = send(self.alice, self.chat, {'text': 'текст', 'attachment_ids': []}, key) + self.assertEqual(plain[2]['id'], same[2]['id']) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_auth.py b/student-tests/test_auth.py new file mode 100644 index 00000000..3a56ee20 --- /dev/null +++ b/student-tests/test_auth.py @@ -0,0 +1,132 @@ +from datetime import datetime, timezone +import unittest +import uuid + +from helpers import basic, call, login, new_user, register + + +class RegistrationTest(unittest.TestCase): + def body(self, password): + return {'username': 'reg_' + uuid.uuid4().hex[:12], 'display_name': 'Регистрация', 'password': password} + + def test_password_is_not_returned(self): + """Ответ регистрации и /users/me не содержат password/hash""" + status, _, user = call('POST', '/api/v1/users', self.body('Secret-password-1')) + self.assertEqual(status, 201) + self.assertEqual(set(user), {'id', 'username', 'display_name', 'created_at'}) + + def test_password_length_limits(self): + """Пароль 12..128 символов; 11 и 129 -> 400; эмодзи считается одним символом""" + self.assertEqual(call('POST', '/api/v1/users', self.body('x' * 11))[0], 400) + self.assertEqual(call('POST', '/api/v1/users', self.body('x' * 129))[0], 400) + self.assertEqual(call('POST', '/api/v1/users', self.body('x' * 128))[0], 201) + self.assertEqual(call('POST', '/api/v1/users', self.body('🔑' * 12))[0], 201) + + def test_password_required_and_not_blank(self): + """Без password или из одних пробелов -> 400""" + body = self.body('x') + del body['password'] + self.assertEqual(call('POST', '/api/v1/users', body)[0], 400) + self.assertEqual(call('POST', '/api/v1/users', self.body(' ' * 20))[0], 400) + + +class LoginTest(unittest.TestCase): + def setUp(self): + self.user, self.password = register('login') + + def assertBasicChallenge(self, result): + status, headers, body = result + self.assertEqual(status, 401) + self.assertEqual(headers['WWW-Authenticate'], 'Basic realm="messenger", charset="UTF-8"') + self.assertEqual(body['error']['code'], 'unauthorized') + return body['error']['message'] + + def test_login_returns_opaque_session(self): + """Вход: 201, token_type=Bearer, токен случайный и не содержит username/UUID""" + first = login(self.user['username'], self.password) + second = login(self.user['username'], self.password) + self.assertEqual(first['token_type'], 'Bearer') + self.assertEqual(first['user'], self.user) + self.assertGreaterEqual(len(first['token']), 43) + self.assertNotEqual(first['token'], second['token']) + self.assertNotIn(self.user['username'], first['token']) + self.assertNotIn(self.user['id'], first['token']) + expires = datetime.fromisoformat(first['expires_at'].replace('Z', '+00:00')) + self.assertGreater(expires, datetime.now(timezone.utc)) + + def test_wrong_password_and_unknown_user_look_the_same(self): + """Неверный пароль и несуществующий пользователь -> одинаковый 401 с Basic challenge""" + wrong = self.assertBasicChallenge(call('POST', '/api/v1/auth/sessions', + headers=basic(self.user['username'], 'Wrong-password-1'))) + unknown = self.assertBasicChallenge(call('POST', '/api/v1/auth/sessions', + headers=basic('nobody_' + uuid.uuid4().hex[:8], 'Wrong-password-1'))) + self.assertEqual(wrong, unknown) + + def test_malformed_basic(self): + """Нет заголовка, не base64, нет двоеточия, Bearer вместо Basic -> 401 Basic""" + for header in (None, 'Basic', 'Basic !!!not-base64!!!', 'Basic ' + 'bm9jb2xvbg==', 'Bearer abc'): + with self.subTest(header=header): + headers = {'Authorization': header} if header else {} + self.assertBasicChallenge(call('POST', '/api/v1/auth/sessions', headers=headers)) + + def test_password_with_colon_and_unicode(self): + """Двоеточие и не-ASCII в пароле допустимы: делим только по первому ':'""" + user, password = register('colon', password='пароль:с:двоеточием🔑') + self.assertEqual(login(user['username'], password)['user']['id'], user['id']) + + def test_login_rejects_body(self): + """POST /auth/sessions принимает только заголовок, JSON body -> 400""" + status = call('POST', '/api/v1/auth/sessions', {'username': self.user['username']}, + headers=basic(self.user['username'], self.password))[0] + self.assertEqual(status, 400) + + +class BearerTest(unittest.TestCase): + def setUp(self): + self.alice = new_user('alice') + self.eve = new_user('eve') + + def assertBearerChallenge(self, headers): + status, response_headers, _ = call('GET', '/api/v1/users/me', headers=headers) + self.assertEqual(status, 401) + self.assertEqual(response_headers['WWW-Authenticate'], 'Bearer realm="messenger"') + + def test_only_bearer_is_accepted(self): + """Нет заголовка, Basic, X-User-Id, мусорный токен -> 401 Bearer""" + self.assertBearerChallenge({}) + self.assertBearerChallenge(basic(self.alice['username'], self.alice['password'])) + self.assertBearerChallenge({'X-User-Id': self.alice['id']}) + self.assertBearerChallenge({'Authorization': 'Bearer ' + uuid.uuid4().hex}) + self.assertBearerChallenge({'Authorization': 'Bearer'}) + + def test_authentication_before_validation(self): + """Аноним с битым JSON на защищённом endpoint получает 401, а не 400; с сессией — 400""" + self.assertEqual(call('POST', '/api/v1/chats', raw=b'{bad')[0], 401) + self.assertEqual(call('POST', '/api/v1/chats', raw=b'{bad', token=self.alice['token'])[0], 400) + self.assertEqual(call('POST', '/api/v1/users', raw=b'{bad')[0], 400) + + def test_scheme_is_case_insensitive(self): + """Схема аутентификации нечувствительна к регистру (RFC 9110)""" + status, _, me = call('GET', '/api/v1/users/me', headers={'Authorization': 'bearer ' + self.alice['token']}) + self.assertEqual((status, me['id']), (200, self.alice['id'])) + + def test_x_user_id_does_not_override_session(self): + """X-User-Id игнорируется: владелец определяется только токеном""" + status, _, me = call('GET', '/api/v1/users/me', token=self.alice['token'], headers={'X-User-Id': self.eve['id']}) + self.assertEqual((status, me['id']), (200, self.alice['id'])) + + +class LogoutTest(unittest.TestCase): + def test_logout_revokes_only_current_session(self): + """Logout -> 204 без тела; этот токен -> 401, вторая сессия того же пользователя работает""" + alice = new_user('alice') + other = login(alice['username'], alice['password'])['token'] + status, _, body = call('DELETE', '/api/v1/auth/sessions/current', token=alice['token']) + self.assertEqual((status, body), (204, None)) + self.assertEqual(call('GET', '/api/v1/users/me', token=alice['token'])[0], 401) + self.assertEqual(call('DELETE', '/api/v1/auth/sessions/current', token=alice['token'])[0], 401) + self.assertEqual(call('GET', '/api/v1/users/me', token=other)[0], 200) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_concurrency.py b/student-tests/test_concurrency.py new file mode 100644 index 00000000..4d258446 --- /dev/null +++ b/student-tests/test_concurrency.py @@ -0,0 +1,68 @@ +from concurrent.futures import ThreadPoolExecutor +import threading +import unittest +import uuid + +from helpers import basic, call, new_chat, new_user, read_all_messages + +CLIENTS = 10 +REGISTRATIONS = 20 +PER_CLIENT = 10 + + +class ConcurrencyTest(unittest.TestCase): + def test_100_messages_from_10_clients(self): + """100 параллельных сообщений от 10 клиентов -> ровно 100 уникальных в истории""" + users = [new_user('client%d' % i) for i in range(CLIENTS)] + chat = new_chat(users[0], *users[1:], title='Нагрузка') + path = '/api/v1/chats/%s/messages' % chat['id'] + barrier = threading.Barrier(CLIENTS) + + def client(index): + barrier.wait() + results = [] + for n in range(PER_CLIENT): + status, _, message = call('POST', path, {'text': 'client %d msg %d' % (index, n)}, users[index]['token']) + results.append((status, message)) + return results + + with ThreadPoolExecutor(max_workers=CLIENTS) as pool: + responses = [r for batch in pool.map(client, range(CLIENTS)) for r in batch] + + self.assertEqual([s for s, _ in responses], [201] * CLIENTS * PER_CLIENT) + sent_ids = {m['id'] for _, m in responses} + items, pages = read_all_messages(chat['id'], users[0]['token'], limit=30) + read_ids = [m['id'] for m in items] + keys = [(m['created_at'], m['id']) for m in items] + + self.assertEqual(len(sent_ids), 100, 'сервер выдал повторяющиеся id') + self.assertEqual(len(read_ids), 100, 'в истории не ровно 100 сообщений') + self.assertEqual(set(read_ids), sent_ids, 'история не совпадает с подтверждёнными сообщениями') + self.assertEqual(len({m['text'] for m in items}), 100, 'потерялся или задвоился текст') + self.assertEqual(keys, sorted(keys), 'история не упорядочена по (created_at, id)') + print('\n отправлено 201: %d, уникальных id: %d, прочитано: %d за %d страниц(ы)' + % (len(responses), len(sent_ids), len(read_ids), pages)) + + def test_parallel_registration_of_same_username(self): + """20 одновременных регистраций одного username -> ровно один 201, остальные 409; вход только с паролем победителя""" + username = 'race_' + uuid.uuid4().hex[:12] + barrier = threading.Barrier(REGISTRATIONS) + + def register(index): + password = 'Race-password-%02d' % index + barrier.wait() + status = call('POST', '/api/v1/users', {'username': username, 'display_name': 'Гонка', 'password': password})[0] + return status, password + + with ThreadPoolExecutor(max_workers=REGISTRATIONS) as pool: + results = list(pool.map(register, range(REGISTRATIONS))) + statuses = sorted(status for status, _ in results) + self.assertEqual(statuses, [201] + [409] * (REGISTRATIONS - 1)) + winner = [password for status, password in results if status == 201][0] + loser = [password for status, password in results if status == 409][0] + self.assertEqual(call('POST', '/api/v1/auth/sessions', headers=basic(username, winner))[0], 201) + self.assertEqual(call('POST', '/api/v1/auth/sessions', headers=basic(username, loser))[0], 401) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_db.py b/student-tests/test_db.py new file mode 100644 index 00000000..5997fd83 --- /dev/null +++ b/student-tests/test_db.py @@ -0,0 +1,58 @@ +import hashlib +import time +import unittest +import uuid + +from helpers import call, compose, new_chat, new_user, sql + + +class DatabaseTest(unittest.TestCase): + def test_chat_creation_is_rolled_back(self): + """Неизвестный участник: INSERT chats выполнен, INSERT chat_members упал по FK, всё откатано""" + alice = new_user('alice') + chats_before = int(sql('SELECT count(*) FROM chats')) + members_before = int(sql('SELECT count(*) FROM chat_members')) + ghost = str(uuid.uuid4()) + since = sql("SELECT to_char(now() - interval '1 second', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')") + status, _, body = call('POST', '/api/v1/chats', {'title': 'Откат', 'member_ids': [alice['id'], ghost]}, + alice['token']) + self.assertEqual(status, 400, body) + time.sleep(0.5) + log = compose('logs', '--no-log-prefix', '--since', since, 'postgres') + self.assertIn('violates foreign key constraint', log) + self.assertIn(ghost, log) + self.assertEqual(int(sql('SELECT count(*) FROM chats')), chats_before) + self.assertEqual(int(sql('SELECT count(*) FROM chat_members')), members_before) + + def test_secrets_are_not_stored_in_plaintext(self): + """В users лежит argon2id-хеш, в sessions — sha256 токена, не сам токен""" + alice = new_user('alice') + password_hash = sql("SELECT password_hash FROM users WHERE id = '%s'" % alice['id']) + self.assertTrue(password_hash.startswith('$argon2id$'), password_hash[:10]) + self.assertNotIn(alice['password'], password_hash) + digest = hashlib.sha256(alice['token'].encode()).hexdigest() + self.assertEqual(sql("SELECT count(*) FROM sessions WHERE token_hash = '\\x%s'" % digest), '1') + dump = sql("SELECT string_agg(encode(token_hash, 'escape'), '') FROM sessions WHERE user_id = '%s'" % alice['id']) + self.assertNotIn(alice['token'], dump) + + def test_migrations_are_idempotent(self): + """Повторный запуск миграций ничего не меняет и завершается успешно""" + before = sql('SELECT version, applied_at FROM schema_migrations ORDER BY version') + users = sql('SELECT count(*) FROM users') + output = compose('run', '--rm', '--no-deps', 'migrate') + self.assertIn('skip 0001_init: already applied', output) + self.assertEqual(sql('SELECT version, applied_at FROM schema_migrations ORDER BY version'), before) + self.assertEqual(sql('SELECT count(*) FROM users'), users) + + def test_history_query_uses_index(self): + """Запрос страницы истории идёт по индексу (chat_id, created_at, id), без Sort""" + alice = new_user('alice') + chat = new_chat(alice) + plan = sql("SET enable_seqscan = off; SET enable_bitmapscan = off; EXPLAIN SELECT id FROM messages WHERE chat_id = '%s' " + "ORDER BY created_at, id LIMIT 51" % chat['id']) + self.assertIn('messages_chat_history_idx', plan) + self.assertNotIn('Sort', plan) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..4ed9185b --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,96 @@ +from concurrent.futures import ThreadPoolExecutor +import threading +import unittest +import uuid + +from helpers import call, new_chat, new_user, read_all_messages + +PARALLEL = 20 + + +def post(chat, user, key, body=None, raw=None): + return call('POST', '/api/v1/chats/%s/messages' % chat['id'], body, user['token'], raw=raw, + headers={'Idempotency-Key': key}) + + +class IdempotencyTest(unittest.TestCase): + def setUp(self): + self.alice, self.bob = new_user('alice'), new_user('bob') + self.chat = new_chat(self.alice, self.bob) + + def history(self): + return read_all_messages(self.chat['id'], self.alice['token'], limit=100)[0] + + def test_replay_returns_same_message(self): + """Повтор того же запроса с тем же ключом -> тот же 201 и тот же Message, дубля нет""" + key = uuid.uuid4().hex + first = post(self.chat, self.alice, key, {'text': 'один раз'}) + second = post(self.chat, self.alice, key, {'text': 'один раз'}) + self.assertEqual((first[0], second[0]), (201, 201)) + self.assertEqual(first[2], second[2]) + self.assertEqual([m['id'] for m in self.history()], [first[2]['id']]) + + def test_json_formatting_does_not_matter(self): + """Порядок/пробелы в JSON семантически равны: повтор с другим форматированием -> тот же Message""" + key = uuid.uuid4().hex + first = post(self.chat, self.alice, key, raw='{"text":"формат"}'.encode()) + second = post(self.chat, self.alice, key, raw='{ "text" : "формат" }'.encode()) + self.assertEqual((first[0], second[0]), (201, 201)) + self.assertEqual(first[2]['id'], second[2]['id']) + + def test_other_body_with_same_key_conflicts(self): + """Тот же ключ с другим телом -> 409, второе сообщение не создаётся""" + key = uuid.uuid4().hex + self.assertEqual(post(self.chat, self.alice, key, {'text': 'первое'})[0], 201) + status, _, body = post(self.chat, self.alice, key, {'text': 'другое'}) + self.assertEqual((status, body['error']['code']), (409, 'conflict')) + self.assertEqual(len(self.history()), 1) + + def test_missing_or_invalid_key(self): + """Нет ключа, пустой, недопустимые символы, длиннее 128 -> 400""" + for key in (None, '', 'bad key', 'bad/key', 'café', 'x' * 129): + with self.subTest(key=key): + self.assertEqual(post(self.chat, self.alice, key, {'text': 'x'})[0], 400) + self.assertEqual(post(self.chat, self.alice, 'A.b_c:d-1' + 'x' * 119, {'text': 'граница'})[0], 201) + + def test_key_scope_is_user_and_chat(self): + """Одинаковый ключ у разных пользователей и в разных чатах — независимые сообщения""" + key = uuid.uuid4().hex + other_chat = new_chat(self.alice, title='Другой') + a = post(self.chat, self.alice, key, {'text': 'от Алисы'}) + b = post(self.chat, self.bob, key, {'text': 'от Боба'}) + c = post(other_chat, self.alice, key, {'text': 'от Алисы'}) + self.assertEqual([a[0], b[0], c[0]], [201, 201, 201]) + self.assertEqual(len({a[2]['id'], b[2]['id'], c[2]['id']}), 3) + + def test_replay_rechecks_session_and_access(self): + """Повтор снова проверяет сессию и доступ: после logout -> 401, посторонний с тем же ключом -> 404""" + key = uuid.uuid4().hex + self.assertEqual(post(self.chat, self.alice, key, {'text': 'до выхода'})[0], 201) + eve = new_user('eve') + self.assertEqual(post(self.chat, eve, key, {'text': 'до выхода'})[0], 404) + call('DELETE', '/api/v1/auth/sessions/current', token=self.alice['token']) + self.assertEqual(post(self.chat, self.alice, key, {'text': 'до выхода'})[0], 401) + + def test_parallel_retries_through_replicas_create_one_message(self): + """20 одновременных POST с одним ключом через LB (разные реплики) -> ровно одно сообщение, все 201 с одним id""" + key = uuid.uuid4().hex + barrier = threading.Barrier(PARALLEL) + + def send(_): + barrier.wait() + status, headers, body = post(self.chat, self.alice, key, {'text': 'конкурентный повтор'}) + return status, headers['X-Instance-Id'], body + + with ThreadPoolExecutor(max_workers=PARALLEL) as pool: + results = list(pool.map(send, range(PARALLEL))) + self.assertEqual({status for status, _, _ in results}, {201}) + self.assertEqual(len({body['id'] for _, _, body in results}), 1) + self.assertGreaterEqual(len({instance for _, instance, _ in results}), 2, 'запросы не попали на разные реплики') + self.assertEqual(len(self.history()), 1) + print('\n 20 повторов: реплики %s, сообщений в истории: %d' + % (sorted({instance for _, instance, _ in results}), len(self.history()))) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_replicas.py b/student-tests/test_replicas.py new file mode 100644 index 00000000..84149108 --- /dev/null +++ b/student-tests/test_replicas.py @@ -0,0 +1,41 @@ +import unittest +import uuid + +from helpers import call, new_chat, new_user + + +class ReplicasTest(unittest.TestCase): + def test_one_session_is_served_by_several_replicas(self): + """Один токен без повторного входа обслуживается разными репликами (X-Instance-Id), все ответы 200""" + alice = new_user('alice') + results = [call('GET', '/api/v1/users/me', token=alice['token']) for _ in range(10)] + self.assertEqual({status for status, _, _ in results}, {200}) + self.assertGreaterEqual(len({headers['X-Instance-Id'] for _, headers, _ in results}), 2) + + def test_write_on_one_replica_is_read_on_another(self): + """Сообщение, подтверждённое одной репликой, сразу видно при чтении через другую""" + alice = new_user('alice') + chat = new_chat(alice) + path = '/api/v1/chats/%s/messages' % chat['id'] + for i in range(6): + _, written_by, message = call('POST', path, {'text': 'm%d' % i}, alice['token']) + status, read_by, page = call('GET', path + '?limit=100', token=alice['token']) + self.assertEqual(status, 200) + self.assertIn(message['id'], [m['id'] for m in page['items']], + 'запись через %s не видна через %s' % (written_by['X-Instance-Id'], read_by['X-Instance-Id'])) + + def test_every_api_response_has_instance_id(self): + """X-Instance-Id есть и в успешных ответах, и в ошибках API, и в health""" + alice = new_user('alice') + for method, path, token, expected in ( + ('GET', '/health/live', None, 200), ('GET', '/health/ready', None, 200), + ('GET', '/api/v1/users/me', None, 401), ('GET', '/api/v1/users/me', alice['token'], 200), + ('GET', '/api/v1/chats/' + str(uuid.uuid4()), alice['token'], 404), ('GET', '/nope', None, 404)): + with self.subTest(path=path, expected=expected): + status, headers, _ = call(method, path, token=token) + self.assertEqual(status, expected) + self.assertTrue(headers['X-Instance-Id']) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_storage.py b/student-tests/test_storage.py new file mode 100644 index 00000000..c51278c5 --- /dev/null +++ b/student-tests/test_storage.py @@ -0,0 +1,81 @@ +import subprocess +import unittest + +from helpers import compose, new_chat, new_user, sql +from imagegen import SAMPLE_PNG, noise_png +from test_attachments import upload + +LIMITED_CLIENT = ''' +import json +from botocore.exceptions import ClientError +from config import settings +from s3 import client +results = {} +for name, action in ( + ("put_get_delete_own_bucket", lambda: (client.put_object(Bucket=settings.s3_bucket, Key="probe/x", Body=b"x"), + client.get_object(Bucket=settings.s3_bucket, Key="probe/x"), + client.delete_object(Bucket=settings.s3_bucket, Key="probe/x"))), + ("list_bucket_objects", lambda: client.list_objects_v2(Bucket=settings.s3_bucket)), + ("create_bucket", lambda: client.create_bucket(Bucket="intruder")), + ("delete_bucket", lambda: client.delete_bucket(Bucket=settings.s3_bucket)), + ("list_all_buckets", lambda: client.list_buckets()["Buckets"])): + try: + value = action() + results[name] = "allowed" if name != "list_all_buckets" else "allowed:%d" % len(value) + except ClientError as error: + results[name] = error.response["Error"]["Code"] +print(json.dumps(results)) +''' + + +class StorageTest(unittest.TestCase): + def setUp(self): + self.alice = new_user('alice') + self.chat = new_chat(self.alice) + + def object_key(self, attachment_id): + return sql("SELECT object_key FROM attachments WHERE id = '%s'" % attachment_id) + + def test_object_key_is_generated_by_server(self): + """Ключ объекта = attachments/, filename в нём не участвует""" + att = upload(self.alice, self.chat, SAMPLE_PNG, filename='мой-файл.png')[2] + self.assertEqual(self.object_key(att['id']), 'attachments/' + att['id']) + + def test_bucket_is_private_outside_api(self): + """Анонимный GET объекта и листинг bucket напрямую в S3 (из сети backend) → 403""" + att = upload(self.alice, self.chat, SAMPLE_PNG)[2] + for path in ('/messenger-attachments/' + self.object_key(att['id']), '/messenger-attachments'): + with self.subTest(path=path): + result = compose('exec', '-T', 'lb', 'wget', '-S', '-O', '/dev/null', 'http://s3:8333' + path, check=False) + self.assertNotIn('200 OK', result) + output = compose('exec', '-T', 'lb', 'sh', '-c', 'wget -S -O /dev/null http://s3:8333%s 2>&1; true' % path) + self.assertIn('403', output) + + def test_api_credentials_are_limited(self): + """Ключи API: свои объекты читать/писать/удалять можно; листинг, создание и удаление bucket — AccessDenied""" + import json + results = json.loads(compose('exec', '-T', 'api', 'python', '-c', 'import sys; sys.path.insert(0, "src")\n' + LIMITED_CLIENT)) + self.assertEqual(results['put_get_delete_own_bucket'], 'allowed') + for action in ('list_bucket_objects', 'create_bucket', 'delete_bucket'): + self.assertEqual(results[action], 'AccessDenied', results) + self.assertNotEqual(results['list_all_buckets'], 'allowed:1', 'API sees the bucket list') + + def test_bootstrap_is_idempotent(self): + """Повторный s3-init: код 0, bucket уже есть, приватность подтверждена""" + output = compose('run', '--rm', '--no-deps', 's3-init') + self.assertIn('already exists, nothing to do', output) + self.assertIn('is private', output) + + def test_no_original_bytes_in_database_or_api_container(self): + """В attachments единственный bytea — sha256 (байтов файла в БД нет); после загрузки 2 МБ в /tmp реплик API пусто""" + binary = sql("SELECT string_agg(column_name, ',') FROM information_schema.columns " + "WHERE table_name = 'attachments' AND data_type = 'bytea'") + self.assertEqual(binary, 'sha256') + self.assertEqual(upload(self.alice, self.chat, noise_png(2 * 1024 * 1024))[0], 201) + for container in compose('ps', '-q', 'api').split(): + files = subprocess.run(['docker', 'exec', container, 'sh', '-c', 'find /tmp -type f | wc -l'], + capture_output=True, text=True).stdout.strip() + self.assertEqual(files, '0', container[:12]) + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/upload_memory.py b/student-tests/upload_memory.py new file mode 100644 index 00000000..c5a0e8cb --- /dev/null +++ b/student-tests/upload_memory.py @@ -0,0 +1,118 @@ +"""Usage (стенд запущен; на время эксперимента лучше остановить другие стенды): + python student-tests/upload_memory.py --output evidence/upload-memory.json + +N одновременных загрузок PNG ~10 MiB через LB при двух настройках буфера тела запроса +(UPLOAD_MEMORY_BYTES = 1 MiB в RAM, дальше временный файл — рабочая настройка; и 16 MiB — весь файл в RAM). +Перед каждым прогоном реплики перезапускаются (memory.peak cgroup считается с момента старта контейнера); +во время прогона каждые 0,1 с снимаются memory.current и anon/file из memory.stat каждой реплики. +""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import http.client +import json +import os +import subprocess +import threading +import time +import urllib.parse + +from helpers import BASE_URL, api_containers, compose, new_chat, new_user, wait_in_rotation, wait_ready +from imagegen import noise_png + +MIB = 1024 * 1024 +SAMPLER = ('while true; do echo "current $(cat /sys/fs/cgroup/memory.current)"; ' + 'grep -E "^(anon|file) " /sys/fs/cgroup/memory.stat; sleep 0.1; done') + + +class Sampler: + def __init__(self, container): + self.container, self.peaks = container, {'current': 0, 'anon': 0, 'file': 0} + self.process = subprocess.Popen(['docker', 'exec', container, 'sh', '-c', SAMPLER], + stdout=subprocess.PIPE, text=True) + self.thread = threading.Thread(target=self.read, daemon=True) + self.thread.start() + + def read(self): + for line in self.process.stdout: + name, _, value = line.partition(' ') + if name in self.peaks and value.strip().isdigit(): + self.peaks[name] = max(self.peaks[name], int(value)) + + def stop(self): + self.process.terminate() + self.process.wait() + return {name: round(value / MIB, 1) for name, value in self.peaks.items()} + + +def cgroup(container, name): + return int(subprocess.run(['docker', 'exec', container, 'cat', '/sys/fs/cgroup/' + name], + capture_output=True, text=True).stdout.strip()) + + +def post(token, chat_id, data): + target = urllib.parse.urlsplit(BASE_URL) + connection = http.client.HTTPConnection(target.hostname, target.port, timeout=120) + started = time.monotonic() + try: + connection.request('POST', '/api/v1/chats/%s/attachments?filename=big.png' % chat_id, body=data, + headers={'Authorization': 'Bearer ' + token, 'Content-Type': 'image/png'}) + status = connection.getresponse().status + except OSError as error: + status = type(error).__name__ + connection.close() + return status, time.monotonic() - started + + +def restart_api(memory_bytes): + env = dict(os.environ, UPLOAD_MEMORY_BYTES=str(memory_bytes)) + compose('up', '-d', '--no-deps', '--no-build', '--force-recreate', 'api', env=env) + time.sleep(3) + wait_ready() + for container in api_containers(): + wait_in_rotation(container) + + +def run(label, memory_bytes, parallel, data): + restart_api(memory_bytes) + alice = new_user('memory') + chat = new_chat(alice) + replicas = api_containers() + idle = {c[:12]: round(cgroup(c, 'memory.current') / MIB, 1) for c in replicas} + samplers = [Sampler(c) for c in replicas] + started = time.monotonic() + with ThreadPoolExecutor(max_workers=parallel) as pool: + results = list(pool.map(lambda _: post(alice['token'], chat['id'], data), range(parallel))) + wall = time.monotonic() - started + time.sleep(0.5) + sampled = {c[:12]: s.stop() for c, s in zip(replicas, samplers)} + peak = {c[:12]: round(cgroup(c, 'memory.peak') / MIB, 1) for c in replicas} + limit = round(cgroup(replicas[0], 'memory.max') / MIB) + statuses = [r[0] for r in results] + summary = {'label': label, 'upload_memory_bytes': memory_bytes, 'parallel': parallel, + 'file_mib': round(len(data) / MIB, 2), 'ok': statuses.count(201), + 'errors': sorted({str(s) for s in statuses if s != 201}), 'wall_s': round(wall, 2), + 'max_request_s': round(max(r[1] for r in results), 2), 'limit_mib': limit, + 'idle_current_mib': idle, 'peak_since_start_mib': peak, 'sampled_peak_mib': sampled} + worst_anon = max(v['anon'] for v in sampled.values()) + print('%-28s ok=%2d/%d wall=%5.1fs peak(memory.peak)=%s anon max=%.1f MiB' + % (label, summary['ok'], parallel, wall, list(peak.values()), worst_anon)) + return summary + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--output') + parser.add_argument('--parallel', default='10,20') + args = parser.parse_args() + data = noise_png(10 * MIB - 50000) + runs = [run('spool 1 MiB, %s parallel' % n, MIB, int(n), data) for n in args.parallel.split(',')] + runs.append(run('all in RAM, %s parallel' % args.parallel.split(',')[-1], 16 * MIB, + int(args.parallel.split(',')[-1]), data)) + restart_api(MIB) + if args.output: + with open(args.output, 'w', encoding='utf-8') as file: + json.dump({'runs': runs}, file, ensure_ascii=False, indent=2) + + +if __name__ == '__main__': + main()