Добавить тесты вложений и хранилища, проверку согласованности и исследование сверщика
This commit is contained in:
@@ -0,0 +1,31 @@
|
||||
name: Application
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
stack:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- run: cp .env.example .env
|
||||
- run: python scripts/init_secrets.py
|
||||
- run: docker compose config --quiet
|
||||
- run: docker compose up -d --build --wait
|
||||
- run: make test BASE_URL=http://127.0.0.1:8080 PYTHON=python
|
||||
- run: make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080 PYTHON=python
|
||||
- run: make student-test BASE_URL=http://127.0.0.1:8080 PYTHON=python
|
||||
- run: python student-tests/expiry_check.py --ttl 2
|
||||
- run: python student-tests/failure_check.py
|
||||
- run: python student-tests/consistency_check.py
|
||||
- run: make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://127.0.0.1:8080 PYTHON=python
|
||||
- run: docker compose start api && sleep 15
|
||||
- run: python student-tests/failover_series.py
|
||||
- run: python student-tests/rolling_update_check.py --no-build
|
||||
- if: failure()
|
||||
run: docker compose ps -a && docker compose logs --no-color
|
||||
- if: always()
|
||||
run: docker compose down -v
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Операции с S3 для внедрения сбоев в треке 2. Запускается внутри контейнера reconcile (ключи сверщика):
|
||||
python research/s3ops.py put SIZE < keys # создать объекты указанного размера
|
||||
python research/s3ops.py delete < keys # удалить объекты
|
||||
python research/s3ops.py exists < keys # вывести число существующих
|
||||
"""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, 'src')
|
||||
|
||||
from botocore.exceptions import ClientError # noqa: E402
|
||||
|
||||
from config import setting, settings # noqa: E402
|
||||
from s3 import make_client # noqa: E402
|
||||
|
||||
client = make_client(setting('S3_RECONCILER_ACCESS_KEY_ID'), setting('S3_RECONCILER_SECRET_ACCESS_KEY'), timeout=30)
|
||||
|
||||
|
||||
def exists(key):
|
||||
try:
|
||||
client.head_object(Bucket=settings.s3_bucket, Key=key)
|
||||
return True
|
||||
except ClientError:
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
command = sys.argv[1]
|
||||
keys = [line.strip() for line in sys.stdin if line.strip()]
|
||||
with ThreadPoolExecutor(max_workers=16) as pool:
|
||||
if command == 'put':
|
||||
body = b'x' * int(sys.argv[2])
|
||||
list(pool.map(lambda k: client.put_object(Bucket=settings.s3_bucket, Key=k, Body=body), keys))
|
||||
print(len(keys))
|
||||
elif command == 'delete':
|
||||
list(pool.map(lambda k: client.delete_object(Bucket=settings.s3_bucket, Key=k), keys))
|
||||
print(len(keys))
|
||||
elif command == 'exists':
|
||||
print(sum(pool.map(exists, keys)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,282 @@
|
||||
"""Трек 2: согласованность PostgreSQL и S3 — исследование сверщика (src/reconcile.py).
|
||||
|
||||
Стенд запущен (docker compose up -d). BASE_URL — адрес LB (по умолчанию http://127.0.0.1:8080).
|
||||
python research/track2.py scenario # H1/H2: внедрённые сбои → dry-run → apply → повторный apply
|
||||
python research/track2.py race # H2: apply в цикле под непрерывными загрузками
|
||||
python research/track2.py cost # H3: время/память обхода при 1k…20k объектов
|
||||
python research/track2.py cleanup # удалить синтетические данные исследования
|
||||
python research/track2.py report # evidence/track2/results.md
|
||||
python research/track2.py all
|
||||
Синтетические строки живут в отдельном чате пользователя track2_*; cleanup удаляет их и их объекты.
|
||||
На время запуска периодический сервис reconciler останавливается и затем запускается снова.
|
||||
"""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'student-tests'))
|
||||
|
||||
from helpers import call, compose, new_chat, new_user, sql # noqa: E402
|
||||
from imagegen import SAMPLE_PNG # noqa: E402
|
||||
from test_attachments import download, upload # noqa: E402
|
||||
|
||||
OUT = ROOT / 'evidence' / 'track2'
|
||||
STATE = OUT / 'state.json'
|
||||
GRACE = 30
|
||||
|
||||
|
||||
def save(name, data):
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
(OUT / name).write_text(json.dumps(data, ensure_ascii=False, indent=2, default=str) + '\n', encoding='utf-8')
|
||||
print('saved evidence/track2/' + name)
|
||||
|
||||
|
||||
def load(name):
|
||||
return json.loads((OUT / name).read_text(encoding='utf-8'))
|
||||
|
||||
|
||||
def s3ops(command, keys, *args):
|
||||
result = subprocess.run(['docker', 'compose', 'run', '--rm', '-T', '-v', str(ROOT / 'research') + ':/app/research:ro',
|
||||
'--entrypoint', 'python', 'reconcile', 'research/s3ops.py', command, *map(str, args)],
|
||||
cwd=ROOT, input='\n'.join(keys), capture_output=True, text=True, encoding='utf-8')
|
||||
if result.returncode:
|
||||
raise SystemExit(result.stderr[-2000:])
|
||||
return int(result.stdout.strip().splitlines()[-1])
|
||||
|
||||
|
||||
def reconcile(*args):
|
||||
return json.loads(compose('run', '--rm', 'reconcile', *map(str, args)))
|
||||
|
||||
|
||||
def research_chat():
|
||||
if STATE.exists():
|
||||
return json.loads(STATE.read_text(encoding='utf-8'))
|
||||
user = new_user('track2')
|
||||
chat = new_chat(user)
|
||||
state = {'user_id': user['id'], 'chat_id': chat['id'], 'token': None}
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
STATE.write_text(json.dumps(state), encoding='utf-8')
|
||||
return state
|
||||
|
||||
|
||||
def insert_rows(state, count, status='ready', age_seconds=0, size=1024):
|
||||
"""Синтетические строки attachments в чате исследования; возвращает ключи объектов."""
|
||||
if count <= 0:
|
||||
return []
|
||||
out = sql("WITH new AS (INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
|
||||
"object_key, status, created_at, updated_at) SELECT id, '%s', '%s', 'synthetic.png', 'image/png', %d, "
|
||||
"decode(repeat('00', 32), 'hex'), 'attachments/' || id, '%s', now() - make_interval(secs => %d), now() "
|
||||
"FROM (SELECT gen_random_uuid() AS id FROM generate_series(1, %d)) g RETURNING object_key) "
|
||||
"SELECT string_agg(object_key, ' ') FROM new"
|
||||
% (state['chat_id'], state['user_id'], size, status, age_seconds, count))
|
||||
return out.split()
|
||||
|
||||
|
||||
def reference(state, keys):
|
||||
"""Привязать вложения к сообщению (используемые вложения)."""
|
||||
message = sql("INSERT INTO messages (chat_id, sender_id, text) VALUES ('%s', '%s', convert_to('track2', 'UTF8')) "
|
||||
"RETURNING id" % (state['chat_id'], state['user_id'])).splitlines()[0]
|
||||
sql("INSERT INTO message_attachments (message_id, attachment_id, position) SELECT '%s', a.id, "
|
||||
"row_number() OVER () FROM attachments a WHERE a.object_key IN (%s)"
|
||||
% (message, ','.join("'%s'" % k for k in keys)))
|
||||
|
||||
|
||||
def orphan_keys(count):
|
||||
return ['attachments/%s' % uuid.uuid4() for _ in range(count)]
|
||||
|
||||
|
||||
def scenario():
|
||||
state = research_chat()
|
||||
real = new_user('track2real')
|
||||
real_chat = new_chat(real)
|
||||
real_att = upload(real, real_chat, SAMPLE_PNG)[2]
|
||||
call('POST', '/api/v1/chats/%s/messages' % real_chat['id'], {'text': '', 'attachment_ids': [real_att['id']]},
|
||||
real['token'])
|
||||
baseline = reconcile('--grace-seconds', GRACE)
|
||||
|
||||
ok = insert_rows(state, 200)
|
||||
s3ops('put', ok, 1024)
|
||||
referenced = ok[:20]
|
||||
reference(state, referenced)
|
||||
missing = ok[20:30] + referenced[:5]
|
||||
s3ops('delete', missing)
|
||||
mismatch = ok[40:50]
|
||||
s3ops('put', mismatch, 2048)
|
||||
stale_with_object = insert_rows(state, 25, 'uploading', 3600)
|
||||
s3ops('put', stale_with_object, 1024)
|
||||
stale_without_object = insert_rows(state, 25, 'uploading', 3600)
|
||||
old_orphans = orphan_keys(25)
|
||||
s3ops('put', old_orphans, 1024)
|
||||
print('waiting %ds so that injected orphans become older than grace…' % (GRACE + 3))
|
||||
time.sleep(GRACE + 3)
|
||||
fresh_uploads = insert_rows(state, 10, 'uploading', 0)
|
||||
s3ops('put', fresh_uploads, 1024)
|
||||
fresh_orphans = orphan_keys(10)
|
||||
s3ops('put', fresh_orphans, 1024)
|
||||
|
||||
expected = {'orphan': 25, 'fresh_orphan': 10, 'stale_upload': 50, 'fresh_upload': 10, 'missing': 15,
|
||||
'size_mismatch': 10}
|
||||
base = baseline['counts']
|
||||
dry = reconcile('--grace-seconds', GRACE)
|
||||
dry_again = reconcile('--grace-seconds', GRACE)
|
||||
applied = reconcile('--apply', '--grace-seconds', GRACE)
|
||||
rerun = reconcile('--apply', '--grace-seconds', GRACE)
|
||||
after = reconcile('--grace-seconds', GRACE)
|
||||
detected = {k: dry['counts'][k] - base.get(k, 0) for k in expected}
|
||||
real_download = download(real, real_att['id'])
|
||||
checks = {
|
||||
'detected_equals_injected': detected == expected,
|
||||
'dry_run_changes_nothing': dry_again['counts'] == dry['counts'] and dry_again['objects_listed'] == dry['objects_listed'],
|
||||
'apply_actions': applied['actions'],
|
||||
'apply_deleted_exactly_old_orphans_and_stale': applied['actions'] == {
|
||||
'orphan_objects_deleted': 25, 'stale_rows_deleted': 50, 'stale_objects_deleted': 25, 'skipped': 0},
|
||||
'rerun_is_noop': not any(rerun['actions'][k] for k in ('orphan_objects_deleted', 'stale_rows_deleted', 'stale_objects_deleted')),
|
||||
'fresh_orphans_kept': s3ops('exists', fresh_orphans) == 10,
|
||||
'fresh_uploads_kept': s3ops('exists', fresh_uploads) == 10 and after['counts']['fresh_upload'] >= 10,
|
||||
'old_orphans_gone': s3ops('exists', old_orphans) == 0,
|
||||
'stale_objects_gone': s3ops('exists', stale_with_object) == 0,
|
||||
'referenced_rows_kept': int(sql("SELECT count(*) FROM attachments WHERE object_key IN (%s)"
|
||||
% ','.join("'%s'" % k for k in referenced))) == 20,
|
||||
'missing_still_reported_not_deleted': after['counts']['missing'] - base.get('missing', 0) == 15,
|
||||
'real_attachment_intact': real_download[0] == 200 and hashlib.sha256(real_download[2]).hexdigest() == real_att['sha256'],
|
||||
}
|
||||
passed = all(v for k, v in checks.items() if k != 'apply_actions')
|
||||
save('scenario.json', {'grace_seconds': GRACE, 'injected': expected, 'baseline_counts': base,
|
||||
'dry_run': dry, 'apply': applied, 'rerun': rerun, 'after': after,
|
||||
'detected': detected, 'checks': checks, 'passed': passed})
|
||||
for name, value in checks.items():
|
||||
print(' %-48s %s' % (name, value))
|
||||
print('PASS' if passed else 'FAIL')
|
||||
|
||||
|
||||
def race(seconds=40):
|
||||
user = new_user('track2race')
|
||||
chat = new_chat(user)
|
||||
uploaded, errors, stop = [], [], threading.Event()
|
||||
|
||||
def uploader():
|
||||
while not stop.is_set():
|
||||
status, _, body = upload(user, chat, SAMPLE_PNG)
|
||||
(uploaded.append(body['id']) if status == 201 else errors.append(status))
|
||||
|
||||
preclean = reconcile('--apply', '--grace-seconds', 0)['actions']
|
||||
threads = [threading.Thread(target=uploader) for _ in range(4)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
runs, started = [], time.monotonic()
|
||||
while time.monotonic() - started < seconds:
|
||||
report = reconcile('--apply', '--grace-seconds', GRACE)
|
||||
runs.append({'counts': report['counts'], 'actions': report['actions'], 'total_s': report['seconds']['total']})
|
||||
stop.set()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
intact = 0
|
||||
for attachment_id in uploaded:
|
||||
status, _, body = download(user, attachment_id)
|
||||
intact += status == 200 and body == SAMPLE_PNG
|
||||
result = {'seconds': seconds, 'grace_seconds': GRACE, 'preclean_before_load': preclean,
|
||||
'reconcile_runs': len(runs), 'uploads_201': len(uploaded),
|
||||
'upload_errors': errors, 'downloaded_intact': intact,
|
||||
'deleted_by_reconciler': sum(r['actions']['orphan_objects_deleted'] + r['actions']['stale_rows_deleted']
|
||||
for r in runs),
|
||||
'fresh_uploads_seen_in_flight': sum(r['counts']['fresh_upload'] for r in runs), 'runs': runs}
|
||||
result['passed'] = intact == len(uploaded) and result['deleted_by_reconciler'] == 0
|
||||
save('race.json', result)
|
||||
print('race: %d uploads, %d intact, reconciler runs %d, deleted %d, in-flight uploads seen %d → %s'
|
||||
% (len(uploaded), intact, len(runs), result['deleted_by_reconciler'], result['fresh_uploads_seen_in_flight'],
|
||||
'PASS' if result['passed'] else 'FAIL'))
|
||||
|
||||
|
||||
def cost(sizes=(1000, 5000, 10000, 20000), repeats=3):
|
||||
state = research_chat()
|
||||
rows = []
|
||||
for target in sizes:
|
||||
current = int(sql("SELECT count(*) FROM attachments"))
|
||||
if target > current:
|
||||
keys = insert_rows(state, target - current)
|
||||
started = time.monotonic()
|
||||
for start in range(0, len(keys), 2000):
|
||||
s3ops('put', keys[start:start + 2000], 256)
|
||||
print(' +%d objects in %.1fs' % (len(keys), time.monotonic() - started))
|
||||
reports = [reconcile('--grace-seconds', GRACE) for _ in range(repeats)]
|
||||
row = {'objects': reports[-1]['objects_listed'], 'rows': reports[-1]['rows_scanned'],
|
||||
'list_pages': reports[-1]['list_pages'],
|
||||
'list_s3_s': statistics.median(r['seconds']['list_s3'] for r in reports),
|
||||
'scan_db_s': statistics.median(r['seconds']['scan_db'] for r in reports),
|
||||
'total_s': statistics.median(r['seconds']['total'] for r in reports),
|
||||
'max_rss_mib': max(r['max_rss_mib'] for r in reports)}
|
||||
row['us_per_object'] = round(row['total_s'] / row['objects'] * 1e6, 1)
|
||||
rows.append(row)
|
||||
print(' %(objects)6d objects: list %(list_s3_s).2fs, db %(scan_db_s).3fs, total %(total_s).2fs, '
|
||||
'rss %(max_rss_mib).0f MiB, %(us_per_object).0f µs/object' % row)
|
||||
save('cost.json', {'repeats': repeats, 'rows': rows})
|
||||
|
||||
|
||||
def cleanup():
|
||||
if not STATE.exists():
|
||||
print('nothing to clean')
|
||||
return
|
||||
state = json.loads(STATE.read_text(encoding='utf-8'))
|
||||
keys = sql("SELECT coalesce(string_agg(object_key, ' '), '') FROM attachments WHERE chat_id = '%s'" % state['chat_id']).split()
|
||||
for start in range(0, len(keys), 2000):
|
||||
s3ops('delete', keys[start:start + 2000])
|
||||
sql("DELETE FROM message_attachments WHERE message_id IN (SELECT id FROM messages WHERE chat_id = '%s')" % state['chat_id'])
|
||||
sql("DELETE FROM messages WHERE chat_id = '%s'" % state['chat_id'])
|
||||
sql("DELETE FROM attachments WHERE chat_id = '%s'" % state['chat_id'])
|
||||
leftovers = reconcile('--apply', '--grace-seconds', 0)
|
||||
STATE.unlink()
|
||||
print('cleanup: %d research objects deleted; reconciler afterwards: %s' % (len(keys), leftovers['counts']))
|
||||
|
||||
|
||||
def report():
|
||||
s, r, c = load('scenario.json'), load('race.json'), load('cost.json')
|
||||
lines = ['# Трек 2: результаты', '', '## H1/H2. Внедрённые сбои (grace %d с)' % s['grace_seconds'], '',
|
||||
'| категория | внедрено | найдено dry-run | действие apply |', '|---|---|---|---|']
|
||||
actions = {'orphan': 'удалено объектов: %d' % s['apply']['actions']['orphan_objects_deleted'],
|
||||
'stale_upload': 'удалено строк: %d, объектов: %d' % (s['apply']['actions']['stale_rows_deleted'],
|
||||
s['apply']['actions']['stale_objects_deleted']),
|
||||
'fresh_orphan': 'не тронуто', 'fresh_upload': 'не тронуто', 'missing': 'только отчёт',
|
||||
'size_mismatch': 'только отчёт'}
|
||||
for name, count in s['injected'].items():
|
||||
lines.append('| %s | %d | %d | %s |' % (name, count, s['detected'][name], actions[name]))
|
||||
lines += ['', '| проверка | результат |', '|---|---|']
|
||||
lines += ['| %s | %s |' % (k, v) for k, v in s['checks'].items() if k != 'apply_actions']
|
||||
lines += ['', '## H2. Сверщик под нагрузкой загрузок', '',
|
||||
'%d с, 4 потока загрузок, apply с grace %d с в цикле: запусков сверщика %d, загрузок 201 — %d, '
|
||||
'скачано без изменений — %d, удалено сверщиком — %d, «свежих» загрузок в полёте увидено — %d.'
|
||||
% (r['seconds'], r['grace_seconds'], r['reconcile_runs'], r['uploads_201'], r['downloaded_intact'],
|
||||
r['deleted_by_reconciler'], r['fresh_uploads_seen_in_flight']),
|
||||
'', '## H3. Стоимость обхода (медиана из %d)' % c['repeats'], '',
|
||||
'| объектов | строк | страниц листинга | листинг S3, с | скан БД, с | всего, с | мкс/объект | RSS, MiB |',
|
||||
'|---|---|---|---|---|---|---|---|']
|
||||
for row in c['rows']:
|
||||
lines.append('| %(objects)d | %(rows)d | %(list_pages)d | %(list_s3_s).2f | %(scan_db_s).3f | %(total_s).2f | '
|
||||
'%(us_per_object).0f | %(max_rss_mib).0f |' % row)
|
||||
(OUT / 'results.md').write_text('\n'.join(lines) + '\n', encoding='utf-8')
|
||||
print('saved evidence/track2/results.md')
|
||||
|
||||
|
||||
COMMANDS = {'scenario': scenario, 'race': race, 'cost': cost, 'cleanup': cleanup, 'report': report}
|
||||
|
||||
if __name__ == '__main__':
|
||||
command = sys.argv[1] if len(sys.argv) > 1 else 'all'
|
||||
# Периодический сервис reconciler (--apply, grace 900 с) удалил бы состаренные синтетические заготовки
|
||||
# посреди эксперимента; на время исследования он остановлен, сверщик запускается явно.
|
||||
paused = 'reconciler' in compose('ps', '--status', 'running', '--services')
|
||||
if paused:
|
||||
compose('stop', 'reconciler')
|
||||
try:
|
||||
for step in (['scenario', 'race', 'cost', 'report', 'cleanup'] if command == 'all' else [command]):
|
||||
print('==', step)
|
||||
COMMANDS[step]()
|
||||
finally:
|
||||
if paused:
|
||||
compose('start', 'reconciler')
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Usage (стенд запущен через docker compose up -d):
|
||||
python student-tests/consistency_check.py
|
||||
|
||||
Проверяет, что сбои при загрузке не оставляют вечных неучтённых данных:
|
||||
1) клиент обрывает загрузку посреди тела → ни строки, ни объекта;
|
||||
2) S3 остановлен → upload/download 503, ни строки, ни объекта; метаданные, текст и ready работают;
|
||||
3) процесс «падает» после S3 PUT до COMMIT ready (FAULTS=after_s3_put, только в этом тесте) →
|
||||
остаётся строка uploading + объект; сверщик не трогает их в пределах grace и убирает после;
|
||||
4) объект без строки в БД (orphan) → свежий не трогается, старый удаляется; повторный запуск ничего не делает;
|
||||
5) периодический сервис reconciler сам удаляет заготовку старше grace (без ручного запуска).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import time
|
||||
import urllib.parse
|
||||
import uuid
|
||||
|
||||
from helpers import BASE_URL, call, compose, new_chat, new_user, sql, wait_ready
|
||||
from imagegen import SAMPLE_PNG
|
||||
from test_attachments import download, upload
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(title, ok, detail=''):
|
||||
print(' %-62s %s %s' % (title, 'OK' if ok else 'FAIL', detail))
|
||||
if not ok:
|
||||
failures.append(title)
|
||||
|
||||
|
||||
def reconcile(*args):
|
||||
return json.loads(compose('run', '--rm', 'reconcile', *args))
|
||||
|
||||
|
||||
def s3_python(code):
|
||||
return compose('run', '--rm', '--entrypoint', 'python', 'reconcile', '-c',
|
||||
'import sys; sys.path.insert(0, "src")\n' + code)
|
||||
|
||||
|
||||
def counts():
|
||||
return sql("SELECT count(*) FILTER (WHERE status = 'ready') || '/' || count(*) FILTER (WHERE status = 'uploading') "
|
||||
"FROM attachments")
|
||||
|
||||
|
||||
def object_exists(key):
|
||||
return s3_python('from config import setting, settings\nfrom s3 import make_client\n'
|
||||
'c = make_client(setting("S3_RECONCILER_ACCESS_KEY_ID"), setting("S3_RECONCILER_SECRET_ACCESS_KEY"))\n'
|
||||
'keys = [o["Key"] for o in c.list_objects_v2(Bucket=settings.s3_bucket, Prefix=%r).get("Contents", [])]\n'
|
||||
'print(len(keys))' % key).strip() == '1'
|
||||
|
||||
|
||||
def restart_api(faults=None):
|
||||
env = dict(os.environ)
|
||||
env['FAULTS'] = faults or ''
|
||||
compose('up', '-d', '--no-deps', '--no-build', 'api', env=env)
|
||||
time.sleep(3)
|
||||
wait_ready()
|
||||
time.sleep(3)
|
||||
|
||||
|
||||
def interrupted_upload(alice, chat):
|
||||
target = urllib.parse.urlsplit(BASE_URL)
|
||||
with socket.create_connection((target.hostname, target.port), timeout=10) as sock:
|
||||
head = ('POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: x\r\n'
|
||||
'Authorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n'
|
||||
% (chat['id'], alice['token'], 5 * 1024 * 1024))
|
||||
sock.sendall(head.encode() + SAMPLE_PNG + bytes(512 * 1024))
|
||||
time.sleep(2)
|
||||
|
||||
|
||||
def main():
|
||||
alice = new_user('consistency')
|
||||
chat = new_chat(alice)
|
||||
|
||||
print('1. Обрыв загрузки посреди тела')
|
||||
before = counts()
|
||||
interrupted_upload(alice, chat)
|
||||
check('строк не прибавилось (ready/uploading)', counts() == before, '%s -> %s' % (before, counts()))
|
||||
|
||||
print('2. S3 остановлен: загрузка, скачивание, остальной API')
|
||||
stored = upload(alice, chat, SAMPLE_PNG)[2]
|
||||
before = counts()
|
||||
compose('stop', 's3')
|
||||
try:
|
||||
started = time.monotonic()
|
||||
status = upload(alice, chat, SAMPLE_PNG)[0]
|
||||
check('upload при остановленном S3 → 503', status == 503, '%s за %.1f с' % (status, time.monotonic() - started))
|
||||
check('заготовка удалена компенсацией', counts() == before, '%s -> %s' % (before, counts()))
|
||||
started = time.monotonic()
|
||||
status = download(alice, stored['id'])[0]
|
||||
check('download при остановленном S3 → 503', status == 503, '%s за %.1f с' % (status, time.monotonic() - started))
|
||||
check('метаданные вложения доступны (они в PostgreSQL)',
|
||||
call('GET', '/api/v1/attachments/' + stored['id'], token=alice['token'])[0] == 200)
|
||||
check('текстовое сообщение → 201', call('POST', '/api/v1/chats/%s/messages' % chat['id'],
|
||||
{'text': 'без S3'}, alice['token'])[0] == 201)
|
||||
check('/health/ready → 200 (S3 не критичен для текстового API)', call('GET', '/health/ready')[0] == 200)
|
||||
finally:
|
||||
compose('up', '-d', '--wait', 's3')
|
||||
check('после старта S3 upload снова 201', upload(alice, chat, SAMPLE_PNG)[0] == 201)
|
||||
|
||||
print('3. Сбой после S3 PUT до фиксации ready (FAULTS=after_s3_put)')
|
||||
restart_api('after_s3_put')
|
||||
try:
|
||||
status = upload(alice, chat, SAMPLE_PNG, filename='fault-%s.png' % uuid.uuid4().hex[:6])[0]
|
||||
row = sql("SELECT id || ' ' || object_key FROM attachments WHERE status = 'uploading' "
|
||||
"AND filename LIKE 'fault-%%' ORDER BY created_at DESC LIMIT 1").split()
|
||||
check('клиент получил ошибку (не 201)', status != 201, str(status))
|
||||
check('осталась строка uploading и объект в S3', bool(row) and object_exists(row[1]), ' '.join(row))
|
||||
finally:
|
||||
restart_api()
|
||||
fresh = reconcile('--grace-seconds', '900')
|
||||
check('dry-run, grace 900 с: заготовка свежая, не трогается', fresh['counts']['fresh_upload'] >= 1
|
||||
and fresh['counts']['stale_upload'] == 0, str(fresh['counts']))
|
||||
applied = reconcile('--apply', '--grace-seconds', '0')
|
||||
check('apply, grace 0: строка и объект удалены', applied['actions']['stale_rows_deleted'] >= 1
|
||||
and applied['actions']['stale_objects_deleted'] >= 1, str(applied['actions']))
|
||||
check('объекта больше нет', not object_exists(row[1]))
|
||||
|
||||
print('4. Объект без строки в БД (orphan)')
|
||||
key = 'attachments/%s' % uuid.uuid4()
|
||||
s3_python('from config import setting, settings\nfrom s3 import make_client\n'
|
||||
'c = make_client(setting("S3_RECONCILER_ACCESS_KEY_ID"), setting("S3_RECONCILER_SECRET_ACCESS_KEY"))\n'
|
||||
'c.put_object(Bucket=settings.s3_bucket, Key=%r, Body=b"orphan")' % key)
|
||||
fresh = reconcile('--grace-seconds', '900')
|
||||
check('dry-run, grace 900 с: свежая сирота не трогается', fresh['counts']['fresh_orphan'] >= 1
|
||||
and fresh['counts']['orphan'] == 0, str(fresh['counts']))
|
||||
applied = reconcile('--apply', '--grace-seconds', '0')
|
||||
check('apply, grace 0: сирота удалена', applied['actions']['orphan_objects_deleted'] >= 1, str(applied['actions']))
|
||||
again = reconcile('--apply', '--grace-seconds', '0')
|
||||
check('повторный запуск ничего не делает', not any(again['actions'][k] for k in
|
||||
('orphan_objects_deleted', 'stale_rows_deleted', 'stale_objects_deleted')), str(again['actions']))
|
||||
check('рабочие вложения не пострадали (missing = 0)', again['counts']['missing'] == 0, str(again['counts']))
|
||||
|
||||
print('5. Периодический сервис reconciler убирает зависшую заготовку сам')
|
||||
stale = str(uuid.uuid4())
|
||||
sql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, "
|
||||
"status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'stale.png', 'image/png', 1, "
|
||||
"decode(repeat('00', 32), 'hex'), 'attachments/%s', 'uploading', now() - interval '1 hour', "
|
||||
"now() - interval '1 hour')" % (stale, chat['id'], alice['id'], stale))
|
||||
running = 'reconciler' in compose('ps', '--status', 'running', '--services')
|
||||
check('сервис reconciler запущен (docker compose ps)', running)
|
||||
if running:
|
||||
compose('restart', 'reconciler') # проход выполняется сразу после старта, дальше — раз в интервал
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline and sql("SELECT count(*) FROM attachments WHERE id = '%s'" % stale) != '0':
|
||||
time.sleep(1)
|
||||
check('заготовка старше grace удалена без ручного запуска',
|
||||
sql("SELECT count(*) FROM attachments WHERE id = '%s'" % stale) == '0')
|
||||
|
||||
print('FAIL: ' + ', '.join(failures) if failures else 'PASS: сбои загрузки не оставляют неучтённых данных')
|
||||
raise SystemExit(1 if failures else 0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Usage (стенд уже запущен через docker compose up -d):
|
||||
python student-tests/expiry_check.py --ttl 2
|
||||
|
||||
Скрипт пересоздаёт только api с SESSION_TTL_SECONDS=<ttl>, проверяет истечение сессии,
|
||||
затем возвращает api с обычным TTL из .env. Volumes и данные не трогаются.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import time
|
||||
|
||||
from helpers import call, compose, new_user, wait_ready
|
||||
|
||||
|
||||
def restart_api(ttl=None):
|
||||
env = dict(os.environ)
|
||||
if ttl is None:
|
||||
env.pop('SESSION_TTL_SECONDS', None)
|
||||
else:
|
||||
env['SESSION_TTL_SECONDS'] = str(ttl)
|
||||
compose('up', '-d', '--no-deps', '--no-build', 'api', env=env)
|
||||
wait_ready()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--ttl', type=int, default=2)
|
||||
args = parser.parse_args()
|
||||
|
||||
print('пересоздаю api с SESSION_TTL_SECONDS=%d' % args.ttl)
|
||||
restart_api(args.ttl)
|
||||
try:
|
||||
alice = new_user('expiry')
|
||||
status = call('GET', '/api/v1/users/me', token=alice['token'])[0]
|
||||
print('сразу после входа: /users/me -> %d' % status)
|
||||
assert status == 200
|
||||
key = 'session:' + hashlib.sha256(alice['token'].encode()).hexdigest()
|
||||
cache_ttl = int(compose('exec', '-T', 'valkey', 'valkey-cli', 'TTL', key))
|
||||
print('TTL ключа этой сессии в Valkey: %d с (-2 = ключа нет)' % cache_ttl)
|
||||
assert cache_ttl <= args.ttl, 'кэш живёт дольше сессии'
|
||||
|
||||
time.sleep(args.ttl + 1.5)
|
||||
status, headers, _ = call('GET', '/api/v1/users/me', token=alice['token'])
|
||||
print('через %.1f с: /users/me -> %d, WWW-Authenticate: %s' % (args.ttl + 1.5, status, headers['WWW-Authenticate']))
|
||||
assert status == 401 and headers['WWW-Authenticate'] == 'Bearer realm="messenger"'
|
||||
status = call('DELETE', '/api/v1/auth/sessions/current', token=alice['token'])[0]
|
||||
print('logout истёкшим токеном -> %d' % status)
|
||||
assert status == 401
|
||||
print('PASS: сессия истекла по SESSION_TTL_SECONDS=%d, кэш не продлил её' % args.ttl)
|
||||
finally:
|
||||
print('возвращаю api с TTL из .env')
|
||||
restart_api()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Usage (стенд запущен через docker compose up -d, обе реплики в ротации):
|
||||
python student-tests/failover_series.py --output evidence/failover-series.json
|
||||
|
||||
Для каждой API-реплики по очереди и в двух режимах (docker stop — штатная остановка с выводом из ротации,
|
||||
docker kill — внезапная смерть процесса) под нагрузкой: останавливает реплику, меряет ошибки, RPS,
|
||||
время до последнего ответа остановленной реплики, восстанавливает её (docker compose start api)
|
||||
и ждёт возврата в ротацию LB. Volumes, БД и LB не трогаются.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from helpers import Load, api_containers, compose, new_chat, new_user, wait_in_rotation
|
||||
|
||||
WARMUP, AFTER = 5, 10
|
||||
|
||||
|
||||
def run(container, mode):
|
||||
alice = new_user('series')
|
||||
chat = new_chat(alice)
|
||||
load = Load(alice['token'], chat['id']).start()
|
||||
time.sleep(WARMUP)
|
||||
acted = time.monotonic()
|
||||
command = ['docker', 'stop', '-t', '15', container] if mode == 'stop' else ['docker', 'kill', container]
|
||||
subprocess.run(command, capture_output=True, check=True)
|
||||
action_seconds = time.monotonic() - acted
|
||||
time.sleep(AFTER)
|
||||
load.stop()
|
||||
instance = container[:12]
|
||||
target = [r for r in load.records if r[2] == 201 and r[3] == instance and r[0] >= acted]
|
||||
others = [r for r in load.records if r[2] == 201 and r[3] != instance and r[0] >= acted]
|
||||
restored = time.monotonic()
|
||||
compose('start', 'api')
|
||||
back = wait_in_rotation(container)
|
||||
result = {
|
||||
'replica': instance, 'mode': mode, 'action_seconds': round(action_seconds, 2),
|
||||
'target_served_until_s': round(max(r[1] for r in target) - acted, 2) if target else 0.0,
|
||||
'first_success_on_other_s': round(min(r[1] for r in others) - acted, 2) if others else None,
|
||||
'before': load.window(acted - WARMUP, acted),
|
||||
'during_10s': load.window(acted, acted + AFTER),
|
||||
'back_in_rotation_s': round(back - restored, 2),
|
||||
'data': load.verify(alice['token']),
|
||||
}
|
||||
print('%-12s %-4s errors=%-3d rps %5.1f -> %5.1f target last answer +%.2fs restore %.1fs lost=%d dup=%d'
|
||||
% (instance, mode, result['during_10s']['errors'], result['before']['rps'], result['during_10s']['rps'],
|
||||
result['target_served_until_s'], result['back_in_rotation_s'], result['data']['lost'],
|
||||
result['data']['duplicate_texts']))
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--output')
|
||||
parser.add_argument('--modes', default='stop,kill')
|
||||
args = parser.parse_args()
|
||||
replicas = api_containers()
|
||||
assert len(replicas) >= 2, 'need at least two running API replicas'
|
||||
results = [run(container, mode) for mode in args.modes.split(',') for container in replicas]
|
||||
bad = [r for r in results if r['data']['lost'] or r['data']['duplicate_texts'] or r['data']['duplicate_ids']]
|
||||
summary = {'replicas': [c[:12] for c in replicas], 'warmup_s': WARMUP, 'observe_s': AFTER, 'workers': 4,
|
||||
'lb_check': 'GET /health/ready every 1s, fall 2, rise 2', 'results': results}
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as file:
|
||||
json.dump(summary, file, ensure_ascii=False, indent=2)
|
||||
print('FAIL: data loss or duplicates' if bad else 'PASS: every replica stopped once per mode; no loss, no duplicates')
|
||||
raise SystemExit(1 if bad else 0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Usage (стенд уже запущен через docker compose up -d):
|
||||
python student-tests/failure_check.py
|
||||
|
||||
Останавливает и замораживает Valkey и PostgreSQL (docker compose stop/pause, без удаления volumes),
|
||||
проверяет поведение API и восстановление.
|
||||
"""
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from helpers import basic, call, compose, new_chat, new_user, wait_ready
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def timed(method, path, token=None, body=None, headers=None):
|
||||
started = time.monotonic()
|
||||
try:
|
||||
status = call(method, path, body=body, token=token, headers=headers, timeout=10)[0]
|
||||
except OSError as error:
|
||||
status = type(error).__name__
|
||||
return status, time.monotonic() - started
|
||||
|
||||
|
||||
def expect(title, result, allowed, max_seconds):
|
||||
status, seconds = result
|
||||
ok = status in allowed and seconds <= max_seconds
|
||||
print(' %-48s -> %-4s за %.2f с %s' % (title, status, seconds, 'OK' if ok else 'FAIL'))
|
||||
if not ok:
|
||||
failures.append(title)
|
||||
|
||||
|
||||
def main():
|
||||
alice = new_user('failure')
|
||||
chat = new_chat(alice)
|
||||
history = '/api/v1/chats/%s/messages' % chat['id']
|
||||
assert call('POST', history, {'text': 'до отказа'}, alice['token'])[0] == 201
|
||||
fake = uuid.uuid4().hex * 2
|
||||
|
||||
print('1. Valkey остановлен: кэш обходится, всё работает через PostgreSQL')
|
||||
compose('stop', 'valkey')
|
||||
try:
|
||||
expect('первый запрос: обнаружение отказа кэша', timed('GET', '/api/v1/users/me', alice['token']), [200], 1.5)
|
||||
expect('GET /users/me действующим токеном', timed('GET', '/api/v1/users/me', alice['token']), [200], 1)
|
||||
expect('POST сообщение', timed('POST', history, alice['token'], {'text': 'без кэша'}), [201], 1)
|
||||
expect('неизвестный токен', timed('GET', '/api/v1/users/me', fake), [401], 1)
|
||||
expect('без Authorization', timed('GET', '/api/v1/users/me'), [401], 1)
|
||||
expect('/health/ready (Valkey некритичен)', timed('GET', '/health/ready'), [200], 1)
|
||||
started = time.monotonic()
|
||||
other = new_user('nocache')
|
||||
expect('регистрация и вход без кэша', (200, time.monotonic() - started), [200], 1)
|
||||
expect('logout без кэша', timed('DELETE', '/api/v1/auth/sessions/current', other['token']), [204], 1.5)
|
||||
expect('отозванный токен', timed('GET', '/api/v1/users/me', other['token']), [401], 1)
|
||||
finally:
|
||||
compose('up', '-d', '--wait', 'valkey')
|
||||
|
||||
print('1б. Valkey на паузе: logout, затем unpause — отозванный токен не оживает из кэша')
|
||||
revoked = new_user('paused')
|
||||
call('GET', '/api/v1/users/me', token=revoked['token'])
|
||||
compose('pause', 'valkey')
|
||||
try:
|
||||
expect('logout при паузе Valkey', timed('DELETE', '/api/v1/auth/sessions/current', revoked['token']), [204], 1.5)
|
||||
finally:
|
||||
compose('unpause', 'valkey')
|
||||
time.sleep(6)
|
||||
expect('через 6 с после unpause: /users/me', timed('GET', '/api/v1/users/me', revoked['token']), [401], 1)
|
||||
expect('через 6 с после unpause: сообщение', timed('POST', history, revoked['token'], {'text': 'после logout'}), [401], 1)
|
||||
|
||||
print('1в. Valkey на паузе: logout, рестарт API, unpause — отзыв переживает рестарт процесса')
|
||||
restarted = new_user('restart')
|
||||
call('GET', '/api/v1/users/me', token=restarted['token'])
|
||||
compose('pause', 'valkey')
|
||||
try:
|
||||
expect('logout при паузе Valkey', timed('DELETE', '/api/v1/auth/sessions/current', restarted['token']), [204], 1.5)
|
||||
compose('restart', 'api')
|
||||
wait_ready()
|
||||
finally:
|
||||
compose('unpause', 'valkey')
|
||||
time.sleep(6)
|
||||
expect('после рестарта API и unpause: /users/me', timed('GET', '/api/v1/users/me', restarted['token']), [401], 1)
|
||||
expect('после рестарта API и unpause: сообщение', timed('POST', history, restarted['token'], {'text': 'после рестарта'}), [401], 1)
|
||||
|
||||
print('2. PostgreSQL остановлен: ограниченная ошибка 503, анонимный запрос не проходит')
|
||||
compose('stop', 'postgres')
|
||||
stopped_at = time.monotonic()
|
||||
try:
|
||||
expect('GET история', timed('GET', history, alice['token']), [503], 3)
|
||||
expect('POST сообщение', timed('POST', history, alice['token'], {'text': 'при отказе БД'}), [503], 3)
|
||||
expect('неизвестный токен (не 200!)', timed('GET', '/api/v1/users/me', fake), [503], 3)
|
||||
expect('без Authorization', timed('GET', '/api/v1/users/me'), [401], 1)
|
||||
expect('вход по Basic верным паролем', timed('POST', '/api/v1/auth/sessions',
|
||||
headers=basic(alice['username'], alice['password'])), [503], 3)
|
||||
expect('/health/live', timed('GET', '/health/live'), [200], 1)
|
||||
expect('/health/ready', timed('GET', '/health/ready'), [503], 3)
|
||||
finally:
|
||||
compose('up', '-d', '--wait', 'postgres')
|
||||
ready_at = wait_ready()
|
||||
print('3. PostgreSQL запущен: готовность через %.1f с после остановки' % (ready_at - stopped_at))
|
||||
status = call('GET', '/api/v1/users/me', token=alice['token'])[0]
|
||||
texts = [m['text'] for m in call('GET', history + '?limit=100', token=alice['token'])[2]['items']]
|
||||
print(' прежний токен -> %d; история: %s' % (status, texts))
|
||||
if status != 200 or texts != ['до отказа', 'без кэша']:
|
||||
failures.append('восстановление')
|
||||
|
||||
print('4. PostgreSQL заморожен (pause): запросы получают 503 в пределах общего срока, а не зависают')
|
||||
compose('pause', 'postgres')
|
||||
try:
|
||||
expect('GET история', timed('GET', history, alice['token']), [503], 4.5)
|
||||
expect('POST сообщение', timed('POST', history, alice['token'], {'text': 'при заморозке'}), [503], 4.5)
|
||||
expect('неизвестный токен (не 200!)', timed('GET', '/api/v1/users/me', fake), [503], 4.5)
|
||||
expect('вход по Basic', timed('POST', '/api/v1/auth/sessions', headers=basic(alice['username'], alice['password'])), [503], 4.5)
|
||||
expect('/health/ready', timed('GET', '/health/ready'), [503], 2)
|
||||
expect('/health/live', timed('GET', '/health/live'), [200], 1)
|
||||
finally:
|
||||
compose('unpause', 'postgres')
|
||||
unpaused = time.monotonic()
|
||||
wait_ready()
|
||||
texts = [m['text'] for m in call('GET', history + '?limit=100', token=alice['token'])[2]['items']]
|
||||
print(' готовность через %.1f с после unpause; история: %s' % (time.monotonic() - unpaused, texts))
|
||||
if texts != ['до отказа', 'без кэша']:
|
||||
failures.append('после заморозки')
|
||||
|
||||
print('FAIL: ' + ', '.join(failures) if failures else 'PASS: отказы Valkey и PostgreSQL обработаны')
|
||||
raise SystemExit(1 if failures else 0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 19 KiB |
@@ -0,0 +1,191 @@
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://127.0.0.1:8080').rstrip('/')
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def call(method, path, body=None, token=None, raw=None, headers=None, timeout=10):
|
||||
all_headers = {}
|
||||
if token:
|
||||
all_headers['Authorization'] = 'Bearer ' + token
|
||||
if body is not None:
|
||||
raw = json.dumps(body, ensure_ascii=False).encode('utf-8')
|
||||
if raw is not None:
|
||||
all_headers['Content-Type'] = 'application/json'
|
||||
all_headers.update(headers or {})
|
||||
if method == 'POST' and path.split('?')[0].endswith('/messages') and 'Idempotency-Key' not in all_headers:
|
||||
all_headers['Idempotency-Key'] = uuid.uuid4().hex
|
||||
all_headers = {name: value for name, value in all_headers.items() if value is not None}
|
||||
request = urllib.request.Request(BASE_URL + path, data=raw, headers=all_headers, method=method)
|
||||
try:
|
||||
response = urllib.request.urlopen(request, timeout=timeout)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
data = response.read()
|
||||
is_json = response.headers.get_content_type() == 'application/json'
|
||||
return response.status, response.headers, json.loads(data) if data and is_json else None
|
||||
|
||||
|
||||
def basic(username, password):
|
||||
return {'Authorization': 'Basic ' + base64.b64encode((username + ':' + password).encode('utf-8')).decode()}
|
||||
|
||||
|
||||
def register(label='user', password=None):
|
||||
username = (label + '_' + uuid.uuid4().hex[:12])[:32]
|
||||
password = password or 'Pw-' + uuid.uuid4().hex
|
||||
status, _, user = call('POST', '/api/v1/users', {'username': username, 'display_name': label, 'password': password})
|
||||
assert status == 201, status
|
||||
return user, password
|
||||
|
||||
|
||||
def login(username, password):
|
||||
status, _, session = call('POST', '/api/v1/auth/sessions', headers=basic(username, password))
|
||||
assert status == 201, status
|
||||
return session
|
||||
|
||||
|
||||
def new_user(label='user'):
|
||||
user, password = register(label)
|
||||
session = login(user['username'], password)
|
||||
return dict(user, password=password, token=session['token'])
|
||||
|
||||
|
||||
def new_chat(owner, *others, title='Тестовый чат'):
|
||||
members = [owner['id']] + [u['id'] for u in others]
|
||||
status, _, chat = call('POST', '/api/v1/chats', {'title': title, 'member_ids': members}, owner['token'])
|
||||
assert status == 201, status
|
||||
return chat
|
||||
|
||||
|
||||
def read_all_messages(chat_id, token, limit):
|
||||
items, cursor, pages = [], None, 0
|
||||
while True:
|
||||
query = '?limit=%d' % limit + ('&cursor=' + urllib.parse.quote(cursor, safe='') if cursor else '')
|
||||
status, _, page = call('GET', '/api/v1/chats/%s/messages%s' % (chat_id, query), token=token)
|
||||
assert status == 200, status
|
||||
items += page['items']
|
||||
pages += 1
|
||||
cursor = page['next_cursor']
|
||||
if cursor is None:
|
||||
return items, pages
|
||||
|
||||
|
||||
def compose(*args, env=None, check=True):
|
||||
result = subprocess.run(['docker', 'compose', *args], cwd=ROOT, capture_output=True, text=True,
|
||||
encoding='utf-8', env=env)
|
||||
if check and result.returncode != 0:
|
||||
raise RuntimeError('docker compose %s failed: %s' % (' '.join(args), result.stderr.strip()))
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def sql(query, db='messenger', service='postgres'):
|
||||
return compose('exec', '-T', service, 'psql', '-U', 'messenger', '-d', db, '-X', '-tA', '-v', 'ON_ERROR_STOP=1',
|
||||
'-c', query)
|
||||
|
||||
|
||||
def wait_ready(seconds=90):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if call('GET', '/health/ready', timeout=2)[0] == 200:
|
||||
return time.monotonic()
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(0.3)
|
||||
raise AssertionError('API is not ready after %ds' % seconds)
|
||||
|
||||
|
||||
def api_containers():
|
||||
return compose('ps', '-q', 'api').split()
|
||||
|
||||
|
||||
def wait_in_rotation(container, seconds=60):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if call('GET', '/health/ready', timeout=2)[1].get('X-Instance-Id') == container[:12]:
|
||||
return time.monotonic()
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
raise AssertionError('container %s did not enter LB rotation' % container[:12])
|
||||
|
||||
|
||||
class Load:
|
||||
"""Отправляет сообщения в несколько потоков; неудачный POST повторяет с тем же Idempotency-Key."""
|
||||
|
||||
def __init__(self, token, chat_id, workers=4):
|
||||
self.token, self.path, self.workers = token, '/api/v1/chats/%s/messages' % chat_id, workers
|
||||
self.records, self.confirmed, self.uncertain = [], [], 0
|
||||
self.stopped = threading.Event()
|
||||
self.threads = []
|
||||
|
||||
def start(self):
|
||||
self.threads = [threading.Thread(target=self.worker, args=(n,), daemon=True) for n in range(self.workers)]
|
||||
for thread in self.threads:
|
||||
thread.start()
|
||||
return self
|
||||
|
||||
def stop(self):
|
||||
self.stopped.set()
|
||||
for thread in self.threads:
|
||||
thread.join()
|
||||
|
||||
def worker(self, n):
|
||||
sent = 0
|
||||
while not self.stopped.is_set():
|
||||
key, text = uuid.uuid4().hex, 'load %d-%d %s' % (n, sent, uuid.uuid4().hex[:6])
|
||||
sent += 1
|
||||
while True:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
status, headers, body = call('POST', self.path, {'text': text}, self.token,
|
||||
headers={'Idempotency-Key': key}, timeout=5)
|
||||
instance = headers.get('X-Instance-Id')
|
||||
except OSError as error:
|
||||
status, instance, body = type(error).__name__, None, None
|
||||
self.records.append((started, time.monotonic(), status, instance))
|
||||
if status == 201:
|
||||
self.confirmed.append(body)
|
||||
break
|
||||
if self.stopped.is_set():
|
||||
self.uncertain += 1
|
||||
break
|
||||
time.sleep(0.05)
|
||||
|
||||
def window(self, start, end):
|
||||
rows = [r for r in self.records if start <= r[0] < end]
|
||||
ok = [r for r in rows if r[2] == 201]
|
||||
latencies = sorted((r[1] - r[0]) * 1000 for r in ok)
|
||||
p = lambda q: round(latencies[min(len(latencies) - 1, int(len(latencies) * q))], 1) if latencies else None
|
||||
return {'seconds': round(end - start, 2), 'ok': len(ok), 'errors': len(rows) - len(ok),
|
||||
'rps': round(len(ok) / (end - start), 1) if end > start else 0,
|
||||
'p50_ms': p(0.5), 'p95_ms': p(0.95), 'p99_ms': p(0.99),
|
||||
'error_kinds': sorted({str(r[2]) for r in rows if r[2] != 201})}
|
||||
|
||||
def verify(self, token):
|
||||
stored, cursor = [], None
|
||||
while True:
|
||||
query = '?limit=100' + ('&cursor=' + urllib.parse.quote(cursor, safe='') if cursor else '')
|
||||
page = call('GET', self.path + query, token=token)[2]
|
||||
stored += page['items']
|
||||
cursor = page['next_cursor']
|
||||
if cursor is None:
|
||||
break
|
||||
ids = [m['id'] for m in stored]
|
||||
texts = [m['text'] for m in stored]
|
||||
return {'confirmed': len(self.confirmed), 'stored': len(stored),
|
||||
'lost': len({m['id'] for m in self.confirmed} - set(ids)),
|
||||
'duplicate_ids': len(ids) - len(set(ids)), 'duplicate_texts': len(texts) - len(set(texts)),
|
||||
'uncertain_at_stop': self.uncertain}
|
||||
@@ -0,0 +1,66 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SAMPLE_PNG = (ROOT / 'tests' / 'fixtures' / 'sample.png').read_bytes()
|
||||
SAMPLE_JPG = (Path(__file__).resolve().parent / 'fixtures' / 'sample.jpg').read_bytes()
|
||||
|
||||
|
||||
def chunk(kind, data):
|
||||
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff)
|
||||
|
||||
|
||||
def png(width, height, rows, color_type=0, level=9):
|
||||
"""rows — итератор байтовых строк пикселей (без байта фильтра)."""
|
||||
compressor = zlib.compressobj(level)
|
||||
parts = []
|
||||
for row in rows:
|
||||
parts.append(compressor.compress(b'\x00' + row))
|
||||
parts.append(compressor.flush())
|
||||
header = struct.pack('>IIBBBBB', width, height, 8, color_type, 0, 0, 0)
|
||||
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', b''.join(parts)) + chunk(b'IEND', b'')
|
||||
|
||||
|
||||
def bomb(width=10000, height=5000):
|
||||
"""50 Мп чёрного: в распакованном виде 50 МБ, в файле — несколько десятков КБ."""
|
||||
row = bytes(width)
|
||||
return png(width, height, (row for _ in range(height)))
|
||||
|
||||
|
||||
def noise_png(target_bytes):
|
||||
"""Шумный RGB PNG (почти не сжимается) размером чуть меньше target_bytes."""
|
||||
width = 1024
|
||||
height = max(1, (target_bytes - 4096) // (width * 3 + 1))
|
||||
return png(width, height, (os.urandom(width * 3) for _ in range(height)), color_type=2, level=0)
|
||||
|
||||
|
||||
def broken_crc(data):
|
||||
"""Испортить один байт внутри IDAT, не пересчитывая CRC."""
|
||||
position = data.index(b'IDAT') + 100
|
||||
return data[:position] + bytes([data[position] ^ 0xff]) + data[position + 1:]
|
||||
|
||||
|
||||
def with_idat(data, transform):
|
||||
"""Заменить пиксельные данные PNG (все IDAT) на transform(idat), с правильными CRC."""
|
||||
position, chunks = 8, []
|
||||
while position < len(data):
|
||||
length, kind = struct.unpack('>I4s', data[position:position + 8])
|
||||
chunks.append((kind, data[position + 8:position + 8 + length]))
|
||||
position += 12 + length
|
||||
idat = b''.join(body for kind, body in chunks if kind == b'IDAT')
|
||||
first = next(index for index, (kind, _) in enumerate(chunks) if kind == b'IDAT')
|
||||
kept = [(kind, body) for kind, body in chunks if kind != b'IDAT']
|
||||
kept.insert(first, (b'IDAT', transform(idat)))
|
||||
return data[:8] + b''.join(chunk(kind, body) for kind, body in kept)
|
||||
|
||||
|
||||
def garbage_idat(data):
|
||||
"""Мусор вместо сжатых пикселей, CRC верные."""
|
||||
return with_idat(data, lambda idat: idat[:2] + os.urandom(len(idat) - 2))
|
||||
|
||||
|
||||
def short_idat(data):
|
||||
"""Корректный zlib-поток, но пикселей меньше, чем требует IHDR."""
|
||||
return with_idat(data, lambda idat: zlib.compress(zlib.decompress(idat)[:-500]))
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Usage (стенд запущен, обе реплики в ротации):
|
||||
python student-tests/rolling_update_check.py --output evidence/rolling-update.json
|
||||
|
||||
Под непрерывной нагрузкой (4 потока, повтор с тем же Idempotency-Key) выполняет scripts/rolling_update.py
|
||||
и проверяет: ошибки клиента, RPS, что все реплики заменены, что подтверждённые сообщения не потеряны и не задвоены.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
from helpers import Load, api_containers, new_chat, new_user
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--output')
|
||||
parser.add_argument('--no-build', action='store_true')
|
||||
args = parser.parse_args()
|
||||
before = api_containers()
|
||||
alice = new_user('rolling')
|
||||
chat = new_chat(alice)
|
||||
load = Load(alice['token'], chat['id']).start()
|
||||
time.sleep(5)
|
||||
started = time.monotonic()
|
||||
command = [sys.executable, str(ROOT / 'scripts' / 'rolling_update.py')] + (['--no-build'] if args.no_build else [])
|
||||
update = subprocess.run(command, capture_output=True, text=True, encoding='utf-8')
|
||||
finished = time.monotonic()
|
||||
time.sleep(5)
|
||||
load.stop()
|
||||
print(update.stdout.strip())
|
||||
after = api_containers()
|
||||
result = {
|
||||
'replicas_before': [c[:12] for c in before], 'replicas_after': [c[:12] for c in after],
|
||||
'update_seconds': round(finished - started, 1), 'update_exit_code': update.returncode,
|
||||
'before_5s': load.window(started - 5, started),
|
||||
'during_update': load.window(started, finished),
|
||||
'after_5s': load.window(finished, finished + 5),
|
||||
'data': load.verify(alice['token']),
|
||||
'log': update.stdout.strip().splitlines(),
|
||||
}
|
||||
replaced = not set(before) & set(after) and len(after) == len(before)
|
||||
ok = update.returncode == 0 and replaced and not result['data']['lost'] and not result['data']['duplicate_texts']
|
||||
print('rolling update %.1fs: errors=%d, rps %.1f -> %.1f -> %.1f, lost=%d, duplicates=%d, all replaced=%s'
|
||||
% (result['update_seconds'], result['during_update']['errors'], result['before_5s']['rps'],
|
||||
result['during_update']['rps'], result['after_5s']['rps'], result['data']['lost'],
|
||||
result['data']['duplicate_texts'], replaced))
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as file:
|
||||
json.dump(result, file, ensure_ascii=False, indent=2)
|
||||
print('PASS' if ok else 'FAIL: ' + update.stderr.strip()[-500:])
|
||||
raise SystemExit(0 if ok else 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Usage (стенд запущен через docker compose up -d):
|
||||
python student-tests/sigterm_check.py
|
||||
docker compose start api
|
||||
|
||||
Останавливает одну API-реплику (docker stop = SIGTERM) и всё это время опрашивает API через LB:
|
||||
реплика сначала выходит из ротации (readiness 503), потом завершается; клиенты не должны получить ошибок.
|
||||
"""
|
||||
import argparse
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
from helpers import BASE_URL, api_containers, call, new_chat, new_user
|
||||
|
||||
|
||||
def docker(*args):
|
||||
return subprocess.run(['docker', *args], capture_output=True, text=True, encoding='utf-8').stdout.strip()
|
||||
|
||||
|
||||
def probe_forever(results, stop_event):
|
||||
while not stop_event.is_set():
|
||||
started = time.monotonic()
|
||||
try:
|
||||
with urllib.request.urlopen(BASE_URL + '/health/live', timeout=1) as response:
|
||||
outcome = str(response.status)
|
||||
except urllib.error.HTTPError as error:
|
||||
outcome = str(error.code)
|
||||
except Exception as error:
|
||||
outcome = type(error).__name__
|
||||
results.append((started, outcome))
|
||||
time.sleep(0.02)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--container', default=None)
|
||||
parser.add_argument('--stop-timeout', type=int, default=10, help='секунд до SIGKILL у docker stop')
|
||||
args = parser.parse_args()
|
||||
args.container = args.container or api_containers()[0]
|
||||
|
||||
marker = 'PRIVATE-' + uuid.uuid4().hex
|
||||
alice = new_user('alice')
|
||||
chat = new_chat(alice)
|
||||
status = call('POST', '/api/v1/chats/%s/messages' % chat['id'], {'text': marker}, alice['token'])[0]
|
||||
logs_before = docker('logs', args.container)
|
||||
print('сообщение с маркером отправлено: HTTP %d; маркер в логах: %s' % (status, marker in logs_before))
|
||||
|
||||
results, stop_event = [], threading.Event()
|
||||
prober = threading.Thread(target=probe_forever, args=(results, stop_event))
|
||||
prober.start()
|
||||
time.sleep(1)
|
||||
|
||||
sigterm_at = time.monotonic()
|
||||
docker('stop', '-t', str(args.stop_timeout), args.container)
|
||||
stopped_at = time.monotonic()
|
||||
time.sleep(1)
|
||||
stop_event.set()
|
||||
prober.join()
|
||||
|
||||
before = [r for t, r in results if t < sigterm_at]
|
||||
after = [r for t, r in results if t >= sigterm_at]
|
||||
exit_code = docker('inspect', '--format', '{{.State.ExitCode}}', args.container)
|
||||
print('docker stop занял: %.2f с (лимит %d с)' % (stopped_at - sigterm_at, args.stop_timeout))
|
||||
print('код выхода контейнера: %s (0 — сам завершился по SIGTERM; 137 — убит SIGKILL)' % exit_code)
|
||||
print('запросы до SIGTERM: %d, все 200: %s' % (len(before), set(before) == {'200'}))
|
||||
print('запросы после SIGTERM: %d, результаты: %s' % (len(after), sorted(set(after))))
|
||||
print('ответов 5xx за всё время: %d' % sum(r.startswith('5') for _, r in results))
|
||||
|
||||
print('\nлоги остановки:')
|
||||
for line in docker('logs', args.container).splitlines():
|
||||
if any(word in line for word in ('draining', 'Shutting down', 'stopping', 'shutdown', 'Finished server')):
|
||||
print(' ' + line)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,157 @@
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
from helpers import call, login, new_chat, new_user, read_all_messages
|
||||
|
||||
|
||||
class ErrorFormatMixin:
|
||||
def assertError(self, result, status, code):
|
||||
actual, headers, body = result
|
||||
self.assertEqual(actual, status)
|
||||
self.assertEqual(body['error']['code'], code)
|
||||
self.assertEqual(body['error']['request_id'], headers['X-Request-Id'])
|
||||
self.assertNotIn('Traceback', body['error']['message'])
|
||||
|
||||
|
||||
class InvalidJsonTest(ErrorFormatMixin, unittest.TestCase):
|
||||
def test_malformed_json(self):
|
||||
"""Битый JSON -> 400 invalid_request"""
|
||||
self.assertError(call('POST', '/api/v1/users', raw=b'{"username": "abc",'), 400, 'invalid_request')
|
||||
|
||||
def test_json_not_object(self):
|
||||
"""JSON-массив вместо объекта -> 400"""
|
||||
self.assertError(call('POST', '/api/v1/users', raw=b'[1, 2, 3]'), 400, 'invalid_request')
|
||||
|
||||
def test_wrong_type(self):
|
||||
"""Число вместо строки -> 400 (без тихого приведения типов)"""
|
||||
self.assertError(call('POST', '/api/v1/users', {'username': 12345, 'display_name': 'x', 'password': 'Long-enough-pass-1'}), 400, 'invalid_request')
|
||||
|
||||
def test_extra_field(self):
|
||||
"""Лишнее поле в теле -> 400"""
|
||||
body = {'username': 'u_' + uuid.uuid4().hex[:8], 'display_name': 'x', 'password': 'Long-enough-pass-1', 'is_admin': True}
|
||||
self.assertError(call('POST', '/api/v1/users', body), 400, 'invalid_request')
|
||||
|
||||
|
||||
class DuplicateUsernameTest(ErrorFormatMixin, unittest.TestCase):
|
||||
def test_duplicate_username(self):
|
||||
"""Повторная регистрация того же username -> 409 conflict"""
|
||||
user = new_user('dup')
|
||||
again = call('POST', '/api/v1/users', {'username': user['username'], 'display_name': 'Другой',
|
||||
'password': 'Other-password-123'})
|
||||
self.assertError(again, 409, 'conflict')
|
||||
|
||||
|
||||
class ForeignChatTest(ErrorFormatMixin, unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.alice, self.bob, self.eve = new_user('alice'), new_user('bob'), new_user('eve')
|
||||
self.chat = new_chat(self.alice, self.bob)
|
||||
self.path = '/api/v1/chats/' + self.chat['id']
|
||||
|
||||
def test_outsider_cannot_read_chat(self):
|
||||
"""Посторонний читает чат -> 404, как будто чата нет"""
|
||||
self.assertError(call('GET', self.path, token=self.eve['token']), 404, 'not_found')
|
||||
missing = call('GET', '/api/v1/chats/' + str(uuid.uuid4()), token=self.eve['token'])
|
||||
self.assertEqual(missing[2]['error']['message'], call('GET', self.path, token=self.eve['token'])[2]['error']['message'])
|
||||
|
||||
def test_outsider_cannot_write_or_read_history(self):
|
||||
"""Посторонний пишет/читает историю -> 404"""
|
||||
self.assertError(call('POST', self.path + '/messages', {'text': 'чужое'}, self.eve['token']), 404, 'not_found')
|
||||
self.assertError(call('GET', self.path + '/messages', token=self.eve['token']), 404, 'not_found')
|
||||
|
||||
def test_outsider_does_not_see_chat_in_list(self):
|
||||
"""Чужой чат не попадает в список чатов"""
|
||||
status, _, page = call('GET', '/api/v1/chats?limit=100', token=self.eve['token'])
|
||||
self.assertEqual(status, 200)
|
||||
self.assertNotIn(self.chat['id'], [c['id'] for c in page['items']])
|
||||
|
||||
def test_member_has_access(self):
|
||||
"""Участник видит чат (контроль: 404 выше не из-за ошибки в тесте)"""
|
||||
self.assertEqual(call('GET', self.path, token=self.bob['token'])[0], 200)
|
||||
|
||||
|
||||
class EmptyMessageTest(ErrorFormatMixin, unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.alice = new_user('alice')
|
||||
self.path = '/api/v1/chats/%s/messages' % new_chat(self.alice)['id']
|
||||
|
||||
def test_empty_text(self):
|
||||
"""Пустой текст -> 400"""
|
||||
self.assertError(call('POST', self.path, {'text': ''}, self.alice['token']), 400, 'invalid_request')
|
||||
|
||||
def test_whitespace_text(self):
|
||||
"""Текст только из пробельных символов -> 400"""
|
||||
self.assertError(call('POST', self.path, {'text': ' \t\n '}, self.alice['token']), 400, 'invalid_request')
|
||||
|
||||
def test_missing_text(self):
|
||||
"""Тело без поля text -> 400"""
|
||||
self.assertError(call('POST', self.path, {}, self.alice['token']), 400, 'invalid_request')
|
||||
|
||||
def test_text_length_limit(self):
|
||||
"""4000 символов можно (эмодзи = 1 символ), 4001 — 400"""
|
||||
self.assertEqual(call('POST', self.path, {'text': '😀' * 4000}, self.alice['token'])[0], 201)
|
||||
self.assertError(call('POST', self.path, {'text': 'x' * 4001}, self.alice['token']), 400, 'invalid_request')
|
||||
|
||||
|
||||
class PaginationTest(ErrorFormatMixin, unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.alice = new_user('alice')
|
||||
self.chat = new_chat(self.alice)
|
||||
self.sent = [call('POST', '/api/v1/chats/%s/messages' % self.chat['id'],
|
||||
{'text': 'сообщение %d' % i}, self.alice['token'])[2] for i in range(7)]
|
||||
|
||||
def test_walk_several_pages(self):
|
||||
"""7 сообщений по 3: три страницы, без потерь, повторов и перестановок"""
|
||||
items, pages = read_all_messages(self.chat['id'], self.alice['token'], limit=3)
|
||||
self.assertEqual(pages, 3)
|
||||
self.assertEqual([m['id'] for m in items], [m['id'] for m in self.sent])
|
||||
keys = [(m['created_at'], m['id']) for m in items]
|
||||
self.assertEqual(keys, sorted(keys))
|
||||
|
||||
def test_cursor_from_other_chat(self):
|
||||
"""cursor от другого чата -> 400"""
|
||||
_, _, page = call('GET', '/api/v1/chats/%s/messages?limit=1' % self.chat['id'], token=self.alice['token'])
|
||||
other = new_chat(self.alice, title='Другой')
|
||||
self.assertError(call('GET', '/api/v1/chats/%s/messages?cursor=%s' % (other['id'], page['next_cursor']),
|
||||
token=self.alice['token']), 400, 'invalid_request')
|
||||
|
||||
def test_invalid_cursor_and_limit(self):
|
||||
"""Мусорный cursor и limit вне 1..100 -> 400"""
|
||||
path = '/api/v1/chats/%s/messages' % self.chat['id']
|
||||
for query in ('?cursor=not-a-cursor', '?limit=0', '?limit=101', '?limit=abc'):
|
||||
with self.subTest(query=query):
|
||||
self.assertError(call('GET', path + query, token=self.alice['token']), 400, 'invalid_request')
|
||||
|
||||
|
||||
class IdentityTest(ErrorFormatMixin, unittest.TestCase):
|
||||
def test_sender_cannot_be_forged(self):
|
||||
"""sender_id в теле запрещён: отправитель только из сессии"""
|
||||
alice, bob = new_user('alice'), new_user('bob')
|
||||
chat = new_chat(alice, bob)
|
||||
forged = call('POST', '/api/v1/chats/%s/messages' % chat['id'], {'text': 'hi', 'sender_id': bob['id']}, alice['token'])
|
||||
self.assertError(forged, 400, 'invalid_request')
|
||||
|
||||
def test_nul_character_is_preserved(self):
|
||||
"""U+0000 — допустимый code point: сообщение, display_name и title сохраняются и читаются без изменений"""
|
||||
username, password = 'nul_' + uuid.uuid4().hex[:12], 'Nul-password-123'
|
||||
status, _, user = call('POST', '/api/v1/users', {'username': username, 'display_name': 'a\u0000lice', 'password': password})
|
||||
self.assertEqual((status, user['display_name']), (201, 'a\u0000lice'))
|
||||
alice = dict(user, token=login(username, password)['token'])
|
||||
self.assertEqual(call('GET', '/api/v1/users/me', token=alice['token'])[2]['display_name'], 'a\u0000lice')
|
||||
status, _, chat = call('POST', '/api/v1/chats', {'title': 't\u0000', 'member_ids': [alice['id']]}, alice['token'])
|
||||
self.assertEqual((status, chat['title']), (201, 't\u0000'))
|
||||
path = '/api/v1/chats/%s/messages' % chat['id']
|
||||
status, _, message = call('POST', path, {'text': 'a\u0000b'}, alice['token'])
|
||||
self.assertEqual((status, message['text']), (201, 'a\u0000b'))
|
||||
self.assertEqual(call('GET', path, token=alice['token'])[2]['items'][0]['text'], 'a\u0000b')
|
||||
self.assertEqual(call('GET', '/api/v1/chats/' + chat['id'], token=alice['token'])[2]['title'], 't\u0000')
|
||||
|
||||
def test_lone_surrogate_is_rejected(self):
|
||||
"""Одиночный суррогат не кодируется в UTF-8 -> 400, а не 500"""
|
||||
alice = new_user('alice')
|
||||
chat = new_chat(alice)
|
||||
self.assertError(call('POST', '/api/v1/chats/%s/messages' % chat['id'], raw=b'{"text": "a\\ud800b"}',
|
||||
token=alice['token']), 400, 'invalid_request')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,192 @@
|
||||
import hashlib
|
||||
import http.client
|
||||
import unittest
|
||||
import urllib.parse
|
||||
import uuid
|
||||
|
||||
from helpers import BASE_URL, call, new_chat, new_user
|
||||
from imagegen import SAMPLE_JPG, SAMPLE_PNG, bomb, broken_crc, garbage_idat, noise_png, short_idat
|
||||
|
||||
MAX = 10 * 1024 * 1024
|
||||
|
||||
|
||||
def upload(user, chat, data, content_type='image/png', filename='photo.png', token=True):
|
||||
query = '' if filename is None else '?filename=' + urllib.parse.quote(filename, safe='')
|
||||
return call('POST', '/api/v1/chats/%s/attachments%s' % (chat['id'], query), raw=data,
|
||||
token=user['token'] if token else None, headers={'Content-Type': content_type})
|
||||
|
||||
|
||||
def download(user, attachment_id):
|
||||
target = urllib.parse.urlsplit(BASE_URL)
|
||||
connection = http.client.HTTPConnection(target.hostname, target.port, timeout=30)
|
||||
connection.request('GET', '/api/v1/attachments/%s/content' % attachment_id,
|
||||
headers={'Authorization': 'Bearer ' + user['token']})
|
||||
response = connection.getresponse()
|
||||
body = response.read()
|
||||
connection.close()
|
||||
return response.status, response.headers, body
|
||||
|
||||
|
||||
def send(user, chat, body, key=None):
|
||||
return call('POST', '/api/v1/chats/%s/messages' % chat['id'], body, user['token'],
|
||||
headers={'Idempotency-Key': key or uuid.uuid4().hex})
|
||||
|
||||
|
||||
class UploadTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.alice, self.bob, self.eve = new_user('alice'), new_user('bob'), new_user('eve')
|
||||
self.chat = new_chat(self.alice, self.bob)
|
||||
|
||||
def assertCode(self, result, status, code=None):
|
||||
self.assertEqual(result[0], status, result[2])
|
||||
if code:
|
||||
self.assertEqual(result[2]['error']['code'], code)
|
||||
|
||||
def test_png_and_jpeg_are_stored_and_downloaded_byte_exact(self):
|
||||
"""PNG и JPEG → 201 ready; второй участник скачивает те же байты с верным Content-Type"""
|
||||
for data, content_type in ((SAMPLE_PNG, 'image/png'), (SAMPLE_JPG, 'image/jpeg')):
|
||||
with self.subTest(content_type=content_type):
|
||||
status, _, att = upload(self.alice, self.chat, data, content_type)
|
||||
self.assertEqual(status, 201)
|
||||
self.assertEqual((att['status'], att['variants'], att['error_code']), ('ready', [], None))
|
||||
self.assertEqual((att['content_type'], att['size_bytes'], att['sha256']),
|
||||
(content_type, len(data), hashlib.sha256(data).hexdigest()))
|
||||
self.assertEqual((att['chat_id'], att['owner_id']), (self.chat['id'], self.alice['id']))
|
||||
self.assertEqual(call('GET', '/api/v1/attachments/' + att['id'], token=self.bob['token'])[2], att)
|
||||
status, headers, body = download(self.bob, att['id'])
|
||||
self.assertEqual((status, headers.get_content_type(), body), (200, content_type, data))
|
||||
self.assertEqual(int(headers['Content-Length']), len(data))
|
||||
|
||||
def test_download_through_every_replica(self):
|
||||
"""Файл скачивается через любую API-реплику (байты не лежат локально у загрузившей)"""
|
||||
att = upload(self.alice, self.chat, SAMPLE_PNG)[2]
|
||||
replicas = set()
|
||||
for _ in range(6):
|
||||
status, headers, body = download(self.bob, att['id'])
|
||||
self.assertEqual((status, body), (200, SAMPLE_PNG))
|
||||
replicas.add(headers['X-Instance-Id'])
|
||||
self.assertGreaterEqual(len(replicas), 2)
|
||||
|
||||
def test_filename_is_display_only_and_validated(self):
|
||||
"""Unicode-имя сохраняется как есть; нет имени, пустое, /, \\, NUL, управляющий символ, >255 → 400"""
|
||||
status, _, att = upload(self.alice, self.chat, SAMPLE_PNG, filename='фото отпуска 😀.png')
|
||||
self.assertEqual((status, att['filename']), (201, 'фото отпуска 😀.png'))
|
||||
for name in (None, '', ' ', '../etc/passwd', 'a\\b.png', 'a\x00b.png', 'bell\x07.png', 'x' * 256):
|
||||
with self.subTest(name=name):
|
||||
self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG, filename=name), 400, 'invalid_request')
|
||||
|
||||
def test_wrong_declared_type(self):
|
||||
"""Content-Type не image/png|jpeg (octet-stream, multipart, gif) → 415"""
|
||||
for content_type in ('application/octet-stream', 'multipart/form-data; boundary=x', 'image/gif', 'text/plain'):
|
||||
with self.subTest(content_type=content_type):
|
||||
self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG, content_type), 415, 'unsupported_media_type')
|
||||
|
||||
def test_mime_spoofing(self):
|
||||
"""Подмена MIME: PNG как jpeg, JPEG как png, текст и GIF под видом png → 415"""
|
||||
for data, content_type in ((SAMPLE_PNG, 'image/jpeg'), (SAMPLE_JPG, 'image/png'),
|
||||
(b'hello, not an image', 'image/png'), (b'GIF89a' + bytes(100), 'image/png'),
|
||||
(b'', 'image/png')):
|
||||
with self.subTest(content_type=content_type, head=data[:6]):
|
||||
self.assertCode(upload(self.alice, self.chat, data, content_type), 415, 'unsupported_media_type')
|
||||
|
||||
def test_corrupted_images(self):
|
||||
"""Обрезанный PNG, битый CRC, мусор или нехватка пикселей в IDAT при верных CRC, обрезанный JPEG → 415"""
|
||||
for data, content_type in ((SAMPLE_PNG[:5000], 'image/png'), (broken_crc(SAMPLE_PNG), 'image/png'),
|
||||
(garbage_idat(SAMPLE_PNG), 'image/png'), (short_idat(SAMPLE_PNG), 'image/png'),
|
||||
(SAMPLE_JPG[:3000], 'image/jpeg')):
|
||||
with self.subTest(size=len(data), content_type=content_type):
|
||||
self.assertCode(upload(self.alice, self.chat, data, content_type), 415, 'unsupported_media_type')
|
||||
|
||||
def test_too_large_file(self):
|
||||
"""10 MiB + 1 байт → 413 (и по Content-Length, и chunked без длины); файл ~10 MiB принимается"""
|
||||
self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG + bytes(MAX + 1 - len(SAMPLE_PNG))), 413, 'payload_too_large')
|
||||
target = urllib.parse.urlsplit(BASE_URL)
|
||||
connection = http.client.HTTPConnection(target.hostname, target.port, timeout=60)
|
||||
chunks = (SAMPLE_PNG if i == 0 else bytes(1024 * 1024) for i in range(12))
|
||||
connection.request('POST', '/api/v1/chats/%s/attachments?filename=big.png' % self.chat['id'], body=chunks,
|
||||
headers={'Authorization': 'Bearer ' + self.alice['token'], 'Content-Type': 'image/png'},
|
||||
encode_chunked=True)
|
||||
self.assertEqual(connection.getresponse().status, 413)
|
||||
connection.close()
|
||||
big = noise_png(MAX - 50000)
|
||||
self.assertLessEqual(len(big), MAX)
|
||||
status, _, att = upload(self.alice, self.chat, big)
|
||||
self.assertEqual((status, att['size_bytes']), (201, len(big)))
|
||||
self.assertEqual(download(self.bob, att['id'])[2], big)
|
||||
|
||||
def test_decompression_bomb(self):
|
||||
"""PNG 10000×5000 (50 Мп) весом в десятки КБ → 413 без распаковки"""
|
||||
data = bomb()
|
||||
self.assertLess(len(data), 1024 * 1024)
|
||||
self.assertCode(upload(self.alice, self.chat, data), 413, 'payload_too_large')
|
||||
|
||||
def test_access_rules(self):
|
||||
"""Без токена 401; посторонний не может загрузить в чужой чат и не видит метаданные/байты (404)"""
|
||||
self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG, token=False), 401, 'unauthorized')
|
||||
self.assertCode(upload(self.eve, self.chat, SAMPLE_PNG), 404, 'not_found')
|
||||
self.assertCode(upload(self.alice, {'id': str(uuid.uuid4())}, SAMPLE_PNG), 404, 'not_found')
|
||||
att = upload(self.alice, self.chat, SAMPLE_PNG)[2]
|
||||
self.assertCode(call('GET', '/api/v1/attachments/' + att['id'], token=self.eve['token']), 404, 'not_found')
|
||||
self.assertEqual(download(self.eve, att['id'])[0], 404)
|
||||
self.assertCode(call('GET', '/api/v1/attachments/' + str(uuid.uuid4()), token=self.bob['token']), 404)
|
||||
self.assertCode(call('GET', '/api/v1/attachments/not-a-uuid', token=self.bob['token']), 400)
|
||||
self.assertCode(call('GET', '/api/v1/attachments/%s/content?variant=thumbnail' % att['id'],
|
||||
token=self.bob['token']), 400)
|
||||
|
||||
|
||||
class MessageAttachmentsTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.alice, self.bob = new_user('alice'), new_user('bob')
|
||||
self.chat = new_chat(self.alice, self.bob)
|
||||
self.other_chat = new_chat(self.alice, title='Другой')
|
||||
self.att = upload(self.alice, self.chat, SAMPLE_PNG)[2]
|
||||
|
||||
def history(self):
|
||||
return call('GET', '/api/v1/chats/%s/messages?limit=100' % self.chat['id'], token=self.bob['token'])[2]['items']
|
||||
|
||||
def test_message_with_attachment_only(self):
|
||||
"""text="" + одно вложение → 201; attachment_ids видны в истории; обычное сообщение получает []"""
|
||||
status, _, message = send(self.alice, self.chat, {'text': '', 'attachment_ids': [self.att['id']]})
|
||||
self.assertEqual((status, message['attachment_ids'], message['text']), (201, [self.att['id']], ''))
|
||||
plain = send(self.alice, self.chat, {'text': 'без вложений'})[2]
|
||||
self.assertEqual(plain['attachment_ids'], [])
|
||||
self.assertEqual([m['attachment_ids'] for m in self.history()], [[self.att['id']], []])
|
||||
|
||||
def test_other_member_can_reference_attachment(self):
|
||||
"""Боб прикрепляет вложение, загруженное Алисой в этот же чат"""
|
||||
status, _, message = send(self.bob, self.chat, {'text': 'смотри', 'attachment_ids': [self.att['id']]})
|
||||
self.assertEqual((status, message['attachment_ids']), (201, [self.att['id']]))
|
||||
|
||||
def test_foreign_or_unknown_attachment(self):
|
||||
"""Вложение другого чата или несуществующее → 404, сообщение не создаётся"""
|
||||
foreign = upload(self.alice, self.other_chat, SAMPLE_PNG)[2]
|
||||
for ids in ([foreign['id']], [str(uuid.uuid4())], [self.att['id'], foreign['id']]):
|
||||
with self.subTest(ids=ids):
|
||||
self.assertEqual(send(self.alice, self.chat, {'text': 'x', 'attachment_ids': ids})[0], 404)
|
||||
self.assertEqual(self.history(), [])
|
||||
|
||||
def test_invalid_attachment_lists(self):
|
||||
"""Повтор id, 11 вложений, не-UUID, пустой текст без вложений, пробелы без вложений → 400"""
|
||||
for body in ({'text': 'x', 'attachment_ids': [self.att['id'], self.att['id']]},
|
||||
{'text': 'x', 'attachment_ids': [str(uuid.uuid4()) for _ in range(11)]},
|
||||
{'text': 'x', 'attachment_ids': ['nope']},
|
||||
{'text': ''}, {'text': ' '}, {'text': '', 'attachment_ids': []}):
|
||||
with self.subTest(body=body):
|
||||
self.assertEqual(send(self.alice, self.chat, body)[0], 400)
|
||||
|
||||
def test_idempotency_includes_attachments(self):
|
||||
"""Тот же ключ: те же вложения → тот же Message; другие вложения → 409; нет поля = [] для текста"""
|
||||
key = uuid.uuid4().hex
|
||||
first = send(self.alice, self.chat, {'text': 'фото', 'attachment_ids': [self.att['id']]}, key)
|
||||
again = send(self.alice, self.chat, {'attachment_ids': [self.att['id']], 'text': 'фото'}, key)
|
||||
self.assertEqual((first[0], again[0], first[2]), (201, 201, again[2]))
|
||||
other = upload(self.alice, self.chat, SAMPLE_JPG, 'image/jpeg', 'b.jpg')[2]
|
||||
self.assertEqual(send(self.alice, self.chat, {'text': 'фото', 'attachment_ids': [other['id']]}, key)[0], 409)
|
||||
key = uuid.uuid4().hex
|
||||
plain = send(self.alice, self.chat, {'text': 'текст'}, key)
|
||||
same = send(self.alice, self.chat, {'text': 'текст', 'attachment_ids': []}, key)
|
||||
self.assertEqual(plain[2]['id'], same[2]['id'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,132 @@
|
||||
from datetime import datetime, timezone
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
from helpers import basic, call, login, new_user, register
|
||||
|
||||
|
||||
class RegistrationTest(unittest.TestCase):
|
||||
def body(self, password):
|
||||
return {'username': 'reg_' + uuid.uuid4().hex[:12], 'display_name': 'Регистрация', 'password': password}
|
||||
|
||||
def test_password_is_not_returned(self):
|
||||
"""Ответ регистрации и /users/me не содержат password/hash"""
|
||||
status, _, user = call('POST', '/api/v1/users', self.body('Secret-password-1'))
|
||||
self.assertEqual(status, 201)
|
||||
self.assertEqual(set(user), {'id', 'username', 'display_name', 'created_at'})
|
||||
|
||||
def test_password_length_limits(self):
|
||||
"""Пароль 12..128 символов; 11 и 129 -> 400; эмодзи считается одним символом"""
|
||||
self.assertEqual(call('POST', '/api/v1/users', self.body('x' * 11))[0], 400)
|
||||
self.assertEqual(call('POST', '/api/v1/users', self.body('x' * 129))[0], 400)
|
||||
self.assertEqual(call('POST', '/api/v1/users', self.body('x' * 128))[0], 201)
|
||||
self.assertEqual(call('POST', '/api/v1/users', self.body('🔑' * 12))[0], 201)
|
||||
|
||||
def test_password_required_and_not_blank(self):
|
||||
"""Без password или из одних пробелов -> 400"""
|
||||
body = self.body('x')
|
||||
del body['password']
|
||||
self.assertEqual(call('POST', '/api/v1/users', body)[0], 400)
|
||||
self.assertEqual(call('POST', '/api/v1/users', self.body(' ' * 20))[0], 400)
|
||||
|
||||
|
||||
class LoginTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.user, self.password = register('login')
|
||||
|
||||
def assertBasicChallenge(self, result):
|
||||
status, headers, body = result
|
||||
self.assertEqual(status, 401)
|
||||
self.assertEqual(headers['WWW-Authenticate'], 'Basic realm="messenger", charset="UTF-8"')
|
||||
self.assertEqual(body['error']['code'], 'unauthorized')
|
||||
return body['error']['message']
|
||||
|
||||
def test_login_returns_opaque_session(self):
|
||||
"""Вход: 201, token_type=Bearer, токен случайный и не содержит username/UUID"""
|
||||
first = login(self.user['username'], self.password)
|
||||
second = login(self.user['username'], self.password)
|
||||
self.assertEqual(first['token_type'], 'Bearer')
|
||||
self.assertEqual(first['user'], self.user)
|
||||
self.assertGreaterEqual(len(first['token']), 43)
|
||||
self.assertNotEqual(first['token'], second['token'])
|
||||
self.assertNotIn(self.user['username'], first['token'])
|
||||
self.assertNotIn(self.user['id'], first['token'])
|
||||
expires = datetime.fromisoformat(first['expires_at'].replace('Z', '+00:00'))
|
||||
self.assertGreater(expires, datetime.now(timezone.utc))
|
||||
|
||||
def test_wrong_password_and_unknown_user_look_the_same(self):
|
||||
"""Неверный пароль и несуществующий пользователь -> одинаковый 401 с Basic challenge"""
|
||||
wrong = self.assertBasicChallenge(call('POST', '/api/v1/auth/sessions',
|
||||
headers=basic(self.user['username'], 'Wrong-password-1')))
|
||||
unknown = self.assertBasicChallenge(call('POST', '/api/v1/auth/sessions',
|
||||
headers=basic('nobody_' + uuid.uuid4().hex[:8], 'Wrong-password-1')))
|
||||
self.assertEqual(wrong, unknown)
|
||||
|
||||
def test_malformed_basic(self):
|
||||
"""Нет заголовка, не base64, нет двоеточия, Bearer вместо Basic -> 401 Basic"""
|
||||
for header in (None, 'Basic', 'Basic !!!not-base64!!!', 'Basic ' + 'bm9jb2xvbg==', 'Bearer abc'):
|
||||
with self.subTest(header=header):
|
||||
headers = {'Authorization': header} if header else {}
|
||||
self.assertBasicChallenge(call('POST', '/api/v1/auth/sessions', headers=headers))
|
||||
|
||||
def test_password_with_colon_and_unicode(self):
|
||||
"""Двоеточие и не-ASCII в пароле допустимы: делим только по первому ':'"""
|
||||
user, password = register('colon', password='пароль:с:двоеточием🔑')
|
||||
self.assertEqual(login(user['username'], password)['user']['id'], user['id'])
|
||||
|
||||
def test_login_rejects_body(self):
|
||||
"""POST /auth/sessions принимает только заголовок, JSON body -> 400"""
|
||||
status = call('POST', '/api/v1/auth/sessions', {'username': self.user['username']},
|
||||
headers=basic(self.user['username'], self.password))[0]
|
||||
self.assertEqual(status, 400)
|
||||
|
||||
|
||||
class BearerTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.alice = new_user('alice')
|
||||
self.eve = new_user('eve')
|
||||
|
||||
def assertBearerChallenge(self, headers):
|
||||
status, response_headers, _ = call('GET', '/api/v1/users/me', headers=headers)
|
||||
self.assertEqual(status, 401)
|
||||
self.assertEqual(response_headers['WWW-Authenticate'], 'Bearer realm="messenger"')
|
||||
|
||||
def test_only_bearer_is_accepted(self):
|
||||
"""Нет заголовка, Basic, X-User-Id, мусорный токен -> 401 Bearer"""
|
||||
self.assertBearerChallenge({})
|
||||
self.assertBearerChallenge(basic(self.alice['username'], self.alice['password']))
|
||||
self.assertBearerChallenge({'X-User-Id': self.alice['id']})
|
||||
self.assertBearerChallenge({'Authorization': 'Bearer ' + uuid.uuid4().hex})
|
||||
self.assertBearerChallenge({'Authorization': 'Bearer'})
|
||||
|
||||
def test_authentication_before_validation(self):
|
||||
"""Аноним с битым JSON на защищённом endpoint получает 401, а не 400; с сессией — 400"""
|
||||
self.assertEqual(call('POST', '/api/v1/chats', raw=b'{bad')[0], 401)
|
||||
self.assertEqual(call('POST', '/api/v1/chats', raw=b'{bad', token=self.alice['token'])[0], 400)
|
||||
self.assertEqual(call('POST', '/api/v1/users', raw=b'{bad')[0], 400)
|
||||
|
||||
def test_scheme_is_case_insensitive(self):
|
||||
"""Схема аутентификации нечувствительна к регистру (RFC 9110)"""
|
||||
status, _, me = call('GET', '/api/v1/users/me', headers={'Authorization': 'bearer ' + self.alice['token']})
|
||||
self.assertEqual((status, me['id']), (200, self.alice['id']))
|
||||
|
||||
def test_x_user_id_does_not_override_session(self):
|
||||
"""X-User-Id игнорируется: владелец определяется только токеном"""
|
||||
status, _, me = call('GET', '/api/v1/users/me', token=self.alice['token'], headers={'X-User-Id': self.eve['id']})
|
||||
self.assertEqual((status, me['id']), (200, self.alice['id']))
|
||||
|
||||
|
||||
class LogoutTest(unittest.TestCase):
|
||||
def test_logout_revokes_only_current_session(self):
|
||||
"""Logout -> 204 без тела; этот токен -> 401, вторая сессия того же пользователя работает"""
|
||||
alice = new_user('alice')
|
||||
other = login(alice['username'], alice['password'])['token']
|
||||
status, _, body = call('DELETE', '/api/v1/auth/sessions/current', token=alice['token'])
|
||||
self.assertEqual((status, body), (204, None))
|
||||
self.assertEqual(call('GET', '/api/v1/users/me', token=alice['token'])[0], 401)
|
||||
self.assertEqual(call('DELETE', '/api/v1/auth/sessions/current', token=alice['token'])[0], 401)
|
||||
self.assertEqual(call('GET', '/api/v1/users/me', token=other)[0], 200)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,68 @@
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import threading
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
from helpers import basic, call, new_chat, new_user, read_all_messages
|
||||
|
||||
CLIENTS = 10
|
||||
REGISTRATIONS = 20
|
||||
PER_CLIENT = 10
|
||||
|
||||
|
||||
class ConcurrencyTest(unittest.TestCase):
|
||||
def test_100_messages_from_10_clients(self):
|
||||
"""100 параллельных сообщений от 10 клиентов -> ровно 100 уникальных в истории"""
|
||||
users = [new_user('client%d' % i) for i in range(CLIENTS)]
|
||||
chat = new_chat(users[0], *users[1:], title='Нагрузка')
|
||||
path = '/api/v1/chats/%s/messages' % chat['id']
|
||||
barrier = threading.Barrier(CLIENTS)
|
||||
|
||||
def client(index):
|
||||
barrier.wait()
|
||||
results = []
|
||||
for n in range(PER_CLIENT):
|
||||
status, _, message = call('POST', path, {'text': 'client %d msg %d' % (index, n)}, users[index]['token'])
|
||||
results.append((status, message))
|
||||
return results
|
||||
|
||||
with ThreadPoolExecutor(max_workers=CLIENTS) as pool:
|
||||
responses = [r for batch in pool.map(client, range(CLIENTS)) for r in batch]
|
||||
|
||||
self.assertEqual([s for s, _ in responses], [201] * CLIENTS * PER_CLIENT)
|
||||
sent_ids = {m['id'] for _, m in responses}
|
||||
items, pages = read_all_messages(chat['id'], users[0]['token'], limit=30)
|
||||
read_ids = [m['id'] for m in items]
|
||||
keys = [(m['created_at'], m['id']) for m in items]
|
||||
|
||||
self.assertEqual(len(sent_ids), 100, 'сервер выдал повторяющиеся id')
|
||||
self.assertEqual(len(read_ids), 100, 'в истории не ровно 100 сообщений')
|
||||
self.assertEqual(set(read_ids), sent_ids, 'история не совпадает с подтверждёнными сообщениями')
|
||||
self.assertEqual(len({m['text'] for m in items}), 100, 'потерялся или задвоился текст')
|
||||
self.assertEqual(keys, sorted(keys), 'история не упорядочена по (created_at, id)')
|
||||
print('\n отправлено 201: %d, уникальных id: %d, прочитано: %d за %d страниц(ы)'
|
||||
% (len(responses), len(sent_ids), len(read_ids), pages))
|
||||
|
||||
def test_parallel_registration_of_same_username(self):
|
||||
"""20 одновременных регистраций одного username -> ровно один 201, остальные 409; вход только с паролем победителя"""
|
||||
username = 'race_' + uuid.uuid4().hex[:12]
|
||||
barrier = threading.Barrier(REGISTRATIONS)
|
||||
|
||||
def register(index):
|
||||
password = 'Race-password-%02d' % index
|
||||
barrier.wait()
|
||||
status = call('POST', '/api/v1/users', {'username': username, 'display_name': 'Гонка', 'password': password})[0]
|
||||
return status, password
|
||||
|
||||
with ThreadPoolExecutor(max_workers=REGISTRATIONS) as pool:
|
||||
results = list(pool.map(register, range(REGISTRATIONS)))
|
||||
statuses = sorted(status for status, _ in results)
|
||||
self.assertEqual(statuses, [201] + [409] * (REGISTRATIONS - 1))
|
||||
winner = [password for status, password in results if status == 201][0]
|
||||
loser = [password for status, password in results if status == 409][0]
|
||||
self.assertEqual(call('POST', '/api/v1/auth/sessions', headers=basic(username, winner))[0], 201)
|
||||
self.assertEqual(call('POST', '/api/v1/auth/sessions', headers=basic(username, loser))[0], 401)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,58 @@
|
||||
import hashlib
|
||||
import time
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
from helpers import call, compose, new_chat, new_user, sql
|
||||
|
||||
|
||||
class DatabaseTest(unittest.TestCase):
|
||||
def test_chat_creation_is_rolled_back(self):
|
||||
"""Неизвестный участник: INSERT chats выполнен, INSERT chat_members упал по FK, всё откатано"""
|
||||
alice = new_user('alice')
|
||||
chats_before = int(sql('SELECT count(*) FROM chats'))
|
||||
members_before = int(sql('SELECT count(*) FROM chat_members'))
|
||||
ghost = str(uuid.uuid4())
|
||||
since = sql("SELECT to_char(now() - interval '1 second', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')")
|
||||
status, _, body = call('POST', '/api/v1/chats', {'title': 'Откат', 'member_ids': [alice['id'], ghost]},
|
||||
alice['token'])
|
||||
self.assertEqual(status, 400, body)
|
||||
time.sleep(0.5)
|
||||
log = compose('logs', '--no-log-prefix', '--since', since, 'postgres')
|
||||
self.assertIn('violates foreign key constraint', log)
|
||||
self.assertIn(ghost, log)
|
||||
self.assertEqual(int(sql('SELECT count(*) FROM chats')), chats_before)
|
||||
self.assertEqual(int(sql('SELECT count(*) FROM chat_members')), members_before)
|
||||
|
||||
def test_secrets_are_not_stored_in_plaintext(self):
|
||||
"""В users лежит argon2id-хеш, в sessions — sha256 токена, не сам токен"""
|
||||
alice = new_user('alice')
|
||||
password_hash = sql("SELECT password_hash FROM users WHERE id = '%s'" % alice['id'])
|
||||
self.assertTrue(password_hash.startswith('$argon2id$'), password_hash[:10])
|
||||
self.assertNotIn(alice['password'], password_hash)
|
||||
digest = hashlib.sha256(alice['token'].encode()).hexdigest()
|
||||
self.assertEqual(sql("SELECT count(*) FROM sessions WHERE token_hash = '\\x%s'" % digest), '1')
|
||||
dump = sql("SELECT string_agg(encode(token_hash, 'escape'), '') FROM sessions WHERE user_id = '%s'" % alice['id'])
|
||||
self.assertNotIn(alice['token'], dump)
|
||||
|
||||
def test_migrations_are_idempotent(self):
|
||||
"""Повторный запуск миграций ничего не меняет и завершается успешно"""
|
||||
before = sql('SELECT version, applied_at FROM schema_migrations ORDER BY version')
|
||||
users = sql('SELECT count(*) FROM users')
|
||||
output = compose('run', '--rm', '--no-deps', 'migrate')
|
||||
self.assertIn('skip 0001_init: already applied', output)
|
||||
self.assertEqual(sql('SELECT version, applied_at FROM schema_migrations ORDER BY version'), before)
|
||||
self.assertEqual(sql('SELECT count(*) FROM users'), users)
|
||||
|
||||
def test_history_query_uses_index(self):
|
||||
"""Запрос страницы истории идёт по индексу (chat_id, created_at, id), без Sort"""
|
||||
alice = new_user('alice')
|
||||
chat = new_chat(alice)
|
||||
plan = sql("SET enable_seqscan = off; SET enable_bitmapscan = off; EXPLAIN SELECT id FROM messages WHERE chat_id = '%s' "
|
||||
"ORDER BY created_at, id LIMIT 51" % chat['id'])
|
||||
self.assertIn('messages_chat_history_idx', plan)
|
||||
self.assertNotIn('Sort', plan)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,96 @@
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import threading
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
from helpers import call, new_chat, new_user, read_all_messages
|
||||
|
||||
PARALLEL = 20
|
||||
|
||||
|
||||
def post(chat, user, key, body=None, raw=None):
|
||||
return call('POST', '/api/v1/chats/%s/messages' % chat['id'], body, user['token'], raw=raw,
|
||||
headers={'Idempotency-Key': key})
|
||||
|
||||
|
||||
class IdempotencyTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.alice, self.bob = new_user('alice'), new_user('bob')
|
||||
self.chat = new_chat(self.alice, self.bob)
|
||||
|
||||
def history(self):
|
||||
return read_all_messages(self.chat['id'], self.alice['token'], limit=100)[0]
|
||||
|
||||
def test_replay_returns_same_message(self):
|
||||
"""Повтор того же запроса с тем же ключом -> тот же 201 и тот же Message, дубля нет"""
|
||||
key = uuid.uuid4().hex
|
||||
first = post(self.chat, self.alice, key, {'text': 'один раз'})
|
||||
second = post(self.chat, self.alice, key, {'text': 'один раз'})
|
||||
self.assertEqual((first[0], second[0]), (201, 201))
|
||||
self.assertEqual(first[2], second[2])
|
||||
self.assertEqual([m['id'] for m in self.history()], [first[2]['id']])
|
||||
|
||||
def test_json_formatting_does_not_matter(self):
|
||||
"""Порядок/пробелы в JSON семантически равны: повтор с другим форматированием -> тот же Message"""
|
||||
key = uuid.uuid4().hex
|
||||
first = post(self.chat, self.alice, key, raw='{"text":"формат"}'.encode())
|
||||
second = post(self.chat, self.alice, key, raw='{ "text" : "формат" }'.encode())
|
||||
self.assertEqual((first[0], second[0]), (201, 201))
|
||||
self.assertEqual(first[2]['id'], second[2]['id'])
|
||||
|
||||
def test_other_body_with_same_key_conflicts(self):
|
||||
"""Тот же ключ с другим телом -> 409, второе сообщение не создаётся"""
|
||||
key = uuid.uuid4().hex
|
||||
self.assertEqual(post(self.chat, self.alice, key, {'text': 'первое'})[0], 201)
|
||||
status, _, body = post(self.chat, self.alice, key, {'text': 'другое'})
|
||||
self.assertEqual((status, body['error']['code']), (409, 'conflict'))
|
||||
self.assertEqual(len(self.history()), 1)
|
||||
|
||||
def test_missing_or_invalid_key(self):
|
||||
"""Нет ключа, пустой, недопустимые символы, длиннее 128 -> 400"""
|
||||
for key in (None, '', 'bad key', 'bad/key', 'café', 'x' * 129):
|
||||
with self.subTest(key=key):
|
||||
self.assertEqual(post(self.chat, self.alice, key, {'text': 'x'})[0], 400)
|
||||
self.assertEqual(post(self.chat, self.alice, 'A.b_c:d-1' + 'x' * 119, {'text': 'граница'})[0], 201)
|
||||
|
||||
def test_key_scope_is_user_and_chat(self):
|
||||
"""Одинаковый ключ у разных пользователей и в разных чатах — независимые сообщения"""
|
||||
key = uuid.uuid4().hex
|
||||
other_chat = new_chat(self.alice, title='Другой')
|
||||
a = post(self.chat, self.alice, key, {'text': 'от Алисы'})
|
||||
b = post(self.chat, self.bob, key, {'text': 'от Боба'})
|
||||
c = post(other_chat, self.alice, key, {'text': 'от Алисы'})
|
||||
self.assertEqual([a[0], b[0], c[0]], [201, 201, 201])
|
||||
self.assertEqual(len({a[2]['id'], b[2]['id'], c[2]['id']}), 3)
|
||||
|
||||
def test_replay_rechecks_session_and_access(self):
|
||||
"""Повтор снова проверяет сессию и доступ: после logout -> 401, посторонний с тем же ключом -> 404"""
|
||||
key = uuid.uuid4().hex
|
||||
self.assertEqual(post(self.chat, self.alice, key, {'text': 'до выхода'})[0], 201)
|
||||
eve = new_user('eve')
|
||||
self.assertEqual(post(self.chat, eve, key, {'text': 'до выхода'})[0], 404)
|
||||
call('DELETE', '/api/v1/auth/sessions/current', token=self.alice['token'])
|
||||
self.assertEqual(post(self.chat, self.alice, key, {'text': 'до выхода'})[0], 401)
|
||||
|
||||
def test_parallel_retries_through_replicas_create_one_message(self):
|
||||
"""20 одновременных POST с одним ключом через LB (разные реплики) -> ровно одно сообщение, все 201 с одним id"""
|
||||
key = uuid.uuid4().hex
|
||||
barrier = threading.Barrier(PARALLEL)
|
||||
|
||||
def send(_):
|
||||
barrier.wait()
|
||||
status, headers, body = post(self.chat, self.alice, key, {'text': 'конкурентный повтор'})
|
||||
return status, headers['X-Instance-Id'], body
|
||||
|
||||
with ThreadPoolExecutor(max_workers=PARALLEL) as pool:
|
||||
results = list(pool.map(send, range(PARALLEL)))
|
||||
self.assertEqual({status for status, _, _ in results}, {201})
|
||||
self.assertEqual(len({body['id'] for _, _, body in results}), 1)
|
||||
self.assertGreaterEqual(len({instance for _, instance, _ in results}), 2, 'запросы не попали на разные реплики')
|
||||
self.assertEqual(len(self.history()), 1)
|
||||
print('\n 20 повторов: реплики %s, сообщений в истории: %d'
|
||||
% (sorted({instance for _, instance, _ in results}), len(self.history())))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,41 @@
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
from helpers import call, new_chat, new_user
|
||||
|
||||
|
||||
class ReplicasTest(unittest.TestCase):
|
||||
def test_one_session_is_served_by_several_replicas(self):
|
||||
"""Один токен без повторного входа обслуживается разными репликами (X-Instance-Id), все ответы 200"""
|
||||
alice = new_user('alice')
|
||||
results = [call('GET', '/api/v1/users/me', token=alice['token']) for _ in range(10)]
|
||||
self.assertEqual({status for status, _, _ in results}, {200})
|
||||
self.assertGreaterEqual(len({headers['X-Instance-Id'] for _, headers, _ in results}), 2)
|
||||
|
||||
def test_write_on_one_replica_is_read_on_another(self):
|
||||
"""Сообщение, подтверждённое одной репликой, сразу видно при чтении через другую"""
|
||||
alice = new_user('alice')
|
||||
chat = new_chat(alice)
|
||||
path = '/api/v1/chats/%s/messages' % chat['id']
|
||||
for i in range(6):
|
||||
_, written_by, message = call('POST', path, {'text': 'm%d' % i}, alice['token'])
|
||||
status, read_by, page = call('GET', path + '?limit=100', token=alice['token'])
|
||||
self.assertEqual(status, 200)
|
||||
self.assertIn(message['id'], [m['id'] for m in page['items']],
|
||||
'запись через %s не видна через %s' % (written_by['X-Instance-Id'], read_by['X-Instance-Id']))
|
||||
|
||||
def test_every_api_response_has_instance_id(self):
|
||||
"""X-Instance-Id есть и в успешных ответах, и в ошибках API, и в health"""
|
||||
alice = new_user('alice')
|
||||
for method, path, token, expected in (
|
||||
('GET', '/health/live', None, 200), ('GET', '/health/ready', None, 200),
|
||||
('GET', '/api/v1/users/me', None, 401), ('GET', '/api/v1/users/me', alice['token'], 200),
|
||||
('GET', '/api/v1/chats/' + str(uuid.uuid4()), alice['token'], 404), ('GET', '/nope', None, 404)):
|
||||
with self.subTest(path=path, expected=expected):
|
||||
status, headers, _ = call(method, path, token=token)
|
||||
self.assertEqual(status, expected)
|
||||
self.assertTrue(headers['X-Instance-Id'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,81 @@
|
||||
import subprocess
|
||||
import unittest
|
||||
|
||||
from helpers import compose, new_chat, new_user, sql
|
||||
from imagegen import SAMPLE_PNG, noise_png
|
||||
from test_attachments import upload
|
||||
|
||||
LIMITED_CLIENT = '''
|
||||
import json
|
||||
from botocore.exceptions import ClientError
|
||||
from config import settings
|
||||
from s3 import client
|
||||
results = {}
|
||||
for name, action in (
|
||||
("put_get_delete_own_bucket", lambda: (client.put_object(Bucket=settings.s3_bucket, Key="probe/x", Body=b"x"),
|
||||
client.get_object(Bucket=settings.s3_bucket, Key="probe/x"),
|
||||
client.delete_object(Bucket=settings.s3_bucket, Key="probe/x"))),
|
||||
("list_bucket_objects", lambda: client.list_objects_v2(Bucket=settings.s3_bucket)),
|
||||
("create_bucket", lambda: client.create_bucket(Bucket="intruder")),
|
||||
("delete_bucket", lambda: client.delete_bucket(Bucket=settings.s3_bucket)),
|
||||
("list_all_buckets", lambda: client.list_buckets()["Buckets"])):
|
||||
try:
|
||||
value = action()
|
||||
results[name] = "allowed" if name != "list_all_buckets" else "allowed:%d" % len(value)
|
||||
except ClientError as error:
|
||||
results[name] = error.response["Error"]["Code"]
|
||||
print(json.dumps(results))
|
||||
'''
|
||||
|
||||
|
||||
class StorageTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.alice = new_user('alice')
|
||||
self.chat = new_chat(self.alice)
|
||||
|
||||
def object_key(self, attachment_id):
|
||||
return sql("SELECT object_key FROM attachments WHERE id = '%s'" % attachment_id)
|
||||
|
||||
def test_object_key_is_generated_by_server(self):
|
||||
"""Ключ объекта = attachments/<uuid вложения>, filename в нём не участвует"""
|
||||
att = upload(self.alice, self.chat, SAMPLE_PNG, filename='мой-файл.png')[2]
|
||||
self.assertEqual(self.object_key(att['id']), 'attachments/' + att['id'])
|
||||
|
||||
def test_bucket_is_private_outside_api(self):
|
||||
"""Анонимный GET объекта и листинг bucket напрямую в S3 (из сети backend) → 403"""
|
||||
att = upload(self.alice, self.chat, SAMPLE_PNG)[2]
|
||||
for path in ('/messenger-attachments/' + self.object_key(att['id']), '/messenger-attachments'):
|
||||
with self.subTest(path=path):
|
||||
result = compose('exec', '-T', 'lb', 'wget', '-S', '-O', '/dev/null', 'http://s3:8333' + path, check=False)
|
||||
self.assertNotIn('200 OK', result)
|
||||
output = compose('exec', '-T', 'lb', 'sh', '-c', 'wget -S -O /dev/null http://s3:8333%s 2>&1; true' % path)
|
||||
self.assertIn('403', output)
|
||||
|
||||
def test_api_credentials_are_limited(self):
|
||||
"""Ключи API: свои объекты читать/писать/удалять можно; листинг, создание и удаление bucket — AccessDenied"""
|
||||
import json
|
||||
results = json.loads(compose('exec', '-T', 'api', 'python', '-c', 'import sys; sys.path.insert(0, "src")\n' + LIMITED_CLIENT))
|
||||
self.assertEqual(results['put_get_delete_own_bucket'], 'allowed')
|
||||
for action in ('list_bucket_objects', 'create_bucket', 'delete_bucket'):
|
||||
self.assertEqual(results[action], 'AccessDenied', results)
|
||||
self.assertNotEqual(results['list_all_buckets'], 'allowed:1', 'API sees the bucket list')
|
||||
|
||||
def test_bootstrap_is_idempotent(self):
|
||||
"""Повторный s3-init: код 0, bucket уже есть, приватность подтверждена"""
|
||||
output = compose('run', '--rm', '--no-deps', 's3-init')
|
||||
self.assertIn('already exists, nothing to do', output)
|
||||
self.assertIn('is private', output)
|
||||
|
||||
def test_no_original_bytes_in_database_or_api_container(self):
|
||||
"""В attachments единственный bytea — sha256 (байтов файла в БД нет); после загрузки 2 МБ в /tmp реплик API пусто"""
|
||||
binary = sql("SELECT string_agg(column_name, ',') FROM information_schema.columns "
|
||||
"WHERE table_name = 'attachments' AND data_type = 'bytea'")
|
||||
self.assertEqual(binary, 'sha256')
|
||||
self.assertEqual(upload(self.alice, self.chat, noise_png(2 * 1024 * 1024))[0], 201)
|
||||
for container in compose('ps', '-q', 'api').split():
|
||||
files = subprocess.run(['docker', 'exec', container, 'sh', '-c', 'find /tmp -type f | wc -l'],
|
||||
capture_output=True, text=True).stdout.strip()
|
||||
self.assertEqual(files, '0', container[:12])
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Usage (стенд запущен; на время эксперимента лучше остановить другие стенды):
|
||||
python student-tests/upload_memory.py --output evidence/upload-memory.json
|
||||
|
||||
N одновременных загрузок PNG ~10 MiB через LB при двух настройках буфера тела запроса
|
||||
(UPLOAD_MEMORY_BYTES = 1 MiB в RAM, дальше временный файл — рабочая настройка; и 16 MiB — весь файл в RAM).
|
||||
Перед каждым прогоном реплики перезапускаются (memory.peak cgroup считается с момента старта контейнера);
|
||||
во время прогона каждые 0,1 с снимаются memory.current и anon/file из memory.stat каждой реплики.
|
||||
"""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
|
||||
from helpers import BASE_URL, api_containers, compose, new_chat, new_user, wait_in_rotation, wait_ready
|
||||
from imagegen import noise_png
|
||||
|
||||
MIB = 1024 * 1024
|
||||
SAMPLER = ('while true; do echo "current $(cat /sys/fs/cgroup/memory.current)"; '
|
||||
'grep -E "^(anon|file) " /sys/fs/cgroup/memory.stat; sleep 0.1; done')
|
||||
|
||||
|
||||
class Sampler:
|
||||
def __init__(self, container):
|
||||
self.container, self.peaks = container, {'current': 0, 'anon': 0, 'file': 0}
|
||||
self.process = subprocess.Popen(['docker', 'exec', container, 'sh', '-c', SAMPLER],
|
||||
stdout=subprocess.PIPE, text=True)
|
||||
self.thread = threading.Thread(target=self.read, daemon=True)
|
||||
self.thread.start()
|
||||
|
||||
def read(self):
|
||||
for line in self.process.stdout:
|
||||
name, _, value = line.partition(' ')
|
||||
if name in self.peaks and value.strip().isdigit():
|
||||
self.peaks[name] = max(self.peaks[name], int(value))
|
||||
|
||||
def stop(self):
|
||||
self.process.terminate()
|
||||
self.process.wait()
|
||||
return {name: round(value / MIB, 1) for name, value in self.peaks.items()}
|
||||
|
||||
|
||||
def cgroup(container, name):
|
||||
return int(subprocess.run(['docker', 'exec', container, 'cat', '/sys/fs/cgroup/' + name],
|
||||
capture_output=True, text=True).stdout.strip())
|
||||
|
||||
|
||||
def post(token, chat_id, data):
|
||||
target = urllib.parse.urlsplit(BASE_URL)
|
||||
connection = http.client.HTTPConnection(target.hostname, target.port, timeout=120)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
connection.request('POST', '/api/v1/chats/%s/attachments?filename=big.png' % chat_id, body=data,
|
||||
headers={'Authorization': 'Bearer ' + token, 'Content-Type': 'image/png'})
|
||||
status = connection.getresponse().status
|
||||
except OSError as error:
|
||||
status = type(error).__name__
|
||||
connection.close()
|
||||
return status, time.monotonic() - started
|
||||
|
||||
|
||||
def restart_api(memory_bytes):
|
||||
env = dict(os.environ, UPLOAD_MEMORY_BYTES=str(memory_bytes))
|
||||
compose('up', '-d', '--no-deps', '--no-build', '--force-recreate', 'api', env=env)
|
||||
time.sleep(3)
|
||||
wait_ready()
|
||||
for container in api_containers():
|
||||
wait_in_rotation(container)
|
||||
|
||||
|
||||
def run(label, memory_bytes, parallel, data):
|
||||
restart_api(memory_bytes)
|
||||
alice = new_user('memory')
|
||||
chat = new_chat(alice)
|
||||
replicas = api_containers()
|
||||
idle = {c[:12]: round(cgroup(c, 'memory.current') / MIB, 1) for c in replicas}
|
||||
samplers = [Sampler(c) for c in replicas]
|
||||
started = time.monotonic()
|
||||
with ThreadPoolExecutor(max_workers=parallel) as pool:
|
||||
results = list(pool.map(lambda _: post(alice['token'], chat['id'], data), range(parallel)))
|
||||
wall = time.monotonic() - started
|
||||
time.sleep(0.5)
|
||||
sampled = {c[:12]: s.stop() for c, s in zip(replicas, samplers)}
|
||||
peak = {c[:12]: round(cgroup(c, 'memory.peak') / MIB, 1) for c in replicas}
|
||||
limit = round(cgroup(replicas[0], 'memory.max') / MIB)
|
||||
statuses = [r[0] for r in results]
|
||||
summary = {'label': label, 'upload_memory_bytes': memory_bytes, 'parallel': parallel,
|
||||
'file_mib': round(len(data) / MIB, 2), 'ok': statuses.count(201),
|
||||
'errors': sorted({str(s) for s in statuses if s != 201}), 'wall_s': round(wall, 2),
|
||||
'max_request_s': round(max(r[1] for r in results), 2), 'limit_mib': limit,
|
||||
'idle_current_mib': idle, 'peak_since_start_mib': peak, 'sampled_peak_mib': sampled}
|
||||
worst_anon = max(v['anon'] for v in sampled.values())
|
||||
print('%-28s ok=%2d/%d wall=%5.1fs peak(memory.peak)=%s anon max=%.1f MiB'
|
||||
% (label, summary['ok'], parallel, wall, list(peak.values()), worst_anon))
|
||||
return summary
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--output')
|
||||
parser.add_argument('--parallel', default='10,20')
|
||||
args = parser.parse_args()
|
||||
data = noise_png(10 * MIB - 50000)
|
||||
runs = [run('spool 1 MiB, %s parallel' % n, MIB, int(n), data) for n in args.parallel.split(',')]
|
||||
runs.append(run('all in RAM, %s parallel' % args.parallel.split(',')[-1], 16 * MIB,
|
||||
int(args.parallel.split(',')[-1]), data))
|
||||
restart_api(MIB)
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as file:
|
||||
json.dump({'runs': runs}, file, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user