Добавить тесты вложений и хранилища, проверку согласованности и исследование сверщика

This commit is contained in:
Pavel Grigorev
2026-11-13 16:22:54 +03:00
parent 064a6c1ce5
commit f84831d0bb
21 changed files with 2105 additions and 0 deletions
+31
View File
@@ -0,0 +1,31 @@
name: Application
on: [push, pull_request]
permissions:
contents: read
jobs:
stack:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- run: cp .env.example .env
- run: python scripts/init_secrets.py
- run: docker compose config --quiet
- run: docker compose up -d --build --wait
- run: make test BASE_URL=http://127.0.0.1:8080 PYTHON=python
- run: make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080 PYTHON=python
- run: make student-test BASE_URL=http://127.0.0.1:8080 PYTHON=python
- run: python student-tests/expiry_check.py --ttl 2
- run: python student-tests/failure_check.py
- run: python student-tests/consistency_check.py
- run: make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://127.0.0.1:8080 PYTHON=python
- run: docker compose start api && sleep 15
- run: python student-tests/failover_series.py
- run: python student-tests/rolling_update_check.py --no-build
- if: failure()
run: docker compose ps -a && docker compose logs --no-color
- if: always()
run: docker compose down -v
+43
View File
@@ -0,0 +1,43 @@
"""Операции с S3 для внедрения сбоев в треке 2. Запускается внутри контейнера reconcile (ключи сверщика):
python research/s3ops.py put SIZE < keys # создать объекты указанного размера
python research/s3ops.py delete < keys # удалить объекты
python research/s3ops.py exists < keys # вывести число существующих
"""
from concurrent.futures import ThreadPoolExecutor
import sys
sys.path.insert(0, 'src')
from botocore.exceptions import ClientError # noqa: E402
from config import setting, settings # noqa: E402
from s3 import make_client # noqa: E402
client = make_client(setting('S3_RECONCILER_ACCESS_KEY_ID'), setting('S3_RECONCILER_SECRET_ACCESS_KEY'), timeout=30)
def exists(key):
try:
client.head_object(Bucket=settings.s3_bucket, Key=key)
return True
except ClientError:
return False
def main():
command = sys.argv[1]
keys = [line.strip() for line in sys.stdin if line.strip()]
with ThreadPoolExecutor(max_workers=16) as pool:
if command == 'put':
body = b'x' * int(sys.argv[2])
list(pool.map(lambda k: client.put_object(Bucket=settings.s3_bucket, Key=k, Body=body), keys))
print(len(keys))
elif command == 'delete':
list(pool.map(lambda k: client.delete_object(Bucket=settings.s3_bucket, Key=k), keys))
print(len(keys))
elif command == 'exists':
print(sum(pool.map(exists, keys)))
if __name__ == '__main__':
main()
+282
View File
@@ -0,0 +1,282 @@
"""Трек 2: согласованность PostgreSQL и S3 — исследование сверщика (src/reconcile.py).
Стенд запущен (docker compose up -d). BASE_URL — адрес LB (по умолчанию http://127.0.0.1:8080).
python research/track2.py scenario # H1/H2: внедрённые сбои → dry-run → apply → повторный apply
python research/track2.py race # H2: apply в цикле под непрерывными загрузками
python research/track2.py cost # H3: время/память обхода при 1k…20k объектов
python research/track2.py cleanup # удалить синтетические данные исследования
python research/track2.py report # evidence/track2/results.md
python research/track2.py all
Синтетические строки живут в отдельном чате пользователя track2_*; cleanup удаляет их и их объекты.
На время запуска периодический сервис reconciler останавливается и затем запускается снова.
"""
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
from pathlib import Path
import statistics
import subprocess
import sys
import threading
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'student-tests'))
from helpers import call, compose, new_chat, new_user, sql # noqa: E402
from imagegen import SAMPLE_PNG # noqa: E402
from test_attachments import download, upload # noqa: E402
OUT = ROOT / 'evidence' / 'track2'
STATE = OUT / 'state.json'
GRACE = 30
def save(name, data):
OUT.mkdir(parents=True, exist_ok=True)
(OUT / name).write_text(json.dumps(data, ensure_ascii=False, indent=2, default=str) + '\n', encoding='utf-8')
print('saved evidence/track2/' + name)
def load(name):
return json.loads((OUT / name).read_text(encoding='utf-8'))
def s3ops(command, keys, *args):
result = subprocess.run(['docker', 'compose', 'run', '--rm', '-T', '-v', str(ROOT / 'research') + ':/app/research:ro',
'--entrypoint', 'python', 'reconcile', 'research/s3ops.py', command, *map(str, args)],
cwd=ROOT, input='\n'.join(keys), capture_output=True, text=True, encoding='utf-8')
if result.returncode:
raise SystemExit(result.stderr[-2000:])
return int(result.stdout.strip().splitlines()[-1])
def reconcile(*args):
return json.loads(compose('run', '--rm', 'reconcile', *map(str, args)))
def research_chat():
if STATE.exists():
return json.loads(STATE.read_text(encoding='utf-8'))
user = new_user('track2')
chat = new_chat(user)
state = {'user_id': user['id'], 'chat_id': chat['id'], 'token': None}
OUT.mkdir(parents=True, exist_ok=True)
STATE.write_text(json.dumps(state), encoding='utf-8')
return state
def insert_rows(state, count, status='ready', age_seconds=0, size=1024):
"""Синтетические строки attachments в чате исследования; возвращает ключи объектов."""
if count <= 0:
return []
out = sql("WITH new AS (INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
"object_key, status, created_at, updated_at) SELECT id, '%s', '%s', 'synthetic.png', 'image/png', %d, "
"decode(repeat('00', 32), 'hex'), 'attachments/' || id, '%s', now() - make_interval(secs => %d), now() "
"FROM (SELECT gen_random_uuid() AS id FROM generate_series(1, %d)) g RETURNING object_key) "
"SELECT string_agg(object_key, ' ') FROM new"
% (state['chat_id'], state['user_id'], size, status, age_seconds, count))
return out.split()
def reference(state, keys):
"""Привязать вложения к сообщению (используемые вложения)."""
message = sql("INSERT INTO messages (chat_id, sender_id, text) VALUES ('%s', '%s', convert_to('track2', 'UTF8')) "
"RETURNING id" % (state['chat_id'], state['user_id'])).splitlines()[0]
sql("INSERT INTO message_attachments (message_id, attachment_id, position) SELECT '%s', a.id, "
"row_number() OVER () FROM attachments a WHERE a.object_key IN (%s)"
% (message, ','.join("'%s'" % k for k in keys)))
def orphan_keys(count):
return ['attachments/%s' % uuid.uuid4() for _ in range(count)]
def scenario():
state = research_chat()
real = new_user('track2real')
real_chat = new_chat(real)
real_att = upload(real, real_chat, SAMPLE_PNG)[2]
call('POST', '/api/v1/chats/%s/messages' % real_chat['id'], {'text': '', 'attachment_ids': [real_att['id']]},
real['token'])
baseline = reconcile('--grace-seconds', GRACE)
ok = insert_rows(state, 200)
s3ops('put', ok, 1024)
referenced = ok[:20]
reference(state, referenced)
missing = ok[20:30] + referenced[:5]
s3ops('delete', missing)
mismatch = ok[40:50]
s3ops('put', mismatch, 2048)
stale_with_object = insert_rows(state, 25, 'uploading', 3600)
s3ops('put', stale_with_object, 1024)
stale_without_object = insert_rows(state, 25, 'uploading', 3600)
old_orphans = orphan_keys(25)
s3ops('put', old_orphans, 1024)
print('waiting %ds so that injected orphans become older than grace…' % (GRACE + 3))
time.sleep(GRACE + 3)
fresh_uploads = insert_rows(state, 10, 'uploading', 0)
s3ops('put', fresh_uploads, 1024)
fresh_orphans = orphan_keys(10)
s3ops('put', fresh_orphans, 1024)
expected = {'orphan': 25, 'fresh_orphan': 10, 'stale_upload': 50, 'fresh_upload': 10, 'missing': 15,
'size_mismatch': 10}
base = baseline['counts']
dry = reconcile('--grace-seconds', GRACE)
dry_again = reconcile('--grace-seconds', GRACE)
applied = reconcile('--apply', '--grace-seconds', GRACE)
rerun = reconcile('--apply', '--grace-seconds', GRACE)
after = reconcile('--grace-seconds', GRACE)
detected = {k: dry['counts'][k] - base.get(k, 0) for k in expected}
real_download = download(real, real_att['id'])
checks = {
'detected_equals_injected': detected == expected,
'dry_run_changes_nothing': dry_again['counts'] == dry['counts'] and dry_again['objects_listed'] == dry['objects_listed'],
'apply_actions': applied['actions'],
'apply_deleted_exactly_old_orphans_and_stale': applied['actions'] == {
'orphan_objects_deleted': 25, 'stale_rows_deleted': 50, 'stale_objects_deleted': 25, 'skipped': 0},
'rerun_is_noop': not any(rerun['actions'][k] for k in ('orphan_objects_deleted', 'stale_rows_deleted', 'stale_objects_deleted')),
'fresh_orphans_kept': s3ops('exists', fresh_orphans) == 10,
'fresh_uploads_kept': s3ops('exists', fresh_uploads) == 10 and after['counts']['fresh_upload'] >= 10,
'old_orphans_gone': s3ops('exists', old_orphans) == 0,
'stale_objects_gone': s3ops('exists', stale_with_object) == 0,
'referenced_rows_kept': int(sql("SELECT count(*) FROM attachments WHERE object_key IN (%s)"
% ','.join("'%s'" % k for k in referenced))) == 20,
'missing_still_reported_not_deleted': after['counts']['missing'] - base.get('missing', 0) == 15,
'real_attachment_intact': real_download[0] == 200 and hashlib.sha256(real_download[2]).hexdigest() == real_att['sha256'],
}
passed = all(v for k, v in checks.items() if k != 'apply_actions')
save('scenario.json', {'grace_seconds': GRACE, 'injected': expected, 'baseline_counts': base,
'dry_run': dry, 'apply': applied, 'rerun': rerun, 'after': after,
'detected': detected, 'checks': checks, 'passed': passed})
for name, value in checks.items():
print(' %-48s %s' % (name, value))
print('PASS' if passed else 'FAIL')
def race(seconds=40):
user = new_user('track2race')
chat = new_chat(user)
uploaded, errors, stop = [], [], threading.Event()
def uploader():
while not stop.is_set():
status, _, body = upload(user, chat, SAMPLE_PNG)
(uploaded.append(body['id']) if status == 201 else errors.append(status))
preclean = reconcile('--apply', '--grace-seconds', 0)['actions']
threads = [threading.Thread(target=uploader) for _ in range(4)]
for thread in threads:
thread.start()
runs, started = [], time.monotonic()
while time.monotonic() - started < seconds:
report = reconcile('--apply', '--grace-seconds', GRACE)
runs.append({'counts': report['counts'], 'actions': report['actions'], 'total_s': report['seconds']['total']})
stop.set()
for thread in threads:
thread.join()
intact = 0
for attachment_id in uploaded:
status, _, body = download(user, attachment_id)
intact += status == 200 and body == SAMPLE_PNG
result = {'seconds': seconds, 'grace_seconds': GRACE, 'preclean_before_load': preclean,
'reconcile_runs': len(runs), 'uploads_201': len(uploaded),
'upload_errors': errors, 'downloaded_intact': intact,
'deleted_by_reconciler': sum(r['actions']['orphan_objects_deleted'] + r['actions']['stale_rows_deleted']
for r in runs),
'fresh_uploads_seen_in_flight': sum(r['counts']['fresh_upload'] for r in runs), 'runs': runs}
result['passed'] = intact == len(uploaded) and result['deleted_by_reconciler'] == 0
save('race.json', result)
print('race: %d uploads, %d intact, reconciler runs %d, deleted %d, in-flight uploads seen %d → %s'
% (len(uploaded), intact, len(runs), result['deleted_by_reconciler'], result['fresh_uploads_seen_in_flight'],
'PASS' if result['passed'] else 'FAIL'))
def cost(sizes=(1000, 5000, 10000, 20000), repeats=3):
state = research_chat()
rows = []
for target in sizes:
current = int(sql("SELECT count(*) FROM attachments"))
if target > current:
keys = insert_rows(state, target - current)
started = time.monotonic()
for start in range(0, len(keys), 2000):
s3ops('put', keys[start:start + 2000], 256)
print(' +%d objects in %.1fs' % (len(keys), time.monotonic() - started))
reports = [reconcile('--grace-seconds', GRACE) for _ in range(repeats)]
row = {'objects': reports[-1]['objects_listed'], 'rows': reports[-1]['rows_scanned'],
'list_pages': reports[-1]['list_pages'],
'list_s3_s': statistics.median(r['seconds']['list_s3'] for r in reports),
'scan_db_s': statistics.median(r['seconds']['scan_db'] for r in reports),
'total_s': statistics.median(r['seconds']['total'] for r in reports),
'max_rss_mib': max(r['max_rss_mib'] for r in reports)}
row['us_per_object'] = round(row['total_s'] / row['objects'] * 1e6, 1)
rows.append(row)
print(' %(objects)6d objects: list %(list_s3_s).2fs, db %(scan_db_s).3fs, total %(total_s).2fs, '
'rss %(max_rss_mib).0f MiB, %(us_per_object).0f µs/object' % row)
save('cost.json', {'repeats': repeats, 'rows': rows})
def cleanup():
if not STATE.exists():
print('nothing to clean')
return
state = json.loads(STATE.read_text(encoding='utf-8'))
keys = sql("SELECT coalesce(string_agg(object_key, ' '), '') FROM attachments WHERE chat_id = '%s'" % state['chat_id']).split()
for start in range(0, len(keys), 2000):
s3ops('delete', keys[start:start + 2000])
sql("DELETE FROM message_attachments WHERE message_id IN (SELECT id FROM messages WHERE chat_id = '%s')" % state['chat_id'])
sql("DELETE FROM messages WHERE chat_id = '%s'" % state['chat_id'])
sql("DELETE FROM attachments WHERE chat_id = '%s'" % state['chat_id'])
leftovers = reconcile('--apply', '--grace-seconds', 0)
STATE.unlink()
print('cleanup: %d research objects deleted; reconciler afterwards: %s' % (len(keys), leftovers['counts']))
def report():
s, r, c = load('scenario.json'), load('race.json'), load('cost.json')
lines = ['# Трек 2: результаты', '', '## H1/H2. Внедрённые сбои (grace %d с)' % s['grace_seconds'], '',
'| категория | внедрено | найдено dry-run | действие apply |', '|---|---|---|---|']
actions = {'orphan': 'удалено объектов: %d' % s['apply']['actions']['orphan_objects_deleted'],
'stale_upload': 'удалено строк: %d, объектов: %d' % (s['apply']['actions']['stale_rows_deleted'],
s['apply']['actions']['stale_objects_deleted']),
'fresh_orphan': 'не тронуто', 'fresh_upload': 'не тронуто', 'missing': 'только отчёт',
'size_mismatch': 'только отчёт'}
for name, count in s['injected'].items():
lines.append('| %s | %d | %d | %s |' % (name, count, s['detected'][name], actions[name]))
lines += ['', '| проверка | результат |', '|---|---|']
lines += ['| %s | %s |' % (k, v) for k, v in s['checks'].items() if k != 'apply_actions']
lines += ['', '## H2. Сверщик под нагрузкой загрузок', '',
'%d с, 4 потока загрузок, apply с grace %d с в цикле: запусков сверщика %d, загрузок 201 — %d, '
'скачано без изменений — %d, удалено сверщиком — %d, «свежих» загрузок в полёте увидено — %d.'
% (r['seconds'], r['grace_seconds'], r['reconcile_runs'], r['uploads_201'], r['downloaded_intact'],
r['deleted_by_reconciler'], r['fresh_uploads_seen_in_flight']),
'', '## H3. Стоимость обхода (медиана из %d)' % c['repeats'], '',
'| объектов | строк | страниц листинга | листинг S3, с | скан БД, с | всего, с | мкс/объект | RSS, MiB |',
'|---|---|---|---|---|---|---|---|']
for row in c['rows']:
lines.append('| %(objects)d | %(rows)d | %(list_pages)d | %(list_s3_s).2f | %(scan_db_s).3f | %(total_s).2f | '
'%(us_per_object).0f | %(max_rss_mib).0f |' % row)
(OUT / 'results.md').write_text('\n'.join(lines) + '\n', encoding='utf-8')
print('saved evidence/track2/results.md')
COMMANDS = {'scenario': scenario, 'race': race, 'cost': cost, 'cleanup': cleanup, 'report': report}
if __name__ == '__main__':
command = sys.argv[1] if len(sys.argv) > 1 else 'all'
# Периодический сервис reconciler (--apply, grace 900 с) удалил бы состаренные синтетические заготовки
# посреди эксперимента; на время исследования он остановлен, сверщик запускается явно.
paused = 'reconciler' in compose('ps', '--status', 'running', '--services')
if paused:
compose('stop', 'reconciler')
try:
for step in (['scenario', 'race', 'cost', 'report', 'cleanup'] if command == 'all' else [command]):
print('==', step)
COMMANDS[step]()
finally:
if paused:
compose('start', 'reconciler')
+156
View File
@@ -0,0 +1,156 @@
"""Usage (стенд запущен через docker compose up -d):
python student-tests/consistency_check.py
Проверяет, что сбои при загрузке не оставляют вечных неучтённых данных:
1) клиент обрывает загрузку посреди тела → ни строки, ни объекта;
2) S3 остановлен → upload/download 503, ни строки, ни объекта; метаданные, текст и ready работают;
3) процесс «падает» после S3 PUT до COMMIT ready (FAULTS=after_s3_put, только в этом тесте) →
остаётся строка uploading + объект; сверщик не трогает их в пределах grace и убирает после;
4) объект без строки в БД (orphan) → свежий не трогается, старый удаляется; повторный запуск ничего не делает;
5) периодический сервис reconciler сам удаляет заготовку старше grace (без ручного запуска).
"""
import json
import os
import socket
import time
import urllib.parse
import uuid
from helpers import BASE_URL, call, compose, new_chat, new_user, sql, wait_ready
from imagegen import SAMPLE_PNG
from test_attachments import download, upload
failures = []
def check(title, ok, detail=''):
print(' %-62s %s %s' % (title, 'OK' if ok else 'FAIL', detail))
if not ok:
failures.append(title)
def reconcile(*args):
return json.loads(compose('run', '--rm', 'reconcile', *args))
def s3_python(code):
return compose('run', '--rm', '--entrypoint', 'python', 'reconcile', '-c',
'import sys; sys.path.insert(0, "src")\n' + code)
def counts():
return sql("SELECT count(*) FILTER (WHERE status = 'ready') || '/' || count(*) FILTER (WHERE status = 'uploading') "
"FROM attachments")
def object_exists(key):
return s3_python('from config import setting, settings\nfrom s3 import make_client\n'
'c = make_client(setting("S3_RECONCILER_ACCESS_KEY_ID"), setting("S3_RECONCILER_SECRET_ACCESS_KEY"))\n'
'keys = [o["Key"] for o in c.list_objects_v2(Bucket=settings.s3_bucket, Prefix=%r).get("Contents", [])]\n'
'print(len(keys))' % key).strip() == '1'
def restart_api(faults=None):
env = dict(os.environ)
env['FAULTS'] = faults or ''
compose('up', '-d', '--no-deps', '--no-build', 'api', env=env)
time.sleep(3)
wait_ready()
time.sleep(3)
def interrupted_upload(alice, chat):
target = urllib.parse.urlsplit(BASE_URL)
with socket.create_connection((target.hostname, target.port), timeout=10) as sock:
head = ('POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: x\r\n'
'Authorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n'
% (chat['id'], alice['token'], 5 * 1024 * 1024))
sock.sendall(head.encode() + SAMPLE_PNG + bytes(512 * 1024))
time.sleep(2)
def main():
alice = new_user('consistency')
chat = new_chat(alice)
print('1. Обрыв загрузки посреди тела')
before = counts()
interrupted_upload(alice, chat)
check('строк не прибавилось (ready/uploading)', counts() == before, '%s -> %s' % (before, counts()))
print('2. S3 остановлен: загрузка, скачивание, остальной API')
stored = upload(alice, chat, SAMPLE_PNG)[2]
before = counts()
compose('stop', 's3')
try:
started = time.monotonic()
status = upload(alice, chat, SAMPLE_PNG)[0]
check('upload при остановленном S3 → 503', status == 503, '%s за %.1f с' % (status, time.monotonic() - started))
check('заготовка удалена компенсацией', counts() == before, '%s -> %s' % (before, counts()))
started = time.monotonic()
status = download(alice, stored['id'])[0]
check('download при остановленном S3 → 503', status == 503, '%s за %.1f с' % (status, time.monotonic() - started))
check('метаданные вложения доступны (они в PostgreSQL)',
call('GET', '/api/v1/attachments/' + stored['id'], token=alice['token'])[0] == 200)
check('текстовое сообщение → 201', call('POST', '/api/v1/chats/%s/messages' % chat['id'],
{'text': 'без S3'}, alice['token'])[0] == 201)
check('/health/ready → 200 (S3 не критичен для текстового API)', call('GET', '/health/ready')[0] == 200)
finally:
compose('up', '-d', '--wait', 's3')
check('после старта S3 upload снова 201', upload(alice, chat, SAMPLE_PNG)[0] == 201)
print('3. Сбой после S3 PUT до фиксации ready (FAULTS=after_s3_put)')
restart_api('after_s3_put')
try:
status = upload(alice, chat, SAMPLE_PNG, filename='fault-%s.png' % uuid.uuid4().hex[:6])[0]
row = sql("SELECT id || ' ' || object_key FROM attachments WHERE status = 'uploading' "
"AND filename LIKE 'fault-%%' ORDER BY created_at DESC LIMIT 1").split()
check('клиент получил ошибку (не 201)', status != 201, str(status))
check('осталась строка uploading и объект в S3', bool(row) and object_exists(row[1]), ' '.join(row))
finally:
restart_api()
fresh = reconcile('--grace-seconds', '900')
check('dry-run, grace 900 с: заготовка свежая, не трогается', fresh['counts']['fresh_upload'] >= 1
and fresh['counts']['stale_upload'] == 0, str(fresh['counts']))
applied = reconcile('--apply', '--grace-seconds', '0')
check('apply, grace 0: строка и объект удалены', applied['actions']['stale_rows_deleted'] >= 1
and applied['actions']['stale_objects_deleted'] >= 1, str(applied['actions']))
check('объекта больше нет', not object_exists(row[1]))
print('4. Объект без строки в БД (orphan)')
key = 'attachments/%s' % uuid.uuid4()
s3_python('from config import setting, settings\nfrom s3 import make_client\n'
'c = make_client(setting("S3_RECONCILER_ACCESS_KEY_ID"), setting("S3_RECONCILER_SECRET_ACCESS_KEY"))\n'
'c.put_object(Bucket=settings.s3_bucket, Key=%r, Body=b"orphan")' % key)
fresh = reconcile('--grace-seconds', '900')
check('dry-run, grace 900 с: свежая сирота не трогается', fresh['counts']['fresh_orphan'] >= 1
and fresh['counts']['orphan'] == 0, str(fresh['counts']))
applied = reconcile('--apply', '--grace-seconds', '0')
check('apply, grace 0: сирота удалена', applied['actions']['orphan_objects_deleted'] >= 1, str(applied['actions']))
again = reconcile('--apply', '--grace-seconds', '0')
check('повторный запуск ничего не делает', not any(again['actions'][k] for k in
('orphan_objects_deleted', 'stale_rows_deleted', 'stale_objects_deleted')), str(again['actions']))
check('рабочие вложения не пострадали (missing = 0)', again['counts']['missing'] == 0, str(again['counts']))
print('5. Периодический сервис reconciler убирает зависшую заготовку сам')
stale = str(uuid.uuid4())
sql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, "
"status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'stale.png', 'image/png', 1, "
"decode(repeat('00', 32), 'hex'), 'attachments/%s', 'uploading', now() - interval '1 hour', "
"now() - interval '1 hour')" % (stale, chat['id'], alice['id'], stale))
running = 'reconciler' in compose('ps', '--status', 'running', '--services')
check('сервис reconciler запущен (docker compose ps)', running)
if running:
compose('restart', 'reconciler') # проход выполняется сразу после старта, дальше — раз в интервал
deadline = time.monotonic() + 60
while time.monotonic() < deadline and sql("SELECT count(*) FROM attachments WHERE id = '%s'" % stale) != '0':
time.sleep(1)
check('заготовка старше grace удалена без ручного запуска',
sql("SELECT count(*) FROM attachments WHERE id = '%s'" % stale) == '0')
print('FAIL: ' + ', '.join(failures) if failures else 'PASS: сбои загрузки не оставляют неучтённых данных')
raise SystemExit(1 if failures else 0)
if __name__ == '__main__':
main()
+56
View File
@@ -0,0 +1,56 @@
"""Usage (стенд уже запущен через docker compose up -d):
python student-tests/expiry_check.py --ttl 2
Скрипт пересоздаёт только api с SESSION_TTL_SECONDS=<ttl>, проверяет истечение сессии,
затем возвращает api с обычным TTL из .env. Volumes и данные не трогаются.
"""
import argparse
import hashlib
import os
import time
from helpers import call, compose, new_user, wait_ready
def restart_api(ttl=None):
env = dict(os.environ)
if ttl is None:
env.pop('SESSION_TTL_SECONDS', None)
else:
env['SESSION_TTL_SECONDS'] = str(ttl)
compose('up', '-d', '--no-deps', '--no-build', 'api', env=env)
wait_ready()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--ttl', type=int, default=2)
args = parser.parse_args()
print('пересоздаю api с SESSION_TTL_SECONDS=%d' % args.ttl)
restart_api(args.ttl)
try:
alice = new_user('expiry')
status = call('GET', '/api/v1/users/me', token=alice['token'])[0]
print('сразу после входа: /users/me -> %d' % status)
assert status == 200
key = 'session:' + hashlib.sha256(alice['token'].encode()).hexdigest()
cache_ttl = int(compose('exec', '-T', 'valkey', 'valkey-cli', 'TTL', key))
print('TTL ключа этой сессии в Valkey: %d с (-2 = ключа нет)' % cache_ttl)
assert cache_ttl <= args.ttl, 'кэш живёт дольше сессии'
time.sleep(args.ttl + 1.5)
status, headers, _ = call('GET', '/api/v1/users/me', token=alice['token'])
print('через %.1f с: /users/me -> %d, WWW-Authenticate: %s' % (args.ttl + 1.5, status, headers['WWW-Authenticate']))
assert status == 401 and headers['WWW-Authenticate'] == 'Bearer realm="messenger"'
status = call('DELETE', '/api/v1/auth/sessions/current', token=alice['token'])[0]
print('logout истёкшим токеном -> %d' % status)
assert status == 401
print('PASS: сессия истекла по SESSION_TTL_SECONDS=%d, кэш не продлил её' % args.ttl)
finally:
print('возвращаю api с TTL из .env')
restart_api()
if __name__ == '__main__':
main()
+71
View File
@@ -0,0 +1,71 @@
"""Usage (стенд запущен через docker compose up -d, обе реплики в ротации):
python student-tests/failover_series.py --output evidence/failover-series.json
Для каждой API-реплики по очереди и в двух режимах (docker stop — штатная остановка с выводом из ротации,
docker kill — внезапная смерть процесса) под нагрузкой: останавливает реплику, меряет ошибки, RPS,
время до последнего ответа остановленной реплики, восстанавливает её (docker compose start api)
и ждёт возврата в ротацию LB. Volumes, БД и LB не трогаются.
"""
import argparse
import json
import subprocess
import time
from helpers import Load, api_containers, compose, new_chat, new_user, wait_in_rotation
WARMUP, AFTER = 5, 10
def run(container, mode):
alice = new_user('series')
chat = new_chat(alice)
load = Load(alice['token'], chat['id']).start()
time.sleep(WARMUP)
acted = time.monotonic()
command = ['docker', 'stop', '-t', '15', container] if mode == 'stop' else ['docker', 'kill', container]
subprocess.run(command, capture_output=True, check=True)
action_seconds = time.monotonic() - acted
time.sleep(AFTER)
load.stop()
instance = container[:12]
target = [r for r in load.records if r[2] == 201 and r[3] == instance and r[0] >= acted]
others = [r for r in load.records if r[2] == 201 and r[3] != instance and r[0] >= acted]
restored = time.monotonic()
compose('start', 'api')
back = wait_in_rotation(container)
result = {
'replica': instance, 'mode': mode, 'action_seconds': round(action_seconds, 2),
'target_served_until_s': round(max(r[1] for r in target) - acted, 2) if target else 0.0,
'first_success_on_other_s': round(min(r[1] for r in others) - acted, 2) if others else None,
'before': load.window(acted - WARMUP, acted),
'during_10s': load.window(acted, acted + AFTER),
'back_in_rotation_s': round(back - restored, 2),
'data': load.verify(alice['token']),
}
print('%-12s %-4s errors=%-3d rps %5.1f -> %5.1f target last answer +%.2fs restore %.1fs lost=%d dup=%d'
% (instance, mode, result['during_10s']['errors'], result['before']['rps'], result['during_10s']['rps'],
result['target_served_until_s'], result['back_in_rotation_s'], result['data']['lost'],
result['data']['duplicate_texts']))
return result
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--output')
parser.add_argument('--modes', default='stop,kill')
args = parser.parse_args()
replicas = api_containers()
assert len(replicas) >= 2, 'need at least two running API replicas'
results = [run(container, mode) for mode in args.modes.split(',') for container in replicas]
bad = [r for r in results if r['data']['lost'] or r['data']['duplicate_texts'] or r['data']['duplicate_ids']]
summary = {'replicas': [c[:12] for c in replicas], 'warmup_s': WARMUP, 'observe_s': AFTER, 'workers': 4,
'lb_check': 'GET /health/ready every 1s, fall 2, rise 2', 'results': results}
if args.output:
with open(args.output, 'w', encoding='utf-8') as file:
json.dump(summary, file, ensure_ascii=False, indent=2)
print('FAIL: data loss or duplicates' if bad else 'PASS: every replica stopped once per mode; no loss, no duplicates')
raise SystemExit(1 if bad else 0)
if __name__ == '__main__':
main()
+127
View File
@@ -0,0 +1,127 @@
"""Usage (стенд уже запущен через docker compose up -d):
python student-tests/failure_check.py
Останавливает и замораживает Valkey и PostgreSQL (docker compose stop/pause, без удаления volumes),
проверяет поведение API и восстановление.
"""
import time
import uuid
from helpers import basic, call, compose, new_chat, new_user, wait_ready
failures = []
def timed(method, path, token=None, body=None, headers=None):
started = time.monotonic()
try:
status = call(method, path, body=body, token=token, headers=headers, timeout=10)[0]
except OSError as error:
status = type(error).__name__
return status, time.monotonic() - started
def expect(title, result, allowed, max_seconds):
status, seconds = result
ok = status in allowed and seconds <= max_seconds
print(' %-48s -> %-4s за %.2f с %s' % (title, status, seconds, 'OK' if ok else 'FAIL'))
if not ok:
failures.append(title)
def main():
alice = new_user('failure')
chat = new_chat(alice)
history = '/api/v1/chats/%s/messages' % chat['id']
assert call('POST', history, {'text': 'до отказа'}, alice['token'])[0] == 201
fake = uuid.uuid4().hex * 2
print('1. Valkey остановлен: кэш обходится, всё работает через PostgreSQL')
compose('stop', 'valkey')
try:
expect('первый запрос: обнаружение отказа кэша', timed('GET', '/api/v1/users/me', alice['token']), [200], 1.5)
expect('GET /users/me действующим токеном', timed('GET', '/api/v1/users/me', alice['token']), [200], 1)
expect('POST сообщение', timed('POST', history, alice['token'], {'text': 'без кэша'}), [201], 1)
expect('неизвестный токен', timed('GET', '/api/v1/users/me', fake), [401], 1)
expect('без Authorization', timed('GET', '/api/v1/users/me'), [401], 1)
expect('/health/ready (Valkey некритичен)', timed('GET', '/health/ready'), [200], 1)
started = time.monotonic()
other = new_user('nocache')
expect('регистрация и вход без кэша', (200, time.monotonic() - started), [200], 1)
expect('logout без кэша', timed('DELETE', '/api/v1/auth/sessions/current', other['token']), [204], 1.5)
expect('отозванный токен', timed('GET', '/api/v1/users/me', other['token']), [401], 1)
finally:
compose('up', '-d', '--wait', 'valkey')
print('1б. Valkey на паузе: logout, затем unpause — отозванный токен не оживает из кэша')
revoked = new_user('paused')
call('GET', '/api/v1/users/me', token=revoked['token'])
compose('pause', 'valkey')
try:
expect('logout при паузе Valkey', timed('DELETE', '/api/v1/auth/sessions/current', revoked['token']), [204], 1.5)
finally:
compose('unpause', 'valkey')
time.sleep(6)
expect('через 6 с после unpause: /users/me', timed('GET', '/api/v1/users/me', revoked['token']), [401], 1)
expect('через 6 с после unpause: сообщение', timed('POST', history, revoked['token'], {'text': 'после logout'}), [401], 1)
print('1в. Valkey на паузе: logout, рестарт API, unpause — отзыв переживает рестарт процесса')
restarted = new_user('restart')
call('GET', '/api/v1/users/me', token=restarted['token'])
compose('pause', 'valkey')
try:
expect('logout при паузе Valkey', timed('DELETE', '/api/v1/auth/sessions/current', restarted['token']), [204], 1.5)
compose('restart', 'api')
wait_ready()
finally:
compose('unpause', 'valkey')
time.sleep(6)
expect('после рестарта API и unpause: /users/me', timed('GET', '/api/v1/users/me', restarted['token']), [401], 1)
expect('после рестарта API и unpause: сообщение', timed('POST', history, restarted['token'], {'text': 'после рестарта'}), [401], 1)
print('2. PostgreSQL остановлен: ограниченная ошибка 503, анонимный запрос не проходит')
compose('stop', 'postgres')
stopped_at = time.monotonic()
try:
expect('GET история', timed('GET', history, alice['token']), [503], 3)
expect('POST сообщение', timed('POST', history, alice['token'], {'text': 'при отказе БД'}), [503], 3)
expect('неизвестный токен (не 200!)', timed('GET', '/api/v1/users/me', fake), [503], 3)
expect('без Authorization', timed('GET', '/api/v1/users/me'), [401], 1)
expect('вход по Basic верным паролем', timed('POST', '/api/v1/auth/sessions',
headers=basic(alice['username'], alice['password'])), [503], 3)
expect('/health/live', timed('GET', '/health/live'), [200], 1)
expect('/health/ready', timed('GET', '/health/ready'), [503], 3)
finally:
compose('up', '-d', '--wait', 'postgres')
ready_at = wait_ready()
print('3. PostgreSQL запущен: готовность через %.1f с после остановки' % (ready_at - stopped_at))
status = call('GET', '/api/v1/users/me', token=alice['token'])[0]
texts = [m['text'] for m in call('GET', history + '?limit=100', token=alice['token'])[2]['items']]
print(' прежний токен -> %d; история: %s' % (status, texts))
if status != 200 or texts != ['до отказа', 'без кэша']:
failures.append('восстановление')
print('4. PostgreSQL заморожен (pause): запросы получают 503 в пределах общего срока, а не зависают')
compose('pause', 'postgres')
try:
expect('GET история', timed('GET', history, alice['token']), [503], 4.5)
expect('POST сообщение', timed('POST', history, alice['token'], {'text': 'при заморозке'}), [503], 4.5)
expect('неизвестный токен (не 200!)', timed('GET', '/api/v1/users/me', fake), [503], 4.5)
expect('вход по Basic', timed('POST', '/api/v1/auth/sessions', headers=basic(alice['username'], alice['password'])), [503], 4.5)
expect('/health/ready', timed('GET', '/health/ready'), [503], 2)
expect('/health/live', timed('GET', '/health/live'), [200], 1)
finally:
compose('unpause', 'postgres')
unpaused = time.monotonic()
wait_ready()
texts = [m['text'] for m in call('GET', history + '?limit=100', token=alice['token'])[2]['items']]
print(' готовность через %.1f с после unpause; история: %s' % (time.monotonic() - unpaused, texts))
if texts != ['до отказа', 'без кэша']:
failures.append('после заморозки')
print('FAIL: ' + ', '.join(failures) if failures else 'PASS: отказы Valkey и PostgreSQL обработаны')
raise SystemExit(1 if failures else 0)
if __name__ == '__main__':
main()
Binary file not shown.

After

Width:  |  Height:  |  Size: 19 KiB

+191
View File
@@ -0,0 +1,191 @@
import base64
import json
import os
from pathlib import Path
import subprocess
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
BASE_URL = os.environ.get('BASE_URL', 'http://127.0.0.1:8080').rstrip('/')
ROOT = Path(__file__).resolve().parents[1]
def call(method, path, body=None, token=None, raw=None, headers=None, timeout=10):
all_headers = {}
if token:
all_headers['Authorization'] = 'Bearer ' + token
if body is not None:
raw = json.dumps(body, ensure_ascii=False).encode('utf-8')
if raw is not None:
all_headers['Content-Type'] = 'application/json'
all_headers.update(headers or {})
if method == 'POST' and path.split('?')[0].endswith('/messages') and 'Idempotency-Key' not in all_headers:
all_headers['Idempotency-Key'] = uuid.uuid4().hex
all_headers = {name: value for name, value in all_headers.items() if value is not None}
request = urllib.request.Request(BASE_URL + path, data=raw, headers=all_headers, method=method)
try:
response = urllib.request.urlopen(request, timeout=timeout)
except urllib.error.HTTPError as error:
response = error
with response:
data = response.read()
is_json = response.headers.get_content_type() == 'application/json'
return response.status, response.headers, json.loads(data) if data and is_json else None
def basic(username, password):
return {'Authorization': 'Basic ' + base64.b64encode((username + ':' + password).encode('utf-8')).decode()}
def register(label='user', password=None):
username = (label + '_' + uuid.uuid4().hex[:12])[:32]
password = password or 'Pw-' + uuid.uuid4().hex
status, _, user = call('POST', '/api/v1/users', {'username': username, 'display_name': label, 'password': password})
assert status == 201, status
return user, password
def login(username, password):
status, _, session = call('POST', '/api/v1/auth/sessions', headers=basic(username, password))
assert status == 201, status
return session
def new_user(label='user'):
user, password = register(label)
session = login(user['username'], password)
return dict(user, password=password, token=session['token'])
def new_chat(owner, *others, title='Тестовый чат'):
members = [owner['id']] + [u['id'] for u in others]
status, _, chat = call('POST', '/api/v1/chats', {'title': title, 'member_ids': members}, owner['token'])
assert status == 201, status
return chat
def read_all_messages(chat_id, token, limit):
items, cursor, pages = [], None, 0
while True:
query = '?limit=%d' % limit + ('&cursor=' + urllib.parse.quote(cursor, safe='') if cursor else '')
status, _, page = call('GET', '/api/v1/chats/%s/messages%s' % (chat_id, query), token=token)
assert status == 200, status
items += page['items']
pages += 1
cursor = page['next_cursor']
if cursor is None:
return items, pages
def compose(*args, env=None, check=True):
result = subprocess.run(['docker', 'compose', *args], cwd=ROOT, capture_output=True, text=True,
encoding='utf-8', env=env)
if check and result.returncode != 0:
raise RuntimeError('docker compose %s failed: %s' % (' '.join(args), result.stderr.strip()))
return result.stdout.strip()
def sql(query, db='messenger', service='postgres'):
return compose('exec', '-T', service, 'psql', '-U', 'messenger', '-d', db, '-X', '-tA', '-v', 'ON_ERROR_STOP=1',
'-c', query)
def wait_ready(seconds=90):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
try:
if call('GET', '/health/ready', timeout=2)[0] == 200:
return time.monotonic()
except OSError:
pass
time.sleep(0.3)
raise AssertionError('API is not ready after %ds' % seconds)
def api_containers():
return compose('ps', '-q', 'api').split()
def wait_in_rotation(container, seconds=60):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
try:
if call('GET', '/health/ready', timeout=2)[1].get('X-Instance-Id') == container[:12]:
return time.monotonic()
except OSError:
pass
time.sleep(0.1)
raise AssertionError('container %s did not enter LB rotation' % container[:12])
class Load:
"""Отправляет сообщения в несколько потоков; неудачный POST повторяет с тем же Idempotency-Key."""
def __init__(self, token, chat_id, workers=4):
self.token, self.path, self.workers = token, '/api/v1/chats/%s/messages' % chat_id, workers
self.records, self.confirmed, self.uncertain = [], [], 0
self.stopped = threading.Event()
self.threads = []
def start(self):
self.threads = [threading.Thread(target=self.worker, args=(n,), daemon=True) for n in range(self.workers)]
for thread in self.threads:
thread.start()
return self
def stop(self):
self.stopped.set()
for thread in self.threads:
thread.join()
def worker(self, n):
sent = 0
while not self.stopped.is_set():
key, text = uuid.uuid4().hex, 'load %d-%d %s' % (n, sent, uuid.uuid4().hex[:6])
sent += 1
while True:
started = time.monotonic()
try:
status, headers, body = call('POST', self.path, {'text': text}, self.token,
headers={'Idempotency-Key': key}, timeout=5)
instance = headers.get('X-Instance-Id')
except OSError as error:
status, instance, body = type(error).__name__, None, None
self.records.append((started, time.monotonic(), status, instance))
if status == 201:
self.confirmed.append(body)
break
if self.stopped.is_set():
self.uncertain += 1
break
time.sleep(0.05)
def window(self, start, end):
rows = [r for r in self.records if start <= r[0] < end]
ok = [r for r in rows if r[2] == 201]
latencies = sorted((r[1] - r[0]) * 1000 for r in ok)
p = lambda q: round(latencies[min(len(latencies) - 1, int(len(latencies) * q))], 1) if latencies else None
return {'seconds': round(end - start, 2), 'ok': len(ok), 'errors': len(rows) - len(ok),
'rps': round(len(ok) / (end - start), 1) if end > start else 0,
'p50_ms': p(0.5), 'p95_ms': p(0.95), 'p99_ms': p(0.99),
'error_kinds': sorted({str(r[2]) for r in rows if r[2] != 201})}
def verify(self, token):
stored, cursor = [], None
while True:
query = '?limit=100' + ('&cursor=' + urllib.parse.quote(cursor, safe='') if cursor else '')
page = call('GET', self.path + query, token=token)[2]
stored += page['items']
cursor = page['next_cursor']
if cursor is None:
break
ids = [m['id'] for m in stored]
texts = [m['text'] for m in stored]
return {'confirmed': len(self.confirmed), 'stored': len(stored),
'lost': len({m['id'] for m in self.confirmed} - set(ids)),
'duplicate_ids': len(ids) - len(set(ids)), 'duplicate_texts': len(texts) - len(set(texts)),
'uncertain_at_stop': self.uncertain}
+66
View File
@@ -0,0 +1,66 @@
import os
from pathlib import Path
import struct
import zlib
ROOT = Path(__file__).resolve().parents[1]
SAMPLE_PNG = (ROOT / 'tests' / 'fixtures' / 'sample.png').read_bytes()
SAMPLE_JPG = (Path(__file__).resolve().parent / 'fixtures' / 'sample.jpg').read_bytes()
def chunk(kind, data):
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff)
def png(width, height, rows, color_type=0, level=9):
"""rows — итератор байтовых строк пикселей (без байта фильтра)."""
compressor = zlib.compressobj(level)
parts = []
for row in rows:
parts.append(compressor.compress(b'\x00' + row))
parts.append(compressor.flush())
header = struct.pack('>IIBBBBB', width, height, 8, color_type, 0, 0, 0)
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', b''.join(parts)) + chunk(b'IEND', b'')
def bomb(width=10000, height=5000):
"""50 Мп чёрного: в распакованном виде 50 МБ, в файле — несколько десятков КБ."""
row = bytes(width)
return png(width, height, (row for _ in range(height)))
def noise_png(target_bytes):
"""Шумный RGB PNG (почти не сжимается) размером чуть меньше target_bytes."""
width = 1024
height = max(1, (target_bytes - 4096) // (width * 3 + 1))
return png(width, height, (os.urandom(width * 3) for _ in range(height)), color_type=2, level=0)
def broken_crc(data):
"""Испортить один байт внутри IDAT, не пересчитывая CRC."""
position = data.index(b'IDAT') + 100
return data[:position] + bytes([data[position] ^ 0xff]) + data[position + 1:]
def with_idat(data, transform):
"""Заменить пиксельные данные PNG (все IDAT) на transform(idat), с правильными CRC."""
position, chunks = 8, []
while position < len(data):
length, kind = struct.unpack('>I4s', data[position:position + 8])
chunks.append((kind, data[position + 8:position + 8 + length]))
position += 12 + length
idat = b''.join(body for kind, body in chunks if kind == b'IDAT')
first = next(index for index, (kind, _) in enumerate(chunks) if kind == b'IDAT')
kept = [(kind, body) for kind, body in chunks if kind != b'IDAT']
kept.insert(first, (b'IDAT', transform(idat)))
return data[:8] + b''.join(chunk(kind, body) for kind, body in kept)
def garbage_idat(data):
"""Мусор вместо сжатых пикселей, CRC верные."""
return with_idat(data, lambda idat: idat[:2] + os.urandom(len(idat) - 2))
def short_idat(data):
"""Корректный zlib-поток, но пикселей меньше, чем требует IHDR."""
return with_idat(data, lambda idat: zlib.compress(zlib.decompress(idat)[:-500]))
+60
View File
@@ -0,0 +1,60 @@
"""Usage (стенд запущен, обе реплики в ротации):
python student-tests/rolling_update_check.py --output evidence/rolling-update.json
Под непрерывной нагрузкой (4 потока, повтор с тем же Idempotency-Key) выполняет scripts/rolling_update.py
и проверяет: ошибки клиента, RPS, что все реплики заменены, что подтверждённые сообщения не потеряны и не задвоены.
"""
import argparse
import json
from pathlib import Path
import subprocess
import sys
import time
from helpers import Load, api_containers, new_chat, new_user
ROOT = Path(__file__).resolve().parents[1]
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--output')
parser.add_argument('--no-build', action='store_true')
args = parser.parse_args()
before = api_containers()
alice = new_user('rolling')
chat = new_chat(alice)
load = Load(alice['token'], chat['id']).start()
time.sleep(5)
started = time.monotonic()
command = [sys.executable, str(ROOT / 'scripts' / 'rolling_update.py')] + (['--no-build'] if args.no_build else [])
update = subprocess.run(command, capture_output=True, text=True, encoding='utf-8')
finished = time.monotonic()
time.sleep(5)
load.stop()
print(update.stdout.strip())
after = api_containers()
result = {
'replicas_before': [c[:12] for c in before], 'replicas_after': [c[:12] for c in after],
'update_seconds': round(finished - started, 1), 'update_exit_code': update.returncode,
'before_5s': load.window(started - 5, started),
'during_update': load.window(started, finished),
'after_5s': load.window(finished, finished + 5),
'data': load.verify(alice['token']),
'log': update.stdout.strip().splitlines(),
}
replaced = not set(before) & set(after) and len(after) == len(before)
ok = update.returncode == 0 and replaced and not result['data']['lost'] and not result['data']['duplicate_texts']
print('rolling update %.1fs: errors=%d, rps %.1f -> %.1f -> %.1f, lost=%d, duplicates=%d, all replaced=%s'
% (result['update_seconds'], result['during_update']['errors'], result['before_5s']['rps'],
result['during_update']['rps'], result['after_5s']['rps'], result['data']['lost'],
result['data']['duplicate_texts'], replaced))
if args.output:
with open(args.output, 'w', encoding='utf-8') as file:
json.dump(result, file, ensure_ascii=False, indent=2)
print('PASS' if ok else 'FAIL: ' + update.stderr.strip()[-500:])
raise SystemExit(0 if ok else 1)
if __name__ == '__main__':
main()
+79
View File
@@ -0,0 +1,79 @@
"""Usage (стенд запущен через docker compose up -d):
python student-tests/sigterm_check.py
docker compose start api
Останавливает одну API-реплику (docker stop = SIGTERM) и всё это время опрашивает API через LB:
реплика сначала выходит из ротации (readiness 503), потом завершается; клиенты не должны получить ошибок.
"""
import argparse
import subprocess
import threading
import time
import urllib.error
import urllib.request
import uuid
from helpers import BASE_URL, api_containers, call, new_chat, new_user
def docker(*args):
return subprocess.run(['docker', *args], capture_output=True, text=True, encoding='utf-8').stdout.strip()
def probe_forever(results, stop_event):
while not stop_event.is_set():
started = time.monotonic()
try:
with urllib.request.urlopen(BASE_URL + '/health/live', timeout=1) as response:
outcome = str(response.status)
except urllib.error.HTTPError as error:
outcome = str(error.code)
except Exception as error:
outcome = type(error).__name__
results.append((started, outcome))
time.sleep(0.02)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--container', default=None)
parser.add_argument('--stop-timeout', type=int, default=10, help='секунд до SIGKILL у docker stop')
args = parser.parse_args()
args.container = args.container or api_containers()[0]
marker = 'PRIVATE-' + uuid.uuid4().hex
alice = new_user('alice')
chat = new_chat(alice)
status = call('POST', '/api/v1/chats/%s/messages' % chat['id'], {'text': marker}, alice['token'])[0]
logs_before = docker('logs', args.container)
print('сообщение с маркером отправлено: HTTP %d; маркер в логах: %s' % (status, marker in logs_before))
results, stop_event = [], threading.Event()
prober = threading.Thread(target=probe_forever, args=(results, stop_event))
prober.start()
time.sleep(1)
sigterm_at = time.monotonic()
docker('stop', '-t', str(args.stop_timeout), args.container)
stopped_at = time.monotonic()
time.sleep(1)
stop_event.set()
prober.join()
before = [r for t, r in results if t < sigterm_at]
after = [r for t, r in results if t >= sigterm_at]
exit_code = docker('inspect', '--format', '{{.State.ExitCode}}', args.container)
print('docker stop занял: %.2f с (лимит %d с)' % (stopped_at - sigterm_at, args.stop_timeout))
print('код выхода контейнера: %s (0 — сам завершился по SIGTERM; 137 — убит SIGKILL)' % exit_code)
print('запросы до SIGTERM: %d, все 200: %s' % (len(before), set(before) == {'200'}))
print('запросы после SIGTERM: %d, результаты: %s' % (len(after), sorted(set(after))))
print('ответов 5xx за всё время: %d' % sum(r.startswith('5') for _, r in results))
print('\nлоги остановки:')
for line in docker('logs', args.container).splitlines():
if any(word in line for word in ('draining', 'Shutting down', 'stopping', 'shutdown', 'Finished server')):
print(' ' + line)
if __name__ == '__main__':
main()
+157
View File
@@ -0,0 +1,157 @@
import unittest
import uuid
from helpers import call, login, new_chat, new_user, read_all_messages
class ErrorFormatMixin:
def assertError(self, result, status, code):
actual, headers, body = result
self.assertEqual(actual, status)
self.assertEqual(body['error']['code'], code)
self.assertEqual(body['error']['request_id'], headers['X-Request-Id'])
self.assertNotIn('Traceback', body['error']['message'])
class InvalidJsonTest(ErrorFormatMixin, unittest.TestCase):
def test_malformed_json(self):
"""Битый JSON -> 400 invalid_request"""
self.assertError(call('POST', '/api/v1/users', raw=b'{"username": "abc",'), 400, 'invalid_request')
def test_json_not_object(self):
"""JSON-массив вместо объекта -> 400"""
self.assertError(call('POST', '/api/v1/users', raw=b'[1, 2, 3]'), 400, 'invalid_request')
def test_wrong_type(self):
"""Число вместо строки -> 400 (без тихого приведения типов)"""
self.assertError(call('POST', '/api/v1/users', {'username': 12345, 'display_name': 'x', 'password': 'Long-enough-pass-1'}), 400, 'invalid_request')
def test_extra_field(self):
"""Лишнее поле в теле -> 400"""
body = {'username': 'u_' + uuid.uuid4().hex[:8], 'display_name': 'x', 'password': 'Long-enough-pass-1', 'is_admin': True}
self.assertError(call('POST', '/api/v1/users', body), 400, 'invalid_request')
class DuplicateUsernameTest(ErrorFormatMixin, unittest.TestCase):
def test_duplicate_username(self):
"""Повторная регистрация того же username -> 409 conflict"""
user = new_user('dup')
again = call('POST', '/api/v1/users', {'username': user['username'], 'display_name': 'Другой',
'password': 'Other-password-123'})
self.assertError(again, 409, 'conflict')
class ForeignChatTest(ErrorFormatMixin, unittest.TestCase):
def setUp(self):
self.alice, self.bob, self.eve = new_user('alice'), new_user('bob'), new_user('eve')
self.chat = new_chat(self.alice, self.bob)
self.path = '/api/v1/chats/' + self.chat['id']
def test_outsider_cannot_read_chat(self):
"""Посторонний читает чат -> 404, как будто чата нет"""
self.assertError(call('GET', self.path, token=self.eve['token']), 404, 'not_found')
missing = call('GET', '/api/v1/chats/' + str(uuid.uuid4()), token=self.eve['token'])
self.assertEqual(missing[2]['error']['message'], call('GET', self.path, token=self.eve['token'])[2]['error']['message'])
def test_outsider_cannot_write_or_read_history(self):
"""Посторонний пишет/читает историю -> 404"""
self.assertError(call('POST', self.path + '/messages', {'text': 'чужое'}, self.eve['token']), 404, 'not_found')
self.assertError(call('GET', self.path + '/messages', token=self.eve['token']), 404, 'not_found')
def test_outsider_does_not_see_chat_in_list(self):
"""Чужой чат не попадает в список чатов"""
status, _, page = call('GET', '/api/v1/chats?limit=100', token=self.eve['token'])
self.assertEqual(status, 200)
self.assertNotIn(self.chat['id'], [c['id'] for c in page['items']])
def test_member_has_access(self):
"""Участник видит чат (контроль: 404 выше не из-за ошибки в тесте)"""
self.assertEqual(call('GET', self.path, token=self.bob['token'])[0], 200)
class EmptyMessageTest(ErrorFormatMixin, unittest.TestCase):
def setUp(self):
self.alice = new_user('alice')
self.path = '/api/v1/chats/%s/messages' % new_chat(self.alice)['id']
def test_empty_text(self):
"""Пустой текст -> 400"""
self.assertError(call('POST', self.path, {'text': ''}, self.alice['token']), 400, 'invalid_request')
def test_whitespace_text(self):
"""Текст только из пробельных символов -> 400"""
self.assertError(call('POST', self.path, {'text': ' \t\n '}, self.alice['token']), 400, 'invalid_request')
def test_missing_text(self):
"""Тело без поля text -> 400"""
self.assertError(call('POST', self.path, {}, self.alice['token']), 400, 'invalid_request')
def test_text_length_limit(self):
"""4000 символов можно (эмодзи = 1 символ), 4001 — 400"""
self.assertEqual(call('POST', self.path, {'text': '😀' * 4000}, self.alice['token'])[0], 201)
self.assertError(call('POST', self.path, {'text': 'x' * 4001}, self.alice['token']), 400, 'invalid_request')
class PaginationTest(ErrorFormatMixin, unittest.TestCase):
def setUp(self):
self.alice = new_user('alice')
self.chat = new_chat(self.alice)
self.sent = [call('POST', '/api/v1/chats/%s/messages' % self.chat['id'],
{'text': 'сообщение %d' % i}, self.alice['token'])[2] for i in range(7)]
def test_walk_several_pages(self):
"""7 сообщений по 3: три страницы, без потерь, повторов и перестановок"""
items, pages = read_all_messages(self.chat['id'], self.alice['token'], limit=3)
self.assertEqual(pages, 3)
self.assertEqual([m['id'] for m in items], [m['id'] for m in self.sent])
keys = [(m['created_at'], m['id']) for m in items]
self.assertEqual(keys, sorted(keys))
def test_cursor_from_other_chat(self):
"""cursor от другого чата -> 400"""
_, _, page = call('GET', '/api/v1/chats/%s/messages?limit=1' % self.chat['id'], token=self.alice['token'])
other = new_chat(self.alice, title='Другой')
self.assertError(call('GET', '/api/v1/chats/%s/messages?cursor=%s' % (other['id'], page['next_cursor']),
token=self.alice['token']), 400, 'invalid_request')
def test_invalid_cursor_and_limit(self):
"""Мусорный cursor и limit вне 1..100 -> 400"""
path = '/api/v1/chats/%s/messages' % self.chat['id']
for query in ('?cursor=not-a-cursor', '?limit=0', '?limit=101', '?limit=abc'):
with self.subTest(query=query):
self.assertError(call('GET', path + query, token=self.alice['token']), 400, 'invalid_request')
class IdentityTest(ErrorFormatMixin, unittest.TestCase):
def test_sender_cannot_be_forged(self):
"""sender_id в теле запрещён: отправитель только из сессии"""
alice, bob = new_user('alice'), new_user('bob')
chat = new_chat(alice, bob)
forged = call('POST', '/api/v1/chats/%s/messages' % chat['id'], {'text': 'hi', 'sender_id': bob['id']}, alice['token'])
self.assertError(forged, 400, 'invalid_request')
def test_nul_character_is_preserved(self):
"""U+0000 — допустимый code point: сообщение, display_name и title сохраняются и читаются без изменений"""
username, password = 'nul_' + uuid.uuid4().hex[:12], 'Nul-password-123'
status, _, user = call('POST', '/api/v1/users', {'username': username, 'display_name': 'a\u0000lice', 'password': password})
self.assertEqual((status, user['display_name']), (201, 'a\u0000lice'))
alice = dict(user, token=login(username, password)['token'])
self.assertEqual(call('GET', '/api/v1/users/me', token=alice['token'])[2]['display_name'], 'a\u0000lice')
status, _, chat = call('POST', '/api/v1/chats', {'title': 't\u0000', 'member_ids': [alice['id']]}, alice['token'])
self.assertEqual((status, chat['title']), (201, 't\u0000'))
path = '/api/v1/chats/%s/messages' % chat['id']
status, _, message = call('POST', path, {'text': 'a\u0000b'}, alice['token'])
self.assertEqual((status, message['text']), (201, 'a\u0000b'))
self.assertEqual(call('GET', path, token=alice['token'])[2]['items'][0]['text'], 'a\u0000b')
self.assertEqual(call('GET', '/api/v1/chats/' + chat['id'], token=alice['token'])[2]['title'], 't\u0000')
def test_lone_surrogate_is_rejected(self):
"""Одиночный суррогат не кодируется в UTF-8 -> 400, а не 500"""
alice = new_user('alice')
chat = new_chat(alice)
self.assertError(call('POST', '/api/v1/chats/%s/messages' % chat['id'], raw=b'{"text": "a\\ud800b"}',
token=alice['token']), 400, 'invalid_request')
if __name__ == '__main__':
unittest.main()
+192
View File
@@ -0,0 +1,192 @@
import hashlib
import http.client
import unittest
import urllib.parse
import uuid
from helpers import BASE_URL, call, new_chat, new_user
from imagegen import SAMPLE_JPG, SAMPLE_PNG, bomb, broken_crc, garbage_idat, noise_png, short_idat
MAX = 10 * 1024 * 1024
def upload(user, chat, data, content_type='image/png', filename='photo.png', token=True):
query = '' if filename is None else '?filename=' + urllib.parse.quote(filename, safe='')
return call('POST', '/api/v1/chats/%s/attachments%s' % (chat['id'], query), raw=data,
token=user['token'] if token else None, headers={'Content-Type': content_type})
def download(user, attachment_id):
target = urllib.parse.urlsplit(BASE_URL)
connection = http.client.HTTPConnection(target.hostname, target.port, timeout=30)
connection.request('GET', '/api/v1/attachments/%s/content' % attachment_id,
headers={'Authorization': 'Bearer ' + user['token']})
response = connection.getresponse()
body = response.read()
connection.close()
return response.status, response.headers, body
def send(user, chat, body, key=None):
return call('POST', '/api/v1/chats/%s/messages' % chat['id'], body, user['token'],
headers={'Idempotency-Key': key or uuid.uuid4().hex})
class UploadTest(unittest.TestCase):
def setUp(self):
self.alice, self.bob, self.eve = new_user('alice'), new_user('bob'), new_user('eve')
self.chat = new_chat(self.alice, self.bob)
def assertCode(self, result, status, code=None):
self.assertEqual(result[0], status, result[2])
if code:
self.assertEqual(result[2]['error']['code'], code)
def test_png_and_jpeg_are_stored_and_downloaded_byte_exact(self):
"""PNG и JPEG → 201 ready; второй участник скачивает те же байты с верным Content-Type"""
for data, content_type in ((SAMPLE_PNG, 'image/png'), (SAMPLE_JPG, 'image/jpeg')):
with self.subTest(content_type=content_type):
status, _, att = upload(self.alice, self.chat, data, content_type)
self.assertEqual(status, 201)
self.assertEqual((att['status'], att['variants'], att['error_code']), ('ready', [], None))
self.assertEqual((att['content_type'], att['size_bytes'], att['sha256']),
(content_type, len(data), hashlib.sha256(data).hexdigest()))
self.assertEqual((att['chat_id'], att['owner_id']), (self.chat['id'], self.alice['id']))
self.assertEqual(call('GET', '/api/v1/attachments/' + att['id'], token=self.bob['token'])[2], att)
status, headers, body = download(self.bob, att['id'])
self.assertEqual((status, headers.get_content_type(), body), (200, content_type, data))
self.assertEqual(int(headers['Content-Length']), len(data))
def test_download_through_every_replica(self):
"""Файл скачивается через любую API-реплику (байты не лежат локально у загрузившей)"""
att = upload(self.alice, self.chat, SAMPLE_PNG)[2]
replicas = set()
for _ in range(6):
status, headers, body = download(self.bob, att['id'])
self.assertEqual((status, body), (200, SAMPLE_PNG))
replicas.add(headers['X-Instance-Id'])
self.assertGreaterEqual(len(replicas), 2)
def test_filename_is_display_only_and_validated(self):
"""Unicode-имя сохраняется как есть; нет имени, пустое, /, \\, NUL, управляющий символ, >255 → 400"""
status, _, att = upload(self.alice, self.chat, SAMPLE_PNG, filename='фото отпуска 😀.png')
self.assertEqual((status, att['filename']), (201, 'фото отпуска 😀.png'))
for name in (None, '', ' ', '../etc/passwd', 'a\\b.png', 'a\x00b.png', 'bell\x07.png', 'x' * 256):
with self.subTest(name=name):
self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG, filename=name), 400, 'invalid_request')
def test_wrong_declared_type(self):
"""Content-Type не image/png|jpeg (octet-stream, multipart, gif) → 415"""
for content_type in ('application/octet-stream', 'multipart/form-data; boundary=x', 'image/gif', 'text/plain'):
with self.subTest(content_type=content_type):
self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG, content_type), 415, 'unsupported_media_type')
def test_mime_spoofing(self):
"""Подмена MIME: PNG как jpeg, JPEG как png, текст и GIF под видом png → 415"""
for data, content_type in ((SAMPLE_PNG, 'image/jpeg'), (SAMPLE_JPG, 'image/png'),
(b'hello, not an image', 'image/png'), (b'GIF89a' + bytes(100), 'image/png'),
(b'', 'image/png')):
with self.subTest(content_type=content_type, head=data[:6]):
self.assertCode(upload(self.alice, self.chat, data, content_type), 415, 'unsupported_media_type')
def test_corrupted_images(self):
"""Обрезанный PNG, битый CRC, мусор или нехватка пикселей в IDAT при верных CRC, обрезанный JPEG → 415"""
for data, content_type in ((SAMPLE_PNG[:5000], 'image/png'), (broken_crc(SAMPLE_PNG), 'image/png'),
(garbage_idat(SAMPLE_PNG), 'image/png'), (short_idat(SAMPLE_PNG), 'image/png'),
(SAMPLE_JPG[:3000], 'image/jpeg')):
with self.subTest(size=len(data), content_type=content_type):
self.assertCode(upload(self.alice, self.chat, data, content_type), 415, 'unsupported_media_type')
def test_too_large_file(self):
"""10 MiB + 1 байт → 413 (и по Content-Length, и chunked без длины); файл ~10 MiB принимается"""
self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG + bytes(MAX + 1 - len(SAMPLE_PNG))), 413, 'payload_too_large')
target = urllib.parse.urlsplit(BASE_URL)
connection = http.client.HTTPConnection(target.hostname, target.port, timeout=60)
chunks = (SAMPLE_PNG if i == 0 else bytes(1024 * 1024) for i in range(12))
connection.request('POST', '/api/v1/chats/%s/attachments?filename=big.png' % self.chat['id'], body=chunks,
headers={'Authorization': 'Bearer ' + self.alice['token'], 'Content-Type': 'image/png'},
encode_chunked=True)
self.assertEqual(connection.getresponse().status, 413)
connection.close()
big = noise_png(MAX - 50000)
self.assertLessEqual(len(big), MAX)
status, _, att = upload(self.alice, self.chat, big)
self.assertEqual((status, att['size_bytes']), (201, len(big)))
self.assertEqual(download(self.bob, att['id'])[2], big)
def test_decompression_bomb(self):
"""PNG 10000×5000 (50 Мп) весом в десятки КБ → 413 без распаковки"""
data = bomb()
self.assertLess(len(data), 1024 * 1024)
self.assertCode(upload(self.alice, self.chat, data), 413, 'payload_too_large')
def test_access_rules(self):
"""Без токена 401; посторонний не может загрузить в чужой чат и не видит метаданные/байты (404)"""
self.assertCode(upload(self.alice, self.chat, SAMPLE_PNG, token=False), 401, 'unauthorized')
self.assertCode(upload(self.eve, self.chat, SAMPLE_PNG), 404, 'not_found')
self.assertCode(upload(self.alice, {'id': str(uuid.uuid4())}, SAMPLE_PNG), 404, 'not_found')
att = upload(self.alice, self.chat, SAMPLE_PNG)[2]
self.assertCode(call('GET', '/api/v1/attachments/' + att['id'], token=self.eve['token']), 404, 'not_found')
self.assertEqual(download(self.eve, att['id'])[0], 404)
self.assertCode(call('GET', '/api/v1/attachments/' + str(uuid.uuid4()), token=self.bob['token']), 404)
self.assertCode(call('GET', '/api/v1/attachments/not-a-uuid', token=self.bob['token']), 400)
self.assertCode(call('GET', '/api/v1/attachments/%s/content?variant=thumbnail' % att['id'],
token=self.bob['token']), 400)
class MessageAttachmentsTest(unittest.TestCase):
def setUp(self):
self.alice, self.bob = new_user('alice'), new_user('bob')
self.chat = new_chat(self.alice, self.bob)
self.other_chat = new_chat(self.alice, title='Другой')
self.att = upload(self.alice, self.chat, SAMPLE_PNG)[2]
def history(self):
return call('GET', '/api/v1/chats/%s/messages?limit=100' % self.chat['id'], token=self.bob['token'])[2]['items']
def test_message_with_attachment_only(self):
"""text="" + одно вложение → 201; attachment_ids видны в истории; обычное сообщение получает []"""
status, _, message = send(self.alice, self.chat, {'text': '', 'attachment_ids': [self.att['id']]})
self.assertEqual((status, message['attachment_ids'], message['text']), (201, [self.att['id']], ''))
plain = send(self.alice, self.chat, {'text': 'без вложений'})[2]
self.assertEqual(plain['attachment_ids'], [])
self.assertEqual([m['attachment_ids'] for m in self.history()], [[self.att['id']], []])
def test_other_member_can_reference_attachment(self):
"""Боб прикрепляет вложение, загруженное Алисой в этот же чат"""
status, _, message = send(self.bob, self.chat, {'text': 'смотри', 'attachment_ids': [self.att['id']]})
self.assertEqual((status, message['attachment_ids']), (201, [self.att['id']]))
def test_foreign_or_unknown_attachment(self):
"""Вложение другого чата или несуществующее → 404, сообщение не создаётся"""
foreign = upload(self.alice, self.other_chat, SAMPLE_PNG)[2]
for ids in ([foreign['id']], [str(uuid.uuid4())], [self.att['id'], foreign['id']]):
with self.subTest(ids=ids):
self.assertEqual(send(self.alice, self.chat, {'text': 'x', 'attachment_ids': ids})[0], 404)
self.assertEqual(self.history(), [])
def test_invalid_attachment_lists(self):
"""Повтор id, 11 вложений, не-UUID, пустой текст без вложений, пробелы без вложений → 400"""
for body in ({'text': 'x', 'attachment_ids': [self.att['id'], self.att['id']]},
{'text': 'x', 'attachment_ids': [str(uuid.uuid4()) for _ in range(11)]},
{'text': 'x', 'attachment_ids': ['nope']},
{'text': ''}, {'text': ' '}, {'text': '', 'attachment_ids': []}):
with self.subTest(body=body):
self.assertEqual(send(self.alice, self.chat, body)[0], 400)
def test_idempotency_includes_attachments(self):
"""Тот же ключ: те же вложения → тот же Message; другие вложения → 409; нет поля = [] для текста"""
key = uuid.uuid4().hex
first = send(self.alice, self.chat, {'text': 'фото', 'attachment_ids': [self.att['id']]}, key)
again = send(self.alice, self.chat, {'attachment_ids': [self.att['id']], 'text': 'фото'}, key)
self.assertEqual((first[0], again[0], first[2]), (201, 201, again[2]))
other = upload(self.alice, self.chat, SAMPLE_JPG, 'image/jpeg', 'b.jpg')[2]
self.assertEqual(send(self.alice, self.chat, {'text': 'фото', 'attachment_ids': [other['id']]}, key)[0], 409)
key = uuid.uuid4().hex
plain = send(self.alice, self.chat, {'text': 'текст'}, key)
same = send(self.alice, self.chat, {'text': 'текст', 'attachment_ids': []}, key)
self.assertEqual(plain[2]['id'], same[2]['id'])
if __name__ == '__main__':
unittest.main()
+132
View File
@@ -0,0 +1,132 @@
from datetime import datetime, timezone
import unittest
import uuid
from helpers import basic, call, login, new_user, register
class RegistrationTest(unittest.TestCase):
def body(self, password):
return {'username': 'reg_' + uuid.uuid4().hex[:12], 'display_name': 'Регистрация', 'password': password}
def test_password_is_not_returned(self):
"""Ответ регистрации и /users/me не содержат password/hash"""
status, _, user = call('POST', '/api/v1/users', self.body('Secret-password-1'))
self.assertEqual(status, 201)
self.assertEqual(set(user), {'id', 'username', 'display_name', 'created_at'})
def test_password_length_limits(self):
"""Пароль 12..128 символов; 11 и 129 -> 400; эмодзи считается одним символом"""
self.assertEqual(call('POST', '/api/v1/users', self.body('x' * 11))[0], 400)
self.assertEqual(call('POST', '/api/v1/users', self.body('x' * 129))[0], 400)
self.assertEqual(call('POST', '/api/v1/users', self.body('x' * 128))[0], 201)
self.assertEqual(call('POST', '/api/v1/users', self.body('🔑' * 12))[0], 201)
def test_password_required_and_not_blank(self):
"""Без password или из одних пробелов -> 400"""
body = self.body('x')
del body['password']
self.assertEqual(call('POST', '/api/v1/users', body)[0], 400)
self.assertEqual(call('POST', '/api/v1/users', self.body(' ' * 20))[0], 400)
class LoginTest(unittest.TestCase):
def setUp(self):
self.user, self.password = register('login')
def assertBasicChallenge(self, result):
status, headers, body = result
self.assertEqual(status, 401)
self.assertEqual(headers['WWW-Authenticate'], 'Basic realm="messenger", charset="UTF-8"')
self.assertEqual(body['error']['code'], 'unauthorized')
return body['error']['message']
def test_login_returns_opaque_session(self):
"""Вход: 201, token_type=Bearer, токен случайный и не содержит username/UUID"""
first = login(self.user['username'], self.password)
second = login(self.user['username'], self.password)
self.assertEqual(first['token_type'], 'Bearer')
self.assertEqual(first['user'], self.user)
self.assertGreaterEqual(len(first['token']), 43)
self.assertNotEqual(first['token'], second['token'])
self.assertNotIn(self.user['username'], first['token'])
self.assertNotIn(self.user['id'], first['token'])
expires = datetime.fromisoformat(first['expires_at'].replace('Z', '+00:00'))
self.assertGreater(expires, datetime.now(timezone.utc))
def test_wrong_password_and_unknown_user_look_the_same(self):
"""Неверный пароль и несуществующий пользователь -> одинаковый 401 с Basic challenge"""
wrong = self.assertBasicChallenge(call('POST', '/api/v1/auth/sessions',
headers=basic(self.user['username'], 'Wrong-password-1')))
unknown = self.assertBasicChallenge(call('POST', '/api/v1/auth/sessions',
headers=basic('nobody_' + uuid.uuid4().hex[:8], 'Wrong-password-1')))
self.assertEqual(wrong, unknown)
def test_malformed_basic(self):
"""Нет заголовка, не base64, нет двоеточия, Bearer вместо Basic -> 401 Basic"""
for header in (None, 'Basic', 'Basic !!!not-base64!!!', 'Basic ' + 'bm9jb2xvbg==', 'Bearer abc'):
with self.subTest(header=header):
headers = {'Authorization': header} if header else {}
self.assertBasicChallenge(call('POST', '/api/v1/auth/sessions', headers=headers))
def test_password_with_colon_and_unicode(self):
"""Двоеточие и не-ASCII в пароле допустимы: делим только по первому ':'"""
user, password = register('colon', password='пароль:с:двоеточием🔑')
self.assertEqual(login(user['username'], password)['user']['id'], user['id'])
def test_login_rejects_body(self):
"""POST /auth/sessions принимает только заголовок, JSON body -> 400"""
status = call('POST', '/api/v1/auth/sessions', {'username': self.user['username']},
headers=basic(self.user['username'], self.password))[0]
self.assertEqual(status, 400)
class BearerTest(unittest.TestCase):
def setUp(self):
self.alice = new_user('alice')
self.eve = new_user('eve')
def assertBearerChallenge(self, headers):
status, response_headers, _ = call('GET', '/api/v1/users/me', headers=headers)
self.assertEqual(status, 401)
self.assertEqual(response_headers['WWW-Authenticate'], 'Bearer realm="messenger"')
def test_only_bearer_is_accepted(self):
"""Нет заголовка, Basic, X-User-Id, мусорный токен -> 401 Bearer"""
self.assertBearerChallenge({})
self.assertBearerChallenge(basic(self.alice['username'], self.alice['password']))
self.assertBearerChallenge({'X-User-Id': self.alice['id']})
self.assertBearerChallenge({'Authorization': 'Bearer ' + uuid.uuid4().hex})
self.assertBearerChallenge({'Authorization': 'Bearer'})
def test_authentication_before_validation(self):
"""Аноним с битым JSON на защищённом endpoint получает 401, а не 400; с сессией — 400"""
self.assertEqual(call('POST', '/api/v1/chats', raw=b'{bad')[0], 401)
self.assertEqual(call('POST', '/api/v1/chats', raw=b'{bad', token=self.alice['token'])[0], 400)
self.assertEqual(call('POST', '/api/v1/users', raw=b'{bad')[0], 400)
def test_scheme_is_case_insensitive(self):
"""Схема аутентификации нечувствительна к регистру (RFC 9110)"""
status, _, me = call('GET', '/api/v1/users/me', headers={'Authorization': 'bearer ' + self.alice['token']})
self.assertEqual((status, me['id']), (200, self.alice['id']))
def test_x_user_id_does_not_override_session(self):
"""X-User-Id игнорируется: владелец определяется только токеном"""
status, _, me = call('GET', '/api/v1/users/me', token=self.alice['token'], headers={'X-User-Id': self.eve['id']})
self.assertEqual((status, me['id']), (200, self.alice['id']))
class LogoutTest(unittest.TestCase):
def test_logout_revokes_only_current_session(self):
"""Logout -> 204 без тела; этот токен -> 401, вторая сессия того же пользователя работает"""
alice = new_user('alice')
other = login(alice['username'], alice['password'])['token']
status, _, body = call('DELETE', '/api/v1/auth/sessions/current', token=alice['token'])
self.assertEqual((status, body), (204, None))
self.assertEqual(call('GET', '/api/v1/users/me', token=alice['token'])[0], 401)
self.assertEqual(call('DELETE', '/api/v1/auth/sessions/current', token=alice['token'])[0], 401)
self.assertEqual(call('GET', '/api/v1/users/me', token=other)[0], 200)
if __name__ == '__main__':
unittest.main()
+68
View File
@@ -0,0 +1,68 @@
from concurrent.futures import ThreadPoolExecutor
import threading
import unittest
import uuid
from helpers import basic, call, new_chat, new_user, read_all_messages
CLIENTS = 10
REGISTRATIONS = 20
PER_CLIENT = 10
class ConcurrencyTest(unittest.TestCase):
def test_100_messages_from_10_clients(self):
"""100 параллельных сообщений от 10 клиентов -> ровно 100 уникальных в истории"""
users = [new_user('client%d' % i) for i in range(CLIENTS)]
chat = new_chat(users[0], *users[1:], title='Нагрузка')
path = '/api/v1/chats/%s/messages' % chat['id']
barrier = threading.Barrier(CLIENTS)
def client(index):
barrier.wait()
results = []
for n in range(PER_CLIENT):
status, _, message = call('POST', path, {'text': 'client %d msg %d' % (index, n)}, users[index]['token'])
results.append((status, message))
return results
with ThreadPoolExecutor(max_workers=CLIENTS) as pool:
responses = [r for batch in pool.map(client, range(CLIENTS)) for r in batch]
self.assertEqual([s for s, _ in responses], [201] * CLIENTS * PER_CLIENT)
sent_ids = {m['id'] for _, m in responses}
items, pages = read_all_messages(chat['id'], users[0]['token'], limit=30)
read_ids = [m['id'] for m in items]
keys = [(m['created_at'], m['id']) for m in items]
self.assertEqual(len(sent_ids), 100, 'сервер выдал повторяющиеся id')
self.assertEqual(len(read_ids), 100, 'в истории не ровно 100 сообщений')
self.assertEqual(set(read_ids), sent_ids, 'история не совпадает с подтверждёнными сообщениями')
self.assertEqual(len({m['text'] for m in items}), 100, 'потерялся или задвоился текст')
self.assertEqual(keys, sorted(keys), 'история не упорядочена по (created_at, id)')
print('\n отправлено 201: %d, уникальных id: %d, прочитано: %d за %d страниц(ы)'
% (len(responses), len(sent_ids), len(read_ids), pages))
def test_parallel_registration_of_same_username(self):
"""20 одновременных регистраций одного username -> ровно один 201, остальные 409; вход только с паролем победителя"""
username = 'race_' + uuid.uuid4().hex[:12]
barrier = threading.Barrier(REGISTRATIONS)
def register(index):
password = 'Race-password-%02d' % index
barrier.wait()
status = call('POST', '/api/v1/users', {'username': username, 'display_name': 'Гонка', 'password': password})[0]
return status, password
with ThreadPoolExecutor(max_workers=REGISTRATIONS) as pool:
results = list(pool.map(register, range(REGISTRATIONS)))
statuses = sorted(status for status, _ in results)
self.assertEqual(statuses, [201] + [409] * (REGISTRATIONS - 1))
winner = [password for status, password in results if status == 201][0]
loser = [password for status, password in results if status == 409][0]
self.assertEqual(call('POST', '/api/v1/auth/sessions', headers=basic(username, winner))[0], 201)
self.assertEqual(call('POST', '/api/v1/auth/sessions', headers=basic(username, loser))[0], 401)
if __name__ == '__main__':
unittest.main()
+58
View File
@@ -0,0 +1,58 @@
import hashlib
import time
import unittest
import uuid
from helpers import call, compose, new_chat, new_user, sql
class DatabaseTest(unittest.TestCase):
def test_chat_creation_is_rolled_back(self):
"""Неизвестный участник: INSERT chats выполнен, INSERT chat_members упал по FK, всё откатано"""
alice = new_user('alice')
chats_before = int(sql('SELECT count(*) FROM chats'))
members_before = int(sql('SELECT count(*) FROM chat_members'))
ghost = str(uuid.uuid4())
since = sql("SELECT to_char(now() - interval '1 second', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')")
status, _, body = call('POST', '/api/v1/chats', {'title': 'Откат', 'member_ids': [alice['id'], ghost]},
alice['token'])
self.assertEqual(status, 400, body)
time.sleep(0.5)
log = compose('logs', '--no-log-prefix', '--since', since, 'postgres')
self.assertIn('violates foreign key constraint', log)
self.assertIn(ghost, log)
self.assertEqual(int(sql('SELECT count(*) FROM chats')), chats_before)
self.assertEqual(int(sql('SELECT count(*) FROM chat_members')), members_before)
def test_secrets_are_not_stored_in_plaintext(self):
"""В users лежит argon2id-хеш, в sessions — sha256 токена, не сам токен"""
alice = new_user('alice')
password_hash = sql("SELECT password_hash FROM users WHERE id = '%s'" % alice['id'])
self.assertTrue(password_hash.startswith('$argon2id$'), password_hash[:10])
self.assertNotIn(alice['password'], password_hash)
digest = hashlib.sha256(alice['token'].encode()).hexdigest()
self.assertEqual(sql("SELECT count(*) FROM sessions WHERE token_hash = '\\x%s'" % digest), '1')
dump = sql("SELECT string_agg(encode(token_hash, 'escape'), '') FROM sessions WHERE user_id = '%s'" % alice['id'])
self.assertNotIn(alice['token'], dump)
def test_migrations_are_idempotent(self):
"""Повторный запуск миграций ничего не меняет и завершается успешно"""
before = sql('SELECT version, applied_at FROM schema_migrations ORDER BY version')
users = sql('SELECT count(*) FROM users')
output = compose('run', '--rm', '--no-deps', 'migrate')
self.assertIn('skip 0001_init: already applied', output)
self.assertEqual(sql('SELECT version, applied_at FROM schema_migrations ORDER BY version'), before)
self.assertEqual(sql('SELECT count(*) FROM users'), users)
def test_history_query_uses_index(self):
"""Запрос страницы истории идёт по индексу (chat_id, created_at, id), без Sort"""
alice = new_user('alice')
chat = new_chat(alice)
plan = sql("SET enable_seqscan = off; SET enable_bitmapscan = off; EXPLAIN SELECT id FROM messages WHERE chat_id = '%s' "
"ORDER BY created_at, id LIMIT 51" % chat['id'])
self.assertIn('messages_chat_history_idx', plan)
self.assertNotIn('Sort', plan)
if __name__ == '__main__':
unittest.main()
+96
View File
@@ -0,0 +1,96 @@
from concurrent.futures import ThreadPoolExecutor
import threading
import unittest
import uuid
from helpers import call, new_chat, new_user, read_all_messages
PARALLEL = 20
def post(chat, user, key, body=None, raw=None):
return call('POST', '/api/v1/chats/%s/messages' % chat['id'], body, user['token'], raw=raw,
headers={'Idempotency-Key': key})
class IdempotencyTest(unittest.TestCase):
def setUp(self):
self.alice, self.bob = new_user('alice'), new_user('bob')
self.chat = new_chat(self.alice, self.bob)
def history(self):
return read_all_messages(self.chat['id'], self.alice['token'], limit=100)[0]
def test_replay_returns_same_message(self):
"""Повтор того же запроса с тем же ключом -> тот же 201 и тот же Message, дубля нет"""
key = uuid.uuid4().hex
first = post(self.chat, self.alice, key, {'text': 'один раз'})
second = post(self.chat, self.alice, key, {'text': 'один раз'})
self.assertEqual((first[0], second[0]), (201, 201))
self.assertEqual(first[2], second[2])
self.assertEqual([m['id'] for m in self.history()], [first[2]['id']])
def test_json_formatting_does_not_matter(self):
"""Порядок/пробелы в JSON семантически равны: повтор с другим форматированием -> тот же Message"""
key = uuid.uuid4().hex
first = post(self.chat, self.alice, key, raw='{"text":"формат"}'.encode())
second = post(self.chat, self.alice, key, raw='{ "text" : "формат" }'.encode())
self.assertEqual((first[0], second[0]), (201, 201))
self.assertEqual(first[2]['id'], second[2]['id'])
def test_other_body_with_same_key_conflicts(self):
"""Тот же ключ с другим телом -> 409, второе сообщение не создаётся"""
key = uuid.uuid4().hex
self.assertEqual(post(self.chat, self.alice, key, {'text': 'первое'})[0], 201)
status, _, body = post(self.chat, self.alice, key, {'text': 'другое'})
self.assertEqual((status, body['error']['code']), (409, 'conflict'))
self.assertEqual(len(self.history()), 1)
def test_missing_or_invalid_key(self):
"""Нет ключа, пустой, недопустимые символы, длиннее 128 -> 400"""
for key in (None, '', 'bad key', 'bad/key', 'café', 'x' * 129):
with self.subTest(key=key):
self.assertEqual(post(self.chat, self.alice, key, {'text': 'x'})[0], 400)
self.assertEqual(post(self.chat, self.alice, 'A.b_c:d-1' + 'x' * 119, {'text': 'граница'})[0], 201)
def test_key_scope_is_user_and_chat(self):
"""Одинаковый ключ у разных пользователей и в разных чатах — независимые сообщения"""
key = uuid.uuid4().hex
other_chat = new_chat(self.alice, title='Другой')
a = post(self.chat, self.alice, key, {'text': 'от Алисы'})
b = post(self.chat, self.bob, key, {'text': 'от Боба'})
c = post(other_chat, self.alice, key, {'text': 'от Алисы'})
self.assertEqual([a[0], b[0], c[0]], [201, 201, 201])
self.assertEqual(len({a[2]['id'], b[2]['id'], c[2]['id']}), 3)
def test_replay_rechecks_session_and_access(self):
"""Повтор снова проверяет сессию и доступ: после logout -> 401, посторонний с тем же ключом -> 404"""
key = uuid.uuid4().hex
self.assertEqual(post(self.chat, self.alice, key, {'text': 'до выхода'})[0], 201)
eve = new_user('eve')
self.assertEqual(post(self.chat, eve, key, {'text': 'до выхода'})[0], 404)
call('DELETE', '/api/v1/auth/sessions/current', token=self.alice['token'])
self.assertEqual(post(self.chat, self.alice, key, {'text': 'до выхода'})[0], 401)
def test_parallel_retries_through_replicas_create_one_message(self):
"""20 одновременных POST с одним ключом через LB (разные реплики) -> ровно одно сообщение, все 201 с одним id"""
key = uuid.uuid4().hex
barrier = threading.Barrier(PARALLEL)
def send(_):
barrier.wait()
status, headers, body = post(self.chat, self.alice, key, {'text': 'конкурентный повтор'})
return status, headers['X-Instance-Id'], body
with ThreadPoolExecutor(max_workers=PARALLEL) as pool:
results = list(pool.map(send, range(PARALLEL)))
self.assertEqual({status for status, _, _ in results}, {201})
self.assertEqual(len({body['id'] for _, _, body in results}), 1)
self.assertGreaterEqual(len({instance for _, instance, _ in results}), 2, 'запросы не попали на разные реплики')
self.assertEqual(len(self.history()), 1)
print('\n 20 повторов: реплики %s, сообщений в истории: %d'
% (sorted({instance for _, instance, _ in results}), len(self.history())))
if __name__ == '__main__':
unittest.main()
+41
View File
@@ -0,0 +1,41 @@
import unittest
import uuid
from helpers import call, new_chat, new_user
class ReplicasTest(unittest.TestCase):
def test_one_session_is_served_by_several_replicas(self):
"""Один токен без повторного входа обслуживается разными репликами (X-Instance-Id), все ответы 200"""
alice = new_user('alice')
results = [call('GET', '/api/v1/users/me', token=alice['token']) for _ in range(10)]
self.assertEqual({status for status, _, _ in results}, {200})
self.assertGreaterEqual(len({headers['X-Instance-Id'] for _, headers, _ in results}), 2)
def test_write_on_one_replica_is_read_on_another(self):
"""Сообщение, подтверждённое одной репликой, сразу видно при чтении через другую"""
alice = new_user('alice')
chat = new_chat(alice)
path = '/api/v1/chats/%s/messages' % chat['id']
for i in range(6):
_, written_by, message = call('POST', path, {'text': 'm%d' % i}, alice['token'])
status, read_by, page = call('GET', path + '?limit=100', token=alice['token'])
self.assertEqual(status, 200)
self.assertIn(message['id'], [m['id'] for m in page['items']],
'запись через %s не видна через %s' % (written_by['X-Instance-Id'], read_by['X-Instance-Id']))
def test_every_api_response_has_instance_id(self):
"""X-Instance-Id есть и в успешных ответах, и в ошибках API, и в health"""
alice = new_user('alice')
for method, path, token, expected in (
('GET', '/health/live', None, 200), ('GET', '/health/ready', None, 200),
('GET', '/api/v1/users/me', None, 401), ('GET', '/api/v1/users/me', alice['token'], 200),
('GET', '/api/v1/chats/' + str(uuid.uuid4()), alice['token'], 404), ('GET', '/nope', None, 404)):
with self.subTest(path=path, expected=expected):
status, headers, _ = call(method, path, token=token)
self.assertEqual(status, expected)
self.assertTrue(headers['X-Instance-Id'])
if __name__ == '__main__':
unittest.main()
+81
View File
@@ -0,0 +1,81 @@
import subprocess
import unittest
from helpers import compose, new_chat, new_user, sql
from imagegen import SAMPLE_PNG, noise_png
from test_attachments import upload
LIMITED_CLIENT = '''
import json
from botocore.exceptions import ClientError
from config import settings
from s3 import client
results = {}
for name, action in (
("put_get_delete_own_bucket", lambda: (client.put_object(Bucket=settings.s3_bucket, Key="probe/x", Body=b"x"),
client.get_object(Bucket=settings.s3_bucket, Key="probe/x"),
client.delete_object(Bucket=settings.s3_bucket, Key="probe/x"))),
("list_bucket_objects", lambda: client.list_objects_v2(Bucket=settings.s3_bucket)),
("create_bucket", lambda: client.create_bucket(Bucket="intruder")),
("delete_bucket", lambda: client.delete_bucket(Bucket=settings.s3_bucket)),
("list_all_buckets", lambda: client.list_buckets()["Buckets"])):
try:
value = action()
results[name] = "allowed" if name != "list_all_buckets" else "allowed:%d" % len(value)
except ClientError as error:
results[name] = error.response["Error"]["Code"]
print(json.dumps(results))
'''
class StorageTest(unittest.TestCase):
def setUp(self):
self.alice = new_user('alice')
self.chat = new_chat(self.alice)
def object_key(self, attachment_id):
return sql("SELECT object_key FROM attachments WHERE id = '%s'" % attachment_id)
def test_object_key_is_generated_by_server(self):
"""Ключ объекта = attachments/<uuid вложения>, filename в нём не участвует"""
att = upload(self.alice, self.chat, SAMPLE_PNG, filename='мой-файл.png')[2]
self.assertEqual(self.object_key(att['id']), 'attachments/' + att['id'])
def test_bucket_is_private_outside_api(self):
"""Анонимный GET объекта и листинг bucket напрямую в S3 (из сети backend) → 403"""
att = upload(self.alice, self.chat, SAMPLE_PNG)[2]
for path in ('/messenger-attachments/' + self.object_key(att['id']), '/messenger-attachments'):
with self.subTest(path=path):
result = compose('exec', '-T', 'lb', 'wget', '-S', '-O', '/dev/null', 'http://s3:8333' + path, check=False)
self.assertNotIn('200 OK', result)
output = compose('exec', '-T', 'lb', 'sh', '-c', 'wget -S -O /dev/null http://s3:8333%s 2>&1; true' % path)
self.assertIn('403', output)
def test_api_credentials_are_limited(self):
"""Ключи API: свои объекты читать/писать/удалять можно; листинг, создание и удаление bucket — AccessDenied"""
import json
results = json.loads(compose('exec', '-T', 'api', 'python', '-c', 'import sys; sys.path.insert(0, "src")\n' + LIMITED_CLIENT))
self.assertEqual(results['put_get_delete_own_bucket'], 'allowed')
for action in ('list_bucket_objects', 'create_bucket', 'delete_bucket'):
self.assertEqual(results[action], 'AccessDenied', results)
self.assertNotEqual(results['list_all_buckets'], 'allowed:1', 'API sees the bucket list')
def test_bootstrap_is_idempotent(self):
"""Повторный s3-init: код 0, bucket уже есть, приватность подтверждена"""
output = compose('run', '--rm', '--no-deps', 's3-init')
self.assertIn('already exists, nothing to do', output)
self.assertIn('is private', output)
def test_no_original_bytes_in_database_or_api_container(self):
"""В attachments единственный bytea — sha256 (байтов файла в БД нет); после загрузки 2 МБ в /tmp реплик API пусто"""
binary = sql("SELECT string_agg(column_name, ',') FROM information_schema.columns "
"WHERE table_name = 'attachments' AND data_type = 'bytea'")
self.assertEqual(binary, 'sha256')
self.assertEqual(upload(self.alice, self.chat, noise_png(2 * 1024 * 1024))[0], 201)
for container in compose('ps', '-q', 'api').split():
files = subprocess.run(['docker', 'exec', container, 'sh', '-c', 'find /tmp -type f | wc -l'],
capture_output=True, text=True).stdout.strip()
self.assertEqual(files, '0', container[:12])
if __name__ == '__main__':
unittest.main()
+118
View File
@@ -0,0 +1,118 @@
"""Usage (стенд запущен; на время эксперимента лучше остановить другие стенды):
python student-tests/upload_memory.py --output evidence/upload-memory.json
N одновременных загрузок PNG ~10 MiB через LB при двух настройках буфера тела запроса
(UPLOAD_MEMORY_BYTES = 1 MiB в RAM, дальше временный файл — рабочая настройка; и 16 MiB — весь файл в RAM).
Перед каждым прогоном реплики перезапускаются (memory.peak cgroup считается с момента старта контейнера);
во время прогона каждые 0,1 с снимаются memory.current и anon/file из memory.stat каждой реплики.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor
import http.client
import json
import os
import subprocess
import threading
import time
import urllib.parse
from helpers import BASE_URL, api_containers, compose, new_chat, new_user, wait_in_rotation, wait_ready
from imagegen import noise_png
MIB = 1024 * 1024
SAMPLER = ('while true; do echo "current $(cat /sys/fs/cgroup/memory.current)"; '
'grep -E "^(anon|file) " /sys/fs/cgroup/memory.stat; sleep 0.1; done')
class Sampler:
def __init__(self, container):
self.container, self.peaks = container, {'current': 0, 'anon': 0, 'file': 0}
self.process = subprocess.Popen(['docker', 'exec', container, 'sh', '-c', SAMPLER],
stdout=subprocess.PIPE, text=True)
self.thread = threading.Thread(target=self.read, daemon=True)
self.thread.start()
def read(self):
for line in self.process.stdout:
name, _, value = line.partition(' ')
if name in self.peaks and value.strip().isdigit():
self.peaks[name] = max(self.peaks[name], int(value))
def stop(self):
self.process.terminate()
self.process.wait()
return {name: round(value / MIB, 1) for name, value in self.peaks.items()}
def cgroup(container, name):
return int(subprocess.run(['docker', 'exec', container, 'cat', '/sys/fs/cgroup/' + name],
capture_output=True, text=True).stdout.strip())
def post(token, chat_id, data):
target = urllib.parse.urlsplit(BASE_URL)
connection = http.client.HTTPConnection(target.hostname, target.port, timeout=120)
started = time.monotonic()
try:
connection.request('POST', '/api/v1/chats/%s/attachments?filename=big.png' % chat_id, body=data,
headers={'Authorization': 'Bearer ' + token, 'Content-Type': 'image/png'})
status = connection.getresponse().status
except OSError as error:
status = type(error).__name__
connection.close()
return status, time.monotonic() - started
def restart_api(memory_bytes):
env = dict(os.environ, UPLOAD_MEMORY_BYTES=str(memory_bytes))
compose('up', '-d', '--no-deps', '--no-build', '--force-recreate', 'api', env=env)
time.sleep(3)
wait_ready()
for container in api_containers():
wait_in_rotation(container)
def run(label, memory_bytes, parallel, data):
restart_api(memory_bytes)
alice = new_user('memory')
chat = new_chat(alice)
replicas = api_containers()
idle = {c[:12]: round(cgroup(c, 'memory.current') / MIB, 1) for c in replicas}
samplers = [Sampler(c) for c in replicas]
started = time.monotonic()
with ThreadPoolExecutor(max_workers=parallel) as pool:
results = list(pool.map(lambda _: post(alice['token'], chat['id'], data), range(parallel)))
wall = time.monotonic() - started
time.sleep(0.5)
sampled = {c[:12]: s.stop() for c, s in zip(replicas, samplers)}
peak = {c[:12]: round(cgroup(c, 'memory.peak') / MIB, 1) for c in replicas}
limit = round(cgroup(replicas[0], 'memory.max') / MIB)
statuses = [r[0] for r in results]
summary = {'label': label, 'upload_memory_bytes': memory_bytes, 'parallel': parallel,
'file_mib': round(len(data) / MIB, 2), 'ok': statuses.count(201),
'errors': sorted({str(s) for s in statuses if s != 201}), 'wall_s': round(wall, 2),
'max_request_s': round(max(r[1] for r in results), 2), 'limit_mib': limit,
'idle_current_mib': idle, 'peak_since_start_mib': peak, 'sampled_peak_mib': sampled}
worst_anon = max(v['anon'] for v in sampled.values())
print('%-28s ok=%2d/%d wall=%5.1fs peak(memory.peak)=%s anon max=%.1f MiB'
% (label, summary['ok'], parallel, wall, list(peak.values()), worst_anon))
return summary
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--output')
parser.add_argument('--parallel', default='10,20')
args = parser.parse_args()
data = noise_png(10 * MIB - 50000)
runs = [run('spool 1 MiB, %s parallel' % n, MIB, int(n), data) for n in args.parallel.split(',')]
runs.append(run('all in RAM, %s parallel' % args.parallel.split(',')[-1], 16 * MIB,
int(args.parallel.split(',')[-1]), data))
restart_api(MIB)
if args.output:
with open(args.output, 'w', encoding='utf-8') as file:
json.dump({'runs': runs}, file, ensure_ascii=False, indent=2)
if __name__ == '__main__':
main()