Files
mtusi2026lab4/scripts/experiments.py
vest1ter 44ae7b2472 ЛР4 на 5: вложения в MinIO, отказоустойчивость, трек 2 (сверка S3 и PostgreSQL)
Уровень 3:
- код lab3 перенесён; MinIO (сборка релиза 2025-10-15, по digest) в общем Compose,
  идемпотентный bootstrap: приватные bucket, отдельный пользователь API с ограниченной политикой
- метаданные вложений в PostgreSQL (миграции 0003, 0004), байты в S3; ключ назначает сервер
- upload PNG/JPEG: лимиты до чтения/декодирования, MIME по содержимому, проверка целостности,
  безопасный filename; 201 только после PUT и COMMIT; download потоком через API
- attachment_ids в сообщениях (только файлы своего чата, FK в БД)
- 503 API больше не повторяется в HAProxy (отказ общей зависимости), S3 не входит в readiness

Уровень 4:
- 45 тестов вложений: размер, повреждение, подмена MIME, чужой id, файл другого чата, отказы S3/БД
- компенсация при сбое COMMIT; слоты загрузки (8 на реплику) против неограниченной памяти
- эксперименты: пик RAM при 10-40 одновременных загрузках, оборванная загрузка и сироты;
  исправлены удержание соединения БД во время загрузки и таймаут отправки в S3

Уровень 5, трек 2:
- reconciler: слияние листинга S3 и курсора БД, 7 категорий расхождений, dry-run, grace period,
  перепроверка перед действием, тормоз --max-deletes, advisory lock; сервис reconciler в Compose
- 8 тестов сверки; эксперимент с 70 внедрёнными ошибками на 10 000 объектах (точное обнаружение,
  безопасное исправление, идемпотентность) и замер стоимости обхода на 1k/10k/50k
2026-11-12 17:38:00 +03:00

414 lines
20 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Эксперименты балансировки (перенесены из lab3: idempotency, failover, rolling). Стенд (docker compose up -d --build --wait) должен быть уже запущен.
failover останавливает каждую API-реплику по очереди под нагрузкой, восстанавливает её и
только потом берёт следующую; считает RPS, ошибки и время восстановления
idempotency параллельно шлёт один Idempotency-Key через LB (разные реплики): должно быть ровно 1 сообщение
rolling поочерёдная замена всех реплик на новую сборку под нагрузкой (migrate запускается один раз)
Результаты (без токенов) пишутся в evidence/*.json. Данные создаются синтетические и не удаляются.
Использует публичный tests/client.py и docker CLI из каталога проекта.
"""
import argparse
import json
import math
import os
from pathlib import Path
import subprocess
import sys
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check # noqa: E402
PROJECT = 'mtusi-lab4'
# ---------- docker ----------
def docker(*args, env=None, timeout=180, ok=True, with_stderr=False):
proc = subprocess.run(['docker', *args], cwd=ROOT, capture_output=True, text=True, encoding='utf-8',
errors='replace', timeout=timeout, env=env)
if ok and proc.returncode != 0:
raise Failure('docker ' + ' '.join(args[:3]) + ' failed: ' + proc.stderr.strip()[:300])
return (proc.stdout + proc.stderr).strip() if with_stderr else proc.stdout.strip()
def api_containers(all_states=False):
"""Короткие ID контейнеров сервиса api. Совпадают с X-Instance-Id (hostname контейнера)."""
args = ['ps', '-q', '--filter', 'label=com.docker.compose.project=' + PROJECT,
'--filter', 'label=com.docker.compose.service=api']
if all_states:
args.insert(1, '-a')
return sorted(x for x in docker(*args).split() if x)
def wait_healthy(ids, timeout=90):
deadline = time.monotonic() + timeout
while True:
states = {i: docker('inspect', '-f', '{{.State.Health.Status}}', i) for i in ids}
if all(s == 'healthy' for s in states.values()):
return
check(time.monotonic() < deadline, 'containers did not become healthy: ' + str(states))
time.sleep(1)
def wait_instances(c, actor, expected, timeout=60):
"""Ждать, пока через LB ответят все ожидаемые реплики (т.е. LB включил их в ротацию)."""
deadline = time.monotonic() + timeout
seen = set()
while True:
for _ in range(20):
try:
seen.add(c.call('GET', '/api/v1/users/me', actor=actor)['headers'].get('X-Instance-Id'))
except Failure:
pass
if set(expected) <= seen:
return
check(time.monotonic() < deadline, 'LB did not route to all replicas: missing ' + str(set(expected) - seen))
time.sleep(0.5)
def resource_snapshot():
out = docker('stats', '--no-stream', '--format', '{{.Name}}: cpu {{.CPUPerc}}, mem {{.MemUsage}}', ok=False)
return [line for line in out.splitlines() if line]
# ---------- нагрузка ----------
class Load:
"""Замкнутая нагрузка: запись сообщения (с ретраями ТОГО ЖЕ ключа при ошибке) + чтение страницы истории."""
def __init__(self, base_url, ca_file, actor, chat, workers):
self.base_url, self.ca_file, self.actor, self.chat, self.workers = base_url, ca_file, actor, chat, workers
self.events, self.acked = [], [] # events: (t_start, t_end, kind, ok, instance, error_reason)
self.stop_event = threading.Event()
self.lock = threading.Lock()
self.threads = []
self.t0 = time.monotonic()
def now(self):
return time.monotonic() - self.t0
def record(self, start, kind, ok, instance, error=None):
# Причина без пути (в нём id чата): "expected [201], received 503" или "connection/TLS failure (...)".
reason = str(error).split(': ', 1)[-1] if error else None
with self.lock:
self.events.append((start, self.now(), kind, ok, instance, reason))
def worker(self, wid):
c = Client(self.base_url, self.ca_file, timeout=2)
path = '/api/v1/chats/' + self.chat['id'] + '/messages'
i = 0
while not self.stop_event.is_set():
key = uuid.uuid4().hex
text = 'exp-%d-%d-%s' % (wid, i, key)
i += 1
for _ in range(10):
start = self.now()
try:
r = c.message(self.actor, self.chat, text, key=key)
self.record(start, 'write', True, r['headers'].get('X-Instance-Id'))
with self.lock:
self.acked.append(r['json'])
break
except Failure as exc:
self.record(start, 'write', False, None, exc) # неопределённый результат: повтор с тем же ключом
if self.stop_event.wait(0.1):
break
start = self.now()
try:
r = c.call('GET', path + '?limit=1', actor=self.actor, schema='MessagePage')
self.record(start, 'read', True, r['headers'].get('X-Instance-Id'))
except Failure as exc:
self.record(start, 'read', False, None, exc)
def start(self):
self.threads = [threading.Thread(target=self.worker, args=(n,), daemon=True) for n in range(self.workers)]
for t in self.threads:
t.start()
def stop(self):
self.stop_event.set()
for t in self.threads:
t.join(timeout=15)
def percentile(sorted_values, p):
if not sorted_values:
return None
return round(1000 * sorted_values[max(0, math.ceil(p * len(sorted_values)) - 1)], 1)
def window(events, lo, hi):
"""Метрики по запросам, завершившимся в [lo, hi)."""
sel = [e for e in events if lo <= e[1] < hi]
ok = [e for e in sel if e[3]]
errors = [e for e in sel if not e[3]]
lat = sorted(e[1] - e[0] for e in ok)
span = max(hi - lo, 1e-9)
reasons = {}
for e in errors:
key = e[2] + ': ' + (e[5] or 'unknown')
reasons[key] = reasons.get(key, 0) + 1
return {
'seconds': round(span, 2),
'successful_requests': len(ok), 'failed_requests': len(errors), 'error_reasons': reasons,
'error_ratio': round(len(errors) / len(sel), 4) if sel else None,
'successful_rps': round(len(ok) / span, 1),
'latency_ms': {'p50': percentile(lat, .5), 'p95': percentile(lat, .95), 'p99': percentile(lat, .99)},
}
def timeline(events, end):
rows = []
for s in range(int(end) + 1):
sel = [e for e in events if s <= e[1] < s + 1]
rows.append({'t': s, 'ok': sum(1 for e in sel if e[3]), 'errors': sum(1 for e in sel if not e[3])})
return rows
def last_error_after(events, moment):
late = [e[1] for e in events if not e[3] and e[1] >= moment]
return round(max(late) - moment, 2) if late else 0.0
def history(c, actor, chat):
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
result, seen = [], set()
for _ in range(100000):
page = c.call('GET', path, actor=actor, schema='MessagePage')['json']
result.extend(page['items'])
cursor = page['next_cursor']
if cursor is None:
return result
check(cursor not in seen, 'cursor cycle')
seen.add(cursor)
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(cursor, safe='')
raise Failure('history pagination exceeded safety bound')
def verify(c, actor, chat, acked):
"""Подтверждённые сообщения не потеряны, а повторы с тем же ключом не создали дублей."""
messages = history(c, actor, chat)
by_id = {m['id']: m for m in messages}
lost = sum(1 for a in acked if by_id.get(a['id']) != a)
texts = [m['text'] for m in messages if m['text'].startswith('exp-')]
duplicates = len(texts) - len(set(texts))
return {'acked_messages': len(acked), 'messages_in_history': len(messages),
'lost_acked_messages': lost, 'duplicate_messages': duplicates}
def write_evidence(out_dir, name, data):
out_dir.mkdir(parents=True, exist_ok=True)
path = out_dir / name
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
print('written', path)
def setup(args):
c = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = c.bootstrap()
return c, alice, bob, chat
# ---------- эксперименты ----------
def run_failover(args):
c, alice, bob, chat = setup(args)
containers = api_containers()
check(len(containers) >= 2, 'need at least 2 running API replicas, found %d' % len(containers))
results = []
for target in containers:
# Предыдущая реплика уже восстановлена; убеждаемся, что в ротации все.
docker('compose', 'up', '-d', '--no-recreate', '--wait', 'api', 'lb', timeout=300)
wait_instances(c, alice, containers)
load = Load(args.base_url, args.ca_file, alice, chat, args.workers)
load.start()
time.sleep(args.warmup)
resources = resource_snapshot()
t_stop = load.now()
docker('stop', '-t', '8', target)
stop_command_seconds = round(load.now() - t_stop, 2)
time.sleep(args.after)
t_restore = load.now()
docker('compose', 'up', '-d', '--no-recreate', '--wait', 'api', 'lb', timeout=300)
wait_instances(c, alice, containers)
t_back = load.now()
time.sleep(args.settle)
load.stop()
end = load.now()
stats = verify(c, bob, chat, load.acked)
after_stop = [e for e in load.events if e[1] >= t_stop and e[3]]
first_other = min((e[1] for e in after_stop if e[4] != target and e[2] == 'write'), default=None)
# Сколько запросов остановленная реплика обслужила после возврата `docker stop` и до восстановления: должно быть 0.
served_by_target_after_stop = sum(1 for e in load.events
if t_stop + stop_command_seconds <= e[0] < t_restore and e[3] and e[4] == target)
results.append({
'stopped_replica': target,
'baseline_before_stop': window(load.events, 0, t_stop),
'stop_to_restore': window(load.events, t_stop, t_restore),
'after_restore': window(load.events, t_restore, end),
'docker_stop_seconds': stop_command_seconds,
'last_error_seconds_after_stop': last_error_after(load.events, t_stop),
'first_success_on_other_replica_seconds': round(first_other - t_stop, 2) if first_other else None,
'requests_served_by_stopped_replica_after_stop_returned': served_by_target_after_stop,
# Включает `docker compose up --wait` (повторный migrate, start_period healthcheck) и rise 2 в LB.
'seconds_until_replica_back_in_rotation': round(t_back - t_restore, 2),
'instances_seen': sorted({e[4] for e in load.events if e[4]}),
'resource_snapshot_under_load': resources,
'timeline_per_second': timeline(load.events, end),
**stats,
})
print('replica %s: errors=%d, last error %.2fs after stop, lost=%d, dup=%d' % (
target, results[-1]['stop_to_restore']['failed_requests'],
results[-1]['last_error_seconds_after_stop'], stats['lost_acked_messages'], stats['duplicate_messages']))
write_evidence(Path(args.output), 'failover-sequential.json', {
'method': 'closed loop %d workers: POST message (retry same Idempotency-Key on error) + GET page; '
'client timeout 2s; each replica stopped with docker stop -t 8, restored, then the next' % args.workers,
'replicas': containers, 'warmup_seconds': args.warmup, 'seconds_down': args.after, 'runs': results})
for r in results:
check(r['lost_acked_messages'] == 0 and r['duplicate_messages'] == 0, 'data integrity violated')
def run_idempotency(args):
c, alice, bob, chat = setup(args)
path = '/api/v1/chats/' + chat['id'] + '/messages'
instances, rounds = set(), []
for _ in range(args.rounds):
key = uuid.uuid4().hex
text = 'idem-' + key
barrier = threading.Barrier(args.parallel)
def send(_):
local = Client(args.base_url, args.ca_file, timeout=10)
barrier.wait()
try:
r = local.message(alice, chat, text, key=key)
return r['status'], r['json']['id'], r['headers'].get('X-Instance-Id')
except Failure as exc:
return None, str(exc), None
with ThreadPoolExecutor(max_workers=args.parallel) as pool:
answers = list(pool.map(send, range(args.parallel)))
ok = [a for a in answers if a[0] == 201]
instances.update(a[2] for a in ok)
rounds.append({'responses_201': len(ok), 'failures': len(answers) - len(ok),
'distinct_message_ids': len({a[1] for a in ok}),
'distinct_instances': len({a[2] for a in ok})})
messages = history(c, bob, chat)
texts = [m['text'] for m in messages]
created = [texts.count(m['text']) for m in messages if m['text'].startswith('idem-')]
# Тот же ключ с другим телом -> 409.
key = uuid.uuid4().hex
c.message(alice, chat, 'idem-first-' + key, key=key)
other_body = c.call('POST', path, 409, actor=alice, body={'text': 'idem-other-' + key},
headers={'Idempotency-Key': key})
report = {
'method': '%d rounds; each round %d simultaneous identical POSTs with one Idempotency-Key through the LB '
'(separate connections, released by a barrier)' % (args.rounds, args.parallel),
'rounds': args.rounds, 'parallel_requests_per_round': args.parallel,
'instances_seen': sorted(i for i in instances if i),
'all_rounds_single_message_id': all(r['distinct_message_ids'] == 1 for r in rounds),
'failed_requests': sum(r['failures'] for r in rounds),
'rounds_with_requests_on_2plus_replicas': sum(1 for r in rounds if r['distinct_instances'] >= 2),
'messages_created_per_key_max': max(created) if created else 0,
'same_key_other_body_status': other_body['status'],
'per_round': rounds,
}
write_evidence(Path(args.output), 'idempotency-parallel.json', report)
check(report['all_rounds_single_message_id'], 'a key produced several message ids')
check(report['messages_created_per_key_max'] == 1, 'a key created several messages')
check(report['failed_requests'] == 0, 'some parallel requests failed')
def run_rolling(args):
c, alice, bob, chat = setup(args)
old = api_containers()
n = len(old)
check(n >= 2, 'need at least 2 running API replicas')
old_image = {i: docker('inspect', '-f', '{{.Image}}', i) for i in old}
# Migrate запускается один раз, до замены; миграции только добавляют (expand), старые реплики их переживают.
# Alembic пишет журнал в stderr.
migrate_log = docker('compose', 'run', '--rm', 'migrate', timeout=300, with_stderr=True)
load = Load(args.base_url, args.ca_file, alice, chat, args.workers)
load.start()
time.sleep(args.warmup)
t_start = load.now()
env = dict(os.environ, APP_VERSION='rolling-' + time.strftime('%H%M%S'))
docker('compose', 'build', 'api', env=env, timeout=600)
docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=%d' % (2 * n), 'api',
env=env, timeout=300)
new = [i for i in api_containers() if i not in old]
check(len(new) == n, 'expected %d new replicas, found %d' % (n, len(new)))
wait_healthy(new)
wait_instances(c, alice, new)
steps = []
for victim in old:
t = load.now()
docker('stop', '-t', '15', victim)
docker('rm', victim)
steps.append({'replaced': victim, 'seconds_into_update': round(t - t_start, 2)})
time.sleep(args.step_pause)
docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=%d' % n, 'api', env=env, timeout=300)
t_done = load.now()
time.sleep(args.settle)
load.stop()
end = load.now()
stats = verify(c, bob, chat, load.acked)
seen_old = [e for e in load.events if e[3] and e[4] in old]
seen_new = [e for e in load.events if e[3] and e[4] in new]
report = {
'method': 'closed loop %d workers under rolling replacement: start N new replicas of the new build, '
'wait until healthy and routed by LB, then stop+remove old replicas one at a time' % args.workers,
'replicas_before': old, 'replicas_after': api_containers(), 'new_replicas': new,
'old_image_ids': old_image, 'new_image_ids': {i: docker('inspect', '-f', '{{.Image}}', i) for i in new},
'app_version_build_arg': env['APP_VERSION'],
'update_seconds': round(t_done - t_start, 2),
'steps': steps,
'before_update': window(load.events, 0, t_start),
'during_update': window(load.events, t_start, t_done),
'after_update': window(load.events, t_done, end),
# Все моменты — секунды от начала обновления, как и steps[].seconds_into_update.
'first_new_replica_response_seconds_into_update': round(min((e[1] for e in seen_new), default=t_start) - t_start, 2),
'last_old_replica_response_seconds_into_update': round(max((e[1] for e in seen_old), default=t_start) - t_start, 2),
'migrate_output_tail': migrate_log.splitlines()[-3:],
'timeline_per_second': timeline(load.events, end),
**stats,
}
write_evidence(Path(args.output), 'rolling-update.json', report)
print('rolling: errors during update=%d, lost=%d, dup=%d' % (
report['during_update']['failed_requests'], stats['lost_acked_messages'], stats['duplicate_messages']))
check(stats['lost_acked_messages'] == 0 and stats['duplicate_messages'] == 0, 'data integrity violated')
def main():
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument('experiment', choices=['failover', 'idempotency', 'rolling'])
p.add_argument('--base-url', default=None)
p.add_argument('--ca-file', default=None)
p.add_argument('--workers', type=int, default=8, help='closed-loop workers (failover, rolling)')
p.add_argument('--warmup', type=float, default=5, help='seconds of load before the action')
p.add_argument('--after', type=float, default=15, help='failover: seconds the replica stays down under load')
p.add_argument('--settle', type=float, default=8, help='seconds of load after the action')
p.add_argument('--step-pause', type=float, default=3, help='rolling: pause between replacing replicas')
p.add_argument('--rounds', type=int, default=30, help='idempotency: number of keys')
p.add_argument('--parallel', type=int, default=16, help='idempotency: simultaneous requests per key')
p.add_argument('--output', default=str(ROOT / 'evidence'))
args = p.parse_args()
try:
{'failover': run_failover, 'idempotency': run_idempotency, 'rolling': run_rolling}[args.experiment](args)
except (Failure, RuntimeError, subprocess.TimeoutExpired) as exc:
print('FAIL:', exc)
raise SystemExit(1)
print('PASS:', args.experiment)
if __name__ == '__main__':
main()