Уровень 3: - код lab3 перенесён; MinIO (сборка релиза 2025-10-15, по digest) в общем Compose, идемпотентный bootstrap: приватные bucket, отдельный пользователь API с ограниченной политикой - метаданные вложений в PostgreSQL (миграции 0003, 0004), байты в S3; ключ назначает сервер - upload PNG/JPEG: лимиты до чтения/декодирования, MIME по содержимому, проверка целостности, безопасный filename; 201 только после PUT и COMMIT; download потоком через API - attachment_ids в сообщениях (только файлы своего чата, FK в БД) - 503 API больше не повторяется в HAProxy (отказ общей зависимости), S3 не входит в readiness Уровень 4: - 45 тестов вложений: размер, повреждение, подмена MIME, чужой id, файл другого чата, отказы S3/БД - компенсация при сбое COMMIT; слоты загрузки (8 на реплику) против неограниченной памяти - эксперименты: пик RAM при 10-40 одновременных загрузках, оборванная загрузка и сироты; исправлены удержание соединения БД во время загрузки и таймаут отправки в S3 Уровень 5, трек 2: - reconciler: слияние листинга S3 и курсора БД, 7 категорий расхождений, dry-run, grace period, перепроверка перед действием, тормоз --max-deletes, advisory lock; сервис reconciler в Compose - 8 тестов сверки; эксперимент с 70 внедрёнными ошибками на 10 000 объектах (точное обнаружение, безопасное исправление, идемпотентность) и замер стоимости обхода на 1k/10k/50k
414 lines
20 KiB
Python
414 lines
20 KiB
Python
#!/usr/bin/env python3
|
||
"""Эксперименты балансировки (перенесены из lab3: idempotency, failover, rolling). Стенд (docker compose up -d --build --wait) должен быть уже запущен.
|
||
|
||
failover останавливает каждую API-реплику по очереди под нагрузкой, восстанавливает её и
|
||
только потом берёт следующую; считает RPS, ошибки и время восстановления
|
||
idempotency параллельно шлёт один Idempotency-Key через LB (разные реплики): должно быть ровно 1 сообщение
|
||
rolling поочерёдная замена всех реплик на новую сборку под нагрузкой (migrate запускается один раз)
|
||
|
||
Результаты (без токенов) пишутся в evidence/*.json. Данные создаются синтетические и не удаляются.
|
||
Использует публичный tests/client.py и docker CLI из каталога проекта.
|
||
"""
|
||
import argparse
|
||
import json
|
||
import math
|
||
import os
|
||
from pathlib import Path
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import uuid
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from urllib.parse import quote
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(ROOT / 'tests'))
|
||
from client import Client, Failure, check # noqa: E402
|
||
|
||
PROJECT = 'mtusi-lab4'
|
||
|
||
|
||
# ---------- docker ----------
|
||
|
||
def docker(*args, env=None, timeout=180, ok=True, with_stderr=False):
|
||
proc = subprocess.run(['docker', *args], cwd=ROOT, capture_output=True, text=True, encoding='utf-8',
|
||
errors='replace', timeout=timeout, env=env)
|
||
if ok and proc.returncode != 0:
|
||
raise Failure('docker ' + ' '.join(args[:3]) + ' failed: ' + proc.stderr.strip()[:300])
|
||
return (proc.stdout + proc.stderr).strip() if with_stderr else proc.stdout.strip()
|
||
|
||
|
||
def api_containers(all_states=False):
|
||
"""Короткие ID контейнеров сервиса api. Совпадают с X-Instance-Id (hostname контейнера)."""
|
||
args = ['ps', '-q', '--filter', 'label=com.docker.compose.project=' + PROJECT,
|
||
'--filter', 'label=com.docker.compose.service=api']
|
||
if all_states:
|
||
args.insert(1, '-a')
|
||
return sorted(x for x in docker(*args).split() if x)
|
||
|
||
|
||
def wait_healthy(ids, timeout=90):
|
||
deadline = time.monotonic() + timeout
|
||
while True:
|
||
states = {i: docker('inspect', '-f', '{{.State.Health.Status}}', i) for i in ids}
|
||
if all(s == 'healthy' for s in states.values()):
|
||
return
|
||
check(time.monotonic() < deadline, 'containers did not become healthy: ' + str(states))
|
||
time.sleep(1)
|
||
|
||
|
||
def wait_instances(c, actor, expected, timeout=60):
|
||
"""Ждать, пока через LB ответят все ожидаемые реплики (т.е. LB включил их в ротацию)."""
|
||
deadline = time.monotonic() + timeout
|
||
seen = set()
|
||
while True:
|
||
for _ in range(20):
|
||
try:
|
||
seen.add(c.call('GET', '/api/v1/users/me', actor=actor)['headers'].get('X-Instance-Id'))
|
||
except Failure:
|
||
pass
|
||
if set(expected) <= seen:
|
||
return
|
||
check(time.monotonic() < deadline, 'LB did not route to all replicas: missing ' + str(set(expected) - seen))
|
||
time.sleep(0.5)
|
||
|
||
|
||
def resource_snapshot():
|
||
out = docker('stats', '--no-stream', '--format', '{{.Name}}: cpu {{.CPUPerc}}, mem {{.MemUsage}}', ok=False)
|
||
return [line for line in out.splitlines() if line]
|
||
|
||
|
||
# ---------- нагрузка ----------
|
||
|
||
class Load:
|
||
"""Замкнутая нагрузка: запись сообщения (с ретраями ТОГО ЖЕ ключа при ошибке) + чтение страницы истории."""
|
||
|
||
def __init__(self, base_url, ca_file, actor, chat, workers):
|
||
self.base_url, self.ca_file, self.actor, self.chat, self.workers = base_url, ca_file, actor, chat, workers
|
||
self.events, self.acked = [], [] # events: (t_start, t_end, kind, ok, instance, error_reason)
|
||
self.stop_event = threading.Event()
|
||
self.lock = threading.Lock()
|
||
self.threads = []
|
||
self.t0 = time.monotonic()
|
||
|
||
def now(self):
|
||
return time.monotonic() - self.t0
|
||
|
||
def record(self, start, kind, ok, instance, error=None):
|
||
# Причина без пути (в нём id чата): "expected [201], received 503" или "connection/TLS failure (...)".
|
||
reason = str(error).split(': ', 1)[-1] if error else None
|
||
with self.lock:
|
||
self.events.append((start, self.now(), kind, ok, instance, reason))
|
||
|
||
def worker(self, wid):
|
||
c = Client(self.base_url, self.ca_file, timeout=2)
|
||
path = '/api/v1/chats/' + self.chat['id'] + '/messages'
|
||
i = 0
|
||
while not self.stop_event.is_set():
|
||
key = uuid.uuid4().hex
|
||
text = 'exp-%d-%d-%s' % (wid, i, key)
|
||
i += 1
|
||
for _ in range(10):
|
||
start = self.now()
|
||
try:
|
||
r = c.message(self.actor, self.chat, text, key=key)
|
||
self.record(start, 'write', True, r['headers'].get('X-Instance-Id'))
|
||
with self.lock:
|
||
self.acked.append(r['json'])
|
||
break
|
||
except Failure as exc:
|
||
self.record(start, 'write', False, None, exc) # неопределённый результат: повтор с тем же ключом
|
||
if self.stop_event.wait(0.1):
|
||
break
|
||
start = self.now()
|
||
try:
|
||
r = c.call('GET', path + '?limit=1', actor=self.actor, schema='MessagePage')
|
||
self.record(start, 'read', True, r['headers'].get('X-Instance-Id'))
|
||
except Failure as exc:
|
||
self.record(start, 'read', False, None, exc)
|
||
|
||
def start(self):
|
||
self.threads = [threading.Thread(target=self.worker, args=(n,), daemon=True) for n in range(self.workers)]
|
||
for t in self.threads:
|
||
t.start()
|
||
|
||
def stop(self):
|
||
self.stop_event.set()
|
||
for t in self.threads:
|
||
t.join(timeout=15)
|
||
|
||
|
||
def percentile(sorted_values, p):
|
||
if not sorted_values:
|
||
return None
|
||
return round(1000 * sorted_values[max(0, math.ceil(p * len(sorted_values)) - 1)], 1)
|
||
|
||
|
||
def window(events, lo, hi):
|
||
"""Метрики по запросам, завершившимся в [lo, hi)."""
|
||
sel = [e for e in events if lo <= e[1] < hi]
|
||
ok = [e for e in sel if e[3]]
|
||
errors = [e for e in sel if not e[3]]
|
||
lat = sorted(e[1] - e[0] for e in ok)
|
||
span = max(hi - lo, 1e-9)
|
||
reasons = {}
|
||
for e in errors:
|
||
key = e[2] + ': ' + (e[5] or 'unknown')
|
||
reasons[key] = reasons.get(key, 0) + 1
|
||
return {
|
||
'seconds': round(span, 2),
|
||
'successful_requests': len(ok), 'failed_requests': len(errors), 'error_reasons': reasons,
|
||
'error_ratio': round(len(errors) / len(sel), 4) if sel else None,
|
||
'successful_rps': round(len(ok) / span, 1),
|
||
'latency_ms': {'p50': percentile(lat, .5), 'p95': percentile(lat, .95), 'p99': percentile(lat, .99)},
|
||
}
|
||
|
||
|
||
def timeline(events, end):
|
||
rows = []
|
||
for s in range(int(end) + 1):
|
||
sel = [e for e in events if s <= e[1] < s + 1]
|
||
rows.append({'t': s, 'ok': sum(1 for e in sel if e[3]), 'errors': sum(1 for e in sel if not e[3])})
|
||
return rows
|
||
|
||
|
||
def last_error_after(events, moment):
|
||
late = [e[1] for e in events if not e[3] and e[1] >= moment]
|
||
return round(max(late) - moment, 2) if late else 0.0
|
||
|
||
|
||
def history(c, actor, chat):
|
||
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
|
||
result, seen = [], set()
|
||
for _ in range(100000):
|
||
page = c.call('GET', path, actor=actor, schema='MessagePage')['json']
|
||
result.extend(page['items'])
|
||
cursor = page['next_cursor']
|
||
if cursor is None:
|
||
return result
|
||
check(cursor not in seen, 'cursor cycle')
|
||
seen.add(cursor)
|
||
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(cursor, safe='')
|
||
raise Failure('history pagination exceeded safety bound')
|
||
|
||
|
||
def verify(c, actor, chat, acked):
|
||
"""Подтверждённые сообщения не потеряны, а повторы с тем же ключом не создали дублей."""
|
||
messages = history(c, actor, chat)
|
||
by_id = {m['id']: m for m in messages}
|
||
lost = sum(1 for a in acked if by_id.get(a['id']) != a)
|
||
texts = [m['text'] for m in messages if m['text'].startswith('exp-')]
|
||
duplicates = len(texts) - len(set(texts))
|
||
return {'acked_messages': len(acked), 'messages_in_history': len(messages),
|
||
'lost_acked_messages': lost, 'duplicate_messages': duplicates}
|
||
|
||
|
||
def write_evidence(out_dir, name, data):
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
path = out_dir / name
|
||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||
print('written', path)
|
||
|
||
|
||
def setup(args):
|
||
c = Client(args.base_url, args.ca_file)
|
||
alice, bob, eve, chat = c.bootstrap()
|
||
return c, alice, bob, chat
|
||
|
||
|
||
# ---------- эксперименты ----------
|
||
|
||
def run_failover(args):
|
||
c, alice, bob, chat = setup(args)
|
||
containers = api_containers()
|
||
check(len(containers) >= 2, 'need at least 2 running API replicas, found %d' % len(containers))
|
||
results = []
|
||
for target in containers:
|
||
# Предыдущая реплика уже восстановлена; убеждаемся, что в ротации все.
|
||
docker('compose', 'up', '-d', '--no-recreate', '--wait', 'api', 'lb', timeout=300)
|
||
wait_instances(c, alice, containers)
|
||
load = Load(args.base_url, args.ca_file, alice, chat, args.workers)
|
||
load.start()
|
||
time.sleep(args.warmup)
|
||
resources = resource_snapshot()
|
||
t_stop = load.now()
|
||
docker('stop', '-t', '8', target)
|
||
stop_command_seconds = round(load.now() - t_stop, 2)
|
||
time.sleep(args.after)
|
||
t_restore = load.now()
|
||
docker('compose', 'up', '-d', '--no-recreate', '--wait', 'api', 'lb', timeout=300)
|
||
wait_instances(c, alice, containers)
|
||
t_back = load.now()
|
||
time.sleep(args.settle)
|
||
load.stop()
|
||
end = load.now()
|
||
stats = verify(c, bob, chat, load.acked)
|
||
after_stop = [e for e in load.events if e[1] >= t_stop and e[3]]
|
||
first_other = min((e[1] for e in after_stop if e[4] != target and e[2] == 'write'), default=None)
|
||
# Сколько запросов остановленная реплика обслужила после возврата `docker stop` и до восстановления: должно быть 0.
|
||
served_by_target_after_stop = sum(1 for e in load.events
|
||
if t_stop + stop_command_seconds <= e[0] < t_restore and e[3] and e[4] == target)
|
||
results.append({
|
||
'stopped_replica': target,
|
||
'baseline_before_stop': window(load.events, 0, t_stop),
|
||
'stop_to_restore': window(load.events, t_stop, t_restore),
|
||
'after_restore': window(load.events, t_restore, end),
|
||
'docker_stop_seconds': stop_command_seconds,
|
||
'last_error_seconds_after_stop': last_error_after(load.events, t_stop),
|
||
'first_success_on_other_replica_seconds': round(first_other - t_stop, 2) if first_other else None,
|
||
'requests_served_by_stopped_replica_after_stop_returned': served_by_target_after_stop,
|
||
# Включает `docker compose up --wait` (повторный migrate, start_period healthcheck) и rise 2 в LB.
|
||
'seconds_until_replica_back_in_rotation': round(t_back - t_restore, 2),
|
||
'instances_seen': sorted({e[4] for e in load.events if e[4]}),
|
||
'resource_snapshot_under_load': resources,
|
||
'timeline_per_second': timeline(load.events, end),
|
||
**stats,
|
||
})
|
||
print('replica %s: errors=%d, last error %.2fs after stop, lost=%d, dup=%d' % (
|
||
target, results[-1]['stop_to_restore']['failed_requests'],
|
||
results[-1]['last_error_seconds_after_stop'], stats['lost_acked_messages'], stats['duplicate_messages']))
|
||
write_evidence(Path(args.output), 'failover-sequential.json', {
|
||
'method': 'closed loop %d workers: POST message (retry same Idempotency-Key on error) + GET page; '
|
||
'client timeout 2s; each replica stopped with docker stop -t 8, restored, then the next' % args.workers,
|
||
'replicas': containers, 'warmup_seconds': args.warmup, 'seconds_down': args.after, 'runs': results})
|
||
for r in results:
|
||
check(r['lost_acked_messages'] == 0 and r['duplicate_messages'] == 0, 'data integrity violated')
|
||
|
||
|
||
def run_idempotency(args):
|
||
c, alice, bob, chat = setup(args)
|
||
path = '/api/v1/chats/' + chat['id'] + '/messages'
|
||
instances, rounds = set(), []
|
||
for _ in range(args.rounds):
|
||
key = uuid.uuid4().hex
|
||
text = 'idem-' + key
|
||
barrier = threading.Barrier(args.parallel)
|
||
|
||
def send(_):
|
||
local = Client(args.base_url, args.ca_file, timeout=10)
|
||
barrier.wait()
|
||
try:
|
||
r = local.message(alice, chat, text, key=key)
|
||
return r['status'], r['json']['id'], r['headers'].get('X-Instance-Id')
|
||
except Failure as exc:
|
||
return None, str(exc), None
|
||
|
||
with ThreadPoolExecutor(max_workers=args.parallel) as pool:
|
||
answers = list(pool.map(send, range(args.parallel)))
|
||
ok = [a for a in answers if a[0] == 201]
|
||
instances.update(a[2] for a in ok)
|
||
rounds.append({'responses_201': len(ok), 'failures': len(answers) - len(ok),
|
||
'distinct_message_ids': len({a[1] for a in ok}),
|
||
'distinct_instances': len({a[2] for a in ok})})
|
||
messages = history(c, bob, chat)
|
||
texts = [m['text'] for m in messages]
|
||
created = [texts.count(m['text']) for m in messages if m['text'].startswith('idem-')]
|
||
# Тот же ключ с другим телом -> 409.
|
||
key = uuid.uuid4().hex
|
||
c.message(alice, chat, 'idem-first-' + key, key=key)
|
||
other_body = c.call('POST', path, 409, actor=alice, body={'text': 'idem-other-' + key},
|
||
headers={'Idempotency-Key': key})
|
||
report = {
|
||
'method': '%d rounds; each round %d simultaneous identical POSTs with one Idempotency-Key through the LB '
|
||
'(separate connections, released by a barrier)' % (args.rounds, args.parallel),
|
||
'rounds': args.rounds, 'parallel_requests_per_round': args.parallel,
|
||
'instances_seen': sorted(i for i in instances if i),
|
||
'all_rounds_single_message_id': all(r['distinct_message_ids'] == 1 for r in rounds),
|
||
'failed_requests': sum(r['failures'] for r in rounds),
|
||
'rounds_with_requests_on_2plus_replicas': sum(1 for r in rounds if r['distinct_instances'] >= 2),
|
||
'messages_created_per_key_max': max(created) if created else 0,
|
||
'same_key_other_body_status': other_body['status'],
|
||
'per_round': rounds,
|
||
}
|
||
write_evidence(Path(args.output), 'idempotency-parallel.json', report)
|
||
check(report['all_rounds_single_message_id'], 'a key produced several message ids')
|
||
check(report['messages_created_per_key_max'] == 1, 'a key created several messages')
|
||
check(report['failed_requests'] == 0, 'some parallel requests failed')
|
||
|
||
|
||
def run_rolling(args):
|
||
c, alice, bob, chat = setup(args)
|
||
old = api_containers()
|
||
n = len(old)
|
||
check(n >= 2, 'need at least 2 running API replicas')
|
||
old_image = {i: docker('inspect', '-f', '{{.Image}}', i) for i in old}
|
||
# Migrate запускается один раз, до замены; миграции только добавляют (expand), старые реплики их переживают.
|
||
# Alembic пишет журнал в stderr.
|
||
migrate_log = docker('compose', 'run', '--rm', 'migrate', timeout=300, with_stderr=True)
|
||
load = Load(args.base_url, args.ca_file, alice, chat, args.workers)
|
||
load.start()
|
||
time.sleep(args.warmup)
|
||
t_start = load.now()
|
||
env = dict(os.environ, APP_VERSION='rolling-' + time.strftime('%H%M%S'))
|
||
docker('compose', 'build', 'api', env=env, timeout=600)
|
||
docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=%d' % (2 * n), 'api',
|
||
env=env, timeout=300)
|
||
new = [i for i in api_containers() if i not in old]
|
||
check(len(new) == n, 'expected %d new replicas, found %d' % (n, len(new)))
|
||
wait_healthy(new)
|
||
wait_instances(c, alice, new)
|
||
steps = []
|
||
for victim in old:
|
||
t = load.now()
|
||
docker('stop', '-t', '15', victim)
|
||
docker('rm', victim)
|
||
steps.append({'replaced': victim, 'seconds_into_update': round(t - t_start, 2)})
|
||
time.sleep(args.step_pause)
|
||
docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=%d' % n, 'api', env=env, timeout=300)
|
||
t_done = load.now()
|
||
time.sleep(args.settle)
|
||
load.stop()
|
||
end = load.now()
|
||
stats = verify(c, bob, chat, load.acked)
|
||
seen_old = [e for e in load.events if e[3] and e[4] in old]
|
||
seen_new = [e for e in load.events if e[3] and e[4] in new]
|
||
report = {
|
||
'method': 'closed loop %d workers under rolling replacement: start N new replicas of the new build, '
|
||
'wait until healthy and routed by LB, then stop+remove old replicas one at a time' % args.workers,
|
||
'replicas_before': old, 'replicas_after': api_containers(), 'new_replicas': new,
|
||
'old_image_ids': old_image, 'new_image_ids': {i: docker('inspect', '-f', '{{.Image}}', i) for i in new},
|
||
'app_version_build_arg': env['APP_VERSION'],
|
||
'update_seconds': round(t_done - t_start, 2),
|
||
'steps': steps,
|
||
'before_update': window(load.events, 0, t_start),
|
||
'during_update': window(load.events, t_start, t_done),
|
||
'after_update': window(load.events, t_done, end),
|
||
# Все моменты — секунды от начала обновления, как и steps[].seconds_into_update.
|
||
'first_new_replica_response_seconds_into_update': round(min((e[1] for e in seen_new), default=t_start) - t_start, 2),
|
||
'last_old_replica_response_seconds_into_update': round(max((e[1] for e in seen_old), default=t_start) - t_start, 2),
|
||
'migrate_output_tail': migrate_log.splitlines()[-3:],
|
||
'timeline_per_second': timeline(load.events, end),
|
||
**stats,
|
||
}
|
||
write_evidence(Path(args.output), 'rolling-update.json', report)
|
||
print('rolling: errors during update=%d, lost=%d, dup=%d' % (
|
||
report['during_update']['failed_requests'], stats['lost_acked_messages'], stats['duplicate_messages']))
|
||
check(stats['lost_acked_messages'] == 0 and stats['duplicate_messages'] == 0, 'data integrity violated')
|
||
|
||
|
||
def main():
|
||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
p.add_argument('experiment', choices=['failover', 'idempotency', 'rolling'])
|
||
p.add_argument('--base-url', default=None)
|
||
p.add_argument('--ca-file', default=None)
|
||
p.add_argument('--workers', type=int, default=8, help='closed-loop workers (failover, rolling)')
|
||
p.add_argument('--warmup', type=float, default=5, help='seconds of load before the action')
|
||
p.add_argument('--after', type=float, default=15, help='failover: seconds the replica stays down under load')
|
||
p.add_argument('--settle', type=float, default=8, help='seconds of load after the action')
|
||
p.add_argument('--step-pause', type=float, default=3, help='rolling: pause between replacing replicas')
|
||
p.add_argument('--rounds', type=int, default=30, help='idempotency: number of keys')
|
||
p.add_argument('--parallel', type=int, default=16, help='idempotency: simultaneous requests per key')
|
||
p.add_argument('--output', default=str(ROOT / 'evidence'))
|
||
args = p.parse_args()
|
||
try:
|
||
{'failover': run_failover, 'idempotency': run_idempotency, 'rolling': run_rolling}[args.experiment](args)
|
||
except (Failure, RuntimeError, subprocess.TimeoutExpired) as exc:
|
||
print('FAIL:', exc)
|
||
raise SystemExit(1)
|
||
print('PASS:', args.experiment)
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|