diff --git a/scripts/chaos.py b/scripts/chaos.py new file mode 100644 index 00000000..11e4b2d1 --- /dev/null +++ b/scripts/chaos.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +"""Нагрузка записью через балансировщик во время отказа или замены реплик. + +stop — по очереди останавливает каждую реплику (SIGTERM), восстанавливая предыдущую; +kill — то же, но SIGKILL, без корректного завершения; +pause — то же, но реплика зависает (docker pause): соединения открыты, ответов нет; +rolling — запускает scripts/rolling-update.sh. + +Клиент ведёт себя как в контракте: при неопределённом ответе повторяет запрос с тем же ключом. +В конце вся история сверяется с подтверждёнными сообщениями. +""" +import argparse +import json +import math +import os +import subprocess +import sys +import threading +import time +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "student-tests")) +from api import call, history, new_chat, new_user, send + +WORKERS = 4 + + +def compose(*args): + command = ["docker", "compose", "--env-file", ".env", *args] + return subprocess.run(command, cwd=ROOT, check=True, capture_output=True, text=True).stdout + + +def api_containers(): + return compose("ps", "-q", "api").split() + + +def wait_ready(seconds=60): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + try: + if call("GET", "/health/ready", timeout=2)[0] == 200: + return + except OSError: + pass + time.sleep(0.2) + raise SystemExit("стенд не стал готов") + + +def wait_replicas(count): + """Ждёт, пока балансировщик начнёт отдавать запросы на count разных реплик.""" + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + seen = {call("GET", "/health/live", timeout=2)[2]["X-Instance-Id"] for _ in range(count * 4)} + if len(seen) >= count: + return + time.sleep(0.2) + raise SystemExit("балансировщик не видит %d реплик" % count) + + +class Load: + """Несколько клиентов пишут в один чат, каждое событие сохраняется с отметкой времени.""" + + def __init__(self): + self.user = new_user("chaos") + self.chat = new_chat(self.user) + self.stop = threading.Event() + self.events = [] # (время, ok, реплика, статус) + self.confirmed = {} + self.lock = threading.Lock() + self.threads = [threading.Thread(target=self.worker, args=(n,)) for n in range(WORKERS)] + + def worker(self, number): + index = 0 + while not self.stop.is_set(): + text = "worker-%d-%d-%s" % (number, index, uuid.uuid4().hex[:8]) + key = uuid.uuid4().hex + index += 1 + while True: + try: + status, body, headers = send(self.user, self.chat, text, key=key, timeout=3) + instance = headers.get("X-Instance-Id") + except OSError: + status, body, instance = 0, None, None + now = time.monotonic() + ok = status == 201 + with self.lock: + self.events.append((now, ok, instance, status)) + if ok: + self.confirmed[body["id"]] = body + if ok or self.stop.is_set(): + break + time.sleep(0.05) # неопределённый результат: тот же текст и тот же ключ + + def start(self): + for thread in self.threads: + thread.start() + + def finish(self): + self.stop.set() + for thread in self.threads: + thread.join() + + +def percentile(values, p): + ordered = sorted(values) + return round(ordered[max(0, math.ceil(p * len(ordered)) - 1)], 3) if ordered else None + + +def summary(events, started, action_at, target=None): + """Показатели одного интервала: от action_at считается время до восстановления записи.""" + ok = [e for e in events if e[1]] + failed = [e for e in events if not e[1]] + elapsed = events[-1][0] - started + gaps = [b[0] - a[0] for a, b in zip(ok, ok[1:])] + after = [e for e in ok if e[0] >= action_at] + last_on_target = [e[0] for e in ok if target and e[2] and target.startswith(e[2])] + return { + "seconds": round(elapsed, 2), + "successful_requests": len(ok), + "failed_attempts": len(failed), + "failed_statuses": sorted({e[3] for e in failed}), + "successful_rps": round(len(ok) / elapsed, 1), + "longest_gap_between_successes_s": round(max(gaps), 3) if gaps else None, + "first_success_after_action_s": round(after[0][0] - action_at, 3) if after else None, + "last_success_on_target_after_action_s": round(last_on_target[-1] - action_at, 3) + if last_on_target and last_on_target[-1] >= action_at else None, + "instances_seen": len({e[2] for e in ok}), + } + + +def verify(load): + items, _ = history(load.user, load.chat, 100) + by_id = {item["id"]: item for item in items} + texts = [item["text"] for item in items] + lost = [mid for mid, message in load.confirmed.items() if by_id.get(mid) != message] + result = { + "confirmed": len(load.confirmed), + "in_history": len(items), + "lost_confirmed": len(lost), + "duplicate_ids": len(items) - len(by_id), + "duplicate_texts": len(texts) - len(set(texts)), + } + result["unconfirmed_but_saved"] = len(items) - len(load.confirmed) + return result + + +def replicas_up(): + """Сколько реплик балансировщик считает рабочими, по его странице статистики.""" + csv = compose("exec", "-T", "lb", "wget", "-q", "-O", "-", "http://127.0.0.1:8404/stats;csv") + return sum(line.startswith("api,api") and ",UP" in line for line in csv.splitlines()) + + +def seconds_until_excluded(replicas, since, limit=8): + while time.monotonic() - since < limit: + if replicas_up() < replicas: + return round(time.monotonic() - since, 2) + time.sleep(0.1) + return None + + +def replica_round(signal): + containers = api_containers() + wait_replicas(len(containers)) + rounds = [] + for container in containers: + load = Load() + started = time.monotonic() + load.start() + time.sleep(5) + action_at = time.monotonic() + command = {"SIGTERM": "stop", "SIGKILL": "kill", "HANG": "pause"}[signal] + subprocess.run(["docker", command, container], check=True, stdout=subprocess.DEVNULL) + stop_took = time.monotonic() - action_at + excluded_after = seconds_until_excluded(len(containers), action_at) + time.sleep(max(0, 10 - (time.monotonic() - action_at))) + command = "unpause" if signal == "HANG" else "start" + subprocess.run(["docker", command, container], check=True, stdout=subprocess.DEVNULL) + wait_replicas(len(containers)) + time.sleep(2) + load.finish() + result = summary(load.events, started, action_at, container) + result.update({"target": container[:12], "signal": signal, "stop_command_s": round(stop_took, 2), + "excluded_by_lb_after_s": excluded_after}) + result["verify"] = verify(load) + print(json.dumps(result, ensure_ascii=False)) + rounds.append(result) + return rounds + + +def images(): + inspect = ["docker", "inspect", "-f", "{{.Image}}"] + return {c[:12]: subprocess.check_output(inspect + [c], text=True).strip()[:19] for c in api_containers()} + + +def rolling_round(): + before = images() + wait_replicas(len(before)) + load = Load() + started = time.monotonic() + load.start() + time.sleep(5) + action_at = time.monotonic() + subprocess.run(["./scripts/rolling-update.sh"], cwd=ROOT, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + update_took = time.monotonic() - action_at + time.sleep(5) + load.finish() + after = images() + result = summary(load.events, started, action_at) + result.update({"update_seconds": round(update_took, 2), "containers_before": before, "containers_after": after, + "app_version": os.environ.get("APP_VERSION", "dev")}) + result["verify"] = verify(load) + print(json.dumps(result, ensure_ascii=False)) + # без этой проверки скрипт, который ничего не заменил, выглядел бы как идеальное обновление + if set(before) & set(after) or set(before.values()) & set(after.values()) or len(after) != len(before): + raise SystemExit("FAIL: реплики не заменены новым образом") + return [result] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("mode", choices=["stop", "kill", "pause", "rolling"]) + parser.add_argument("--output") + args = parser.parse_args() + wait_ready() + if args.mode == "rolling": + rounds = rolling_round() + else: + rounds = replica_round({"stop": "SIGTERM", "kill": "SIGKILL", "pause": "HANG"}[args.mode]) + report = { + "workload": "%d клиента, closed loop, POST сообщений через балансировщик, повтор с тем же ключом" % WORKERS, + "mode": args.mode, + "rounds": rounds, + } + if args.output: + Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n") + bad = [r for r in rounds if r["verify"]["lost_confirmed"] or r["verify"]["duplicate_ids"] or r["verify"]["duplicate_texts"]] + if bad: + raise SystemExit("FAIL: потеряны или продублированы сообщения") + print("PASS: подтверждённые сообщения на месте, дублей нет") + + +if __name__ == "__main__": + main() diff --git a/scripts/explain_history.sh b/scripts/explain_history.sh new file mode 100755 index 00000000..371e03ee --- /dev/null +++ b/scripts/explain_history.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# Наполняет БД синтетической историей и печатает планы запросов пагинации. +set -eu +cd "$(dirname "$0")/.." + +run_sql() { + docker compose --env-file .env exec -T postgres psql -U messenger -d messenger -v ON_ERROR_STOP=1 -q +} + +run_sql < scripts/seed_history.sql +run_sql < scripts/explain_history.sql diff --git a/scripts/explain_history.sql b/scripts/explain_history.sql new file mode 100644 index 00000000..88d64441 --- /dev/null +++ b/scripts/explain_history.sql @@ -0,0 +1,44 @@ +\set chat '00000000-0000-4000-8000-000000000100' +\pset pager off + +\echo '== Объём данных' +SELECT count(*) AS messages_total, + count(*) FILTER (WHERE chat_id = :'chat') AS messages_in_chat, + pg_size_pretty(pg_relation_size('messages')) AS table_size, + pg_size_pretty(pg_relation_size('messages_chat_history_idx')) AS index_size +FROM messages; + +\echo +\echo '== 1. Первая страница (limit 50, запрос берёт 51 строку)' +EXPLAIN (ANALYZE, BUFFERS) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'chat' +ORDER BY created_at, id LIMIT 51; + +-- cursor на 5000-е сообщение чата, как его получил бы клиент после 100 страниц +SELECT created_at AS after_time, id AS after_id FROM messages +WHERE chat_id = :'chat' ORDER BY created_at, id OFFSET 4999 LIMIT 1 \gset + +\echo +\echo '== 2. Страница из середины истории: keyset после 5000-го сообщения' +EXPLAIN (ANALYZE, BUFFERS) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'chat' AND (created_at, id) > (:'after_time', :'after_id') +ORDER BY created_at, id LIMIT 51; + +\echo +\echo '== 3. Та же страница через OFFSET (так делать не стали)' +EXPLAIN (ANALYZE, BUFFERS) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'chat' +ORDER BY created_at, id OFFSET 5000 LIMIT 51; + +\echo +\echo '== 4. Запрос из пункта 2 без индекса messages_chat_history_idx (DROP INDEX внутри транзакции с ROLLBACK)' +BEGIN; +DROP INDEX messages_chat_history_idx; +EXPLAIN (ANALYZE, BUFFERS) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'chat' AND (created_at, id) > (:'after_time', :'after_id') +ORDER BY created_at, id LIMIT 51; +ROLLBACK; diff --git a/scripts/seed_history.sql b/scripts/seed_history.sql new file mode 100644 index 00000000..38bf2eb3 --- /dev/null +++ b/scripts/seed_history.sql @@ -0,0 +1,26 @@ +-- Синтетические данные для EXPLAIN: 21 чат и 60 000 сообщений, из них 10 000 в исследуемом чате. +-- Повторный запуск ничего не добавляет. Войти под этим пользователем нельзя: вместо hash стоит заглушка. + +INSERT INTO users (id, username, display_name, password_hash) +VALUES ('00000000-0000-4000-8000-000000000001', 'explain_seed', 'Данные для EXPLAIN', '!') +ON CONFLICT DO NOTHING; + +INSERT INTO chats (id, title) +SELECT ('00000000-0000-4000-8000-0000000001' || lpad(n::text, 2, '0'))::uuid, 'EXPLAIN чат ' || n +FROM generate_series(0, 20) AS n +ON CONFLICT DO NOTHING; + +INSERT INTO chat_members (chat_id, user_id, position) +SELECT id, '00000000-0000-4000-8000-000000000001', 0 FROM chats WHERE title LIKE 'EXPLAIN чат %' +ON CONFLICT DO NOTHING; + +-- чат 00: 10 000 сообщений; чаты 01..20: по 2 500. Время идёт с шагом в секунду и перемешано между чатами. +INSERT INTO messages (chat_id, sender_id, text, created_at) +SELECT ('00000000-0000-4000-8000-0000000001' || lpad((CASE WHEN n % 6 = 0 THEN 0 ELSE 1 + n % 20 END)::text, 2, '0'))::uuid, + '00000000-0000-4000-8000-000000000001', + 'сообщение ' || n, + now() - make_interval(secs => 60000 - n) +FROM generate_series(1, 60000) AS n +WHERE NOT EXISTS (SELECT 1 FROM messages WHERE chat_id = '00000000-0000-4000-8000-000000000100'); + +ANALYZE messages; diff --git a/scripts/sigterm_check.py b/scripts/sigterm_check.py new file mode 100644 index 00000000..388410a3 --- /dev/null +++ b/scripts/sigterm_check.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Проверка SIGTERM через балансировщик: начатые запросы завершаются, новые уходят на другую реплику, +процесс выходит быстрее 10 секунд. API наружу не опубликован, поэтому незаконченные запросы открываются +на все реплики сразу: часть из них обязательно окажется на останавливаемой.""" +import json +import socket +import subprocess +import sys +import time +import urllib.parse +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "student-tests")) +from api import BASE_URL, call, new_chat, new_user + +LIMIT_SECONDS = 10 +OPEN_REQUESTS = 6 + + +def main(): + containers = subprocess.run( + ["docker", "compose", "--env-file", ".env", "ps", "-q", "api"], + cwd=ROOT, check=True, capture_output=True, text=True, + ).stdout.split() + target = containers[0] + address = urllib.parse.urlsplit(BASE_URL) + user = new_user("sigterm") + chat = new_chat(user) + + # заголовки и половина тела: запрос уже начат, но ещё не закончен + body = json.dumps({"text": "сообщение, отправленное во время остановки"}).encode() + connections = [] + for _ in range(OPEN_REQUESTS): + head = ( + "POST /api/v1/chats/%s/messages HTTP/1.1\r\nHost: %s\r\nContent-Type: application/json\r\n" + "Content-Length: %d\r\nAuthorization: Bearer %s\r\nIdempotency-Key: %s\r\n\r\n" + % (chat["id"], address.netloc, len(body), user["token"], uuid.uuid4().hex) + ).encode() + connection = socket.create_connection((address.hostname, address.port), timeout=LIMIT_SECONDS) + connection.sendall(head + body[:10]) + connections.append(connection) + time.sleep(0.5) + + print("$ docker kill --signal=SIGTERM", target[:12]) + started = time.monotonic() + subprocess.run(["docker", "kill", "--signal=SIGTERM", target], check=True, stdout=subprocess.DEVNULL) + time.sleep(1) + + served = [call("GET", "/health/live")[2]["X-Instance-Id"] for _ in range(20)] + new_on_target = sum(target.startswith(instance) for instance in served) + print("новых запросов после SIGTERM: 20, попало на останавливаемую реплику:", new_on_target) + + finished_on_target = 0 + statuses = [] + for connection in connections: + connection.sendall(body[10:]) + answer = connection.recv(65536).decode("utf-8", "replace") + connection.close() + statuses.append(answer.split("\r\n")[0]) + for line in answer.split("\r\n"): + if line.lower().startswith("x-instance-id:") and target.startswith(line.split(":")[1].strip()): + finished_on_target += 1 + print("начатые до SIGTERM запросы:", sorted(set(statuses))) + print("из них завершила останавливаемая реплика:", finished_on_target) + + exit_code = subprocess.run(["docker", "wait", target], check=True, capture_output=True, text=True).stdout.strip() + elapsed = time.monotonic() - started + print("процесс завершился за %.2f с, код выхода %s" % (elapsed, exit_code)) + + # контейнер, остановленный через docker kill, сам не перезапускается: поднимаем его обратно + subprocess.run(["docker", "start", target], check=True, stdout=subprocess.DEVNULL) + + good = all(" 201 " in status for status in statuses) + if not good or new_on_target or not finished_on_target or elapsed > LIMIT_SECONDS or exit_code != "0": + raise SystemExit("FAIL") + print("PASS: остановка корректная и укладывается в %d секунд" % LIMIT_SECONDS) + + +if __name__ == "__main__": + main()