forked from mtusi2026/mtusi2026lab3
Добавить нагрузку при отказах и проверку SIGTERM через балансировщик
This commit is contained in:
@@ -0,0 +1,245 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Нагрузка записью через балансировщик во время отказа или замены реплик.
|
||||
|
||||
stop — по очереди останавливает каждую реплику (SIGTERM), восстанавливая предыдущую;
|
||||
kill — то же, но SIGKILL, без корректного завершения;
|
||||
pause — то же, но реплика зависает (docker pause): соединения открыты, ответов нет;
|
||||
rolling — запускает scripts/rolling-update.sh.
|
||||
|
||||
Клиент ведёт себя как в контракте: при неопределённом ответе повторяет запрос с тем же ключом.
|
||||
В конце вся история сверяется с подтверждёнными сообщениями.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||||
from api import call, history, new_chat, new_user, send
|
||||
|
||||
WORKERS = 4
|
||||
|
||||
|
||||
def compose(*args):
|
||||
command = ["docker", "compose", "--env-file", ".env", *args]
|
||||
return subprocess.run(command, cwd=ROOT, check=True, capture_output=True, text=True).stdout
|
||||
|
||||
|
||||
def api_containers():
|
||||
return compose("ps", "-q", "api").split()
|
||||
|
||||
|
||||
def wait_ready(seconds=60):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if call("GET", "/health/ready", timeout=2)[0] == 200:
|
||||
return
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(0.2)
|
||||
raise SystemExit("стенд не стал готов")
|
||||
|
||||
|
||||
def wait_replicas(count):
|
||||
"""Ждёт, пока балансировщик начнёт отдавать запросы на count разных реплик."""
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline:
|
||||
seen = {call("GET", "/health/live", timeout=2)[2]["X-Instance-Id"] for _ in range(count * 4)}
|
||||
if len(seen) >= count:
|
||||
return
|
||||
time.sleep(0.2)
|
||||
raise SystemExit("балансировщик не видит %d реплик" % count)
|
||||
|
||||
|
||||
class Load:
|
||||
"""Несколько клиентов пишут в один чат, каждое событие сохраняется с отметкой времени."""
|
||||
|
||||
def __init__(self):
|
||||
self.user = new_user("chaos")
|
||||
self.chat = new_chat(self.user)
|
||||
self.stop = threading.Event()
|
||||
self.events = [] # (время, ok, реплика, статус)
|
||||
self.confirmed = {}
|
||||
self.lock = threading.Lock()
|
||||
self.threads = [threading.Thread(target=self.worker, args=(n,)) for n in range(WORKERS)]
|
||||
|
||||
def worker(self, number):
|
||||
index = 0
|
||||
while not self.stop.is_set():
|
||||
text = "worker-%d-%d-%s" % (number, index, uuid.uuid4().hex[:8])
|
||||
key = uuid.uuid4().hex
|
||||
index += 1
|
||||
while True:
|
||||
try:
|
||||
status, body, headers = send(self.user, self.chat, text, key=key, timeout=3)
|
||||
instance = headers.get("X-Instance-Id")
|
||||
except OSError:
|
||||
status, body, instance = 0, None, None
|
||||
now = time.monotonic()
|
||||
ok = status == 201
|
||||
with self.lock:
|
||||
self.events.append((now, ok, instance, status))
|
||||
if ok:
|
||||
self.confirmed[body["id"]] = body
|
||||
if ok or self.stop.is_set():
|
||||
break
|
||||
time.sleep(0.05) # неопределённый результат: тот же текст и тот же ключ
|
||||
|
||||
def start(self):
|
||||
for thread in self.threads:
|
||||
thread.start()
|
||||
|
||||
def finish(self):
|
||||
self.stop.set()
|
||||
for thread in self.threads:
|
||||
thread.join()
|
||||
|
||||
|
||||
def percentile(values, p):
|
||||
ordered = sorted(values)
|
||||
return round(ordered[max(0, math.ceil(p * len(ordered)) - 1)], 3) if ordered else None
|
||||
|
||||
|
||||
def summary(events, started, action_at, target=None):
|
||||
"""Показатели одного интервала: от action_at считается время до восстановления записи."""
|
||||
ok = [e for e in events if e[1]]
|
||||
failed = [e for e in events if not e[1]]
|
||||
elapsed = events[-1][0] - started
|
||||
gaps = [b[0] - a[0] for a, b in zip(ok, ok[1:])]
|
||||
after = [e for e in ok if e[0] >= action_at]
|
||||
last_on_target = [e[0] for e in ok if target and e[2] and target.startswith(e[2])]
|
||||
return {
|
||||
"seconds": round(elapsed, 2),
|
||||
"successful_requests": len(ok),
|
||||
"failed_attempts": len(failed),
|
||||
"failed_statuses": sorted({e[3] for e in failed}),
|
||||
"successful_rps": round(len(ok) / elapsed, 1),
|
||||
"longest_gap_between_successes_s": round(max(gaps), 3) if gaps else None,
|
||||
"first_success_after_action_s": round(after[0][0] - action_at, 3) if after else None,
|
||||
"last_success_on_target_after_action_s": round(last_on_target[-1] - action_at, 3)
|
||||
if last_on_target and last_on_target[-1] >= action_at else None,
|
||||
"instances_seen": len({e[2] for e in ok}),
|
||||
}
|
||||
|
||||
|
||||
def verify(load):
|
||||
items, _ = history(load.user, load.chat, 100)
|
||||
by_id = {item["id"]: item for item in items}
|
||||
texts = [item["text"] for item in items]
|
||||
lost = [mid for mid, message in load.confirmed.items() if by_id.get(mid) != message]
|
||||
result = {
|
||||
"confirmed": len(load.confirmed),
|
||||
"in_history": len(items),
|
||||
"lost_confirmed": len(lost),
|
||||
"duplicate_ids": len(items) - len(by_id),
|
||||
"duplicate_texts": len(texts) - len(set(texts)),
|
||||
}
|
||||
result["unconfirmed_but_saved"] = len(items) - len(load.confirmed)
|
||||
return result
|
||||
|
||||
|
||||
def replicas_up():
|
||||
"""Сколько реплик балансировщик считает рабочими, по его странице статистики."""
|
||||
csv = compose("exec", "-T", "lb", "wget", "-q", "-O", "-", "http://127.0.0.1:8404/stats;csv")
|
||||
return sum(line.startswith("api,api") and ",UP" in line for line in csv.splitlines())
|
||||
|
||||
|
||||
def seconds_until_excluded(replicas, since, limit=8):
|
||||
while time.monotonic() - since < limit:
|
||||
if replicas_up() < replicas:
|
||||
return round(time.monotonic() - since, 2)
|
||||
time.sleep(0.1)
|
||||
return None
|
||||
|
||||
|
||||
def replica_round(signal):
|
||||
containers = api_containers()
|
||||
wait_replicas(len(containers))
|
||||
rounds = []
|
||||
for container in containers:
|
||||
load = Load()
|
||||
started = time.monotonic()
|
||||
load.start()
|
||||
time.sleep(5)
|
||||
action_at = time.monotonic()
|
||||
command = {"SIGTERM": "stop", "SIGKILL": "kill", "HANG": "pause"}[signal]
|
||||
subprocess.run(["docker", command, container], check=True, stdout=subprocess.DEVNULL)
|
||||
stop_took = time.monotonic() - action_at
|
||||
excluded_after = seconds_until_excluded(len(containers), action_at)
|
||||
time.sleep(max(0, 10 - (time.monotonic() - action_at)))
|
||||
command = "unpause" if signal == "HANG" else "start"
|
||||
subprocess.run(["docker", command, container], check=True, stdout=subprocess.DEVNULL)
|
||||
wait_replicas(len(containers))
|
||||
time.sleep(2)
|
||||
load.finish()
|
||||
result = summary(load.events, started, action_at, container)
|
||||
result.update({"target": container[:12], "signal": signal, "stop_command_s": round(stop_took, 2),
|
||||
"excluded_by_lb_after_s": excluded_after})
|
||||
result["verify"] = verify(load)
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
rounds.append(result)
|
||||
return rounds
|
||||
|
||||
|
||||
def images():
|
||||
inspect = ["docker", "inspect", "-f", "{{.Image}}"]
|
||||
return {c[:12]: subprocess.check_output(inspect + [c], text=True).strip()[:19] for c in api_containers()}
|
||||
|
||||
|
||||
def rolling_round():
|
||||
before = images()
|
||||
wait_replicas(len(before))
|
||||
load = Load()
|
||||
started = time.monotonic()
|
||||
load.start()
|
||||
time.sleep(5)
|
||||
action_at = time.monotonic()
|
||||
subprocess.run(["./scripts/rolling-update.sh"], cwd=ROOT, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
update_took = time.monotonic() - action_at
|
||||
time.sleep(5)
|
||||
load.finish()
|
||||
after = images()
|
||||
result = summary(load.events, started, action_at)
|
||||
result.update({"update_seconds": round(update_took, 2), "containers_before": before, "containers_after": after,
|
||||
"app_version": os.environ.get("APP_VERSION", "dev")})
|
||||
result["verify"] = verify(load)
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
# без этой проверки скрипт, который ничего не заменил, выглядел бы как идеальное обновление
|
||||
if set(before) & set(after) or set(before.values()) & set(after.values()) or len(after) != len(before):
|
||||
raise SystemExit("FAIL: реплики не заменены новым образом")
|
||||
return [result]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("mode", choices=["stop", "kill", "pause", "rolling"])
|
||||
parser.add_argument("--output")
|
||||
args = parser.parse_args()
|
||||
wait_ready()
|
||||
if args.mode == "rolling":
|
||||
rounds = rolling_round()
|
||||
else:
|
||||
rounds = replica_round({"stop": "SIGTERM", "kill": "SIGKILL", "pause": "HANG"}[args.mode])
|
||||
report = {
|
||||
"workload": "%d клиента, closed loop, POST сообщений через балансировщик, повтор с тем же ключом" % WORKERS,
|
||||
"mode": args.mode,
|
||||
"rounds": rounds,
|
||||
}
|
||||
if args.output:
|
||||
Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
||||
bad = [r for r in rounds if r["verify"]["lost_confirmed"] or r["verify"]["duplicate_ids"] or r["verify"]["duplicate_texts"]]
|
||||
if bad:
|
||||
raise SystemExit("FAIL: потеряны или продублированы сообщения")
|
||||
print("PASS: подтверждённые сообщения на месте, дублей нет")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
# Наполняет БД синтетической историей и печатает планы запросов пагинации.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
run_sql() {
|
||||
docker compose --env-file .env exec -T postgres psql -U messenger -d messenger -v ON_ERROR_STOP=1 -q
|
||||
}
|
||||
|
||||
run_sql < scripts/seed_history.sql
|
||||
run_sql < scripts/explain_history.sql
|
||||
@@ -0,0 +1,44 @@
|
||||
\set chat '00000000-0000-4000-8000-000000000100'
|
||||
\pset pager off
|
||||
|
||||
\echo '== Объём данных'
|
||||
SELECT count(*) AS messages_total,
|
||||
count(*) FILTER (WHERE chat_id = :'chat') AS messages_in_chat,
|
||||
pg_size_pretty(pg_relation_size('messages')) AS table_size,
|
||||
pg_size_pretty(pg_relation_size('messages_chat_history_idx')) AS index_size
|
||||
FROM messages;
|
||||
|
||||
\echo
|
||||
\echo '== 1. Первая страница (limit 50, запрос берёт 51 строку)'
|
||||
EXPLAIN (ANALYZE, BUFFERS)
|
||||
SELECT id, chat_id, sender_id, text, created_at FROM messages
|
||||
WHERE chat_id = :'chat'
|
||||
ORDER BY created_at, id LIMIT 51;
|
||||
|
||||
-- cursor на 5000-е сообщение чата, как его получил бы клиент после 100 страниц
|
||||
SELECT created_at AS after_time, id AS after_id FROM messages
|
||||
WHERE chat_id = :'chat' ORDER BY created_at, id OFFSET 4999 LIMIT 1 \gset
|
||||
|
||||
\echo
|
||||
\echo '== 2. Страница из середины истории: keyset после 5000-го сообщения'
|
||||
EXPLAIN (ANALYZE, BUFFERS)
|
||||
SELECT id, chat_id, sender_id, text, created_at FROM messages
|
||||
WHERE chat_id = :'chat' AND (created_at, id) > (:'after_time', :'after_id')
|
||||
ORDER BY created_at, id LIMIT 51;
|
||||
|
||||
\echo
|
||||
\echo '== 3. Та же страница через OFFSET (так делать не стали)'
|
||||
EXPLAIN (ANALYZE, BUFFERS)
|
||||
SELECT id, chat_id, sender_id, text, created_at FROM messages
|
||||
WHERE chat_id = :'chat'
|
||||
ORDER BY created_at, id OFFSET 5000 LIMIT 51;
|
||||
|
||||
\echo
|
||||
\echo '== 4. Запрос из пункта 2 без индекса messages_chat_history_idx (DROP INDEX внутри транзакции с ROLLBACK)'
|
||||
BEGIN;
|
||||
DROP INDEX messages_chat_history_idx;
|
||||
EXPLAIN (ANALYZE, BUFFERS)
|
||||
SELECT id, chat_id, sender_id, text, created_at FROM messages
|
||||
WHERE chat_id = :'chat' AND (created_at, id) > (:'after_time', :'after_id')
|
||||
ORDER BY created_at, id LIMIT 51;
|
||||
ROLLBACK;
|
||||
@@ -0,0 +1,26 @@
|
||||
-- Синтетические данные для EXPLAIN: 21 чат и 60 000 сообщений, из них 10 000 в исследуемом чате.
|
||||
-- Повторный запуск ничего не добавляет. Войти под этим пользователем нельзя: вместо hash стоит заглушка.
|
||||
|
||||
INSERT INTO users (id, username, display_name, password_hash)
|
||||
VALUES ('00000000-0000-4000-8000-000000000001', 'explain_seed', 'Данные для EXPLAIN', '!')
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO chats (id, title)
|
||||
SELECT ('00000000-0000-4000-8000-0000000001' || lpad(n::text, 2, '0'))::uuid, 'EXPLAIN чат ' || n
|
||||
FROM generate_series(0, 20) AS n
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO chat_members (chat_id, user_id, position)
|
||||
SELECT id, '00000000-0000-4000-8000-000000000001', 0 FROM chats WHERE title LIKE 'EXPLAIN чат %'
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
-- чат 00: 10 000 сообщений; чаты 01..20: по 2 500. Время идёт с шагом в секунду и перемешано между чатами.
|
||||
INSERT INTO messages (chat_id, sender_id, text, created_at)
|
||||
SELECT ('00000000-0000-4000-8000-0000000001' || lpad((CASE WHEN n % 6 = 0 THEN 0 ELSE 1 + n % 20 END)::text, 2, '0'))::uuid,
|
||||
'00000000-0000-4000-8000-000000000001',
|
||||
'сообщение ' || n,
|
||||
now() - make_interval(secs => 60000 - n)
|
||||
FROM generate_series(1, 60000) AS n
|
||||
WHERE NOT EXISTS (SELECT 1 FROM messages WHERE chat_id = '00000000-0000-4000-8000-000000000100');
|
||||
|
||||
ANALYZE messages;
|
||||
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Проверка SIGTERM через балансировщик: начатые запросы завершаются, новые уходят на другую реплику,
|
||||
процесс выходит быстрее 10 секунд. API наружу не опубликован, поэтому незаконченные запросы открываются
|
||||
на все реплики сразу: часть из них обязательно окажется на останавливаемой."""
|
||||
import json
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||||
from api import BASE_URL, call, new_chat, new_user
|
||||
|
||||
LIMIT_SECONDS = 10
|
||||
OPEN_REQUESTS = 6
|
||||
|
||||
|
||||
def main():
|
||||
containers = subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", "ps", "-q", "api"],
|
||||
cwd=ROOT, check=True, capture_output=True, text=True,
|
||||
).stdout.split()
|
||||
target = containers[0]
|
||||
address = urllib.parse.urlsplit(BASE_URL)
|
||||
user = new_user("sigterm")
|
||||
chat = new_chat(user)
|
||||
|
||||
# заголовки и половина тела: запрос уже начат, но ещё не закончен
|
||||
body = json.dumps({"text": "сообщение, отправленное во время остановки"}).encode()
|
||||
connections = []
|
||||
for _ in range(OPEN_REQUESTS):
|
||||
head = (
|
||||
"POST /api/v1/chats/%s/messages HTTP/1.1\r\nHost: %s\r\nContent-Type: application/json\r\n"
|
||||
"Content-Length: %d\r\nAuthorization: Bearer %s\r\nIdempotency-Key: %s\r\n\r\n"
|
||||
% (chat["id"], address.netloc, len(body), user["token"], uuid.uuid4().hex)
|
||||
).encode()
|
||||
connection = socket.create_connection((address.hostname, address.port), timeout=LIMIT_SECONDS)
|
||||
connection.sendall(head + body[:10])
|
||||
connections.append(connection)
|
||||
time.sleep(0.5)
|
||||
|
||||
print("$ docker kill --signal=SIGTERM", target[:12])
|
||||
started = time.monotonic()
|
||||
subprocess.run(["docker", "kill", "--signal=SIGTERM", target], check=True, stdout=subprocess.DEVNULL)
|
||||
time.sleep(1)
|
||||
|
||||
served = [call("GET", "/health/live")[2]["X-Instance-Id"] for _ in range(20)]
|
||||
new_on_target = sum(target.startswith(instance) for instance in served)
|
||||
print("новых запросов после SIGTERM: 20, попало на останавливаемую реплику:", new_on_target)
|
||||
|
||||
finished_on_target = 0
|
||||
statuses = []
|
||||
for connection in connections:
|
||||
connection.sendall(body[10:])
|
||||
answer = connection.recv(65536).decode("utf-8", "replace")
|
||||
connection.close()
|
||||
statuses.append(answer.split("\r\n")[0])
|
||||
for line in answer.split("\r\n"):
|
||||
if line.lower().startswith("x-instance-id:") and target.startswith(line.split(":")[1].strip()):
|
||||
finished_on_target += 1
|
||||
print("начатые до SIGTERM запросы:", sorted(set(statuses)))
|
||||
print("из них завершила останавливаемая реплика:", finished_on_target)
|
||||
|
||||
exit_code = subprocess.run(["docker", "wait", target], check=True, capture_output=True, text=True).stdout.strip()
|
||||
elapsed = time.monotonic() - started
|
||||
print("процесс завершился за %.2f с, код выхода %s" % (elapsed, exit_code))
|
||||
|
||||
# контейнер, остановленный через docker kill, сам не перезапускается: поднимаем его обратно
|
||||
subprocess.run(["docker", "start", target], check=True, stdout=subprocess.DEVNULL)
|
||||
|
||||
good = all(" 201 " in status for status in statuses)
|
||||
if not good or new_on_target or not finished_on_target or elapsed > LIMIT_SECONDS or exit_code != "0":
|
||||
raise SystemExit("FAIL")
|
||||
print("PASS: остановка корректная и укладывается в %d секунд" % LIMIT_SECONDS)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user