forked from mtusi2026/mtusi2026lab3
481 lines
18 KiB
Python
481 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Нагрузка через LB во время отказа/замены API-реплик; метрики и проверка данных.
|
|
|
|
failover — для каждой API-реплики по очереди: нагрузка → остановка этой реплики
|
|
(docker stop = SIGTERM, затем docker kill = SIGKILL) → восстановление
|
|
(scripts/restore.sh) → нагрузка после восстановления.
|
|
rolling — нагрузка во время scripts/rolling-update.sh (поочерёдная замена реплик).
|
|
|
|
Клиент ведёт себя как настоящий: неподтверждённый POST повторяет с тем же
|
|
Idempotency-Key и телом. После прогона вся история читается заново: каждое
|
|
подтверждённое сообщение на месте и не изменено, дублей нет.
|
|
Python 3.9+, стандартная библиотека; tokens не печатаются и не сохраняются.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import math
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from urllib.parse import quote
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
|
|
from client import Client # noqa: E402
|
|
|
|
REQUEST_TIMEOUT_SEC = 5
|
|
RETRY_PAUSE_SEC = 0.05
|
|
STATS_FORMAT = "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"
|
|
|
|
|
|
@dataclass
|
|
class Attempt:
|
|
"""Одна HTTP-попытка клиента; время — секунды от начала прогона"""
|
|
|
|
op: str
|
|
started: float
|
|
finished: float
|
|
status: int | None
|
|
error: str | None
|
|
instance: str | None
|
|
retry: bool
|
|
|
|
@property
|
|
def ok(self) -> bool:
|
|
return self.status in (200, 201)
|
|
|
|
|
|
class Recorder:
|
|
def __init__(self) -> None:
|
|
self.started = time.monotonic()
|
|
self.lock = threading.Lock()
|
|
self.attempts: list[Attempt] = []
|
|
self.confirmed: list[dict] = []
|
|
self.events: dict[str, float] = {}
|
|
|
|
def now(self) -> float:
|
|
return time.monotonic() - self.started
|
|
|
|
def mark(self, name: str) -> None:
|
|
self.events[name] = round(self.now(), 3)
|
|
|
|
def add(self, attempt: Attempt, message: dict | None = None) -> None:
|
|
with self.lock:
|
|
self.attempts.append(attempt)
|
|
if message is not None:
|
|
self.confirmed.append(message)
|
|
|
|
|
|
def request(
|
|
base: str, method: str, path: str, token: str, body: dict | None, key: str | None
|
|
) -> tuple[int | None, str | None, str | None, dict | None]:
|
|
headers = {"Authorization": "Bearer " + token, "Accept": "application/json"}
|
|
data = None
|
|
if body is not None:
|
|
data = json.dumps(body, ensure_ascii=False).encode()
|
|
headers["Content-Type"] = "application/json"
|
|
if key is not None:
|
|
headers["Idempotency-Key"] = key
|
|
req = urllib.request.Request(base + path, data=data, headers=headers, method=method)
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT_SEC) as response:
|
|
payload = json.loads(response.read())
|
|
return response.status, None, response.headers.get("X-Instance-Id"), payload
|
|
except urllib.error.HTTPError as error:
|
|
error.read()
|
|
return error.code, None, error.headers.get("X-Instance-Id"), None
|
|
except OSError as error:
|
|
return None, type(error).__name__, None, None
|
|
|
|
|
|
def worker(
|
|
base: str,
|
|
token: str,
|
|
chat_id: str,
|
|
worker_id: int,
|
|
recorder: Recorder,
|
|
stop: threading.Event,
|
|
) -> None:
|
|
"""Запись с повтором по тому же ключу до подтверждения, затем чтение"""
|
|
path = f"/api/v1/chats/{chat_id}/messages"
|
|
index = 0
|
|
while not stop.is_set():
|
|
key = uuid.uuid4().hex
|
|
body = {"text": f"exp-{worker_id}-{index}-{key[:8]}"}
|
|
retry = False
|
|
while not stop.is_set():
|
|
started = recorder.now()
|
|
status, error, instance, payload = request(
|
|
base, "POST", path, token, body, key
|
|
)
|
|
attempt = Attempt(
|
|
"write", started, recorder.now(), status, error, instance, retry
|
|
)
|
|
recorder.add(attempt, payload if status == 201 else None)
|
|
if status == 201:
|
|
break
|
|
retry = True
|
|
time.sleep(RETRY_PAUSE_SEC)
|
|
started = recorder.now()
|
|
status, error, instance, _ = request(
|
|
base, "GET", path + "?limit=1", token, None, None
|
|
)
|
|
recorder.add(
|
|
Attempt("read", started, recorder.now(), status, error, instance, False)
|
|
)
|
|
index += 1
|
|
|
|
|
|
def percentile(values: list[float], p: float) -> float | None:
|
|
if not values:
|
|
return None
|
|
ordered = sorted(values)
|
|
return round(1000 * ordered[max(0, math.ceil(p * len(ordered)) - 1)], 1)
|
|
|
|
|
|
def phase_stats(attempts: list[Attempt], start: float, end: float) -> dict:
|
|
inside = [a for a in attempts if start <= a.finished < end]
|
|
duration = max(end - start, 1e-9)
|
|
result: dict = {"seconds": round(duration, 3)}
|
|
for op in ("write", "read"):
|
|
ops = [a for a in inside if a.op == op]
|
|
ok = [a for a in ops if a.ok]
|
|
result[op] = {
|
|
"ok": len(ok),
|
|
"errors": len(ops) - len(ok),
|
|
"ok_rps": round(len(ok) / duration, 1),
|
|
"latency_ms": {
|
|
"p50": percentile([a.finished - a.started for a in ok], 0.50),
|
|
"p95": percentile([a.finished - a.started for a in ok], 0.95),
|
|
"p99": percentile([a.finished - a.started for a in ok], 0.99),
|
|
},
|
|
}
|
|
return result
|
|
|
|
|
|
def recovery_stats(attempts: list[Attempt], action_start: float, end: float) -> dict:
|
|
"""
|
|
Время восстановления записи: от первой ошибки после воздействия до первой
|
|
успешной записи после последней ошибки; без ошибок — 0. Отдельно —
|
|
наибольший разрыв между подтверждениями записи (задержка без ошибок).
|
|
"""
|
|
window = sorted(
|
|
(a for a in attempts if a.op == "write" and a.finished >= action_start - 1),
|
|
key=lambda a: a.finished,
|
|
)
|
|
errors = [a for a in window if not a.ok and a.finished >= action_start]
|
|
kinds: dict[str, int] = {}
|
|
for a in errors:
|
|
kind = a.error or f"HTTP {a.status}"
|
|
kinds[kind] = kinds.get(kind, 0) + 1
|
|
if errors:
|
|
last_error = errors[-1].finished
|
|
first_ok_after = next(
|
|
(a.finished for a in window if a.ok and a.finished > last_error), end
|
|
)
|
|
recovery = first_ok_after - errors[0].finished
|
|
else:
|
|
recovery = 0.0
|
|
oks = [a.finished for a in window if a.ok and a.finished <= end]
|
|
gap = max((b - a for a, b in zip(oks, oks[1:])), default=None)
|
|
return {
|
|
"write_errors": len(errors),
|
|
"error_kinds": kinds,
|
|
"write_recovery_seconds": round(recovery, 3),
|
|
"max_write_gap_seconds": None if gap is None else round(gap, 3),
|
|
}
|
|
|
|
|
|
def timeline(attempts: list[Attempt], end: float) -> list[dict]:
|
|
rows = []
|
|
for second in range(int(end) + 1):
|
|
inside = [a for a in attempts if second <= a.finished < second + 1]
|
|
instances: dict[str, int] = {}
|
|
for a in inside:
|
|
if a.ok and a.instance:
|
|
instances[a.instance] = instances.get(a.instance, 0) + 1
|
|
rows.append(
|
|
{
|
|
"t": second,
|
|
"ok": sum(a.ok for a in inside),
|
|
"errors": sum(not a.ok for a in inside),
|
|
"instances": instances,
|
|
}
|
|
)
|
|
return rows
|
|
|
|
|
|
def read_history(client: Client, actor: dict, chat_id: str) -> list[dict]:
|
|
path = f"/api/v1/chats/{chat_id}/messages?limit=100"
|
|
items: list[dict] = []
|
|
while True:
|
|
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
|
|
items.extend(page["items"])
|
|
if page["next_cursor"] is None:
|
|
return items
|
|
path = f"/api/v1/chats/{chat_id}/messages?limit=100&cursor=" + quote(
|
|
page["next_cursor"], safe=""
|
|
)
|
|
|
|
|
|
def integrity(client: Client, actor: dict, chat_id: str, recorder: Recorder) -> dict:
|
|
history = read_history(client, actor, chat_id)
|
|
by_id = {m["id"]: m for m in history}
|
|
texts = [m["text"] for m in history]
|
|
confirmed = {m["id"]: m for m in recorder.confirmed}
|
|
return {
|
|
"messages_in_history": len(history),
|
|
"confirmed_unique": len(confirmed),
|
|
"confirmed_missing_or_changed": sum(
|
|
by_id.get(i) != m for i, m in confirmed.items()
|
|
),
|
|
"duplicate_ids": len(history) - len(by_id),
|
|
"duplicate_texts": len(texts) - len(set(texts)),
|
|
"retried_writes_confirmed_as_replay": len(recorder.confirmed) - len(confirmed),
|
|
}
|
|
|
|
|
|
def run(command: list[str], env: dict | None = None) -> str:
|
|
result = subprocess.run(
|
|
command, cwd=ROOT, env=env, check=True, capture_output=True, text=True
|
|
)
|
|
return result.stdout.strip()
|
|
|
|
|
|
def api_containers() -> list[str]:
|
|
ids = run(["docker", "compose", "--env-file", ".env", "ps", "-q", "api"]).split()
|
|
names = [run(["docker", "inspect", "-f", "{{.Name}}", i]).lstrip("/") for i in ids]
|
|
return sorted(names)
|
|
|
|
|
|
def instance_of(container: str) -> str:
|
|
return run(["docker", "exec", container, "cat", "/tmp/instance-id"])
|
|
|
|
|
|
def container_stats() -> list[str]:
|
|
names = ["messenger-lab3-nginx-1", "messenger-lab3-postgres-1", *api_containers()]
|
|
output = run(["docker", "stats", "--no-stream", "--format", STATS_FORMAT, *names])
|
|
return output.splitlines()
|
|
|
|
|
|
def scenario(
|
|
args: argparse.Namespace,
|
|
label: str,
|
|
action: list[str],
|
|
restore: list[str] | None,
|
|
target: str | None = None,
|
|
env: dict | None = None,
|
|
) -> dict:
|
|
client = Client(args.base_url, timeout=REQUEST_TIMEOUT_SEC)
|
|
alice, _, _, chat = client.bootstrap()
|
|
base = client.base
|
|
recorder = Recorder()
|
|
stop = threading.Event()
|
|
threads = [
|
|
threading.Thread(
|
|
target=worker,
|
|
args=(base, alice["token"], chat["id"], n, recorder, stop),
|
|
daemon=True,
|
|
)
|
|
for n in range(args.workers)
|
|
]
|
|
for thread in threads:
|
|
thread.start()
|
|
time.sleep(args.before / 2)
|
|
stats = container_stats()
|
|
time.sleep(args.before / 2)
|
|
|
|
instances_before = sorted(
|
|
{a.instance for a in recorder.attempts if a.ok and a.instance}
|
|
)
|
|
recorder.mark("action_start")
|
|
action_output = run(action, env)
|
|
recorder.mark("action_end")
|
|
restore_output = ""
|
|
if restore is not None:
|
|
time.sleep(args.outage)
|
|
recorder.mark("restore_start")
|
|
restore_output = run(restore)
|
|
recorder.mark("restore_end")
|
|
time.sleep(args.after)
|
|
stop.set()
|
|
for thread in threads:
|
|
thread.join(timeout=REQUEST_TIMEOUT_SEC + 1)
|
|
recorder.mark("end")
|
|
|
|
events = recorder.events
|
|
end = events["end"]
|
|
after_start = events.get("restore_end", events["action_end"])
|
|
attempts = recorder.attempts
|
|
seen_after = [
|
|
a for a in attempts if a.ok and a.finished >= after_start and a.instance
|
|
]
|
|
back_in_rotation = next(
|
|
(
|
|
round(a.finished - after_start, 3)
|
|
for a in sorted(seen_after, key=lambda a: a.finished)
|
|
if a.instance not in instances_before
|
|
),
|
|
None,
|
|
)
|
|
served = sorted(
|
|
(a for a in attempts if a.ok and a.instance), key=lambda a: a.finished
|
|
)
|
|
instances_seen: dict[str, list[float]] = {}
|
|
for a in served:
|
|
offset = round(a.finished - events["action_start"], 3)
|
|
instances_seen.setdefault(a.instance, [offset, offset])[1] = offset
|
|
phases = {
|
|
"before": phase_stats(attempts, 0, events["action_start"]),
|
|
"during": phase_stats(attempts, events["action_start"], after_start),
|
|
"after": phase_stats(attempts, after_start, end),
|
|
}
|
|
target_after_stop = None
|
|
if target is not None:
|
|
target_after_stop = sum(
|
|
1
|
|
for a in attempts
|
|
if a.ok
|
|
and a.instance == target
|
|
and a.finished > events["action_end"]
|
|
and a.finished < events.get("restore_start", end)
|
|
)
|
|
return {
|
|
"scenario": label,
|
|
"target_instance": target,
|
|
"workers": args.workers,
|
|
"events_seconds": events,
|
|
"instances_before": instances_before,
|
|
"instances_after": sorted({a.instance for a in seen_after}),
|
|
"target_responses_while_stopped": target_after_stop,
|
|
"restored_replica_in_rotation_seconds": back_in_rotation
|
|
if restore is not None
|
|
else None,
|
|
# alias → [первый, последний] успешный ответ, секунды от начала воздействия
|
|
"instances_seen_from_action": instances_seen,
|
|
"phases": phases,
|
|
"recovery": recovery_stats(attempts, events["action_start"], end),
|
|
"integrity": integrity(client, alice, chat["id"], recorder),
|
|
"container_stats_before_action": stats,
|
|
"action_output": action_output.splitlines(),
|
|
"restore_output_tail": restore_output.splitlines()[-2:],
|
|
"timeline": timeline(attempts, end),
|
|
}
|
|
|
|
|
|
def failover(args: argparse.Namespace) -> list[dict]:
|
|
results = []
|
|
for signal in args.signals:
|
|
for container in api_containers():
|
|
target = instance_of(container)
|
|
if signal == "SIGTERM":
|
|
action = ["docker", "stop", "--timeout", "10", container]
|
|
else:
|
|
action = ["docker", "kill", "--signal", "SIGKILL", container]
|
|
label = f"{signal} {container}"
|
|
print(f"== {label} (instance {target})", flush=True)
|
|
result = scenario(
|
|
args, label, action, ["./scripts/restore.sh"], target=target
|
|
)
|
|
print(json.dumps(brief(result), ensure_ascii=False), flush=True)
|
|
results.append(result)
|
|
return results
|
|
|
|
|
|
def rolling(args: argparse.Namespace) -> list[dict]:
|
|
env = dict(os.environ)
|
|
env.setdefault("APP_REVISION", f"exp-{int(time.time())}")
|
|
print(f"== rolling update to revision {env['APP_REVISION']}", flush=True)
|
|
result = scenario(
|
|
args, "rolling update", ["./scripts/rolling-update.sh"], None, env=env
|
|
)
|
|
result["revisions_after"] = run(["./scripts/instances.sh"]).splitlines()
|
|
print(json.dumps(brief(result), ensure_ascii=False), flush=True)
|
|
return [result]
|
|
|
|
|
|
def brief(result: dict) -> dict:
|
|
"""Краткая строка прогона для терминала; полная версия — в --output"""
|
|
return {
|
|
"scenario": result["scenario"],
|
|
"rps_ok_write": {k: v["write"]["ok_rps"] for k, v in result["phases"].items()},
|
|
"rps_ok_read": {k: v["read"]["ok_rps"] for k, v in result["phases"].items()},
|
|
"errors": {
|
|
k: v["write"]["errors"] + v["read"]["errors"]
|
|
for k, v in result["phases"].items()
|
|
},
|
|
"recovery": result["recovery"],
|
|
"target_responses_while_stopped": result["target_responses_while_stopped"],
|
|
"restored_replica_in_rotation_seconds": result[
|
|
"restored_replica_in_rotation_seconds"
|
|
],
|
|
"instances_seen_from_action": result["instances_seen_from_action"],
|
|
"integrity": result["integrity"],
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("mode", choices=["failover", "rolling"])
|
|
parser.add_argument("--base-url", default=None)
|
|
parser.add_argument("--workers", type=int, default=4)
|
|
parser.add_argument("--before", type=float, default=8, help="нагрузка до отказа")
|
|
parser.add_argument("--outage", type=float, default=10, help="реплика остановлена")
|
|
parser.add_argument("--after", type=float, default=8, help="после восстановления")
|
|
parser.add_argument(
|
|
"--signals",
|
|
nargs="+",
|
|
default=["SIGTERM", "SIGKILL"],
|
|
choices=["SIGTERM", "SIGKILL"],
|
|
)
|
|
parser.add_argument("--output", default=None, help="JSON с полными результатами")
|
|
args = parser.parse_args()
|
|
started = time.strftime("%Y-%m-%dT%H:%M:%S%z")
|
|
results = failover(args) if args.mode == "failover" else rolling(args)
|
|
report = {
|
|
"mode": args.mode,
|
|
"started": started,
|
|
"profile": {
|
|
"workers": args.workers,
|
|
"loop": "closed: POST message (retry same key until 201) + GET limit=1",
|
|
"client_timeout_seconds": REQUEST_TIMEOUT_SEC,
|
|
"phases_seconds": {
|
|
"before": args.before,
|
|
"outage": args.outage,
|
|
"after": args.after,
|
|
},
|
|
},
|
|
"runs": results,
|
|
}
|
|
if args.output:
|
|
destination = ROOT / args.output
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
|
print(f"saved: {args.output}")
|
|
failed = [
|
|
r["scenario"]
|
|
for r in results
|
|
if r["integrity"]["confirmed_missing_or_changed"]
|
|
or r["integrity"]["duplicate_ids"]
|
|
or r["integrity"]["duplicate_texts"]
|
|
or r["target_responses_while_stopped"]
|
|
]
|
|
if failed:
|
|
raise SystemExit(f"FAIL: потеря/дубль или отказ не изолирован: {failed}")
|
|
print(f"PASS: прогонов {len(results)}, подтверждённые сообщения целы, дублей нет")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|