Реализация лабораторной работы 4 на 4
This commit is contained in:
@@ -0,0 +1,480 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Нагрузка через LB во время отказа/замены API-реплик; метрики и проверка данных.
|
||||
|
||||
failover — для каждой API-реплики по очереди: нагрузка → остановка этой реплики
|
||||
(docker stop = SIGTERM, затем docker kill = SIGKILL) → восстановление
|
||||
(scripts/restore.sh) → нагрузка после восстановления.
|
||||
rolling — нагрузка во время scripts/rolling-update.sh (поочерёдная замена реплик).
|
||||
|
||||
Клиент ведёт себя как настоящий: неподтверждённый POST повторяет с тем же
|
||||
Idempotency-Key и телом. После прогона вся история читается заново: каждое
|
||||
подтверждённое сообщение на месте и не изменено, дублей нет.
|
||||
Python 3.9+, стандартная библиотека; tokens не печатаются и не сохраняются.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client # noqa: E402
|
||||
|
||||
REQUEST_TIMEOUT_SEC = 5
|
||||
RETRY_PAUSE_SEC = 0.05
|
||||
STATS_FORMAT = "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Attempt:
|
||||
"""Одна HTTP-попытка клиента; время — секунды от начала прогона"""
|
||||
|
||||
op: str
|
||||
started: float
|
||||
finished: float
|
||||
status: int | None
|
||||
error: str | None
|
||||
instance: str | None
|
||||
retry: bool
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return self.status in (200, 201)
|
||||
|
||||
|
||||
class Recorder:
|
||||
def __init__(self) -> None:
|
||||
self.started = time.monotonic()
|
||||
self.lock = threading.Lock()
|
||||
self.attempts: list[Attempt] = []
|
||||
self.confirmed: list[dict] = []
|
||||
self.events: dict[str, float] = {}
|
||||
|
||||
def now(self) -> float:
|
||||
return time.monotonic() - self.started
|
||||
|
||||
def mark(self, name: str) -> None:
|
||||
self.events[name] = round(self.now(), 3)
|
||||
|
||||
def add(self, attempt: Attempt, message: dict | None = None) -> None:
|
||||
with self.lock:
|
||||
self.attempts.append(attempt)
|
||||
if message is not None:
|
||||
self.confirmed.append(message)
|
||||
|
||||
|
||||
def request(
|
||||
base: str, method: str, path: str, token: str, body: dict | None, key: str | None
|
||||
) -> tuple[int | None, str | None, str | None, dict | None]:
|
||||
headers = {"Authorization": "Bearer " + token, "Accept": "application/json"}
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body, ensure_ascii=False).encode()
|
||||
headers["Content-Type"] = "application/json"
|
||||
if key is not None:
|
||||
headers["Idempotency-Key"] = key
|
||||
req = urllib.request.Request(base + path, data=data, headers=headers, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT_SEC) as response:
|
||||
payload = json.loads(response.read())
|
||||
return response.status, None, response.headers.get("X-Instance-Id"), payload
|
||||
except urllib.error.HTTPError as error:
|
||||
error.read()
|
||||
return error.code, None, error.headers.get("X-Instance-Id"), None
|
||||
except OSError as error:
|
||||
return None, type(error).__name__, None, None
|
||||
|
||||
|
||||
def worker(
|
||||
base: str,
|
||||
token: str,
|
||||
chat_id: str,
|
||||
worker_id: int,
|
||||
recorder: Recorder,
|
||||
stop: threading.Event,
|
||||
) -> None:
|
||||
"""Запись с повтором по тому же ключу до подтверждения, затем чтение"""
|
||||
path = f"/api/v1/chats/{chat_id}/messages"
|
||||
index = 0
|
||||
while not stop.is_set():
|
||||
key = uuid.uuid4().hex
|
||||
body = {"text": f"exp-{worker_id}-{index}-{key[:8]}"}
|
||||
retry = False
|
||||
while not stop.is_set():
|
||||
started = recorder.now()
|
||||
status, error, instance, payload = request(
|
||||
base, "POST", path, token, body, key
|
||||
)
|
||||
attempt = Attempt(
|
||||
"write", started, recorder.now(), status, error, instance, retry
|
||||
)
|
||||
recorder.add(attempt, payload if status == 201 else None)
|
||||
if status == 201:
|
||||
break
|
||||
retry = True
|
||||
time.sleep(RETRY_PAUSE_SEC)
|
||||
started = recorder.now()
|
||||
status, error, instance, _ = request(
|
||||
base, "GET", path + "?limit=1", token, None, None
|
||||
)
|
||||
recorder.add(
|
||||
Attempt("read", started, recorder.now(), status, error, instance, False)
|
||||
)
|
||||
index += 1
|
||||
|
||||
|
||||
def percentile(values: list[float], p: float) -> float | None:
|
||||
if not values:
|
||||
return None
|
||||
ordered = sorted(values)
|
||||
return round(1000 * ordered[max(0, math.ceil(p * len(ordered)) - 1)], 1)
|
||||
|
||||
|
||||
def phase_stats(attempts: list[Attempt], start: float, end: float) -> dict:
|
||||
inside = [a for a in attempts if start <= a.finished < end]
|
||||
duration = max(end - start, 1e-9)
|
||||
result: dict = {"seconds": round(duration, 3)}
|
||||
for op in ("write", "read"):
|
||||
ops = [a for a in inside if a.op == op]
|
||||
ok = [a for a in ops if a.ok]
|
||||
result[op] = {
|
||||
"ok": len(ok),
|
||||
"errors": len(ops) - len(ok),
|
||||
"ok_rps": round(len(ok) / duration, 1),
|
||||
"latency_ms": {
|
||||
"p50": percentile([a.finished - a.started for a in ok], 0.50),
|
||||
"p95": percentile([a.finished - a.started for a in ok], 0.95),
|
||||
"p99": percentile([a.finished - a.started for a in ok], 0.99),
|
||||
},
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def recovery_stats(attempts: list[Attempt], action_start: float, end: float) -> dict:
|
||||
"""
|
||||
Время восстановления записи: от первой ошибки после воздействия до первой
|
||||
успешной записи после последней ошибки; без ошибок — 0. Отдельно —
|
||||
наибольший разрыв между подтверждениями записи (задержка без ошибок).
|
||||
"""
|
||||
window = sorted(
|
||||
(a for a in attempts if a.op == "write" and a.finished >= action_start - 1),
|
||||
key=lambda a: a.finished,
|
||||
)
|
||||
errors = [a for a in window if not a.ok and a.finished >= action_start]
|
||||
kinds: dict[str, int] = {}
|
||||
for a in errors:
|
||||
kind = a.error or f"HTTP {a.status}"
|
||||
kinds[kind] = kinds.get(kind, 0) + 1
|
||||
if errors:
|
||||
last_error = errors[-1].finished
|
||||
first_ok_after = next(
|
||||
(a.finished for a in window if a.ok and a.finished > last_error), end
|
||||
)
|
||||
recovery = first_ok_after - errors[0].finished
|
||||
else:
|
||||
recovery = 0.0
|
||||
oks = [a.finished for a in window if a.ok and a.finished <= end]
|
||||
gap = max((b - a for a, b in zip(oks, oks[1:])), default=None)
|
||||
return {
|
||||
"write_errors": len(errors),
|
||||
"error_kinds": kinds,
|
||||
"write_recovery_seconds": round(recovery, 3),
|
||||
"max_write_gap_seconds": None if gap is None else round(gap, 3),
|
||||
}
|
||||
|
||||
|
||||
def timeline(attempts: list[Attempt], end: float) -> list[dict]:
|
||||
rows = []
|
||||
for second in range(int(end) + 1):
|
||||
inside = [a for a in attempts if second <= a.finished < second + 1]
|
||||
instances: dict[str, int] = {}
|
||||
for a in inside:
|
||||
if a.ok and a.instance:
|
||||
instances[a.instance] = instances.get(a.instance, 0) + 1
|
||||
rows.append(
|
||||
{
|
||||
"t": second,
|
||||
"ok": sum(a.ok for a in inside),
|
||||
"errors": sum(not a.ok for a in inside),
|
||||
"instances": instances,
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def read_history(client: Client, actor: dict, chat_id: str) -> list[dict]:
|
||||
path = f"/api/v1/chats/{chat_id}/messages?limit=100"
|
||||
items: list[dict] = []
|
||||
while True:
|
||||
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
|
||||
items.extend(page["items"])
|
||||
if page["next_cursor"] is None:
|
||||
return items
|
||||
path = f"/api/v1/chats/{chat_id}/messages?limit=100&cursor=" + quote(
|
||||
page["next_cursor"], safe=""
|
||||
)
|
||||
|
||||
|
||||
def integrity(client: Client, actor: dict, chat_id: str, recorder: Recorder) -> dict:
|
||||
history = read_history(client, actor, chat_id)
|
||||
by_id = {m["id"]: m for m in history}
|
||||
texts = [m["text"] for m in history]
|
||||
confirmed = {m["id"]: m for m in recorder.confirmed}
|
||||
return {
|
||||
"messages_in_history": len(history),
|
||||
"confirmed_unique": len(confirmed),
|
||||
"confirmed_missing_or_changed": sum(
|
||||
by_id.get(i) != m for i, m in confirmed.items()
|
||||
),
|
||||
"duplicate_ids": len(history) - len(by_id),
|
||||
"duplicate_texts": len(texts) - len(set(texts)),
|
||||
"retried_writes_confirmed_as_replay": len(recorder.confirmed) - len(confirmed),
|
||||
}
|
||||
|
||||
|
||||
def run(command: list[str], env: dict | None = None) -> str:
|
||||
result = subprocess.run(
|
||||
command, cwd=ROOT, env=env, check=True, capture_output=True, text=True
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def api_containers() -> list[str]:
|
||||
ids = run(["docker", "compose", "--env-file", ".env", "ps", "-q", "api"]).split()
|
||||
names = [run(["docker", "inspect", "-f", "{{.Name}}", i]).lstrip("/") for i in ids]
|
||||
return sorted(names)
|
||||
|
||||
|
||||
def instance_of(container: str) -> str:
|
||||
return run(["docker", "exec", container, "cat", "/tmp/instance-id"])
|
||||
|
||||
|
||||
def container_stats() -> list[str]:
|
||||
names = ["messenger-lab3-nginx-1", "messenger-lab3-postgres-1", *api_containers()]
|
||||
output = run(["docker", "stats", "--no-stream", "--format", STATS_FORMAT, *names])
|
||||
return output.splitlines()
|
||||
|
||||
|
||||
def scenario(
|
||||
args: argparse.Namespace,
|
||||
label: str,
|
||||
action: list[str],
|
||||
restore: list[str] | None,
|
||||
target: str | None = None,
|
||||
env: dict | None = None,
|
||||
) -> dict:
|
||||
client = Client(args.base_url, timeout=REQUEST_TIMEOUT_SEC)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
base = client.base
|
||||
recorder = Recorder()
|
||||
stop = threading.Event()
|
||||
threads = [
|
||||
threading.Thread(
|
||||
target=worker,
|
||||
args=(base, alice["token"], chat["id"], n, recorder, stop),
|
||||
daemon=True,
|
||||
)
|
||||
for n in range(args.workers)
|
||||
]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
time.sleep(args.before / 2)
|
||||
stats = container_stats()
|
||||
time.sleep(args.before / 2)
|
||||
|
||||
instances_before = sorted(
|
||||
{a.instance for a in recorder.attempts if a.ok and a.instance}
|
||||
)
|
||||
recorder.mark("action_start")
|
||||
action_output = run(action, env)
|
||||
recorder.mark("action_end")
|
||||
restore_output = ""
|
||||
if restore is not None:
|
||||
time.sleep(args.outage)
|
||||
recorder.mark("restore_start")
|
||||
restore_output = run(restore)
|
||||
recorder.mark("restore_end")
|
||||
time.sleep(args.after)
|
||||
stop.set()
|
||||
for thread in threads:
|
||||
thread.join(timeout=REQUEST_TIMEOUT_SEC + 1)
|
||||
recorder.mark("end")
|
||||
|
||||
events = recorder.events
|
||||
end = events["end"]
|
||||
after_start = events.get("restore_end", events["action_end"])
|
||||
attempts = recorder.attempts
|
||||
seen_after = [
|
||||
a for a in attempts if a.ok and a.finished >= after_start and a.instance
|
||||
]
|
||||
back_in_rotation = next(
|
||||
(
|
||||
round(a.finished - after_start, 3)
|
||||
for a in sorted(seen_after, key=lambda a: a.finished)
|
||||
if a.instance not in instances_before
|
||||
),
|
||||
None,
|
||||
)
|
||||
served = sorted(
|
||||
(a for a in attempts if a.ok and a.instance), key=lambda a: a.finished
|
||||
)
|
||||
instances_seen: dict[str, list[float]] = {}
|
||||
for a in served:
|
||||
offset = round(a.finished - events["action_start"], 3)
|
||||
instances_seen.setdefault(a.instance, [offset, offset])[1] = offset
|
||||
phases = {
|
||||
"before": phase_stats(attempts, 0, events["action_start"]),
|
||||
"during": phase_stats(attempts, events["action_start"], after_start),
|
||||
"after": phase_stats(attempts, after_start, end),
|
||||
}
|
||||
target_after_stop = None
|
||||
if target is not None:
|
||||
target_after_stop = sum(
|
||||
1
|
||||
for a in attempts
|
||||
if a.ok
|
||||
and a.instance == target
|
||||
and a.finished > events["action_end"]
|
||||
and a.finished < events.get("restore_start", end)
|
||||
)
|
||||
return {
|
||||
"scenario": label,
|
||||
"target_instance": target,
|
||||
"workers": args.workers,
|
||||
"events_seconds": events,
|
||||
"instances_before": instances_before,
|
||||
"instances_after": sorted({a.instance for a in seen_after}),
|
||||
"target_responses_while_stopped": target_after_stop,
|
||||
"restored_replica_in_rotation_seconds": back_in_rotation
|
||||
if restore is not None
|
||||
else None,
|
||||
# alias → [первый, последний] успешный ответ, секунды от начала воздействия
|
||||
"instances_seen_from_action": instances_seen,
|
||||
"phases": phases,
|
||||
"recovery": recovery_stats(attempts, events["action_start"], end),
|
||||
"integrity": integrity(client, alice, chat["id"], recorder),
|
||||
"container_stats_before_action": stats,
|
||||
"action_output": action_output.splitlines(),
|
||||
"restore_output_tail": restore_output.splitlines()[-2:],
|
||||
"timeline": timeline(attempts, end),
|
||||
}
|
||||
|
||||
|
||||
def failover(args: argparse.Namespace) -> list[dict]:
|
||||
results = []
|
||||
for signal in args.signals:
|
||||
for container in api_containers():
|
||||
target = instance_of(container)
|
||||
if signal == "SIGTERM":
|
||||
action = ["docker", "stop", "--timeout", "10", container]
|
||||
else:
|
||||
action = ["docker", "kill", "--signal", "SIGKILL", container]
|
||||
label = f"{signal} {container}"
|
||||
print(f"== {label} (instance {target})", flush=True)
|
||||
result = scenario(
|
||||
args, label, action, ["./scripts/restore.sh"], target=target
|
||||
)
|
||||
print(json.dumps(brief(result), ensure_ascii=False), flush=True)
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
|
||||
def rolling(args: argparse.Namespace) -> list[dict]:
|
||||
env = dict(os.environ)
|
||||
env.setdefault("APP_REVISION", f"exp-{int(time.time())}")
|
||||
print(f"== rolling update to revision {env['APP_REVISION']}", flush=True)
|
||||
result = scenario(
|
||||
args, "rolling update", ["./scripts/rolling-update.sh"], None, env=env
|
||||
)
|
||||
result["revisions_after"] = run(["./scripts/instances.sh"]).splitlines()
|
||||
print(json.dumps(brief(result), ensure_ascii=False), flush=True)
|
||||
return [result]
|
||||
|
||||
|
||||
def brief(result: dict) -> dict:
|
||||
"""Краткая строка прогона для терминала; полная версия — в --output"""
|
||||
return {
|
||||
"scenario": result["scenario"],
|
||||
"rps_ok_write": {k: v["write"]["ok_rps"] for k, v in result["phases"].items()},
|
||||
"rps_ok_read": {k: v["read"]["ok_rps"] for k, v in result["phases"].items()},
|
||||
"errors": {
|
||||
k: v["write"]["errors"] + v["read"]["errors"]
|
||||
for k, v in result["phases"].items()
|
||||
},
|
||||
"recovery": result["recovery"],
|
||||
"target_responses_while_stopped": result["target_responses_while_stopped"],
|
||||
"restored_replica_in_rotation_seconds": result[
|
||||
"restored_replica_in_rotation_seconds"
|
||||
],
|
||||
"instances_seen_from_action": result["instances_seen_from_action"],
|
||||
"integrity": result["integrity"],
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("mode", choices=["failover", "rolling"])
|
||||
parser.add_argument("--base-url", default=None)
|
||||
parser.add_argument("--workers", type=int, default=4)
|
||||
parser.add_argument("--before", type=float, default=8, help="нагрузка до отказа")
|
||||
parser.add_argument("--outage", type=float, default=10, help="реплика остановлена")
|
||||
parser.add_argument("--after", type=float, default=8, help="после восстановления")
|
||||
parser.add_argument(
|
||||
"--signals",
|
||||
nargs="+",
|
||||
default=["SIGTERM", "SIGKILL"],
|
||||
choices=["SIGTERM", "SIGKILL"],
|
||||
)
|
||||
parser.add_argument("--output", default=None, help="JSON с полными результатами")
|
||||
args = parser.parse_args()
|
||||
started = time.strftime("%Y-%m-%dT%H:%M:%S%z")
|
||||
results = failover(args) if args.mode == "failover" else rolling(args)
|
||||
report = {
|
||||
"mode": args.mode,
|
||||
"started": started,
|
||||
"profile": {
|
||||
"workers": args.workers,
|
||||
"loop": "closed: POST message (retry same key until 201) + GET limit=1",
|
||||
"client_timeout_seconds": REQUEST_TIMEOUT_SEC,
|
||||
"phases_seconds": {
|
||||
"before": args.before,
|
||||
"outage": args.outage,
|
||||
"after": args.after,
|
||||
},
|
||||
},
|
||||
"runs": results,
|
||||
}
|
||||
if args.output:
|
||||
destination = ROOT / args.output
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
||||
print(f"saved: {args.output}")
|
||||
failed = [
|
||||
r["scenario"]
|
||||
for r in results
|
||||
if r["integrity"]["confirmed_missing_or_changed"]
|
||||
or r["integrity"]["duplicate_ids"]
|
||||
or r["integrity"]["duplicate_texts"]
|
||||
or r["target_responses_while_stopped"]
|
||||
]
|
||||
if failed:
|
||||
raise SystemExit(f"FAIL: потеря/дубль или отказ не изолирован: {failed}")
|
||||
print(f"PASS: прогонов {len(results)}, подтверждённые сообщения целы, дублей нет")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user