forked from mtusi2026/mtusi2026lab3
149 lines
6.4 KiB
Python
149 lines
6.4 KiB
Python
"""Сценарий поочерёдного отказа всех API-реплик под записью."""
|
|
|
|
from collections import Counter
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
from threading import Event, Lock
|
|
import time
|
|
from urllib.parse import quote
|
|
from uuid import uuid4
|
|
|
|
from scripts.deployment import ComposeProject, ROOT, deployment_lock
|
|
from tests.client import Client, Failure, arguments, check, run
|
|
|
|
|
|
def failure_kind(error: Failure) -> str:
|
|
"""Различает HTTP-отказы, ошибки соединения и проверки контракта."""
|
|
match = re.search(r"received (\d{3})$", str(error))
|
|
if match:
|
|
return "http_" + match.group(1)
|
|
return "connection" if "connection/TLS failure" in str(error) else "contract"
|
|
|
|
|
|
def snapshot(project: ComposeProject, phase: str, target: str) -> list:
|
|
"""Печатает состояние API без окружения и секретов."""
|
|
replicas = project.replicas()
|
|
print(json.dumps({
|
|
"scenario": "replica_failure", "phase": phase, "target": target,
|
|
"replicas": [{"instance": item.instance_id, "running": item.running, "health": item.health} for item in replicas],
|
|
}), flush=True)
|
|
return replicas
|
|
|
|
|
|
def history(client: Client, actor: dict, chat: dict) -> list[dict]:
|
|
"""Читает всю историю без повторов курсора."""
|
|
path = f"/api/v1/chats/{chat['id']}/messages?limit=100"
|
|
items = []
|
|
cursors = set()
|
|
for _ in range(1000):
|
|
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
|
|
items.extend(page["items"])
|
|
cursor = page["next_cursor"]
|
|
if cursor is None:
|
|
return items
|
|
check(cursor not in cursors, "cursor cycle")
|
|
cursors.add(cursor)
|
|
path = f"/api/v1/chats/{chat['id']}/messages?limit=100&cursor={quote(cursor, safe='')}"
|
|
raise Failure("history exceeded safety bound")
|
|
|
|
|
|
def exercise(project: ComposeProject, args) -> None:
|
|
"""Останавливает каждую реплику и восстанавливает её перед следующей."""
|
|
replicas = project.replicas()
|
|
check(len(replicas) >= 2, "at least two API replicas are required")
|
|
project.wait_healthy({item.id for item in replicas})
|
|
client = Client(args.base_url, args.ca_file, timeout=2)
|
|
for item in replicas:
|
|
project.wait_routed(item.instance_id, client.base, args.ca_file)
|
|
actor, _, _, chat = client.bootstrap()
|
|
confirmed = set()
|
|
|
|
for target in replicas:
|
|
snapshot(project, "before", target.instance_id)
|
|
stop = Event()
|
|
lock = Lock()
|
|
successes = []
|
|
errors = []
|
|
|
|
def worker():
|
|
local = Client(client.base, args.ca_file, timeout=2)
|
|
key = uuid4().hex
|
|
while not stop.is_set():
|
|
started = time.monotonic()
|
|
try:
|
|
result = local.message(actor, chat, "failover-" + key, key=key)
|
|
with lock:
|
|
successes.append((started, time.monotonic(), result["json"]["id"]))
|
|
key = uuid4().hex
|
|
except Failure as error:
|
|
with lock:
|
|
errors.append(failure_kind(error))
|
|
time.sleep(0.1)
|
|
|
|
started = time.monotonic()
|
|
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
|
jobs = [pool.submit(worker) for _ in range(args.concurrency)]
|
|
try:
|
|
time.sleep(2)
|
|
stop_started = time.monotonic()
|
|
subprocess.run(
|
|
[str(ROOT / "scripts/stop-one.sh")], cwd=ROOT,
|
|
env={**os.environ, "TARGET_INSTANCE_ID": target.instance_id},
|
|
check=True, timeout=10, capture_output=True,
|
|
)
|
|
stop_finished = time.monotonic()
|
|
stopped = snapshot(project, "stopped", target.instance_id)
|
|
check(any(item.id == target.id and not item.running for item in stopped), "target API was not stopped")
|
|
check(any(item.id != target.id and item.running and item.health == "healthy" for item in stopped), "no healthy surviving API")
|
|
time.sleep(args.duration)
|
|
finally:
|
|
stop.set()
|
|
try:
|
|
for job in jobs:
|
|
job.result(timeout=5)
|
|
finally:
|
|
project.docker("start", target.id)
|
|
project.wait_healthy({item.id for item in replicas})
|
|
project.wait_routed(target.instance_id, client.base, args.ca_file)
|
|
snapshot(project, "restored", target.instance_id)
|
|
|
|
elapsed = max(end for _, end, _ in successes) - started if successes else 0
|
|
recovered = [end for begin, end, _ in successes if begin >= stop_finished]
|
|
check(bool(recovered), "no confirmed writes after failure")
|
|
recovery = min(recovered) - stop_started
|
|
check(recovery <= 10, "writes did not recover within 10 seconds")
|
|
confirmed.update(id for _, _, id in successes)
|
|
items = history(client, actor, chat)
|
|
check(confirmed <= {item["id"] for item in items}, "confirmed messages were lost")
|
|
texts = [item["text"] for item in items]
|
|
check(len(texts) == len(set(texts)), "uncertain retry created duplicates")
|
|
print(json.dumps({
|
|
"instance": target.instance_id,
|
|
"successful_writes": len(successes),
|
|
"failed_attempts": len(errors),
|
|
"errors_by_kind": dict(Counter(errors)),
|
|
"elapsed_seconds": round(elapsed, 3),
|
|
"successful_rps": round(len(successes) / elapsed, 3) if elapsed else 0,
|
|
"first_recovery_seconds": round(recovery, 3),
|
|
}), flush=True)
|
|
|
|
|
|
def main() -> None:
|
|
"""Готовит параметры эксперимента отказа реплик."""
|
|
def configure(parser):
|
|
parser.add_argument("--duration", type=float, default=15)
|
|
parser.add_argument("--concurrency", type=int, default=2)
|
|
|
|
args = arguments("Fail each API replica under write load", configure)
|
|
check(1 <= args.duration <= 120 and 1 <= args.concurrency <= 8, "invalid workload")
|
|
project = ComposeProject()
|
|
with deployment_lock(project):
|
|
exercise(project, args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|