Files
mtusi2026lab3/scripts/verify_cluster.py
T

293 lines
10 KiB
Python

"""Проверки реальных процессов, failover, rolling и общей идемпотентности."""
import argparse
import json
import os
import subprocess
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from pathlib import Path
from uuid import uuid4
import httpx
from scripts import cluster
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
def snapshot():
ids = compose("ps", "-q").split()
result = []
for c in json.loads(command(["docker", "inspect", *ids])):
role = c["Config"]["Labels"]["com.docker.compose.service"]
entry = {
"role": role,
"container": c["Id"][:12],
"pid": c["State"]["Pid"],
"started_at": c["State"]["StartedAt"],
"ports": c["HostConfig"]["PortBindings"],
"cpu_nano": c["HostConfig"]["NanoCpus"],
"memory_limit": c["HostConfig"]["Memory"],
}
if role == "api":
entry["instance"] = cluster.alias(c["Id"])
assert not entry["ports"]
result.append(entry)
return result
def exercise(client, session, chat, action, *, seconds=15):
path = f"/api/v1/chats/{chat['id']}/messages"
observations, confirmed, errors, reads = [], {}, 0, 0
started = time.monotonic()
finished = threading.Event()
action_result = {}
def operate():
time.sleep(0.5)
try:
action()
except BaseException as exc:
action_result["error"] = type(exc).__name__
finally:
action_result["finished"] = round(time.monotonic() - started, 3)
finished.set()
thread = threading.Thread(target=operate)
thread.start()
key = uuid4().hex
try:
while time.monotonic() - started < seconds or not finished.is_set():
h = {**auth(session), "Idempotency-Key": key}
try:
r = client.post(path, headers=h, json={"text": key})
if r.status_code != 201:
errors += 1
continue
m = r.json()
confirmed[m["id"]] = m
observations.append(
{
"at": round(time.monotonic() - started, 3),
"instance": r.headers["x-instance-id"],
}
)
key = uuid4().hex
read = client.get(
path, headers=auth(session), params={"limit": 100}
)
reads += read.status_code == 200
errors += read.status_code != 200
except httpx.HTTPError:
errors += 1
time.sleep(0.04)
finally:
thread.join(timeout=90)
assert not action_result.get("error"), action_result
found, cursor = [], None
while True:
r = client.get(
path,
headers=auth(session),
params={"limit": 100, **({"cursor": cursor} if cursor else {})},
)
assert r.status_code == 200
page = r.json()
found.extend(page["items"])
cursor = page["next_cursor"]
if not cursor:
break
by_id = {m["id"]: m for m in found}
assert all(by_id.get(k) == v for k, v in confirmed.items())
texts = [m["text"] for m in found]
assert len(texts) == len(set(texts))
recovery = next(
(
x["at"]
for x in observations
if x["at"] >= action_result["finished"]
),
None,
)
assert recovery is not None and (seconds > 20 or recovery <= 10)
elapsed = time.monotonic() - started
return {
"confirmed_writes": len(confirmed),
"successful_reads": reads,
"errors": errors,
"successful_rps": round((len(confirmed) + reads) / elapsed, 2),
"first_success_after_action_seconds": recovery,
"action": action_result,
"observations": observations,
"confirmed_preserved": True,
"duplicate_texts": False,
}
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/cluster-repeat.json")
args = parser.parse_args()
report = {
"date": datetime.now(timezone.utc).isoformat(),
"source_commit": command(["git", "rev-parse", "HEAD"]),
"complete": False,
}
output = Path(args.output)
output.parent.mkdir(parents=True, exist_ok=True)
try:
ready()
initial = snapshot()
apis = [c for c in initial if c["role"] == "api"]
assert len(apis) == 2 and len({c["pid"] for c in apis}) == 2
report["before"] = initial
with httpx.Client(base_url=BASE, trust_env=False, timeout=8) as client:
user, _, session = actor(client)
r = client.post(
"/api/v1/chats",
headers=auth(session),
json={
"title": "Cluster verification",
"member_ids": [user["id"]],
},
)
assert r.status_code == 201
chat = r.json()
path = f"/api/v1/chats/{chat['id']}/messages"
key = uuid4().hex
def repeat(_):
with httpx.Client(
base_url=BASE, trust_env=False, timeout=8
) as c:
r = c.post(
path,
headers={**auth(session), "Idempotency-Key": key},
json={"text": "Same concurrent request"},
)
assert r.status_code == 201
return r.json(), r.headers["x-instance-id"]
with ThreadPoolExecutor(max_workers=10) as pool:
repeated = list(pool.map(repeat, range(30)))
assert len({m[0]["id"] for m in repeated}) == 1
assert len({m[1] for m in repeated}) == 2
report["parallel_replay"] = {
"requests": 30,
"messages": 1,
"instances": sorted({m[1] for m in repeated}),
}
# Отдельный чат каждой серии: проверки дубликатов не смешивают тексты.
report["failovers"] = []
for api in apis:
def stop(target=api["instance"]):
env = {**os.environ, "TARGET_INSTANCE_ID": target}
subprocess.run(
["scripts/stop-one.sh"],
cwd=ROOT,
env=env,
check=True,
capture_output=True,
text=True,
)
r = client.post(
"/api/v1/chats",
headers=auth(session),
json={"title": "Failover", "member_ids": [user["id"]]},
)
result = exercise(client, session, r.json(), stop)
assert api["instance"] not in [
o["instance"] for o in result["observations"][-5:]
]
assert (
command(
[
"docker",
"inspect",
"-f",
"{{.State.Running}}",
api["container"],
]
)
== "false"
)
result["stopped"] = api
report["failovers"].append(result)
compose("up", "-d", "--no-build", "--wait")
ready()
r = client.post(
"/api/v1/chats",
headers=auth(session),
json={"title": "Rolling", "member_ids": [user["id"]]},
)
before_roll = snapshot()
report["rolling"] = exercise(
client,
session,
r.json(),
lambda: command(["scripts/rolling-update.sh"]),
seconds=45,
)
after_roll = snapshot()
assert not (
{c["container"] for c in before_roll if c["role"] == "api"}
& {c["container"] for c in after_roll if c["role"] == "api"}
)
report["rolling"]["before"] = before_roll
report["rolling"]["after"] = after_roll
replay = repeat(0)[0]
assert replay == repeated[0][0]
report["replay_after_replacement"] = True
before = snapshot()
env = {
**os.environ,
"NO_PROXY": "localhost,127.0.0.1",
"no_proxy": "localhost,127.0.0.1",
}
for mode, hook in [
("failover", "scripts/stop-one.sh"),
("persistence", "scripts/restart.sh"),
]:
if mode == "persistence":
compose("up", "-d", "--no-build", "--wait")
ready()
before = snapshot()
run = subprocess.run(
[
".venv/bin/python",
"tests/resilience.py",
"--mode",
mode,
"--action-script",
hook,
"--base-url",
BASE,
],
cwd=ROOT,
env=env,
capture_output=True,
text=True,
)
assert run.returncode == 0, run.stdout + run.stderr
report["public_" + mode] = run.stdout.strip()
after = snapshot()
for role in ["postgres", "cache", "lb"]:
old = next(c for c in before if c["role"] == role)
new = next(c for c in after if c["role"] == role)
assert old["started_at"] != new["started_at"]
report["after_persistence"] = after
report["complete"] = True
finally:
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print("PASS: process failover, rolling, replay and full persistence")
if __name__ == "__main__":
main()