Files
mtusi2026lab5/scripts/verify_cluster.py
T

348 lines
12 KiB
Python

"""Проверки реальных процессов, failover, rolling и общей идемпотентности."""
import argparse
import json
import os
import subprocess
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from pathlib import Path
from uuid import uuid4
import httpx
from scripts import cluster
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
def snapshot():
ids = compose("ps", "-q").split()
result = []
for c in json.loads(command(["docker", "inspect", *ids])):
role = c["Config"]["Labels"]["com.docker.compose.service"]
entry = {
"role": role,
"container": c["Id"][:12],
"pid": c["State"]["Pid"],
"started_at": c["State"]["StartedAt"],
"ports": c["HostConfig"]["PortBindings"],
"cpu_nano": c["HostConfig"]["NanoCpus"],
"memory_limit": c["HostConfig"]["Memory"],
}
if role == "api":
entry["instance"] = cluster.alias(c["Id"])
assert not entry["ports"]
result.append(entry)
return result
def exercise(client, session, chat, action, *, seconds=15):
path = f"/api/v1/chats/{chat['id']}/messages"
observations, confirmed, errors, reads = [], {}, 0, 0
started = time.monotonic()
finished = threading.Event()
action_result = {}
def operate():
time.sleep(0.5)
try:
action()
except BaseException as exc:
action_result["error"] = type(exc).__name__
finally:
action_result["finished"] = round(time.monotonic() - started, 3)
finished.set()
thread = threading.Thread(target=operate)
thread.start()
key = uuid4().hex
try:
while (
time.monotonic() - started < seconds
or not finished.is_set()
or time.monotonic() - started
< action_result.get("finished", 0) + 2
):
h = {**auth(session), "Idempotency-Key": key}
try:
r = client.post(path, headers=h, json={"text": key})
if r.status_code != 201:
errors += 1
continue
m = r.json()
confirmed[m["id"]] = m
observations.append(
{
"at": round(time.monotonic() - started, 3),
"instance": r.headers["x-instance-id"],
}
)
key = uuid4().hex
read = client.get(
path, headers=auth(session), params={"limit": 100}
)
reads += read.status_code == 200
errors += read.status_code != 200
except httpx.HTTPError:
errors += 1
time.sleep(0.04)
finally:
thread.join(timeout=90)
assert not action_result.get("error"), action_result
found, cursor = [], None
while True:
r = client.get(
path,
headers=auth(session),
params={"limit": 100, **({"cursor": cursor} if cursor else {})},
)
assert r.status_code == 200
page = r.json()
found.extend(page["items"])
cursor = page["next_cursor"]
if not cursor:
break
by_id = {m["id"]: m for m in found}
assert all(by_id.get(k) == v for k, v in confirmed.items())
texts = [m["text"] for m in found]
assert len(texts) == len(set(texts))
recovery = next(
(
x["at"]
for x in observations
if x["at"] >= action_result["finished"]
),
None,
)
assert recovery is not None and (seconds > 20 or recovery <= 10)
elapsed = time.monotonic() - started
return {
"confirmed_writes": len(confirmed),
"successful_reads": reads,
"errors": errors,
"successful_rps": round((len(confirmed) + reads) / elapsed, 2),
"first_success_after_action_seconds": recovery,
"action": action_result,
"observations": observations,
"confirmed_preserved": True,
"duplicate_texts": False,
}
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/cluster-repeat.json")
args = parser.parse_args()
report = {
"date": datetime.now(timezone.utc).isoformat(),
"source_commit": command(["git", "rev-parse", "HEAD"]),
"complete": False,
}
output = Path(args.output)
output.parent.mkdir(parents=True, exist_ok=True)
try:
ready()
initial = snapshot()
apis = [c for c in initial if c["role"] == "api"]
assert len(apis) == 2 and len({c["pid"] for c in apis}) == 2
report["before"] = initial
with httpx.Client(base_url=BASE, trust_env=False, timeout=8) as client:
user, _, session = actor(client)
r = client.post(
"/api/v1/chats",
headers=auth(session),
json={
"title": "Cluster verification",
"member_ids": [user["id"]],
},
)
assert r.status_code == 201
chat = r.json()
path = f"/api/v1/chats/{chat['id']}/messages"
key = uuid4().hex
def repeat(_):
with httpx.Client(
base_url=BASE, trust_env=False, timeout=8
) as c:
r = c.post(
path,
headers={**auth(session), "Idempotency-Key": key},
json={"text": "Same concurrent request"},
)
assert r.status_code == 201
return r.json(), r.headers["x-instance-id"]
with ThreadPoolExecutor(max_workers=10) as pool:
repeated = list(pool.map(repeat, range(30)))
assert len({m[0]["id"] for m in repeated}) == 1
assert len({m[1] for m in repeated}) == 2
report["parallel_replay"] = {
"requests": 30,
"messages": 1,
"instances": sorted({m[1] for m in repeated}),
}
# Сбой второй записи транзакции не должен оставить сообщение.
fault_key = uuid4().hex
fault_text = "Atomic rollback " + fault_key
sql(
"CREATE FUNCTION student_idempo_fault() RETURNS trigger "
"LANGUAGE plpgsql AS $$ BEGIN IF NEW.key='"
+ fault_key
+ "' THEN RAISE EXCEPTION 'synthetic'; END IF; RETURN NEW; END $$; "
"CREATE TRIGGER student_idempo_fault BEFORE INSERT ON idempotency "
"FOR EACH ROW EXECUTE FUNCTION student_idempo_fault()"
)
try:
failed = client.post(
path,
headers={**auth(session), "Idempotency-Key": fault_key},
json={"text": fault_text},
)
assert failed.status_code == 500
assert (
sql(
"SELECT count(*) FROM messages WHERE text='"
+ fault_text
+ "'"
)
== "0"
)
finally:
sql(
"DROP TRIGGER student_idempo_fault ON idempotency; "
"DROP FUNCTION student_idempo_fault()"
)
retried = client.post(
path,
headers={**auth(session), "Idempotency-Key": fault_key},
json={"text": fault_text},
)
assert retried.status_code == 201
report["atomic_rollback_and_5xx_retry"] = True
sql(
"UPDATE idempotency SET expires_at=clock_timestamp()-interval '1 second' "
"WHERE key='" + fault_key + "'"
)
expired = client.post(
path,
headers={**auth(session), "Idempotency-Key": fault_key},
json={"text": "New retention window"},
)
assert expired.status_code == 201
assert expired.json()["id"] != retried.json()["id"]
report["expired_key_can_be_reused"] = True
# Отдельный чат каждой серии: проверки дубликатов не смешивают тексты.
report["failovers"] = []
for api in apis:
def stop(target=api["instance"]):
env = {**os.environ, "TARGET_INSTANCE_ID": target}
subprocess.run(
["scripts/stop-one.sh"],
cwd=ROOT,
env=env,
check=True,
capture_output=True,
text=True,
)
r = client.post(
"/api/v1/chats",
headers=auth(session),
json={"title": "Failover", "member_ids": [user["id"]]},
)
result = exercise(client, session, r.json(), stop)
assert api["instance"] not in [
o["instance"] for o in result["observations"][-5:]
]
assert (
command(
[
"docker",
"inspect",
"-f",
"{{.State.Running}}",
api["container"],
]
)
== "false"
)
result["stopped"] = api
report["failovers"].append(result)
compose("up", "-d", "--no-build", "--wait")
ready()
r = client.post(
"/api/v1/chats",
headers=auth(session),
json={"title": "Rolling", "member_ids": [user["id"]]},
)
before_roll = snapshot()
report["rolling"] = exercise(
client,
session,
r.json(),
lambda: command(["scripts/rolling-update.sh"]),
seconds=45,
)
after_roll = snapshot()
assert not (
{c["container"] for c in before_roll if c["role"] == "api"}
& {c["container"] for c in after_roll if c["role"] == "api"}
)
report["rolling"]["before"] = before_roll
report["rolling"]["after"] = after_roll
replay = repeat(0)[0]
assert replay == repeated[0][0]
report["replay_after_replacement"] = True
before = snapshot()
env = {
**os.environ,
"NO_PROXY": "localhost,127.0.0.1",
"no_proxy": "localhost,127.0.0.1",
}
for mode, hook in [
("failover", "scripts/stop-one.sh"),
("persistence", "scripts/restart.sh"),
]:
if mode == "persistence":
compose("up", "-d", "--no-build", "--wait")
ready()
before = snapshot()
run = subprocess.run(
[
".venv/bin/python",
"tests/resilience.py",
"--mode",
mode,
"--action-script",
hook,
"--base-url",
BASE,
],
cwd=ROOT,
env=env,
capture_output=True,
text=True,
)
assert run.returncode == 0, run.stdout + run.stderr
report["public_" + mode] = run.stdout.strip()
after = snapshot()
for role in ["postgres", "cache", "s3", "broker", "lb"]:
old = next(c for c in before if c["role"] == role)
new = next(c for c in after if c["role"] == role)
assert old["started_at"] != new["started_at"]
report["after_persistence"] = after
report["complete"] = True
finally:
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print("PASS: process failover, rolling, replay and full persistence")
if __name__ == "__main__":
main()