348 lines
12 KiB
Python
348 lines
12 KiB
Python
"""Проверки реальных процессов, failover, rolling и общей идемпотентности."""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
import httpx
|
|
|
|
from scripts import cluster
|
|
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
|
|
|
|
|
|
def snapshot():
|
|
ids = compose("ps", "-q").split()
|
|
result = []
|
|
for c in json.loads(command(["docker", "inspect", *ids])):
|
|
role = c["Config"]["Labels"]["com.docker.compose.service"]
|
|
entry = {
|
|
"role": role,
|
|
"container": c["Id"][:12],
|
|
"pid": c["State"]["Pid"],
|
|
"started_at": c["State"]["StartedAt"],
|
|
"ports": c["HostConfig"]["PortBindings"],
|
|
"cpu_nano": c["HostConfig"]["NanoCpus"],
|
|
"memory_limit": c["HostConfig"]["Memory"],
|
|
}
|
|
if role == "api":
|
|
entry["instance"] = cluster.alias(c["Id"])
|
|
assert not entry["ports"]
|
|
result.append(entry)
|
|
return result
|
|
|
|
|
|
def exercise(client, session, chat, action, *, seconds=15):
|
|
path = f"/api/v1/chats/{chat['id']}/messages"
|
|
observations, confirmed, errors, reads = [], {}, 0, 0
|
|
started = time.monotonic()
|
|
finished = threading.Event()
|
|
action_result = {}
|
|
|
|
def operate():
|
|
time.sleep(0.5)
|
|
try:
|
|
action()
|
|
except BaseException as exc:
|
|
action_result["error"] = type(exc).__name__
|
|
finally:
|
|
action_result["finished"] = round(time.monotonic() - started, 3)
|
|
finished.set()
|
|
|
|
thread = threading.Thread(target=operate)
|
|
thread.start()
|
|
key = uuid4().hex
|
|
try:
|
|
while (
|
|
time.monotonic() - started < seconds
|
|
or not finished.is_set()
|
|
or time.monotonic() - started
|
|
< action_result.get("finished", 0) + 2
|
|
):
|
|
h = {**auth(session), "Idempotency-Key": key}
|
|
try:
|
|
r = client.post(path, headers=h, json={"text": key})
|
|
if r.status_code != 201:
|
|
errors += 1
|
|
continue
|
|
m = r.json()
|
|
confirmed[m["id"]] = m
|
|
observations.append(
|
|
{
|
|
"at": round(time.monotonic() - started, 3),
|
|
"instance": r.headers["x-instance-id"],
|
|
}
|
|
)
|
|
key = uuid4().hex
|
|
read = client.get(
|
|
path, headers=auth(session), params={"limit": 100}
|
|
)
|
|
reads += read.status_code == 200
|
|
errors += read.status_code != 200
|
|
except httpx.HTTPError:
|
|
errors += 1
|
|
time.sleep(0.04)
|
|
finally:
|
|
thread.join(timeout=90)
|
|
assert not action_result.get("error"), action_result
|
|
found, cursor = [], None
|
|
while True:
|
|
r = client.get(
|
|
path,
|
|
headers=auth(session),
|
|
params={"limit": 100, **({"cursor": cursor} if cursor else {})},
|
|
)
|
|
assert r.status_code == 200
|
|
page = r.json()
|
|
found.extend(page["items"])
|
|
cursor = page["next_cursor"]
|
|
if not cursor:
|
|
break
|
|
by_id = {m["id"]: m for m in found}
|
|
assert all(by_id.get(k) == v for k, v in confirmed.items())
|
|
texts = [m["text"] for m in found]
|
|
assert len(texts) == len(set(texts))
|
|
recovery = next(
|
|
(
|
|
x["at"]
|
|
for x in observations
|
|
if x["at"] >= action_result["finished"]
|
|
),
|
|
None,
|
|
)
|
|
assert recovery is not None and (seconds > 20 or recovery <= 10)
|
|
elapsed = time.monotonic() - started
|
|
return {
|
|
"confirmed_writes": len(confirmed),
|
|
"successful_reads": reads,
|
|
"errors": errors,
|
|
"successful_rps": round((len(confirmed) + reads) / elapsed, 2),
|
|
"first_success_after_action_seconds": recovery,
|
|
"action": action_result,
|
|
"observations": observations,
|
|
"confirmed_preserved": True,
|
|
"duplicate_texts": False,
|
|
}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--output", default="evidence/cluster-repeat.json")
|
|
args = parser.parse_args()
|
|
report = {
|
|
"date": datetime.now(timezone.utc).isoformat(),
|
|
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
|
"complete": False,
|
|
}
|
|
output = Path(args.output)
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
ready()
|
|
initial = snapshot()
|
|
apis = [c for c in initial if c["role"] == "api"]
|
|
assert len(apis) == 2 and len({c["pid"] for c in apis}) == 2
|
|
report["before"] = initial
|
|
with httpx.Client(base_url=BASE, trust_env=False, timeout=8) as client:
|
|
user, _, session = actor(client)
|
|
r = client.post(
|
|
"/api/v1/chats",
|
|
headers=auth(session),
|
|
json={
|
|
"title": "Cluster verification",
|
|
"member_ids": [user["id"]],
|
|
},
|
|
)
|
|
assert r.status_code == 201
|
|
chat = r.json()
|
|
path = f"/api/v1/chats/{chat['id']}/messages"
|
|
key = uuid4().hex
|
|
|
|
def repeat(_):
|
|
with httpx.Client(
|
|
base_url=BASE, trust_env=False, timeout=8
|
|
) as c:
|
|
r = c.post(
|
|
path,
|
|
headers={**auth(session), "Idempotency-Key": key},
|
|
json={"text": "Same concurrent request"},
|
|
)
|
|
assert r.status_code == 201
|
|
return r.json(), r.headers["x-instance-id"]
|
|
|
|
with ThreadPoolExecutor(max_workers=10) as pool:
|
|
repeated = list(pool.map(repeat, range(30)))
|
|
assert len({m[0]["id"] for m in repeated}) == 1
|
|
assert len({m[1] for m in repeated}) == 2
|
|
report["parallel_replay"] = {
|
|
"requests": 30,
|
|
"messages": 1,
|
|
"instances": sorted({m[1] for m in repeated}),
|
|
}
|
|
# Сбой второй записи транзакции не должен оставить сообщение.
|
|
fault_key = uuid4().hex
|
|
fault_text = "Atomic rollback " + fault_key
|
|
sql(
|
|
"CREATE FUNCTION student_idempo_fault() RETURNS trigger "
|
|
"LANGUAGE plpgsql AS $$ BEGIN IF NEW.key='"
|
|
+ fault_key
|
|
+ "' THEN RAISE EXCEPTION 'synthetic'; END IF; RETURN NEW; END $$; "
|
|
"CREATE TRIGGER student_idempo_fault BEFORE INSERT ON idempotency "
|
|
"FOR EACH ROW EXECUTE FUNCTION student_idempo_fault()"
|
|
)
|
|
try:
|
|
failed = client.post(
|
|
path,
|
|
headers={**auth(session), "Idempotency-Key": fault_key},
|
|
json={"text": fault_text},
|
|
)
|
|
assert failed.status_code == 500
|
|
assert (
|
|
sql(
|
|
"SELECT count(*) FROM messages WHERE text='"
|
|
+ fault_text
|
|
+ "'"
|
|
)
|
|
== "0"
|
|
)
|
|
finally:
|
|
sql(
|
|
"DROP TRIGGER student_idempo_fault ON idempotency; "
|
|
"DROP FUNCTION student_idempo_fault()"
|
|
)
|
|
retried = client.post(
|
|
path,
|
|
headers={**auth(session), "Idempotency-Key": fault_key},
|
|
json={"text": fault_text},
|
|
)
|
|
assert retried.status_code == 201
|
|
report["atomic_rollback_and_5xx_retry"] = True
|
|
sql(
|
|
"UPDATE idempotency SET expires_at=clock_timestamp()-interval '1 second' "
|
|
"WHERE key='" + fault_key + "'"
|
|
)
|
|
expired = client.post(
|
|
path,
|
|
headers={**auth(session), "Idempotency-Key": fault_key},
|
|
json={"text": "New retention window"},
|
|
)
|
|
assert expired.status_code == 201
|
|
assert expired.json()["id"] != retried.json()["id"]
|
|
report["expired_key_can_be_reused"] = True
|
|
# Отдельный чат каждой серии: проверки дубликатов не смешивают тексты.
|
|
report["failovers"] = []
|
|
for api in apis:
|
|
|
|
def stop(target=api["instance"]):
|
|
env = {**os.environ, "TARGET_INSTANCE_ID": target}
|
|
subprocess.run(
|
|
["scripts/stop-one.sh"],
|
|
cwd=ROOT,
|
|
env=env,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
r = client.post(
|
|
"/api/v1/chats",
|
|
headers=auth(session),
|
|
json={"title": "Failover", "member_ids": [user["id"]]},
|
|
)
|
|
result = exercise(client, session, r.json(), stop)
|
|
assert api["instance"] not in [
|
|
o["instance"] for o in result["observations"][-5:]
|
|
]
|
|
assert (
|
|
command(
|
|
[
|
|
"docker",
|
|
"inspect",
|
|
"-f",
|
|
"{{.State.Running}}",
|
|
api["container"],
|
|
]
|
|
)
|
|
== "false"
|
|
)
|
|
result["stopped"] = api
|
|
report["failovers"].append(result)
|
|
compose("up", "-d", "--no-build", "--wait")
|
|
ready()
|
|
r = client.post(
|
|
"/api/v1/chats",
|
|
headers=auth(session),
|
|
json={"title": "Rolling", "member_ids": [user["id"]]},
|
|
)
|
|
before_roll = snapshot()
|
|
report["rolling"] = exercise(
|
|
client,
|
|
session,
|
|
r.json(),
|
|
lambda: command(["scripts/rolling-update.sh"]),
|
|
seconds=45,
|
|
)
|
|
after_roll = snapshot()
|
|
assert not (
|
|
{c["container"] for c in before_roll if c["role"] == "api"}
|
|
& {c["container"] for c in after_roll if c["role"] == "api"}
|
|
)
|
|
report["rolling"]["before"] = before_roll
|
|
report["rolling"]["after"] = after_roll
|
|
replay = repeat(0)[0]
|
|
assert replay == repeated[0][0]
|
|
report["replay_after_replacement"] = True
|
|
before = snapshot()
|
|
env = {
|
|
**os.environ,
|
|
"NO_PROXY": "localhost,127.0.0.1",
|
|
"no_proxy": "localhost,127.0.0.1",
|
|
}
|
|
for mode, hook in [
|
|
("failover", "scripts/stop-one.sh"),
|
|
("persistence", "scripts/restart.sh"),
|
|
]:
|
|
if mode == "persistence":
|
|
compose("up", "-d", "--no-build", "--wait")
|
|
ready()
|
|
before = snapshot()
|
|
run = subprocess.run(
|
|
[
|
|
".venv/bin/python",
|
|
"tests/resilience.py",
|
|
"--mode",
|
|
mode,
|
|
"--action-script",
|
|
hook,
|
|
"--base-url",
|
|
BASE,
|
|
],
|
|
cwd=ROOT,
|
|
env=env,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert run.returncode == 0, run.stdout + run.stderr
|
|
report["public_" + mode] = run.stdout.strip()
|
|
after = snapshot()
|
|
for role in ["postgres", "cache", "s3", "broker", "lb"]:
|
|
old = next(c for c in before if c["role"] == role)
|
|
new = next(c for c in after if c["role"] == role)
|
|
assert old["started_at"] != new["started_at"]
|
|
report["after_persistence"] = after
|
|
report["complete"] = True
|
|
finally:
|
|
output.write_text(
|
|
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
|
)
|
|
print("PASS: process failover, rolling, replay and full persistence")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|