101 lines
3.9 KiB
Python
Executable File
101 lines
3.9 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Проверяет поочерёдную замену двух API-реплик под записью через LB."""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from urllib.parse import quote
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
|
from client import Client, Failure, arguments, check, run # noqa: E402
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def history(client: Client, actor: dict, chat: dict) -> list[dict]:
|
|
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100"
|
|
messages, cursors = [], set()
|
|
while True:
|
|
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
|
|
messages.extend(page["items"])
|
|
cursor = page["next_cursor"]
|
|
if cursor is None:
|
|
return messages
|
|
check(cursor not in cursors, "history cursor cycle after rolling update")
|
|
cursors.add(cursor)
|
|
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100&cursor=" + quote(cursor, safe="")
|
|
|
|
|
|
def main() -> None:
|
|
args = arguments("Sequential API replacement under writes")
|
|
action = ROOT / "scripts" / "rolling-update.sh"
|
|
check(action.is_file() and os.access(action, os.X_OK), "rolling-update.sh must be executable")
|
|
client = Client(args.base_url, args.ca_file, timeout=2)
|
|
alice, _bob, _eve, chat = client.bootstrap()
|
|
|
|
before = {
|
|
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"]["X-Instance-Id"]
|
|
for _ in range(40)
|
|
}
|
|
check(len(before) >= 2, "less than two API instances observed before rolling update")
|
|
|
|
process = subprocess.Popen([str(action)], cwd=ROOT)
|
|
started = time.monotonic()
|
|
latencies, failures, successful = [], 0, 0
|
|
confirmed_ids = []
|
|
try:
|
|
while process.poll() is None:
|
|
check(time.monotonic() - started < 90, "rolling update hook exceeded 90 seconds")
|
|
key = uuid.uuid4().hex
|
|
began = time.monotonic()
|
|
try:
|
|
message = client.message(alice, chat, "rolling-" + key, key=key)["json"]
|
|
successful += 1
|
|
confirmed_ids.append(message["id"])
|
|
latencies.append(time.monotonic() - began)
|
|
except Failure:
|
|
failures += 1
|
|
time.sleep(0.02)
|
|
check(process.wait(timeout=5) == 0, "rolling update hook failed")
|
|
finally:
|
|
if process.poll() is None:
|
|
process.terminate()
|
|
process.wait(timeout=5)
|
|
|
|
after = {
|
|
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"]["X-Instance-Id"]
|
|
for _ in range(40)
|
|
}
|
|
check(len(after) >= 2, "less than two API instances observed after rolling update")
|
|
check(before.isdisjoint(after), "old API instance remained after sequential replacement")
|
|
check(successful > 0, "no writes succeeded during rolling update")
|
|
stored_ids = {message["id"] for message in history(client, alice, chat)}
|
|
check(all(message_id in stored_ids for message_id in confirmed_ids), "confirmed write lost during rolling update")
|
|
|
|
elapsed = time.monotonic() - started
|
|
ordered = sorted(latencies)
|
|
def percentile(fraction: float) -> float | None:
|
|
if not ordered:
|
|
return None
|
|
return round(1000 * ordered[max(0, int(len(ordered) * fraction + 0.999999) - 1)], 3)
|
|
|
|
print(json.dumps({
|
|
"instances_before": len(before),
|
|
"instances_after": len(after),
|
|
"successful_writes": successful,
|
|
"failed_writes": failures,
|
|
"elapsed_seconds": round(elapsed, 3),
|
|
"successful_rps": round(successful / elapsed, 3),
|
|
"successful_latency_ms": {"p50": percentile(0.50), "p95": percentile(0.95), "p99": percentile(0.99)},
|
|
}, ensure_ascii=False))
|
|
print("PASS: sequential API replacement preserved writes through load balancer")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|