feat: add replicated API with transactional message idempotency
This commit is contained in:
@@ -0,0 +1,47 @@
|
||||
"""Создаёт только отсутствующие локальные файлы; секреты не печатает."""
|
||||
|
||||
import os
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def create(path, value):
|
||||
try:
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
|
||||
except FileExistsError:
|
||||
return
|
||||
with os.fdopen(fd, "w") as stream:
|
||||
stream.write(value + "\n")
|
||||
|
||||
|
||||
def main():
|
||||
directory = ROOT / "secrets"
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
pg = directory / "postgres_password"
|
||||
cache = directory / "cache_password"
|
||||
create(pg, secrets.token_urlsafe(32))
|
||||
create(cache, secrets.token_urlsafe(32))
|
||||
create(
|
||||
directory / "database_url",
|
||||
"postgresql://messenger:"
|
||||
+ pg.read_text().strip()
|
||||
+ "@postgres:5432/messenger",
|
||||
)
|
||||
create(
|
||||
directory / "cache_url",
|
||||
"redis://:" + cache.read_text().strip() + "@cache:6379/0",
|
||||
)
|
||||
create(
|
||||
directory / "valkey.conf",
|
||||
"bind 0.0.0.0\nprotected-mode yes\nport 6379\nmaxmemory 64mb\n"
|
||||
'maxmemory-policy allkeys-lru\nsave ""\nappendonly no\nrequirepass '
|
||||
+ cache.read_text().strip(),
|
||||
)
|
||||
create(ROOT / ".env", (ROOT / ".env.example").read_text().rstrip())
|
||||
print("Local configuration ready; existing files preserved")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Явные операторские действия только над API текущего Compose-проекта."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def command(*args):
|
||||
return subprocess.check_output(args, cwd=ROOT, text=True).strip()
|
||||
|
||||
|
||||
def containers():
|
||||
ids = command("docker", "compose", "ps", "-q", "api").split()
|
||||
return json.loads(command("docker", "inspect", *ids)) if ids else []
|
||||
|
||||
|
||||
def alias(cid):
|
||||
code = (
|
||||
"import urllib.request; r=urllib.request.urlopen("
|
||||
"'http://127.0.0.1:8080/health/live',timeout=2); "
|
||||
"print(r.headers['X-Instance-Id'])"
|
||||
)
|
||||
return command("docker", "exec", cid, "python", "-c", code)
|
||||
|
||||
|
||||
def wait_count(count):
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
data = containers()
|
||||
if len(data) == count and all(
|
||||
c["State"].get("Health", {}).get("Status") == "healthy"
|
||||
for c in data
|
||||
):
|
||||
# DNS и проверки HAProxy должны успеть увидеть новый адрес.
|
||||
time.sleep(3)
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise TimeoutError("Replica readiness")
|
||||
|
||||
|
||||
def main():
|
||||
action = sys.argv[1]
|
||||
data = containers()
|
||||
if action == "stop":
|
||||
target = os.environ["TARGET_INSTANCE_ID"]
|
||||
matches = [c for c in data if alias(c["Id"]) == target]
|
||||
if len(matches) != 1 or len(data) < 2:
|
||||
raise RuntimeError("Target must identify one running replica")
|
||||
command("docker", "stop", "--time", "10", matches[0]["Id"])
|
||||
print("Selected API stopped")
|
||||
elif action == "rolling":
|
||||
count = len(data)
|
||||
if count < 2:
|
||||
raise RuntimeError("Need at least two replicas")
|
||||
# Сначала добавляется готовая реплика, затем удаляется одна старая.
|
||||
for old in data:
|
||||
command(
|
||||
"docker",
|
||||
"compose",
|
||||
"up",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--no-recreate",
|
||||
"--scale",
|
||||
f"api={count + 1}",
|
||||
"api",
|
||||
)
|
||||
wait_count(count + 1)
|
||||
command("docker", "stop", "--time", "10", old["Id"])
|
||||
command("docker", "rm", old["Id"])
|
||||
wait_count(count)
|
||||
print("Sequential replacement completed")
|
||||
else:
|
||||
raise ValueError("Unknown action")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
docker compose restart api postgres cache lb
|
||||
docker compose up -d --no-build --no-recreate --wait
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
exec python3 scripts/cluster.py rolling
|
||||
@@ -0,0 +1,137 @@
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8083")
|
||||
|
||||
|
||||
def command(args, *, input=None):
|
||||
result = subprocess.run(
|
||||
args, cwd=ROOT, input=input, capture_output=True, text=True
|
||||
)
|
||||
if result.returncode:
|
||||
raise RuntimeError(
|
||||
"Command failed: "
|
||||
+ args[0]
|
||||
+ "; "
|
||||
+ result.stderr.replace(str(ROOT), "<repository>")
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return command(["docker", "compose", *args])
|
||||
|
||||
|
||||
def sql(query, *, database="messenger"):
|
||||
return compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"psql",
|
||||
"-U",
|
||||
"messenger",
|
||||
"-d",
|
||||
database,
|
||||
"-A",
|
||||
"-t",
|
||||
"-v",
|
||||
"ON_ERROR_STOP=1",
|
||||
"-c",
|
||||
query,
|
||||
)
|
||||
|
||||
|
||||
def cache(command_name, *args):
|
||||
script = (
|
||||
'export REDISCLI_AUTH="$(cat /run/secrets/cache_password)"; '
|
||||
'exec valkey-cli "$@"'
|
||||
)
|
||||
return compose(
|
||||
"exec", "-T", "cache", "sh", "-c", script, "sh", command_name, *args
|
||||
)
|
||||
|
||||
|
||||
def ready(base=BASE, *, timeout=30):
|
||||
end = time.monotonic() + timeout
|
||||
with httpx.Client(base_url=base, trust_env=False, timeout=6) as client:
|
||||
while time.monotonic() < end:
|
||||
try:
|
||||
if client.get("/health/ready").status_code == 200:
|
||||
return
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
time.sleep(0.2)
|
||||
raise TimeoutError("API readiness")
|
||||
|
||||
|
||||
def actor(client):
|
||||
username = "verify_" + uuid4().hex[:20]
|
||||
password = "Synthetic-" + uuid4().hex + ":🌍"
|
||||
result = client.post(
|
||||
"/api/v1/users",
|
||||
json={
|
||||
"username": username,
|
||||
"display_name": "Synthetic",
|
||||
"password": password,
|
||||
},
|
||||
)
|
||||
assert result.status_code == 201
|
||||
user = result.json()
|
||||
session = client.post(
|
||||
"/api/v1/auth/sessions", auth=httpx.BasicAuth(username, password)
|
||||
)
|
||||
assert session.status_code == 201
|
||||
return user, password, session.json()
|
||||
|
||||
|
||||
def auth(session):
|
||||
return {"Authorization": "Bearer " + session["token"]}
|
||||
|
||||
|
||||
def extra_api(*, ttl=3600, pool_size=1):
|
||||
name = "mtusi-lab3-test-" + uuid4().hex[:10]
|
||||
compose(
|
||||
"run",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--name",
|
||||
name,
|
||||
"-e",
|
||||
"SESSION_TTL_SECONDS=" + str(ttl),
|
||||
"-e",
|
||||
"DB_POOL_SIZE=" + str(pool_size),
|
||||
"-p",
|
||||
"127.0.0.1::8080",
|
||||
"api",
|
||||
)
|
||||
try:
|
||||
info = json.loads(command(["docker", "inspect", name]))[0]
|
||||
base = (
|
||||
"http://127.0.0.1:"
|
||||
+ info["NetworkSettings"]["Ports"]["8080/tcp"][0]["HostPort"]
|
||||
)
|
||||
ready(base)
|
||||
return name, base
|
||||
except Exception:
|
||||
command(["docker", "rm", "-f", name])
|
||||
raise
|
||||
|
||||
|
||||
def instances():
|
||||
data = {}
|
||||
for role in ["api", "postgres", "cache"]:
|
||||
cid = compose("ps", "-q", role)
|
||||
info = json.loads(command(["docker", "inspect", cid]))[0]
|
||||
data[role] = {
|
||||
"container": cid[:12],
|
||||
"pid": info["State"]["Pid"],
|
||||
"started_at": info["State"]["StartedAt"],
|
||||
}
|
||||
return data
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
exec python3 scripts/cluster.py stop
|
||||
@@ -0,0 +1,292 @@
|
||||
"""Проверки реальных процессов, failover, rolling и общей идемпотентности."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
|
||||
from scripts import cluster
|
||||
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
|
||||
|
||||
|
||||
def snapshot():
|
||||
ids = compose("ps", "-q").split()
|
||||
result = []
|
||||
for c in json.loads(command(["docker", "inspect", *ids])):
|
||||
role = c["Config"]["Labels"]["com.docker.compose.service"]
|
||||
entry = {
|
||||
"role": role,
|
||||
"container": c["Id"][:12],
|
||||
"pid": c["State"]["Pid"],
|
||||
"started_at": c["State"]["StartedAt"],
|
||||
"ports": c["HostConfig"]["PortBindings"],
|
||||
"cpu_nano": c["HostConfig"]["NanoCpus"],
|
||||
"memory_limit": c["HostConfig"]["Memory"],
|
||||
}
|
||||
if role == "api":
|
||||
entry["instance"] = cluster.alias(c["Id"])
|
||||
assert not entry["ports"]
|
||||
result.append(entry)
|
||||
return result
|
||||
|
||||
|
||||
def exercise(client, session, chat, action, *, seconds=15):
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
observations, confirmed, errors, reads = [], {}, 0, 0
|
||||
started = time.monotonic()
|
||||
finished = threading.Event()
|
||||
action_result = {}
|
||||
|
||||
def operate():
|
||||
time.sleep(0.5)
|
||||
try:
|
||||
action()
|
||||
except BaseException as exc:
|
||||
action_result["error"] = type(exc).__name__
|
||||
finally:
|
||||
action_result["finished"] = round(time.monotonic() - started, 3)
|
||||
finished.set()
|
||||
|
||||
thread = threading.Thread(target=operate)
|
||||
thread.start()
|
||||
key = uuid4().hex
|
||||
try:
|
||||
while time.monotonic() - started < seconds or not finished.is_set():
|
||||
h = {**auth(session), "Idempotency-Key": key}
|
||||
try:
|
||||
r = client.post(path, headers=h, json={"text": key})
|
||||
if r.status_code != 201:
|
||||
errors += 1
|
||||
continue
|
||||
m = r.json()
|
||||
confirmed[m["id"]] = m
|
||||
observations.append(
|
||||
{
|
||||
"at": round(time.monotonic() - started, 3),
|
||||
"instance": r.headers["x-instance-id"],
|
||||
}
|
||||
)
|
||||
key = uuid4().hex
|
||||
read = client.get(
|
||||
path, headers=auth(session), params={"limit": 100}
|
||||
)
|
||||
reads += read.status_code == 200
|
||||
errors += read.status_code != 200
|
||||
except httpx.HTTPError:
|
||||
errors += 1
|
||||
time.sleep(0.04)
|
||||
finally:
|
||||
thread.join(timeout=90)
|
||||
assert not action_result.get("error"), action_result
|
||||
found, cursor = [], None
|
||||
while True:
|
||||
r = client.get(
|
||||
path,
|
||||
headers=auth(session),
|
||||
params={"limit": 100, **({"cursor": cursor} if cursor else {})},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
page = r.json()
|
||||
found.extend(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if not cursor:
|
||||
break
|
||||
by_id = {m["id"]: m for m in found}
|
||||
assert all(by_id.get(k) == v for k, v in confirmed.items())
|
||||
texts = [m["text"] for m in found]
|
||||
assert len(texts) == len(set(texts))
|
||||
recovery = next(
|
||||
(
|
||||
x["at"]
|
||||
for x in observations
|
||||
if x["at"] >= action_result["finished"]
|
||||
),
|
||||
None,
|
||||
)
|
||||
assert recovery is not None and (seconds > 20 or recovery <= 10)
|
||||
elapsed = time.monotonic() - started
|
||||
return {
|
||||
"confirmed_writes": len(confirmed),
|
||||
"successful_reads": reads,
|
||||
"errors": errors,
|
||||
"successful_rps": round((len(confirmed) + reads) / elapsed, 2),
|
||||
"first_success_after_action_seconds": recovery,
|
||||
"action": action_result,
|
||||
"observations": observations,
|
||||
"confirmed_preserved": True,
|
||||
"duplicate_texts": False,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", default="evidence/cluster-repeat.json")
|
||||
args = parser.parse_args()
|
||||
report = {
|
||||
"date": datetime.now(timezone.utc).isoformat(),
|
||||
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
||||
"complete": False,
|
||||
}
|
||||
output = Path(args.output)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
ready()
|
||||
initial = snapshot()
|
||||
apis = [c for c in initial if c["role"] == "api"]
|
||||
assert len(apis) == 2 and len({c["pid"] for c in apis}) == 2
|
||||
report["before"] = initial
|
||||
with httpx.Client(base_url=BASE, trust_env=False, timeout=8) as client:
|
||||
user, _, session = actor(client)
|
||||
r = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=auth(session),
|
||||
json={
|
||||
"title": "Cluster verification",
|
||||
"member_ids": [user["id"]],
|
||||
},
|
||||
)
|
||||
assert r.status_code == 201
|
||||
chat = r.json()
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
key = uuid4().hex
|
||||
|
||||
def repeat(_):
|
||||
with httpx.Client(
|
||||
base_url=BASE, trust_env=False, timeout=8
|
||||
) as c:
|
||||
r = c.post(
|
||||
path,
|
||||
headers={**auth(session), "Idempotency-Key": key},
|
||||
json={"text": "Same concurrent request"},
|
||||
)
|
||||
assert r.status_code == 201
|
||||
return r.json(), r.headers["x-instance-id"]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
repeated = list(pool.map(repeat, range(30)))
|
||||
assert len({m[0]["id"] for m in repeated}) == 1
|
||||
assert len({m[1] for m in repeated}) == 2
|
||||
report["parallel_replay"] = {
|
||||
"requests": 30,
|
||||
"messages": 1,
|
||||
"instances": sorted({m[1] for m in repeated}),
|
||||
}
|
||||
# Отдельный чат каждой серии: проверки дубликатов не смешивают тексты.
|
||||
report["failovers"] = []
|
||||
for api in apis:
|
||||
|
||||
def stop(target=api["instance"]):
|
||||
env = {**os.environ, "TARGET_INSTANCE_ID": target}
|
||||
subprocess.run(
|
||||
["scripts/stop-one.sh"],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
r = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=auth(session),
|
||||
json={"title": "Failover", "member_ids": [user["id"]]},
|
||||
)
|
||||
result = exercise(client, session, r.json(), stop)
|
||||
assert api["instance"] not in [
|
||||
o["instance"] for o in result["observations"][-5:]
|
||||
]
|
||||
assert (
|
||||
command(
|
||||
[
|
||||
"docker",
|
||||
"inspect",
|
||||
"-f",
|
||||
"{{.State.Running}}",
|
||||
api["container"],
|
||||
]
|
||||
)
|
||||
== "false"
|
||||
)
|
||||
result["stopped"] = api
|
||||
report["failovers"].append(result)
|
||||
compose("up", "-d", "--no-build", "--wait")
|
||||
ready()
|
||||
r = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=auth(session),
|
||||
json={"title": "Rolling", "member_ids": [user["id"]]},
|
||||
)
|
||||
before_roll = snapshot()
|
||||
report["rolling"] = exercise(
|
||||
client,
|
||||
session,
|
||||
r.json(),
|
||||
lambda: command(["scripts/rolling-update.sh"]),
|
||||
seconds=45,
|
||||
)
|
||||
after_roll = snapshot()
|
||||
assert not (
|
||||
{c["container"] for c in before_roll if c["role"] == "api"}
|
||||
& {c["container"] for c in after_roll if c["role"] == "api"}
|
||||
)
|
||||
report["rolling"]["before"] = before_roll
|
||||
report["rolling"]["after"] = after_roll
|
||||
replay = repeat(0)[0]
|
||||
assert replay == repeated[0][0]
|
||||
report["replay_after_replacement"] = True
|
||||
before = snapshot()
|
||||
env = {
|
||||
**os.environ,
|
||||
"NO_PROXY": "localhost,127.0.0.1",
|
||||
"no_proxy": "localhost,127.0.0.1",
|
||||
}
|
||||
for mode, hook in [
|
||||
("failover", "scripts/stop-one.sh"),
|
||||
("persistence", "scripts/restart.sh"),
|
||||
]:
|
||||
if mode == "persistence":
|
||||
compose("up", "-d", "--no-build", "--wait")
|
||||
ready()
|
||||
before = snapshot()
|
||||
run = subprocess.run(
|
||||
[
|
||||
".venv/bin/python",
|
||||
"tests/resilience.py",
|
||||
"--mode",
|
||||
mode,
|
||||
"--action-script",
|
||||
hook,
|
||||
"--base-url",
|
||||
BASE,
|
||||
],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert run.returncode == 0, run.stdout + run.stderr
|
||||
report["public_" + mode] = run.stdout.strip()
|
||||
after = snapshot()
|
||||
for role in ["postgres", "cache", "lb"]:
|
||||
old = next(c for c in before if c["role"] == role)
|
||||
new = next(c for c in after if c["role"] == role)
|
||||
assert old["started_at"] != new["started_at"]
|
||||
report["after_persistence"] = after
|
||||
report["complete"] = True
|
||||
finally:
|
||||
output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print("PASS: process failover, rolling, replay and full persistence")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user