Добавить балансировку API и атомарные повторы сообщений

This commit is contained in:
Горбанюк
2026-10-17 18:41:13 +03:00
parent 0ab3a341aa
commit 534ddd1371
39 changed files with 4113 additions and 1 deletions
+29
View File
@@ -0,0 +1,29 @@
#!/usr/bin/env python3
"""Create local synthetic credentials once, without printing their values."""
import os
import secrets
from pathlib import Path
root = Path(__file__).resolve().parents[1]
directory = root / "secrets"
directory.mkdir(mode=0o700, exist_ok=True)
password_file = directory / "postgres_password"
if not password_file.exists():
password_file.write_text(secrets.token_urlsafe(32) + "\n")
password = password_file.read_text().strip()
for name, value in {
"database_url": f"postgresql://messenger:{password}@postgres:5432/messenger",
"cache_url": "redis://valkey:6379/0",
}.items():
path = directory / name
if not path.exists():
path.write_text(value + "\n")
# Compose bind-mount secrets must be readable by the non-root container process.
# The parent directory remains 0700 on the host.
for name in ("postgres_password", "database_url", "cache_url"):
os.chmod(directory / name, 0o644)
env = root / ".env"
if not env.exists():
env.write_text((root / ".env.example").read_text())
print("Local secret files and environment are ready; existing values preserved.")
+82
View File
@@ -0,0 +1,82 @@
"""Read-only discovery and scoped Compose operations for the local lab3 stack."""
import json
import subprocess
import time
import urllib.error
import urllib.request
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def compose(*args, env=None):
return subprocess.run(
["docker", "compose", "--env-file", ".env", *args],
cwd=ROOT,
env=env,
check=True,
capture_output=True,
text=True,
timeout=90,
).stdout
def inspect(container):
template = (
'{"id":{{json .Id}},"started_at":{{json .State.StartedAt}},'
'"pid":{{.State.Pid}},"running":{{json .State.Running}},'
'"image":{{json .Image}},"memory":{{.HostConfig.Memory}},'
'"nano_cpus":{{.HostConfig.NanoCpus}},'
'"published_ports":{{json .HostConfig.PortBindings}}}'
)
r = subprocess.run(
["docker", "inspect", "--format", template, container],
capture_output=True,
text=True,
check=True,
timeout=5,
)
return json.loads(r.stdout)
def instance(container):
code = (
"import urllib.request; r=urllib.request.urlopen("
"'http://127.0.0.1:8080/health/live',timeout=1); print(r.headers['X-Instance-Id'])"
)
r = subprocess.run(
["docker", "exec", container, "python", "-c", code],
capture_output=True,
text=True,
check=True,
timeout=3,
)
value = inspect(container)
value["instance"] = r.stdout.strip()
return value
def instances():
ids = compose("ps", "-q", "api").split()
with ThreadPoolExecutor(max_workers=8) as pool:
return list(pool.map(instance, ids))
def wait_lb(base="http://127.0.0.1:8083", count=2, timeout=45):
deadline, seen = time.monotonic() + timeout, set()
while time.monotonic() < deadline:
try:
current = {value["instance"] for value in instances()}
if len(current) != count:
time.sleep(0.2)
continue
with urllib.request.urlopen(base + "/health/ready", timeout=2) as response:
seen.add(response.headers["X-Instance-Id"])
if current <= seen:
return sorted(current)
except (OSError, urllib.error.URLError, subprocess.SubprocessError):
pass
time.sleep(0.1)
raise RuntimeError("Expected healthy API replicas were not observed through LB")
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
# All running application/state services; no migration, down -v, flush or seed.
docker compose --env-file .env restart --timeout 7 api postgres valkey lb
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env python3
"""Sequential process replacement; at least one existing API stays running."""
import argparse
import json
import subprocess
from cluster import compose, instances, wait_lb
def replace_all(base="http://127.0.0.1:8083"):
original = instances()
if len(original) < 2:
raise RuntimeError("Rolling replacement requires at least two healthy replicas")
wait_lb(base, len(original))
replacements = []
for current in original:
subprocess.run(
["docker", "stop", "--time", "5", current["id"]],
check=True,
capture_output=True,
timeout=8,
)
subprocess.run(["docker", "rm", current["id"]], check=True, capture_output=True)
compose(
"up", "-d", "--no-deps", "--no-recreate", "--scale", "api=" + str(len(original)), "api"
)
aliases = wait_lb(base, len(original))
if current["instance"] in aliases:
raise RuntimeError("Old API process is still serving after replacement")
replacements.append(
{
"old_container": current["id"],
"old_instance": current["instance"],
"live_instances": aliases,
}
)
return replacements
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8083")
args = parser.parse_args()
print(json.dumps(replace_all(args.base_url)))
if __name__ == "__main__":
main()
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env python3
"""Map a process alias to ONE container belonging to this Compose service."""
import json
import os
import re
import subprocess
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def alias(container):
code = (
"import urllib.request; r=urllib.request.urlopen("
"'http://127.0.0.1:8080/health/live',timeout=1); print(r.headers['X-Instance-Id'])"
)
r = subprocess.run(
["docker", "exec", container, "python", "-c", code],
cwd=ROOT,
capture_output=True,
text=True,
timeout=3,
check=True,
)
return container, r.stdout.strip()
def main():
target = os.environ.get("TARGET_INSTANCE_ID", "")
if not re.fullmatch(r"[a-f0-9]{32}", target):
raise SystemExit("TARGET_INSTANCE_ID must name a live API process")
ids = subprocess.check_output(
["docker", "compose", "ps", "-q", "api"], cwd=ROOT, text=True
).split()
with ThreadPoolExecutor(max_workers=8) as pool:
matches = [cid for cid, identity in pool.map(alias, ids) if identity == target]
if len(matches) != 1:
raise SystemExit("Expected exactly one matching API in this project")
subprocess.run(
["docker", "stop", "--time", "5", matches[0]], check=True, capture_output=True, timeout=7
)
print(json.dumps({"stopped_container": matches[0], "instance": target}))
if __name__ == "__main__":
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/stop-one.py
+253
View File
@@ -0,0 +1,253 @@
#!/usr/bin/env python3
"""Real LB requests during each replica failure and sequential replacement."""
import argparse
import json
import os
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from uuid import uuid4
from cluster import ROOT, compose, inspect, instance, instances, wait_lb
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check # noqa: E402
from resilience import history # noqa: E402
def percentile(values, fraction):
values = sorted(values)
return values[min(int((len(values) - 1) * fraction), len(values) - 1)] if values else None
class Traffic:
def __init__(self, base, actor, room, workers=6):
self.base, self.actor, self.room = base, actor, room
self.prefix = uuid4().hex
self.stop = threading.Event()
self.lock = threading.Lock()
self.events = []
self.executor = ThreadPoolExecutor(max_workers=workers)
self.workers = workers
def worker(self, worker):
client, iteration = Client(self.base, timeout=3), 0
while not self.stop.is_set():
started = time.monotonic()
write = iteration % 10 == 9
try:
if write:
key = f"{self.prefix}:{worker}:{iteration}"
response = client.message(self.actor, self.room, key, key=key)
else:
response = client.call(
"GET", "/api/v1/users/me", actor=self.actor, schema="User"
)
event = {
"at": time.monotonic(),
"ok": True,
"instance": response["headers"]["X-Instance-Id"],
"latency_ms": (time.monotonic() - started) * 1000,
"message": response["json"] if write else None,
}
iteration += 1
except Failure:
event = {
"at": time.monotonic(),
"ok": False,
"latency_ms": (time.monotonic() - started) * 1000,
}
with self.lock:
self.events.append(event)
def __enter__(self):
self.started = time.monotonic()
self.futures = [self.executor.submit(self.worker, i) for i in range(self.workers)]
return self
def __exit__(self, *args):
self.stop.set()
self.executor.shutdown(wait=True)
for future in self.futures:
future.result()
self.finished = time.monotonic()
def report(self, action_start, action_end):
events = sorted(self.events, key=lambda row: row["at"])
ok = [row for row in events if row["ok"]]
after = [row for row in ok if row["at"] >= action_end]
check(bool(after), "No successful request after action")
elapsed = self.finished - self.started
latency = [row["latency_ms"] for row in ok]
return {
"duration_seconds": elapsed,
"workers": self.workers,
"requests": len(events),
"successes": len(ok),
"errors": len(events) - len(ok),
"successful_rps": len(ok) / elapsed,
"error_fraction": (len(events) - len(ok)) / len(events),
"first_success_after_action_seconds": after[0]["at"] - action_start,
"action_seconds": action_end - action_start,
"p50_ms": percentile(latency, 0.5),
"p95_ms": percentile(latency, 0.95),
"p99_ms": percentile(latency, 0.99),
"instances": sorted({row["instance"] for row in ok}),
"tail_instances": sorted({row["instance"] for row in after[-10:]}),
"confirmed_writes": sum(row["message"] is not None for row in ok),
}
def verify_messages(self, client, actor, room):
actual = history(client, actor, room)
by_id = {value["id"]: value for value in actual}
check(len(by_id) == len(actual), "History has duplicate IDs")
texts = [row["text"] for row in actual if row["text"].startswith(self.prefix)]
check(len(texts) == len(set(texts)), "An uncertain retry created a duplicate effect")
for event in self.events:
if event["ok"] and event["message"]:
check(
by_id.get(event["message"]["id"]) == event["message"],
"A confirmed message was lost or changed",
)
def public_test(target, base):
result = subprocess.run(
[
"make",
target,
"ACTION_SCRIPT=scripts/"
+ ("restart.sh" if target == "test-persistence" else "stop-one.sh"),
"BASE_URL=" + base,
],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
timeout=150,
)
return "\n".join(line.rstrip() for line in result.stdout.splitlines()) + "\n"
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8083")
parser.add_argument(
"--output", type=Path, default=ROOT / f"evidence/cluster-{time.time_ns()}.json"
)
args = parser.parse_args()
if args.output.exists():
raise SystemExit("Use a new evidence filename")
os.environ["NO_PROXY"] = os.environ["no_proxy"] = "127.0.0.1,localhost"
baseline = instances()
check(len(baseline) == 2, "This verification expects the default two API replicas")
check(not any(row["published_ports"] for row in baseline), "API must not publish host ports")
wait_lb(args.base_url, 2)
result = {"initial_instances": baseline, "failures": []}
client = Client(args.base_url, timeout=4)
alice, bob, eve, room = client.bootstrap()
key = uuid4().hex
def same_request(_):
response = Client(args.base_url).message(alice, room, "concurrent replay", key=key)
return response["json"], response["headers"]["X-Instance-Id"]
with ThreadPoolExecutor(max_workers=10) as pool:
responses = list(pool.map(same_request, range(30)))
check(
len({row[0]["id"] for row in responses}) == 1, "Cross-replica replay duplicated a message"
)
check(len({row[1] for row in responses}) == 2, "Concurrent replay did not cross real replicas")
result["concurrent_replays"] = 30
result["concurrent_effects"] = 1
try:
for selected in baseline:
selected = instance(selected["id"])
with Traffic(args.base_url, alice, room) as traffic:
time.sleep(1)
action_start = time.monotonic()
env = {**os.environ, "TARGET_INSTANCE_ID": selected["instance"]}
subprocess.run(
[str(ROOT / "scripts/stop-one.sh")],
cwd=ROOT,
env=env,
check=True,
capture_output=True,
timeout=10,
)
action_end = time.monotonic()
check(not inspect(selected["id"])["running"], "Chosen process was not stopped")
time.sleep(3)
report = traffic.report(action_start, action_end)
check(report["first_success_after_action_seconds"] <= 10, "Failover took too long")
check(selected["instance"] not in report["tail_instances"], "Stopped API still serves")
report["stopped"] = selected
traffic.verify_messages(client, bob, room)
result["failures"].append(report)
subprocess.run(["docker", "start", selected["id"]], check=True, capture_output=True)
wait_lb(args.base_url, 2)
print("Verified failure and restoration of one API", flush=True)
with Traffic(args.base_url, alice, room) as traffic:
time.sleep(1)
action_start = time.monotonic()
replaced = subprocess.run(
[
sys.executable,
str(ROOT / "scripts/rolling-update.py"),
"--base-url",
args.base_url,
],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
timeout=120,
)
action_end = time.monotonic()
time.sleep(3)
traffic.verify_messages(client, bob, room)
result["rolling"] = traffic.report(action_start, action_end)
result["rolling"]["replacements"] = json.loads(replaced.stdout)
result["rolling"]["confirmed_messages_preserved"] = True
print("Verified sequential replacement under traffic", flush=True)
(ROOT / "evidence/failover.txt").write_text(public_test("test-failover", args.base_url))
for value in compose("ps", "-aq", "api").split():
if not inspect(value)["running"]:
subprocess.run(["docker", "start", value], check=True, capture_output=True)
wait_lb(args.base_url, 2)
before = {
service: inspect(compose("ps", "-q", service).strip())
for service in ("postgres", "valkey", "lb")
}
(ROOT / "evidence/persistence.txt").write_text(
public_test("test-persistence", args.base_url)
)
after = {service: inspect(compose("ps", "-q", service).strip()) for service in before}
check(
all(before[s]["started_at"] != after[s]["started_at"] for s in before),
"Persistence hook did not restart all state/LB services",
)
result["persistence"] = {"before": before, "after": after, "same_token_public_test": True}
result["final_instances"] = instances()
logs = compose("logs", "--no-color", "api", "lb")
check(
not any(
a["token"] in logs or a["registration"]["password"] in logs
for a in (alice, bob, eve)
),
"Credential appeared in logs",
)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
print("PASS: real replica failures, replay, rolling replacement and persistence")
finally:
for value in compose("ps", "-aq", "api").split():
if not inspect(value)["running"]:
subprocess.run(["docker", "start", value], check=True, capture_output=True)
if __name__ == "__main__":
main()