forked from mtusi2026/mtusi2026lab3
Добавить балансировку API и атомарные повторы сообщений
This commit is contained in:
@@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create local synthetic credentials once, without printing their values."""
|
||||
|
||||
import os
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
directory = root / "secrets"
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
password_file = directory / "postgres_password"
|
||||
if not password_file.exists():
|
||||
password_file.write_text(secrets.token_urlsafe(32) + "\n")
|
||||
password = password_file.read_text().strip()
|
||||
for name, value in {
|
||||
"database_url": f"postgresql://messenger:{password}@postgres:5432/messenger",
|
||||
"cache_url": "redis://valkey:6379/0",
|
||||
}.items():
|
||||
path = directory / name
|
||||
if not path.exists():
|
||||
path.write_text(value + "\n")
|
||||
# Compose bind-mount secrets must be readable by the non-root container process.
|
||||
# The parent directory remains 0700 on the host.
|
||||
for name in ("postgres_password", "database_url", "cache_url"):
|
||||
os.chmod(directory / name, 0o644)
|
||||
env = root / ".env"
|
||||
if not env.exists():
|
||||
env.write_text((root / ".env.example").read_text())
|
||||
print("Local secret files and environment are ready; existing values preserved.")
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Read-only discovery and scoped Compose operations for the local lab3 stack."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def compose(*args, env=None):
|
||||
return subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", *args],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=90,
|
||||
).stdout
|
||||
|
||||
|
||||
def inspect(container):
|
||||
template = (
|
||||
'{"id":{{json .Id}},"started_at":{{json .State.StartedAt}},'
|
||||
'"pid":{{.State.Pid}},"running":{{json .State.Running}},'
|
||||
'"image":{{json .Image}},"memory":{{.HostConfig.Memory}},'
|
||||
'"nano_cpus":{{.HostConfig.NanoCpus}},'
|
||||
'"published_ports":{{json .HostConfig.PortBindings}}}'
|
||||
)
|
||||
r = subprocess.run(
|
||||
["docker", "inspect", "--format", template, container],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=5,
|
||||
)
|
||||
return json.loads(r.stdout)
|
||||
|
||||
|
||||
def instance(container):
|
||||
code = (
|
||||
"import urllib.request; r=urllib.request.urlopen("
|
||||
"'http://127.0.0.1:8080/health/live',timeout=1); print(r.headers['X-Instance-Id'])"
|
||||
)
|
||||
r = subprocess.run(
|
||||
["docker", "exec", container, "python", "-c", code],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=3,
|
||||
)
|
||||
value = inspect(container)
|
||||
value["instance"] = r.stdout.strip()
|
||||
return value
|
||||
|
||||
|
||||
def instances():
|
||||
ids = compose("ps", "-q", "api").split()
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
return list(pool.map(instance, ids))
|
||||
|
||||
|
||||
def wait_lb(base="http://127.0.0.1:8083", count=2, timeout=45):
|
||||
deadline, seen = time.monotonic() + timeout, set()
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
current = {value["instance"] for value in instances()}
|
||||
if len(current) != count:
|
||||
time.sleep(0.2)
|
||||
continue
|
||||
with urllib.request.urlopen(base + "/health/ready", timeout=2) as response:
|
||||
seen.add(response.headers["X-Instance-Id"])
|
||||
if current <= seen:
|
||||
return sorted(current)
|
||||
except (OSError, urllib.error.URLError, subprocess.SubprocessError):
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError("Expected healthy API replicas were not observed through LB")
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# All running application/state services; no migration, down -v, flush or seed.
|
||||
docker compose --env-file .env restart --timeout 7 api postgres valkey lb
|
||||
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sequential process replacement; at least one existing API stays running."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
from cluster import compose, instances, wait_lb
|
||||
|
||||
|
||||
def replace_all(base="http://127.0.0.1:8083"):
|
||||
original = instances()
|
||||
if len(original) < 2:
|
||||
raise RuntimeError("Rolling replacement requires at least two healthy replicas")
|
||||
wait_lb(base, len(original))
|
||||
replacements = []
|
||||
for current in original:
|
||||
subprocess.run(
|
||||
["docker", "stop", "--time", "5", current["id"]],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
timeout=8,
|
||||
)
|
||||
subprocess.run(["docker", "rm", current["id"]], check=True, capture_output=True)
|
||||
compose(
|
||||
"up", "-d", "--no-deps", "--no-recreate", "--scale", "api=" + str(len(original)), "api"
|
||||
)
|
||||
aliases = wait_lb(base, len(original))
|
||||
if current["instance"] in aliases:
|
||||
raise RuntimeError("Old API process is still serving after replacement")
|
||||
replacements.append(
|
||||
{
|
||||
"old_container": current["id"],
|
||||
"old_instance": current["instance"],
|
||||
"live_instances": aliases,
|
||||
}
|
||||
)
|
||||
return replacements
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8083")
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(replace_all(args.base_url)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Map a process alias to ONE container belonging to this Compose service."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def alias(container):
|
||||
code = (
|
||||
"import urllib.request; r=urllib.request.urlopen("
|
||||
"'http://127.0.0.1:8080/health/live',timeout=1); print(r.headers['X-Instance-Id'])"
|
||||
)
|
||||
r = subprocess.run(
|
||||
["docker", "exec", container, "python", "-c", code],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=3,
|
||||
check=True,
|
||||
)
|
||||
return container, r.stdout.strip()
|
||||
|
||||
|
||||
def main():
|
||||
target = os.environ.get("TARGET_INSTANCE_ID", "")
|
||||
if not re.fullmatch(r"[a-f0-9]{32}", target):
|
||||
raise SystemExit("TARGET_INSTANCE_ID must name a live API process")
|
||||
ids = subprocess.check_output(
|
||||
["docker", "compose", "ps", "-q", "api"], cwd=ROOT, text=True
|
||||
).split()
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
matches = [cid for cid, identity in pool.map(alias, ids) if identity == target]
|
||||
if len(matches) != 1:
|
||||
raise SystemExit("Expected exactly one matching API in this project")
|
||||
subprocess.run(
|
||||
["docker", "stop", "--time", "5", matches[0]], check=True, capture_output=True, timeout=7
|
||||
)
|
||||
print(json.dumps({"stopped_container": matches[0], "instance": target}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/stop-one.py
|
||||
@@ -0,0 +1,253 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Real LB requests during each replica failure and sequential replacement."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from cluster import ROOT, compose, inspect, instance, instances, wait_lb
|
||||
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
from resilience import history # noqa: E402
|
||||
|
||||
|
||||
def percentile(values, fraction):
|
||||
values = sorted(values)
|
||||
return values[min(int((len(values) - 1) * fraction), len(values) - 1)] if values else None
|
||||
|
||||
|
||||
class Traffic:
|
||||
def __init__(self, base, actor, room, workers=6):
|
||||
self.base, self.actor, self.room = base, actor, room
|
||||
self.prefix = uuid4().hex
|
||||
self.stop = threading.Event()
|
||||
self.lock = threading.Lock()
|
||||
self.events = []
|
||||
self.executor = ThreadPoolExecutor(max_workers=workers)
|
||||
self.workers = workers
|
||||
|
||||
def worker(self, worker):
|
||||
client, iteration = Client(self.base, timeout=3), 0
|
||||
while not self.stop.is_set():
|
||||
started = time.monotonic()
|
||||
write = iteration % 10 == 9
|
||||
try:
|
||||
if write:
|
||||
key = f"{self.prefix}:{worker}:{iteration}"
|
||||
response = client.message(self.actor, self.room, key, key=key)
|
||||
else:
|
||||
response = client.call(
|
||||
"GET", "/api/v1/users/me", actor=self.actor, schema="User"
|
||||
)
|
||||
event = {
|
||||
"at": time.monotonic(),
|
||||
"ok": True,
|
||||
"instance": response["headers"]["X-Instance-Id"],
|
||||
"latency_ms": (time.monotonic() - started) * 1000,
|
||||
"message": response["json"] if write else None,
|
||||
}
|
||||
iteration += 1
|
||||
except Failure:
|
||||
event = {
|
||||
"at": time.monotonic(),
|
||||
"ok": False,
|
||||
"latency_ms": (time.monotonic() - started) * 1000,
|
||||
}
|
||||
with self.lock:
|
||||
self.events.append(event)
|
||||
|
||||
def __enter__(self):
|
||||
self.started = time.monotonic()
|
||||
self.futures = [self.executor.submit(self.worker, i) for i in range(self.workers)]
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
self.stop.set()
|
||||
self.executor.shutdown(wait=True)
|
||||
for future in self.futures:
|
||||
future.result()
|
||||
self.finished = time.monotonic()
|
||||
|
||||
def report(self, action_start, action_end):
|
||||
events = sorted(self.events, key=lambda row: row["at"])
|
||||
ok = [row for row in events if row["ok"]]
|
||||
after = [row for row in ok if row["at"] >= action_end]
|
||||
check(bool(after), "No successful request after action")
|
||||
elapsed = self.finished - self.started
|
||||
latency = [row["latency_ms"] for row in ok]
|
||||
return {
|
||||
"duration_seconds": elapsed,
|
||||
"workers": self.workers,
|
||||
"requests": len(events),
|
||||
"successes": len(ok),
|
||||
"errors": len(events) - len(ok),
|
||||
"successful_rps": len(ok) / elapsed,
|
||||
"error_fraction": (len(events) - len(ok)) / len(events),
|
||||
"first_success_after_action_seconds": after[0]["at"] - action_start,
|
||||
"action_seconds": action_end - action_start,
|
||||
"p50_ms": percentile(latency, 0.5),
|
||||
"p95_ms": percentile(latency, 0.95),
|
||||
"p99_ms": percentile(latency, 0.99),
|
||||
"instances": sorted({row["instance"] for row in ok}),
|
||||
"tail_instances": sorted({row["instance"] for row in after[-10:]}),
|
||||
"confirmed_writes": sum(row["message"] is not None for row in ok),
|
||||
}
|
||||
|
||||
def verify_messages(self, client, actor, room):
|
||||
actual = history(client, actor, room)
|
||||
by_id = {value["id"]: value for value in actual}
|
||||
check(len(by_id) == len(actual), "History has duplicate IDs")
|
||||
texts = [row["text"] for row in actual if row["text"].startswith(self.prefix)]
|
||||
check(len(texts) == len(set(texts)), "An uncertain retry created a duplicate effect")
|
||||
for event in self.events:
|
||||
if event["ok"] and event["message"]:
|
||||
check(
|
||||
by_id.get(event["message"]["id"]) == event["message"],
|
||||
"A confirmed message was lost or changed",
|
||||
)
|
||||
|
||||
|
||||
def public_test(target, base):
|
||||
result = subprocess.run(
|
||||
[
|
||||
"make",
|
||||
target,
|
||||
"ACTION_SCRIPT=scripts/"
|
||||
+ ("restart.sh" if target == "test-persistence" else "stop-one.sh"),
|
||||
"BASE_URL=" + base,
|
||||
],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=150,
|
||||
)
|
||||
return "\n".join(line.rstrip() for line in result.stdout.splitlines()) + "\n"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8083")
|
||||
parser.add_argument(
|
||||
"--output", type=Path, default=ROOT / f"evidence/cluster-{time.time_ns()}.json"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.output.exists():
|
||||
raise SystemExit("Use a new evidence filename")
|
||||
os.environ["NO_PROXY"] = os.environ["no_proxy"] = "127.0.0.1,localhost"
|
||||
baseline = instances()
|
||||
check(len(baseline) == 2, "This verification expects the default two API replicas")
|
||||
check(not any(row["published_ports"] for row in baseline), "API must not publish host ports")
|
||||
wait_lb(args.base_url, 2)
|
||||
result = {"initial_instances": baseline, "failures": []}
|
||||
client = Client(args.base_url, timeout=4)
|
||||
alice, bob, eve, room = client.bootstrap()
|
||||
key = uuid4().hex
|
||||
|
||||
def same_request(_):
|
||||
response = Client(args.base_url).message(alice, room, "concurrent replay", key=key)
|
||||
return response["json"], response["headers"]["X-Instance-Id"]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
responses = list(pool.map(same_request, range(30)))
|
||||
check(
|
||||
len({row[0]["id"] for row in responses}) == 1, "Cross-replica replay duplicated a message"
|
||||
)
|
||||
check(len({row[1] for row in responses}) == 2, "Concurrent replay did not cross real replicas")
|
||||
result["concurrent_replays"] = 30
|
||||
result["concurrent_effects"] = 1
|
||||
try:
|
||||
for selected in baseline:
|
||||
selected = instance(selected["id"])
|
||||
with Traffic(args.base_url, alice, room) as traffic:
|
||||
time.sleep(1)
|
||||
action_start = time.monotonic()
|
||||
env = {**os.environ, "TARGET_INSTANCE_ID": selected["instance"]}
|
||||
subprocess.run(
|
||||
[str(ROOT / "scripts/stop-one.sh")],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
)
|
||||
action_end = time.monotonic()
|
||||
check(not inspect(selected["id"])["running"], "Chosen process was not stopped")
|
||||
time.sleep(3)
|
||||
report = traffic.report(action_start, action_end)
|
||||
check(report["first_success_after_action_seconds"] <= 10, "Failover took too long")
|
||||
check(selected["instance"] not in report["tail_instances"], "Stopped API still serves")
|
||||
report["stopped"] = selected
|
||||
traffic.verify_messages(client, bob, room)
|
||||
result["failures"].append(report)
|
||||
subprocess.run(["docker", "start", selected["id"]], check=True, capture_output=True)
|
||||
wait_lb(args.base_url, 2)
|
||||
print("Verified failure and restoration of one API", flush=True)
|
||||
with Traffic(args.base_url, alice, room) as traffic:
|
||||
time.sleep(1)
|
||||
action_start = time.monotonic()
|
||||
replaced = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(ROOT / "scripts/rolling-update.py"),
|
||||
"--base-url",
|
||||
args.base_url,
|
||||
],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
action_end = time.monotonic()
|
||||
time.sleep(3)
|
||||
traffic.verify_messages(client, bob, room)
|
||||
result["rolling"] = traffic.report(action_start, action_end)
|
||||
result["rolling"]["replacements"] = json.loads(replaced.stdout)
|
||||
result["rolling"]["confirmed_messages_preserved"] = True
|
||||
print("Verified sequential replacement under traffic", flush=True)
|
||||
(ROOT / "evidence/failover.txt").write_text(public_test("test-failover", args.base_url))
|
||||
for value in compose("ps", "-aq", "api").split():
|
||||
if not inspect(value)["running"]:
|
||||
subprocess.run(["docker", "start", value], check=True, capture_output=True)
|
||||
wait_lb(args.base_url, 2)
|
||||
before = {
|
||||
service: inspect(compose("ps", "-q", service).strip())
|
||||
for service in ("postgres", "valkey", "lb")
|
||||
}
|
||||
(ROOT / "evidence/persistence.txt").write_text(
|
||||
public_test("test-persistence", args.base_url)
|
||||
)
|
||||
after = {service: inspect(compose("ps", "-q", service).strip()) for service in before}
|
||||
check(
|
||||
all(before[s]["started_at"] != after[s]["started_at"] for s in before),
|
||||
"Persistence hook did not restart all state/LB services",
|
||||
)
|
||||
result["persistence"] = {"before": before, "after": after, "same_token_public_test": True}
|
||||
result["final_instances"] = instances()
|
||||
logs = compose("logs", "--no-color", "api", "lb")
|
||||
check(
|
||||
not any(
|
||||
a["token"] in logs or a["registration"]["password"] in logs
|
||||
for a in (alice, bob, eve)
|
||||
),
|
||||
"Credential appeared in logs",
|
||||
)
|
||||
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
||||
print("PASS: real replica failures, replay, rolling replacement and persistence")
|
||||
finally:
|
||||
for value in compose("ps", "-aq", "api").split():
|
||||
if not inspect(value)["running"]:
|
||||
subprocess.run(["docker", "start", value], check=True, capture_output=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user