forked from mtusi2026/mtusi2026lab3
Собрать стенд lab3 с двумя репликами API за Traefik.
Общие PostgreSQL, Valkey и идемпотентность сообщений нужны, чтобы чтение и запись переживали остановку одной реплики.
This commit is contained in:
@@ -0,0 +1,325 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Нагрузка, поочерёдный отказ каждой реплики и rolling update. Токены не печатает."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client, Failure # noqa: E402
|
||||
|
||||
|
||||
def run(cmd, env=None, check=True):
|
||||
return subprocess.run(
|
||||
cmd,
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
check=check,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return run(["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", *args])
|
||||
|
||||
|
||||
def api_ids():
|
||||
proc = compose("ps", "-q", "api")
|
||||
return [line.strip() for line in proc.stdout.splitlines() if line.strip()]
|
||||
|
||||
|
||||
def inspect(cid, fmt):
|
||||
return run(["docker", "inspect", "-f", fmt, cid]).stdout.strip()
|
||||
|
||||
|
||||
def percentile(values, p):
|
||||
if not values:
|
||||
return None
|
||||
ordered = sorted(values)
|
||||
index = max(0, int(round((len(ordered) - 1) * p)))
|
||||
return round(ordered[index] * 1000, 3)
|
||||
|
||||
|
||||
class Load:
|
||||
def __init__(self, base, actor, chat, concurrency=4):
|
||||
self.base = base
|
||||
self.actor = actor
|
||||
self.chat = chat
|
||||
self.concurrency = concurrency
|
||||
self.events = []
|
||||
self._stop = threading.Event()
|
||||
self._origin = time.monotonic()
|
||||
self._pool = None
|
||||
self._futures = []
|
||||
|
||||
def now(self):
|
||||
return time.monotonic() - self._origin
|
||||
|
||||
def start(self):
|
||||
def worker(worker_id):
|
||||
local = Client(self.base, timeout=3)
|
||||
index = 0
|
||||
while not self._stop.is_set():
|
||||
stamp = self.now()
|
||||
try:
|
||||
response = local.message(
|
||||
self.actor,
|
||||
self.chat,
|
||||
"load-%s-%s-%s" % (worker_id, index, uuid.uuid4().hex[:8]),
|
||||
)
|
||||
self.events.append({
|
||||
"ok": True,
|
||||
"t": stamp,
|
||||
"dt": self.now() - stamp,
|
||||
"instance": response["headers"].get("X-Instance-Id"),
|
||||
})
|
||||
except Failure:
|
||||
self.events.append({
|
||||
"ok": False,
|
||||
"t": stamp,
|
||||
"dt": self.now() - stamp,
|
||||
"instance": None,
|
||||
})
|
||||
index += 1
|
||||
|
||||
self._pool = ThreadPoolExecutor(max_workers=self.concurrency)
|
||||
self._futures = [self._pool.submit(worker, i) for i in range(self.concurrency)]
|
||||
|
||||
def stop(self):
|
||||
self._stop.set()
|
||||
for future in self._futures:
|
||||
future.result()
|
||||
self._pool.shutdown(wait=True)
|
||||
|
||||
def summary(self, start, end):
|
||||
window = [item for item in self.events if start <= item["t"] < end]
|
||||
ok = [item for item in window if item["ok"]]
|
||||
bad = [item for item in window if not item["ok"]]
|
||||
elapsed = max(end - start, 0.001)
|
||||
return {
|
||||
"seconds": round(end - start, 3),
|
||||
"successful_requests": len(ok),
|
||||
"failed_requests": len(bad),
|
||||
"successful_rps": round(len(ok) / elapsed, 3),
|
||||
"error_ratio": round(len(bad) / len(window), 6) if window else None,
|
||||
"success_latency_ms": {
|
||||
"p50": percentile([item["dt"] for item in ok], 0.50),
|
||||
"p95": percentile([item["dt"] for item in ok], 0.95),
|
||||
"p99": percentile([item["dt"] for item in ok], 0.99),
|
||||
},
|
||||
"instances": sorted({item["instance"] for item in ok if item["instance"]}),
|
||||
}
|
||||
|
||||
|
||||
def wait_ready(base, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
try:
|
||||
Client(base, timeout=2).call("GET", "/health/ready", schema="Health")
|
||||
return
|
||||
except Failure:
|
||||
if time.monotonic() > deadline:
|
||||
raise SystemExit("API did not become ready")
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def wait_instance(base, actor, wanted, timeout):
|
||||
started = time.monotonic()
|
||||
seen = set()
|
||||
while time.monotonic() - started < timeout:
|
||||
try:
|
||||
seen.add(
|
||||
Client(base, timeout=2).call(
|
||||
"GET", "/api/v1/users/me", actor=actor, schema="User"
|
||||
)["headers"]["X-Instance-Id"]
|
||||
)
|
||||
except Failure:
|
||||
pass
|
||||
if wanted in seen:
|
||||
return round(time.monotonic() - started, 3), sorted(seen)
|
||||
time.sleep(0.2)
|
||||
return None, sorted(seen)
|
||||
|
||||
|
||||
def container_limits():
|
||||
rows = []
|
||||
proc = compose("ps", "-q")
|
||||
for cid in [line.strip() for line in proc.stdout.splitlines() if line.strip()]:
|
||||
rows.append({
|
||||
"name": inspect(cid, "{{.Name}}").lstrip("/"),
|
||||
"nano_cpus": int(inspect(cid, "{{.HostConfig.NanoCpus}}") or "0"),
|
||||
"memory_bytes": int(inspect(cid, "{{.HostConfig.Memory}}") or "0"),
|
||||
"published": inspect(cid, "{{json .NetworkSettings.Ports}}"),
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def main():
|
||||
base = os.environ.get("BASE_URL", "http://localhost:8080")
|
||||
wait_ready(base)
|
||||
client = Client(base, timeout=10)
|
||||
alice, _bob, _eve, chat = client.bootstrap()
|
||||
spread = [
|
||||
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"]["X-Instance-Id"]
|
||||
for _ in range(30)
|
||||
]
|
||||
observed = sorted(set(spread))
|
||||
if len(observed) < 2:
|
||||
raise SystemExit("need two instance ids, saw %s" % observed)
|
||||
|
||||
key = "probe-" + uuid.uuid4().hex
|
||||
text = "параллельный ключ " + key
|
||||
barrier = threading.Barrier(8)
|
||||
|
||||
def once(_n):
|
||||
local = Client(base, timeout=10)
|
||||
barrier.wait()
|
||||
response = local.message(alice, chat, text, key=key)
|
||||
return response["json"]["id"], response["headers"]["X-Instance-Id"]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
parallel = list(pool.map(once, range(8)))
|
||||
parallel_ids = sorted({item[0] for item in parallel})
|
||||
parallel_instances = sorted({item[1] for item in parallel})
|
||||
if len(parallel_ids) != 1:
|
||||
raise SystemExit("parallel idempotency created %s messages" % len(parallel_ids))
|
||||
|
||||
load = Load(base, alice, chat)
|
||||
load.start()
|
||||
time.sleep(8)
|
||||
baseline_end = load.now()
|
||||
baseline = load.summary(0, baseline_end)
|
||||
|
||||
failovers = []
|
||||
for target in observed:
|
||||
time.sleep(1)
|
||||
mark = load.now()
|
||||
env = os.environ.copy()
|
||||
env["TARGET_INSTANCE_ID"] = target
|
||||
started = time.monotonic()
|
||||
proc = run([str(ROOT / "scripts" / "stop-one.sh")], env=env, check=False)
|
||||
stop_s = round(time.monotonic() - started, 3)
|
||||
stopped_at = load.now()
|
||||
time.sleep(6)
|
||||
window_end = load.now()
|
||||
after = [item for item in load.events if item["t"] >= stopped_at and item["ok"]]
|
||||
served = sorted({item["instance"] for item in after if item["instance"]})
|
||||
first = None
|
||||
for item in load.events:
|
||||
if item["t"] >= mark and item["ok"] and item["instance"] and item["instance"] != target:
|
||||
first = round(item["t"] - mark, 3)
|
||||
break
|
||||
restore_started = time.monotonic()
|
||||
run([str(ROOT / "scripts" / "restore-api.sh")])
|
||||
back_s, seen_after = wait_instance(base, alice, target, 40)
|
||||
failovers.append({
|
||||
"target": target,
|
||||
"stop_hook_exit": proc.returncode,
|
||||
"stop_hook_seconds": stop_s,
|
||||
"window": load.summary(mark, window_end),
|
||||
"first_other_replica_success_seconds": first,
|
||||
"target_seen_after_hook_returned": target in served,
|
||||
"instances_after_hook": served,
|
||||
"restore_until_target_seen_seconds": back_s,
|
||||
"instances_after_restore": seen_after,
|
||||
"restore_command_seconds": round(time.monotonic() - restore_started, 3),
|
||||
})
|
||||
if proc.returncode != 0:
|
||||
load.stop()
|
||||
raise SystemExit("stop-one failed for %s\n%s" % (target, proc.stderr))
|
||||
if first is None or first > 10 or target in served:
|
||||
load.stop()
|
||||
raise SystemExit("failover criteria failed for %s" % target)
|
||||
|
||||
images_before = sorted({inspect(cid, "{{.Image}}")[:19] for cid in api_ids()})
|
||||
roll_mark = load.now()
|
||||
holder = {}
|
||||
|
||||
def run_roll():
|
||||
proc = run([str(ROOT / "scripts" / "rolling-update.sh")], check=False)
|
||||
holder["code"] = proc.returncode
|
||||
holder["err"] = (proc.stderr or "")[-800:]
|
||||
holder["out"] = (proc.stdout or "")[-800:]
|
||||
|
||||
roller = threading.Thread(target=run_roll)
|
||||
roller.start()
|
||||
roller.join()
|
||||
roll_end = load.now()
|
||||
load.stop()
|
||||
images_after = sorted({inspect(cid, "{{.Image}}")[:19] for cid in api_ids()})
|
||||
rolling = load.summary(roll_mark, roll_end)
|
||||
rolling["exit"] = holder.get("code")
|
||||
rolling["stderr_tail"] = holder.get("err")
|
||||
if holder.get("code") != 0:
|
||||
raise SystemExit("rolling update failed: %s\n%s" % (holder.get("err"), holder.get("out")))
|
||||
|
||||
report = {
|
||||
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"platform": platform.platform(),
|
||||
"cpu_count": os.cpu_count(),
|
||||
"workload": "closed-loop POST, concurrency 4, client timeout 3s, no client retry",
|
||||
"observed_instances": observed,
|
||||
"parallel_idempotency": {
|
||||
"requests": 8,
|
||||
"unique_message_ids": len(parallel_ids),
|
||||
"instances": parallel_instances,
|
||||
},
|
||||
"baseline": baseline,
|
||||
"failovers": failovers,
|
||||
"images_before_roll": images_before,
|
||||
"images_after_roll": images_after,
|
||||
"rolling_under_load": rolling,
|
||||
"container_limits": container_limits(),
|
||||
"lb_timeouts": {
|
||||
"check_interval_seconds": 1,
|
||||
"unhealthy_after_failures": 1,
|
||||
"check_timeout_seconds": 1,
|
||||
"max_detection_seconds": 2,
|
||||
"connect_timeout_seconds": 1,
|
||||
"server_timeout_seconds": 8,
|
||||
"client_timeout_seconds": 8,
|
||||
"drain_seconds": 3,
|
||||
"stop_grace_seconds": 10,
|
||||
},
|
||||
}
|
||||
dest = ROOT / "evidence" / "grade4.json"
|
||||
dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({
|
||||
"observed_instances": observed,
|
||||
"parallel_unique_messages": len(parallel_ids),
|
||||
"parallel_instances": parallel_instances,
|
||||
"baseline_rps": baseline["successful_rps"],
|
||||
"baseline_errors": baseline["failed_requests"],
|
||||
"failovers": [
|
||||
{
|
||||
"target": item["target"],
|
||||
"stop_s": item["stop_hook_seconds"],
|
||||
"first_other_s": item["first_other_replica_success_seconds"],
|
||||
"errors": item["window"]["failed_requests"],
|
||||
"rps": item["window"]["successful_rps"],
|
||||
"target_seen_after_hook": item["target_seen_after_hook_returned"],
|
||||
"restore_s": item["restore_until_target_seen_seconds"],
|
||||
}
|
||||
for item in failovers
|
||||
],
|
||||
"rolling_exit": rolling["exit"],
|
||||
"rolling_errors": rolling["failed_requests"],
|
||||
"rolling_rps": rolling["successful_rps"],
|
||||
"images_changed": images_before != images_after,
|
||||
"evidence": "evidence/grade4.json",
|
||||
}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# Перезапуск API и хранилищ. Volumes не удаляются, схема и данные остаются.
|
||||
docker compose --env-file .env -f compose.yaml restart api postgres valkey
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
replicas=$(sed -n 's/^API_REPLICAS=//p' .env | tr -d '\r')
|
||||
if [ -z "${replicas}" ]; then
|
||||
replicas=2
|
||||
fi
|
||||
docker compose --env-file .env -f compose.yaml up -d --no-deps --scale "api=${replicas}" --no-recreate api
|
||||
Executable
+56
@@ -0,0 +1,56 @@
|
||||
#!/bin/sh
|
||||
# Поочерёдная замена API: сначала поднимается новая реплика, затем останавливается старая.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
compose() {
|
||||
docker compose --env-file .env -f compose.yaml "$@"
|
||||
}
|
||||
|
||||
bid=$(date +%s)
|
||||
docker build --build-arg "BUILD_ID=${bid}" -t mtusi-messenger:lab3 .
|
||||
|
||||
old=$(compose ps -q api)
|
||||
if [ -z "$old" ]; then
|
||||
echo "no running api replicas" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
healthy_count() {
|
||||
n=0
|
||||
for id in $(compose ps -q api); do
|
||||
status=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$id")
|
||||
if [ "$status" = "healthy" ]; then
|
||||
n=$((n + 1))
|
||||
fi
|
||||
done
|
||||
printf '%s' "$n"
|
||||
}
|
||||
|
||||
wait_healthy() {
|
||||
want=$1
|
||||
i=0
|
||||
while [ "$i" -lt 45 ]; do
|
||||
n=$(healthy_count)
|
||||
if [ "$n" -ge "$want" ]; then
|
||||
return 0
|
||||
fi
|
||||
i=$((i + 1))
|
||||
sleep 1
|
||||
done
|
||||
echo "wanted ${want} healthy api, saw $(healthy_count)" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
for cid in $old; do
|
||||
echo "replacement before stop ${cid}"
|
||||
compose up -d --no-deps --scale api=3 --no-recreate api
|
||||
wait_healthy 3
|
||||
docker stop -t 8 "$cid" >/dev/null
|
||||
docker rm "$cid" >/dev/null
|
||||
compose up -d --no-deps --scale api=2 --no-recreate api
|
||||
wait_healthy 2
|
||||
done
|
||||
|
||||
echo "rolling update done"
|
||||
compose ps api
|
||||
Executable
+22
@@ -0,0 +1,22 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
found=""
|
||||
for cid in $(docker compose --env-file .env -f compose.yaml ps -q api); do
|
||||
host=$(docker inspect -f '{{.Config.Hostname}}' "$cid")
|
||||
short=$(printf '%s' "$cid" | cut -c1-12)
|
||||
if [ "$host" = "$TARGET_INSTANCE_ID" ] || [ "$short" = "$TARGET_INSTANCE_ID" ]; then
|
||||
found=$cid
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$found" ]; then
|
||||
echo "API replica was not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# docker stop + restart: unless-stopped оставляет реплику выключенной.
|
||||
docker stop -t 8 "$found" >/dev/null
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
if [ ! -f .env ]; then
|
||||
echo "Copy .env.example to .env" >&2
|
||||
exit 1
|
||||
fi
|
||||
replicas=$(sed -n 's/^API_REPLICAS=//p' .env | tr -d '\r')
|
||||
if [ -z "${replicas}" ]; then
|
||||
replicas=2
|
||||
fi
|
||||
docker compose --env-file .env -f compose.yaml up -d --build --scale "api=${replicas}" --remove-orphans
|
||||
Reference in New Issue
Block a user