feat: rolling update без простоя и серия отказов реплик под нагрузкой (уровень 4)

- scripts/rolling-update.sh: сборка, однократные миграции новым образом,
  затем для каждой старой реплики: +1 новая (healthy и UP в HAProxy),
  graceful stop старой; ревизия образа видна в label и логе "starting".
- bench/loadgen.py: фоновая нагрузка через LB (POST с Idempotency-Key +
  GET), неопределённый POST повторяется тем же ключом, проверка истории
  на потери и дубли.
- bench/failover_series.py: каждая реплика по очереди в режимах stop /
  kill / pause; RPS, ошибки, время выхода из ротации и возврата.
- bench/rolling_under_load.py: rolling update под нагрузкой.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alexey Bodanyuk
2026-10-16 13:39:54 +03:00
co-authored by Claude Opus 5.5
parent 78b95acabf
commit c050642ec2
7 changed files with 481 additions and 3 deletions
+6
View File
@@ -42,6 +42,12 @@ ENV PYTHONPATH=/app/src \
SESSION_TTL_SECONDS=3600 \
DATABASE_URL_FILE=/run/secrets/database_url
# Build revision (scripts/rolling-update.sh passes git commit + time): makes every rolled-out
# image distinguishable in `docker inspect` and in the "starting" log line of each replica.
ARG BUILD_REVISION=dev
LABEL org.opencontainers.image.revision=$BUILD_REVISION
ENV APP_REVISION=$BUILD_REVISION
USER app:app
EXPOSE 8080
STOPSIGNAL SIGTERM
+141
View File
@@ -0,0 +1,141 @@
#!/usr/bin/env python3
"""Level 4: stop EACH API replica in turn under load (restoring the previous one first) and
measure RPS, errors and recovery. Modes: graceful `stop` (SIGTERM + drain), `kill` (SIGKILL)
and `pause` (frozen process: the kernel still accepts TCP, nothing answers - the worst case
for detection, only the active health check can find it).
python bench/failover_series.py --output evidence/failover # ~5 min for 2 replicas x 3 modes
Timeline of one round (all times from the moment the stop command is issued):
warm-up 6 s -> stop replica -> 12 s with it down -> start it again -> wait until HAProxy
has every replica UP (rejoin time) -> 6 s steady state.
Measured per round: requests/ok/failed and ok-RPS before / during / after, the last failed
request (+s), the first moment after which the stopped replica never answered again
(= removal from rotation), time until the restarted replica is UP in HAProxy and serves
traffic again, plus the history check (no confirmed message lost, no duplicate).
Output: <output>/series.json (summary), <output>/timeline-<mode>-<replica>.csv (per second),
<output>/series.md (table for REPORT.md). No tokens are written.
"""
from __future__ import annotations
import argparse
import csv
import datetime as dt
import json
import platform
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
sys.path.insert(0, str(ROOT / "bench"))
import lbctl # noqa: E402
from loadgen import Load # noqa: E402
def docker(*args: str) -> str:
return subprocess.run(["docker", *args], capture_output=True, text=True, check=True, timeout=120).stdout
def one_round(replica: dict, mode: str, replicas: int, workers: int, out: Path) -> dict:
load = Load(workers).start()
time.sleep(6)
t_stop = load.now()
if mode == "pause":
docker("pause", replica["id"])
else:
docker("stop", "--time", "15" if mode == "stop" else "0", replica["id"])
t_stopped = load.now()
time.sleep(12)
t_start = load.now()
docker("unpause" if mode == "pause" else "start", replica["id"])
lbctl.wait_up(replicas, timeout=90)
t_up = load.now()
# The restarted container keeps its hostname: wait until it serves requests again.
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
if any(e[3] == replica["hostname"] and e[0] > t_up for e in load.events):
break
time.sleep(0.1)
t_serves = load.now()
time.sleep(6)
load.stop()
t_end = load.now()
failed_after_stop = [e for e in load.events if e[0] >= t_stop and e[2] not in (200, 201)]
served_by_target = [e[0] for e in load.events if e[3] == replica["hostname"] and t_stop <= e[0] < t_start]
out_of_rotation = (max(served_by_target) - t_stop) if served_by_target else 0.0
rows = load.per_second()
with (out / f"timeline-{mode}-{replica['name']}.csv").open("w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=["second", "ok", "failed", "instances"])
writer.writeheader()
for row in rows:
row = dict(row)
row["instances"] = row["instances"].replace(replica["hostname"], "TARGET")
writer.writerow(row)
state = json.loads(docker("inspect", replica["id"]))[0]["State"]
return {
"mode": mode, "replica": replica["name"], "instance_id": replica["hostname"],
"stop_command_seconds": round(t_stopped - t_stop, 2),
"before": load.window(0.5, t_stop), "during": load.window(t_stop, t_start), "after": load.window(t_serves, t_end),
"failed_after_stop": len(failed_after_stop),
"failed_statuses": sorted({e[2] for e in failed_after_stop}),
"last_failure_after_stop_s": round(max((e[0] for e in failed_after_stop), default=t_stop) - t_stop, 3),
"target_last_served_after_stop_s": round(out_of_rotation, 3),
"rejoin_up_in_lb_s": round(t_up - t_start, 2), "rejoin_serving_s": round(t_serves - t_start, 2),
"client_retries_same_key": load.retries,
"history": load.verify_history(),
"restarted_state": f"{state['Status']} health={(state.get('Health') or {}).get('Status')}",
}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--output", default="evidence/failover")
parser.add_argument("--modes", default="stop,kill,pause")
parser.add_argument("--workers", type=int, default=4)
args = parser.parse_args()
out = ROOT / args.output
out.mkdir(parents=True, exist_ok=True)
replicas = sorted((r for r in lbctl.replicas() if r["state"] == "running"), key=lambda r: r["name"])
assert len(replicas) >= 2, "need at least two running replicas"
lbctl.wait_up(len(replicas))
results = []
for mode in args.modes.split(","):
for replica in replicas: # each replica in turn; the previous one is already restored
print(f"== {mode} {replica['name']} ...", flush=True)
result = one_round(replica, mode, len(replicas), args.workers, out)
results.append(result)
print(json.dumps({k: result[k] for k in ("failed_after_stop", "last_failure_after_stop_s",
"target_last_served_after_stop_s", "rejoin_serving_s")}), flush=True)
meta = {"date": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
"commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True,
text=True).stdout.strip(),
"host": f"{platform.system()} {platform.release()}, docker {docker('version', '--format', '{{.Server.Version}}').strip()}",
"workload": f"{args.workers} closed-loop workers from the host via 127.0.0.1:8080: POST message "
"(Idempotency-Key, uncertain -> same key) + GET last page; client timeout 3 s",
"lb": "HAProxy: check GET /health/ready inter 1s fastinter 500ms fall 2 rise 2, timeout check 1s; "
"retry-on conn-failure + redispatch",
"api": "DRAIN_SECONDS=3, SHUTDOWN_TIMEOUT_SECONDS=8, stop_grace_period 15s"}
(out / "series.json").write_text(json.dumps({"meta": meta, "rounds": results}, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8")
lines = ["| Режим | Реплика | Запросов во время отказа | Ошибок после stop | ok-RPS до / во время / после | "
"Последняя ошибка, с | Цель ушла из ротации, с | Возврат в LB / обслуживает, с | Потери / дубли |",
"| --- | --- | --- | --- | --- | --- | --- | --- | --- |"]
for r in results:
h = r["history"]
lines.append(f"| {r['mode']} | {r['replica']} | {r['during']['requests']} | {r['failed_after_stop']} "
f"{r['failed_statuses'] or ''} | {r['before']['ok_rps']} / {r['during']['ok_rps']} / {r['after']['ok_rps']} | "
f"{r['last_failure_after_stop_s']} | {r['target_last_served_after_stop_s']} | "
f"{r['rejoin_up_in_lb_s']} / {r['rejoin_serving_s']} | "
f"{h['lost_confirmed']} / {h['duplicate_ids'] + h['duplicate_texts']} |")
(out / "series.md").write_text("\n".join([f"Дата {meta['date']}, commit {meta['commit']}. {meta['workload']}.", "",
*lines, ""]), encoding="utf-8")
print("\n".join(lines))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+212
View File
@@ -0,0 +1,212 @@
#!/usr/bin/env python3
"""Background HTTP traffic through the LB for failover / rolling-update experiments (level 4).
Closed loop, N worker threads, each: POST a message (Idempotency-Key) + GET the last page.
An uncertain POST (timeout, connection error, 5xx) is retried with the SAME key and body,
exactly as a correct client must do; the history is checked afterwards for losses and
duplicates. Every request is recorded as (t, kind, status, instance, latency).
This is a correctness/availability probe, not a capacity benchmark (see bench/track1 for that).
python bench/loadgen.py --duration 30 --workers 4 # prints a summary JSON
"""
from __future__ import annotations
import argparse
import base64
import json
import math
import os
import threading
import time
import uuid
from collections import Counter, defaultdict
import urllib.error
import urllib.request
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
class Http:
"""Minimal stdlib client with keep-alive per thread (http.client under urllib is not)."""
def __init__(self, base_url: str, timeout: float):
import http.client
from urllib.parse import urlsplit
parts = urlsplit(base_url)
self.host, self.port = parts.hostname, parts.port or 80
self.timeout = timeout
self.factory = lambda: http.client.HTTPConnection(self.host, self.port, timeout=self.timeout)
self.conn = self.factory()
def request(self, method: str, path: str, body: bytes | None = None, headers: dict | None = None):
try:
self.conn.request(method, path, body=body, headers=headers or {})
response = self.conn.getresponse()
data = response.read()
if response.getheader("connection", "").lower() == "close":
self.conn.close()
self.conn = self.factory()
return response.status, dict(response.getheaders()), data
except Exception:
self.conn.close()
self.conn = self.factory()
raise
def call(method: str, path: str, body=None, headers=None, timeout=10.0):
data = json.dumps(body).encode() if body is not None else None
h = {"Content-Type": "application/json", **(headers or {})}
req = urllib.request.Request(BASE_URL + path, data=data, headers=h, method=method)
try:
with urllib.request.urlopen(req, timeout=timeout) as r:
return r.status, json.loads(r.read() or b"null")
except urllib.error.HTTPError as e:
return e.code, json.loads(e.read() or b"null")
def make_user(label: str) -> dict:
username = f"{label}_{uuid.uuid4().hex[:12]}"
password = "Load-password-" + uuid.uuid4().hex
status, user = call("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
assert status == 201, (status, user)
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
status, session = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
assert status == 201, (status, session)
return {"id": user["id"], "token": session["token"]}
class Load:
def __init__(self, workers: int = 4, timeout: float = 3.0, pause: float = 0.0):
self.workers = workers
self.timeout = timeout
self.pause = pause
self.alice, self.bob = make_user("load"), make_user("load")
status, chat = call("POST", "/api/v1/chats", {"title": "load", "member_ids": [self.alice["id"], self.bob["id"]]},
headers={"Authorization": "Bearer " + self.alice["token"]})
assert status == 201, chat
self.chat = chat
self.stop_event = threading.Event()
self.lock = threading.Lock()
self.events: list[tuple[float, str, int, str, float]] = []
self.confirmed: dict[str, dict] = {}
self.retries = 0
self.t0 = time.monotonic()
self.threads = [threading.Thread(target=self._worker, args=(i,), daemon=True) for i in range(workers)]
def now(self) -> float:
return time.monotonic() - self.t0
def _record(self, kind: str, status: int, instance: str, latency: float) -> None:
with self.lock:
self.events.append((round(self.now(), 4), kind, status, instance, round(latency * 1000, 2)))
def _worker(self, index: int) -> None:
http = Http(BASE_URL, self.timeout)
auth = {"Authorization": "Bearer " + self.alice["token"]}
path = f"/api/v1/chats/{self.chat['id']}/messages"
n, key = 0, uuid.uuid4().hex
text = f"w{index}-0-{key[:6]}"
while not self.stop_event.is_set():
started = time.monotonic()
try:
status, headers, data = http.request("POST", path, json.dumps({"text": text}).encode(),
{**auth, "Idempotency-Key": key, "Content-Type": "application/json"})
instance = headers.get("x-instance-id") or headers.get("X-Instance-Id") or "-"
except Exception:
status, instance, data = 0, "-", b""
self._record("POST", status, instance, time.monotonic() - started)
if status == 201:
msg = json.loads(data)
with self.lock:
self.confirmed[msg["id"]] = msg
n += 1
key = uuid.uuid4().hex
text = f"w{index}-{n}-{key[:6]}"
elif status in (0, 502, 503, 504, 500):
with self.lock:
self.retries += 1 # same key + same text on the next iteration
started = time.monotonic()
try:
status, headers, _ = http.request("GET", path + "?limit=5", None, auth)
instance = headers.get("x-instance-id") or headers.get("X-Instance-Id") or "-"
except Exception:
status, instance = 0, "-"
self._record("GET", status, instance, time.monotonic() - started)
if self.pause:
time.sleep(self.pause)
def start(self) -> "Load":
self.t0 = time.monotonic()
for t in self.threads:
t.start()
return self
def stop(self) -> None:
self.stop_event.set()
for t in self.threads:
t.join(timeout=15)
def verify_history(self) -> dict:
"""Walk the whole history with the second user: no confirmed message lost, no duplicate."""
items, cursor = [], None
while True:
q = "?limit=100" + (f"&cursor={cursor}" if cursor else "")
status, page = call("GET", f"/api/v1/chats/{self.chat['id']}/messages{q}",
headers={"Authorization": "Bearer " + self.bob["token"]}, timeout=30)
assert status == 200, page
items += page["items"]
cursor = page["next_cursor"]
if not cursor:
break
ids = [m["id"] for m in items]
texts = [m["text"] for m in items]
lost = [i for i in self.confirmed if i not in set(ids)]
changed = [m for m in items if m["id"] in self.confirmed and self.confirmed[m["id"]] != m]
return {"history_messages": len(items), "confirmed_messages": len(self.confirmed),
"lost_confirmed": len(lost), "altered_confirmed": len(changed),
"duplicate_ids": len(ids) - len(set(ids)), "duplicate_texts": len(texts) - len(set(texts)),
"unconfirmed_but_stored": len(set(ids) - set(self.confirmed))}
# ------------------------------------------------------------------ summaries
def window(self, start: float, end: float) -> dict:
ev = [e for e in self.events if start <= e[0] < end]
ok = [e for e in ev if e[2] in (200, 201)]
lat = sorted(e[4] for e in ok)
def pct(p: float):
return lat[max(0, math.ceil(p * len(lat)) - 1)] if lat else None
span = max(end - start, 1e-9)
return {"seconds": round(span, 2), "requests": len(ev), "ok": len(ok), "failed": len(ev) - len(ok),
"ok_rps": round(len(ok) / span, 1), "error_ratio": round((len(ev) - len(ok)) / len(ev), 5) if ev else None,
"statuses": dict(Counter(e[2] for e in ev)), "instances": dict(Counter(e[3] for e in ok)),
"p50_ms": pct(.5), "p95_ms": pct(.95), "p99_ms": pct(.99)}
def per_second(self) -> list[dict]:
buckets: dict[int, list] = defaultdict(list)
for e in self.events:
buckets[int(e[0])].append(e)
rows = []
for second in sorted(buckets):
ev = buckets[second]
rows.append({"second": second, "ok": sum(1 for e in ev if e[2] in (200, 201)),
"failed": sum(1 for e in ev if e[2] not in (200, 201)),
"instances": "|".join(f"{k}:{v}" for k, v in sorted(Counter(e[3] for e in ev).items()))})
return rows
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--duration", type=float, default=20)
parser.add_argument("--workers", type=int, default=4)
args = parser.parse_args()
load = Load(args.workers).start()
time.sleep(args.duration)
load.stop()
print(json.dumps({"total": load.window(0, load.now()), "history": load.verify_history(),
"client_retries_same_key": load.retries}, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env python3
"""Level 4: scripts/rolling-update.sh while bench/loadgen.py traffic runs through the LB.
python bench/rolling_under_load.py --output evidence/rolling-update
Writes <output>/summary.json (errors, RPS, history check, revisions before/after),
<output>/timeline.csv (per second ok/failed/instances) and <output>/script-output.txt.
"""
from __future__ import annotations
import argparse
import csv
import json
import os
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
sys.path.insert(0, str(ROOT / "bench"))
import lbctl # noqa: E402
from loadgen import Load # noqa: E402
def revisions() -> dict[str, str]:
result = {}
for r in lbctl.replicas():
if r["state"] == "running":
label = subprocess.run(["docker", "inspect", "-f", '{{index .Config.Labels "org.opencontainers.image.revision"}}',
r["id"]], capture_output=True, text=True).stdout.strip()
result[r["name"]] = f"{r['hostname']} revision={label}"
return result
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--output", default="evidence/rolling-update")
parser.add_argument("--workers", type=int, default=4)
args = parser.parse_args()
out = ROOT / args.output
out.mkdir(parents=True, exist_ok=True)
before = revisions()
load = Load(args.workers).start()
time.sleep(5)
t_begin = load.now()
bash = (os.path.join(os.environ.get("ProgramFiles", r"C:\Program Files"), "Git", "bin", "bash.exe")
if os.name == "nt" else "sh")
env = {**os.environ, "PYTHON": sys.executable}
script = subprocess.run([bash, str(ROOT / "scripts" / "rolling-update.sh")], cwd=ROOT, env=env,
capture_output=True, text=True, encoding="utf-8", timeout=900)
t_end = load.now()
time.sleep(5)
load.stop()
(out / "script-output.txt").write_text(f"$ sh scripts/rolling-update.sh (exit {script.returncode})\n"
+ script.stdout + script.stderr, encoding="utf-8")
with (out / "timeline.csv").open("w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=["second", "ok", "failed", "instances"])
writer.writeheader()
writer.writerows(load.per_second())
summary = {"script_exit_code": script.returncode, "update_seconds": round(t_end - t_begin, 1),
"before": before, "after": revisions(),
"during_update": load.window(t_begin, t_end), "whole_run": load.window(0, load.now()),
"client_retries_same_key": load.retries, "history": load.verify_history(),
"workload": f"{args.workers} closed-loop workers: POST (Idempotency-Key) + GET page via 127.0.0.1:8080"}
(out / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({k: summary[k] for k in ("script_exit_code", "update_seconds", "before", "after", "history")},
ensure_ascii=False, indent=2))
print(json.dumps(summary["during_update"], ensure_ascii=False))
return 0 if script.returncode == 0 else 1
if __name__ == "__main__":
raise SystemExit(main())
+8 -2
View File
@@ -80,7 +80,10 @@ services:
# One-shot schema migration: runs ONCE per `up`, before any API replica starts
# (api depends_on service_completed_successfully). Replicas never migrate themselves.
migrate:
build: .
build:
context: .
args:
BUILD_REVISION: ${BUILD_REVISION:-dev}
image: mtusi-messenger:lab3
command: ["python", "-m", "messenger.migrate"]
environment:
@@ -97,7 +100,10 @@ services:
limits: {cpus: "0.5", memory: 256m}
api:
build: .
build:
context: .
args:
BUILD_REVISION: ${BUILD_REVISION:-dev}
image: mtusi-messenger:lab3
environment: *api-env
secrets: [database_url, cache_url]
+37
View File
@@ -0,0 +1,37 @@
#!/bin/sh
# Rolling update of the API without downtime (Compose has no built-in one for a scaled service).
# 1. build the new image from the working tree;
# 2. run migrations ONCE with the new image (they must be additive / compatible with the
# running old version: expand first, contract in a later release);
# 3. for every OLD replica: start one NEW replica (scale N+1, old ones untouched), wait until
# Docker says healthy AND HAProxy has N+1 replicas UP, then stop the old one gracefully
# (SIGTERM -> readiness 503 -> HAProxy DOWN -> drain -> exit) and remove it.
# At every moment at least N replicas serve traffic. Stops on the first error (set -e);
# the old replicas that were not replaced yet keep running.
set -eu
cd "$(dirname "$0")/.."
PY="${PYTHON:-$(command -v python3 || command -v python)}"
# New image = new revision label (git commit + UTC time), visible in docker inspect and logs.
BUILD_REVISION="${BUILD_REVISION:-$(git rev-parse --short HEAD 2>/dev/null || echo dev)-$(date -u +%Y%m%dT%H%M%SZ)}"
export BUILD_REVISION
replicas="$(docker compose ps -q api | wc -l | tr -d ' ')"
[ "$replicas" -ge 1 ] || { echo "rolling-update: no running api replica" >&2; exit 2; }
old="$(docker compose ps -q api)"
echo "rolling-update: $replicas replica(s) to replace, new revision $BUILD_REVISION"
docker compose build api
docker compose run --rm --no-deps migrate
for id in $old; do
name="$(docker inspect -f '{{.Name}}' "$id" | tr -d '/')"
docker compose up -d --no-deps --no-recreate --wait --scale "api=$((replicas + 1))" api
"$PY" scripts/lbctl.py wait-up "$((replicas + 1))" --timeout 60
docker stop --time 15 "$id" >/dev/null
docker rm "$id" >/dev/null
"$PY" scripts/lbctl.py wait-up "$replicas" --timeout 60
echo "rolling-update: replaced $name"
done
for id in $(docker compose ps -q api); do
docker inspect -f '{{.Name}} revision={{index .Config.Labels "org.opencontainers.image.revision"}} {{.State.Status}}' "$id"
done
+2 -1
View File
@@ -13,6 +13,7 @@ A second signal during the drain skips the rest of it. Data stays in PostgreSQL
from __future__ import annotations
import logging
import os
import sys
import threading
from types import FrameType
@@ -75,7 +76,7 @@ def main() -> int:
)
server = GracefulServer(config, settings.drain_seconds)
log.info("starting", extra={"fields": {
"version": __version__, "env": settings.env, "host": settings.host, "port": settings.port,
"version": __version__, "revision": os.environ.get("APP_REVISION", "dev"), "env": settings.env, "host": settings.host, "port": settings.port,
"python": sys.version.split()[0], "storage": "postgresql", "instance_id": settings.instance_id,
}})
server.run()