forked from mtusi2026/mtusi2026lab3
feat: rolling update без простоя и серия отказов реплик под нагрузкой (уровень 4)
- scripts/rolling-update.sh: сборка, однократные миграции новым образом, затем для каждой старой реплики: +1 новая (healthy и UP в HAProxy), graceful stop старой; ревизия образа видна в label и логе "starting". - bench/loadgen.py: фоновая нагрузка через LB (POST с Idempotency-Key + GET), неопределённый POST повторяется тем же ключом, проверка истории на потери и дубли. - bench/failover_series.py: каждая реплика по очереди в режимах stop / kill / pause; RPS, ошибки, время выхода из ротации и возврата. - bench/rolling_under_load.py: rolling update под нагрузкой. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
78b95acabf
commit
c050642ec2
@@ -42,6 +42,12 @@ ENV PYTHONPATH=/app/src \
|
||||
SESSION_TTL_SECONDS=3600 \
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
|
||||
# Build revision (scripts/rolling-update.sh passes git commit + time): makes every rolled-out
|
||||
# image distinguishable in `docker inspect` and in the "starting" log line of each replica.
|
||||
ARG BUILD_REVISION=dev
|
||||
LABEL org.opencontainers.image.revision=$BUILD_REVISION
|
||||
ENV APP_REVISION=$BUILD_REVISION
|
||||
|
||||
USER app:app
|
||||
EXPOSE 8080
|
||||
STOPSIGNAL SIGTERM
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Level 4: stop EACH API replica in turn under load (restoring the previous one first) and
|
||||
measure RPS, errors and recovery. Modes: graceful `stop` (SIGTERM + drain), `kill` (SIGKILL)
|
||||
and `pause` (frozen process: the kernel still accepts TCP, nothing answers - the worst case
|
||||
for detection, only the active health check can find it).
|
||||
|
||||
python bench/failover_series.py --output evidence/failover # ~5 min for 2 replicas x 3 modes
|
||||
|
||||
Timeline of one round (all times from the moment the stop command is issued):
|
||||
warm-up 6 s -> stop replica -> 12 s with it down -> start it again -> wait until HAProxy
|
||||
has every replica UP (rejoin time) -> 6 s steady state.
|
||||
Measured per round: requests/ok/failed and ok-RPS before / during / after, the last failed
|
||||
request (+s), the first moment after which the stopped replica never answered again
|
||||
(= removal from rotation), time until the restarted replica is UP in HAProxy and serves
|
||||
traffic again, plus the history check (no confirmed message lost, no duplicate).
|
||||
Output: <output>/series.json (summary), <output>/timeline-<mode>-<replica>.csv (per second),
|
||||
<output>/series.md (table for REPORT.md). No tokens are written.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import datetime as dt
|
||||
import json
|
||||
import platform
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
sys.path.insert(0, str(ROOT / "bench"))
|
||||
import lbctl # noqa: E402
|
||||
from loadgen import Load # noqa: E402
|
||||
|
||||
|
||||
def docker(*args: str) -> str:
|
||||
return subprocess.run(["docker", *args], capture_output=True, text=True, check=True, timeout=120).stdout
|
||||
|
||||
|
||||
def one_round(replica: dict, mode: str, replicas: int, workers: int, out: Path) -> dict:
|
||||
load = Load(workers).start()
|
||||
time.sleep(6)
|
||||
t_stop = load.now()
|
||||
if mode == "pause":
|
||||
docker("pause", replica["id"])
|
||||
else:
|
||||
docker("stop", "--time", "15" if mode == "stop" else "0", replica["id"])
|
||||
t_stopped = load.now()
|
||||
time.sleep(12)
|
||||
t_start = load.now()
|
||||
docker("unpause" if mode == "pause" else "start", replica["id"])
|
||||
lbctl.wait_up(replicas, timeout=90)
|
||||
t_up = load.now()
|
||||
# The restarted container keeps its hostname: wait until it serves requests again.
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline:
|
||||
if any(e[3] == replica["hostname"] and e[0] > t_up for e in load.events):
|
||||
break
|
||||
time.sleep(0.1)
|
||||
t_serves = load.now()
|
||||
time.sleep(6)
|
||||
load.stop()
|
||||
t_end = load.now()
|
||||
|
||||
failed_after_stop = [e for e in load.events if e[0] >= t_stop and e[2] not in (200, 201)]
|
||||
served_by_target = [e[0] for e in load.events if e[3] == replica["hostname"] and t_stop <= e[0] < t_start]
|
||||
out_of_rotation = (max(served_by_target) - t_stop) if served_by_target else 0.0
|
||||
rows = load.per_second()
|
||||
with (out / f"timeline-{mode}-{replica['name']}.csv").open("w", newline="", encoding="utf-8") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=["second", "ok", "failed", "instances"])
|
||||
writer.writeheader()
|
||||
for row in rows:
|
||||
row = dict(row)
|
||||
row["instances"] = row["instances"].replace(replica["hostname"], "TARGET")
|
||||
writer.writerow(row)
|
||||
state = json.loads(docker("inspect", replica["id"]))[0]["State"]
|
||||
return {
|
||||
"mode": mode, "replica": replica["name"], "instance_id": replica["hostname"],
|
||||
"stop_command_seconds": round(t_stopped - t_stop, 2),
|
||||
"before": load.window(0.5, t_stop), "during": load.window(t_stop, t_start), "after": load.window(t_serves, t_end),
|
||||
"failed_after_stop": len(failed_after_stop),
|
||||
"failed_statuses": sorted({e[2] for e in failed_after_stop}),
|
||||
"last_failure_after_stop_s": round(max((e[0] for e in failed_after_stop), default=t_stop) - t_stop, 3),
|
||||
"target_last_served_after_stop_s": round(out_of_rotation, 3),
|
||||
"rejoin_up_in_lb_s": round(t_up - t_start, 2), "rejoin_serving_s": round(t_serves - t_start, 2),
|
||||
"client_retries_same_key": load.retries,
|
||||
"history": load.verify_history(),
|
||||
"restarted_state": f"{state['Status']} health={(state.get('Health') or {}).get('Status')}",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("--output", default="evidence/failover")
|
||||
parser.add_argument("--modes", default="stop,kill,pause")
|
||||
parser.add_argument("--workers", type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
out = ROOT / args.output
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
replicas = sorted((r for r in lbctl.replicas() if r["state"] == "running"), key=lambda r: r["name"])
|
||||
assert len(replicas) >= 2, "need at least two running replicas"
|
||||
lbctl.wait_up(len(replicas))
|
||||
results = []
|
||||
for mode in args.modes.split(","):
|
||||
for replica in replicas: # each replica in turn; the previous one is already restored
|
||||
print(f"== {mode} {replica['name']} ...", flush=True)
|
||||
result = one_round(replica, mode, len(replicas), args.workers, out)
|
||||
results.append(result)
|
||||
print(json.dumps({k: result[k] for k in ("failed_after_stop", "last_failure_after_stop_s",
|
||||
"target_last_served_after_stop_s", "rejoin_serving_s")}), flush=True)
|
||||
meta = {"date": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
|
||||
"commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True,
|
||||
text=True).stdout.strip(),
|
||||
"host": f"{platform.system()} {platform.release()}, docker {docker('version', '--format', '{{.Server.Version}}').strip()}",
|
||||
"workload": f"{args.workers} closed-loop workers from the host via 127.0.0.1:8080: POST message "
|
||||
"(Idempotency-Key, uncertain -> same key) + GET last page; client timeout 3 s",
|
||||
"lb": "HAProxy: check GET /health/ready inter 1s fastinter 500ms fall 2 rise 2, timeout check 1s; "
|
||||
"retry-on conn-failure + redispatch",
|
||||
"api": "DRAIN_SECONDS=3, SHUTDOWN_TIMEOUT_SECONDS=8, stop_grace_period 15s"}
|
||||
(out / "series.json").write_text(json.dumps({"meta": meta, "rounds": results}, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8")
|
||||
lines = ["| Режим | Реплика | Запросов во время отказа | Ошибок после stop | ok-RPS до / во время / после | "
|
||||
"Последняя ошибка, с | Цель ушла из ротации, с | Возврат в LB / обслуживает, с | Потери / дубли |",
|
||||
"| --- | --- | --- | --- | --- | --- | --- | --- | --- |"]
|
||||
for r in results:
|
||||
h = r["history"]
|
||||
lines.append(f"| {r['mode']} | {r['replica']} | {r['during']['requests']} | {r['failed_after_stop']} "
|
||||
f"{r['failed_statuses'] or ''} | {r['before']['ok_rps']} / {r['during']['ok_rps']} / {r['after']['ok_rps']} | "
|
||||
f"{r['last_failure_after_stop_s']} | {r['target_last_served_after_stop_s']} | "
|
||||
f"{r['rejoin_up_in_lb_s']} / {r['rejoin_serving_s']} | "
|
||||
f"{h['lost_confirmed']} / {h['duplicate_ids'] + h['duplicate_texts']} |")
|
||||
(out / "series.md").write_text("\n".join([f"Дата {meta['date']}, commit {meta['commit']}. {meta['workload']}.", "",
|
||||
*lines, ""]), encoding="utf-8")
|
||||
print("\n".join(lines))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,212 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Background HTTP traffic through the LB for failover / rolling-update experiments (level 4).
|
||||
|
||||
Closed loop, N worker threads, each: POST a message (Idempotency-Key) + GET the last page.
|
||||
An uncertain POST (timeout, connection error, 5xx) is retried with the SAME key and body,
|
||||
exactly as a correct client must do; the history is checked afterwards for losses and
|
||||
duplicates. Every request is recorded as (t, kind, status, instance, latency).
|
||||
This is a correctness/availability probe, not a capacity benchmark (see bench/track1 for that).
|
||||
|
||||
python bench/loadgen.py --duration 30 --workers 4 # prints a summary JSON
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
||||
|
||||
|
||||
class Http:
|
||||
"""Minimal stdlib client with keep-alive per thread (http.client under urllib is not)."""
|
||||
|
||||
def __init__(self, base_url: str, timeout: float):
|
||||
import http.client
|
||||
from urllib.parse import urlsplit
|
||||
parts = urlsplit(base_url)
|
||||
self.host, self.port = parts.hostname, parts.port or 80
|
||||
self.timeout = timeout
|
||||
self.factory = lambda: http.client.HTTPConnection(self.host, self.port, timeout=self.timeout)
|
||||
self.conn = self.factory()
|
||||
|
||||
def request(self, method: str, path: str, body: bytes | None = None, headers: dict | None = None):
|
||||
try:
|
||||
self.conn.request(method, path, body=body, headers=headers or {})
|
||||
response = self.conn.getresponse()
|
||||
data = response.read()
|
||||
if response.getheader("connection", "").lower() == "close":
|
||||
self.conn.close()
|
||||
self.conn = self.factory()
|
||||
return response.status, dict(response.getheaders()), data
|
||||
except Exception:
|
||||
self.conn.close()
|
||||
self.conn = self.factory()
|
||||
raise
|
||||
|
||||
|
||||
def call(method: str, path: str, body=None, headers=None, timeout=10.0):
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
h = {"Content-Type": "application/json", **(headers or {})}
|
||||
req = urllib.request.Request(BASE_URL + path, data=data, headers=h, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return r.status, json.loads(r.read() or b"null")
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, json.loads(e.read() or b"null")
|
||||
|
||||
|
||||
def make_user(label: str) -> dict:
|
||||
username = f"{label}_{uuid.uuid4().hex[:12]}"
|
||||
password = "Load-password-" + uuid.uuid4().hex
|
||||
status, user = call("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
|
||||
assert status == 201, (status, user)
|
||||
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
status, session = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
|
||||
assert status == 201, (status, session)
|
||||
return {"id": user["id"], "token": session["token"]}
|
||||
|
||||
|
||||
class Load:
|
||||
def __init__(self, workers: int = 4, timeout: float = 3.0, pause: float = 0.0):
|
||||
self.workers = workers
|
||||
self.timeout = timeout
|
||||
self.pause = pause
|
||||
self.alice, self.bob = make_user("load"), make_user("load")
|
||||
status, chat = call("POST", "/api/v1/chats", {"title": "load", "member_ids": [self.alice["id"], self.bob["id"]]},
|
||||
headers={"Authorization": "Bearer " + self.alice["token"]})
|
||||
assert status == 201, chat
|
||||
self.chat = chat
|
||||
self.stop_event = threading.Event()
|
||||
self.lock = threading.Lock()
|
||||
self.events: list[tuple[float, str, int, str, float]] = []
|
||||
self.confirmed: dict[str, dict] = {}
|
||||
self.retries = 0
|
||||
self.t0 = time.monotonic()
|
||||
self.threads = [threading.Thread(target=self._worker, args=(i,), daemon=True) for i in range(workers)]
|
||||
|
||||
def now(self) -> float:
|
||||
return time.monotonic() - self.t0
|
||||
|
||||
def _record(self, kind: str, status: int, instance: str, latency: float) -> None:
|
||||
with self.lock:
|
||||
self.events.append((round(self.now(), 4), kind, status, instance, round(latency * 1000, 2)))
|
||||
|
||||
def _worker(self, index: int) -> None:
|
||||
http = Http(BASE_URL, self.timeout)
|
||||
auth = {"Authorization": "Bearer " + self.alice["token"]}
|
||||
path = f"/api/v1/chats/{self.chat['id']}/messages"
|
||||
n, key = 0, uuid.uuid4().hex
|
||||
text = f"w{index}-0-{key[:6]}"
|
||||
while not self.stop_event.is_set():
|
||||
started = time.monotonic()
|
||||
try:
|
||||
status, headers, data = http.request("POST", path, json.dumps({"text": text}).encode(),
|
||||
{**auth, "Idempotency-Key": key, "Content-Type": "application/json"})
|
||||
instance = headers.get("x-instance-id") or headers.get("X-Instance-Id") or "-"
|
||||
except Exception:
|
||||
status, instance, data = 0, "-", b""
|
||||
self._record("POST", status, instance, time.monotonic() - started)
|
||||
if status == 201:
|
||||
msg = json.loads(data)
|
||||
with self.lock:
|
||||
self.confirmed[msg["id"]] = msg
|
||||
n += 1
|
||||
key = uuid.uuid4().hex
|
||||
text = f"w{index}-{n}-{key[:6]}"
|
||||
elif status in (0, 502, 503, 504, 500):
|
||||
with self.lock:
|
||||
self.retries += 1 # same key + same text on the next iteration
|
||||
started = time.monotonic()
|
||||
try:
|
||||
status, headers, _ = http.request("GET", path + "?limit=5", None, auth)
|
||||
instance = headers.get("x-instance-id") or headers.get("X-Instance-Id") or "-"
|
||||
except Exception:
|
||||
status, instance = 0, "-"
|
||||
self._record("GET", status, instance, time.monotonic() - started)
|
||||
if self.pause:
|
||||
time.sleep(self.pause)
|
||||
|
||||
def start(self) -> "Load":
|
||||
self.t0 = time.monotonic()
|
||||
for t in self.threads:
|
||||
t.start()
|
||||
return self
|
||||
|
||||
def stop(self) -> None:
|
||||
self.stop_event.set()
|
||||
for t in self.threads:
|
||||
t.join(timeout=15)
|
||||
|
||||
def verify_history(self) -> dict:
|
||||
"""Walk the whole history with the second user: no confirmed message lost, no duplicate."""
|
||||
items, cursor = [], None
|
||||
while True:
|
||||
q = "?limit=100" + (f"&cursor={cursor}" if cursor else "")
|
||||
status, page = call("GET", f"/api/v1/chats/{self.chat['id']}/messages{q}",
|
||||
headers={"Authorization": "Bearer " + self.bob["token"]}, timeout=30)
|
||||
assert status == 200, page
|
||||
items += page["items"]
|
||||
cursor = page["next_cursor"]
|
||||
if not cursor:
|
||||
break
|
||||
ids = [m["id"] for m in items]
|
||||
texts = [m["text"] for m in items]
|
||||
lost = [i for i in self.confirmed if i not in set(ids)]
|
||||
changed = [m for m in items if m["id"] in self.confirmed and self.confirmed[m["id"]] != m]
|
||||
return {"history_messages": len(items), "confirmed_messages": len(self.confirmed),
|
||||
"lost_confirmed": len(lost), "altered_confirmed": len(changed),
|
||||
"duplicate_ids": len(ids) - len(set(ids)), "duplicate_texts": len(texts) - len(set(texts)),
|
||||
"unconfirmed_but_stored": len(set(ids) - set(self.confirmed))}
|
||||
|
||||
# ------------------------------------------------------------------ summaries
|
||||
def window(self, start: float, end: float) -> dict:
|
||||
ev = [e for e in self.events if start <= e[0] < end]
|
||||
ok = [e for e in ev if e[2] in (200, 201)]
|
||||
lat = sorted(e[4] for e in ok)
|
||||
|
||||
def pct(p: float):
|
||||
return lat[max(0, math.ceil(p * len(lat)) - 1)] if lat else None
|
||||
span = max(end - start, 1e-9)
|
||||
return {"seconds": round(span, 2), "requests": len(ev), "ok": len(ok), "failed": len(ev) - len(ok),
|
||||
"ok_rps": round(len(ok) / span, 1), "error_ratio": round((len(ev) - len(ok)) / len(ev), 5) if ev else None,
|
||||
"statuses": dict(Counter(e[2] for e in ev)), "instances": dict(Counter(e[3] for e in ok)),
|
||||
"p50_ms": pct(.5), "p95_ms": pct(.95), "p99_ms": pct(.99)}
|
||||
|
||||
def per_second(self) -> list[dict]:
|
||||
buckets: dict[int, list] = defaultdict(list)
|
||||
for e in self.events:
|
||||
buckets[int(e[0])].append(e)
|
||||
rows = []
|
||||
for second in sorted(buckets):
|
||||
ev = buckets[second]
|
||||
rows.append({"second": second, "ok": sum(1 for e in ev if e[2] in (200, 201)),
|
||||
"failed": sum(1 for e in ev if e[2] not in (200, 201)),
|
||||
"instances": "|".join(f"{k}:{v}" for k, v in sorted(Counter(e[3] for e in ev).items()))})
|
||||
return rows
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("--duration", type=float, default=20)
|
||||
parser.add_argument("--workers", type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
load = Load(args.workers).start()
|
||||
time.sleep(args.duration)
|
||||
load.stop()
|
||||
print(json.dumps({"total": load.window(0, load.now()), "history": load.verify_history(),
|
||||
"client_retries_same_key": load.retries}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Level 4: scripts/rolling-update.sh while bench/loadgen.py traffic runs through the LB.
|
||||
|
||||
python bench/rolling_under_load.py --output evidence/rolling-update
|
||||
|
||||
Writes <output>/summary.json (errors, RPS, history check, revisions before/after),
|
||||
<output>/timeline.csv (per second ok/failed/instances) and <output>/script-output.txt.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
sys.path.insert(0, str(ROOT / "bench"))
|
||||
import lbctl # noqa: E402
|
||||
from loadgen import Load # noqa: E402
|
||||
|
||||
|
||||
def revisions() -> dict[str, str]:
|
||||
result = {}
|
||||
for r in lbctl.replicas():
|
||||
if r["state"] == "running":
|
||||
label = subprocess.run(["docker", "inspect", "-f", '{{index .Config.Labels "org.opencontainers.image.revision"}}',
|
||||
r["id"]], capture_output=True, text=True).stdout.strip()
|
||||
result[r["name"]] = f"{r['hostname']} revision={label}"
|
||||
return result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("--output", default="evidence/rolling-update")
|
||||
parser.add_argument("--workers", type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
out = ROOT / args.output
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
before = revisions()
|
||||
load = Load(args.workers).start()
|
||||
time.sleep(5)
|
||||
t_begin = load.now()
|
||||
bash = (os.path.join(os.environ.get("ProgramFiles", r"C:\Program Files"), "Git", "bin", "bash.exe")
|
||||
if os.name == "nt" else "sh")
|
||||
env = {**os.environ, "PYTHON": sys.executable}
|
||||
script = subprocess.run([bash, str(ROOT / "scripts" / "rolling-update.sh")], cwd=ROOT, env=env,
|
||||
capture_output=True, text=True, encoding="utf-8", timeout=900)
|
||||
t_end = load.now()
|
||||
time.sleep(5)
|
||||
load.stop()
|
||||
(out / "script-output.txt").write_text(f"$ sh scripts/rolling-update.sh (exit {script.returncode})\n"
|
||||
+ script.stdout + script.stderr, encoding="utf-8")
|
||||
with (out / "timeline.csv").open("w", newline="", encoding="utf-8") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=["second", "ok", "failed", "instances"])
|
||||
writer.writeheader()
|
||||
writer.writerows(load.per_second())
|
||||
summary = {"script_exit_code": script.returncode, "update_seconds": round(t_end - t_begin, 1),
|
||||
"before": before, "after": revisions(),
|
||||
"during_update": load.window(t_begin, t_end), "whole_run": load.window(0, load.now()),
|
||||
"client_retries_same_key": load.retries, "history": load.verify_history(),
|
||||
"workload": f"{args.workers} closed-loop workers: POST (Idempotency-Key) + GET page via 127.0.0.1:8080"}
|
||||
(out / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({k: summary[k] for k in ("script_exit_code", "update_seconds", "before", "after", "history")},
|
||||
ensure_ascii=False, indent=2))
|
||||
print(json.dumps(summary["during_update"], ensure_ascii=False))
|
||||
return 0 if script.returncode == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+8
-2
@@ -80,7 +80,10 @@ services:
|
||||
# One-shot schema migration: runs ONCE per `up`, before any API replica starts
|
||||
# (api depends_on service_completed_successfully). Replicas never migrate themselves.
|
||||
migrate:
|
||||
build: .
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
BUILD_REVISION: ${BUILD_REVISION:-dev}
|
||||
image: mtusi-messenger:lab3
|
||||
command: ["python", "-m", "messenger.migrate"]
|
||||
environment:
|
||||
@@ -97,7 +100,10 @@ services:
|
||||
limits: {cpus: "0.5", memory: 256m}
|
||||
|
||||
api:
|
||||
build: .
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
BUILD_REVISION: ${BUILD_REVISION:-dev}
|
||||
image: mtusi-messenger:lab3
|
||||
environment: *api-env
|
||||
secrets: [database_url, cache_url]
|
||||
|
||||
Executable
+37
@@ -0,0 +1,37 @@
|
||||
#!/bin/sh
|
||||
# Rolling update of the API without downtime (Compose has no built-in one for a scaled service).
|
||||
# 1. build the new image from the working tree;
|
||||
# 2. run migrations ONCE with the new image (they must be additive / compatible with the
|
||||
# running old version: expand first, contract in a later release);
|
||||
# 3. for every OLD replica: start one NEW replica (scale N+1, old ones untouched), wait until
|
||||
# Docker says healthy AND HAProxy has N+1 replicas UP, then stop the old one gracefully
|
||||
# (SIGTERM -> readiness 503 -> HAProxy DOWN -> drain -> exit) and remove it.
|
||||
# At every moment at least N replicas serve traffic. Stops on the first error (set -e);
|
||||
# the old replicas that were not replaced yet keep running.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
PY="${PYTHON:-$(command -v python3 || command -v python)}"
|
||||
# New image = new revision label (git commit + UTC time), visible in docker inspect and logs.
|
||||
BUILD_REVISION="${BUILD_REVISION:-$(git rev-parse --short HEAD 2>/dev/null || echo dev)-$(date -u +%Y%m%dT%H%M%SZ)}"
|
||||
export BUILD_REVISION
|
||||
|
||||
replicas="$(docker compose ps -q api | wc -l | tr -d ' ')"
|
||||
[ "$replicas" -ge 1 ] || { echo "rolling-update: no running api replica" >&2; exit 2; }
|
||||
old="$(docker compose ps -q api)"
|
||||
echo "rolling-update: $replicas replica(s) to replace, new revision $BUILD_REVISION"
|
||||
|
||||
docker compose build api
|
||||
docker compose run --rm --no-deps migrate
|
||||
|
||||
for id in $old; do
|
||||
name="$(docker inspect -f '{{.Name}}' "$id" | tr -d '/')"
|
||||
docker compose up -d --no-deps --no-recreate --wait --scale "api=$((replicas + 1))" api
|
||||
"$PY" scripts/lbctl.py wait-up "$((replicas + 1))" --timeout 60
|
||||
docker stop --time 15 "$id" >/dev/null
|
||||
docker rm "$id" >/dev/null
|
||||
"$PY" scripts/lbctl.py wait-up "$replicas" --timeout 60
|
||||
echo "rolling-update: replaced $name"
|
||||
done
|
||||
for id in $(docker compose ps -q api); do
|
||||
docker inspect -f '{{.Name}} revision={{index .Config.Labels "org.opencontainers.image.revision"}} {{.State.Status}}' "$id"
|
||||
done
|
||||
@@ -13,6 +13,7 @@ A second signal during the drain skips the rest of it. Data stays in PostgreSQL
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
from types import FrameType
|
||||
@@ -75,7 +76,7 @@ def main() -> int:
|
||||
)
|
||||
server = GracefulServer(config, settings.drain_seconds)
|
||||
log.info("starting", extra={"fields": {
|
||||
"version": __version__, "env": settings.env, "host": settings.host, "port": settings.port,
|
||||
"version": __version__, "revision": os.environ.get("APP_REVISION", "dev"), "env": settings.env, "host": settings.host, "port": settings.port,
|
||||
"python": sys.version.split()[0], "storage": "postgresql", "instance_id": settings.instance_id,
|
||||
}})
|
||||
server.run()
|
||||
|
||||
Reference in New Issue
Block a user