From c050642ec25104f3e9445ec5d8f8d7876a686c0d Mon Sep 17 00:00:00 2001 From: Alexey Bodanyuk <144894236+wfa22@users.noreply.github.com> Date: Fri, 16 Oct 2026 13:39:54 +0300 Subject: [PATCH] =?UTF-8?q?feat:=20rolling=20update=20=D0=B1=D0=B5=D0=B7?= =?UTF-8?q?=20=D0=BF=D1=80=D0=BE=D1=81=D1=82=D0=BE=D1=8F=20=D0=B8=20=D1=81?= =?UTF-8?q?=D0=B5=D1=80=D0=B8=D1=8F=20=D0=BE=D1=82=D0=BA=D0=B0=D0=B7=D0=BE?= =?UTF-8?q?=D0=B2=20=D1=80=D0=B5=D0=BF=D0=BB=D0=B8=D0=BA=20=D0=BF=D0=BE?= =?UTF-8?q?=D0=B4=20=D0=BD=D0=B0=D0=B3=D1=80=D1=83=D0=B7=D0=BA=D0=BE=D0=B9?= =?UTF-8?q?=20(=D1=83=D1=80=D0=BE=D0=B2=D0=B5=D0=BD=D1=8C=204)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - scripts/rolling-update.sh: сборка, однократные миграции новым образом, затем для каждой старой реплики: +1 новая (healthy и UP в HAProxy), graceful stop старой; ревизия образа видна в label и логе "starting". - bench/loadgen.py: фоновая нагрузка через LB (POST с Idempotency-Key + GET), неопределённый POST повторяется тем же ключом, проверка истории на потери и дубли. - bench/failover_series.py: каждая реплика по очереди в режимах stop / kill / pause; RPS, ошибки, время выхода из ротации и возврата. - bench/rolling_under_load.py: rolling update под нагрузкой. Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 6 + bench/failover_series.py | 141 ++++++++++++++++++++++++ bench/loadgen.py | 212 ++++++++++++++++++++++++++++++++++++ bench/rolling_under_load.py | 75 +++++++++++++ compose.yaml | 10 +- scripts/rolling-update.sh | 37 +++++++ src/messenger/__main__.py | 3 +- 7 files changed, 481 insertions(+), 3 deletions(-) create mode 100644 bench/failover_series.py create mode 100644 bench/loadgen.py create mode 100644 bench/rolling_under_load.py create mode 100755 scripts/rolling-update.sh diff --git a/Dockerfile b/Dockerfile index 12bb32c..f46ac0b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,6 +42,12 @@ ENV PYTHONPATH=/app/src \ SESSION_TTL_SECONDS=3600 \ DATABASE_URL_FILE=/run/secrets/database_url +# Build revision (scripts/rolling-update.sh passes git commit + time): makes every rolled-out +# image distinguishable in `docker inspect` and in the "starting" log line of each replica. +ARG BUILD_REVISION=dev +LABEL org.opencontainers.image.revision=$BUILD_REVISION +ENV APP_REVISION=$BUILD_REVISION + USER app:app EXPOSE 8080 STOPSIGNAL SIGTERM diff --git a/bench/failover_series.py b/bench/failover_series.py new file mode 100644 index 0000000..b4c5daf --- /dev/null +++ b/bench/failover_series.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Level 4: stop EACH API replica in turn under load (restoring the previous one first) and +measure RPS, errors and recovery. Modes: graceful `stop` (SIGTERM + drain), `kill` (SIGKILL) +and `pause` (frozen process: the kernel still accepts TCP, nothing answers - the worst case +for detection, only the active health check can find it). + + python bench/failover_series.py --output evidence/failover # ~5 min for 2 replicas x 3 modes + +Timeline of one round (all times from the moment the stop command is issued): + warm-up 6 s -> stop replica -> 12 s with it down -> start it again -> wait until HAProxy + has every replica UP (rejoin time) -> 6 s steady state. +Measured per round: requests/ok/failed and ok-RPS before / during / after, the last failed +request (+s), the first moment after which the stopped replica never answered again +(= removal from rotation), time until the restarted replica is UP in HAProxy and serves +traffic again, plus the history check (no confirmed message lost, no duplicate). +Output: /series.json (summary), /timeline--.csv (per second), +/series.md (table for REPORT.md). No tokens are written. +""" +from __future__ import annotations + +import argparse +import csv +import datetime as dt +import json +import platform +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +sys.path.insert(0, str(ROOT / "bench")) +import lbctl # noqa: E402 +from loadgen import Load # noqa: E402 + + +def docker(*args: str) -> str: + return subprocess.run(["docker", *args], capture_output=True, text=True, check=True, timeout=120).stdout + + +def one_round(replica: dict, mode: str, replicas: int, workers: int, out: Path) -> dict: + load = Load(workers).start() + time.sleep(6) + t_stop = load.now() + if mode == "pause": + docker("pause", replica["id"]) + else: + docker("stop", "--time", "15" if mode == "stop" else "0", replica["id"]) + t_stopped = load.now() + time.sleep(12) + t_start = load.now() + docker("unpause" if mode == "pause" else "start", replica["id"]) + lbctl.wait_up(replicas, timeout=90) + t_up = load.now() + # The restarted container keeps its hostname: wait until it serves requests again. + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + if any(e[3] == replica["hostname"] and e[0] > t_up for e in load.events): + break + time.sleep(0.1) + t_serves = load.now() + time.sleep(6) + load.stop() + t_end = load.now() + + failed_after_stop = [e for e in load.events if e[0] >= t_stop and e[2] not in (200, 201)] + served_by_target = [e[0] for e in load.events if e[3] == replica["hostname"] and t_stop <= e[0] < t_start] + out_of_rotation = (max(served_by_target) - t_stop) if served_by_target else 0.0 + rows = load.per_second() + with (out / f"timeline-{mode}-{replica['name']}.csv").open("w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=["second", "ok", "failed", "instances"]) + writer.writeheader() + for row in rows: + row = dict(row) + row["instances"] = row["instances"].replace(replica["hostname"], "TARGET") + writer.writerow(row) + state = json.loads(docker("inspect", replica["id"]))[0]["State"] + return { + "mode": mode, "replica": replica["name"], "instance_id": replica["hostname"], + "stop_command_seconds": round(t_stopped - t_stop, 2), + "before": load.window(0.5, t_stop), "during": load.window(t_stop, t_start), "after": load.window(t_serves, t_end), + "failed_after_stop": len(failed_after_stop), + "failed_statuses": sorted({e[2] for e in failed_after_stop}), + "last_failure_after_stop_s": round(max((e[0] for e in failed_after_stop), default=t_stop) - t_stop, 3), + "target_last_served_after_stop_s": round(out_of_rotation, 3), + "rejoin_up_in_lb_s": round(t_up - t_start, 2), "rejoin_serving_s": round(t_serves - t_start, 2), + "client_retries_same_key": load.retries, + "history": load.verify_history(), + "restarted_state": f"{state['Status']} health={(state.get('Health') or {}).get('Status')}", + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--output", default="evidence/failover") + parser.add_argument("--modes", default="stop,kill,pause") + parser.add_argument("--workers", type=int, default=4) + args = parser.parse_args() + out = ROOT / args.output + out.mkdir(parents=True, exist_ok=True) + replicas = sorted((r for r in lbctl.replicas() if r["state"] == "running"), key=lambda r: r["name"]) + assert len(replicas) >= 2, "need at least two running replicas" + lbctl.wait_up(len(replicas)) + results = [] + for mode in args.modes.split(","): + for replica in replicas: # each replica in turn; the previous one is already restored + print(f"== {mode} {replica['name']} ...", flush=True) + result = one_round(replica, mode, len(replicas), args.workers, out) + results.append(result) + print(json.dumps({k: result[k] for k in ("failed_after_stop", "last_failure_after_stop_s", + "target_last_served_after_stop_s", "rejoin_serving_s")}), flush=True) + meta = {"date": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"), + "commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, + text=True).stdout.strip(), + "host": f"{platform.system()} {platform.release()}, docker {docker('version', '--format', '{{.Server.Version}}').strip()}", + "workload": f"{args.workers} closed-loop workers from the host via 127.0.0.1:8080: POST message " + "(Idempotency-Key, uncertain -> same key) + GET last page; client timeout 3 s", + "lb": "HAProxy: check GET /health/ready inter 1s fastinter 500ms fall 2 rise 2, timeout check 1s; " + "retry-on conn-failure + redispatch", + "api": "DRAIN_SECONDS=3, SHUTDOWN_TIMEOUT_SECONDS=8, stop_grace_period 15s"} + (out / "series.json").write_text(json.dumps({"meta": meta, "rounds": results}, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8") + lines = ["| Режим | Реплика | Запросов во время отказа | Ошибок после stop | ok-RPS до / во время / после | " + "Последняя ошибка, с | Цель ушла из ротации, с | Возврат в LB / обслуживает, с | Потери / дубли |", + "| --- | --- | --- | --- | --- | --- | --- | --- | --- |"] + for r in results: + h = r["history"] + lines.append(f"| {r['mode']} | {r['replica']} | {r['during']['requests']} | {r['failed_after_stop']} " + f"{r['failed_statuses'] or ''} | {r['before']['ok_rps']} / {r['during']['ok_rps']} / {r['after']['ok_rps']} | " + f"{r['last_failure_after_stop_s']} | {r['target_last_served_after_stop_s']} | " + f"{r['rejoin_up_in_lb_s']} / {r['rejoin_serving_s']} | " + f"{h['lost_confirmed']} / {h['duplicate_ids'] + h['duplicate_texts']} |") + (out / "series.md").write_text("\n".join([f"Дата {meta['date']}, commit {meta['commit']}. {meta['workload']}.", "", + *lines, ""]), encoding="utf-8") + print("\n".join(lines)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bench/loadgen.py b/bench/loadgen.py new file mode 100644 index 0000000..97a94de --- /dev/null +++ b/bench/loadgen.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""Background HTTP traffic through the LB for failover / rolling-update experiments (level 4). + +Closed loop, N worker threads, each: POST a message (Idempotency-Key) + GET the last page. +An uncertain POST (timeout, connection error, 5xx) is retried with the SAME key and body, +exactly as a correct client must do; the history is checked afterwards for losses and +duplicates. Every request is recorded as (t, kind, status, instance, latency). +This is a correctness/availability probe, not a capacity benchmark (see bench/track1 for that). + + python bench/loadgen.py --duration 30 --workers 4 # prints a summary JSON +""" +from __future__ import annotations + +import argparse +import base64 +import json +import math +import os +import threading +import time +import uuid +from collections import Counter, defaultdict + +import urllib.error +import urllib.request + +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") + + +class Http: + """Minimal stdlib client with keep-alive per thread (http.client under urllib is not).""" + + def __init__(self, base_url: str, timeout: float): + import http.client + from urllib.parse import urlsplit + parts = urlsplit(base_url) + self.host, self.port = parts.hostname, parts.port or 80 + self.timeout = timeout + self.factory = lambda: http.client.HTTPConnection(self.host, self.port, timeout=self.timeout) + self.conn = self.factory() + + def request(self, method: str, path: str, body: bytes | None = None, headers: dict | None = None): + try: + self.conn.request(method, path, body=body, headers=headers or {}) + response = self.conn.getresponse() + data = response.read() + if response.getheader("connection", "").lower() == "close": + self.conn.close() + self.conn = self.factory() + return response.status, dict(response.getheaders()), data + except Exception: + self.conn.close() + self.conn = self.factory() + raise + + +def call(method: str, path: str, body=None, headers=None, timeout=10.0): + data = json.dumps(body).encode() if body is not None else None + h = {"Content-Type": "application/json", **(headers or {})} + req = urllib.request.Request(BASE_URL + path, data=data, headers=h, method=method) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.status, json.loads(r.read() or b"null") + except urllib.error.HTTPError as e: + return e.code, json.loads(e.read() or b"null") + + +def make_user(label: str) -> dict: + username = f"{label}_{uuid.uuid4().hex[:12]}" + password = "Load-password-" + uuid.uuid4().hex + status, user = call("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password}) + assert status == 201, (status, user) + cred = base64.b64encode(f"{username}:{password}".encode()).decode() + status, session = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) + assert status == 201, (status, session) + return {"id": user["id"], "token": session["token"]} + + +class Load: + def __init__(self, workers: int = 4, timeout: float = 3.0, pause: float = 0.0): + self.workers = workers + self.timeout = timeout + self.pause = pause + self.alice, self.bob = make_user("load"), make_user("load") + status, chat = call("POST", "/api/v1/chats", {"title": "load", "member_ids": [self.alice["id"], self.bob["id"]]}, + headers={"Authorization": "Bearer " + self.alice["token"]}) + assert status == 201, chat + self.chat = chat + self.stop_event = threading.Event() + self.lock = threading.Lock() + self.events: list[tuple[float, str, int, str, float]] = [] + self.confirmed: dict[str, dict] = {} + self.retries = 0 + self.t0 = time.monotonic() + self.threads = [threading.Thread(target=self._worker, args=(i,), daemon=True) for i in range(workers)] + + def now(self) -> float: + return time.monotonic() - self.t0 + + def _record(self, kind: str, status: int, instance: str, latency: float) -> None: + with self.lock: + self.events.append((round(self.now(), 4), kind, status, instance, round(latency * 1000, 2))) + + def _worker(self, index: int) -> None: + http = Http(BASE_URL, self.timeout) + auth = {"Authorization": "Bearer " + self.alice["token"]} + path = f"/api/v1/chats/{self.chat['id']}/messages" + n, key = 0, uuid.uuid4().hex + text = f"w{index}-0-{key[:6]}" + while not self.stop_event.is_set(): + started = time.monotonic() + try: + status, headers, data = http.request("POST", path, json.dumps({"text": text}).encode(), + {**auth, "Idempotency-Key": key, "Content-Type": "application/json"}) + instance = headers.get("x-instance-id") or headers.get("X-Instance-Id") or "-" + except Exception: + status, instance, data = 0, "-", b"" + self._record("POST", status, instance, time.monotonic() - started) + if status == 201: + msg = json.loads(data) + with self.lock: + self.confirmed[msg["id"]] = msg + n += 1 + key = uuid.uuid4().hex + text = f"w{index}-{n}-{key[:6]}" + elif status in (0, 502, 503, 504, 500): + with self.lock: + self.retries += 1 # same key + same text on the next iteration + started = time.monotonic() + try: + status, headers, _ = http.request("GET", path + "?limit=5", None, auth) + instance = headers.get("x-instance-id") or headers.get("X-Instance-Id") or "-" + except Exception: + status, instance = 0, "-" + self._record("GET", status, instance, time.monotonic() - started) + if self.pause: + time.sleep(self.pause) + + def start(self) -> "Load": + self.t0 = time.monotonic() + for t in self.threads: + t.start() + return self + + def stop(self) -> None: + self.stop_event.set() + for t in self.threads: + t.join(timeout=15) + + def verify_history(self) -> dict: + """Walk the whole history with the second user: no confirmed message lost, no duplicate.""" + items, cursor = [], None + while True: + q = "?limit=100" + (f"&cursor={cursor}" if cursor else "") + status, page = call("GET", f"/api/v1/chats/{self.chat['id']}/messages{q}", + headers={"Authorization": "Bearer " + self.bob["token"]}, timeout=30) + assert status == 200, page + items += page["items"] + cursor = page["next_cursor"] + if not cursor: + break + ids = [m["id"] for m in items] + texts = [m["text"] for m in items] + lost = [i for i in self.confirmed if i not in set(ids)] + changed = [m for m in items if m["id"] in self.confirmed and self.confirmed[m["id"]] != m] + return {"history_messages": len(items), "confirmed_messages": len(self.confirmed), + "lost_confirmed": len(lost), "altered_confirmed": len(changed), + "duplicate_ids": len(ids) - len(set(ids)), "duplicate_texts": len(texts) - len(set(texts)), + "unconfirmed_but_stored": len(set(ids) - set(self.confirmed))} + + # ------------------------------------------------------------------ summaries + def window(self, start: float, end: float) -> dict: + ev = [e for e in self.events if start <= e[0] < end] + ok = [e for e in ev if e[2] in (200, 201)] + lat = sorted(e[4] for e in ok) + + def pct(p: float): + return lat[max(0, math.ceil(p * len(lat)) - 1)] if lat else None + span = max(end - start, 1e-9) + return {"seconds": round(span, 2), "requests": len(ev), "ok": len(ok), "failed": len(ev) - len(ok), + "ok_rps": round(len(ok) / span, 1), "error_ratio": round((len(ev) - len(ok)) / len(ev), 5) if ev else None, + "statuses": dict(Counter(e[2] for e in ev)), "instances": dict(Counter(e[3] for e in ok)), + "p50_ms": pct(.5), "p95_ms": pct(.95), "p99_ms": pct(.99)} + + def per_second(self) -> list[dict]: + buckets: dict[int, list] = defaultdict(list) + for e in self.events: + buckets[int(e[0])].append(e) + rows = [] + for second in sorted(buckets): + ev = buckets[second] + rows.append({"second": second, "ok": sum(1 for e in ev if e[2] in (200, 201)), + "failed": sum(1 for e in ev if e[2] not in (200, 201)), + "instances": "|".join(f"{k}:{v}" for k, v in sorted(Counter(e[3] for e in ev).items()))}) + return rows + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--duration", type=float, default=20) + parser.add_argument("--workers", type=int, default=4) + args = parser.parse_args() + load = Load(args.workers).start() + time.sleep(args.duration) + load.stop() + print(json.dumps({"total": load.window(0, load.now()), "history": load.verify_history(), + "client_retries_same_key": load.retries}, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bench/rolling_under_load.py b/bench/rolling_under_load.py new file mode 100644 index 0000000..c4fd9ba --- /dev/null +++ b/bench/rolling_under_load.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Level 4: scripts/rolling-update.sh while bench/loadgen.py traffic runs through the LB. + + python bench/rolling_under_load.py --output evidence/rolling-update + +Writes /summary.json (errors, RPS, history check, revisions before/after), +/timeline.csv (per second ok/failed/instances) and /script-output.txt. +""" +from __future__ import annotations + +import argparse +import csv +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +sys.path.insert(0, str(ROOT / "bench")) +import lbctl # noqa: E402 +from loadgen import Load # noqa: E402 + + +def revisions() -> dict[str, str]: + result = {} + for r in lbctl.replicas(): + if r["state"] == "running": + label = subprocess.run(["docker", "inspect", "-f", '{{index .Config.Labels "org.opencontainers.image.revision"}}', + r["id"]], capture_output=True, text=True).stdout.strip() + result[r["name"]] = f"{r['hostname']} revision={label}" + return result + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--output", default="evidence/rolling-update") + parser.add_argument("--workers", type=int, default=4) + args = parser.parse_args() + out = ROOT / args.output + out.mkdir(parents=True, exist_ok=True) + before = revisions() + load = Load(args.workers).start() + time.sleep(5) + t_begin = load.now() + bash = (os.path.join(os.environ.get("ProgramFiles", r"C:\Program Files"), "Git", "bin", "bash.exe") + if os.name == "nt" else "sh") + env = {**os.environ, "PYTHON": sys.executable} + script = subprocess.run([bash, str(ROOT / "scripts" / "rolling-update.sh")], cwd=ROOT, env=env, + capture_output=True, text=True, encoding="utf-8", timeout=900) + t_end = load.now() + time.sleep(5) + load.stop() + (out / "script-output.txt").write_text(f"$ sh scripts/rolling-update.sh (exit {script.returncode})\n" + + script.stdout + script.stderr, encoding="utf-8") + with (out / "timeline.csv").open("w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=["second", "ok", "failed", "instances"]) + writer.writeheader() + writer.writerows(load.per_second()) + summary = {"script_exit_code": script.returncode, "update_seconds": round(t_end - t_begin, 1), + "before": before, "after": revisions(), + "during_update": load.window(t_begin, t_end), "whole_run": load.window(0, load.now()), + "client_retries_same_key": load.retries, "history": load.verify_history(), + "workload": f"{args.workers} closed-loop workers: POST (Idempotency-Key) + GET page via 127.0.0.1:8080"} + (out / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps({k: summary[k] for k in ("script_exit_code", "update_seconds", "before", "after", "history")}, + ensure_ascii=False, indent=2)) + print(json.dumps(summary["during_update"], ensure_ascii=False)) + return 0 if script.returncode == 0 else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/compose.yaml b/compose.yaml index 09abcfd..22bf943 100644 --- a/compose.yaml +++ b/compose.yaml @@ -80,7 +80,10 @@ services: # One-shot schema migration: runs ONCE per `up`, before any API replica starts # (api depends_on service_completed_successfully). Replicas never migrate themselves. migrate: - build: . + build: + context: . + args: + BUILD_REVISION: ${BUILD_REVISION:-dev} image: mtusi-messenger:lab3 command: ["python", "-m", "messenger.migrate"] environment: @@ -97,7 +100,10 @@ services: limits: {cpus: "0.5", memory: 256m} api: - build: . + build: + context: . + args: + BUILD_REVISION: ${BUILD_REVISION:-dev} image: mtusi-messenger:lab3 environment: *api-env secrets: [database_url, cache_url] diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 0000000..5a8e426 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Rolling update of the API without downtime (Compose has no built-in one for a scaled service). +# 1. build the new image from the working tree; +# 2. run migrations ONCE with the new image (they must be additive / compatible with the +# running old version: expand first, contract in a later release); +# 3. for every OLD replica: start one NEW replica (scale N+1, old ones untouched), wait until +# Docker says healthy AND HAProxy has N+1 replicas UP, then stop the old one gracefully +# (SIGTERM -> readiness 503 -> HAProxy DOWN -> drain -> exit) and remove it. +# At every moment at least N replicas serve traffic. Stops on the first error (set -e); +# the old replicas that were not replaced yet keep running. +set -eu +cd "$(dirname "$0")/.." +PY="${PYTHON:-$(command -v python3 || command -v python)}" +# New image = new revision label (git commit + UTC time), visible in docker inspect and logs. +BUILD_REVISION="${BUILD_REVISION:-$(git rev-parse --short HEAD 2>/dev/null || echo dev)-$(date -u +%Y%m%dT%H%M%SZ)}" +export BUILD_REVISION + +replicas="$(docker compose ps -q api | wc -l | tr -d ' ')" +[ "$replicas" -ge 1 ] || { echo "rolling-update: no running api replica" >&2; exit 2; } +old="$(docker compose ps -q api)" +echo "rolling-update: $replicas replica(s) to replace, new revision $BUILD_REVISION" + +docker compose build api +docker compose run --rm --no-deps migrate + +for id in $old; do + name="$(docker inspect -f '{{.Name}}' "$id" | tr -d '/')" + docker compose up -d --no-deps --no-recreate --wait --scale "api=$((replicas + 1))" api + "$PY" scripts/lbctl.py wait-up "$((replicas + 1))" --timeout 60 + docker stop --time 15 "$id" >/dev/null + docker rm "$id" >/dev/null + "$PY" scripts/lbctl.py wait-up "$replicas" --timeout 60 + echo "rolling-update: replaced $name" +done +for id in $(docker compose ps -q api); do + docker inspect -f '{{.Name}} revision={{index .Config.Labels "org.opencontainers.image.revision"}} {{.State.Status}}' "$id" +done diff --git a/src/messenger/__main__.py b/src/messenger/__main__.py index 5701b66..ef22eaf 100644 --- a/src/messenger/__main__.py +++ b/src/messenger/__main__.py @@ -13,6 +13,7 @@ A second signal during the drain skips the rest of it. Data stays in PostgreSQL from __future__ import annotations import logging +import os import sys import threading from types import FrameType @@ -75,7 +76,7 @@ def main() -> int: ) server = GracefulServer(config, settings.drain_seconds) log.info("starting", extra={"fields": { - "version": __version__, "env": settings.env, "host": settings.host, "port": settings.port, + "version": __version__, "revision": os.environ.get("APP_REVISION", "dev"), "env": settings.env, "host": settings.host, "port": settings.port, "python": sys.version.split()[0], "storage": "postgresql", "instance_id": settings.instance_id, }}) server.run()