forked from mtusi2026/mtusi2026lab3
- scripts/rolling-update.sh: сборка, однократные миграции новым образом, затем для каждой старой реплики: +1 новая (healthy и UP в HAProxy), graceful stop старой; ревизия образа видна в label и логе "starting". - bench/loadgen.py: фоновая нагрузка через LB (POST с Idempotency-Key + GET), неопределённый POST повторяется тем же ключом, проверка истории на потери и дубли. - bench/failover_series.py: каждая реплика по очереди в режимах stop / kill / pause; RPS, ошибки, время выхода из ротации и возврата. - bench/rolling_under_load.py: rolling update под нагрузкой. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
89 lines
3.5 KiB
Python
89 lines
3.5 KiB
Python
"""Entry point: `python -m messenger`. The HTTP server is the main process (PID 1 in Docker).
|
|
|
|
Shutdown sequence on SIGTERM/SIGINT (lab 3: the replica stands behind HAProxy):
|
|
1. readiness flips to 503 (`app.state.draining = True`), requests are still served;
|
|
2. after DRAIN_SECONDS (compose: 3 s > HAProxy detection 2 x 1 s) the load balancer has
|
|
marked this replica DOWN and sends nothing new here;
|
|
3. uvicorn closes the listening socket; in-flight requests get up to
|
|
SHUTDOWN_TIMEOUT_SECONDS (compose: 8 s) to finish;
|
|
4. the lifespan closes the DB pool and the Valkey client; the process exits with code 0.
|
|
DRAIN + SHUTDOWN (11 s) < compose stop_grace_period (15 s), so Docker never needs SIGKILL.
|
|
A second signal during the drain skips the rest of it. Data stays in PostgreSQL and Valkey.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
import sys
|
|
import threading
|
|
from types import FrameType
|
|
|
|
import uvicorn
|
|
|
|
from . import __version__, logging_setup
|
|
from .app import create_app
|
|
from .config import ConfigError, Settings
|
|
|
|
log = logging.getLogger("messenger")
|
|
|
|
|
|
class GracefulServer(uvicorn.Server):
|
|
"""uvicorn.Server that drains (readiness 503, still serving) before the normal graceful stop."""
|
|
|
|
def __init__(self, config: uvicorn.Config, drain_seconds: float):
|
|
super().__init__(config)
|
|
self.drain_seconds = drain_seconds
|
|
self.drain_timer: threading.Timer | None = None
|
|
|
|
def handle_exit(self, sig: int, frame: FrameType | None) -> None:
|
|
self.config.app.state.draining = True # config.app is our Starlette instance
|
|
if self.drain_timer is None and self.drain_seconds > 0 and not self.should_exit:
|
|
log.info("drain_started", extra={"fields": {"signal": sig, "drain_s": self.drain_seconds,
|
|
"timeout_s": self.config.timeout_graceful_shutdown}})
|
|
# uvicorn polls should_exit in its main loop, so setting it from a timer thread is safe.
|
|
self.drain_timer = threading.Timer(self.drain_seconds, self._stop, args=(sig, frame))
|
|
self.drain_timer.daemon = True
|
|
self.drain_timer.start()
|
|
return
|
|
if self.drain_timer is not None:
|
|
self.drain_timer.cancel() # second signal: stop draining now
|
|
self._stop(sig, frame)
|
|
|
|
def _stop(self, sig: int, frame: FrameType | None) -> None:
|
|
if not self.should_exit:
|
|
log.info("shutdown_requested", extra={"fields": {"signal": sig}})
|
|
super().handle_exit(sig, frame)
|
|
|
|
|
|
def main() -> int:
|
|
try:
|
|
settings = Settings.from_env()
|
|
except ConfigError as exc:
|
|
print(f"configuration error: {exc}", file=sys.stderr)
|
|
return 2
|
|
logging_setup.configure(settings.log_level)
|
|
app = create_app(settings)
|
|
config = uvicorn.Config(
|
|
app,
|
|
host=settings.host,
|
|
port=settings.port,
|
|
log_config=None,
|
|
access_log=False,
|
|
server_header=False,
|
|
proxy_headers=False,
|
|
lifespan="on",
|
|
timeout_graceful_shutdown=settings.shutdown_timeout,
|
|
)
|
|
server = GracefulServer(config, settings.drain_seconds)
|
|
log.info("starting", extra={"fields": {
|
|
"version": __version__, "revision": os.environ.get("APP_REVISION", "dev"), "env": settings.env, "host": settings.host, "port": settings.port,
|
|
"python": sys.version.split()[0], "storage": "postgresql", "instance_id": settings.instance_id,
|
|
}})
|
|
server.run()
|
|
log.info("stopped")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|