Лабораторная работа 3

This commit is contained in:
2026-10-26 23:23:12 +03:00
parent 0ab3a341aa
commit fafd1828c0
70 changed files with 6665 additions and 25 deletions
+174
View File
@@ -0,0 +1,174 @@
#!/usr/bin/env python3
"""Прогон всех проверок lab3 на запущенном compose-стенде с записью в evidence/.
python scripts/collect_evidence.py # ~8-10 минут
python scripts/collect_evidence.py --quick # без kill-серии и нагрузки
Перед запуском: стенд поднят (`docker compose --env-file .env up -d --build`,
`python scripts/ops.py wait`). Скрипт останавливает и восстанавливает реплики,
перезапускает хранилища (volumes сохраняются) и делает rolling update
lab3 -> lab3.1 -> lab3 (откат), поэтому запускайте его только на учебном стенде.
Токены и пароли в файлы не пишутся (тесты держат их в памяти).
"""
import argparse
import json
import os
import subprocess
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
from urllib.request import Request, urlopen
ROOT = Path(__file__).resolve().parents[1]
EV = ROOT / "evidence"
PY = sys.executable or "python"
WIN = os.name == "nt"
HOOK_STOP = str(ROOT / "scripts" / ("stop-one.cmd" if WIN else "stop-one.sh"))
HOOK_RESTART = str(ROOT / "scripts" / ("restart.cmd" if WIN else "restart.sh"))
BASE = "http://127.0.0.1:" + os.environ.get("APP_PORT", "8080")
RESULTS = []
try:
sys.stdout.reconfigure(errors="replace")
except AttributeError:
pass
def stamp():
return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
def step(title, cmd, out_file, append=False, env=None, timeout=900):
"""Выполнить команду, сохранить «$ команда + вывод» в evidence/<out_file>."""
print("\n== %s\n$ %s" % (title, " ".join(cmd)), flush=True)
started = time.monotonic()
try:
proc = subprocess.run(cmd, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
timeout=timeout, env=dict(os.environ, **(env or {})),
encoding="utf-8", errors="replace")
output, rc = proc.stdout, proc.returncode
except subprocess.TimeoutExpired as exc:
output, rc = (exc.stdout or "") + "\nTIMEOUT", 124
elapsed = time.monotonic() - started
shown = " ".join(c if c != PY else "python" for c in cmd).replace(str(ROOT) + os.sep, "")
text = "# %s | %s | exit %d | %.1fs\n$ %s\n%s\n" % (stamp(), title, rc, elapsed, shown, output.rstrip())
path = EV / out_file
with path.open("a" if append else "w", encoding="utf-8") as f:
f.write(text + ("\n" if append else ""))
print(output.rstrip()[-1500:], flush=True)
RESULTS.append((title, rc, out_file))
return rc, output
def instances_through_lb(n=20):
"""Одна сессия, n запросов через LB -> какие реплики отвечали."""
import base64
import uuid
name = "inst_" + uuid.uuid4().hex[:12]
pw = "Inst-" + uuid.uuid4().hex
body = json.dumps({"username": name, "display_name": "Inst", "password": pw}).encode()
urlopen(Request(BASE + "/api/v1/users", data=body, headers={"Content-Type": "application/json"}), timeout=10)
cred = base64.b64encode((name + ":" + pw).encode()).decode()
sess = json.loads(urlopen(Request(BASE + "/api/v1/auth/sessions", data=b"", method="POST",
headers={"Authorization": "Basic " + cred}), timeout=10).read())
lines = []
for i in range(n):
with urlopen(Request(BASE + "/api/v1/users/me", headers={"Authorization": "Bearer " + sess["token"]}), timeout=5) as r:
lines.append("%2d %s X-Instance-Id=%s X-Request-Id=%s" % (
i + 1, r.status, r.headers.get("X-Instance-Id"), r.headers.get("X-Request-Id")))
return lines
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--quick", action="store_true")
args = ap.parse_args()
EV.mkdir(exist_ok=True)
(EV / "ops.log").write_text("", encoding="utf-8")
step("Контракт (offline)", [PY, "scripts/check_contract.py"], "check.txt")
step("Compose config", ["docker", "compose", "--env-file", ".env", "config", "--quiet"], "check.txt", append=True)
step("Стенд: контейнеры", ["docker", "compose", "--env-file", ".env", "ps", "-a"], "stand.txt")
step("Стенд: реплики и статус в LB", [PY, "scripts/ops.py", "ps"], "stand.txt", append=True)
step("Стенд: лимиты CPU/RAM", ["docker", "stats", "--no-stream", "--format",
"table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}"], "stand.txt", append=True)
step("Стенд: опубликованные порты (только lb)", ["docker", "compose", "--env-file", ".env", "ps",
"--format", "table {{.Service}}\t{{.Ports}}"], "stand.txt", append=True)
lines = instances_through_lb()
(EV / "instances.txt").write_text("# %s | одна сессия, 20 запросов GET /users/me через LB\n%s\nразных реплик: %d\n"
% (stamp(), "\n".join(lines), len({l.split("X-Instance-Id=")[1].split()[0] for l in lines})),
encoding="utf-8")
print("\n".join(lines))
step("Публичный smoke", [PY, "tests/smoke.py", "--base-url", BASE], "smoke.txt")
step("Собственные тесты (HTTP через LB)", [PY, "student-tests/run_all.py", "--base-url", BASE], "student-tests.txt")
step("Тесты с доступом к БД (внутри compose-сети)", [PY, "scripts/ops.py", "in-network", "storage"],
"student-tests.txt", append=True)
step("Истечение сессии (api-short-ttl, TTL 2 с)", [PY, "scripts/ops.py", "in-network", "expiry"],
"student-tests.txt", append=True)
# Публичный тест останавливает реплику, обслужившую его первое сообщение.
# Round-robin детерминирован, поэтому для второй реплики сдвигаем очередь
# лишними запросами, пока цель не окажется другой (README: «повторить для второй»).
targets = []
for i in range(1, 5):
if len(set(targets)) >= 2:
break
for _ in range(i - 1):
urlopen(BASE + "/health/live", timeout=5).read()
step("Failover #%d: реплики до" % i, [PY, "scripts/ops.py", "ps"], "failover.txt", append=i > 1)
step("Failover #%d: make test-failover" % i,
[PY, "tests/resilience.py", "--mode", "failover", "--action-script", HOOK_STOP, "--base-url", BASE],
"failover.txt", append=True)
step("Failover #%d: реплики после (остановлена именно цель теста)" % i, [PY, "scripts/ops.py", "ps"],
"failover.txt", append=True)
step("Failover #%d: restore" % i, [PY, "scripts/ops.py", "restore"], "failover.txt", append=True)
stops = [l for l in (EV / "ops.log").read_text(encoding="utf-8").splitlines() if "stop-one: mode=" in l]
if stops:
targets.append(stops[-1].split("target=")[1].split()[0])
with (EV / "failover.txt").open("a", encoding="utf-8") as f:
f.write("# остановленные публичным тестом реплики: %s\n" % targets)
step("Серия отказов: каждая реплика, graceful stop", [PY, "student-tests/failover_series.py", "--base-url", BASE,
"--output", "evidence/failover_series.json"], "failover_series.txt")
if not args.quick:
step("Серия отказов: каждая реплика, SIGKILL", [PY, "student-tests/failover_series.py", "--base-url", BASE,
"--mode", "kill", "--output", "evidence/failover_series_kill.json"],
"failover_series.txt", append=True)
step("Rolling update lab3 -> lab3.1 под нагрузкой",
[PY, "student-tests/rolling_update_load.py", "--base-url", BASE, "--output", "evidence/rolling_update.json"],
"rolling_update.txt", env={"APP_VERSION": "lab3.1"})
step("Rolling update обратно lab3.1 -> lab3 (откат) под нагрузкой",
[PY, "student-tests/rolling_update_load.py", "--base-url", BASE, "--output", "evidence/rolling_back.json"],
"rolling_update.txt", append=True, env={"APP_VERSION": "lab3"})
step("Реплики после обновлений", [PY, "scripts/ops.py", "ps"], "rolling_update.txt", append=True)
step("Persistence: контейнеры до", ["docker", "compose", "--env-file", ".env", "ps", "-a", "--format",
"table {{.Name}}\t{{.Status}}\t{{.RunningFor}}"], "persistence.txt")
step("Persistence: make test-persistence",
[PY, "tests/resilience.py", "--mode", "persistence", "--action-script", HOOK_RESTART, "--base-url", BASE],
"persistence.txt", append=True)
step("Persistence: контейнеры после (Status = Up N seconds)", ["docker", "compose", "--env-file", ".env", "ps", "-a",
"--format", "table {{.Name}}\t{{.Status}}\t{{.RunningFor}}"],
"persistence.txt", append=True)
if not args.quick:
step("Нагрузка 30 с, concurrency 8", [PY, "tests/load.py", "--base-url", BASE, "--duration", "30",
"--concurrency", "8", "--output", "evidence/load.json"], "load.txt")
step("CPU/RAM после нагрузки", ["docker", "stats", "--no-stream", "--format",
"table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}"], "load.txt", append=True)
step("Лог LB (последние строки, без секретов)", ["docker", "compose", "--env-file", ".env", "logs", "--no-color",
"--tail", "60", "lb"], "lb_log_sample.txt")
summary = ["# %s" % stamp()] + ["%-4s %-60s evidence/%s" % ("OK" if rc == 0 else "FAIL", t, f) for t, rc, f in RESULTS]
(EV / "SUMMARY.txt").write_text("\n".join(summary) + "\n", encoding="utf-8")
print("\n" + "\n".join(summary))
sys.exit(0 if all(rc == 0 for _, rc, _ in RESULTS) else 1)
if __name__ == "__main__":
main()
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env python3
"""Создаёт локальные учебные секреты в ./secrets (каталог в .gitignore).
Повторный запуск ничего не перезаписывает: пароль PostgreSQL задаётся только
при первой инициализации volume, и смена файла сломала бы подключение.
Значения секретов не печатаются.
"""
import os
import secrets
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
DIR = ROOT / "secrets"
def write(name: str, value: str) -> None:
path = DIR / name
if path.exists():
print("keep secrets/%s" % name)
return
path.write_text(value + "\n", encoding="utf-8")
# Каталог 0700, файлы 0644: внутри контейнера их читают непривилегированные
# пользователи (appuser, redis), а на хосте каталог закрыт.
os.chmod(path, 0o644)
print("create secrets/%s" % name)
def read(name: str) -> str:
return (DIR / name).read_text(encoding="utf-8").strip()
def main() -> None:
DIR.mkdir(exist_ok=True)
try:
os.chmod(DIR, 0o700)
except OSError:
pass
write("postgres_password", secrets.token_hex(24))
write("redis_password", secrets.token_hex(24))
write("database_url", "postgresql://messenger:%s@postgres:5432/messenger" % read("postgres_password"))
write("cache_url", "redis://:%s@redis:6379/0" % read("redis_password"))
if __name__ == "__main__":
main()
+398
View File
@@ -0,0 +1,398 @@
#!/usr/bin/env python3
"""Операции над compose-стендом lab3. Python 3.9+, только стандартная библиотека.
Работает одинаково в Windows (PowerShell/cmd), Linux и macOS: вызывает
`docker compose` и `docker` без shell-интерполяции. Секреты не печатает.
python scripts/ops.py wait # дождаться N здоровых реплик и LB
python scripts/ops.py ps # реплики: контейнер, X-Instance-Id, health, статус в LB
python scripts/ops.py stop-one # TARGET_INSTANCE_ID=<X-Instance-Id>; оставляет остановленной
python scripts/ops.py restore # запустить остановленные реплики и дождаться UP в LB
python scripts/ops.py restart # hook persistence: API + PostgreSQL + Redis + LB, volumes целы
python scripts/ops.py rolling-update # поочерёдная замена реплик без остановки сервиса
python scripts/ops.py in-network storage # тесты с прямым доступом к БД — внутри compose-сети
python scripts/ops.py in-network expiry # истечение сессии на api-short-ttl (TTL 2 с)
Переменные окружения:
COMPOSE_PROJECT_NAME имя проекта (по умолчанию messenger-lab3, как `name:` в compose.yaml)
ENV_FILE файл переменных (по умолчанию .env, если существует)
API_REPLICAS ожидаемое число реплик (по умолчанию из .env, иначе 2)
APP_PORT, LB_STATS_PORT опубликованные на 127.0.0.1 порты LB (8080, 8404)
STOP_MODE stop-one: stop (SIGTERM, graceful, по умолчанию) | kill (SIGKILL, «падение»)
Каждое действие дописывается в evidence/ops.log (время, контейнер, результат) —
это доказательство того, какой контейнер реально остановлен/заменён.
"""
import csv
import io
import json
import os
import subprocess
import sys
import time
import urllib.error
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
OPS_LOG = ROOT / "evidence" / "ops.log"
try: # консоль Windows может быть не в UTF-8
sys.stdout.reconfigure(errors="replace")
sys.stderr.reconfigure(errors="replace")
except AttributeError:
pass
# ---------------------------------------------------------------- settings
def _dotenv():
path = Path(os.environ.get("ENV_FILE", str(ROOT / ".env")))
values = {}
if path.is_file():
for line in path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if line and not line.startswith("#") and "=" in line:
k, v = line.split("=", 1)
values[k.strip()] = v.strip().strip('"').strip("'")
return path if path.is_file() else None, values
ENV_PATH, DOTENV = _dotenv()
def setting(name, default):
return os.environ.get(name) or DOTENV.get(name) or default
PROJECT = setting("COMPOSE_PROJECT_NAME", "messenger-lab3")
REPLICAS = int(setting("API_REPLICAS", "2"))
APP_PORT = int(setting("APP_PORT", "8080"))
STATS_PORT = int(setting("LB_STATS_PORT", "8404"))
LB_URL = "http://127.0.0.1:%d" % APP_PORT
STATS_URL = "http://127.0.0.1:%d/stats;csv" % STATS_PORT
def now_iso():
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
def log(msg):
line = "%s %s" % (now_iso(), msg)
print(line, flush=True)
try:
OPS_LOG.parent.mkdir(exist_ok=True)
with OPS_LOG.open("a", encoding="utf-8") as f:
f.write(line + "\n")
except OSError:
pass
# ---------------------------------------------------------------- docker
def compose_cmd(*args):
cmd = ["docker", "compose", "-p", PROJECT]
if ENV_PATH:
cmd += ["--env-file", str(ENV_PATH)]
return cmd + list(args)
def run(cmd, check=True, capture=False, timeout=300, show=False):
"""capture: вернуть вывод; show: показать вывод в консоли; иначе — тихо."""
out = None if show else (subprocess.PIPE if capture else subprocess.DEVNULL)
result = subprocess.run(
cmd, cwd=ROOT, check=False, timeout=timeout,
stdout=out, stderr=out, text=True,
)
if check and result.returncode != 0:
detail = (result.stderr or "").strip()[-400:] if capture else ""
raise SystemExit("command failed (%d): %s %s" % (result.returncode, " ".join(cmd[:4]), detail))
return result
def compose(*args, timeout=300):
return run(compose_cmd(*args), timeout=timeout, capture=True)
def api_containers(include_stopped=True):
"""[{id, short, name, state, health, image, ip}] — только сервис api, без one-off."""
cmd = ["docker", "ps", "--no-trunc", "--format", "{{.ID}}",
"--filter", "label=com.docker.compose.project=" + PROJECT,
"--filter", "label=com.docker.compose.service=api",
"--filter", "label=com.docker.compose.oneoff=False"]
if include_stopped:
cmd.insert(2, "-a")
ids = [x for x in run(cmd, capture=True).stdout.split() if x]
if not ids:
return []
raw = run(["docker", "inspect"] + ids, capture=True).stdout
result = []
for item in json.loads(raw):
state = item["State"]
nets = item.get("NetworkSettings", {}).get("Networks") or {}
ips = [n.get("IPAddress") for n in nets.values() if n.get("IPAddress")]
result.append({
"id": item["Id"],
"short": item["Id"][:12],
"hostname": item["Config"].get("Hostname", ""),
"name": item["Name"].lstrip("/"),
"state": state.get("Status"),
"health": (state.get("Health") or {}).get("Status", "-"),
"image": item["Config"].get("Image"),
"ip": ips[0] if ips else "",
"created": item.get("Created", "")[:19],
})
result.sort(key=lambda c: c["name"])
return result
def running(containers=None):
return [c for c in (containers if containers is not None else api_containers()) if c["state"] == "running"]
# ---------------------------------------------------------------- load balancer
def lb_servers():
"""[{name, status, addr}] backend api из CSV-статистики HAProxy (слоты с адресом)."""
data = urllib.request.urlopen(STATS_URL, timeout=2).read().decode("utf-8", "replace")
rows = csv.DictReader(io.StringIO(data.lstrip("# ")))
servers = []
for r in rows:
if r.get("pxname") != "api" or r.get("svname") in ("BACKEND", "FRONTEND"):
continue
addr = r.get("addr") or ""
status = r.get("status", "")
if status.startswith("MAINT") and not addr:
continue # пустой слот server-template
servers.append({"name": r["svname"], "status": status, "addr": addr})
return servers
def lb_up_count():
return sum(1 for s in lb_servers() if s["status"].startswith("UP"))
def lb_ready():
try:
with urllib.request.urlopen(LB_URL + "/health/ready", timeout=2) as r:
return r.status == 200
except (urllib.error.URLError, OSError):
return False
def wait_until(predicate, timeout, what, interval=0.3):
deadline = time.monotonic() + timeout
last_exc = None
while time.monotonic() < deadline:
try:
if predicate():
return True
except (urllib.error.URLError, OSError, ValueError, KeyError) as exc:
last_exc = exc
time.sleep(interval)
raise SystemExit("timeout after %ss waiting for %s%s" % (
timeout, what, (" (%s)" % type(last_exc).__name__) if last_exc else ""))
def wait_healthy(ids, timeout=90):
def ok():
states = {c["id"]: c for c in api_containers()}
return all(states.get(i, {}).get("health") == "healthy" for i in ids)
wait_until(ok, timeout, "containers healthy: " + ", ".join(i[:12] for i in ids))
def wait_stack(replicas, timeout=120):
wait_until(lambda: len([c for c in running() if c["health"] == "healthy"]) >= replicas,
timeout, "%d healthy api replicas" % replicas)
wait_until(lambda: lb_up_count() >= replicas, 30, "%d replicas UP in HAProxy" % replicas)
wait_until(lb_ready, 30, "LB /health/ready = 200")
# ---------------------------------------------------------------- commands
def cmd_wait(_args):
t0 = time.monotonic()
wait_stack(REPLICAS)
print("ready: %d api replicas healthy and UP in LB (%.1fs)" % (REPLICAS, time.monotonic() - t0))
def cmd_ps(_args):
containers = api_containers()
try:
by_addr = {s["addr"].rsplit(":", 1)[0]: s for s in lb_servers()}
except (urllib.error.URLError, OSError):
by_addr = {}
print("%-14s %-28s %-9s %-10s %-22s %-6s %s" % ("X-Instance-Id", "container", "state", "health", "image", "lb", "lb_status"))
for c in containers:
s = by_addr.get(c["ip"], {})
print("%-14s %-28s %-9s %-10s %-22s %-6s %s" % (
c["hostname"], c["name"], c["state"], c["health"], c["image"], s.get("name", "-"), s.get("status", "-")))
def find_target(target):
for c in api_containers():
# X-Instance-Id = hostname контейнера = первые 12 символов его ID.
if target and (c["hostname"] == target or c["id"].startswith(target)):
return c
return None
def cmd_stop_one(_args):
target = os.environ.get("TARGET_INSTANCE_ID", "").strip()
if not target:
raise SystemExit("TARGET_INSTANCE_ID is required (X-Instance-Id of the replica to stop)")
mode = os.environ.get("STOP_MODE", "stop").strip().lower()
c = find_target(target)
if c is None:
raise SystemExit("no api container of project %s matches X-Instance-Id %s" % (PROJECT, target))
if c["state"] != "running":
raise SystemExit("container %s is already %s" % (c["name"], c["state"]))
t0 = time.monotonic()
log("stop-one: mode=%s target=%s container=%s ip=%s" % (mode, target, c["name"], c["ip"]))
if mode == "kill":
# «Падение» процесса. Политику restart снимаем, чтобы Docker не поднял
# контейнер сам: тест требует, чтобы реплика оставалась остановленной.
run(["docker", "update", "--restart", "no", c["id"]])
run(["docker", "kill", "--signal", "KILL", c["id"]])
else:
# SIGTERM -> осушение 3 с (readiness 503) -> graceful stop; SIGKILL не позже 20 с.
run(["docker", "stop", "--time", "20", c["id"]], timeout=40)
after = find_target(target) or {}
log("stop-one: container=%s state=%s after %.2fs; left stopped (restore: python scripts/ops.py restore)"
% (c["name"], after.get("state", "removed"), time.monotonic() - t0))
def cmd_restore(_args):
t0 = time.monotonic()
stopped = [c for c in api_containers() if c["state"] != "running"]
for c in stopped: # вернуть политику, снятую в режиме kill
run(["docker", "update", "--restart", "unless-stopped", c["id"]], check=False)
# --no-recreate: не пересоздавать работающие; up запускает остановленные
# и добирает число контейнеров до deploy.replicas.
compose("up", "-d", "--no-deps", "--no-recreate", "api")
wait_stack(REPLICAS)
log("restore: started %s; %d replicas healthy and UP in LB after %.1fs"
% ([c["name"] for c in stopped] or "nothing", REPLICAS, time.monotonic() - t0))
def cmd_restart(_args):
"""Hook для make test-persistence: перезапуск API, ВСЕХ хранилищ и LB.
Volumes не удаляются, данные не пересоздаются. Порядок: сначала LB и API
(новые записи не принимаются), затем PostgreSQL (штатный shutdown с
checkpoint) и Redis, затем API и LB в порядке зависимостей.
"""
t0 = time.monotonic()
before = {c["name"]: c["short"] for c in running()}
pg_before = container_started("postgres")
redis_before = container_started("redis")
compose("stop", "lb", "api", timeout=120)
compose("restart", "postgres", "redis", timeout=120)
compose("start", "api", timeout=120)
wait_until(lambda: len([c for c in running() if c["health"] == "healthy"]) >= REPLICAS,
80, "api healthy after restart")
compose("start", "lb", timeout=60)
log("restart: postgres started_at %s -> %s; redis %s -> %s; api %s restarted; lb restarted; %.1fs"
% (pg_before, container_started("postgres"), redis_before, container_started("redis"),
sorted(before), time.monotonic() - t0))
def container_started(service):
ids = run(["docker", "ps", "-aq", "--filter", "label=com.docker.compose.project=" + PROJECT,
"--filter", "label=com.docker.compose.service=" + service], capture=True).stdout.split()
if not ids:
return "-"
return run(["docker", "inspect", "-f", "{{.State.StartedAt}}", ids[0]], capture=True).stdout.strip()[:23]
def cmd_rolling_update(args):
"""Поочерёдная замена реплик: +1 новая -> healthy и UP в LB -> graceful stop старой.
В любой момент в ротации >= N реплик. Миграции — один раз, до замены, и только
расширяющие (expand): старая версия продолжает работать на новой схеме.
"""
t_all = time.monotonic()
if [c for c in api_containers() if c["state"] != "running"]:
raise SystemExit("some api containers are stopped; run `python scripts/ops.py restore` first")
old = running()
n = len(old)
if n < 1:
raise SystemExit("no running api replicas")
log("rolling-update: start; replicas=%d old=%s" % (n, [c["short"] for c in old]))
if "--no-build" not in args:
t = time.monotonic()
compose("build", "api", "migrate", timeout=900)
log("rolling-update: images built in %.1fs (tag %s)" % (time.monotonic() - t, setting("APP_VERSION", "lab3")))
t = time.monotonic()
out = compose("run", "--rm", "-T", "migrate", timeout=300).stdout.strip().splitlines()
log("rolling-update: migrations once in %.1fs: %s" % (time.monotonic() - t, out[-1] if out else ""))
for i, victim in enumerate(old, 1):
before = {c["id"] for c in api_containers()}
t = time.monotonic()
compose("up", "-d", "--no-deps", "--no-recreate", "--scale", "api=%d" % (n + 1), "api", timeout=180)
new = [c for c in api_containers() if c["id"] not in before]
if len(new) != 1:
raise SystemExit("expected exactly one new container, got %d" % len(new))
fresh = new[0]
wait_healthy([fresh["id"]], timeout=90)
wait_until(lambda: lb_up_count() >= n + 1, 30, "new replica UP in HAProxy")
t_ready = time.monotonic() - t
t = time.monotonic()
run(["docker", "stop", "--time", "20", victim["id"]], timeout=40)
run(["docker", "rm", victim["id"]])
log("rolling-update: step %d/%d new=%s (%s, ready+UP in %.1fs) replaced old=%s (graceful stop %.1fs)"
% (i, n, fresh["short"], fresh["image"], t_ready, victim["short"], time.monotonic() - t))
wait_stack(n)
log("rolling-update: done in %.1fs; replicas now %s" % (time.monotonic() - t_all, [c["short"] for c in running()]))
def cmd_in_network(args):
"""Тесты, которым нужен прямой доступ к PostgreSQL/Redis или экземпляр api-short-ttl.
Запускаются в one-off контейнере образа API внутри сети backend: порты БД и
реплик наружу не публикуются. One-off контейнер не получает DNS-имя `api`,
поэтому LB его не видит.
"""
which = args[0] if args else ""
mount = "%s:/app/student-tests:ro" % (ROOT / "student-tests")
base = compose_cmd("run", "--rm", "-T", "--no-deps", "-v", mount)
if which == "storage":
for pattern in ("test_storage_*.py", "test_session_cache.py"):
run(base + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests", "-p", pattern, "-v"],
show=True, timeout=600)
elif which == "expiry":
prof = compose_cmd("--profile", "ttl")
run(prof + ["up", "-d", "--no-recreate", "api-short-ttl"], show=True, timeout=300)
env = ["-e", "EXPIRY_BASE_URL=http://api-short-ttl:8080", "-e", "BASE_URL=http://lb:8080"]
try:
run(base + env + ["api", "python", "student-tests/wait_url.py", "http://api-short-ttl:8080/health/ready"],
show=True, timeout=120)
run(base + env + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests",
"-p", "test_expiry.py", "-v"], show=True, timeout=300)
finally:
run(prof + ["rm", "-sf", "api-short-ttl"], check=False, timeout=120)
else:
raise SystemExit("usage: python scripts/ops.py in-network storage|expiry")
COMMANDS = {
"wait": cmd_wait,
"ps": cmd_ps,
"stop-one": cmd_stop_one,
"restore": cmd_restore,
"restart": cmd_restart,
"rolling-update": cmd_rolling_update,
"in-network": cmd_in_network,
}
def main():
if len(sys.argv) < 2 or sys.argv[1] not in COMMANDS:
print(__doc__)
sys.exit(2)
COMMANDS[sys.argv[1]](sys.argv[2:])
if __name__ == "__main__":
main()
+9
View File
@@ -0,0 +1,9 @@
@echo off
rem Windows-вариант restart.sh: API + PostgreSQL + Redis + LB, volumes сохраняются.
rem python tests\resilience.py --mode persistence --action-script scripts\restart.cmd
setlocal
cd /d "%~dp0.."
set PY=python
where python >nul 2>nul || set PY=py -3
%PY% scripts\ops.py restart
exit /b %ERRORLEVEL%
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
# Hook для make test-persistence: перезапускает API (все реплики), PostgreSQL,
# Redis и LB, сохраняя volumes. Никаких down -v / flush / повторного seed.
# Возвращает 0, когда API снова healthy; готовность LB тест ждёт сам.
set -eu
cd "$(dirname "$0")/.."
PY=$(command -v python3 || command -v python)
exec "$PY" scripts/ops.py restart
+93
View File
@@ -0,0 +1,93 @@
-- Обоснование индекса истории: EXPLAIN (ANALYZE, BUFFERS) на >= 10 000 сообщений.
-- Всё выполняется в ОДНОЙ транзакции и откатывается (ROLLBACK): синтетические
-- данные и временное удаление индекса не остаются в БД.
-- Запуск (compose):
-- docker compose exec -T postgres psql -U messenger -d messenger -v ON_ERROR_STOP=1 \
-- < scripts/sql/explain_history.sql > evidence/explain_history.txt
-- Внимание: DROP INDEX внутри транзакции берёт эксклюзивную блокировку messages
-- до ROLLBACK (несколько секунд) — запускать на учебном стенде.
\set ON_ERROR_STOP 1
\timing off
SET TimeZone = 'UTC';
-- Убираем мёртвые версии строк от прошлых запусков (VACUUM нельзя внутри транзакции).
VACUUM (ANALYZE) messages, chat_members, chats, users;
BEGIN;
INSERT INTO users (id, username, display_name, password_hash)
VALUES ('00000000-0000-4000-8000-00000000e001', 'explain_alice', 'Explain', 'x'),
('00000000-0000-4000-8000-00000000e002', 'explain_bob', 'Explain', 'x');
-- 1 большой чат (12 000 сообщений) + 40 чатов по 1 000 = 52 000 сообщений.
INSERT INTO chats (id, title, created_at)
SELECT CASE WHEN g = 0 THEN '00000000-0000-4000-8000-0000000c0000'::uuid ELSE gen_random_uuid() END,
'explain-' || g, timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second'
FROM generate_series(0, 40) AS g;
INSERT INTO chat_members (chat_id, user_id, position)
SELECT c.id, u.id, u.pos
FROM chats c
CROSS JOIN (VALUES ('00000000-0000-4000-8000-00000000e001'::uuid, 0),
('00000000-0000-4000-8000-00000000e002'::uuid, 1)) AS u(id, pos)
WHERE c.title LIKE 'explain-%';
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), c.id,
CASE WHEN g % 2 = 0 THEN '00000000-0000-4000-8000-00000000e001'::uuid
ELSE '00000000-0000-4000-8000-00000000e002'::uuid END,
'synthetic message ' || g,
timestamptz '2026-01-02 00:00:00+00' + g * interval '1 millisecond' + (c.created_at - timestamptz '2026-01-01 00:00:00+00')
FROM chats c
CROSS JOIN LATERAL generate_series(1, CASE WHEN c.title = 'explain-0' THEN 12000 ELSE 1000 END) AS g
WHERE c.title LIKE 'explain-%';
ANALYZE users, chats, chat_members, messages;
\echo '=== dataset'
SELECT count(*) AS total_messages,
count(*) FILTER (WHERE chat_id = '00000000-0000-4000-8000-0000000c0000') AS big_chat_messages
FROM messages;
-- Позиция курсора после 10 000-го сообщения большого чата (как в next_cursor).
SELECT created_at AS cur_ts, id AS cur_id
FROM messages
WHERE chat_id = '00000000-0000-4000-8000-0000000c0000'
ORDER BY created_at, id OFFSET 9999 LIMIT 1 \gset
\echo '=== 1. first page (no cursor), LIMIT 51 = limit+1'
EXPLAIN (ANALYZE, BUFFERS, COSTS, TIMING)
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = '00000000-0000-4000-8000-0000000c0000'
ORDER BY created_at, id
LIMIT 51;
\echo '=== 2. deep page: keyset cursor after message #10000 (query used by API)'
EXPLAIN (ANALYZE, BUFFERS, COSTS, TIMING)
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = '00000000-0000-4000-8000-0000000c0000'
AND (created_at, id) > (:'cur_ts'::timestamptz, :'cur_id'::uuid)
ORDER BY created_at, id
LIMIT 51;
\echo '=== 3. for comparison: OFFSET pagination to the same page (NOT used by API)'
EXPLAIN (ANALYZE, BUFFERS, COSTS, TIMING)
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = '00000000-0000-4000-8000-0000000c0000'
ORDER BY created_at, id
OFFSET 10000 LIMIT 51;
\echo '=== 4. same keyset query WITHOUT messages_chat_created_id_idx (index dropped inside the transaction)'
DROP INDEX messages_chat_created_id_idx;
EXPLAIN (ANALYZE, BUFFERS, COSTS, TIMING)
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = '00000000-0000-4000-8000-0000000c0000'
AND (created_at, id) > (:'cur_ts'::timestamptz, :'cur_id'::uuid)
ORDER BY created_at, id
LIMIT 51;
ROLLBACK;
\echo '=== rolled back: synthetic data removed, index restored'
+9
View File
@@ -0,0 +1,9 @@
@echo off
rem Windows-вариант stop-one.sh: TARGET_INSTANCE_ID -> один контейнер api, остаётся остановленным.
rem python tests\resilience.py --mode failover --action-script scripts\stop-one.cmd
setlocal
cd /d "%~dp0.."
set PY=python
where python >nul 2>nul || set PY=py -3
%PY% scripts\ops.py stop-one
exit /b %ERRORLEVEL%
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
# Hook для make test-failover: останавливает ОДНУ API-реплику, которую тест передал
# в TARGET_INSTANCE_ID (= X-Instance-Id = hostname контейнера = короткий ID), и
# оставляет её остановленной. По умолчанию graceful (docker stop: SIGTERM ->
# осушение 3 с -> graceful shutdown); STOP_MODE=kill — «падение» (SIGKILL).
# БД, Redis, LB и volumes не трогает. Восстановление: python3 scripts/ops.py restore
set -eu
cd "$(dirname "$0")/.."
PY=$(command -v python3 || command -v python)
exec "$PY" scripts/ops.py stop-one