diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml index 0bca8e17..80b088e3 100644 --- a/.github/workflows/application.yml +++ b/.github/workflows/application.yml @@ -19,7 +19,7 @@ jobs: - run: make student-test VENV=.venv acceptance: - name: Стенд Compose за HAProxy, smoke, рестарт и отказы зависимостей + name: Стенд Compose за HAProxy, smoke, отказ реплики, рестарт и отказы зависимостей runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -34,6 +34,15 @@ jobs: - name: Публичный smoke через балансировщик run: make test + - name: Несколько реплик на одном токене и гонка одного ключа + run: make lb-check + + - name: Остановка реплики под запросами, затем возврат + run: | + make test-failover ACTION_SCRIPT=scripts/stop-one.sh + docker compose ps -a api + make restore + - name: Рестарт LB, API и хранилищ с сохранением volumes run: make test-persistence ACTION_SCRIPT=scripts/restart.sh diff --git a/Makefile b/Makefile index 8f96d97c..5439667f 100644 --- a/Makefile +++ b/Makefile @@ -11,12 +11,14 @@ PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q .PHONY: help check validate test load test-persistence test-failover \ install lint format typecheck student-test \ - secrets up down reset-data migrate ps logs persistence-evidence storage-evidence cache-evidence test-faults seed explain + secrets up down reset-data migrate ps logs restore lb-check \ + persistence-evidence storage-evidence cache-evidence test-faults seed explain help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @echo "install/lint/typecheck/student-test: студенческий тулчейн" @echo "secrets/up/down/migrate/ps/logs: стенд Docker Compose; reset-data: намеренное удаление volumes" + @echo "restore: вернуть остановленные реплики; lb-check: реплики на одном токене и гонка одного ключа" check: $(PYTHON) scripts/check_contract.py @@ -42,13 +44,13 @@ install: $(UV) sync --group dev lint: - $(VENV)/bin/ruff check src student-tests - $(VENV)/bin/ruff format --check src student-tests + $(VENV)/bin/ruff check src student-tests scripts/probe.py + $(VENV)/bin/ruff format --check src student-tests scripts/probe.py $(VENV)/bin/ty check src student-tests format: - $(VENV)/bin/ruff check --fix src student-tests - $(VENV)/bin/ruff format src student-tests + $(VENV)/bin/ruff check --fix src student-tests scripts/probe.py + $(VENV)/bin/ruff format src student-tests scripts/probe.py typecheck: $(VENV)/bin/ty check src student-tests @@ -78,6 +80,12 @@ ps: logs: $(COMPOSE) logs --tail 200 +restore: + scripts/restore.sh + +lb-check: + $(PYTHON) scripts/probe.py check --out evidence/lb-check.json + persistence-evidence: sh scripts/persistence-evidence.sh "$(BASE_URL)" > evidence/persistence.txt cat evidence/persistence.txt diff --git a/openspec/changes/add-lab3-load-balanced-replicas/tasks.md b/openspec/changes/add-lab3-load-balanced-replicas/tasks.md index bf4bd921..d015e1fa 100644 --- a/openspec/changes/add-lab3-load-balanced-replicas/tasks.md +++ b/openspec/changes/add-lab3-load-balanced-replicas/tasks.md @@ -31,8 +31,8 @@ ## 6. Хуки и зонд -- [ ] 6.1 `scripts/stop-one.sh`, `scripts/restart.sh` (вместо версии лабы 2), `scripts/restore.sh`, `scripts/lb-wait.sh` -- [ ] 6.2 `scripts/probe.py` (`check`), ruff для `scripts/*.py` +- [x] 6.1 `scripts/stop-one.sh`, `scripts/restart.sh` (вместо версии лабы 2), `scripts/restore.sh`, `scripts/lb-wait.sh` +- [x] 6.2 `scripts/probe.py` (`check`), ruff для `scripts/*.py` - [ ] 6.3 Скрипт доказательства отказа: `docker ps -a` до и после, `make test-failover`, какой контейнер остановлен - [ ] 6.4 Проверка группы: `make test`, `make lb-check`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` для обеих реплик с `scripts/restore.sh` между ними, `make test-persistence ACTION_SCRIPT=scripts/restart.sh` diff --git a/pyproject.toml b/pyproject.toml index 52b34dc2..b7e26dc3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,7 @@ ignore = [ [tool.ruff.lint.per-file-ignores] "student-tests/**" = ["S101", "S105", "S106", "S603", "S607", "PLR2004", "ARG001", "T20", "TID251"] +"scripts/*.py" = ["T20", "S603", "S607", "S310", "PLR2004", "TID251"] "src/messenger/*/infrastructure/**" = ["TID251", "ARG002"] "src/messenger/*/domain/errors.py" = ["N818"] "src/messenger/*/application/errors.py" = ["N818"] diff --git a/scripts/lb-wait.sh b/scripts/lb-wait.sh new file mode 100755 index 00000000..f1ead888 --- /dev/null +++ b/scripts/lb-wait.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +service=${API_SERVICE:-api} +want=${1:-$(docker compose ps -q "$service" | wc -l | tr -d ' ')} +url=${LB_STATS_URL:-http://$(docker compose port lb 8404)/stats;csv} +deadline=$(($(date +%s) + ${WAIT_TIMEOUT:-60})) +until [ "$(curl -fsS "$url" | awk -F, -v b="$service" '$1 == b && $2 != "BACKEND" && $18 == "UP"' | wc -l)" -ge "$want" ]; do + [ "$(date +%s)" -lt "$deadline" ] || { echo "LB does not see $want UP $service servers" >&2; exit 4; } + sleep 0.5 +done diff --git a/scripts/probe.py b/scripts/probe.py new file mode 100755 index 00000000..44a896bd --- /dev/null +++ b/scripts/probe.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +import argparse +import base64 +import collections +import functools +import json +import os +import subprocess +import sys +import threading +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +@functools.cache +def published(port, fallback): + command = ["docker", "compose", "port", "lb", str(port)] + found = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False) + return found.stdout.strip() or fallback + + +def base_url(): + return os.environ.get("BASE_URL") or "http://" + published(8080, "localhost:8080") + + +def call(method, path, body=None, headers=None, timeout=2.0): + data = None if body is None else json.dumps(body, ensure_ascii=False).encode() + head = {"Accept": "application/json"} + if data is not None: + head["Content-Type"] = "application/json" + head.update(headers or {}) + request = urllib.request.Request( + base_url().rstrip("/") + path, data=data, headers=head, method=method + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + payload = json.loads(response.read() or b"null") + return response.status, response.headers.get("X-Instance-Id", ""), payload + except urllib.error.HTTPError as error: + return error.code, error.headers.get("X-Instance-Id", ""), None + except (OSError, ValueError): + return 0, "", None + + +def emit(report, out=None): + text = json.dumps(report, ensure_ascii=False, indent=2) + if out: + Path(out).parent.mkdir(parents=True, exist_ok=True) + Path(out).write_text(text + "\n") + print(text) + + +def created(result, what): + if result[0] != 201: + raise SystemExit(f"{what} failed: HTTP {result[0]}") + return result[2] + + +def actor(label): + username = f"p_{label}_{uuid.uuid4().hex[:12]}" + password = "Probe-" + uuid.uuid4().hex + body = {"username": username, "display_name": "probe " + label, "password": password} + user = created(call("POST", "/api/v1/users", body, timeout=10), "register") + basic = base64.b64encode(f"{username}:{password}".encode()).decode() + login = call( + "POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}, timeout=10 + ) + return { + "id": user["id"], + "auth": {"Authorization": "Bearer " + created(login, "login")["token"]}, + } + + +def chat_of(owner, other): + body = {"title": "probe", "member_ids": [owner["id"], other["id"]]} + return created(call("POST", "/api/v1/chats", body, owner["auth"], timeout=10), "chat")["id"] + + +def send(owner, chat, text, key): + headers = dict(owner["auth"], **{"Idempotency-Key": key}) + return call("POST", f"/api/v1/chats/{chat}/messages", {"text": text}, headers) + + +def check(args): + alice, bob = actor("alice"), actor("bob") + chat = chat_of(alice, bob) + seen = collections.Counter( + call("GET", "/api/v1/users/me", headers=alice["auth"])[1] for _ in range(args.samples) + ) + key, text = uuid.uuid4().hex, "same key " + uuid.uuid4().hex + barrier, results = threading.Barrier(args.parallel), [] + + def worker(): + barrier.wait() + results.append(send(alice, chat, text, key)) + + threads = [threading.Thread(target=worker) for _ in range(args.parallel)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + page = call("GET", f"/api/v1/chats/{chat}/messages?limit=100", headers=bob["auth"])[2] + bodies = [r[2] for r in results if r[0] == 201] + race = { + "parallel": args.parallel, + "statuses": dict(collections.Counter(r[0] for r in results)), + "instances": dict(collections.Counter(r[1] for r in results)), + "identical_bodies": bool(bodies) and all(b == bodies[0] for b in bodies), + "stored_copies": sum(1 for m in (page or {"items": []})["items"] if m.get("text") == text), + "other_body_status": send(alice, chat, text + " changed", key)[0], + } + passed = ( + len(seen) >= 2 + and race["statuses"] == {201: args.parallel} + and race["identical_bodies"] + and race["stored_copies"] == 1 + and race["other_body_status"] == 409 + ) + emit({"instances_one_token": dict(seen), "same_key": race, "pass": passed}, args.out) + return 0 if passed else 1 + + +def main(): + parser = argparse.ArgumentParser(description="LB probes over the public HTTP contract") + sub = parser.add_subparsers(dest="mode", required=True) + one = sub.add_parser("check", help="instances behind one token and a same-key race") + one.add_argument("--samples", type=int, default=40) + one.add_argument("--parallel", type=int, default=8) + one.add_argument("--out", help="JSON file, written only after a complete report") + return check(parser.parse_args()) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/restart.sh b/scripts/restart.sh index c58c6532..20b252c9 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,6 +1,12 @@ #!/bin/sh set -eu cd "$(dirname "$0")/.." -docker compose stop api -docker compose restart postgres valkey -docker compose up -d --wait api +defined=$(docker compose config --services) +stores="" +for service in ${STATE_SERVICES:-postgres valkey}; do + if printf '%s\n' "$defined" | grep -qx "$service"; then stores="$stores $service"; fi +done +[ -n "$stores" ] || { echo "no state services found" >&2; exit 3; } +docker compose stop lb "${API_SERVICE:-api}" +docker compose restart $stores +docker compose up -d --wait --wait-timeout "${WAIT_TIMEOUT:-75}" diff --git a/scripts/restore.sh b/scripts/restore.sh new file mode 100755 index 00000000..705a092d --- /dev/null +++ b/scripts/restore.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose up -d --no-deps --no-recreate --wait --wait-timeout "${WAIT_TIMEOUT:-60}" "${API_SERVICE:-api}" +scripts/lb-wait.sh diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..222cd887 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +cd "$(dirname "$0")/.." +ids=$(docker compose ps -q "${API_SERVICE:-api}") +[ -n "$ids" ] || { echo "no running ${API_SERVICE:-api} replicas" >&2; exit 3; } +target=$(docker inspect -f '{{.Id}} {{.Config.Hostname}}' $ids | awk -v t="$TARGET_INSTANCE_ID" '$2 == t {print $1}') +[ -n "$target" ] || { echo "no running ${API_SERVICE:-api} replica answers as $TARGET_INSTANCE_ID" >&2; exit 3; } +docker stop -t "${STOP_TIMEOUT:-6}" "$target" >/dev/null