diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml index e92b653a..b990fb84 100644 --- a/.github/workflows/application.yml +++ b/.github/workflows/application.yml @@ -37,6 +37,9 @@ jobs: - name: Несколько реплик на одном токене и гонка одного ключа run: make lb-check + - name: Повтор неподтверждённой записи после потерянного ответа и убитой реплики + run: make retry-check + - name: Остановка реплики под запросами, затем возврат run: | make test-failover ACTION_SCRIPT=scripts/stop-one.sh diff --git a/Makefile b/Makefile index 7f7e8426..a55f0439 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ PYTEST_ARGS ?= .PHONY: help check validate test load test-persistence test-failover \ install lint format typecheck student-test \ secrets up down reset-data migrate ps logs restore lb-check failover-evidence stack-evidence \ - persistence-evidence storage-evidence cache-evidence test-faults faults-evidence seed explain \ + persistence-evidence storage-evidence cache-evidence test-faults faults-evidence seed explain retry-check \ limits detection failover-each rolling-update level4 \ track1 track1-host track1-a track1-b track1-compare track1-control plots @@ -29,6 +29,7 @@ help: @echo "install/lint/typecheck/student-test: студенческий тулчейн" @echo "secrets/up/down/migrate/ps/logs: стенд Docker Compose; reset-data: намеренное удаление volumes" @echo "restore: вернуть остановленные реплики; lb-check: реплики на одном токене и 20 гонок одного ключа" + @echo "retry-check: потерянный ответ на живой и на убитой реплике, повтор тем же ключом" @echo "level4 = limits, lb-check, detection, failover-each, rolling-update [FROM_REV=<коммит>]; RATE=$(RATE) rps" @echo "track1 = track1-host, track1-a (1x4/2x2/4x1), track1-b (1x1/2x1/8x1), track1-compare [BEFORE_REV=$(BEFORE_REV)], track1-control, plots → $(TRACK1)" @echo "BASE_URL=$(BASE_URL) (порт lb из docker compose); failover-evidence SHIFT=1 — сдвиг round robin на вторую реплику" @@ -101,6 +102,9 @@ restore: lb-check: $(PYTHON) scripts/probe.py check --rounds 20 --out evidence/lb-check.json +retry-check: + $(PYTHON) scripts/probe.py retry --out evidence/retry.json + limits: $(PYTHON) scripts/limits.py --out evidence/limits.txt diff --git a/evidence/track1/README.md b/evidence/track1/README.md index 220ae4f8..3baa59a9 100644 --- a/evidence/track1/README.md +++ b/evidence/track1/README.md @@ -3,7 +3,7 @@ | Файл | Что внутри | Чем получен | | --- | --- | --- | | `host.txt` | `docker stats --all` всего хоста, контейнеры, версии Docker и образов перед серией A | `make track1-host` | -| `host-others.csv` | раз в 10 с весь период замеров: суммарный CPU чужих контейнеров (стенды других лаб) и самый загруженный из них | цикл `docker stats --no-stream` по контейнерам не из проекта `mtusi2026lab3`, запущенный рядом с сериями | +| `host-others.csv` | раз в 10 с с 2026-10-28T17:32Z до 2026-10-28T22:42Z (20:32–01:42 MSK; `ts` — Unix-время): суммарный CPU чужих контейнеров (стенды других лаб) и самый загруженный из них. Покрывает серии A, B и сравнение, но не контрольные прогоны 30.10: для них чужая нагрузка есть только в столбце `cpu_other` их `summary.csv` | цикл `docker stats --no-stream` по контейнерам не из проекта `mtusi2026lab3`, запущенный рядом с сериями | | `aborted/series-a-1-summary.csv` | первая попытка серии A, прерванная из-за сборки образа соседней лабы: шаги с API 410 % при 1 350 rps и паузами Docker API по 15 мин | `make track1-a` до добавления отбраковки | | `series-a/summary.csv` | серия A: 1×4, 2×2, 4×1 при 4 CPU / 2 GiB API, строка на прогон | `make track1-a` | | `series-b/summary.csv` | серия B: 1×1, 2×1, 8×1 по 1 CPU / 512 MiB (4×1 — из серии A) | `make track1-b` | @@ -26,4 +26,8 @@ `idle_tx` — транзакция открыта и ждёт приложение, `lock` — ждут блокировку строки), `lb_qcur_max` — очередь HAProxy к backend `api`, `db_mb`/`messages` — размер базы и число сообщений перед прогоном, `gap_s` — наибольший разрыв между сэмплами, `wall_s` — длительность прогона k6, `saturated` — доля ошибок > 1 %, или сброшенных -итераций > 1 %, или p95 > 500 мс, `disturbed` — прогон отбракован. +итераций > 1 %, или p95 > 500 мс, `disturbed` — прогон отбракован, `started` — начало окна замера. + +Время: `started` во всех строках, снятых в этом каталоге, и метки `== …` в `compose-up-failed.log` — местное время +хоста MSK (UTC+3) без суффикса; `*.txt` и `action.txt` в `evidence/` — UTC с `Z`, `ts` в CSV сэмплеров — Unix-время. +Новые прогоны `scripts/scaling.py` пишут `started` в UTC с `Z`, поэтому строка без суффикса — MSK, с `Z` — UTC. diff --git a/openspec/changes/close-lab3-second-review-gaps/tasks.md b/openspec/changes/close-lab3-second-review-gaps/tasks.md index 9b2a5865..3dc125e9 100644 --- a/openspec/changes/close-lab3-second-review-gaps/tasks.md +++ b/openspec/changes/close-lab3-second-review-gaps/tasks.md @@ -10,15 +10,15 @@ ## 2. Стенд и CI - [x] 2.1 `make up` ждёт все реплики в LB (`scripts/lb-wait.sh`) -- [ ] 2.2 CI: non-root по числовому uid `docker top -eo pid,uid,comm`; шаг `make retry-check` +- [x] 2.2 CI: non-root по числовому uid `docker top -eo pid,uid,comm`; шаг `make retry-check` - [x] 2.3 Проверка группы: `make up` и сразу `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`; проверка non-root на стенде и на контейнере от root ## 3. Обвязка доказательств -- [ ] 3.1 `probe.py retry` и `make retry-check`: потерянный ответ на живой и на убитой реплике → `evidence/retry.json` -- [ ] 3.2 `scripts/stand-usage.sh` в действиях `failover-each` и `rolling-update`; тег и ID образа в `rolling-update.sh` -- [ ] 3.3 `scaling.py`: `started` и метки `compose-up-failed.log` в UTC; `evidence/track1/README.md` — время прежних строк (MSK) -- [ ] 3.4 Проверка группы: `make lint`, `make check`, `sh -n` скриптов, `make retry-check` на стенде +- [x] 3.1 `probe.py retry` и `make retry-check`: потерянный ответ на живой и на убитой реплике → `evidence/retry.json` +- [x] 3.2 `scripts/stand-usage.sh` в действиях `failover-each` и `rolling-update`; тег и ID образа в `rolling-update.sh` +- [x] 3.3 `scaling.py`: `started` и метки `compose-up-failed.log` в UTC; `evidence/track1/README.md` — время прежних строк (MSK) +- [x] 3.4 Проверка группы: `make lint`, `make check`, `sh -n` скриптов, `make retry-check` на стенде ## 4. Доказательства и отчёт diff --git a/scripts/failover-each.sh b/scripts/failover-each.sh index 0fba2652..02f1b7ca 100755 --- a/scripts/failover-each.sh +++ b/scripts/failover-each.sh @@ -10,7 +10,7 @@ failed=0 set -- for host in $hosts; do TARGET_INSTANCE_ID=$host python3 scripts/probe.py load --expect dip --out "$out/$host" \ - -- sh -c "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' ${API_SERVICE:-api} && sleep ${HOLD:-10} && scripts/restore.sh" || failed=1 + -- sh -c "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' ${API_SERVICE:-api} && { scripts/stand-usage.sh & sleep ${HOLD:-10}; wait; } && scripts/restore.sh" || failed=1 scripts/restore.sh set -- "$@" "$out/$host/result.json" done diff --git a/scripts/probe.py b/scripts/probe.py index a624bd0b..7a23927a 100755 --- a/scripts/probe.py +++ b/scripts/probe.py @@ -12,10 +12,13 @@ import time import urllib.error import urllib.request import uuid +from datetime import datetime from pathlib import Path ROOT = Path(__file__).resolve().parents[1] COUNTERS = ("wretr", "wredis", "econ", "eresp", "hrsp_5xx") +PSQL = ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger"] +PSQL += ["-X", "-q", "-At"] @functools.cache @@ -207,6 +210,101 @@ def check(args): return 0 if passed else 1 +def output(*command): + done = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False) + return done.stdout + done.stderr + + +def replicas(): + ids = output("docker", "compose", "ps", "-q", os.environ.get("API_SERVICE", "api")).split() + form = "{{.Id}} {{.Config.Hostname}} {{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}" + return [line.split() for line in output("docker", "inspect", "-f", form, *ids).splitlines()] + + +def answered(container, request_id): + for line in output("docker", "logs", container).splitlines(): + if request_id in line and "request.handled" in line: + return json.loads(line).get("status") + return None + + +def lost(alice, bob, chat, kill, args): + key, text = uuid.uuid4().hex, "lost " + uuid.uuid4().hex + request_id = "lost-" + key[:12] + lock = subprocess.Popen( + PSQL, cwd=str(ROOT), stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True + ) + lock.stdin.write( + f"BEGIN;\nSELECT 'locked' FROM chats WHERE id = '{uuid.UUID(chat)}' FOR UPDATE;\n" + f"SELECT pg_sleep({args.hold});\nCOMMIT;\n" + ) + lock.stdin.flush() + lock.stdout.readline() + sent = time.time() + headers = dict(alice["auth"], **{"Idempotency-Key": key, "X-Request-Id": request_id}) + path = f"/api/v1/chats/{chat}/messages" + first = call("POST", path, {"text": text}, headers, args.timeout)[0] + sql = "SELECT host(client_addr) FROM pg_stat_activity WHERE wait_event_type = 'Lock'" + waiting = set(output(*PSQL, "-c", sql).split()) + holder = next((r for r in replicas() if waiting & set(r[2:])), [None, None]) + if kill and holder[0]: + subprocess.call(["docker", "kill", holder[0]], stdout=subprocess.DEVNULL) + lock.stdin.close() + lock.wait() + retried = time.time() + attempts, (status, instance, body) = 1, send(alice, chat, text, key) + while status != 201 and attempts < 20: + time.sleep(0.2) + attempts, (status, instance, body) = attempts + 1, send(alice, chat, text, key) + page = call("GET", path + "?limit=100", headers=bob["auth"])[2] or {"items": []} + copies = [m for m in page["items"] if m["text"] == text] + stamp = copies[0]["created_at"].replace("Z", "+00:00") if copies else None + report = { + "variant": "kill" if kill else "timeout", + "lost_attempt": { + "status": first, + "client_timeout_s": args.timeout, + "chat_lock_held_s": args.hold, + "replica": holder[1], + "replica_killed": bool(kill and holder[0]), + "replica_answered": answered(holder[0], request_id) if holder[0] else None, + }, + "retry": { + "after_s": round(retried - sent, 3), + "status": status, + "attempts": attempts, + "replica": instance, + }, + "copies_in_history": len(copies), + "same_message": bool(copies) and copies[0]["id"] == (body or {}).get("id"), + "written_by": None + if stamp is None + else ("lost attempt" if datetime.fromisoformat(stamp).timestamp() < retried else "retry"), + } + if kill and holder[0]: + subprocess.call( + [str(ROOT / "scripts/restore.sh")], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + report["pass"] = ( + first == 0 + and holder[0] is not None + and status == 201 + and report["copies_in_history"] == 1 + and report["same_message"] + and (kill or report["lost_attempt"]["replica_answered"] == 201) + ) + return report + + +def retry(args): + alice, bob = actor("alice"), actor("bob") + chat = chat_of(alice, bob) + variants = [lost(alice, bob, chat, kill, args) for kill in (False, True)] + passed = all(v["pass"] for v in variants) + emit({"variants": variants, "pass": passed}, args.out) + return 0 if passed else 1 + + def record(stop, alice, chat): probes, trace = [], [] @@ -413,6 +511,10 @@ def main(): under.add_argument("--max-failed-ratio", type=float, default=0.01) under.add_argument("--max-recovery", type=float, default=10.0) under.add_argument("command", nargs=argparse.REMAINDER) + again = sub.add_parser("retry", help="retry a write whose answer was lost, same key") + again.add_argument("--hold", type=float, default=1.5, help="chat row lock, s (< lock_timeout)") + again.add_argument("--timeout", type=float, default=0.5, help="first attempt timeout, s") + again.add_argument("--out", help="JSON file, written only after a complete report") watch = sub.add_parser("detect", help="time until the LB takes replicas out of rotation") watch.add_argument("--max", type=float, required=True, help="declared worst detection, s") watch.add_argument("--slack", type=float, default=0.25, help="docker CLI and polling, s") @@ -426,6 +528,8 @@ def main(): args = parser.parse_args() if args.mode == "check": return check(args) + if args.mode == "retry": + return retry(args) args.command = [c for c in args.command if c != "--"] if not args.command: parser.error(f"{args.mode} needs an action command after --") diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh index 59735a75..777c6671 100755 --- a/scripts/rolling-update.sh +++ b/scripts/rolling-update.sh @@ -2,7 +2,8 @@ set -eu cd "$(dirname "$0")/.." service=${API_SERVICE:-api} -show() { docker inspect -f '{{.Config.Hostname}} image={{slice .Image 7 19}}' "$1"; } +show() { docker inspect -f '{{.Config.Hostname}} image={{.Config.Image}}@{{slice .Image 7 19}}' "$1"; } +scripts/stand-usage.sh & echo "$(date -u +%T) migrate once before the first replacement" docker compose run --rm --no-deps -T migrate 2>&1 old=$(docker compose ps -q "$service") @@ -17,4 +18,5 @@ for id in $old; do docker rm "$id" >/dev/null done scripts/lb-wait.sh "$count" +wait echo "$(date -u +%T) done: $count replicas replaced" diff --git a/scripts/scaling.py b/scripts/scaling.py index f1290f4d..657102d1 100755 --- a/scripts/scaling.py +++ b/scripts/scaling.py @@ -217,7 +217,8 @@ def apply(base, settings, env, out): except subprocess.CalledProcessError as error: log = Path(out) / "compose-up-failed.log" with log.open("a") as handle: - handle.write(f"== {time.strftime('%FT%T')} {base} attempt {attempt + 1}\n") + stamp = time.strftime("%FT%TZ", time.gmtime()) + handle.write(f"== {stamp} {base} attempt {attempt + 1}\n") handle.write(error.stdout + error.stderr + "\n") print(json.dumps({"compose_up_failed": attempt + 1, "log": str(log)})) if attempt == 2: @@ -266,7 +267,7 @@ def step(base, rate, repeat, args, env): or metrics["p95_ms"] > args.slo_p95_ms ) row = dict(base, rate=rate, repeat=repeat, k6_exit=code, saturated=saturated, **metrics) - row["started"] = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(start)) + row["started"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(start)) row["db_mb"] = round(int(database[0]) / 2**20, 1) row["messages"] = int(database[1]) row["p95_post_ms"] = trend(summary, "post_message") diff --git a/scripts/stand-usage.sh b/scripts/stand-usage.sh new file mode 100755 index 00000000..1b29ec26 --- /dev/null +++ b/scripts/stand-usage.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +echo "$(date -u +%T) database bytes,messages: $(sh scripts/pg-stats.sh size)" +docker stats --no-stream --format '{{.Name}} cpu={{.CPUPerc}} mem={{.MemUsage}}' $(docker compose ps -q)