From e049c265b86ef5829d66a6120867eabde07c4b0a Mon Sep 17 00:00:00 2001 From: Artyom Date: Fri, 30 Oct 2026 05:43:44 +0300 Subject: [PATCH] Reproduce a lost answer on the stand and enrich the run records No failover run ever lost an answer, so the retry of an unconfirmed write was only covered in process. make retry-check holds the chat row with SELECT ... FOR UPDATE for 1.5 s (below lock_timeout), sends a message with a 0.5 s client timeout, finds the replica whose request waits on the lock in pg_stat_activity and then retries with the same key and text through the LB, once with that replica alive and once after docker kill. It checks the 201, a single copy with the id of the retry answer and, for the live replica, the 201 it logged without a client; created_at tells which attempt wrote the message. CI runs it. failover-each and rolling-update now record the database size and a docker stats snapshot in the background of the action, rolling-update names the tag and id of every image it replaces, and scaling.py writes started and the compose-up log marks in UTC with Z; the track README states that the recorded rows are host time (MSK) and what host-others.csv covers. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27 --- .github/workflows/application.yml | 3 + Makefile | 6 +- evidence/track1/README.md | 8 +- .../close-lab3-second-review-gaps/tasks.md | 10 +- scripts/failover-each.sh | 2 +- scripts/probe.py | 104 ++++++++++++++++++ scripts/rolling-update.sh | 4 +- scripts/scaling.py | 5 +- scripts/stand-usage.sh | 5 + 9 files changed, 135 insertions(+), 12 deletions(-) create mode 100755 scripts/stand-usage.sh diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml index e92b653a..b990fb84 100644 --- a/.github/workflows/application.yml +++ b/.github/workflows/application.yml @@ -37,6 +37,9 @@ jobs: - name: Несколько реплик на одном токене и гонка одного ключа run: make lb-check + - name: Повтор неподтверждённой записи после потерянного ответа и убитой реплики + run: make retry-check + - name: Остановка реплики под запросами, затем возврат run: | make test-failover ACTION_SCRIPT=scripts/stop-one.sh diff --git a/Makefile b/Makefile index 7f7e8426..a55f0439 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ PYTEST_ARGS ?= .PHONY: help check validate test load test-persistence test-failover \ install lint format typecheck student-test \ secrets up down reset-data migrate ps logs restore lb-check failover-evidence stack-evidence \ - persistence-evidence storage-evidence cache-evidence test-faults faults-evidence seed explain \ + persistence-evidence storage-evidence cache-evidence test-faults faults-evidence seed explain retry-check \ limits detection failover-each rolling-update level4 \ track1 track1-host track1-a track1-b track1-compare track1-control plots @@ -29,6 +29,7 @@ help: @echo "install/lint/typecheck/student-test: студенческий тулчейн" @echo "secrets/up/down/migrate/ps/logs: стенд Docker Compose; reset-data: намеренное удаление volumes" @echo "restore: вернуть остановленные реплики; lb-check: реплики на одном токене и 20 гонок одного ключа" + @echo "retry-check: потерянный ответ на живой и на убитой реплике, повтор тем же ключом" @echo "level4 = limits, lb-check, detection, failover-each, rolling-update [FROM_REV=<коммит>]; RATE=$(RATE) rps" @echo "track1 = track1-host, track1-a (1x4/2x2/4x1), track1-b (1x1/2x1/8x1), track1-compare [BEFORE_REV=$(BEFORE_REV)], track1-control, plots → $(TRACK1)" @echo "BASE_URL=$(BASE_URL) (порт lb из docker compose); failover-evidence SHIFT=1 — сдвиг round robin на вторую реплику" @@ -101,6 +102,9 @@ restore: lb-check: $(PYTHON) scripts/probe.py check --rounds 20 --out evidence/lb-check.json +retry-check: + $(PYTHON) scripts/probe.py retry --out evidence/retry.json + limits: $(PYTHON) scripts/limits.py --out evidence/limits.txt diff --git a/evidence/track1/README.md b/evidence/track1/README.md index 220ae4f8..3baa59a9 100644 --- a/evidence/track1/README.md +++ b/evidence/track1/README.md @@ -3,7 +3,7 @@ | Файл | Что внутри | Чем получен | | --- | --- | --- | | `host.txt` | `docker stats --all` всего хоста, контейнеры, версии Docker и образов перед серией A | `make track1-host` | -| `host-others.csv` | раз в 10 с весь период замеров: суммарный CPU чужих контейнеров (стенды других лаб) и самый загруженный из них | цикл `docker stats --no-stream` по контейнерам не из проекта `mtusi2026lab3`, запущенный рядом с сериями | +| `host-others.csv` | раз в 10 с с 2026-10-28T17:32Z до 2026-10-28T22:42Z (20:32–01:42 MSK; `ts` — Unix-время): суммарный CPU чужих контейнеров (стенды других лаб) и самый загруженный из них. Покрывает серии A, B и сравнение, но не контрольные прогоны 30.10: для них чужая нагрузка есть только в столбце `cpu_other` их `summary.csv` | цикл `docker stats --no-stream` по контейнерам не из проекта `mtusi2026lab3`, запущенный рядом с сериями | | `aborted/series-a-1-summary.csv` | первая попытка серии A, прерванная из-за сборки образа соседней лабы: шаги с API 410 % при 1 350 rps и паузами Docker API по 15 мин | `make track1-a` до добавления отбраковки | | `series-a/summary.csv` | серия A: 1×4, 2×2, 4×1 при 4 CPU / 2 GiB API, строка на прогон | `make track1-a` | | `series-b/summary.csv` | серия B: 1×1, 2×1, 8×1 по 1 CPU / 512 MiB (4×1 — из серии A) | `make track1-b` | @@ -26,4 +26,8 @@ `idle_tx` — транзакция открыта и ждёт приложение, `lock` — ждут блокировку строки), `lb_qcur_max` — очередь HAProxy к backend `api`, `db_mb`/`messages` — размер базы и число сообщений перед прогоном, `gap_s` — наибольший разрыв между сэмплами, `wall_s` — длительность прогона k6, `saturated` — доля ошибок > 1 %, или сброшенных -итераций > 1 %, или p95 > 500 мс, `disturbed` — прогон отбракован. +итераций > 1 %, или p95 > 500 мс, `disturbed` — прогон отбракован, `started` — начало окна замера. + +Время: `started` во всех строках, снятых в этом каталоге, и метки `== …` в `compose-up-failed.log` — местное время +хоста MSK (UTC+3) без суффикса; `*.txt` и `action.txt` в `evidence/` — UTC с `Z`, `ts` в CSV сэмплеров — Unix-время. +Новые прогоны `scripts/scaling.py` пишут `started` в UTC с `Z`, поэтому строка без суффикса — MSK, с `Z` — UTC. diff --git a/openspec/changes/close-lab3-second-review-gaps/tasks.md b/openspec/changes/close-lab3-second-review-gaps/tasks.md index 9b2a5865..3dc125e9 100644 --- a/openspec/changes/close-lab3-second-review-gaps/tasks.md +++ b/openspec/changes/close-lab3-second-review-gaps/tasks.md @@ -10,15 +10,15 @@ ## 2. Стенд и CI - [x] 2.1 `make up` ждёт все реплики в LB (`scripts/lb-wait.sh`) -- [ ] 2.2 CI: non-root по числовому uid `docker top -eo pid,uid,comm`; шаг `make retry-check` +- [x] 2.2 CI: non-root по числовому uid `docker top -eo pid,uid,comm`; шаг `make retry-check` - [x] 2.3 Проверка группы: `make up` и сразу `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`; проверка non-root на стенде и на контейнере от root ## 3. Обвязка доказательств -- [ ] 3.1 `probe.py retry` и `make retry-check`: потерянный ответ на живой и на убитой реплике → `evidence/retry.json` -- [ ] 3.2 `scripts/stand-usage.sh` в действиях `failover-each` и `rolling-update`; тег и ID образа в `rolling-update.sh` -- [ ] 3.3 `scaling.py`: `started` и метки `compose-up-failed.log` в UTC; `evidence/track1/README.md` — время прежних строк (MSK) -- [ ] 3.4 Проверка группы: `make lint`, `make check`, `sh -n` скриптов, `make retry-check` на стенде +- [x] 3.1 `probe.py retry` и `make retry-check`: потерянный ответ на живой и на убитой реплике → `evidence/retry.json` +- [x] 3.2 `scripts/stand-usage.sh` в действиях `failover-each` и `rolling-update`; тег и ID образа в `rolling-update.sh` +- [x] 3.3 `scaling.py`: `started` и метки `compose-up-failed.log` в UTC; `evidence/track1/README.md` — время прежних строк (MSK) +- [x] 3.4 Проверка группы: `make lint`, `make check`, `sh -n` скриптов, `make retry-check` на стенде ## 4. Доказательства и отчёт diff --git a/scripts/failover-each.sh b/scripts/failover-each.sh index 0fba2652..02f1b7ca 100755 --- a/scripts/failover-each.sh +++ b/scripts/failover-each.sh @@ -10,7 +10,7 @@ failed=0 set -- for host in $hosts; do TARGET_INSTANCE_ID=$host python3 scripts/probe.py load --expect dip --out "$out/$host" \ - -- sh -c "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' ${API_SERVICE:-api} && sleep ${HOLD:-10} && scripts/restore.sh" || failed=1 + -- sh -c "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' ${API_SERVICE:-api} && { scripts/stand-usage.sh & sleep ${HOLD:-10}; wait; } && scripts/restore.sh" || failed=1 scripts/restore.sh set -- "$@" "$out/$host/result.json" done diff --git a/scripts/probe.py b/scripts/probe.py index a624bd0b..7a23927a 100755 --- a/scripts/probe.py +++ b/scripts/probe.py @@ -12,10 +12,13 @@ import time import urllib.error import urllib.request import uuid +from datetime import datetime from pathlib import Path ROOT = Path(__file__).resolve().parents[1] COUNTERS = ("wretr", "wredis", "econ", "eresp", "hrsp_5xx") +PSQL = ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger"] +PSQL += ["-X", "-q", "-At"] @functools.cache @@ -207,6 +210,101 @@ def check(args): return 0 if passed else 1 +def output(*command): + done = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True, check=False) + return done.stdout + done.stderr + + +def replicas(): + ids = output("docker", "compose", "ps", "-q", os.environ.get("API_SERVICE", "api")).split() + form = "{{.Id}} {{.Config.Hostname}} {{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}" + return [line.split() for line in output("docker", "inspect", "-f", form, *ids).splitlines()] + + +def answered(container, request_id): + for line in output("docker", "logs", container).splitlines(): + if request_id in line and "request.handled" in line: + return json.loads(line).get("status") + return None + + +def lost(alice, bob, chat, kill, args): + key, text = uuid.uuid4().hex, "lost " + uuid.uuid4().hex + request_id = "lost-" + key[:12] + lock = subprocess.Popen( + PSQL, cwd=str(ROOT), stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True + ) + lock.stdin.write( + f"BEGIN;\nSELECT 'locked' FROM chats WHERE id = '{uuid.UUID(chat)}' FOR UPDATE;\n" + f"SELECT pg_sleep({args.hold});\nCOMMIT;\n" + ) + lock.stdin.flush() + lock.stdout.readline() + sent = time.time() + headers = dict(alice["auth"], **{"Idempotency-Key": key, "X-Request-Id": request_id}) + path = f"/api/v1/chats/{chat}/messages" + first = call("POST", path, {"text": text}, headers, args.timeout)[0] + sql = "SELECT host(client_addr) FROM pg_stat_activity WHERE wait_event_type = 'Lock'" + waiting = set(output(*PSQL, "-c", sql).split()) + holder = next((r for r in replicas() if waiting & set(r[2:])), [None, None]) + if kill and holder[0]: + subprocess.call(["docker", "kill", holder[0]], stdout=subprocess.DEVNULL) + lock.stdin.close() + lock.wait() + retried = time.time() + attempts, (status, instance, body) = 1, send(alice, chat, text, key) + while status != 201 and attempts < 20: + time.sleep(0.2) + attempts, (status, instance, body) = attempts + 1, send(alice, chat, text, key) + page = call("GET", path + "?limit=100", headers=bob["auth"])[2] or {"items": []} + copies = [m for m in page["items"] if m["text"] == text] + stamp = copies[0]["created_at"].replace("Z", "+00:00") if copies else None + report = { + "variant": "kill" if kill else "timeout", + "lost_attempt": { + "status": first, + "client_timeout_s": args.timeout, + "chat_lock_held_s": args.hold, + "replica": holder[1], + "replica_killed": bool(kill and holder[0]), + "replica_answered": answered(holder[0], request_id) if holder[0] else None, + }, + "retry": { + "after_s": round(retried - sent, 3), + "status": status, + "attempts": attempts, + "replica": instance, + }, + "copies_in_history": len(copies), + "same_message": bool(copies) and copies[0]["id"] == (body or {}).get("id"), + "written_by": None + if stamp is None + else ("lost attempt" if datetime.fromisoformat(stamp).timestamp() < retried else "retry"), + } + if kill and holder[0]: + subprocess.call( + [str(ROOT / "scripts/restore.sh")], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + report["pass"] = ( + first == 0 + and holder[0] is not None + and status == 201 + and report["copies_in_history"] == 1 + and report["same_message"] + and (kill or report["lost_attempt"]["replica_answered"] == 201) + ) + return report + + +def retry(args): + alice, bob = actor("alice"), actor("bob") + chat = chat_of(alice, bob) + variants = [lost(alice, bob, chat, kill, args) for kill in (False, True)] + passed = all(v["pass"] for v in variants) + emit({"variants": variants, "pass": passed}, args.out) + return 0 if passed else 1 + + def record(stop, alice, chat): probes, trace = [], [] @@ -413,6 +511,10 @@ def main(): under.add_argument("--max-failed-ratio", type=float, default=0.01) under.add_argument("--max-recovery", type=float, default=10.0) under.add_argument("command", nargs=argparse.REMAINDER) + again = sub.add_parser("retry", help="retry a write whose answer was lost, same key") + again.add_argument("--hold", type=float, default=1.5, help="chat row lock, s (< lock_timeout)") + again.add_argument("--timeout", type=float, default=0.5, help="first attempt timeout, s") + again.add_argument("--out", help="JSON file, written only after a complete report") watch = sub.add_parser("detect", help="time until the LB takes replicas out of rotation") watch.add_argument("--max", type=float, required=True, help="declared worst detection, s") watch.add_argument("--slack", type=float, default=0.25, help="docker CLI and polling, s") @@ -426,6 +528,8 @@ def main(): args = parser.parse_args() if args.mode == "check": return check(args) + if args.mode == "retry": + return retry(args) args.command = [c for c in args.command if c != "--"] if not args.command: parser.error(f"{args.mode} needs an action command after --") diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh index 59735a75..777c6671 100755 --- a/scripts/rolling-update.sh +++ b/scripts/rolling-update.sh @@ -2,7 +2,8 @@ set -eu cd "$(dirname "$0")/.." service=${API_SERVICE:-api} -show() { docker inspect -f '{{.Config.Hostname}} image={{slice .Image 7 19}}' "$1"; } +show() { docker inspect -f '{{.Config.Hostname}} image={{.Config.Image}}@{{slice .Image 7 19}}' "$1"; } +scripts/stand-usage.sh & echo "$(date -u +%T) migrate once before the first replacement" docker compose run --rm --no-deps -T migrate 2>&1 old=$(docker compose ps -q "$service") @@ -17,4 +18,5 @@ for id in $old; do docker rm "$id" >/dev/null done scripts/lb-wait.sh "$count" +wait echo "$(date -u +%T) done: $count replicas replaced" diff --git a/scripts/scaling.py b/scripts/scaling.py index f1290f4d..657102d1 100755 --- a/scripts/scaling.py +++ b/scripts/scaling.py @@ -217,7 +217,8 @@ def apply(base, settings, env, out): except subprocess.CalledProcessError as error: log = Path(out) / "compose-up-failed.log" with log.open("a") as handle: - handle.write(f"== {time.strftime('%FT%T')} {base} attempt {attempt + 1}\n") + stamp = time.strftime("%FT%TZ", time.gmtime()) + handle.write(f"== {stamp} {base} attempt {attempt + 1}\n") handle.write(error.stdout + error.stderr + "\n") print(json.dumps({"compose_up_failed": attempt + 1, "log": str(log)})) if attempt == 2: @@ -266,7 +267,7 @@ def step(base, rate, repeat, args, env): or metrics["p95_ms"] > args.slo_p95_ms ) row = dict(base, rate=rate, repeat=repeat, k6_exit=code, saturated=saturated, **metrics) - row["started"] = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(start)) + row["started"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(start)) row["db_mb"] = round(int(database[0]) / 2**20, 1) row["messages"] = int(database[1]) row["p95_post_ms"] = trend(summary, "post_message") diff --git a/scripts/stand-usage.sh b/scripts/stand-usage.sh new file mode 100755 index 00000000..1b29ec26 --- /dev/null +++ b/scripts/stand-usage.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +echo "$(date -u +%T) database bytes,messages: $(sh scripts/pg-stats.sh size)" +docker stats --no-stream --format '{{.Name}} cpu={{.CPUPerc}} mem={{.MemUsage}}' $(docker compose ps -q)