From 252e3e3ffeea5004cfd9dc342e686af1417a77a2 Mon Sep 17 00:00:00 2001 From: Artyom Date: Thu, 5 Nov 2026 22:15:18 +0300 Subject: [PATCH] Measure successful recovery and verify history under load --- .../verify-lab3-grade4-recovery/tasks.md | 4 +- scripts/probe.py | 66 +++++++++++++-- student-tests/unit/test_probe.py | 83 +++++++++++++++++++ 3 files changed, 144 insertions(+), 9 deletions(-) create mode 100644 student-tests/unit/test_probe.py diff --git a/openspec/changes/verify-lab3-grade4-recovery/tasks.md b/openspec/changes/verify-lab3-grade4-recovery/tasks.md index 4e22c9c3..8d7af8fa 100644 --- a/openspec/changes/verify-lab3-grade4-recovery/tasks.md +++ b/openspec/changes/verify-lab3-grade4-recovery/tasks.md @@ -4,8 +4,8 @@ ## 2. Нагрузочный зонд -- [ ] 2.1 Исправить измерение восстановления, добавить сверку полной истории и отбраковку неполной нагрузки/статистики. -- [ ] 2.2 Проверить негативные случаи собственными тестами; `make check`, `make lint`, `make student-test`. +- [x] 2.1 Исправить измерение восстановления, добавить сверку полной истории и отбраковку неполной нагрузки/статистики. +- [x] 2.2 Проверить негативные случаи собственными тестами; `make check`, `make lint`, `make student-test`. ## 3. Стенд уровня «4» diff --git a/scripts/probe.py b/scripts/probe.py index 7a23927a..71652a59 100755 --- a/scripts/probe.py +++ b/scripts/probe.py @@ -10,6 +10,7 @@ import sys import threading import time import urllib.error +import urllib.parse import urllib.request import uuid from datetime import datetime @@ -306,15 +307,16 @@ def retry(args): def record(stop, alice, chat): - probes, trace = [], [] + probes, trace, confirmed = [], [], [] def prober(): key = uuid.uuid4().hex while not stop.is_set(): started = time.time() - status, instance, _ = send(alice, chat, "probe " + key, key) + status, instance, body = send(alice, chat, "probe " + key, key) probes.append((started, status, round(1000 * (time.time() - started), 1), instance)) if status == 201: + confirmed.append(body) key = uuid.uuid4().hex time.sleep(0.1) @@ -326,7 +328,38 @@ def record(stop, alice, chat): threads = [threading.Thread(target=t, daemon=True) for t in (prober, poller)] for thread in threads: thread.start() - return probes, trace, threads + return probes, trace, threads, confirmed + + +def history_check(reader, chat, confirmed): + items, cursors = [], set() + cursor = None + for _ in range(len(confirmed) // 100 + 2): + path = f"/api/v1/chats/{chat}/messages?limit=100" + if cursor: + path += "&cursor=" + urllib.parse.quote(cursor, safe="") + status, _, page = call("GET", path, headers=reader["auth"]) + if status != 200 or not isinstance(page, dict): + break + items.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + same = sorted(items, key=lambda m: m["id"]) == sorted(confirmed, key=lambda m: m["id"]) + return { + "confirmed": len(confirmed), + "stored": len(items), + "exact_match": bool(confirmed) and same, + "unique": len({m["id"] for m in items}) == len(items), + } + if cursor in cursors: + break + cursors.add(cursor) + return { + "confirmed": len(confirmed), + "stored": len(items), + "exact_match": False, + "unique": False, + } def write_csv(path, header, rows): @@ -339,6 +372,10 @@ def summarize(args, probes, trace, action, expected): t0, end = action["start"], action["end"] label = args.label or os.environ.get("TARGET_INSTANCE_ID", "action") failures = [t - t0 for t, s, _, _ in probes if s != 201 and t >= t0] + successes = [(t, t + ms / 1000 - t0) for t, s, ms, _ in probes if s == 201 and t >= t0] + recovery = ( + next((end for t, end in successes if t - t0 > failures[-1]), None) if failures else 0.0 + ) down = [t for t, up in trace if t >= t0 and up is not None and up < expected] back = [t for t, up in trace if down and t > down[0] and up is not None and up >= expected] outage = [t for t in down if not back or t < back[0]] or [end] @@ -354,13 +391,15 @@ def summarize(args, probes, trace, action, expected): "action_finished_before_load_end": end < action["load_end"], "lb_up_before": expected, "lb_up_min": min((up for _, up in trace if up is not None), default=None), + "lb_samples_missing": sum(up is None for _, up in trace), "detection_seconds": round(down[0] - t0, 3) if down else None, "rejoin_seconds": round(back[0] - t0, 3) if back else None, "probe": { "attempts": len(probes), "failed_attempts": sum(1 for _, s, _, _ in probes if s != 201), "first_failure_s": round(failures[0], 3) if failures else None, - "recovery_s": round(failures[-1], 3) if failures else 0.0, + "first_success_s": round(successes[0][1], 3) if successes else None, + "recovery_s": round(recovery, 3) if recovery is not None else None, "instances_before": dict(before), "instances_during_outage": dict(during), "instances_after": dict(after), @@ -373,7 +412,13 @@ def verdict(args, result): checks = { "action_ok": result["action_exit"] == 0 and result["action_finished_before_load_end"], "load_errors_ok": k6["exit"] == 0 and k6["failed_ratio"] <= args.max_failed_ratio, - "write_recovery_ok": probe["recovery_s"] <= args.max_recovery, + "load_delivered": k6["requests"] > 0 and k6["dropped_iterations"] == 0, + "lb_observed": result["lb_up_before"] >= 2 and result["lb_samples_missing"] == 0, + "write_recovery_ok": all( + probe[k] is not None and probe[k] <= args.max_recovery + for k in ("first_success_s", "recovery_s") + ), + "history_preserved": result["history"]["exact_match"] and result["history"]["unique"], } if args.expect == "dip": checks["detected"] = result["detection_seconds"] is not None @@ -394,7 +439,8 @@ def load(args): expected = lb_up() or 0 alice, bob = actor("alice"), actor("bob") stop, action = threading.Event(), {} - probes, trace, threads = record(stop, alice, chat_of(alice, bob)) + chat = chat_of(alice, bob) + probes, trace, threads, confirmed = record(stop, alice, chat) def act(): time.sleep(args.warmup + args.lead) @@ -425,6 +471,7 @@ def load(args): ) write_csv(out / "lb_up.csv", "t_rel_s,up", ((round(t - t0, 3), up) for t, up in trace)) result = summarize(args, probes, trace, action, expected) + result["history"] = history_check(bob, chat, confirmed) result["lb_counters"] = {k: v - counters[k] for k, v in lb_counters().items()} result["k6"] = dict( k6_metrics(out / "k6/summary.json", args.duration), exit=code, rate=args.rate @@ -442,10 +489,13 @@ def detect(args): started = time.time() process = subprocess.Popen(args.command, cwd=str(ROOT)) detected = acted = None + missing = 0 while time.time() - started < args.timeout and (detected is None or acted is None): if acted is None and process.poll() is not None: acted = time.time() - started - if detected is None and (lb_up() or 0) <= until: + up = lb_up() + missing += up is None + if detected is None and up is not None and up <= until: detected = time.time() - started time.sleep(0.05) code = process.wait() @@ -460,6 +510,7 @@ def detect(args): restored = round(time.time() - begun, 3) if (lb_up() or 0) >= before else None checks = { "action_ok": code == 0, + "lb_observed": before >= 2 and missing == 0, "detected_in_time": detected is not None and detected <= args.max + args.slack, "lb_answer_ok": status == args.lb_status, } @@ -472,6 +523,7 @@ def detect(args): "action_exit": code, "action_seconds": round(acted, 3), "lb_up_before": before, + "lb_samples_missing": missing, "lb_up_counted_as_detected": until, "detection_seconds": None if detected is None else round(detected, 3), "declared_max_seconds": args.max, diff --git a/student-tests/unit/test_probe.py b/student-tests/unit/test_probe.py new file mode 100644 index 00000000..c0764d67 --- /dev/null +++ b/student-tests/unit/test_probe.py @@ -0,0 +1,83 @@ +import importlib.util +from pathlib import Path +from types import SimpleNamespace + +import pytest + +SPEC = importlib.util.spec_from_file_location( + "grade4_probe", Path(__file__).resolve().parents[2] / "scripts/probe.py" +) +assert SPEC is not None and SPEC.loader is not None +probe = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(probe) +ARGS = SimpleNamespace( + label="old", command=[], expect="replace", max_failed_ratio=0.01, max_recovery=10 +) + + +def result_of(probes: list[tuple]) -> dict: + result = probe.summarize( + ARGS, probes, [(99, 2), (101, 2)], {"start": 100, "end": 101, "exit": 0, "load_end": 120}, 2 + ) + result["k6"] = {"exit": 0, "failed_ratio": 0, "requests": 100, "dropped_iterations": 0} + result["history"] = {"exact_match": True, "unique": True} + return result + + +@pytest.mark.parametrize( + ("attempts", "recovery", "passed"), + [ + ([(100, 503, 20, "old")], None, False), + ([(100, 503, 20, "old"), (109, 201, 2000, "new")], 11, False), + ([(100, 503, 20, "old"), (101, 201, 500, "new")], 1.5, True), + ([(101, 201, 500, "new")], 0, True), + ([], 0, False), + ], +) +def test_recovery_requires_completed_success(attempts: list, recovery: float, passed: bool) -> None: + result = result_of([(99, 201, 1, "old"), *attempts]) + assert result["probe"]["recovery_s"] == recovery + assert probe.verdict(ARGS, result)["write_recovery_ok"] is passed + + +@pytest.mark.parametrize( + ("field", "value", "check"), + [("requests", 0, "load_delivered"), ("dropped_iterations", 1, "load_delivered")], +) +def test_incomplete_load_fails(field: str, value: int, check: str) -> None: + result = result_of([(99, 201, 1, "old"), (102, 201, 1, "new")]) + result["k6"][field] = value + assert not probe.verdict(ARGS, result)[check] + + +def test_missing_lb_sample_fails() -> None: + result = result_of([(99, 201, 1, "old"), (102, 201, 1, "new")]) + result["lb_samples_missing"] = 1 + assert not probe.verdict(ARGS, result)["lb_observed"] + + +@pytest.mark.parametrize("corruption", ["none", "missing", "duplicate", "changed", "http", "cycle"]) +def test_full_history_matches_confirmed_bodies( + monkeypatch: pytest.MonkeyPatch, corruption: str +) -> None: + messages = [{"id": str(n), "text": "original"} for n in range(101)] + stored = list(reversed(messages)) + if corruption == "missing": + stored.pop() + if corruption == "duplicate": + stored.append(stored[0]) + if corruption == "changed": + stored[0] = dict(stored[0], text="changed") + pages = iter( + [ + (200, "", {"items": stored[:100], "next_cursor": "next"}), + ( + 503 if corruption == "http" else 200, + "", + {"items": stored[100:], "next_cursor": "next" if corruption == "cycle" else None}, + ), + ] + ) + monkeypatch.setattr(probe, "call", lambda *_a, **_kw: next(pages)) + result = probe.history_check({"auth": {}}, "chat", messages) + assert (result["exact_match"] and result["unique"]) is (corruption == "none")