Measure successful recovery and verify history under load

This commit is contained in:
Artyom
2026-11-05 22:15:18 +03:00
parent 8a7e027925
commit 252e3e3ffe
3 changed files with 144 additions and 9 deletions
@@ -4,8 +4,8 @@
## 2. Нагрузочный зонд
- [ ] 2.1 Исправить измерение восстановления, добавить сверку полной истории и отбраковку неполной нагрузки/статистики.
- [ ] 2.2 Проверить негативные случаи собственными тестами; `make check`, `make lint`, `make student-test`.
- [x] 2.1 Исправить измерение восстановления, добавить сверку полной истории и отбраковку неполной нагрузки/статистики.
- [x] 2.2 Проверить негативные случаи собственными тестами; `make check`, `make lint`, `make student-test`.
## 3. Стенд уровня «4»
+59 -7
View File
@@ -10,6 +10,7 @@ import sys
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from datetime import datetime
@@ -306,15 +307,16 @@ def retry(args):
def record(stop, alice, chat):
probes, trace = [], []
probes, trace, confirmed = [], [], []
def prober():
key = uuid.uuid4().hex
while not stop.is_set():
started = time.time()
status, instance, _ = send(alice, chat, "probe " + key, key)
status, instance, body = send(alice, chat, "probe " + key, key)
probes.append((started, status, round(1000 * (time.time() - started), 1), instance))
if status == 201:
confirmed.append(body)
key = uuid.uuid4().hex
time.sleep(0.1)
@@ -326,7 +328,38 @@ def record(stop, alice, chat):
threads = [threading.Thread(target=t, daemon=True) for t in (prober, poller)]
for thread in threads:
thread.start()
return probes, trace, threads
return probes, trace, threads, confirmed
def history_check(reader, chat, confirmed):
items, cursors = [], set()
cursor = None
for _ in range(len(confirmed) // 100 + 2):
path = f"/api/v1/chats/{chat}/messages?limit=100"
if cursor:
path += "&cursor=" + urllib.parse.quote(cursor, safe="")
status, _, page = call("GET", path, headers=reader["auth"])
if status != 200 or not isinstance(page, dict):
break
items.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
same = sorted(items, key=lambda m: m["id"]) == sorted(confirmed, key=lambda m: m["id"])
return {
"confirmed": len(confirmed),
"stored": len(items),
"exact_match": bool(confirmed) and same,
"unique": len({m["id"] for m in items}) == len(items),
}
if cursor in cursors:
break
cursors.add(cursor)
return {
"confirmed": len(confirmed),
"stored": len(items),
"exact_match": False,
"unique": False,
}
def write_csv(path, header, rows):
@@ -339,6 +372,10 @@ def summarize(args, probes, trace, action, expected):
t0, end = action["start"], action["end"]
label = args.label or os.environ.get("TARGET_INSTANCE_ID", "action")
failures = [t - t0 for t, s, _, _ in probes if s != 201 and t >= t0]
successes = [(t, t + ms / 1000 - t0) for t, s, ms, _ in probes if s == 201 and t >= t0]
recovery = (
next((end for t, end in successes if t - t0 > failures[-1]), None) if failures else 0.0
)
down = [t for t, up in trace if t >= t0 and up is not None and up < expected]
back = [t for t, up in trace if down and t > down[0] and up is not None and up >= expected]
outage = [t for t in down if not back or t < back[0]] or [end]
@@ -354,13 +391,15 @@ def summarize(args, probes, trace, action, expected):
"action_finished_before_load_end": end < action["load_end"],
"lb_up_before": expected,
"lb_up_min": min((up for _, up in trace if up is not None), default=None),
"lb_samples_missing": sum(up is None for _, up in trace),
"detection_seconds": round(down[0] - t0, 3) if down else None,
"rejoin_seconds": round(back[0] - t0, 3) if back else None,
"probe": {
"attempts": len(probes),
"failed_attempts": sum(1 for _, s, _, _ in probes if s != 201),
"first_failure_s": round(failures[0], 3) if failures else None,
"recovery_s": round(failures[-1], 3) if failures else 0.0,
"first_success_s": round(successes[0][1], 3) if successes else None,
"recovery_s": round(recovery, 3) if recovery is not None else None,
"instances_before": dict(before),
"instances_during_outage": dict(during),
"instances_after": dict(after),
@@ -373,7 +412,13 @@ def verdict(args, result):
checks = {
"action_ok": result["action_exit"] == 0 and result["action_finished_before_load_end"],
"load_errors_ok": k6["exit"] == 0 and k6["failed_ratio"] <= args.max_failed_ratio,
"write_recovery_ok": probe["recovery_s"] <= args.max_recovery,
"load_delivered": k6["requests"] > 0 and k6["dropped_iterations"] == 0,
"lb_observed": result["lb_up_before"] >= 2 and result["lb_samples_missing"] == 0,
"write_recovery_ok": all(
probe[k] is not None and probe[k] <= args.max_recovery
for k in ("first_success_s", "recovery_s")
),
"history_preserved": result["history"]["exact_match"] and result["history"]["unique"],
}
if args.expect == "dip":
checks["detected"] = result["detection_seconds"] is not None
@@ -394,7 +439,8 @@ def load(args):
expected = lb_up() or 0
alice, bob = actor("alice"), actor("bob")
stop, action = threading.Event(), {}
probes, trace, threads = record(stop, alice, chat_of(alice, bob))
chat = chat_of(alice, bob)
probes, trace, threads, confirmed = record(stop, alice, chat)
def act():
time.sleep(args.warmup + args.lead)
@@ -425,6 +471,7 @@ def load(args):
)
write_csv(out / "lb_up.csv", "t_rel_s,up", ((round(t - t0, 3), up) for t, up in trace))
result = summarize(args, probes, trace, action, expected)
result["history"] = history_check(bob, chat, confirmed)
result["lb_counters"] = {k: v - counters[k] for k, v in lb_counters().items()}
result["k6"] = dict(
k6_metrics(out / "k6/summary.json", args.duration), exit=code, rate=args.rate
@@ -442,10 +489,13 @@ def detect(args):
started = time.time()
process = subprocess.Popen(args.command, cwd=str(ROOT))
detected = acted = None
missing = 0
while time.time() - started < args.timeout and (detected is None or acted is None):
if acted is None and process.poll() is not None:
acted = time.time() - started
if detected is None and (lb_up() or 0) <= until:
up = lb_up()
missing += up is None
if detected is None and up is not None and up <= until:
detected = time.time() - started
time.sleep(0.05)
code = process.wait()
@@ -460,6 +510,7 @@ def detect(args):
restored = round(time.time() - begun, 3) if (lb_up() or 0) >= before else None
checks = {
"action_ok": code == 0,
"lb_observed": before >= 2 and missing == 0,
"detected_in_time": detected is not None and detected <= args.max + args.slack,
"lb_answer_ok": status == args.lb_status,
}
@@ -472,6 +523,7 @@ def detect(args):
"action_exit": code,
"action_seconds": round(acted, 3),
"lb_up_before": before,
"lb_samples_missing": missing,
"lb_up_counted_as_detected": until,
"detection_seconds": None if detected is None else round(detected, 3),
"declared_max_seconds": args.max,
+83
View File
@@ -0,0 +1,83 @@
import importlib.util
from pathlib import Path
from types import SimpleNamespace
import pytest
SPEC = importlib.util.spec_from_file_location(
"grade4_probe", Path(__file__).resolve().parents[2] / "scripts/probe.py"
)
assert SPEC is not None and SPEC.loader is not None
probe = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(probe)
ARGS = SimpleNamespace(
label="old", command=[], expect="replace", max_failed_ratio=0.01, max_recovery=10
)
def result_of(probes: list[tuple]) -> dict:
result = probe.summarize(
ARGS, probes, [(99, 2), (101, 2)], {"start": 100, "end": 101, "exit": 0, "load_end": 120}, 2
)
result["k6"] = {"exit": 0, "failed_ratio": 0, "requests": 100, "dropped_iterations": 0}
result["history"] = {"exact_match": True, "unique": True}
return result
@pytest.mark.parametrize(
("attempts", "recovery", "passed"),
[
([(100, 503, 20, "old")], None, False),
([(100, 503, 20, "old"), (109, 201, 2000, "new")], 11, False),
([(100, 503, 20, "old"), (101, 201, 500, "new")], 1.5, True),
([(101, 201, 500, "new")], 0, True),
([], 0, False),
],
)
def test_recovery_requires_completed_success(attempts: list, recovery: float, passed: bool) -> None:
result = result_of([(99, 201, 1, "old"), *attempts])
assert result["probe"]["recovery_s"] == recovery
assert probe.verdict(ARGS, result)["write_recovery_ok"] is passed
@pytest.mark.parametrize(
("field", "value", "check"),
[("requests", 0, "load_delivered"), ("dropped_iterations", 1, "load_delivered")],
)
def test_incomplete_load_fails(field: str, value: int, check: str) -> None:
result = result_of([(99, 201, 1, "old"), (102, 201, 1, "new")])
result["k6"][field] = value
assert not probe.verdict(ARGS, result)[check]
def test_missing_lb_sample_fails() -> None:
result = result_of([(99, 201, 1, "old"), (102, 201, 1, "new")])
result["lb_samples_missing"] = 1
assert not probe.verdict(ARGS, result)["lb_observed"]
@pytest.mark.parametrize("corruption", ["none", "missing", "duplicate", "changed", "http", "cycle"])
def test_full_history_matches_confirmed_bodies(
monkeypatch: pytest.MonkeyPatch, corruption: str
) -> None:
messages = [{"id": str(n), "text": "original"} for n in range(101)]
stored = list(reversed(messages))
if corruption == "missing":
stored.pop()
if corruption == "duplicate":
stored.append(stored[0])
if corruption == "changed":
stored[0] = dict(stored[0], text="changed")
pages = iter(
[
(200, "", {"items": stored[:100], "next_cursor": "next"}),
(
503 if corruption == "http" else 200,
"",
{"items": stored[100:], "next_cursor": "next" if corruption == "cycle" else None},
),
]
)
monkeypatch.setattr(probe, "call", lambda *_a, **_kw: next(pages))
result = probe.history_check({"auth": {}}, "chat", messages)
assert (result["exact_match"] and result["unique"]) is (corruption == "none")