forked from mtusi2026/mtusi2026lab3
Measure successful recovery and verify history under load
This commit is contained in:
+59
-7
@@ -10,6 +10,7 @@ import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
@@ -306,15 +307,16 @@ def retry(args):
|
||||
|
||||
|
||||
def record(stop, alice, chat):
|
||||
probes, trace = [], []
|
||||
probes, trace, confirmed = [], [], []
|
||||
|
||||
def prober():
|
||||
key = uuid.uuid4().hex
|
||||
while not stop.is_set():
|
||||
started = time.time()
|
||||
status, instance, _ = send(alice, chat, "probe " + key, key)
|
||||
status, instance, body = send(alice, chat, "probe " + key, key)
|
||||
probes.append((started, status, round(1000 * (time.time() - started), 1), instance))
|
||||
if status == 201:
|
||||
confirmed.append(body)
|
||||
key = uuid.uuid4().hex
|
||||
time.sleep(0.1)
|
||||
|
||||
@@ -326,7 +328,38 @@ def record(stop, alice, chat):
|
||||
threads = [threading.Thread(target=t, daemon=True) for t in (prober, poller)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
return probes, trace, threads
|
||||
return probes, trace, threads, confirmed
|
||||
|
||||
|
||||
def history_check(reader, chat, confirmed):
|
||||
items, cursors = [], set()
|
||||
cursor = None
|
||||
for _ in range(len(confirmed) // 100 + 2):
|
||||
path = f"/api/v1/chats/{chat}/messages?limit=100"
|
||||
if cursor:
|
||||
path += "&cursor=" + urllib.parse.quote(cursor, safe="")
|
||||
status, _, page = call("GET", path, headers=reader["auth"])
|
||||
if status != 200 or not isinstance(page, dict):
|
||||
break
|
||||
items.extend(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
same = sorted(items, key=lambda m: m["id"]) == sorted(confirmed, key=lambda m: m["id"])
|
||||
return {
|
||||
"confirmed": len(confirmed),
|
||||
"stored": len(items),
|
||||
"exact_match": bool(confirmed) and same,
|
||||
"unique": len({m["id"] for m in items}) == len(items),
|
||||
}
|
||||
if cursor in cursors:
|
||||
break
|
||||
cursors.add(cursor)
|
||||
return {
|
||||
"confirmed": len(confirmed),
|
||||
"stored": len(items),
|
||||
"exact_match": False,
|
||||
"unique": False,
|
||||
}
|
||||
|
||||
|
||||
def write_csv(path, header, rows):
|
||||
@@ -339,6 +372,10 @@ def summarize(args, probes, trace, action, expected):
|
||||
t0, end = action["start"], action["end"]
|
||||
label = args.label or os.environ.get("TARGET_INSTANCE_ID", "action")
|
||||
failures = [t - t0 for t, s, _, _ in probes if s != 201 and t >= t0]
|
||||
successes = [(t, t + ms / 1000 - t0) for t, s, ms, _ in probes if s == 201 and t >= t0]
|
||||
recovery = (
|
||||
next((end for t, end in successes if t - t0 > failures[-1]), None) if failures else 0.0
|
||||
)
|
||||
down = [t for t, up in trace if t >= t0 and up is not None and up < expected]
|
||||
back = [t for t, up in trace if down and t > down[0] and up is not None and up >= expected]
|
||||
outage = [t for t in down if not back or t < back[0]] or [end]
|
||||
@@ -354,13 +391,15 @@ def summarize(args, probes, trace, action, expected):
|
||||
"action_finished_before_load_end": end < action["load_end"],
|
||||
"lb_up_before": expected,
|
||||
"lb_up_min": min((up for _, up in trace if up is not None), default=None),
|
||||
"lb_samples_missing": sum(up is None for _, up in trace),
|
||||
"detection_seconds": round(down[0] - t0, 3) if down else None,
|
||||
"rejoin_seconds": round(back[0] - t0, 3) if back else None,
|
||||
"probe": {
|
||||
"attempts": len(probes),
|
||||
"failed_attempts": sum(1 for _, s, _, _ in probes if s != 201),
|
||||
"first_failure_s": round(failures[0], 3) if failures else None,
|
||||
"recovery_s": round(failures[-1], 3) if failures else 0.0,
|
||||
"first_success_s": round(successes[0][1], 3) if successes else None,
|
||||
"recovery_s": round(recovery, 3) if recovery is not None else None,
|
||||
"instances_before": dict(before),
|
||||
"instances_during_outage": dict(during),
|
||||
"instances_after": dict(after),
|
||||
@@ -373,7 +412,13 @@ def verdict(args, result):
|
||||
checks = {
|
||||
"action_ok": result["action_exit"] == 0 and result["action_finished_before_load_end"],
|
||||
"load_errors_ok": k6["exit"] == 0 and k6["failed_ratio"] <= args.max_failed_ratio,
|
||||
"write_recovery_ok": probe["recovery_s"] <= args.max_recovery,
|
||||
"load_delivered": k6["requests"] > 0 and k6["dropped_iterations"] == 0,
|
||||
"lb_observed": result["lb_up_before"] >= 2 and result["lb_samples_missing"] == 0,
|
||||
"write_recovery_ok": all(
|
||||
probe[k] is not None and probe[k] <= args.max_recovery
|
||||
for k in ("first_success_s", "recovery_s")
|
||||
),
|
||||
"history_preserved": result["history"]["exact_match"] and result["history"]["unique"],
|
||||
}
|
||||
if args.expect == "dip":
|
||||
checks["detected"] = result["detection_seconds"] is not None
|
||||
@@ -394,7 +439,8 @@ def load(args):
|
||||
expected = lb_up() or 0
|
||||
alice, bob = actor("alice"), actor("bob")
|
||||
stop, action = threading.Event(), {}
|
||||
probes, trace, threads = record(stop, alice, chat_of(alice, bob))
|
||||
chat = chat_of(alice, bob)
|
||||
probes, trace, threads, confirmed = record(stop, alice, chat)
|
||||
|
||||
def act():
|
||||
time.sleep(args.warmup + args.lead)
|
||||
@@ -425,6 +471,7 @@ def load(args):
|
||||
)
|
||||
write_csv(out / "lb_up.csv", "t_rel_s,up", ((round(t - t0, 3), up) for t, up in trace))
|
||||
result = summarize(args, probes, trace, action, expected)
|
||||
result["history"] = history_check(bob, chat, confirmed)
|
||||
result["lb_counters"] = {k: v - counters[k] for k, v in lb_counters().items()}
|
||||
result["k6"] = dict(
|
||||
k6_metrics(out / "k6/summary.json", args.duration), exit=code, rate=args.rate
|
||||
@@ -442,10 +489,13 @@ def detect(args):
|
||||
started = time.time()
|
||||
process = subprocess.Popen(args.command, cwd=str(ROOT))
|
||||
detected = acted = None
|
||||
missing = 0
|
||||
while time.time() - started < args.timeout and (detected is None or acted is None):
|
||||
if acted is None and process.poll() is not None:
|
||||
acted = time.time() - started
|
||||
if detected is None and (lb_up() or 0) <= until:
|
||||
up = lb_up()
|
||||
missing += up is None
|
||||
if detected is None and up is not None and up <= until:
|
||||
detected = time.time() - started
|
||||
time.sleep(0.05)
|
||||
code = process.wait()
|
||||
@@ -460,6 +510,7 @@ def detect(args):
|
||||
restored = round(time.time() - begun, 3) if (lb_up() or 0) >= before else None
|
||||
checks = {
|
||||
"action_ok": code == 0,
|
||||
"lb_observed": before >= 2 and missing == 0,
|
||||
"detected_in_time": detected is not None and detected <= args.max + args.slack,
|
||||
"lb_answer_ok": status == args.lb_status,
|
||||
}
|
||||
@@ -472,6 +523,7 @@ def detect(args):
|
||||
"action_exit": code,
|
||||
"action_seconds": round(acted, 3),
|
||||
"lb_up_before": before,
|
||||
"lb_samples_missing": missing,
|
||||
"lb_up_counted_as_detected": until,
|
||||
"detection_seconds": None if detected is None else round(detected, 3),
|
||||
"declared_max_seconds": args.max,
|
||||
|
||||
Reference in New Issue
Block a user