Files
mtusi2026lab3-fork/scripts/lab3_evidence.py
T
2026-11-02 20:05:33 +03:00

416 lines
15 KiB
Python

#!/usr/bin/env python3
"""Явные эксперименты lab3: останавливает и восстанавливает API, заменяет контейнеры."""
import argparse
import http.client
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import platform
import subprocess
import sys
import threading
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check
from resilience import history
from replicas import (
COMPOSE,
SERVICES,
compose,
inventory,
probe,
replace,
undrain,
wait_balanced,
wait_ready,
)
def save_json(path, value):
path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n")
def record_command(out, name, command, env=None):
"""Записать реальный вывод команды; не запрашивать полный inspect или env."""
started = time.monotonic()
result = subprocess.run(command, cwd=ROOT, text=True, capture_output=True, env=env)
(out / (name + ".txt")).write_text(
"$ "
+ " ".join(command)
+ "\n"
+ result.stdout
+ result.stderr
+ f"\nexit_code={result.returncode}; elapsed_seconds={time.monotonic() - started:.3f}\n"
)
check(result.returncode == 0, name + " failed; inspect evidence file")
def observe(c, actor):
seen = set()
for _ in range(50):
try:
resp = c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
instance_id = resp["headers"].get("X-Instance-Id")
if instance_id:
seen.add(instance_id)
except Exception:
pass
if len(seen) >= 2:
break
time.sleep(0.1)
check(len(seen) >= 2, "one session must be accepted by at least two API processes")
return sorted(seen)
def concurrent_replay(base):
"""Одновременный один ключ через LB; один id, одна строка истории, обе реплики."""
c = Client(base)
alice, bob, eve, chat = c.bootstrap()
key = uuid.uuid4().hex
barrier = threading.Barrier(12)
def worker(_):
local = Client(base)
barrier.wait()
return local.message(alice, chat, "одновременный повтор", key=key)
with ThreadPoolExecutor(max_workers=12) as pool:
responses = list(pool.map(worker, range(12)))
ids = {r["json"]["id"] for r in responses}
instances = {r["headers"]["X-Instance-Id"] for r in responses}
check(len(ids) == 1, "concurrent replay created different messages")
check(len(instances) >= 2, "replay did not cross two API processes")
rows = history(c, bob, chat)
check(
len(rows) == 1 and rows[0] == responses[0]["json"],
"history must contain one exact response",
)
c.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/messages",
409,
actor=alice,
body={"text": "другое тело"},
headers={"Idempotency-Key": key},
)
replay = c.message(alice, chat, "одновременный повтор", key=key)
check(replay["json"] == responses[0]["json"], "409 changed original result")
return {
"parallel_requests": 12,
"unique_message_ids": len(ids),
"history_rows": len(rows),
"instances": sorted(instances),
"conflict_status": 409,
}
def experiment(base, service, mode, duration):
"""Closed-loop чтение/запись во время stop или поочерёдной замены."""
c = Client(base, timeout=2)
alice, bob, eve, chat = c.bootstrap()
initial = c.message(alice, chat, "подтверждено до действия")["json"]
seen = observe(c, alice)
before = inventory()
target = probe(service)["instance_id"]
events, confirmed = [], [initial]
lock = threading.Lock()
halt = threading.Event()
started = time.monotonic()
def worker(number):
local = Client(base, timeout=2)
key, text = uuid.uuid4().hex, ""
while not halt.is_set():
if not text:
text = "load-" + key
for operation in ("write", "read"):
t0 = time.monotonic()
try:
response = (
local.message(alice, chat, text, key=key)
if operation == "write"
else local.call(
"GET",
"/api/v1/chats/" + chat["id"] + "/messages?limit=1",
actor=bob,
schema="MessagePage",
)
)
event = {
"start": t0,
"end": time.monotonic(),
"kind": operation,
"ok": True,
"instance": response["headers"]["X-Instance-Id"],
}
with lock:
if operation == "write":
confirmed.append(response["json"])
if operation == "write":
key, text = uuid.uuid4().hex, ""
except (Failure, OSError, http.client.HTTPException):
event = {
"start": t0,
"end": time.monotonic(),
"kind": operation,
"ok": False,
}
with lock:
events.append(event)
time.sleep(0.02)
with ThreadPoolExecutor(max_workers=2) as pool:
futures = [pool.submit(worker, n) for n in range(2)]
time.sleep(1)
action_started = time.monotonic()
try:
if mode == "stop":
env = {**os.environ, "TARGET_INSTANCE_ID": target}
# Вызываем replicas.py stop-target напрямую через текущий интерпретатор Python
result = subprocess.run(
[sys.executable, str(ROOT / "scripts/replicas.py"), "stop-target"],
cwd=ROOT,
env=env,
text=True,
capture_output=True,
timeout=10,
)
check(result.returncode == 0, "stop hook failed")
action_detail = result.stdout
else:
action_detail = replace(service)
action_finished = time.monotonic()
after_action = inventory()
if mode == "stop":
stopped = next(row for row in after_action if row["service"] == service)
check(
stopped["state"] == "exited" and stopped["pid"] == 0,
"target process is still running",
)
deadline = max(action_started + duration, action_finished + 5)
while time.monotonic() < deadline:
time.sleep(0.1)
finally:
halt.set()
for future in futures:
future.result()
finished = time.monotonic()
rows = history(c, bob, chat)
by_id = {row["id"]: row for row in rows}
check(
all(by_id.get(message["id"]) == message for message in confirmed),
"confirmed data was lost",
)
texts = [row["text"] for row in rows]
check(len(texts) == len(set(texts)), "uncertain retry created duplicate text")
tail = [e for e in events if e["start"] >= action_finished]
check(any(e["ok"] and e["kind"] == "write" for e in tail), "no writes after action")
check(any(e["ok"] and e["kind"] == "read" for e in tail), "no reads after action")
if mode == "stop":
check(
all(e.get("instance") != target for e in tail if e["ok"]),
"stopped instance still responds",
)
for kind in ("write", "read"):
recovered = min(e["end"] for e in tail if e["ok"] and e["kind"] == kind)
check(recovered - action_started <= 10, kind + " recovery exceeded 10s")
def metrics(items, elapsed):
good = [e for e in items if e["ok"]]
failed = len(items) - len(good)
latencies = sorted((e["end"] - e["start"]) * 1000 for e in good)
def percentile(fraction):
return (
round(latencies[max(0, int(len(latencies) * fraction + 0.999) - 1)], 3)
if latencies
else None
)
return {
"elapsed_seconds": round(elapsed, 3),
"attempts": len(items),
"successes": len(good),
"errors": failed,
"error_ratio": round(failed / len(items), 6) if items else None,
"successful_rps": round(len(good) / elapsed, 3),
"successful_latency_ms": {
"p50": percentile(0.5),
"p95": percentile(0.95),
"p99": percentile(0.99),
},
}
result = {
"mode": mode,
"service": service,
"target_instance": target,
"observed_with_one_session": seen,
"before": before,
"after_action": after_action,
"action": action_detail,
"action_seconds": round(action_finished - action_started, 3),
"concurrency": 2,
"total": metrics(events, finished - started),
"during_and_after_action": metrics(
[e for e in events if e["start"] >= action_started],
finished - action_started,
),
"confirmed_responses": len(confirmed),
"history_messages": len(rows),
"duplicates": 0,
"existing_session_works": True,
}
for kind in ("write", "read"):
after = [e for e in tail if e["ok"] and e["kind"] == kind]
result["first_" + kind + "_after_action_seconds"] = round(
min(e["end"] for e in after) - action_started, 3
)
result[kind] = metrics(
[e for e in events if e["kind"] == kind], finished - started
)
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--output", default="evidence/lab3")
parser.add_argument("--duration", type=float, default=20)
args = parser.parse_args()
check(15 <= args.duration <= 120, "duration must be 15..120 seconds")
out = ROOT / args.output
out.mkdir(parents=True, exist_ok=True)
save_json(out / "result.json", {"status": "RUNNING"})
try:
os.environ["BASE_URL"] = args.base_url
save_json(
out / "00_environment.json",
{
"at_utc": datetime.now(timezone.utc).isoformat(),
"platform": platform.platform(),
"cpu_count": os.cpu_count(),
"python": platform.python_version(),
"duration": args.duration,
"workload": "2 closed-loop clients, POST+GET; uncertain POST retried with same key/body",
},
)
record_command(out, "01_docker", ["docker", "version"])
record_command(out, "02_compose_config", COMPOSE + ["config", "--quiet"])
wait_balanced()
save_json(out / "03_processes.json", inventory())
record_command(out, "03_limits", COMPOSE + ["ps", "-q"])
ids = compose("ps", "-q", capture=True).splitlines()
record_command(
out,
"03_container_limits",
[
"docker",
"inspect",
"--format",
"{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}} ports={{json .HostConfig.PortBindings}}",
*ids,
],
)
record_command(
out,
"04_smoke",
[sys.executable, "tests/smoke.py", "--base-url", args.base_url],
)
save_json(
out / "05_concurrent_idempotency.json", concurrent_replay(args.base_url)
)
try:
for service in SERVICES:
wait_balanced()
report = experiment(args.base_url, service, "stop", args.duration)
save_json(out / ("06_failover_" + service + ".json"), report)
compose("up", "-d", "--no-deps", service)
wait_balanced()
record_command(
out,
"07_public_failover",
[
sys.executable,
"tests/resilience.py",
"--mode",
"failover",
"--action-script",
"scripts/stop-one.sh",
"--base-url",
args.base_url,
],
)
for service in SERVICES:
compose("up", "-d", "--no-deps", service)
wait_balanced()
for service in SERVICES:
save_json(
out / ("08_rolling_" + service + ".json"),
experiment(args.base_url, service, "replace", args.duration),
)
record_command(
out,
"09_persistence",
[
sys.executable,
"tests/resilience.py",
"--mode",
"persistence",
"--action-script",
"scripts/restart.sh",
"--base-url",
args.base_url,
],
)
wait_balanced()
save_json(out / "10_processes_after.json", inventory())
record_command(
out,
"11_nginx",
COMPOSE
+ ["exec", "-T", "nginx", "nginx", "-T", "-c", "/opt/lb/nginx.conf"],
)
record_command(
out,
"12_student_tests",
COMPOSE + ["--profile", "test", "run", "--rm", "--build", "tests"],
)
save_json(
out / "result.json",
{"status": "PASS", "at_utc": datetime.now(timezone.utc).isoformat()},
)
except Exception:
save_json(
out / "result.json",
{
"status": "FAIL",
"note": "Inspect completed evidence files; failed runs are not evidence of passing.",
},
)
raise
finally:
for service in SERVICES:
compose("up", "-d", "--no-deps", service)
undrain(service)
print("PASS: evidence saved to " + str(out.relative_to(ROOT)))
except BaseException:
save_json(
out / "result.json",
{
"status": "FAIL",
"note": "Run did not finish successfully; inspect completed outputs.",
},
)
raise
if __name__ == "__main__":
main()