forked from mtusi2026/mtusi2026lab3
416 lines
15 KiB
Python
416 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""Явные эксперименты lab3: останавливает и восстанавливает API, заменяет контейнеры."""
|
|
|
|
import argparse
|
|
import http.client
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timezone
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure, check
|
|
from resilience import history
|
|
from replicas import (
|
|
COMPOSE,
|
|
SERVICES,
|
|
compose,
|
|
inventory,
|
|
probe,
|
|
replace,
|
|
undrain,
|
|
wait_balanced,
|
|
wait_ready,
|
|
)
|
|
|
|
|
|
def save_json(path, value):
|
|
path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n")
|
|
|
|
|
|
def record_command(out, name, command, env=None):
|
|
"""Записать реальный вывод команды; не запрашивать полный inspect или env."""
|
|
started = time.monotonic()
|
|
result = subprocess.run(command, cwd=ROOT, text=True, capture_output=True, env=env)
|
|
(out / (name + ".txt")).write_text(
|
|
"$ "
|
|
+ " ".join(command)
|
|
+ "\n"
|
|
+ result.stdout
|
|
+ result.stderr
|
|
+ f"\nexit_code={result.returncode}; elapsed_seconds={time.monotonic() - started:.3f}\n"
|
|
)
|
|
check(result.returncode == 0, name + " failed; inspect evidence file")
|
|
|
|
|
|
def observe(c, actor):
|
|
seen = set()
|
|
for _ in range(50):
|
|
try:
|
|
resp = c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
|
instance_id = resp["headers"].get("X-Instance-Id")
|
|
if instance_id:
|
|
seen.add(instance_id)
|
|
except Exception:
|
|
pass
|
|
if len(seen) >= 2:
|
|
break
|
|
time.sleep(0.1)
|
|
check(len(seen) >= 2, "one session must be accepted by at least two API processes")
|
|
return sorted(seen)
|
|
|
|
|
|
def concurrent_replay(base):
|
|
"""Одновременный один ключ через LB; один id, одна строка истории, обе реплики."""
|
|
c = Client(base)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
key = uuid.uuid4().hex
|
|
barrier = threading.Barrier(12)
|
|
|
|
def worker(_):
|
|
local = Client(base)
|
|
barrier.wait()
|
|
return local.message(alice, chat, "одновременный повтор", key=key)
|
|
|
|
with ThreadPoolExecutor(max_workers=12) as pool:
|
|
responses = list(pool.map(worker, range(12)))
|
|
ids = {r["json"]["id"] for r in responses}
|
|
instances = {r["headers"]["X-Instance-Id"] for r in responses}
|
|
check(len(ids) == 1, "concurrent replay created different messages")
|
|
check(len(instances) >= 2, "replay did not cross two API processes")
|
|
rows = history(c, bob, chat)
|
|
check(
|
|
len(rows) == 1 and rows[0] == responses[0]["json"],
|
|
"history must contain one exact response",
|
|
)
|
|
c.call(
|
|
"POST",
|
|
"/api/v1/chats/" + chat["id"] + "/messages",
|
|
409,
|
|
actor=alice,
|
|
body={"text": "другое тело"},
|
|
headers={"Idempotency-Key": key},
|
|
)
|
|
replay = c.message(alice, chat, "одновременный повтор", key=key)
|
|
check(replay["json"] == responses[0]["json"], "409 changed original result")
|
|
return {
|
|
"parallel_requests": 12,
|
|
"unique_message_ids": len(ids),
|
|
"history_rows": len(rows),
|
|
"instances": sorted(instances),
|
|
"conflict_status": 409,
|
|
}
|
|
|
|
|
|
def experiment(base, service, mode, duration):
|
|
"""Closed-loop чтение/запись во время stop или поочерёдной замены."""
|
|
c = Client(base, timeout=2)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
initial = c.message(alice, chat, "подтверждено до действия")["json"]
|
|
seen = observe(c, alice)
|
|
before = inventory()
|
|
target = probe(service)["instance_id"]
|
|
events, confirmed = [], [initial]
|
|
lock = threading.Lock()
|
|
halt = threading.Event()
|
|
started = time.monotonic()
|
|
|
|
def worker(number):
|
|
local = Client(base, timeout=2)
|
|
key, text = uuid.uuid4().hex, ""
|
|
while not halt.is_set():
|
|
if not text:
|
|
text = "load-" + key
|
|
for operation in ("write", "read"):
|
|
t0 = time.monotonic()
|
|
try:
|
|
response = (
|
|
local.message(alice, chat, text, key=key)
|
|
if operation == "write"
|
|
else local.call(
|
|
"GET",
|
|
"/api/v1/chats/" + chat["id"] + "/messages?limit=1",
|
|
actor=bob,
|
|
schema="MessagePage",
|
|
)
|
|
)
|
|
event = {
|
|
"start": t0,
|
|
"end": time.monotonic(),
|
|
"kind": operation,
|
|
"ok": True,
|
|
"instance": response["headers"]["X-Instance-Id"],
|
|
}
|
|
with lock:
|
|
if operation == "write":
|
|
confirmed.append(response["json"])
|
|
if operation == "write":
|
|
key, text = uuid.uuid4().hex, ""
|
|
except (Failure, OSError, http.client.HTTPException):
|
|
event = {
|
|
"start": t0,
|
|
"end": time.monotonic(),
|
|
"kind": operation,
|
|
"ok": False,
|
|
}
|
|
with lock:
|
|
events.append(event)
|
|
time.sleep(0.02)
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
futures = [pool.submit(worker, n) for n in range(2)]
|
|
time.sleep(1)
|
|
action_started = time.monotonic()
|
|
try:
|
|
if mode == "stop":
|
|
env = {**os.environ, "TARGET_INSTANCE_ID": target}
|
|
# Вызываем replicas.py stop-target напрямую через текущий интерпретатор Python
|
|
result = subprocess.run(
|
|
[sys.executable, str(ROOT / "scripts/replicas.py"), "stop-target"],
|
|
cwd=ROOT,
|
|
env=env,
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=10,
|
|
)
|
|
check(result.returncode == 0, "stop hook failed")
|
|
action_detail = result.stdout
|
|
else:
|
|
action_detail = replace(service)
|
|
action_finished = time.monotonic()
|
|
after_action = inventory()
|
|
if mode == "stop":
|
|
stopped = next(row for row in after_action if row["service"] == service)
|
|
check(
|
|
stopped["state"] == "exited" and stopped["pid"] == 0,
|
|
"target process is still running",
|
|
)
|
|
deadline = max(action_started + duration, action_finished + 5)
|
|
while time.monotonic() < deadline:
|
|
time.sleep(0.1)
|
|
finally:
|
|
halt.set()
|
|
for future in futures:
|
|
future.result()
|
|
finished = time.monotonic()
|
|
rows = history(c, bob, chat)
|
|
by_id = {row["id"]: row for row in rows}
|
|
check(
|
|
all(by_id.get(message["id"]) == message for message in confirmed),
|
|
"confirmed data was lost",
|
|
)
|
|
texts = [row["text"] for row in rows]
|
|
check(len(texts) == len(set(texts)), "uncertain retry created duplicate text")
|
|
tail = [e for e in events if e["start"] >= action_finished]
|
|
check(any(e["ok"] and e["kind"] == "write" for e in tail), "no writes after action")
|
|
check(any(e["ok"] and e["kind"] == "read" for e in tail), "no reads after action")
|
|
if mode == "stop":
|
|
check(
|
|
all(e.get("instance") != target for e in tail if e["ok"]),
|
|
"stopped instance still responds",
|
|
)
|
|
for kind in ("write", "read"):
|
|
recovered = min(e["end"] for e in tail if e["ok"] and e["kind"] == kind)
|
|
check(recovered - action_started <= 10, kind + " recovery exceeded 10s")
|
|
|
|
def metrics(items, elapsed):
|
|
good = [e for e in items if e["ok"]]
|
|
failed = len(items) - len(good)
|
|
latencies = sorted((e["end"] - e["start"]) * 1000 for e in good)
|
|
|
|
def percentile(fraction):
|
|
return (
|
|
round(latencies[max(0, int(len(latencies) * fraction + 0.999) - 1)], 3)
|
|
if latencies
|
|
else None
|
|
)
|
|
|
|
return {
|
|
"elapsed_seconds": round(elapsed, 3),
|
|
"attempts": len(items),
|
|
"successes": len(good),
|
|
"errors": failed,
|
|
"error_ratio": round(failed / len(items), 6) if items else None,
|
|
"successful_rps": round(len(good) / elapsed, 3),
|
|
"successful_latency_ms": {
|
|
"p50": percentile(0.5),
|
|
"p95": percentile(0.95),
|
|
"p99": percentile(0.99),
|
|
},
|
|
}
|
|
|
|
result = {
|
|
"mode": mode,
|
|
"service": service,
|
|
"target_instance": target,
|
|
"observed_with_one_session": seen,
|
|
"before": before,
|
|
"after_action": after_action,
|
|
"action": action_detail,
|
|
"action_seconds": round(action_finished - action_started, 3),
|
|
"concurrency": 2,
|
|
"total": metrics(events, finished - started),
|
|
"during_and_after_action": metrics(
|
|
[e for e in events if e["start"] >= action_started],
|
|
finished - action_started,
|
|
),
|
|
"confirmed_responses": len(confirmed),
|
|
"history_messages": len(rows),
|
|
"duplicates": 0,
|
|
"existing_session_works": True,
|
|
}
|
|
for kind in ("write", "read"):
|
|
after = [e for e in tail if e["ok"] and e["kind"] == kind]
|
|
result["first_" + kind + "_after_action_seconds"] = round(
|
|
min(e["end"] for e in after) - action_started, 3
|
|
)
|
|
result[kind] = metrics(
|
|
[e for e in events if e["kind"] == kind], finished - started
|
|
)
|
|
return result
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--base-url", default="http://localhost:8080")
|
|
parser.add_argument("--output", default="evidence/lab3")
|
|
parser.add_argument("--duration", type=float, default=20)
|
|
args = parser.parse_args()
|
|
check(15 <= args.duration <= 120, "duration must be 15..120 seconds")
|
|
out = ROOT / args.output
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
save_json(out / "result.json", {"status": "RUNNING"})
|
|
try:
|
|
os.environ["BASE_URL"] = args.base_url
|
|
save_json(
|
|
out / "00_environment.json",
|
|
{
|
|
"at_utc": datetime.now(timezone.utc).isoformat(),
|
|
"platform": platform.platform(),
|
|
"cpu_count": os.cpu_count(),
|
|
"python": platform.python_version(),
|
|
"duration": args.duration,
|
|
"workload": "2 closed-loop clients, POST+GET; uncertain POST retried with same key/body",
|
|
},
|
|
)
|
|
record_command(out, "01_docker", ["docker", "version"])
|
|
record_command(out, "02_compose_config", COMPOSE + ["config", "--quiet"])
|
|
wait_balanced()
|
|
save_json(out / "03_processes.json", inventory())
|
|
record_command(out, "03_limits", COMPOSE + ["ps", "-q"])
|
|
ids = compose("ps", "-q", capture=True).splitlines()
|
|
record_command(
|
|
out,
|
|
"03_container_limits",
|
|
[
|
|
"docker",
|
|
"inspect",
|
|
"--format",
|
|
"{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}} ports={{json .HostConfig.PortBindings}}",
|
|
*ids,
|
|
],
|
|
)
|
|
record_command(
|
|
out,
|
|
"04_smoke",
|
|
[sys.executable, "tests/smoke.py", "--base-url", args.base_url],
|
|
)
|
|
save_json(
|
|
out / "05_concurrent_idempotency.json", concurrent_replay(args.base_url)
|
|
)
|
|
try:
|
|
for service in SERVICES:
|
|
wait_balanced()
|
|
report = experiment(args.base_url, service, "stop", args.duration)
|
|
save_json(out / ("06_failover_" + service + ".json"), report)
|
|
compose("up", "-d", "--no-deps", service)
|
|
wait_balanced()
|
|
record_command(
|
|
out,
|
|
"07_public_failover",
|
|
[
|
|
sys.executable,
|
|
"tests/resilience.py",
|
|
"--mode",
|
|
"failover",
|
|
"--action-script",
|
|
"scripts/stop-one.sh",
|
|
"--base-url",
|
|
args.base_url,
|
|
],
|
|
)
|
|
for service in SERVICES:
|
|
compose("up", "-d", "--no-deps", service)
|
|
wait_balanced()
|
|
for service in SERVICES:
|
|
save_json(
|
|
out / ("08_rolling_" + service + ".json"),
|
|
experiment(args.base_url, service, "replace", args.duration),
|
|
)
|
|
record_command(
|
|
out,
|
|
"09_persistence",
|
|
[
|
|
sys.executable,
|
|
"tests/resilience.py",
|
|
"--mode",
|
|
"persistence",
|
|
"--action-script",
|
|
"scripts/restart.sh",
|
|
"--base-url",
|
|
args.base_url,
|
|
],
|
|
)
|
|
wait_balanced()
|
|
save_json(out / "10_processes_after.json", inventory())
|
|
record_command(
|
|
out,
|
|
"11_nginx",
|
|
COMPOSE
|
|
+ ["exec", "-T", "nginx", "nginx", "-T", "-c", "/opt/lb/nginx.conf"],
|
|
)
|
|
record_command(
|
|
out,
|
|
"12_student_tests",
|
|
COMPOSE + ["--profile", "test", "run", "--rm", "--build", "tests"],
|
|
)
|
|
save_json(
|
|
out / "result.json",
|
|
{"status": "PASS", "at_utc": datetime.now(timezone.utc).isoformat()},
|
|
)
|
|
except Exception:
|
|
save_json(
|
|
out / "result.json",
|
|
{
|
|
"status": "FAIL",
|
|
"note": "Inspect completed evidence files; failed runs are not evidence of passing.",
|
|
},
|
|
)
|
|
raise
|
|
finally:
|
|
for service in SERVICES:
|
|
compose("up", "-d", "--no-deps", service)
|
|
undrain(service)
|
|
print("PASS: evidence saved to " + str(out.relative_to(ROOT)))
|
|
except BaseException:
|
|
save_json(
|
|
out / "result.json",
|
|
{
|
|
"status": "FAIL",
|
|
"note": "Run did not finish successfully; inspect completed outputs.",
|
|
},
|
|
)
|
|
raise
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|