forked from mtusi2026/mtusi2026lab3
ЛР 3: балансировка, идемпотентность и проверки на 4
This commit is contained in:
@@ -0,0 +1,208 @@
|
||||
"""Repeatable API failures and sequential replacement under HTTP load."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from uuid import uuid4
|
||||
|
||||
from ops import ROOT, command, compose, containers, summary
|
||||
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure
|
||||
|
||||
|
||||
def wait_instances(client, actor, expected, excluded=(), version=None):
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
seen = {}
|
||||
try:
|
||||
for _ in range(20):
|
||||
r = client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
seen[r["headers"]["X-Instance-Id"]] = r["headers"].get("X-App-Version")
|
||||
if len(seen) >= expected and not any(any(s.startswith(x + "-") for x in excluded) for s in seen):
|
||||
if version is None or all(v == version for v in seen.values()):
|
||||
return seen
|
||||
except Failure:
|
||||
pass
|
||||
time.sleep(.2)
|
||||
raise AssertionError("Expected API replicas were not observed through the load balancer")
|
||||
|
||||
|
||||
def history(client, actor, chat):
|
||||
from urllib.parse import quote
|
||||
result, cursor = [], None
|
||||
while True:
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100"
|
||||
if cursor:
|
||||
path += "&cursor=" + quote(cursor, safe="")
|
||||
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
|
||||
result.extend(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
return result
|
||||
|
||||
|
||||
def resource_sample():
|
||||
ids = [r["id"] for r in summary() if r["status"] == "running"]
|
||||
return [json.loads(line) for line in command(["docker", "stats", "--no-stream", "--format", "{{json .}}", *ids]).splitlines()]
|
||||
|
||||
|
||||
def workload(base_url, label, action, duration=24, concurrency=4):
|
||||
client = Client(base_url, timeout=3)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
before_message = client.message(alice, chat, "confirmed before " + label)["json"]
|
||||
observed_before = wait_instances(client, alice, 2)
|
||||
started = time.monotonic()
|
||||
deadline = started + duration
|
||||
events, confirmations = [], {}
|
||||
action_result = {}
|
||||
lock = threading.Lock()
|
||||
|
||||
def worker(index):
|
||||
local = Client(base_url, timeout=3)
|
||||
key = uuid4().hex
|
||||
attempt = 0
|
||||
while time.monotonic() < deadline:
|
||||
begin = time.monotonic()
|
||||
try:
|
||||
if index == 0 and attempt % 4 == 0:
|
||||
response = local.call("GET", "/api/v1/users/me", actor=alice, schema="User")
|
||||
kind = "read"
|
||||
else:
|
||||
response = local.message(alice, chat, "load-" + key, key=key)
|
||||
kind = "write"
|
||||
with lock:
|
||||
confirmations[response["json"]["id"]] = response["json"]
|
||||
key = uuid4().hex
|
||||
event = {"at": time.monotonic() - started, "ok": True, "kind": kind,
|
||||
"ms": (time.monotonic() - begin) * 1000,
|
||||
"instance": response["headers"]["X-Instance-Id"]}
|
||||
except Failure:
|
||||
event = {"at": time.monotonic() - started, "ok": False,
|
||||
"ms": (time.monotonic() - begin) * 1000}
|
||||
# Preserve the key after uncertain POST responses.
|
||||
with lock:
|
||||
events.append(event)
|
||||
attempt += 1
|
||||
|
||||
def disrupt():
|
||||
time.sleep(4)
|
||||
action_result["started_seconds"] = time.monotonic() - started
|
||||
action_result["detail"] = action(client, alice)
|
||||
action_result["finished_seconds"] = time.monotonic() - started
|
||||
action_result["resources_during_load"] = resource_sample()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=concurrency + 1) as pool:
|
||||
workers = [pool.submit(worker, i) for i in range(concurrency)]
|
||||
change = pool.submit(disrupt)
|
||||
for future in workers:
|
||||
future.result()
|
||||
change.result()
|
||||
elapsed = time.monotonic() - started
|
||||
rows = history(client, bob, chat)
|
||||
by_id = {r["id"]: r for r in rows}
|
||||
assert by_id[before_message["id"]] == before_message
|
||||
assert all(by_id.get(mid) == msg for mid, msg in confirmations.items()), "Confirmed write lost"
|
||||
assert len(rows) == len(by_id) == len({r["text"] for r in rows}), "Retry created duplicate messages"
|
||||
success = [e for e in events if e["ok"]]
|
||||
failure = [e for e in events if not e["ok"]]
|
||||
finish = action_result["finished_seconds"]
|
||||
writes_after = [e for e in success if e["at"] >= finish and e["kind"] == "write"]
|
||||
reads_after = [e for e in success if e["at"] >= finish and e["kind"] == "read"]
|
||||
assert writes_after and reads_after, "Read/write did not resume"
|
||||
recovery = min(e["at"] for e in writes_after) - action_result["started_seconds"]
|
||||
if label.startswith("failure"):
|
||||
assert recovery <= 10, "Failover exceeded 10 seconds"
|
||||
hostname = action_result["detail"]["hostname"]
|
||||
assert all(not e["instance"].startswith(hostname + "-") for e in writes_after[-20:])
|
||||
latencies = sorted(e["ms"] for e in success)
|
||||
def percentile(q):
|
||||
return round(latencies[max(0, math.ceil(len(latencies) * q) - 1)], 3)
|
||||
buckets = []
|
||||
for start in range(0, math.ceil(elapsed), 2):
|
||||
group = [e for e in events if start <= e["at"] < start + 2]
|
||||
buckets.append({"start_seconds": start, "rps": sum(e["ok"] for e in group) / 2,
|
||||
"errors": sum(not e["ok"] for e in group)})
|
||||
report = {"scenario": label, "utc": datetime.now(timezone.utc).isoformat(),
|
||||
"duration_seconds": round(elapsed, 3), "concurrency": concurrency,
|
||||
"successful_requests": len(success), "failed_attempts": len(failure),
|
||||
"error_ratio": round(len(failure) / len(events), 6),
|
||||
"successful_rps": round(len(success) / elapsed, 3),
|
||||
"successful_latency_ms": {"p50": percentile(.5), "p95": percentile(.95), "p99": percentile(.99)},
|
||||
"first_write_after_action_seconds": round(recovery, 3), "action": action_result,
|
||||
"confirmed_writes_checked": len(confirmations) + 1, "history_messages": len(rows),
|
||||
"unique_history_messages": len(by_id), "instances_before": list(observed_before),
|
||||
"series_2_seconds": buckets, "deployment_after": summary(),
|
||||
"note": "Closed-loop mixed HTTP load, 4 clients; failures are attempts, retries retain keys; latency covers successes only."}
|
||||
(ROOT / "evidence" / (label + ".json")).write_text(json.dumps(report, indent=2) + "\n")
|
||||
print(label + ": " + str(report["successful_rps"]) + " RPS; " + str(len(failure)) + " errors", flush=True)
|
||||
return report
|
||||
|
||||
|
||||
def failure_action(target):
|
||||
def stop(client, actor):
|
||||
begin = time.monotonic()
|
||||
command(["docker", "stop", "--time", "6", target["Id"]], timeout=10)
|
||||
row = json.loads(command(["docker", "inspect", target["Id"]]))[0]
|
||||
assert not row["State"]["Running"] and row["State"]["ExitCode"] == 0
|
||||
elapsed = time.monotonic() - begin
|
||||
assert elapsed <= 10
|
||||
return {"container": row["Id"], "hostname": row["Config"]["Hostname"],
|
||||
"stop_seconds": round(elapsed, 3), "exit_code": row["State"]["ExitCode"]}
|
||||
return stop
|
||||
|
||||
|
||||
def rolling_action(client, actor):
|
||||
original = containers()
|
||||
assert len(original) == 2
|
||||
updates = []
|
||||
env = dict(os.environ, APP_VERSION="lab3-v2")
|
||||
# Both migrations are additive. Check/replay once before replacing processes.
|
||||
migration = compose("run", "--rm", "--no-deps", "migrate")
|
||||
for old in original:
|
||||
address = next(n["IPAddress"] for name, n in old["NetworkSettings"]["Networks"].items() if name.endswith("_edge"))
|
||||
slot = compose("run", "--rm", "--no-deps", "checks", "python", "scripts/ops.py", "drain", address)
|
||||
command(["docker", "stop", "--time", "6", old["Id"]])
|
||||
command(["docker", "rm", old["Id"]])
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "--scale", "api=2", "api", env=env)
|
||||
compose("run", "--rm", "--no-deps", "checks", "python", "scripts/ops.py", "ready", slot)
|
||||
current = containers()
|
||||
fresh = [r for r in current if r["Id"] not in {x["Id"] for x in original} and r["Id"] not in {x["new_container"] for x in updates}]
|
||||
assert len(fresh) == 1
|
||||
wait_instances(client, actor, 2, excluded=[old["Config"]["Hostname"]])
|
||||
updates.append({"old_container": old["Id"], "new_container": fresh[0]["Id"], "backend_slot": slot,
|
||||
"new_image": fresh[0]["Image"]})
|
||||
wait_instances(client, actor, 2, version="lab3-v2")
|
||||
return {"updates": updates, "migration": migration,
|
||||
"method": "drain, stop, remove and recreate each container; same application image, new APP_VERSION configuration"}
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
args = p.parse_args()
|
||||
originals = containers()
|
||||
assert len(originals) == 2
|
||||
reports = []
|
||||
for index, target in enumerate(originals, 1):
|
||||
try:
|
||||
reports.append(workload(args.base_url, "failure-api-" + str(index), failure_action(target)))
|
||||
finally:
|
||||
command(["docker", "start", target["Id"]])
|
||||
c = Client(args.base_url)
|
||||
alice = c.user("restore")
|
||||
wait_instances(c, alice, 2)
|
||||
reports.append(workload(args.base_url, "rolling-update", rolling_action, duration=45))
|
||||
(ROOT / "evidence" / "experiments.json").write_text(json.dumps([
|
||||
{k: r[k] for k in ("scenario", "successful_rps", "failed_attempts", "first_write_after_action_seconds",
|
||||
"confirmed_writes_checked", "successful_latency_ms")} for r in reports], indent=2) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user