forked from mtusi2026/mtusi2026lab3
оценка 4
This commit is contained in:
Executable
+397
@@ -0,0 +1,397 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the repeatable, secret-free resilience experiment required for grade 4."""
|
||||
import argparse
|
||||
from collections import Counter, defaultdict
|
||||
import datetime as dt
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure # noqa: E402
|
||||
|
||||
|
||||
def command(args, env=None, timeout=240, check=True):
|
||||
result = subprocess.run(
|
||||
[str(value) for value in args],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
)
|
||||
if check and result.returncode:
|
||||
raise RuntimeError(f"{' '.join(map(str, args[:3]))} exited {result.returncode}")
|
||||
return result
|
||||
|
||||
|
||||
def compose(env_file, *args):
|
||||
return command(["docker", "compose", "--env-file", env_file, *args])
|
||||
|
||||
|
||||
def api_ids(env_file):
|
||||
return [line for line in compose(env_file, "ps", "-q", "api").stdout.splitlines() if line]
|
||||
|
||||
|
||||
def inspect(container_id):
|
||||
data = command(["docker", "inspect", "--format", "{{json .}}", container_id]).stdout
|
||||
return json.loads(data)
|
||||
|
||||
|
||||
def wait_api(env_file, expected, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
healthy = []
|
||||
for container_id in api_ids(env_file):
|
||||
state = inspect(container_id)["State"]
|
||||
if state.get("Health", {}).get("Status") == "healthy":
|
||||
healthy.append(container_id)
|
||||
if len(healthy) >= expected:
|
||||
return healthy
|
||||
time.sleep(0.5)
|
||||
raise RuntimeError(f"fewer than {expected} API replicas became healthy")
|
||||
|
||||
|
||||
def percentile(values, ratio):
|
||||
if not values:
|
||||
return None
|
||||
values = sorted(values)
|
||||
return round(values[max(0, math.ceil(ratio * len(values)) - 1)], 3)
|
||||
|
||||
|
||||
def memory_mib(raw):
|
||||
value = raw.split("/", 1)[0].strip()
|
||||
units = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}
|
||||
for suffix in ("GiB", "MiB", "KiB", "B"):
|
||||
if value.endswith(suffix):
|
||||
return round(float(value[:-len(suffix)]) * units[suffix] / 1024**2, 3)
|
||||
return 0.0
|
||||
|
||||
|
||||
def service(name):
|
||||
for candidate in ("load-balancer", "postgres", "redis", "api"):
|
||||
if f"-{candidate}-" in name:
|
||||
return candidate
|
||||
return "other"
|
||||
|
||||
|
||||
def sample_stats():
|
||||
result = command(
|
||||
["docker", "stats", "--no-stream", "--format", "{{json .}}"],
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
samples = []
|
||||
if result.returncode:
|
||||
return samples
|
||||
for line in result.stdout.splitlines():
|
||||
try:
|
||||
item = json.loads(line)
|
||||
samples.append({
|
||||
"service": service(item.get("Name", "")),
|
||||
"cpu_percent": float(item.get("CPUPerc", "0%").rstrip("%")),
|
||||
"memory_mib": memory_mib(item.get("MemUsage", "0B / 0B")),
|
||||
})
|
||||
except (ValueError, json.JSONDecodeError):
|
||||
continue
|
||||
return samples
|
||||
|
||||
|
||||
def aggregate_resources(samples):
|
||||
grouped = defaultdict(lambda: {"cpu": [], "memory": []})
|
||||
for item in samples:
|
||||
if item["service"] != "other":
|
||||
grouped[item["service"]]["cpu"].append(item["cpu_percent"])
|
||||
grouped[item["service"]]["memory"].append(item["memory_mib"])
|
||||
return {
|
||||
name: {
|
||||
"cpu_percent_avg": round(sum(values["cpu"]) / len(values["cpu"]), 3),
|
||||
"cpu_percent_max": round(max(values["cpu"]), 3),
|
||||
"memory_mib_avg": round(sum(values["memory"]) / len(values["memory"]), 3),
|
||||
"memory_mib_max": round(max(values["memory"]), 3),
|
||||
}
|
||||
for name, values in sorted(grouped.items())
|
||||
}
|
||||
|
||||
|
||||
def load_phase(name, actor, chat, args, action=None, target=None):
|
||||
records, resources = [], []
|
||||
lock = threading.Lock()
|
||||
stop = threading.Event()
|
||||
barrier = threading.Barrier(args.concurrency)
|
||||
started = time.monotonic()
|
||||
|
||||
def worker(worker_id):
|
||||
client = Client(args.base_url, timeout=4)
|
||||
index = 0
|
||||
barrier.wait()
|
||||
while not stop.is_set():
|
||||
request_started = time.monotonic()
|
||||
try:
|
||||
response = client.message(actor, chat, f"grade4-{name}-{worker_id}-{index}")
|
||||
row = {
|
||||
"t": round(time.monotonic() - started, 3),
|
||||
"ok": True,
|
||||
"latency_ms": round((time.monotonic() - request_started) * 1000, 3),
|
||||
"instance": response["headers"].get("X-Instance-Id", ""),
|
||||
}
|
||||
except Failure as exc:
|
||||
row = {
|
||||
"t": round(time.monotonic() - started, 3),
|
||||
"ok": False,
|
||||
"latency_ms": round((time.monotonic() - request_started) * 1000, 3),
|
||||
"error": str(exc),
|
||||
}
|
||||
with lock:
|
||||
records.append(row)
|
||||
index += 1
|
||||
|
||||
def sampler():
|
||||
while not stop.is_set():
|
||||
resources.extend(sample_stats())
|
||||
stop.wait(1)
|
||||
|
||||
workers = [threading.Thread(target=worker, args=(index,), daemon=True)
|
||||
for index in range(args.concurrency)]
|
||||
stats_thread = threading.Thread(target=sampler, daemon=True)
|
||||
stats_thread.start()
|
||||
for thread in workers:
|
||||
thread.start()
|
||||
|
||||
action_data = None
|
||||
action_error = None
|
||||
try:
|
||||
if action is None:
|
||||
time.sleep(args.baseline_seconds)
|
||||
else:
|
||||
time.sleep(args.action_after_seconds)
|
||||
action_started = time.monotonic()
|
||||
try:
|
||||
summary = action()
|
||||
exit_code = 0
|
||||
except Exception as exc:
|
||||
summary, exit_code, action_error = type(exc).__name__, 1, exc
|
||||
action_ended = time.monotonic()
|
||||
action_data = {
|
||||
"started_at": round(action_started - started, 3),
|
||||
"ended_at": round(action_ended - started, 3),
|
||||
"duration_seconds": round(action_ended - action_started, 3),
|
||||
"exit_code": exit_code,
|
||||
"summary": summary,
|
||||
}
|
||||
time.sleep(args.post_action_seconds)
|
||||
finally:
|
||||
stop.set()
|
||||
for thread in workers:
|
||||
thread.join(10)
|
||||
stats_thread.join(5)
|
||||
if action_error:
|
||||
raise action_error
|
||||
|
||||
elapsed = time.monotonic() - started
|
||||
successes = [row for row in records if row["ok"]]
|
||||
failures = [row for row in records if not row["ok"]]
|
||||
latencies = [row["latency_ms"] for row in successes]
|
||||
buckets = defaultdict(lambda: {"successful": 0, "failed": 0})
|
||||
for row in records:
|
||||
buckets[str(int(row["t"]))]["successful" if row["ok"] else "failed"] += 1
|
||||
recovery = None
|
||||
if action_data and target:
|
||||
recovery_row = next((
|
||||
row for row in successes
|
||||
if row["t"] >= action_data["ended_at"] and row.get("instance") != target
|
||||
), None)
|
||||
if recovery_row:
|
||||
recovery = round(recovery_row["t"] - action_data["started_at"], 3)
|
||||
total = len(records)
|
||||
return {
|
||||
"name": name,
|
||||
"elapsed_seconds": round(elapsed, 3),
|
||||
"successful_requests": len(successes),
|
||||
"failed_requests": len(failures),
|
||||
"error_ratio": round(len(failures) / total, 6) if total else None,
|
||||
"errors": dict(sorted(Counter(row["error"] for row in failures).items())),
|
||||
"successful_rps": round(len(successes) / elapsed, 3),
|
||||
"successful_latency_ms": {
|
||||
"p50": percentile(latencies, 0.50),
|
||||
"p95": percentile(latencies, 0.95),
|
||||
"p99": percentile(latencies, 0.99),
|
||||
},
|
||||
"instances": sorted({row.get("instance") for row in successes if row.get("instance")}),
|
||||
"recovery_seconds": recovery,
|
||||
"action": action_data,
|
||||
"per_second": dict(sorted(buckets.items(), key=lambda item: int(item[0]))),
|
||||
"resources": aggregate_resources(resources),
|
||||
}
|
||||
|
||||
|
||||
def idempotency(actor, chat, base_url, count=32):
|
||||
key, body = uuid.uuid4().hex, "grade4-idempotency-" + uuid.uuid4().hex
|
||||
barrier, lock = threading.Barrier(count), threading.Lock()
|
||||
rows, errors = [], []
|
||||
|
||||
def request():
|
||||
client = Client(base_url, timeout=8)
|
||||
barrier.wait()
|
||||
try:
|
||||
response = client.message(actor, chat, body, key=key)
|
||||
row = (response["json"]["id"], response["headers"].get("X-Instance-Id", ""))
|
||||
with lock:
|
||||
rows.append(row)
|
||||
except Failure as exc:
|
||||
with lock:
|
||||
errors.append(str(exc))
|
||||
|
||||
threads = [threading.Thread(target=request) for _ in range(count)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join(30)
|
||||
message_ids = {row[0] for row in rows}
|
||||
if errors or len(rows) != count or len(message_ids) != 1:
|
||||
raise RuntimeError("parallel idempotency check failed")
|
||||
return {
|
||||
"parallel_requests": count,
|
||||
"successful_responses": len(rows),
|
||||
"failed_responses": len(errors),
|
||||
"unique_message_ids": len(message_ids),
|
||||
"observed_instances": sorted({row[1] for row in rows}),
|
||||
}
|
||||
|
||||
|
||||
def environment(env_file):
|
||||
mem_total_kib = 0
|
||||
with open("/proc/meminfo", encoding="utf-8") as meminfo:
|
||||
for line in meminfo:
|
||||
if line.startswith("MemTotal:"):
|
||||
mem_total_kib = int(line.split()[1])
|
||||
break
|
||||
limits = []
|
||||
for container_id in compose(env_file, "ps", "-q").stdout.splitlines():
|
||||
data = inspect(container_id)
|
||||
limits.append({
|
||||
"service": service(data["Name"].lstrip("/")),
|
||||
"nano_cpus": data["HostConfig"].get("NanoCpus", 0),
|
||||
"memory_limit_bytes": data["HostConfig"].get("Memory", 0),
|
||||
})
|
||||
return {
|
||||
"utc_started_at": dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
"git_commit": command(["git", "rev-parse", "HEAD"]).stdout.strip(),
|
||||
"working_tree_dirty": bool(command(["git", "status", "--porcelain"]).stdout.strip()),
|
||||
"logical_cpu_count": os.cpu_count(),
|
||||
"host_memory_mib": round(mem_total_kib / 1024, 1),
|
||||
"docker_engine": command(["docker", "version", "--format", "{{.Server.Version}}"]).stdout.strip(),
|
||||
"docker_compose": command(["docker", "compose", "version", "--short"]).stdout.strip(),
|
||||
"container_limits": limits,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://localhost:8080")
|
||||
parser.add_argument("--env-file", default=".env")
|
||||
parser.add_argument("--concurrency", type=int, default=8)
|
||||
parser.add_argument("--baseline-seconds", type=float, default=12)
|
||||
parser.add_argument("--action-after-seconds", type=float, default=3)
|
||||
parser.add_argument("--post-action-seconds", type=float, default=8)
|
||||
parser.add_argument("--output", default="evidence/grade4/results.json")
|
||||
args = parser.parse_args()
|
||||
if not 1 <= args.concurrency <= 64:
|
||||
raise SystemExit("concurrency must be 1..64")
|
||||
|
||||
original = wait_api(args.env_file, 2)
|
||||
if len(original) != 2:
|
||||
raise RuntimeError("experiment requires exactly two initial API replicas")
|
||||
client = Client(args.base_url, timeout=5)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
for index in range(12):
|
||||
client.message(alice, chat, f"grade4-warmup-{index}")
|
||||
|
||||
phases = [load_phase("baseline", alice, chat, args)]
|
||||
targets = [(container_id, inspect(container_id)["Config"]["Hostname"])
|
||||
for container_id in original]
|
||||
for ordinal, (container_id, target) in enumerate(targets, start=1):
|
||||
def stop_target(instance=target):
|
||||
hook_env = dict(os.environ)
|
||||
hook_env.update({"ENV_FILE": args.env_file, "TARGET_INSTANCE_ID": instance})
|
||||
result = command([ROOT / "scripts" / "stop-one.sh"], env=hook_env, timeout=30)
|
||||
return result.stdout.strip().splitlines()[-1]
|
||||
|
||||
phases.append(load_phase(
|
||||
f"failover-{ordinal}", alice, chat, args,
|
||||
action=stop_target, target=target,
|
||||
))
|
||||
command(["docker", "start", container_id])
|
||||
wait_api(args.env_file, 2)
|
||||
|
||||
idem = idempotency(alice, chat, args.base_url)
|
||||
|
||||
def rolling():
|
||||
hook_env = dict(os.environ)
|
||||
hook_env.update({"ENV_FILE": args.env_file, "DESIRED_REPLICAS": "2"})
|
||||
result = command(
|
||||
[ROOT / "scripts" / "rolling-update.sh"],
|
||||
env=hook_env,
|
||||
timeout=240,
|
||||
)
|
||||
return result.stdout.strip().splitlines()[-1]
|
||||
|
||||
phases.append(load_phase("rolling-update", alice, chat, args, action=rolling))
|
||||
wait_api(args.env_file, 2)
|
||||
|
||||
report = {
|
||||
"schema_version": 1,
|
||||
"environment": environment(args.env_file),
|
||||
"methodology": {
|
||||
"base_url": args.base_url,
|
||||
"concurrency": args.concurrency,
|
||||
"baseline_seconds": args.baseline_seconds,
|
||||
"action_after_seconds": args.action_after_seconds,
|
||||
"post_action_seconds": args.post_action_seconds,
|
||||
"warmup_requests": 12,
|
||||
"workload": "closed-loop POST messages via Nginx; no automatic retries",
|
||||
},
|
||||
"phases": phases,
|
||||
"idempotency": idem,
|
||||
"dataset": {
|
||||
"warmup_requests": 12,
|
||||
"minimum_confirmed_messages": 13 + sum(
|
||||
phase["successful_requests"] for phase in phases
|
||||
),
|
||||
},
|
||||
"limitations": [
|
||||
"single Docker host",
|
||||
"closed-loop coordinated omission",
|
||||
"successful-response latency percentiles only",
|
||||
"PostgreSQL and Nginx remain single points of failure",
|
||||
],
|
||||
}
|
||||
destination = ROOT / args.output
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
||||
try:
|
||||
displayed_output = destination.relative_to(ROOT)
|
||||
except ValueError:
|
||||
displayed_output = destination
|
||||
print(json.dumps({
|
||||
"output": str(displayed_output),
|
||||
"phases": [{
|
||||
"name": phase["name"],
|
||||
"successful_rps": phase["successful_rps"],
|
||||
"failed_requests": phase["failed_requests"],
|
||||
"recovery_seconds": phase["recovery_seconds"],
|
||||
} for phase in phases],
|
||||
"idempotency": idem,
|
||||
}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+203
@@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render portable SVG graphs and a Markdown summary from grade-4 results."""
|
||||
import argparse
|
||||
from collections import defaultdict
|
||||
import html
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
STYLE = """
|
||||
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
|
||||
.title { font-size: 26px; font-weight: 650; }
|
||||
.label { font-size: 14px; fill: #c8cbd0; }
|
||||
.small { font-size: 12px; fill: #a2a5aa; }
|
||||
.grid { stroke: #34373d; stroke-width: 1; }
|
||||
.bar { fill: #cfd2d6; }
|
||||
.bar2 { fill: #73777f; }
|
||||
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
|
||||
.error { fill: #8f949c; stroke: #f4f5f6; stroke-width: 1; }
|
||||
.event { stroke: #a2a5aa; stroke-width: 2; stroke-dasharray: 6 5; }
|
||||
"""
|
||||
|
||||
|
||||
def svg(title, body, width=1100, height=560):
|
||||
return f"""<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" viewBox="0 0 {width} {height}">
|
||||
<rect width="100%" height="100%" fill="#0e1013"/>
|
||||
<style>{STYLE}</style>
|
||||
<text x="48" y="42" class="title">{html.escape(title)}</text>
|
||||
{body}
|
||||
</svg>
|
||||
"""
|
||||
|
||||
|
||||
def grid(maximum, left, top, width, height):
|
||||
parts = []
|
||||
for step in range(6):
|
||||
value = maximum * step / 5
|
||||
y = top + height - height * step / 5
|
||||
parts.append(f'<line x1="{left}" y1="{y:.1f}" x2="{left+width}" y2="{y:.1f}" class="grid"/>')
|
||||
parts.append(f'<text x="{left-12}" y="{y+4:.1f}" text-anchor="end" class="small">{value:.0f}</text>')
|
||||
return parts
|
||||
|
||||
|
||||
def throughput(phases, output):
|
||||
left, top, width, height = 80, 90, 960, 360
|
||||
maximum = max((phase["successful_rps"] for phase in phases), default=1) or 1
|
||||
parts = grid(maximum, left, top, width, height)
|
||||
slot = width / max(1, len(phases))
|
||||
for index, phase in enumerate(phases):
|
||||
bar_width = min(120, slot * 0.5)
|
||||
x = left + slot * index + (slot - bar_width) / 2
|
||||
bar_height = height * phase["successful_rps"] / maximum
|
||||
y = top + height - bar_height
|
||||
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_width:.1f}" height="{bar_height:.1f}" rx="10" class="bar"/>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{y-10:.1f}" text-anchor="middle" class="label">{phase["successful_rps"]:.1f} RPS</text>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+28}" text-anchor="middle" class="label">{html.escape(phase["name"])}</text>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+48}" text-anchor="middle" class="small">errors {phase["error_ratio"]*100:.2f}%</text>')
|
||||
parts.append(f'<text x="24" y="{top+height/2}" transform="rotate(-90 24 {top+height/2})" class="label">Successful requests per second</text>')
|
||||
output.write_text(svg("Grade 4: throughput and errors", "\n".join(parts)))
|
||||
|
||||
|
||||
def latency(phases, output):
|
||||
left, top, width, height = 80, 90, 960, 360
|
||||
values = [value for phase in phases
|
||||
for value in phase["successful_latency_ms"].values()
|
||||
if value is not None]
|
||||
maximum = max(values, default=1) or 1
|
||||
parts = grid(maximum, left, top, width, height)
|
||||
slot = width / max(1, len(phases))
|
||||
for phase_index, phase in enumerate(phases):
|
||||
group_x = left + slot * phase_index
|
||||
bar_width = min(42, slot / 5)
|
||||
for metric_index, metric in enumerate(("p50", "p95", "p99")):
|
||||
value = phase["successful_latency_ms"][metric] or 0
|
||||
bar_height = height * value / maximum
|
||||
x = group_x + slot * 0.2 + metric_index * (bar_width + 8)
|
||||
y = top + height - bar_height
|
||||
css = "bar" if metric_index == 0 else "bar2"
|
||||
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_width:.1f}" height="{bar_height:.1f}" rx="7" class="{css}"/>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{y-7:.1f}" text-anchor="middle" class="small">{value:.0f}</text>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+20}" text-anchor="middle" class="small">{metric}</text>')
|
||||
parts.append(f'<text x="{group_x+slot/2:.1f}" y="{top+height+48}" text-anchor="middle" class="label">{html.escape(phase["name"])}</text>')
|
||||
parts.append(f'<text x="24" y="{top+height/2}" transform="rotate(-90 24 {top+height/2})" class="label">Successful latency, ms</text>')
|
||||
output.write_text(svg("Grade 4: successful latency percentiles", "\n".join(parts)))
|
||||
|
||||
|
||||
def timeline(phases, output):
|
||||
left, top, width, height = 70, 90, 980, 390
|
||||
rows, events, offset = [], [], 0
|
||||
for phase in phases:
|
||||
buckets = phase["per_second"]
|
||||
for second, counts in sorted(buckets.items(), key=lambda item: int(item[0])):
|
||||
rows.append((offset + int(second), counts))
|
||||
if phase.get("action"):
|
||||
events.append((offset + phase["action"]["started_at"], phase["name"]))
|
||||
offset += max([int(key) for key in buckets] or [0]) + 2
|
||||
maximum = max((counts["successful"] for _, counts in rows), default=1) or 1
|
||||
max_x = max((second for second, _ in rows), default=1) or 1
|
||||
parts, points = grid(maximum, left, top, width, height), []
|
||||
for second, counts in rows:
|
||||
x = left + width * second / max_x
|
||||
y = top + height - height * counts["successful"] / maximum
|
||||
points.append(f"{x:.1f},{y:.1f}")
|
||||
if counts["failed"]:
|
||||
radius = min(12, 3 + counts["failed"])
|
||||
parts.append(f'<circle cx="{x:.1f}" cy="{top+height-8}" r="{radius}" class="error"/>')
|
||||
parts.append(f'<polyline points="{" ".join(points)}" class="line"/>')
|
||||
for second, label in events:
|
||||
x = left + width * second / max_x
|
||||
parts.append(f'<line x1="{x:.1f}" y1="{top}" x2="{x:.1f}" y2="{top+height}" class="event"/>')
|
||||
parts.append(f'<text x="{x+6:.1f}" y="{top+18}" class="small">{html.escape(label)} action</text>')
|
||||
parts.append(f'<text x="{left}" y="{top+height+32}" class="label">One-second buckets; circles mark failed requests</text>')
|
||||
output.write_text(svg("Grade 4: request timeline", "\n".join(parts), height=600))
|
||||
|
||||
|
||||
def resources(phases, output):
|
||||
combined = defaultdict(lambda: {"cpu": [], "memory": []})
|
||||
for phase in phases:
|
||||
for name, values in phase["resources"].items():
|
||||
combined[name]["cpu"].append(values["cpu_percent_max"])
|
||||
combined[name]["memory"].append(values["memory_mib_max"])
|
||||
names, parts = sorted(combined), []
|
||||
panels = [
|
||||
("Peak CPU, %", "cpu", 80, 90, 420, 360),
|
||||
("Peak memory, MiB", "memory", 600, 90, 420, 360),
|
||||
]
|
||||
for title, metric, left, top, width, height in panels:
|
||||
maximum = max((max(combined[name][metric]) for name in names), default=1) or 1
|
||||
parts.append(f'<text x="{left}" y="{top-18}" class="label">{title}</text>')
|
||||
slot = width / max(1, len(names))
|
||||
for index, name in enumerate(names):
|
||||
value = max(combined[name][metric])
|
||||
bar_width = min(70, slot * 0.55)
|
||||
x = left + slot * index + (slot - bar_width) / 2
|
||||
bar_height = height * value / maximum
|
||||
y = top + height - bar_height
|
||||
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_width:.1f}" height="{bar_height:.1f}" rx="9" class="bar"/>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{y-8:.1f}" text-anchor="middle" class="small">{value:.1f}</text>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+25}" text-anchor="middle" class="small">{html.escape(name)}</text>')
|
||||
output.write_text(svg("Grade 4: peak container resources", "\n".join(parts), height=600))
|
||||
|
||||
|
||||
def summary(report, output):
|
||||
method = report["methodology"]
|
||||
lines = [
|
||||
"# Lab 3 grade-4 experiment", "",
|
||||
f"Generated: {report['environment']['utc_started_at']}.", "",
|
||||
"## Method", "",
|
||||
f"- concurrency: {method['concurrency']};",
|
||||
f"- baseline duration: {method['baseline_seconds']} s;",
|
||||
"- load: closed-loop POST messages through Nginx;",
|
||||
"- failover: both original API containers stopped and restored in turn;",
|
||||
"- rolling update: one surge replica, old replicas replaced one by one;",
|
||||
"- percentiles cover successful responses; failures are separate.", "",
|
||||
"## Results", "",
|
||||
"| Phase | RPS | Errors | p50 ms | p95 ms | p99 ms | Recovery s |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
|
||||
]
|
||||
for phase in report["phases"]:
|
||||
lat = phase["successful_latency_ms"]
|
||||
recovery = phase["recovery_seconds"]
|
||||
lines.append(
|
||||
f"| {phase['name']} | {phase['successful_rps']} | "
|
||||
f"{phase['failed_requests']} ({phase['error_ratio']*100:.3f}%) | "
|
||||
f"{lat['p50']} | {lat['p95']} | {lat['p99']} | "
|
||||
f"{recovery if recovery is not None else 'n/a'} |"
|
||||
)
|
||||
idem = report["idempotency"]
|
||||
lines += [
|
||||
"", "## Parallel idempotency", "",
|
||||
f"{idem['parallel_requests']} requests produced "
|
||||
f"{idem['unique_message_ids']} unique message ID across "
|
||||
f"{len(idem['observed_instances'])} API instances.", "",
|
||||
"## Graphs", "",
|
||||
"", "",
|
||||
"", "",
|
||||
"", "",
|
||||
"", "",
|
||||
"## Scope", "",
|
||||
"Single-host process failure and rolling replacement only. PostgreSQL "
|
||||
"and Nginx remain single points of failure.", "",
|
||||
]
|
||||
output.write_text("\n".join(lines))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("results", nargs="?", default="evidence/grade4/results.json")
|
||||
args = parser.parse_args()
|
||||
source = Path(args.results)
|
||||
report = json.loads(source.read_text())
|
||||
destination = source.parent
|
||||
phases = report["phases"]
|
||||
throughput(phases, destination / "throughput-errors.svg")
|
||||
latency(phases, destination / "latency.svg")
|
||||
timeline(phases, destination / "timeline.svg")
|
||||
resources(phases, destination / "resources.svg")
|
||||
summary(report, destination / "README.md")
|
||||
print(f"rendered grade-4 evidence in {destination}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+89
@@ -0,0 +1,89 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
project_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$project_root"
|
||||
|
||||
env_file=${ENV_FILE:-.env}
|
||||
desired_replicas=${DESIRED_REPLICAS:-2}
|
||||
wait_timeout=${ROLLING_WAIT_TIMEOUT:-90}
|
||||
stop_timeout=${ROLLING_STOP_TIMEOUT:-15}
|
||||
dns_settle_delay=${ROLLING_DNS_SETTLE_DELAY:-2}
|
||||
|
||||
case "$desired_replicas" in
|
||||
''|*[!0-9]*)
|
||||
printf '%s\n' "DESIRED_REPLICAS must be an integer" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
case "$dns_settle_delay" in
|
||||
''|*[!0-9]*)
|
||||
printf '%s\n' "ROLLING_DNS_SETTLE_DELAY must be a non-negative integer" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
if [ "$desired_replicas" -lt 2 ]; then
|
||||
printf '%s\n' "rolling update requires at least two desired replicas" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
compose() {
|
||||
docker compose --env-file "$env_file" "$@"
|
||||
}
|
||||
|
||||
healthy_api_count() {
|
||||
healthy=0
|
||||
for container_id in $(compose ps -q api); do
|
||||
status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container_id")
|
||||
if [ "$status" = "healthy" ] || [ "$status" = "running" ]; then
|
||||
healthy=$((healthy + 1))
|
||||
fi
|
||||
done
|
||||
printf '%s\n' "$healthy"
|
||||
}
|
||||
|
||||
wait_for_healthy() {
|
||||
expected=$1
|
||||
deadline=$(( $(date +%s) + wait_timeout ))
|
||||
while [ "$(healthy_api_count)" -lt "$expected" ]; do
|
||||
if [ "$(date +%s)" -ge "$deadline" ]; then
|
||||
printf '%s\n' "timed out waiting for $expected healthy API replicas" >&2
|
||||
return 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
old_containers=$(compose ps -q api)
|
||||
old_count=$(printf '%s\n' "$old_containers" | awk 'NF { count++ } END { print count+0 }')
|
||||
if [ "$old_count" -ne "$desired_replicas" ]; then
|
||||
printf '%s\n' "expected $desired_replicas running API containers, found $old_count" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
surge_replicas=$((desired_replicas + 1))
|
||||
|
||||
printf '%s\n' "building replacement API image"
|
||||
compose build api
|
||||
|
||||
printf '%s\n' "starting one surge replica"
|
||||
compose up -d --no-deps --scale api="$surge_replicas" --no-recreate api
|
||||
wait_for_healthy "$surge_replicas"
|
||||
sleep "$dns_settle_delay"
|
||||
|
||||
for old_container in $old_containers; do
|
||||
old_instance=$(docker inspect --format '{{.Config.Hostname}}' "$old_container")
|
||||
printf '%s\n' "draining and replacing API instance $old_instance"
|
||||
docker stop --time "$stop_timeout" "$old_container" >/dev/null
|
||||
docker rm "$old_container" >/dev/null
|
||||
compose up -d --no-deps --scale api="$surge_replicas" --no-recreate api
|
||||
wait_for_healthy "$surge_replicas"
|
||||
sleep "$dns_settle_delay"
|
||||
done
|
||||
|
||||
printf '%s\n' "removing surge capacity"
|
||||
compose up -d --no-deps --scale api="$desired_replicas" --no-recreate api
|
||||
wait_for_healthy "$desired_replicas"
|
||||
sleep "$dns_settle_delay"
|
||||
|
||||
printf '%s\n' "rolling update completed with $desired_replicas healthy API replicas"
|
||||
Reference in New Issue
Block a user