оценка 4

This commit is contained in:
2026-10-23 16:20:43 +03:00
parent 34618bd423
commit 5300a1a1e0
18 changed files with 1861 additions and 29 deletions
+397
View File
@@ -0,0 +1,397 @@
#!/usr/bin/env python3
"""Run the repeatable, secret-free resilience experiment required for grade 4."""
import argparse
from collections import Counter, defaultdict
import datetime as dt
import json
import math
import os
from pathlib import Path
import subprocess
import sys
import threading
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure # noqa: E402
def command(args, env=None, timeout=240, check=True):
result = subprocess.run(
[str(value) for value in args],
cwd=ROOT,
env=env,
text=True,
capture_output=True,
timeout=timeout,
check=False,
)
if check and result.returncode:
raise RuntimeError(f"{' '.join(map(str, args[:3]))} exited {result.returncode}")
return result
def compose(env_file, *args):
return command(["docker", "compose", "--env-file", env_file, *args])
def api_ids(env_file):
return [line for line in compose(env_file, "ps", "-q", "api").stdout.splitlines() if line]
def inspect(container_id):
data = command(["docker", "inspect", "--format", "{{json .}}", container_id]).stdout
return json.loads(data)
def wait_api(env_file, expected, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
healthy = []
for container_id in api_ids(env_file):
state = inspect(container_id)["State"]
if state.get("Health", {}).get("Status") == "healthy":
healthy.append(container_id)
if len(healthy) >= expected:
return healthy
time.sleep(0.5)
raise RuntimeError(f"fewer than {expected} API replicas became healthy")
def percentile(values, ratio):
if not values:
return None
values = sorted(values)
return round(values[max(0, math.ceil(ratio * len(values)) - 1)], 3)
def memory_mib(raw):
value = raw.split("/", 1)[0].strip()
units = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}
for suffix in ("GiB", "MiB", "KiB", "B"):
if value.endswith(suffix):
return round(float(value[:-len(suffix)]) * units[suffix] / 1024**2, 3)
return 0.0
def service(name):
for candidate in ("load-balancer", "postgres", "redis", "api"):
if f"-{candidate}-" in name:
return candidate
return "other"
def sample_stats():
result = command(
["docker", "stats", "--no-stream", "--format", "{{json .}}"],
timeout=10,
check=False,
)
samples = []
if result.returncode:
return samples
for line in result.stdout.splitlines():
try:
item = json.loads(line)
samples.append({
"service": service(item.get("Name", "")),
"cpu_percent": float(item.get("CPUPerc", "0%").rstrip("%")),
"memory_mib": memory_mib(item.get("MemUsage", "0B / 0B")),
})
except (ValueError, json.JSONDecodeError):
continue
return samples
def aggregate_resources(samples):
grouped = defaultdict(lambda: {"cpu": [], "memory": []})
for item in samples:
if item["service"] != "other":
grouped[item["service"]]["cpu"].append(item["cpu_percent"])
grouped[item["service"]]["memory"].append(item["memory_mib"])
return {
name: {
"cpu_percent_avg": round(sum(values["cpu"]) / len(values["cpu"]), 3),
"cpu_percent_max": round(max(values["cpu"]), 3),
"memory_mib_avg": round(sum(values["memory"]) / len(values["memory"]), 3),
"memory_mib_max": round(max(values["memory"]), 3),
}
for name, values in sorted(grouped.items())
}
def load_phase(name, actor, chat, args, action=None, target=None):
records, resources = [], []
lock = threading.Lock()
stop = threading.Event()
barrier = threading.Barrier(args.concurrency)
started = time.monotonic()
def worker(worker_id):
client = Client(args.base_url, timeout=4)
index = 0
barrier.wait()
while not stop.is_set():
request_started = time.monotonic()
try:
response = client.message(actor, chat, f"grade4-{name}-{worker_id}-{index}")
row = {
"t": round(time.monotonic() - started, 3),
"ok": True,
"latency_ms": round((time.monotonic() - request_started) * 1000, 3),
"instance": response["headers"].get("X-Instance-Id", ""),
}
except Failure as exc:
row = {
"t": round(time.monotonic() - started, 3),
"ok": False,
"latency_ms": round((time.monotonic() - request_started) * 1000, 3),
"error": str(exc),
}
with lock:
records.append(row)
index += 1
def sampler():
while not stop.is_set():
resources.extend(sample_stats())
stop.wait(1)
workers = [threading.Thread(target=worker, args=(index,), daemon=True)
for index in range(args.concurrency)]
stats_thread = threading.Thread(target=sampler, daemon=True)
stats_thread.start()
for thread in workers:
thread.start()
action_data = None
action_error = None
try:
if action is None:
time.sleep(args.baseline_seconds)
else:
time.sleep(args.action_after_seconds)
action_started = time.monotonic()
try:
summary = action()
exit_code = 0
except Exception as exc:
summary, exit_code, action_error = type(exc).__name__, 1, exc
action_ended = time.monotonic()
action_data = {
"started_at": round(action_started - started, 3),
"ended_at": round(action_ended - started, 3),
"duration_seconds": round(action_ended - action_started, 3),
"exit_code": exit_code,
"summary": summary,
}
time.sleep(args.post_action_seconds)
finally:
stop.set()
for thread in workers:
thread.join(10)
stats_thread.join(5)
if action_error:
raise action_error
elapsed = time.monotonic() - started
successes = [row for row in records if row["ok"]]
failures = [row for row in records if not row["ok"]]
latencies = [row["latency_ms"] for row in successes]
buckets = defaultdict(lambda: {"successful": 0, "failed": 0})
for row in records:
buckets[str(int(row["t"]))]["successful" if row["ok"] else "failed"] += 1
recovery = None
if action_data and target:
recovery_row = next((
row for row in successes
if row["t"] >= action_data["ended_at"] and row.get("instance") != target
), None)
if recovery_row:
recovery = round(recovery_row["t"] - action_data["started_at"], 3)
total = len(records)
return {
"name": name,
"elapsed_seconds": round(elapsed, 3),
"successful_requests": len(successes),
"failed_requests": len(failures),
"error_ratio": round(len(failures) / total, 6) if total else None,
"errors": dict(sorted(Counter(row["error"] for row in failures).items())),
"successful_rps": round(len(successes) / elapsed, 3),
"successful_latency_ms": {
"p50": percentile(latencies, 0.50),
"p95": percentile(latencies, 0.95),
"p99": percentile(latencies, 0.99),
},
"instances": sorted({row.get("instance") for row in successes if row.get("instance")}),
"recovery_seconds": recovery,
"action": action_data,
"per_second": dict(sorted(buckets.items(), key=lambda item: int(item[0]))),
"resources": aggregate_resources(resources),
}
def idempotency(actor, chat, base_url, count=32):
key, body = uuid.uuid4().hex, "grade4-idempotency-" + uuid.uuid4().hex
barrier, lock = threading.Barrier(count), threading.Lock()
rows, errors = [], []
def request():
client = Client(base_url, timeout=8)
barrier.wait()
try:
response = client.message(actor, chat, body, key=key)
row = (response["json"]["id"], response["headers"].get("X-Instance-Id", ""))
with lock:
rows.append(row)
except Failure as exc:
with lock:
errors.append(str(exc))
threads = [threading.Thread(target=request) for _ in range(count)]
for thread in threads:
thread.start()
for thread in threads:
thread.join(30)
message_ids = {row[0] for row in rows}
if errors or len(rows) != count or len(message_ids) != 1:
raise RuntimeError("parallel idempotency check failed")
return {
"parallel_requests": count,
"successful_responses": len(rows),
"failed_responses": len(errors),
"unique_message_ids": len(message_ids),
"observed_instances": sorted({row[1] for row in rows}),
}
def environment(env_file):
mem_total_kib = 0
with open("/proc/meminfo", encoding="utf-8") as meminfo:
for line in meminfo:
if line.startswith("MemTotal:"):
mem_total_kib = int(line.split()[1])
break
limits = []
for container_id in compose(env_file, "ps", "-q").stdout.splitlines():
data = inspect(container_id)
limits.append({
"service": service(data["Name"].lstrip("/")),
"nano_cpus": data["HostConfig"].get("NanoCpus", 0),
"memory_limit_bytes": data["HostConfig"].get("Memory", 0),
})
return {
"utc_started_at": dt.datetime.now(dt.timezone.utc).isoformat(),
"git_commit": command(["git", "rev-parse", "HEAD"]).stdout.strip(),
"working_tree_dirty": bool(command(["git", "status", "--porcelain"]).stdout.strip()),
"logical_cpu_count": os.cpu_count(),
"host_memory_mib": round(mem_total_kib / 1024, 1),
"docker_engine": command(["docker", "version", "--format", "{{.Server.Version}}"]).stdout.strip(),
"docker_compose": command(["docker", "compose", "version", "--short"]).stdout.strip(),
"container_limits": limits,
}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--env-file", default=".env")
parser.add_argument("--concurrency", type=int, default=8)
parser.add_argument("--baseline-seconds", type=float, default=12)
parser.add_argument("--action-after-seconds", type=float, default=3)
parser.add_argument("--post-action-seconds", type=float, default=8)
parser.add_argument("--output", default="evidence/grade4/results.json")
args = parser.parse_args()
if not 1 <= args.concurrency <= 64:
raise SystemExit("concurrency must be 1..64")
original = wait_api(args.env_file, 2)
if len(original) != 2:
raise RuntimeError("experiment requires exactly two initial API replicas")
client = Client(args.base_url, timeout=5)
alice, _, _, chat = client.bootstrap()
for index in range(12):
client.message(alice, chat, f"grade4-warmup-{index}")
phases = [load_phase("baseline", alice, chat, args)]
targets = [(container_id, inspect(container_id)["Config"]["Hostname"])
for container_id in original]
for ordinal, (container_id, target) in enumerate(targets, start=1):
def stop_target(instance=target):
hook_env = dict(os.environ)
hook_env.update({"ENV_FILE": args.env_file, "TARGET_INSTANCE_ID": instance})
result = command([ROOT / "scripts" / "stop-one.sh"], env=hook_env, timeout=30)
return result.stdout.strip().splitlines()[-1]
phases.append(load_phase(
f"failover-{ordinal}", alice, chat, args,
action=stop_target, target=target,
))
command(["docker", "start", container_id])
wait_api(args.env_file, 2)
idem = idempotency(alice, chat, args.base_url)
def rolling():
hook_env = dict(os.environ)
hook_env.update({"ENV_FILE": args.env_file, "DESIRED_REPLICAS": "2"})
result = command(
[ROOT / "scripts" / "rolling-update.sh"],
env=hook_env,
timeout=240,
)
return result.stdout.strip().splitlines()[-1]
phases.append(load_phase("rolling-update", alice, chat, args, action=rolling))
wait_api(args.env_file, 2)
report = {
"schema_version": 1,
"environment": environment(args.env_file),
"methodology": {
"base_url": args.base_url,
"concurrency": args.concurrency,
"baseline_seconds": args.baseline_seconds,
"action_after_seconds": args.action_after_seconds,
"post_action_seconds": args.post_action_seconds,
"warmup_requests": 12,
"workload": "closed-loop POST messages via Nginx; no automatic retries",
},
"phases": phases,
"idempotency": idem,
"dataset": {
"warmup_requests": 12,
"minimum_confirmed_messages": 13 + sum(
phase["successful_requests"] for phase in phases
),
},
"limitations": [
"single Docker host",
"closed-loop coordinated omission",
"successful-response latency percentiles only",
"PostgreSQL and Nginx remain single points of failure",
],
}
destination = ROOT / args.output
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
try:
displayed_output = destination.relative_to(ROOT)
except ValueError:
displayed_output = destination
print(json.dumps({
"output": str(displayed_output),
"phases": [{
"name": phase["name"],
"successful_rps": phase["successful_rps"],
"failed_requests": phase["failed_requests"],
"recovery_seconds": phase["recovery_seconds"],
} for phase in phases],
"idempotency": idem,
}, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+203
View File
@@ -0,0 +1,203 @@
#!/usr/bin/env python3
"""Render portable SVG graphs and a Markdown summary from grade-4 results."""
import argparse
from collections import defaultdict
import html
import json
from pathlib import Path
STYLE = """
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
.title { font-size: 26px; font-weight: 650; }
.label { font-size: 14px; fill: #c8cbd0; }
.small { font-size: 12px; fill: #a2a5aa; }
.grid { stroke: #34373d; stroke-width: 1; }
.bar { fill: #cfd2d6; }
.bar2 { fill: #73777f; }
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
.error { fill: #8f949c; stroke: #f4f5f6; stroke-width: 1; }
.event { stroke: #a2a5aa; stroke-width: 2; stroke-dasharray: 6 5; }
"""
def svg(title, body, width=1100, height=560):
return f"""<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" viewBox="0 0 {width} {height}">
<rect width="100%" height="100%" fill="#0e1013"/>
<style>{STYLE}</style>
<text x="48" y="42" class="title">{html.escape(title)}</text>
{body}
</svg>
"""
def grid(maximum, left, top, width, height):
parts = []
for step in range(6):
value = maximum * step / 5
y = top + height - height * step / 5
parts.append(f'<line x1="{left}" y1="{y:.1f}" x2="{left+width}" y2="{y:.1f}" class="grid"/>')
parts.append(f'<text x="{left-12}" y="{y+4:.1f}" text-anchor="end" class="small">{value:.0f}</text>')
return parts
def throughput(phases, output):
left, top, width, height = 80, 90, 960, 360
maximum = max((phase["successful_rps"] for phase in phases), default=1) or 1
parts = grid(maximum, left, top, width, height)
slot = width / max(1, len(phases))
for index, phase in enumerate(phases):
bar_width = min(120, slot * 0.5)
x = left + slot * index + (slot - bar_width) / 2
bar_height = height * phase["successful_rps"] / maximum
y = top + height - bar_height
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_width:.1f}" height="{bar_height:.1f}" rx="10" class="bar"/>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{y-10:.1f}" text-anchor="middle" class="label">{phase["successful_rps"]:.1f} RPS</text>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+28}" text-anchor="middle" class="label">{html.escape(phase["name"])}</text>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+48}" text-anchor="middle" class="small">errors {phase["error_ratio"]*100:.2f}%</text>')
parts.append(f'<text x="24" y="{top+height/2}" transform="rotate(-90 24 {top+height/2})" class="label">Successful requests per second</text>')
output.write_text(svg("Grade 4: throughput and errors", "\n".join(parts)))
def latency(phases, output):
left, top, width, height = 80, 90, 960, 360
values = [value for phase in phases
for value in phase["successful_latency_ms"].values()
if value is not None]
maximum = max(values, default=1) or 1
parts = grid(maximum, left, top, width, height)
slot = width / max(1, len(phases))
for phase_index, phase in enumerate(phases):
group_x = left + slot * phase_index
bar_width = min(42, slot / 5)
for metric_index, metric in enumerate(("p50", "p95", "p99")):
value = phase["successful_latency_ms"][metric] or 0
bar_height = height * value / maximum
x = group_x + slot * 0.2 + metric_index * (bar_width + 8)
y = top + height - bar_height
css = "bar" if metric_index == 0 else "bar2"
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_width:.1f}" height="{bar_height:.1f}" rx="7" class="{css}"/>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{y-7:.1f}" text-anchor="middle" class="small">{value:.0f}</text>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+20}" text-anchor="middle" class="small">{metric}</text>')
parts.append(f'<text x="{group_x+slot/2:.1f}" y="{top+height+48}" text-anchor="middle" class="label">{html.escape(phase["name"])}</text>')
parts.append(f'<text x="24" y="{top+height/2}" transform="rotate(-90 24 {top+height/2})" class="label">Successful latency, ms</text>')
output.write_text(svg("Grade 4: successful latency percentiles", "\n".join(parts)))
def timeline(phases, output):
left, top, width, height = 70, 90, 980, 390
rows, events, offset = [], [], 0
for phase in phases:
buckets = phase["per_second"]
for second, counts in sorted(buckets.items(), key=lambda item: int(item[0])):
rows.append((offset + int(second), counts))
if phase.get("action"):
events.append((offset + phase["action"]["started_at"], phase["name"]))
offset += max([int(key) for key in buckets] or [0]) + 2
maximum = max((counts["successful"] for _, counts in rows), default=1) or 1
max_x = max((second for second, _ in rows), default=1) or 1
parts, points = grid(maximum, left, top, width, height), []
for second, counts in rows:
x = left + width * second / max_x
y = top + height - height * counts["successful"] / maximum
points.append(f"{x:.1f},{y:.1f}")
if counts["failed"]:
radius = min(12, 3 + counts["failed"])
parts.append(f'<circle cx="{x:.1f}" cy="{top+height-8}" r="{radius}" class="error"/>')
parts.append(f'<polyline points="{" ".join(points)}" class="line"/>')
for second, label in events:
x = left + width * second / max_x
parts.append(f'<line x1="{x:.1f}" y1="{top}" x2="{x:.1f}" y2="{top+height}" class="event"/>')
parts.append(f'<text x="{x+6:.1f}" y="{top+18}" class="small">{html.escape(label)} action</text>')
parts.append(f'<text x="{left}" y="{top+height+32}" class="label">One-second buckets; circles mark failed requests</text>')
output.write_text(svg("Grade 4: request timeline", "\n".join(parts), height=600))
def resources(phases, output):
combined = defaultdict(lambda: {"cpu": [], "memory": []})
for phase in phases:
for name, values in phase["resources"].items():
combined[name]["cpu"].append(values["cpu_percent_max"])
combined[name]["memory"].append(values["memory_mib_max"])
names, parts = sorted(combined), []
panels = [
("Peak CPU, %", "cpu", 80, 90, 420, 360),
("Peak memory, MiB", "memory", 600, 90, 420, 360),
]
for title, metric, left, top, width, height in panels:
maximum = max((max(combined[name][metric]) for name in names), default=1) or 1
parts.append(f'<text x="{left}" y="{top-18}" class="label">{title}</text>')
slot = width / max(1, len(names))
for index, name in enumerate(names):
value = max(combined[name][metric])
bar_width = min(70, slot * 0.55)
x = left + slot * index + (slot - bar_width) / 2
bar_height = height * value / maximum
y = top + height - bar_height
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_width:.1f}" height="{bar_height:.1f}" rx="9" class="bar"/>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{y-8:.1f}" text-anchor="middle" class="small">{value:.1f}</text>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+25}" text-anchor="middle" class="small">{html.escape(name)}</text>')
output.write_text(svg("Grade 4: peak container resources", "\n".join(parts), height=600))
def summary(report, output):
method = report["methodology"]
lines = [
"# Lab 3 grade-4 experiment", "",
f"Generated: {report['environment']['utc_started_at']}.", "",
"## Method", "",
f"- concurrency: {method['concurrency']};",
f"- baseline duration: {method['baseline_seconds']} s;",
"- load: closed-loop POST messages through Nginx;",
"- failover: both original API containers stopped and restored in turn;",
"- rolling update: one surge replica, old replicas replaced one by one;",
"- percentiles cover successful responses; failures are separate.", "",
"## Results", "",
"| Phase | RPS | Errors | p50 ms | p95 ms | p99 ms | Recovery s |",
"| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
]
for phase in report["phases"]:
lat = phase["successful_latency_ms"]
recovery = phase["recovery_seconds"]
lines.append(
f"| {phase['name']} | {phase['successful_rps']} | "
f"{phase['failed_requests']} ({phase['error_ratio']*100:.3f}%) | "
f"{lat['p50']} | {lat['p95']} | {lat['p99']} | "
f"{recovery if recovery is not None else 'n/a'} |"
)
idem = report["idempotency"]
lines += [
"", "## Parallel idempotency", "",
f"{idem['parallel_requests']} requests produced "
f"{idem['unique_message_ids']} unique message ID across "
f"{len(idem['observed_instances'])} API instances.", "",
"## Graphs", "",
"![Throughput and errors](throughput-errors.svg)", "",
"![Latency](latency.svg)", "",
"![Timeline](timeline.svg)", "",
"![Resources](resources.svg)", "",
"## Scope", "",
"Single-host process failure and rolling replacement only. PostgreSQL "
"and Nginx remain single points of failure.", "",
]
output.write_text("\n".join(lines))
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("results", nargs="?", default="evidence/grade4/results.json")
args = parser.parse_args()
source = Path(args.results)
report = json.loads(source.read_text())
destination = source.parent
phases = report["phases"]
throughput(phases, destination / "throughput-errors.svg")
latency(phases, destination / "latency.svg")
timeline(phases, destination / "timeline.svg")
resources(phases, destination / "resources.svg")
summary(report, destination / "README.md")
print(f"rendered grade-4 evidence in {destination}")
if __name__ == "__main__":
main()
+89
View File
@@ -0,0 +1,89 @@
#!/bin/sh
set -eu
project_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$project_root"
env_file=${ENV_FILE:-.env}
desired_replicas=${DESIRED_REPLICAS:-2}
wait_timeout=${ROLLING_WAIT_TIMEOUT:-90}
stop_timeout=${ROLLING_STOP_TIMEOUT:-15}
dns_settle_delay=${ROLLING_DNS_SETTLE_DELAY:-2}
case "$desired_replicas" in
''|*[!0-9]*)
printf '%s\n' "DESIRED_REPLICAS must be an integer" >&2
exit 2
;;
esac
case "$dns_settle_delay" in
''|*[!0-9]*)
printf '%s\n' "ROLLING_DNS_SETTLE_DELAY must be a non-negative integer" >&2
exit 2
;;
esac
if [ "$desired_replicas" -lt 2 ]; then
printf '%s\n' "rolling update requires at least two desired replicas" >&2
exit 2
fi
compose() {
docker compose --env-file "$env_file" "$@"
}
healthy_api_count() {
healthy=0
for container_id in $(compose ps -q api); do
status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container_id")
if [ "$status" = "healthy" ] || [ "$status" = "running" ]; then
healthy=$((healthy + 1))
fi
done
printf '%s\n' "$healthy"
}
wait_for_healthy() {
expected=$1
deadline=$(( $(date +%s) + wait_timeout ))
while [ "$(healthy_api_count)" -lt "$expected" ]; do
if [ "$(date +%s)" -ge "$deadline" ]; then
printf '%s\n' "timed out waiting for $expected healthy API replicas" >&2
return 1
fi
sleep 1
done
}
old_containers=$(compose ps -q api)
old_count=$(printf '%s\n' "$old_containers" | awk 'NF { count++ } END { print count+0 }')
if [ "$old_count" -ne "$desired_replicas" ]; then
printf '%s\n' "expected $desired_replicas running API containers, found $old_count" >&2
exit 1
fi
surge_replicas=$((desired_replicas + 1))
printf '%s\n' "building replacement API image"
compose build api
printf '%s\n' "starting one surge replica"
compose up -d --no-deps --scale api="$surge_replicas" --no-recreate api
wait_for_healthy "$surge_replicas"
sleep "$dns_settle_delay"
for old_container in $old_containers; do
old_instance=$(docker inspect --format '{{.Config.Hostname}}' "$old_container")
printf '%s\n' "draining and replacing API instance $old_instance"
docker stop --time "$stop_timeout" "$old_container" >/dev/null
docker rm "$old_container" >/dev/null
compose up -d --no-deps --scale api="$surge_replicas" --no-recreate api
wait_for_healthy "$surge_replicas"
sleep "$dns_settle_delay"
done
printf '%s\n' "removing surge capacity"
compose up -d --no-deps --scale api="$desired_replicas" --no-recreate api
wait_for_healthy "$desired_replicas"
sleep "$dns_settle_delay"
printf '%s\n' "rolling update completed with $desired_replicas healthy API replicas"