forked from mtusi2026/mtusi2026lab3
Reject steps disturbed by foreign load on the shared VM
The first run of series A was spoiled by other stands on the same Docker VM: an image build of another lab is invisible to docker stats, yet it took CPU from the pinned replicas, doubled their CPU per request and stalled the Docker API for fifteen minutes in the middle of two steps. docker-stats.sh now also samples the VM-wide /proc/stat counters, so a step records the CPU of the whole VM, the part not used by this project, the largest gap between samples and its wall time. A step with more than one foreign core, a gap over 10 s or a k6 run 90 s longer than its load is written to disturbed.csv and repeated after the VM has been quiet for 15 s, up to three times. On a quiet VM the foreign part is 2 % of a core at 600 rps and 25 % at 4 560 rps, which is Docker networking and the samplers themselves. The growth series starts at 300 rps per CPU and searches up to 9 000, so its knees are found with fewer idle steps. The plots fit the universal scalability law to the growth series. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
e8804b05d0
commit
67f6d6866e
@@ -12,7 +12,7 @@ API_IMAGE ?= mtusi-messenger:lab3
|
||||
FROM_REV ?=
|
||||
SCRIPTS_PY = scripts/probe.py scripts/limits.py scripts/scaling.py tools
|
||||
TRACK1 = evidence/track1
|
||||
SCALING = $(PYTHON) scripts/scaling.py --fresh-db --start-per-cpu 150
|
||||
SCALING = $(PYTHON) scripts/scaling.py --fresh-db
|
||||
PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover \
|
||||
@@ -124,10 +124,12 @@ track1-host:
|
||||
sh scripts/host-snapshot.sh $(TRACK1)/host.txt
|
||||
|
||||
track1-a:
|
||||
$(SCALING) --out $(TRACK1)/series-a --configs 1x4,2x2,4x1 --api-memory-total 2048
|
||||
$(SCALING) --out $(TRACK1)/series-a --configs 1x4,2x2,4x1 --api-memory-total 2048 \
|
||||
--start-per-cpu 150
|
||||
|
||||
track1-b:
|
||||
$(SCALING) --out $(TRACK1)/series-b --configs 1x1,2x1,8x1 --api-memory 512
|
||||
$(SCALING) --out $(TRACK1)/series-b --configs 1x1,2x1,8x1 --api-memory 512 \
|
||||
--start-per-cpu 300 --search 1.5:9000
|
||||
|
||||
plots:
|
||||
MPLBACKEND=Agg $(UV) run --no-project --python 3.14 --with 'matplotlib==3.11.2' \
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
out=${1:?usage: docker-stats.sh <csv>}
|
||||
vm=$(dirname "$out")/vm_stat.csv
|
||||
cd "$(dirname "$0")/.."
|
||||
project=$(docker compose config | sed -n 's/^name: //p')
|
||||
echo 'ts,name,cpu_pct,mem_usage,mem_pct,net_io,block_io,pids' > "$out"
|
||||
echo 'ts,busy,total,cpus' > "$vm"
|
||||
while :; do
|
||||
ts=$(date +%s)
|
||||
docker compose exec -T valkey cat /proc/stat 2>/dev/null | awk -v ts="$ts" '/^cpu /{b=$2+$3+$4+$7+$8+$9; t=b+$5+$6} /^cpu[0-9]/{n++} END{if (t) print ts "," b "," t "," n}' >> "$vm" || true
|
||||
ids=$(docker ps -q --filter "label=com.docker.compose.project=$project")
|
||||
[ -z "$ids" ] || docker stats --no-stream --format "$(date +%s),{{.Name}},{{.CPUPerc}},{{.MemUsage}},{{.MemPerc}},{{.NetIO}},{{.BlockIO}},{{.PIDs}}" $ids >> "$out" || true
|
||||
[ -z "$ids" ] || docker stats --no-stream --format "$ts,{{.Name}},{{.CPUPerc}},{{.MemUsage}},{{.MemPerc}},{{.NetIO}},{{.BlockIO}},{{.PIDs}}" $ids >> "$out" || true
|
||||
sleep "${INTERVAL:-1}"
|
||||
done
|
||||
|
||||
+73
-11
@@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import collections
|
||||
import contextlib
|
||||
import csv
|
||||
import json
|
||||
import math
|
||||
@@ -32,7 +33,8 @@ FIELDS = (
|
||||
"dropped_iterations p50_ms p95_ms p99_ms p95_post_ms p95_get_ms cpu_api cpu_postgres "
|
||||
"cpu_lb cpu_valkey cpu_k6 mem_api_mb mem_postgres_mb mem_k6_mb pg_backends_max "
|
||||
"pg_active_avg pg_active_max pg_idle_avg pg_idle_tx_avg pg_lock_avg pg_lock_max pg_lwlock_avg "
|
||||
"pg_io_avg lb_scur_max lb_qcur_max lb_qtime_max_ms lb_rtime_avg_ms"
|
||||
"pg_io_avg lb_scur_max lb_qcur_max lb_qtime_max_ms lb_rtime_avg_ms cpu_vm cpu_other "
|
||||
"gap_s wall_s disturbed"
|
||||
).split()
|
||||
|
||||
|
||||
@@ -73,6 +75,38 @@ def docker_usage(path, project, start, end):
|
||||
return usage
|
||||
|
||||
|
||||
def vm_usage(path, start, end):
|
||||
samples = [(int(r["busy"]), int(r["total"]), int(r["cpus"])) for r in rows(path, start, end)]
|
||||
if len(samples) < 2 or samples[-1][1] == samples[0][1]:
|
||||
return 0.0
|
||||
first, last = samples[0], samples[-1]
|
||||
return round(100 * last[2] * (last[0] - first[0]) / (last[1] - first[1]), 1)
|
||||
|
||||
|
||||
def gap(path, start, end):
|
||||
ticks = sorted({float(r["ts"]) for r in rows(path, start - 5, end + 5)})
|
||||
edges = [start, *[t for t in ticks if start <= t <= end], end]
|
||||
return max(b - a for a, b in zip(edges, edges[1:]))
|
||||
|
||||
|
||||
def vm_counters():
|
||||
lines = compose("exec", "-T", "valkey", "cat", "/proc/stat").splitlines()
|
||||
values = [int(v) for v in lines[0].split()[1:]]
|
||||
cpus = sum(1 for line in lines if re.match(r"cpu\d", line))
|
||||
return sum(values) - values[3] - values[4], sum(values), cpus
|
||||
|
||||
|
||||
def wait_quiet(args):
|
||||
deadline, calm = time.monotonic() + args.quiet_timeout, 0
|
||||
while calm < 3 and time.monotonic() < deadline:
|
||||
first = vm_counters()
|
||||
time.sleep(5)
|
||||
last = vm_counters()
|
||||
busy = 100 * last[2] * (last[0] - first[0]) / max(1, last[1] - first[1])
|
||||
calm = calm + 1 if busy < args.max_other else 0
|
||||
print(json.dumps({"waiting_for_quiet_vm_cpu": round(busy, 1)}), flush=True)
|
||||
|
||||
|
||||
def pg_usage(path, start, end):
|
||||
ticks = collections.defaultdict(collections.Counter)
|
||||
for r in rows(path, start, end):
|
||||
@@ -192,9 +226,11 @@ def measure(folder, args, rate, env):
|
||||
sampler("pg-stats.sh", "activity", str(folder / "pg_activity.csv")),
|
||||
]
|
||||
load = dict(env, RATE=str(rate), WARMUP=str(args.warmup), DURATION=str(args.duration))
|
||||
began = time.time()
|
||||
try:
|
||||
code, started = run_k6(folder / "k6", load)
|
||||
finally:
|
||||
wall = time.time() - began
|
||||
for process in samplers:
|
||||
stop(process)
|
||||
subprocess.run(
|
||||
@@ -202,13 +238,13 @@ def measure(folder, args, rate, env):
|
||||
)
|
||||
if started is None:
|
||||
raise SystemExit(f"k6 failed in setup, see {folder / 'k6/k6.txt'}")
|
||||
return code, started + args.warmup, database
|
||||
return code, started + args.warmup, database, wall
|
||||
|
||||
|
||||
def step(base, rate, repeat, args, env):
|
||||
folder = Path(args.out) / f"{base['variant']}-{base['config']}_rate{rate}_rep{repeat}"
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
code, start, database = measure(folder, args, rate, env)
|
||||
code, start, database, wall = measure(folder, args, rate, env)
|
||||
end = start + args.duration
|
||||
summary = folder / "k6/summary.json"
|
||||
metrics = k6_metrics(summary, args.duration)
|
||||
@@ -226,6 +262,16 @@ def step(base, rate, repeat, args, env):
|
||||
row.update(docker_usage(folder / "docker_stats.csv", args.project, start, end))
|
||||
row.update(pg_usage(folder / "pg_activity.csv", start, end))
|
||||
row.update(lb_usage(folder / "lb_stats.csv", start, end))
|
||||
row["cpu_vm"] = vm_usage(folder / "vm_stat.csv", start, end)
|
||||
own = sum(row[f"cpu_{name}"] for name in ("api", "postgres", "lb", "valkey", "k6"))
|
||||
row["cpu_other"] = round(row["cpu_vm"] - own, 1)
|
||||
row["gap_s"] = round(gap(folder / "docker_stats.csv", start, end), 1)
|
||||
row["wall_s"] = round(wall, 1)
|
||||
row["disturbed"] = (
|
||||
row["cpu_other"] > args.max_other
|
||||
or row["gap_s"] > args.max_gap
|
||||
or wall > args.warmup + args.duration + args.max_overhead
|
||||
)
|
||||
print(json.dumps({k: row[k] for k in FIELDS}, ensure_ascii=False), flush=True)
|
||||
return row
|
||||
|
||||
@@ -269,6 +315,11 @@ def arguments():
|
||||
parser.add_argument("--slo-p95-ms", type=float, default=500.0)
|
||||
parser.add_argument("--max-error-ratio", type=float, default=0.01)
|
||||
parser.add_argument("--keep", action="store_true", help="leave the last layout running")
|
||||
parser.add_argument("--max-other", type=float, default=100.0, help="foreign VM CPU, %% core")
|
||||
parser.add_argument("--max-gap", type=float, default=10.0, help="sampling gap, s")
|
||||
parser.add_argument("--max-overhead", type=float, default=90.0, help="k6 run beyond load, s")
|
||||
parser.add_argument("--retries", type=int, default=3, help="reruns of a disturbed step")
|
||||
parser.add_argument("--quiet-timeout", type=float, default=1200.0)
|
||||
args = parser.parse_args()
|
||||
args.variants = args.variants or ["base"]
|
||||
if len(args.variants) > 1 and not args.rates:
|
||||
@@ -282,8 +333,14 @@ def block(args, output, config, variant, repeats):
|
||||
for rate in rates(args, base):
|
||||
results = []
|
||||
for repeat in repeats:
|
||||
results.append(step(base, rate, repeat, args, current))
|
||||
time.sleep(args.pause)
|
||||
for _ in range(args.retries + 1):
|
||||
row = step(base, rate, repeat, args, current)
|
||||
time.sleep(args.pause)
|
||||
if not row["disturbed"]:
|
||||
break
|
||||
args.rejected.writerow(row)
|
||||
wait_quiet(args)
|
||||
results.append(row)
|
||||
output.writerows(results)
|
||||
streak = streak + 1 if not args.rates and all(r["saturated"] for r in results) else 0
|
||||
if streak == 2:
|
||||
@@ -303,19 +360,24 @@ def series(args, output):
|
||||
block(args, output, config, variant, [repeat])
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def table(path):
|
||||
fresh = not path.exists()
|
||||
with path.open("a", newline="", buffering=1) as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=FIELDS, extrasaction="ignore")
|
||||
if fresh:
|
||||
writer.writeheader()
|
||||
yield writer
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments()
|
||||
out = Path(args.out)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
args.project = json.loads(compose("config", "--format", "json"))["name"]
|
||||
args.base_env = dict(os.environ, **(PINNED if args.pin else {}))
|
||||
summary = out / "summary.csv"
|
||||
fresh = not summary.exists()
|
||||
try:
|
||||
with summary.open("a", newline="", buffering=1) as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=FIELDS, extrasaction="ignore")
|
||||
if fresh:
|
||||
writer.writeheader()
|
||||
with table(out / "summary.csv") as writer, table(out / "disturbed.csv") as args.rejected:
|
||||
series(args, writer)
|
||||
finally:
|
||||
if not args.keep:
|
||||
|
||||
+31
-3
@@ -9,6 +9,7 @@ import matplotlib.pyplot as plt
|
||||
ORDER = ("1x1", "2x1", "4x1", "8x1", "2x2", "1x4")
|
||||
PALETTE = ("#2a78d6", "#eb6834", "#1baf7a", "#eda100", "#e87ba4", "#008300", "#6250d6", "#e34948")
|
||||
MARKERS = ("o", "s", "^", "D", "v", "P", "X", "*")
|
||||
USL_POINTS = 3
|
||||
COMPONENTS = (("api", "API"), ("postgres", "PostgreSQL"), ("lb", "HAProxy"), ("k6", "k6"))
|
||||
INK, MUTED, GRID, SURFACE = "#0b0b0b", "#52514e", "#e4e3df", "#fcfcfb"
|
||||
SUMMARY = (
|
||||
@@ -170,6 +171,17 @@ def cpu(series, keys, groups, out):
|
||||
plt.close(figure)
|
||||
|
||||
|
||||
def usl(points):
|
||||
base = points[0][1] / points[0][0]
|
||||
pairs = [(n - 1, n * (n - 1), n * base / x - 1) for n, x in points]
|
||||
a, b, c = (sum(p[i] * p[j] for p in pairs) for i, j in ((0, 0), (0, 1), (1, 1)))
|
||||
d, e = (sum(p[i] * p[2] for p in pairs) for i in (0, 1))
|
||||
det = a * c - b * b
|
||||
sigma, kappa = (d * c - b * e) / det, (a * e - b * d) / det
|
||||
peak = ((1 - sigma) / kappa) ** 0.5 if kappa > 0 else None
|
||||
return base, sigma, kappa, peak
|
||||
|
||||
|
||||
def scaling(rows, out):
|
||||
by = {(r["series"], r["config"]): r for r in rows if r["variant"] == "base"}
|
||||
growth = [
|
||||
@@ -183,6 +195,15 @@ def scaling(rows, out):
|
||||
unit = growth[0]["peak_ok_rps"] / cpus[0]
|
||||
axis.plot(cpus, [unit * c for c in cpus], color=MUTED, linestyle="--", linewidth=1)
|
||||
axis.annotate("линейный рост", (cpus[-1], unit * cpus[-1]), color=MUTED, fontsize=9)
|
||||
fit = None
|
||||
if len(growth) >= USL_POINTS:
|
||||
fit = usl([(c, r["peak_ok_rps"]) for c, r in zip(cpus, growth, strict=True)])
|
||||
base, sigma, kappa, _ = fit
|
||||
grid = [1 + i * (cpus[-1] - 1) / 60 for i in range(61)]
|
||||
model = [base * n / (1 + sigma * (n - 1) + kappa * n * (n - 1)) for n in grid]
|
||||
axis.plot(
|
||||
grid, model, color=PALETTE[6], linewidth=1.5, label=f"USL: σ={sigma:.3f}, κ={kappa:.4f}"
|
||||
)
|
||||
axis.errorbar(
|
||||
cpus,
|
||||
[r["peak_ok_rps"] for r in growth],
|
||||
@@ -210,6 +231,7 @@ def scaling(rows, out):
|
||||
figure.tight_layout()
|
||||
figure.savefig(out / "scaling.png", dpi=150)
|
||||
plt.close(figure)
|
||||
return fit
|
||||
|
||||
|
||||
def utilisation(rows, out):
|
||||
@@ -251,7 +273,7 @@ def utilisation(rows, out):
|
||||
plt.close(figure)
|
||||
|
||||
|
||||
def write(rows, out):
|
||||
def write(rows, out, fit):
|
||||
with (out / "capacity.csv").open("w", newline="") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=SUMMARY, extrasaction="ignore")
|
||||
writer.writeheader()
|
||||
@@ -269,6 +291,12 @@ def write(rows, out):
|
||||
f"{r['cpu_api']:.0f} / {r['cpu_api_limit']} | {r['cpu_postgres']:.0f} | "
|
||||
f"{r['cpu_lb']:.0f} | {r['pg_lock_avg']:.1f} / {r['pool_total']} |"
|
||||
)
|
||||
if fit:
|
||||
_, sigma, kappa, peak = fit
|
||||
best = f"{peak:.1f}" if peak else "—"
|
||||
lines_.append(
|
||||
f"\nUSL по серии B: σ = {sigma:.4f}, κ = {kappa:.5f}, пик при N* = {best} CPU"
|
||||
)
|
||||
(out / "capacity.md").write_text("\n".join(lines_) + "\n")
|
||||
print("\n".join(lines_))
|
||||
|
||||
@@ -288,9 +316,9 @@ def main():
|
||||
chosen = [k for k in keys if k[0] == series]
|
||||
throughput(series, chosen, groups, out)
|
||||
cpu(series, chosen, groups, out)
|
||||
scaling(rows, out)
|
||||
fit = scaling(rows, out)
|
||||
utilisation(rows, out)
|
||||
write(rows, out)
|
||||
write(rows, out, fit)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user