forked from mtusi2026/mtusi2026lab4
Перенесён стенд lab3 (API, Redis-сессии, идемпотентность, HAProxy, реплики API). Добавлены приватный bucket SeaweedFS с отдельным ключом API, загрузка PNG/JPEG с лимитами и полной проверкой, компенсация и reconciler, плавная остановка API, тесты и evidence.
285 lines
12 KiB
Python
285 lines
12 KiB
Python
#!/usr/bin/env python3
|
||
"""Experiment: peak memory of API replicas, LB and S3 under N simultaneous ~9.5 MiB uploads.
|
||
|
||
Hypothesis: streaming intake (64 KiB chunks → validator → temp file → S3 PUT from
|
||
file) keeps the API process RSS growth far below N × file size; total in-flight
|
||
work per replica is capped by MAX_CONCURRENT_UPLOADS.
|
||
|
||
Method (per concurrency level, fresh replicas):
|
||
1. `docker compose restart s3`, then `s3-gw api lb` → new processes; wait for /health/ready;
|
||
2. one warm-up upload per replica (imports, pools, TLS-less S3 connections);
|
||
3. reset VmHWM of PID 1 (echo 5 > /proc/1/clear_refs) and record baseline;
|
||
4. fire N uploads at once through the LB (barrier), sample cgroup memory.stat
|
||
(anon/file) of every replica, the LB and the s3 container from the host every ~0.25 s;
|
||
5. read VmHWM (peak RSS of the uvicorn process) and cgroup memory.peak.
|
||
|
||
Usage: python scripts/upload_memory_experiment.py [--levels 10 20] [--size-mib 9.5]
|
||
Writes evidence/upload-memory.json. Tokens are kept in memory only.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import base64
|
||
import json
|
||
import os
|
||
import statistics
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
import uuid
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||
import imagegen # noqa: E402
|
||
|
||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
||
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
||
|
||
|
||
def sh(args, check=True) -> str:
|
||
proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8")
|
||
if check and proc.returncode != 0:
|
||
raise RuntimeError(f"{args[:4]} rc={proc.returncode}: {proc.stderr[-300:]}")
|
||
return proc.stdout
|
||
|
||
|
||
def request(method, path, body=None, token=None, raw=None, headers=None, timeout=180):
|
||
h = {"Accept": "application/json"}
|
||
if token:
|
||
h["Authorization"] = "Bearer " + token
|
||
if body is not None:
|
||
raw = json.dumps(body).encode()
|
||
h["Content-Type"] = "application/json"
|
||
h.update(headers or {})
|
||
req = urllib.request.Request(BASE + path, data=raw, headers=h, method=method)
|
||
try:
|
||
resp = urllib.request.urlopen(req, timeout=timeout)
|
||
except urllib.error.HTTPError as err:
|
||
resp = err
|
||
data = resp.read()
|
||
return resp.code, resp.headers, data
|
||
|
||
|
||
def actor(label):
|
||
username = f"mem_{label}_{uuid.uuid4().hex[:10]}"
|
||
password = "T3st-" + uuid.uuid4().hex
|
||
status, _, data = request("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
|
||
assert status == 201, status
|
||
user = json.loads(data)
|
||
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||
status, _, data = request("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
|
||
assert status == 201, status
|
||
return user["id"], json.loads(data)["token"]
|
||
|
||
|
||
def wait_ready(timeout=120):
|
||
deadline = time.time() + timeout
|
||
while time.time() < deadline:
|
||
try:
|
||
if request("GET", "/health/ready", timeout=3)[0] == 200:
|
||
seen = {request("GET", "/health/live", timeout=3)[1].get("X-Instance-Id") for _ in range(20)}
|
||
if len(seen) >= 2:
|
||
return
|
||
except OSError:
|
||
pass
|
||
time.sleep(1)
|
||
raise RuntimeError("stack not ready")
|
||
|
||
|
||
def containers():
|
||
ids = sh(COMPOSE + ["ps", "-q", "api"]).split()
|
||
return {sh(["docker", "inspect", "-f", "{{.Config.Hostname}}", c]).strip(): c for c in ids}
|
||
|
||
|
||
def proc_status(cid):
|
||
out = sh(["docker", "exec", cid, "cat", "/proc/1/status"])
|
||
fields = {}
|
||
for line in out.splitlines():
|
||
if line.startswith(("VmRSS", "VmHWM")):
|
||
k, v = line.split(":", 1)
|
||
fields[k] = int(v.split()[0]) // 1024 # MiB
|
||
return fields
|
||
|
||
|
||
def cgroup(cid):
|
||
stat = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.stat"])
|
||
vals = dict(line.split() for line in stat.splitlines() if line)
|
||
peak = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.peak"], check=False).strip()
|
||
current = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.current"]).strip()
|
||
mib = 1024 * 1024
|
||
return {
|
||
"anon_mib": round(int(vals["anon"]) / mib, 1),
|
||
"file_mib": round(int(vals["file"]) / mib, 1),
|
||
"current_mib": round(int(current) / mib, 1),
|
||
"peak_mib": round(int(peak) / mib, 1) if peak.isdigit() else None,
|
||
}
|
||
|
||
|
||
def sampler(cid, stop, out):
|
||
best = {"anon_mib": 0.0, "file_mib": 0.0, "current_mib": 0.0, "samples": 0}
|
||
while not stop.is_set():
|
||
try:
|
||
c = cgroup(cid)
|
||
for k in ("anon_mib", "file_mib", "current_mib"):
|
||
best[k] = max(best[k], c[k])
|
||
best["samples"] += 1
|
||
except Exception:
|
||
pass
|
||
time.sleep(0.25)
|
||
out[cid] = best
|
||
|
||
|
||
def pct(values, q):
|
||
values = sorted(values)
|
||
return round(values[min(len(values) - 1, int(round(q * (len(values) - 1))))], 3)
|
||
|
||
|
||
def run_level(n, payload, chat_id, token):
|
||
print(f"== level {n}: restarting s3, api replicas and lb", flush=True)
|
||
# s3-gw shares the s3 network namespace and must be restarted after it.
|
||
sh(COMPOSE + ["restart", "s3"])
|
||
sh(COMPOSE + ["restart", "s3-gw", "api", "lb"])
|
||
wait_ready()
|
||
by_host = containers()
|
||
lb = sh(COMPOSE + ["ps", "-q", "lb"]).split()[0]
|
||
s3c = sh(COMPOSE + ["ps", "-q", "s3"]).split()[0]
|
||
# Warm-up: at least one upload served by every replica.
|
||
served = set()
|
||
for _ in range(20):
|
||
_, h, _ = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=warm.png", token=token,
|
||
raw=imagegen.rgb_png(64, 64), headers={"Content-Type": "image/png"})
|
||
served.add(h.get("X-Instance-Id"))
|
||
if served >= set(by_host):
|
||
break
|
||
baseline = {}
|
||
for host, cid in by_host.items():
|
||
sh(["docker", "exec", cid, "sh", "-c", "echo 5 > /proc/1/clear_refs"])
|
||
baseline[host] = {**proc_status(cid), **cgroup(cid)}
|
||
lb_baseline = cgroup(lb)
|
||
s3_baseline = cgroup(s3c)
|
||
stop = threading.Event()
|
||
sampled = {}
|
||
threads = [threading.Thread(target=sampler, args=(cid, stop, sampled)) for cid in [*by_host.values(), lb, s3c]]
|
||
for t in threads:
|
||
t.start()
|
||
time.sleep(0.5)
|
||
barrier = threading.Barrier(n)
|
||
results = [None] * n
|
||
|
||
def one(i):
|
||
barrier.wait()
|
||
t0 = time.monotonic()
|
||
try:
|
||
status, h, _ = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=load{i}.png",
|
||
token=token, raw=payload, headers={"Content-Type": "image/png"})
|
||
results[i] = (status, time.monotonic() - t0, h.get("X-Instance-Id"))
|
||
except OSError as exc:
|
||
results[i] = (type(exc).__name__, time.monotonic() - t0, None)
|
||
|
||
workers = [threading.Thread(target=one, args=(i,)) for i in range(n)]
|
||
started = time.monotonic()
|
||
for w in workers:
|
||
w.start()
|
||
for w in workers:
|
||
w.join()
|
||
wall = time.monotonic() - started
|
||
time.sleep(1.0)
|
||
stop.set()
|
||
for t in threads:
|
||
t.join()
|
||
replicas = {}
|
||
for host, cid in by_host.items():
|
||
after = proc_status(cid)
|
||
cg = cgroup(cid)
|
||
replicas[host] = {
|
||
"uploads_served": sum(1 for r in results if r[2] == host),
|
||
"rss_baseline_mib": baseline[host]["VmRSS"],
|
||
"rss_peak_vmhwm_mib": after["VmHWM"],
|
||
"rss_peak_growth_mib": after["VmHWM"] - baseline[host]["VmRSS"],
|
||
"cgroup_peak_mib": cg["peak_mib"],
|
||
"cgroup_baseline_current_mib": baseline[host]["current_mib"],
|
||
"sampled_max_anon_mib": sampled.get(cid, {}).get("anon_mib"),
|
||
"sampled_max_file_cache_mib": sampled.get(cid, {}).get("file_mib"),
|
||
"samples": sampled.get(cid, {}).get("samples"),
|
||
}
|
||
latencies = [r[1] for r in results if r[0] == 201]
|
||
statuses = {}
|
||
for r in results:
|
||
statuses[str(r[0])] = statuses.get(str(r[0]), 0) + 1
|
||
per_replica_max = max(v["uploads_served"] for v in replicas.values())
|
||
return {
|
||
"concurrency": n,
|
||
"payload_bytes": len(payload),
|
||
"statuses": statuses,
|
||
"wall_seconds": round(wall, 3),
|
||
"latency_ok_p50_s": pct(latencies, 0.5) if latencies else None,
|
||
"latency_ok_p95_s": pct(latencies, 0.95) if latencies else None,
|
||
"latency_ok_max_s": round(max(latencies), 3) if latencies else None,
|
||
"throughput_mib_s": round(len(latencies) * len(payload) / (1024 * 1024) / wall, 1),
|
||
"naive_full_buffer_estimate_mib": round(min(per_replica_max, 8) * len(payload) / (1024 * 1024), 1),
|
||
"replicas": replicas,
|
||
"lb": {
|
||
"mem_limit_mib": int(sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}}", lb]).strip()) // (1024 * 1024),
|
||
"baseline_current_mib": lb_baseline["current_mib"],
|
||
"cgroup_peak_mib": cgroup(lb)["peak_mib"],
|
||
"sampled_max_anon_mib": sampled.get(lb, {}).get("anon_mib"),
|
||
"sampled_max_current_mib": sampled.get(lb, {}).get("current_mib"),
|
||
"restarts": int(sh(["docker", "inspect", "-f", "{{.RestartCount}}", lb]).strip()),
|
||
"oom_killed": sh(["docker", "inspect", "-f", "{{.State.OOMKilled}}", lb]).strip() == "true",
|
||
},
|
||
"s3": _s3_report(s3c, s3_baseline, sampled.get(s3c, {})),
|
||
}
|
||
|
||
|
||
def _s3_report(cid, baseline, sampled):
|
||
running = sh(["docker", "inspect", "-f", "{{.State.Running}}", cid]).strip() == "true"
|
||
return {
|
||
"mem_limit_mib": int(sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}}", cid]).strip()) // (1024 * 1024),
|
||
"baseline_current_mib": baseline["current_mib"],
|
||
# memory.peak includes reclaimable page cache of the volume files; anon is the weed heap.
|
||
"cgroup_peak_mib": cgroup(cid)["peak_mib"] if running else None,
|
||
"sampled_max_anon_mib": sampled.get("anon_mib"),
|
||
"sampled_max_current_mib": sampled.get("current_mib"),
|
||
"running": running,
|
||
"oom_killed": sh(["docker", "inspect", "-f", "{{.State.OOMKilled}}", cid]).strip() == "true",
|
||
}
|
||
|
||
|
||
def main():
|
||
p = argparse.ArgumentParser(description="peak memory under concurrent uploads")
|
||
p.add_argument("--levels", type=int, nargs="+", default=[10, 20])
|
||
p.add_argument("--size-mib", type=float, default=9.5)
|
||
p.add_argument("--output", default=str(ROOT / "evidence" / "upload-memory.json"))
|
||
args = p.parse_args()
|
||
payload = imagegen.large_png(int(args.size_mib * 1024 * 1024))
|
||
_, token = actor("a")
|
||
bob_id, _ = actor("b")
|
||
me = json.loads(request("GET", "/api/v1/users/me", token=token)[2])
|
||
status, _, data = request("POST", "/api/v1/chats", {"title": "memory", "member_ids": [me["id"], bob_id]}, token=token)
|
||
assert status == 201, status
|
||
chat_id = json.loads(data)["id"]
|
||
limits = sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}} {{.HostConfig.NanoCpus}}",
|
||
list(containers().values())[0]]).split()
|
||
report = {
|
||
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||
"api_mem_limit_mib": int(limits[0]) // (1024 * 1024),
|
||
"api_cpus": int(limits[1]) / 1e9,
|
||
"max_concurrent_uploads_per_replica": 8,
|
||
"levels": [],
|
||
}
|
||
for n in args.levels:
|
||
level = run_level(n, payload, chat_id, token)
|
||
report["levels"].append(level)
|
||
print(json.dumps(level, ensure_ascii=False), flush=True)
|
||
Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
|
||
print("WROTE", args.output)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|