Files
mtusi2026lab4/scripts/upload_memory_experiment.py
T
Mikhail Verkovykh 728e53cc55 Реализовать lab4 на оценку 4: SeaweedFS, вложения и согласованность БД/S3.
Перенесён стенд lab3 (API, Redis-сессии, идемпотентность, HAProxy, реплики API). Добавлены приватный bucket SeaweedFS с отдельным ключом API, загрузка PNG/JPEG с лимитами и полной проверкой, компенсация и reconciler, плавная остановка API, тесты и evidence.
2026-11-06 19:48:37 +03:00

285 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Experiment: peak memory of API replicas, LB and S3 under N simultaneous ~9.5 MiB uploads.
Hypothesis: streaming intake (64 KiB chunks → validator → temp file → S3 PUT from
file) keeps the API process RSS growth far below N × file size; total in-flight
work per replica is capped by MAX_CONCURRENT_UPLOADS.
Method (per concurrency level, fresh replicas):
1. `docker compose restart s3`, then `s3-gw api lb` → new processes; wait for /health/ready;
2. one warm-up upload per replica (imports, pools, TLS-less S3 connections);
3. reset VmHWM of PID 1 (echo 5 > /proc/1/clear_refs) and record baseline;
4. fire N uploads at once through the LB (barrier), sample cgroup memory.stat
(anon/file) of every replica, the LB and the s3 container from the host every ~0.25 s;
5. read VmHWM (peak RSS of the uvicorn process) and cgroup memory.peak.
Usage: python scripts/upload_memory_experiment.py [--levels 10 20] [--size-mib 9.5]
Writes evidence/upload-memory.json. Tokens are kept in memory only.
"""
from __future__ import annotations
import argparse
import base64
import json
import os
import statistics
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
import imagegen # noqa: E402
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
COMPOSE = ["docker", "compose", "--env-file", ".env"]
def sh(args, check=True) -> str:
proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8")
if check and proc.returncode != 0:
raise RuntimeError(f"{args[:4]} rc={proc.returncode}: {proc.stderr[-300:]}")
return proc.stdout
def request(method, path, body=None, token=None, raw=None, headers=None, timeout=180):
h = {"Accept": "application/json"}
if token:
h["Authorization"] = "Bearer " + token
if body is not None:
raw = json.dumps(body).encode()
h["Content-Type"] = "application/json"
h.update(headers or {})
req = urllib.request.Request(BASE + path, data=raw, headers=h, method=method)
try:
resp = urllib.request.urlopen(req, timeout=timeout)
except urllib.error.HTTPError as err:
resp = err
data = resp.read()
return resp.code, resp.headers, data
def actor(label):
username = f"mem_{label}_{uuid.uuid4().hex[:10]}"
password = "T3st-" + uuid.uuid4().hex
status, _, data = request("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
assert status == 201, status
user = json.loads(data)
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
status, _, data = request("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
assert status == 201, status
return user["id"], json.loads(data)["token"]
def wait_ready(timeout=120):
deadline = time.time() + timeout
while time.time() < deadline:
try:
if request("GET", "/health/ready", timeout=3)[0] == 200:
seen = {request("GET", "/health/live", timeout=3)[1].get("X-Instance-Id") for _ in range(20)}
if len(seen) >= 2:
return
except OSError:
pass
time.sleep(1)
raise RuntimeError("stack not ready")
def containers():
ids = sh(COMPOSE + ["ps", "-q", "api"]).split()
return {sh(["docker", "inspect", "-f", "{{.Config.Hostname}}", c]).strip(): c for c in ids}
def proc_status(cid):
out = sh(["docker", "exec", cid, "cat", "/proc/1/status"])
fields = {}
for line in out.splitlines():
if line.startswith(("VmRSS", "VmHWM")):
k, v = line.split(":", 1)
fields[k] = int(v.split()[0]) // 1024 # MiB
return fields
def cgroup(cid):
stat = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.stat"])
vals = dict(line.split() for line in stat.splitlines() if line)
peak = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.peak"], check=False).strip()
current = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.current"]).strip()
mib = 1024 * 1024
return {
"anon_mib": round(int(vals["anon"]) / mib, 1),
"file_mib": round(int(vals["file"]) / mib, 1),
"current_mib": round(int(current) / mib, 1),
"peak_mib": round(int(peak) / mib, 1) if peak.isdigit() else None,
}
def sampler(cid, stop, out):
best = {"anon_mib": 0.0, "file_mib": 0.0, "current_mib": 0.0, "samples": 0}
while not stop.is_set():
try:
c = cgroup(cid)
for k in ("anon_mib", "file_mib", "current_mib"):
best[k] = max(best[k], c[k])
best["samples"] += 1
except Exception:
pass
time.sleep(0.25)
out[cid] = best
def pct(values, q):
values = sorted(values)
return round(values[min(len(values) - 1, int(round(q * (len(values) - 1))))], 3)
def run_level(n, payload, chat_id, token):
print(f"== level {n}: restarting s3, api replicas and lb", flush=True)
# s3-gw shares the s3 network namespace and must be restarted after it.
sh(COMPOSE + ["restart", "s3"])
sh(COMPOSE + ["restart", "s3-gw", "api", "lb"])
wait_ready()
by_host = containers()
lb = sh(COMPOSE + ["ps", "-q", "lb"]).split()[0]
s3c = sh(COMPOSE + ["ps", "-q", "s3"]).split()[0]
# Warm-up: at least one upload served by every replica.
served = set()
for _ in range(20):
_, h, _ = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=warm.png", token=token,
raw=imagegen.rgb_png(64, 64), headers={"Content-Type": "image/png"})
served.add(h.get("X-Instance-Id"))
if served >= set(by_host):
break
baseline = {}
for host, cid in by_host.items():
sh(["docker", "exec", cid, "sh", "-c", "echo 5 > /proc/1/clear_refs"])
baseline[host] = {**proc_status(cid), **cgroup(cid)}
lb_baseline = cgroup(lb)
s3_baseline = cgroup(s3c)
stop = threading.Event()
sampled = {}
threads = [threading.Thread(target=sampler, args=(cid, stop, sampled)) for cid in [*by_host.values(), lb, s3c]]
for t in threads:
t.start()
time.sleep(0.5)
barrier = threading.Barrier(n)
results = [None] * n
def one(i):
barrier.wait()
t0 = time.monotonic()
try:
status, h, _ = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=load{i}.png",
token=token, raw=payload, headers={"Content-Type": "image/png"})
results[i] = (status, time.monotonic() - t0, h.get("X-Instance-Id"))
except OSError as exc:
results[i] = (type(exc).__name__, time.monotonic() - t0, None)
workers = [threading.Thread(target=one, args=(i,)) for i in range(n)]
started = time.monotonic()
for w in workers:
w.start()
for w in workers:
w.join()
wall = time.monotonic() - started
time.sleep(1.0)
stop.set()
for t in threads:
t.join()
replicas = {}
for host, cid in by_host.items():
after = proc_status(cid)
cg = cgroup(cid)
replicas[host] = {
"uploads_served": sum(1 for r in results if r[2] == host),
"rss_baseline_mib": baseline[host]["VmRSS"],
"rss_peak_vmhwm_mib": after["VmHWM"],
"rss_peak_growth_mib": after["VmHWM"] - baseline[host]["VmRSS"],
"cgroup_peak_mib": cg["peak_mib"],
"cgroup_baseline_current_mib": baseline[host]["current_mib"],
"sampled_max_anon_mib": sampled.get(cid, {}).get("anon_mib"),
"sampled_max_file_cache_mib": sampled.get(cid, {}).get("file_mib"),
"samples": sampled.get(cid, {}).get("samples"),
}
latencies = [r[1] for r in results if r[0] == 201]
statuses = {}
for r in results:
statuses[str(r[0])] = statuses.get(str(r[0]), 0) + 1
per_replica_max = max(v["uploads_served"] for v in replicas.values())
return {
"concurrency": n,
"payload_bytes": len(payload),
"statuses": statuses,
"wall_seconds": round(wall, 3),
"latency_ok_p50_s": pct(latencies, 0.5) if latencies else None,
"latency_ok_p95_s": pct(latencies, 0.95) if latencies else None,
"latency_ok_max_s": round(max(latencies), 3) if latencies else None,
"throughput_mib_s": round(len(latencies) * len(payload) / (1024 * 1024) / wall, 1),
"naive_full_buffer_estimate_mib": round(min(per_replica_max, 8) * len(payload) / (1024 * 1024), 1),
"replicas": replicas,
"lb": {
"mem_limit_mib": int(sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}}", lb]).strip()) // (1024 * 1024),
"baseline_current_mib": lb_baseline["current_mib"],
"cgroup_peak_mib": cgroup(lb)["peak_mib"],
"sampled_max_anon_mib": sampled.get(lb, {}).get("anon_mib"),
"sampled_max_current_mib": sampled.get(lb, {}).get("current_mib"),
"restarts": int(sh(["docker", "inspect", "-f", "{{.RestartCount}}", lb]).strip()),
"oom_killed": sh(["docker", "inspect", "-f", "{{.State.OOMKilled}}", lb]).strip() == "true",
},
"s3": _s3_report(s3c, s3_baseline, sampled.get(s3c, {})),
}
def _s3_report(cid, baseline, sampled):
running = sh(["docker", "inspect", "-f", "{{.State.Running}}", cid]).strip() == "true"
return {
"mem_limit_mib": int(sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}}", cid]).strip()) // (1024 * 1024),
"baseline_current_mib": baseline["current_mib"],
# memory.peak includes reclaimable page cache of the volume files; anon is the weed heap.
"cgroup_peak_mib": cgroup(cid)["peak_mib"] if running else None,
"sampled_max_anon_mib": sampled.get("anon_mib"),
"sampled_max_current_mib": sampled.get("current_mib"),
"running": running,
"oom_killed": sh(["docker", "inspect", "-f", "{{.State.OOMKilled}}", cid]).strip() == "true",
}
def main():
p = argparse.ArgumentParser(description="peak memory under concurrent uploads")
p.add_argument("--levels", type=int, nargs="+", default=[10, 20])
p.add_argument("--size-mib", type=float, default=9.5)
p.add_argument("--output", default=str(ROOT / "evidence" / "upload-memory.json"))
args = p.parse_args()
payload = imagegen.large_png(int(args.size_mib * 1024 * 1024))
_, token = actor("a")
bob_id, _ = actor("b")
me = json.loads(request("GET", "/api/v1/users/me", token=token)[2])
status, _, data = request("POST", "/api/v1/chats", {"title": "memory", "member_ids": [me["id"], bob_id]}, token=token)
assert status == 201, status
chat_id = json.loads(data)["id"]
limits = sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}} {{.HostConfig.NanoCpus}}",
list(containers().values())[0]]).split()
report = {
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"api_mem_limit_mib": int(limits[0]) // (1024 * 1024),
"api_cpus": int(limits[1]) / 1e9,
"max_concurrent_uploads_per_replica": 8,
"levels": [],
}
for n in args.levels:
level = run_level(n, payload, chat_id, token)
report["levels"].append(level)
print(json.dumps(level, ensure_ascii=False), flush=True)
Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
print("WROTE", args.output)
if __name__ == "__main__":
main()