forked from mtusi2026/mtusi2026lab4
Реализовать lab4 на оценку 4: SeaweedFS, вложения и согласованность БД/S3.
Перенесён стенд lab3 (API, Redis-сессии, идемпотентность, HAProxy, реплики API). Добавлены приватный bucket SeaweedFS с отдельным ключом API, загрузка PNG/JPEG с лимитами и полной проверкой, компенсация и reconciler, плавная остановка API, тесты и evidence.
This commit is contained in:
@@ -0,0 +1,284 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Experiment: peak memory of API replicas, LB and S3 under N simultaneous ~9.5 MiB uploads.
|
||||
|
||||
Hypothesis: streaming intake (64 KiB chunks → validator → temp file → S3 PUT from
|
||||
file) keeps the API process RSS growth far below N × file size; total in-flight
|
||||
work per replica is capped by MAX_CONCURRENT_UPLOADS.
|
||||
|
||||
Method (per concurrency level, fresh replicas):
|
||||
1. `docker compose restart s3`, then `s3-gw api lb` → new processes; wait for /health/ready;
|
||||
2. one warm-up upload per replica (imports, pools, TLS-less S3 connections);
|
||||
3. reset VmHWM of PID 1 (echo 5 > /proc/1/clear_refs) and record baseline;
|
||||
4. fire N uploads at once through the LB (barrier), sample cgroup memory.stat
|
||||
(anon/file) of every replica, the LB and the s3 container from the host every ~0.25 s;
|
||||
5. read VmHWM (peak RSS of the uvicorn process) and cgroup memory.peak.
|
||||
|
||||
Usage: python scripts/upload_memory_experiment.py [--levels 10 20] [--size-mib 9.5]
|
||||
Writes evidence/upload-memory.json. Tokens are kept in memory only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||||
import imagegen # noqa: E402
|
||||
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
||||
|
||||
|
||||
def sh(args, check=True) -> str:
|
||||
proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8")
|
||||
if check and proc.returncode != 0:
|
||||
raise RuntimeError(f"{args[:4]} rc={proc.returncode}: {proc.stderr[-300:]}")
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def request(method, path, body=None, token=None, raw=None, headers=None, timeout=180):
|
||||
h = {"Accept": "application/json"}
|
||||
if token:
|
||||
h["Authorization"] = "Bearer " + token
|
||||
if body is not None:
|
||||
raw = json.dumps(body).encode()
|
||||
h["Content-Type"] = "application/json"
|
||||
h.update(headers or {})
|
||||
req = urllib.request.Request(BASE + path, data=raw, headers=h, method=method)
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=timeout)
|
||||
except urllib.error.HTTPError as err:
|
||||
resp = err
|
||||
data = resp.read()
|
||||
return resp.code, resp.headers, data
|
||||
|
||||
|
||||
def actor(label):
|
||||
username = f"mem_{label}_{uuid.uuid4().hex[:10]}"
|
||||
password = "T3st-" + uuid.uuid4().hex
|
||||
status, _, data = request("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
|
||||
assert status == 201, status
|
||||
user = json.loads(data)
|
||||
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
status, _, data = request("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
|
||||
assert status == 201, status
|
||||
return user["id"], json.loads(data)["token"]
|
||||
|
||||
|
||||
def wait_ready(timeout=120):
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
if request("GET", "/health/ready", timeout=3)[0] == 200:
|
||||
seen = {request("GET", "/health/live", timeout=3)[1].get("X-Instance-Id") for _ in range(20)}
|
||||
if len(seen) >= 2:
|
||||
return
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise RuntimeError("stack not ready")
|
||||
|
||||
|
||||
def containers():
|
||||
ids = sh(COMPOSE + ["ps", "-q", "api"]).split()
|
||||
return {sh(["docker", "inspect", "-f", "{{.Config.Hostname}}", c]).strip(): c for c in ids}
|
||||
|
||||
|
||||
def proc_status(cid):
|
||||
out = sh(["docker", "exec", cid, "cat", "/proc/1/status"])
|
||||
fields = {}
|
||||
for line in out.splitlines():
|
||||
if line.startswith(("VmRSS", "VmHWM")):
|
||||
k, v = line.split(":", 1)
|
||||
fields[k] = int(v.split()[0]) // 1024 # MiB
|
||||
return fields
|
||||
|
||||
|
||||
def cgroup(cid):
|
||||
stat = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.stat"])
|
||||
vals = dict(line.split() for line in stat.splitlines() if line)
|
||||
peak = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.peak"], check=False).strip()
|
||||
current = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.current"]).strip()
|
||||
mib = 1024 * 1024
|
||||
return {
|
||||
"anon_mib": round(int(vals["anon"]) / mib, 1),
|
||||
"file_mib": round(int(vals["file"]) / mib, 1),
|
||||
"current_mib": round(int(current) / mib, 1),
|
||||
"peak_mib": round(int(peak) / mib, 1) if peak.isdigit() else None,
|
||||
}
|
||||
|
||||
|
||||
def sampler(cid, stop, out):
|
||||
best = {"anon_mib": 0.0, "file_mib": 0.0, "current_mib": 0.0, "samples": 0}
|
||||
while not stop.is_set():
|
||||
try:
|
||||
c = cgroup(cid)
|
||||
for k in ("anon_mib", "file_mib", "current_mib"):
|
||||
best[k] = max(best[k], c[k])
|
||||
best["samples"] += 1
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.25)
|
||||
out[cid] = best
|
||||
|
||||
|
||||
def pct(values, q):
|
||||
values = sorted(values)
|
||||
return round(values[min(len(values) - 1, int(round(q * (len(values) - 1))))], 3)
|
||||
|
||||
|
||||
def run_level(n, payload, chat_id, token):
|
||||
print(f"== level {n}: restarting s3, api replicas and lb", flush=True)
|
||||
# s3-gw shares the s3 network namespace and must be restarted after it.
|
||||
sh(COMPOSE + ["restart", "s3"])
|
||||
sh(COMPOSE + ["restart", "s3-gw", "api", "lb"])
|
||||
wait_ready()
|
||||
by_host = containers()
|
||||
lb = sh(COMPOSE + ["ps", "-q", "lb"]).split()[0]
|
||||
s3c = sh(COMPOSE + ["ps", "-q", "s3"]).split()[0]
|
||||
# Warm-up: at least one upload served by every replica.
|
||||
served = set()
|
||||
for _ in range(20):
|
||||
_, h, _ = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=warm.png", token=token,
|
||||
raw=imagegen.rgb_png(64, 64), headers={"Content-Type": "image/png"})
|
||||
served.add(h.get("X-Instance-Id"))
|
||||
if served >= set(by_host):
|
||||
break
|
||||
baseline = {}
|
||||
for host, cid in by_host.items():
|
||||
sh(["docker", "exec", cid, "sh", "-c", "echo 5 > /proc/1/clear_refs"])
|
||||
baseline[host] = {**proc_status(cid), **cgroup(cid)}
|
||||
lb_baseline = cgroup(lb)
|
||||
s3_baseline = cgroup(s3c)
|
||||
stop = threading.Event()
|
||||
sampled = {}
|
||||
threads = [threading.Thread(target=sampler, args=(cid, stop, sampled)) for cid in [*by_host.values(), lb, s3c]]
|
||||
for t in threads:
|
||||
t.start()
|
||||
time.sleep(0.5)
|
||||
barrier = threading.Barrier(n)
|
||||
results = [None] * n
|
||||
|
||||
def one(i):
|
||||
barrier.wait()
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
status, h, _ = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=load{i}.png",
|
||||
token=token, raw=payload, headers={"Content-Type": "image/png"})
|
||||
results[i] = (status, time.monotonic() - t0, h.get("X-Instance-Id"))
|
||||
except OSError as exc:
|
||||
results[i] = (type(exc).__name__, time.monotonic() - t0, None)
|
||||
|
||||
workers = [threading.Thread(target=one, args=(i,)) for i in range(n)]
|
||||
started = time.monotonic()
|
||||
for w in workers:
|
||||
w.start()
|
||||
for w in workers:
|
||||
w.join()
|
||||
wall = time.monotonic() - started
|
||||
time.sleep(1.0)
|
||||
stop.set()
|
||||
for t in threads:
|
||||
t.join()
|
||||
replicas = {}
|
||||
for host, cid in by_host.items():
|
||||
after = proc_status(cid)
|
||||
cg = cgroup(cid)
|
||||
replicas[host] = {
|
||||
"uploads_served": sum(1 for r in results if r[2] == host),
|
||||
"rss_baseline_mib": baseline[host]["VmRSS"],
|
||||
"rss_peak_vmhwm_mib": after["VmHWM"],
|
||||
"rss_peak_growth_mib": after["VmHWM"] - baseline[host]["VmRSS"],
|
||||
"cgroup_peak_mib": cg["peak_mib"],
|
||||
"cgroup_baseline_current_mib": baseline[host]["current_mib"],
|
||||
"sampled_max_anon_mib": sampled.get(cid, {}).get("anon_mib"),
|
||||
"sampled_max_file_cache_mib": sampled.get(cid, {}).get("file_mib"),
|
||||
"samples": sampled.get(cid, {}).get("samples"),
|
||||
}
|
||||
latencies = [r[1] for r in results if r[0] == 201]
|
||||
statuses = {}
|
||||
for r in results:
|
||||
statuses[str(r[0])] = statuses.get(str(r[0]), 0) + 1
|
||||
per_replica_max = max(v["uploads_served"] for v in replicas.values())
|
||||
return {
|
||||
"concurrency": n,
|
||||
"payload_bytes": len(payload),
|
||||
"statuses": statuses,
|
||||
"wall_seconds": round(wall, 3),
|
||||
"latency_ok_p50_s": pct(latencies, 0.5) if latencies else None,
|
||||
"latency_ok_p95_s": pct(latencies, 0.95) if latencies else None,
|
||||
"latency_ok_max_s": round(max(latencies), 3) if latencies else None,
|
||||
"throughput_mib_s": round(len(latencies) * len(payload) / (1024 * 1024) / wall, 1),
|
||||
"naive_full_buffer_estimate_mib": round(min(per_replica_max, 8) * len(payload) / (1024 * 1024), 1),
|
||||
"replicas": replicas,
|
||||
"lb": {
|
||||
"mem_limit_mib": int(sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}}", lb]).strip()) // (1024 * 1024),
|
||||
"baseline_current_mib": lb_baseline["current_mib"],
|
||||
"cgroup_peak_mib": cgroup(lb)["peak_mib"],
|
||||
"sampled_max_anon_mib": sampled.get(lb, {}).get("anon_mib"),
|
||||
"sampled_max_current_mib": sampled.get(lb, {}).get("current_mib"),
|
||||
"restarts": int(sh(["docker", "inspect", "-f", "{{.RestartCount}}", lb]).strip()),
|
||||
"oom_killed": sh(["docker", "inspect", "-f", "{{.State.OOMKilled}}", lb]).strip() == "true",
|
||||
},
|
||||
"s3": _s3_report(s3c, s3_baseline, sampled.get(s3c, {})),
|
||||
}
|
||||
|
||||
|
||||
def _s3_report(cid, baseline, sampled):
|
||||
running = sh(["docker", "inspect", "-f", "{{.State.Running}}", cid]).strip() == "true"
|
||||
return {
|
||||
"mem_limit_mib": int(sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}}", cid]).strip()) // (1024 * 1024),
|
||||
"baseline_current_mib": baseline["current_mib"],
|
||||
# memory.peak includes reclaimable page cache of the volume files; anon is the weed heap.
|
||||
"cgroup_peak_mib": cgroup(cid)["peak_mib"] if running else None,
|
||||
"sampled_max_anon_mib": sampled.get("anon_mib"),
|
||||
"sampled_max_current_mib": sampled.get("current_mib"),
|
||||
"running": running,
|
||||
"oom_killed": sh(["docker", "inspect", "-f", "{{.State.OOMKilled}}", cid]).strip() == "true",
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description="peak memory under concurrent uploads")
|
||||
p.add_argument("--levels", type=int, nargs="+", default=[10, 20])
|
||||
p.add_argument("--size-mib", type=float, default=9.5)
|
||||
p.add_argument("--output", default=str(ROOT / "evidence" / "upload-memory.json"))
|
||||
args = p.parse_args()
|
||||
payload = imagegen.large_png(int(args.size_mib * 1024 * 1024))
|
||||
_, token = actor("a")
|
||||
bob_id, _ = actor("b")
|
||||
me = json.loads(request("GET", "/api/v1/users/me", token=token)[2])
|
||||
status, _, data = request("POST", "/api/v1/chats", {"title": "memory", "member_ids": [me["id"], bob_id]}, token=token)
|
||||
assert status == 201, status
|
||||
chat_id = json.loads(data)["id"]
|
||||
limits = sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}} {{.HostConfig.NanoCpus}}",
|
||||
list(containers().values())[0]]).split()
|
||||
report = {
|
||||
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"api_mem_limit_mib": int(limits[0]) // (1024 * 1024),
|
||||
"api_cpus": int(limits[1]) / 1e9,
|
||||
"max_concurrent_uploads_per_replica": 8,
|
||||
"levels": [],
|
||||
}
|
||||
for n in args.levels:
|
||||
level = run_level(n, payload, chat_id, token)
|
||||
report["levels"].append(level)
|
||||
print(json.dumps(level, ensure_ascii=False), flush=True)
|
||||
Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
print("WROTE", args.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user