forked from mtusi2026/mtusi2026lab5
223 lines
11 KiB
Python
223 lines
11 KiB
Python
"""Эксперимент: >= 10 одновременных загрузок и пик памяти API-реплик.
|
||
|
||
Гипотеза: память реплики почти не растёт от числа и размера одновременных загрузок, потому что
|
||
тело пишется во временный файл (в RAM не больше 256 KiB на загрузку), PNG проверяется потоково
|
||
(куски по 64 KiB), JPEG декодируется в масштабе 1/8, а одновременно проверяется не больше
|
||
DECODE_CONCURRENCY файлов на реплику. Наивный вариант (тело целиком в память + полное декодирование)
|
||
при тех же 10 загрузках не помещается в лимит 512 MiB.
|
||
|
||
Методика: перед каждым сценарием `docker compose restart api` (сбросить пик), прогрев, затем N потоков
|
||
одновременно (barrier) загружают файл через балансировщик. Пик RSS процесса API — VmHWM из /proc/1/status
|
||
(high-water mark ядра, не пропускает короткие всплески); дополнительно память cgroup контейнера
|
||
(memory.peak, если ядро даёт; иначе максимум из опроса memory.current) и опрос VmRSS раз в ~0,1 с.
|
||
|
||
python student-tests/upload_memory.py [--concurrency 10] [--output evidence/upload-memory.json]
|
||
"""
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import platform
|
||
import statistics
|
||
import threading
|
||
import time
|
||
|
||
import images
|
||
from common import (ROOT, app_python, call, check, compose, docker, new_chat, new_user, upload, wait_attachment,
|
||
wait_balanced, wait_ready)
|
||
|
||
|
||
def proc_status(container):
|
||
"""VmRSS и VmHWM (КиБ) процесса API (PID 1) и память cgroup контейнера (байты)."""
|
||
script = ("cat /proc/1/status | grep -E '^Vm(RSS|HWM)'; "
|
||
"for f in /sys/fs/cgroup/memory.peak /sys/fs/cgroup/memory.current "
|
||
"/sys/fs/cgroup/memory/memory.max_usage_in_bytes /sys/fs/cgroup/memory/memory.usage_in_bytes; "
|
||
"do [ -r $f ] && echo \"$f $(cat $f)\"; done; true")
|
||
out = docker("exec", container, "sh", "-c", script)
|
||
values = {}
|
||
for line in out.splitlines():
|
||
parts = line.replace(":", " ").split()
|
||
if parts[0] in ("VmRSS", "VmHWM"):
|
||
values[parts[0]] = int(parts[1])
|
||
elif parts[0].endswith(("memory.peak", "max_usage_in_bytes")):
|
||
values["cgroup_peak"] = int(parts[1])
|
||
elif parts[0].endswith(("memory.current", "usage_in_bytes")):
|
||
values["cgroup_current"] = int(parts[1])
|
||
return values
|
||
|
||
|
||
class Sampler(threading.Thread):
|
||
def __init__(self, containers):
|
||
super().__init__(daemon=True)
|
||
self.containers = containers
|
||
self.stop = threading.Event()
|
||
self.rss = {c: 0 for c in containers}
|
||
self.cgroup = {c: 0 for c in containers}
|
||
self.samples = 0
|
||
|
||
def run(self):
|
||
while not self.stop.is_set():
|
||
for c in self.containers:
|
||
try:
|
||
v = proc_status(c)
|
||
except Exception: # noqa: BLE001 — один пропущенный замер не важен
|
||
continue
|
||
self.rss[c] = max(self.rss[c], v.get("VmRSS", 0))
|
||
self.cgroup[c] = max(self.cgroup[c], v.get("cgroup_current", 0))
|
||
self.samples += 1
|
||
|
||
|
||
def api_containers():
|
||
from common import api_containers as running
|
||
return running()
|
||
|
||
|
||
def fresh_replicas():
|
||
compose("restart", "api")
|
||
containers = wait_balanced(2, seconds=90)
|
||
wait_ready()
|
||
return containers
|
||
|
||
|
||
def warm_up(user, chat):
|
||
"""По несколько загрузок на каждую реплику: в базовую линию входят импортированный Pillow, клиенты S3 и т.п."""
|
||
for data, ctype in ((images.sample_png(), "image/png"), (images.sample_jpeg(), "image/jpeg")) * 4:
|
||
upload(user, chat["id"], data, ctype)
|
||
|
||
|
||
def scenario(name, payloads, concurrency):
|
||
user = new_user("mem")
|
||
chat = new_chat(user)
|
||
containers = fresh_replicas()
|
||
warm_up(user, chat)
|
||
before = {c: proc_status(c) for c in containers}
|
||
sampler = Sampler(containers)
|
||
sampler.start()
|
||
barrier = threading.Barrier(concurrency)
|
||
results = [None] * concurrency
|
||
|
||
def worker(i):
|
||
data, ctype = payloads[i % len(payloads)]
|
||
barrier.wait()
|
||
started = time.monotonic()
|
||
try:
|
||
# lab5: замеряем только приём (202 queued); обработку worker'ом ждём ниже, вне замера
|
||
reply = upload(user, chat["id"], data, ctype, f"load-{i}.png", timeout=180, wait=False)
|
||
results[i] = (reply.status, time.monotonic() - started, reply.instance, reply.json, data)
|
||
except AssertionError as exc:
|
||
results[i] = (str(exc)[:120], time.monotonic() - started, None, None, data)
|
||
|
||
threads = [threading.Thread(target=worker, args=(i,)) for i in range(concurrency)]
|
||
wall = time.monotonic()
|
||
for t in threads:
|
||
t.start()
|
||
for t in threads:
|
||
t.join()
|
||
wall = time.monotonic() - wall
|
||
time.sleep(0.5)
|
||
sampler.stop.set()
|
||
sampler.join()
|
||
after = {c: proc_status(c) for c in containers}
|
||
|
||
ok = [r for r in results if r[0] == 202]
|
||
check(len(ok) == concurrency, f"{name}: not all uploads succeeded: {[r[0] for r in results if r[0] != 202]}")
|
||
for status, _, _, att, data in ok[:3]:
|
||
att = wait_attachment(user, att["id"], seconds=120)
|
||
check(att["status"] == "ready", f"{name}: attachment {att['id']} is {att['status']}, expected ready")
|
||
got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=user["token"])
|
||
check(got.data == data and att["sha256"] == hashlib.sha256(data).hexdigest(), "stored file differs")
|
||
latencies = sorted(r[1] for r in results)
|
||
per_replica = []
|
||
for c in containers:
|
||
b, a = before[c], after[c]
|
||
per_replica.append({
|
||
"replica": c[:12],
|
||
"uploads_served": sum(1 for r in ok if r[2] and c.startswith(r[2])),
|
||
"rss_before_mib": round(b["VmRSS"] / 1024, 1),
|
||
"rss_peak_mib": round(a["VmHWM"] / 1024, 1),
|
||
"rss_growth_mib": round((a["VmHWM"] - b["VmRSS"]) / 1024, 1),
|
||
"rss_peak_sampled_mib": round(sampler.rss[c] / 1024, 1),
|
||
"cgroup_peak_mib": round(a["cgroup_peak"] / 2**20, 1) if "cgroup_peak" in a else None,
|
||
"cgroup_peak_sampled_mib": round(sampler.cgroup[c] / 2**20, 1) if sampler.cgroup[c] else None,
|
||
})
|
||
sizes = sorted({len(d) for d, _ in payloads})
|
||
summary = {
|
||
"scenario": name,
|
||
"concurrency": concurrency,
|
||
"file_bytes": sizes,
|
||
"total_mib": round(sum(len(r[4]) for r in results) / 2**20, 1),
|
||
"all_202": True,
|
||
"wall_seconds": round(wall, 2),
|
||
"latency_s": {"p50": round(statistics.median(latencies), 2), "max": round(latencies[-1], 2)},
|
||
"sampler_rounds": sampler.samples,
|
||
"replicas": per_replica,
|
||
}
|
||
print(json.dumps(summary, ensure_ascii=False))
|
||
return summary
|
||
|
||
|
||
# Для сравнения: то же внутри контейнера api, но «наивно» — тело целиком в памяти и полное декодирование.
|
||
NAIVE_VS_STREAMING = r'''
|
||
import json, os, subprocess, sys
|
||
code = r"""
|
||
import io, os, resource, struct, sys, zlib
|
||
sys.path.insert(0, os.getcwd())
|
||
os.environ.setdefault("S3_ACCESS_KEY_ID", "x")
|
||
from PIL import Image
|
||
import uploads # импорт до замера: сравниваем только саму проверку файла
|
||
Image.MAX_IMAGE_PIXELS = None
|
||
def chunk(k, d): return struct.pack(">I", len(d)) + k + d + struct.pack(">I", zlib.crc32(k + d))
|
||
w, h = 8000, 5000
|
||
comp = zlib.compressobj(6); row = b"\x00" + bytes(w * 3)
|
||
idat = b"".join(comp.compress(row) for _ in range(h)) + comp.flush()
|
||
data = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) + chunk(b"IDAT", idat) + chunk(b"IEND", b"")
|
||
del idat, comp, row
|
||
base = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
||
if sys.argv[1] == "naive":
|
||
body = bytes(data) + bytes(9 * 1024 * 1024) # тело запроса целиком в памяти (9 MiB)
|
||
im = Image.open(io.BytesIO(data)); im.load()
|
||
else:
|
||
f = io.BytesIO(data); uploads.check_png(f)
|
||
print((resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - base) // 1024)
|
||
"""
|
||
out = {}
|
||
for mode in ("naive", "streaming"):
|
||
r = subprocess.run([sys.executable, "-c", code, mode], capture_output=True, text=True, cwd=os.getcwd())
|
||
out[mode] = int(r.stdout.strip().splitlines()[-1]) if r.returncode == 0 else r.stderr[-200:]
|
||
print(json.dumps(out))
|
||
'''
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser()
|
||
parser.add_argument("--concurrency", type=int, default=10)
|
||
parser.add_argument("--output", default="evidence/upload-memory.json")
|
||
args = parser.parse_args()
|
||
check(args.concurrency >= 10, "the experiment needs at least 10 concurrent uploads")
|
||
|
||
print(f"client: Python {platform.python_version()} on {platform.system()} {platform.release()}")
|
||
print("preparing synthetic images (random-noise PNG ~9.5 MiB each, 40 Mpx PNG)...")
|
||
large = [(images.noise_png(int(9.5 * 2**20)), "image/png") for _ in range(4)]
|
||
pixels = [(images.png(8000, 5000), "image/png")]
|
||
jpeg = [(images.sample_jpeg(), "image/jpeg")]
|
||
print(f"large: {len(large[0][0])} bytes; 40 Mpx: {len(pixels[0][0])} bytes")
|
||
|
||
runs = [
|
||
scenario("10 x ~9.5 MiB PNG (noise)", large, args.concurrency),
|
||
scenario("10 x 40 Mpx PNG (8000x5000 RGB)", pixels, args.concurrency),
|
||
scenario(f"{2 * args.concurrency} mixed: large + 40 Mpx + JPEG", large + pixels + jpeg, 2 * args.concurrency),
|
||
]
|
||
naive = json.loads(app_python(NAIVE_VS_STREAMING).splitlines()[-1])
|
||
print(f"one 8000x5000 RGB PNG inside the api container, extra peak RSS: naive (body in RAM + full decode) "
|
||
f"{naive['naive']} MiB, streaming check {naive['streaming']} MiB")
|
||
limits = docker("inspect", "-f", "{{.HostConfig.Memory}} {{.HostConfig.NanoCpus}}", api_containers()[0]).split()
|
||
result = {"date": time.strftime("%Y-%m-%d %H:%M"), "api_limit_mib": int(limits[0]) // 2**20,
|
||
"api_limit_cpus": int(limits[1]) / 1e9, "runs": runs, "single_file_extra_rss_mib": naive}
|
||
(ROOT / args.output).write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
worst = max(r["rss_growth_mib"] for run in runs for r in run["replicas"])
|
||
print(f"PASS: all uploads succeeded; largest RSS growth of one replica: {worst} MiB "
|
||
f"(limit {result['api_limit_mib']} MiB); results in {args.output}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|