Files
mtusi2026lab4/research/upload_memory.py
T

151 lines
7.4 KiB
Python

#!/usr/bin/env python3
"""Уровень 4: ≥10 одновременных загрузок крупных файлов и пик памяти API.
Для каждого значения UPLOAD_CONCURRENCY (одновременных загрузок на реплику) пересоздаётся ОДНА реплика API
(пик RSS обнуляется), затем 12 клиентов одновременно грузят ~8 MiB PNG. Пик памяти — VmHWM из /proc/1/status контейнера
(максимум резидентной памяти за время жизни процесса). Результат: evidence/upload_memory.json.
python research/upload_memory.py --out evidence
Требует Pillow и aiohttp (tools/.venv). Стенд должен быть запущен (docker compose up -d).
"""
import argparse
import asyncio
import base64
import hashlib
import io
import json
import os
import subprocess
import time
import uuid
import aiohttp
from PIL import Image
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
BASE = "http://127.0.0.1:8080"
PROJECT = "mtusi-lab4"
def sh(*args, env=None):
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, check=True,
env=dict(os.environ, **(env or {}))).stdout.strip()
def noisy_png(side, seed):
"""Почти несжимаемый PNG (шум): размер файла близок к side*side*3 байт."""
raw = os.urandom(side * side * 3)
img = Image.frombytes("RGB", (side, side), raw)
buf = io.BytesIO()
img.save(buf, "PNG", compress_level=1)
return buf.getvalue()
def api_containers():
return sh("docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + PROJECT,
"--filter", "label=com.docker.compose.service=api").split()
def set_api(concurrency, replicas):
sh("docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--force-recreate", "--scale", "api=%d" % replicas, "api",
env={"UPLOAD_CONCURRENCY": str(concurrency)})
deadline = time.time() + 90
while time.time() < deadline:
ids = api_containers()
if len(ids) == replicas and all(sh("docker", "inspect", "-f", "{{.State.Health.Status}}", i) == "healthy" for i in ids):
break
time.sleep(1)
else:
raise RuntimeError("API replicas did not become healthy within 90 seconds")
time.sleep(6)
def peak_rss_mib(cid):
out = sh("docker", "exec", cid, "sh", "-c", "grep -E 'VmHWM|VmRSS' /proc/1/status")
vals = {l.split(":")[0]: int(l.split()[1]) / 1024 for l in out.splitlines()}
return round(vals.get("VmHWM", 0), 1), round(vals.get("VmRSS", 0), 1)
async def register(s):
name = "up_%s" % uuid.uuid4().hex[:12]
pw = "upload-test-pw-" + uuid.uuid4().hex[:6]
async with s.post(BASE + "/api/v1/users", json={"username": name, "display_name": "u", "password": pw}) as r:
user = await r.json()
cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode()
async with s.post(BASE + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r:
sess = await r.json()
return user["id"], sess["token"]
async def one_run(clients, data, token, chat_id, timeout=120):
results = []
t0 = time.perf_counter()
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=timeout), connector=aiohttp.TCPConnector(limit=clients)) as s:
async def up(i):
t = time.perf_counter()
try:
async with s.post("%s/api/v1/chats/%s/attachments?filename=load%d.png" % (BASE, chat_id, i), data=data,
headers={"Authorization": "Bearer " + token, "Content-Type": "image/png"}) as r:
body = await r.json(content_type=None)
results.append((r.status, time.perf_counter() - t, body.get("id") if r.status < 300 else body.get("error", {}).get("code")))
except Exception as exc:
results.append((type(exc).__name__, time.perf_counter() - t, None))
await asyncio.gather(*(up(i) for i in range(clients)))
return results, time.perf_counter() - t0
async def main(args):
data = noisy_png(args.side, 1)
print("размер файла: %.1f MiB, sha256 %s" % (len(data) / 1048576, hashlib.sha256(data).hexdigest()[:12]))
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "file_mib": round(len(data) / 1048576, 2), "clients": args.clients,
"api_memory_limit": "256m", "runs": []}
for conc in args.concurrency:
set_api(conc, 1)
cid = api_containers()[0]
limits = json.loads(sh("docker", "inspect", "-f",
'{"memory_bytes":{{.HostConfig.Memory}},"nano_cpus":{{.HostConfig.NanoCpus}}}', cid))
assert limits == {"memory_bytes": 256 * 1024 * 1024, "nano_cpus": 1000000000}, limits
async with aiohttp.ClientSession() as s:
uid, token = await register(s)
async with s.post(BASE + "/api/v1/chats", json={"title": "mem", "member_ids": [uid]}, headers={"Authorization": "Bearer " + token}) as r:
chat = await r.json()
baseline_hwm, _ = peak_rss_mib(cid)
results, wall = await one_run(args.clients, data, token, chat["id"])
peak, rss = peak_rss_mib(cid)
statuses = {}
for st, _, _ in results:
statuses[str(st)] = statuses.get(str(st), 0) + 1
ok_lat = sorted(t for st, t, _ in results if st == 201)
# проверка целостности: скачиваем каждый принятый файл и сверяем sha256
intact = 0
async with aiohttp.ClientSession() as s:
for st, _, att in results:
if st == 201:
async with s.get("%s/api/v1/attachments/%s/content" % (BASE, att), headers={"Authorization": "Bearer " + token}) as r:
if hashlib.sha256(await r.read()).hexdigest() == hashlib.sha256(data).hexdigest():
intact += 1
oom = sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", cid)
entry = {"UPLOAD_CONCURRENCY": conc, "statuses": statuses, "wall_seconds": round(wall, 2), "peak_rss_mib": peak,
"resource_limits": limits,
"rss_before_mib": baseline_hwm, "rss_after_mib": rss, "oom_killed": oom,
"accepted_files_intact": "%d/%d" % (intact, statuses.get("201", 0)),
"p50_s": round(ok_lat[len(ok_lat) // 2], 2) if ok_lat else None, "max_s": round(ok_lat[-1], 2) if ok_lat else None}
print(json.dumps(entry, ensure_ascii=False))
report["runs"].append(entry)
set_api(4, 2)
with open(os.path.join(args.out, "upload_memory.json"), "w", encoding="utf-8") as fh:
json.dump(report, fh, ensure_ascii=False, indent=2)
for entry in report["runs"]:
assert entry["statuses"] == {"201": args.clients}, entry
assert entry["accepted_files_intact"] == "%d/%d" % (args.clients, args.clients), entry
assert entry["oom_killed"] == "false" and 0 < entry["peak_rss_mib"] < 256, entry
if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument("--out", default="evidence")
ap.add_argument("--clients", type=int, default=12)
ap.add_argument("--side", type=int, default=1700, help="сторона квадрата PNG-шума; 1700 -> около 8.7 MiB")
ap.add_argument("--concurrency", type=int, nargs="+", default=[1, 4, 12])
asyncio.run(main(ap.parse_args()))