forked from mtusi2026/mtusi2026lab4
151 lines
7.4 KiB
Python
151 lines
7.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Уровень 4: ≥10 одновременных загрузок крупных файлов и пик памяти API.
|
|
|
|
Для каждого значения UPLOAD_CONCURRENCY (одновременных загрузок на реплику) пересоздаётся ОДНА реплика API
|
|
(пик RSS обнуляется), затем 12 клиентов одновременно грузят ~8 MiB PNG. Пик памяти — VmHWM из /proc/1/status контейнера
|
|
(максимум резидентной памяти за время жизни процесса). Результат: evidence/upload_memory.json.
|
|
|
|
python research/upload_memory.py --out evidence
|
|
Требует Pillow и aiohttp (tools/.venv). Стенд должен быть запущен (docker compose up -d).
|
|
"""
|
|
import argparse
|
|
import asyncio
|
|
import base64
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
|
|
import aiohttp
|
|
from PIL import Image
|
|
|
|
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
|
|
BASE = "http://127.0.0.1:8080"
|
|
PROJECT = "mtusi-lab4"
|
|
|
|
|
|
def sh(*args, env=None):
|
|
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, check=True,
|
|
env=dict(os.environ, **(env or {}))).stdout.strip()
|
|
|
|
|
|
def noisy_png(side, seed):
|
|
"""Почти несжимаемый PNG (шум): размер файла близок к side*side*3 байт."""
|
|
raw = os.urandom(side * side * 3)
|
|
img = Image.frombytes("RGB", (side, side), raw)
|
|
buf = io.BytesIO()
|
|
img.save(buf, "PNG", compress_level=1)
|
|
return buf.getvalue()
|
|
|
|
|
|
def api_containers():
|
|
return sh("docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + PROJECT,
|
|
"--filter", "label=com.docker.compose.service=api").split()
|
|
|
|
|
|
def set_api(concurrency, replicas):
|
|
sh("docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--force-recreate", "--scale", "api=%d" % replicas, "api",
|
|
env={"UPLOAD_CONCURRENCY": str(concurrency)})
|
|
deadline = time.time() + 90
|
|
while time.time() < deadline:
|
|
ids = api_containers()
|
|
if len(ids) == replicas and all(sh("docker", "inspect", "-f", "{{.State.Health.Status}}", i) == "healthy" for i in ids):
|
|
break
|
|
time.sleep(1)
|
|
else:
|
|
raise RuntimeError("API replicas did not become healthy within 90 seconds")
|
|
time.sleep(6)
|
|
|
|
|
|
def peak_rss_mib(cid):
|
|
out = sh("docker", "exec", cid, "sh", "-c", "grep -E 'VmHWM|VmRSS' /proc/1/status")
|
|
vals = {l.split(":")[0]: int(l.split()[1]) / 1024 for l in out.splitlines()}
|
|
return round(vals.get("VmHWM", 0), 1), round(vals.get("VmRSS", 0), 1)
|
|
|
|
|
|
async def register(s):
|
|
name = "up_%s" % uuid.uuid4().hex[:12]
|
|
pw = "upload-test-pw-" + uuid.uuid4().hex[:6]
|
|
async with s.post(BASE + "/api/v1/users", json={"username": name, "display_name": "u", "password": pw}) as r:
|
|
user = await r.json()
|
|
cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode()
|
|
async with s.post(BASE + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r:
|
|
sess = await r.json()
|
|
return user["id"], sess["token"]
|
|
|
|
|
|
async def one_run(clients, data, token, chat_id, timeout=120):
|
|
results = []
|
|
t0 = time.perf_counter()
|
|
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=timeout), connector=aiohttp.TCPConnector(limit=clients)) as s:
|
|
async def up(i):
|
|
t = time.perf_counter()
|
|
try:
|
|
async with s.post("%s/api/v1/chats/%s/attachments?filename=load%d.png" % (BASE, chat_id, i), data=data,
|
|
headers={"Authorization": "Bearer " + token, "Content-Type": "image/png"}) as r:
|
|
body = await r.json(content_type=None)
|
|
results.append((r.status, time.perf_counter() - t, body.get("id") if r.status < 300 else body.get("error", {}).get("code")))
|
|
except Exception as exc:
|
|
results.append((type(exc).__name__, time.perf_counter() - t, None))
|
|
await asyncio.gather(*(up(i) for i in range(clients)))
|
|
return results, time.perf_counter() - t0
|
|
|
|
|
|
async def main(args):
|
|
data = noisy_png(args.side, 1)
|
|
print("размер файла: %.1f MiB, sha256 %s" % (len(data) / 1048576, hashlib.sha256(data).hexdigest()[:12]))
|
|
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "file_mib": round(len(data) / 1048576, 2), "clients": args.clients,
|
|
"api_memory_limit": "256m", "runs": []}
|
|
for conc in args.concurrency:
|
|
set_api(conc, 1)
|
|
cid = api_containers()[0]
|
|
limits = json.loads(sh("docker", "inspect", "-f",
|
|
'{"memory_bytes":{{.HostConfig.Memory}},"nano_cpus":{{.HostConfig.NanoCpus}}}', cid))
|
|
assert limits == {"memory_bytes": 256 * 1024 * 1024, "nano_cpus": 1000000000}, limits
|
|
async with aiohttp.ClientSession() as s:
|
|
uid, token = await register(s)
|
|
async with s.post(BASE + "/api/v1/chats", json={"title": "mem", "member_ids": [uid]}, headers={"Authorization": "Bearer " + token}) as r:
|
|
chat = await r.json()
|
|
baseline_hwm, _ = peak_rss_mib(cid)
|
|
results, wall = await one_run(args.clients, data, token, chat["id"])
|
|
peak, rss = peak_rss_mib(cid)
|
|
statuses = {}
|
|
for st, _, _ in results:
|
|
statuses[str(st)] = statuses.get(str(st), 0) + 1
|
|
ok_lat = sorted(t for st, t, _ in results if st == 201)
|
|
# проверка целостности: скачиваем каждый принятый файл и сверяем sha256
|
|
intact = 0
|
|
async with aiohttp.ClientSession() as s:
|
|
for st, _, att in results:
|
|
if st == 201:
|
|
async with s.get("%s/api/v1/attachments/%s/content" % (BASE, att), headers={"Authorization": "Bearer " + token}) as r:
|
|
if hashlib.sha256(await r.read()).hexdigest() == hashlib.sha256(data).hexdigest():
|
|
intact += 1
|
|
oom = sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", cid)
|
|
entry = {"UPLOAD_CONCURRENCY": conc, "statuses": statuses, "wall_seconds": round(wall, 2), "peak_rss_mib": peak,
|
|
"resource_limits": limits,
|
|
"rss_before_mib": baseline_hwm, "rss_after_mib": rss, "oom_killed": oom,
|
|
"accepted_files_intact": "%d/%d" % (intact, statuses.get("201", 0)),
|
|
"p50_s": round(ok_lat[len(ok_lat) // 2], 2) if ok_lat else None, "max_s": round(ok_lat[-1], 2) if ok_lat else None}
|
|
print(json.dumps(entry, ensure_ascii=False))
|
|
report["runs"].append(entry)
|
|
set_api(4, 2)
|
|
with open(os.path.join(args.out, "upload_memory.json"), "w", encoding="utf-8") as fh:
|
|
json.dump(report, fh, ensure_ascii=False, indent=2)
|
|
for entry in report["runs"]:
|
|
assert entry["statuses"] == {"201": args.clients}, entry
|
|
assert entry["accepted_files_intact"] == "%d/%d" % (args.clients, args.clients), entry
|
|
assert entry["oom_killed"] == "false" and 0 < entry["peak_rss_mib"] < 256, entry
|
|
|
|
|
|
if __name__ == "__main__":
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--out", default="evidence")
|
|
ap.add_argument("--clients", type=int, default=12)
|
|
ap.add_argument("--side", type=int, default=1700, help="сторона квадрата PNG-шума; 1700 -> около 8.7 MiB")
|
|
ap.add_argument("--concurrency", type=int, nargs="+", default=[1, 4, 12])
|
|
asyncio.run(main(ap.parse_args()))
|