#!/usr/bin/env python3 """Уровень 4: ≥10 одновременных загрузок крупных файлов и пик памяти API. Для каждого значения UPLOAD_CONCURRENCY (одновременных загрузок на реплику) пересоздаётся ОДНА реплика API (пик RSS обнуляется), затем 12 клиентов одновременно грузят ~8 MiB PNG. Пик памяти — VmHWM из /proc/1/status контейнера (максимум резидентной памяти за время жизни процесса). Результат: evidence/upload_memory.json. python research/upload_memory.py --out evidence Требует Pillow и aiohttp (tools/.venv). Стенд должен быть запущен (docker compose up -d). """ import argparse import asyncio import base64 import hashlib import io import json import os import subprocess import time import uuid import aiohttp from PIL import Image ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") BASE = "http://127.0.0.1:8080" PROJECT = "mtusi-lab4" def sh(*args, env=None): return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, check=True, env=dict(os.environ, **(env or {}))).stdout.strip() def noisy_png(side, seed): """Почти несжимаемый PNG (шум): размер файла близок к side*side*3 байт.""" raw = os.urandom(side * side * 3) img = Image.frombytes("RGB", (side, side), raw) buf = io.BytesIO() img.save(buf, "PNG", compress_level=1) return buf.getvalue() def api_containers(): return sh("docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + PROJECT, "--filter", "label=com.docker.compose.service=api").split() def set_api(concurrency, replicas): sh("docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--force-recreate", "--scale", "api=%d" % replicas, "api", env={"UPLOAD_CONCURRENCY": str(concurrency)}) deadline = time.time() + 90 while time.time() < deadline: ids = api_containers() if len(ids) == replicas and all(sh("docker", "inspect", "-f", "{{.State.Health.Status}}", i) == "healthy" for i in ids): break time.sleep(1) else: raise RuntimeError("API replicas did not become healthy within 90 seconds") time.sleep(6) def peak_rss_mib(cid): out = sh("docker", "exec", cid, "sh", "-c", "grep -E 'VmHWM|VmRSS' /proc/1/status") vals = {l.split(":")[0]: int(l.split()[1]) / 1024 for l in out.splitlines()} return round(vals.get("VmHWM", 0), 1), round(vals.get("VmRSS", 0), 1) async def register(s): name = "up_%s" % uuid.uuid4().hex[:12] pw = "upload-test-pw-" + uuid.uuid4().hex[:6] async with s.post(BASE + "/api/v1/users", json={"username": name, "display_name": "u", "password": pw}) as r: user = await r.json() cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode() async with s.post(BASE + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r: sess = await r.json() return user["id"], sess["token"] async def one_run(clients, data, token, chat_id, timeout=120): results = [] t0 = time.perf_counter() async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=timeout), connector=aiohttp.TCPConnector(limit=clients)) as s: async def up(i): t = time.perf_counter() try: async with s.post("%s/api/v1/chats/%s/attachments?filename=load%d.png" % (BASE, chat_id, i), data=data, headers={"Authorization": "Bearer " + token, "Content-Type": "image/png"}) as r: body = await r.json(content_type=None) results.append((r.status, time.perf_counter() - t, body.get("id") if r.status < 300 else body.get("error", {}).get("code"))) except Exception as exc: results.append((type(exc).__name__, time.perf_counter() - t, None)) await asyncio.gather(*(up(i) for i in range(clients))) return results, time.perf_counter() - t0 async def main(args): data = noisy_png(args.side, 1) print("размер файла: %.1f MiB, sha256 %s" % (len(data) / 1048576, hashlib.sha256(data).hexdigest()[:12])) report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "file_mib": round(len(data) / 1048576, 2), "clients": args.clients, "api_memory_limit": "256m", "runs": []} for conc in args.concurrency: set_api(conc, 1) cid = api_containers()[0] limits = json.loads(sh("docker", "inspect", "-f", '{"memory_bytes":{{.HostConfig.Memory}},"nano_cpus":{{.HostConfig.NanoCpus}}}', cid)) assert limits == {"memory_bytes": 256 * 1024 * 1024, "nano_cpus": 1000000000}, limits async with aiohttp.ClientSession() as s: uid, token = await register(s) async with s.post(BASE + "/api/v1/chats", json={"title": "mem", "member_ids": [uid]}, headers={"Authorization": "Bearer " + token}) as r: chat = await r.json() baseline_hwm, _ = peak_rss_mib(cid) results, wall = await one_run(args.clients, data, token, chat["id"]) peak, rss = peak_rss_mib(cid) statuses = {} for st, _, _ in results: statuses[str(st)] = statuses.get(str(st), 0) + 1 ok_lat = sorted(t for st, t, _ in results if st == 201) # проверка целостности: скачиваем каждый принятый файл и сверяем sha256 intact = 0 async with aiohttp.ClientSession() as s: for st, _, att in results: if st == 201: async with s.get("%s/api/v1/attachments/%s/content" % (BASE, att), headers={"Authorization": "Bearer " + token}) as r: if hashlib.sha256(await r.read()).hexdigest() == hashlib.sha256(data).hexdigest(): intact += 1 oom = sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", cid) entry = {"UPLOAD_CONCURRENCY": conc, "statuses": statuses, "wall_seconds": round(wall, 2), "peak_rss_mib": peak, "resource_limits": limits, "rss_before_mib": baseline_hwm, "rss_after_mib": rss, "oom_killed": oom, "accepted_files_intact": "%d/%d" % (intact, statuses.get("201", 0)), "p50_s": round(ok_lat[len(ok_lat) // 2], 2) if ok_lat else None, "max_s": round(ok_lat[-1], 2) if ok_lat else None} print(json.dumps(entry, ensure_ascii=False)) report["runs"].append(entry) set_api(4, 2) with open(os.path.join(args.out, "upload_memory.json"), "w", encoding="utf-8") as fh: json.dump(report, fh, ensure_ascii=False, indent=2) for entry in report["runs"]: assert entry["statuses"] == {"201": args.clients}, entry assert entry["accepted_files_intact"] == "%d/%d" % (args.clients, args.clients), entry assert entry["oom_killed"] == "false" and 0 < entry["peak_rss_mib"] < 256, entry if __name__ == "__main__": ap = argparse.ArgumentParser() ap.add_argument("--out", default="evidence") ap.add_argument("--clients", type=int, default=12) ap.add_argument("--side", type=int, default=1700, help="сторона квадрата PNG-шума; 1700 -> около 8.7 MiB") ap.add_argument("--concurrency", type=int, nargs="+", default=[1, 4, 12]) asyncio.run(main(ap.parse_args()))