#!/usr/bin/env python3 """Пик памяти API при одновременных загрузках файлов ~9.4 MiB. Для каждого значения лимита слотов (APP_CONFIG__ATTACHMENTS__MAX_CONCURRENT_UPLOADS через UPLOAD_CONCURRENCY) реплики API пересоздаются с этим лимитом. Перед каждым сценарием: прогрев (маленькие загрузки, пока каждая реплика не обслужит хотя бы одну) -> сброс пика RSS процесса (echo 5 > /proc/1/clear_refs) -> базовый VmHWM -> N одновременных загрузок через LB -> VmHWM после. Разница — сколько памяти добавили сами загрузки. Запуск из venv (нужен Pillow для генерации файла), стенд уже поднят: python scripts/upload_memory_experiment.py --limits 8,64 --concurrency 1,10,20 """ import argparse import base64 import io import json import os import statistics import subprocess import threading import time import urllib.error import urllib.request import uuid from collections import Counter from pathlib import Path from PIL import Image ROOT = Path(__file__).resolve().parents[1] PROJECT = "mtusi-lab4" MIB = 1024 * 1024 def call(base, method, path, body=None, headers=None, timeout=60): req = urllib.request.Request(base + path, data=body, method=method, headers=headers or {}) start = time.monotonic() try: with urllib.request.urlopen(req, timeout=timeout) as resp: status, resp_headers, data = resp.status, resp.headers, resp.read() except urllib.error.HTTPError as exc: status, resp_headers, data = exc.code, exc.headers, exc.read() except OSError as exc: return type(exc).__name__, {}, b"", time.monotonic() - start return status, resp_headers, data, time.monotonic() - start def compose(*args, env=None): subprocess.run(["docker", "compose", *args], check=True, cwd=ROOT, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) def api_containers(): out = subprocess.run(["docker", "ps", "-q", "--filter", f"label=com.docker.compose.project={PROJECT}", "--filter", "label=com.docker.compose.service=api"], check=True, capture_output=True, text=True).stdout return out.split() def reset_peak(container): # Сбросить VmHWM процесса API (PID 1 = python main.py; владелец — тот же appuser). subprocess.run(["docker", "exec", container, "sh", "-c", "echo 5 > /proc/1/clear_refs"], check=True) def memory_of(container): """VmHWM/VmRSS процесса API, MiB.""" status = subprocess.run(["docker", "exec", container, "cat", "/proc/1/status"], check=True, capture_output=True, text=True).stdout values = {line.split(":")[0]: int(line.split()[1]) / 1024 for line in status.splitlines() if line.startswith(("VmHWM", "VmRSS"))} return {"rss_peak_mib": round(values["VmHWM"], 1), "rss_now_mib": round(values["VmRSS"], 1)} def make_noise_png(target_mib): # Случайный шум почти не сжимается: файл PNG ~ размер сырых пикселей. side = int((target_mib * MIB / 3) ** 0.5) image = Image.frombytes("RGB", (side, side), os.urandom(side * side * 3)) buffer = io.BytesIO() image.save(buffer, "PNG", compress_level=1) return buffer.getvalue() def warm_up(base, auth, small_png, chat_id, expected): """Маленькие загрузки, пока каждая реплика не обслужит хотя бы одну (импорты, пулы соединений).""" seen = set() deadline = time.monotonic() + 60 while len(seen) < expected and time.monotonic() < deadline: status, headers, _, _ = call(base, "POST", f"/api/v1/chats/{chat_id}/attachments?filename=warm.png", small_png, {**auth, "Content-Type": "image/png"}) if status == 201: seen.add(headers.get("X-Instance-Id")) else: time.sleep(0.5) if len(seen) < expected: raise SystemExit("не дождались обеих реплик") def run_scenario(base, auth, chat_id, payload, small_png, concurrency): containers = api_containers() warm_up(base, auth, small_png, chat_id, len(containers)) for container in containers: reset_peak(container) before = {c: memory_of(c) for c in containers} results = [] barrier = threading.Barrier(concurrency) def worker(): barrier.wait() status, headers, _, seconds = call(base, "POST", f"/api/v1/chats/{chat_id}/attachments?filename=big.png", payload, {**auth, "Content-Type": "image/png"}) results.append({"status": status, "instance": headers.get("X-Instance-Id") if headers else None, "seconds": seconds}) started = time.monotonic() threads = [threading.Thread(target=worker) for _ in range(concurrency)] for thread in threads: thread.start() for thread in threads: thread.join() wall = time.monotonic() - started after = {c: memory_of(c) for c in containers} ok = sorted(r["seconds"] for r in results if r["status"] == 201) served = Counter(r["instance"] for r in results if r["status"] == 201) replicas = {} for c in containers: replicas[c] = {"uploads_201": served.get(c, 0), "before": before[c], "after": after[c], "rss_peak_growth_mib": round(after[c]["rss_peak_mib"] - before[c]["rss_now_mib"], 1)} return { "concurrency": concurrency, "statuses": dict(Counter(str(r["status"]) for r in results)), "wall_seconds": round(wall, 2), "latency_201_seconds": {"p50": round(statistics.median(ok), 2), "max": round(ok[-1], 2)} if ok else None, "replicas": replicas, "max_rss_peak_mib": max(r["after"]["rss_peak_mib"] for r in replicas.values()), "max_rss_peak_growth_mib": max(r["rss_peak_growth_mib"] for r in replicas.values()), } def main(): parser = argparse.ArgumentParser() parser.add_argument("--base-url", default="http://localhost:8080") parser.add_argument("--limits", default="8,64", help="значения лимита слотов на реплику") parser.add_argument("--concurrency", default="1,10,20") parser.add_argument("--file-mib", type=float, default=9.4) parser.add_argument("--output", default=str(ROOT / "evidence" / "upload-memory.json")) args = parser.parse_args() base = args.base_url.rstrip("/") payload = make_noise_png(args.file_mib) small_png = make_noise_png(0.01) if len(payload) > 10 * MIB: raise SystemExit("файл больше 10 MiB") username, password = f"mem_{uuid.uuid4().hex[:8]}", "memory-experiment-pass" user = json.loads(call(base, "POST", "/api/v1/users", json.dumps( {"username": username, "display_name": "mem", "password": password}).encode(), {"Content-Type": "application/json"})[2]) basic = base64.b64encode(f"{username}:{password}".encode()).decode() token = json.loads(call(base, "POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})[2])["token"] auth = {"Authorization": "Bearer " + token} chat_id = json.loads(call(base, "POST", "/api/v1/chats", json.dumps( {"title": "memory", "member_ids": [user["id"]]}).encode(), {"Content-Type": "application/json", **auth})[2])["id"] report = {"file_bytes": len(payload), "api_limits": "1 CPU, 512M per replica, 2 replicas", "method": "peak RSS (VmHWM) of the API process, reset after warm-up; growth = peak after - RSS before", "runs": []} try: for limit in [int(x) for x in args.limits.split(",")]: env = dict(os.environ, UPLOAD_CONCURRENCY=str(limit)) compose("up", "-d", "--wait", "api", "lb", env=env) for concurrency in [int(x) for x in args.concurrency.split(",")]: result = run_scenario(base, auth, chat_id, payload, small_png, concurrency) result["upload_slots_per_replica"] = limit report["runs"].append(result) print(json.dumps({k: v for k, v in result.items() if k != "replicas"}, ensure_ascii=False)) finally: # Вернуть реплики к лимиту по умолчанию. env = {k: v for k, v in os.environ.items() if k != "UPLOAD_CONCURRENCY"} compose("up", "-d", "--wait", "api", "lb", env=env) Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") if __name__ == "__main__": main()