#!/usr/bin/env python3 """Эксперимент: N одновременных загрузок вложений и пик памяти API-реплик. Гипотеза: память реплики на загрузку ограничена (spool ≤ 1 MiB RAM, потоковый PUT в S3, проверка изображения в пуле из IMAGE_INSPECT_WORKERS потоков), поэтому пик RSS растёт не пропорционально N × размер файла. Профили: png — PNG ~9 MiB из шума (почти не сжимается): нагрузка на приём тела, spool, SHA-256 и PUT в S3. Генерируется стандартной библиотекой. jpeg — progressive JPEG 8000×5000 = 40 Мпикс (предел): худший случай декодера libjpeg. Нужен Pillow (poetry run python ...). Перед каждым прогоном API-реплики перезапускаются: VmHWM (пиковый RSS процесса) начинается заново. Память опрашивается внутри контейнеров каждые 0.2 с (cgroup memory.current и VmRSS процесса). Все N запросов стартуют по одному барьеру. Tokens не печатаются и не сохраняются. """ from __future__ import annotations import argparse import hashlib import io import json import os import platform import statistics import struct import subprocess import sys import threading import time import urllib.error import urllib.request import zlib from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client # noqa: E402 REQUEST_TIMEOUT_SEC = 120 READY_TIMEOUT_SEC = 90 SAMPLE_INTERVAL_SEC = 0.2 COMPOSE = ["docker", "compose", "--env-file", ".env"] SAMPLER = ( "while :; do " "echo $(cat /sys/fs/cgroup/memory.current) " "$(awk '/VmRSS/ {print $2}' /proc/1/status); " f"sleep {SAMPLE_INTERVAL_SEC}; done" ) def png_noise(width: int, height: int) -> bytes: """Настоящий RGB PNG из случайных байт: размер ≈ width × height × 3""" def chunk(kind: bytes, data: bytes) -> bytes: return ( struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data)) ) raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height)) header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) return ( b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + chunk(b"IDAT", zlib.compress(raw, 1)) + chunk(b"IEND", b"") ) def jpeg_worst_case() -> bytes: from PIL import Image Image.MAX_IMAGE_PIXELS = None image = Image.effect_noise((8000, 5000), 64).convert("RGB") buffer = io.BytesIO() image.save(buffer, "JPEG", quality=10, progressive=True, subsampling=0) return buffer.getvalue() def compose(*args: str) -> str: return subprocess.run( [*COMPOSE, *args], cwd=ROOT, capture_output=True, text=True, check=True ).stdout def api_containers() -> list[str]: return compose("ps", "-q", "api").split() def container_name(container: str) -> str: return subprocess.run( ["docker", "inspect", "-f", "{{.Name}}", container], capture_output=True, text=True, check=True, ).stdout.strip().lstrip("/") def process_status(container: str) -> dict: """VmRSS/VmHWM процесса API (PID 1), байты, и его X-Instance-Id""" output = subprocess.run( ["docker", "exec", container, "sh", "-c", "grep -E 'VmRSS|VmHWM' /proc/1/status; cat /tmp/instance-id"], capture_output=True, text=True, check=True, ).stdout.split("\n") values: dict = {line.split(":")[0]: int(line.split()[1]) * 1024 for line in output[:2]} values["instance"] = output[2].strip() return values def restart_api(base_url: str) -> None: compose("restart", "api") deadline = time.monotonic() + READY_TIMEOUT_SEC while time.monotonic() < deadline: states = [ subprocess.run( ["docker", "inspect", "-f", "{{.State.Health.Status}}", c], capture_output=True, text=True, ).stdout.strip() for c in api_containers() ] if states and all(state == "healthy" for state in states): return time.sleep(1) raise SystemExit("API replicas are not healthy after restart") class Sampler(threading.Thread): """Опрос памяти одного контейнера изнутри одним долгим docker exec""" def __init__(self, container: str) -> None: super().__init__(daemon=True) self.container = container self.cgroup_max = 0 self.rss_max = 0 self.samples = 0 self._process = subprocess.Popen( ["docker", "exec", container, "sh", "-c", SAMPLER], stdout=subprocess.PIPE, text=True, ) def run(self) -> None: assert self._process.stdout is not None for line in self._process.stdout: parts = line.split() if len(parts) == 2: self.cgroup_max = max(self.cgroup_max, int(parts[0])) self.rss_max = max(self.rss_max, int(parts[1]) * 1024) self.samples += 1 def stop(self) -> None: self._process.terminate() self._process.wait() def upload_once(base: str, token: str, chat_id: str, content: bytes, content_type: str) -> dict: request = urllib.request.Request( f"{base}/api/v1/chats/{chat_id}/attachments?filename=experiment", data=content, method="POST", headers={"Authorization": "Bearer " + token, "Content-Type": content_type}, ) started = time.monotonic() try: with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT_SEC) as response: body, status, instance = response.read(), response.status, response.headers.get("X-Instance-Id") except urllib.error.HTTPError as error: body, status, instance = error.read(), error.code, error.headers.get("X-Instance-Id") except OSError as error: return {"status": None, "error": type(error).__name__, "seconds": time.monotonic() - started} result = {"status": status, "instance": instance, "seconds": time.monotonic() - started} if status == 202: result["id"] = json.loads(body)["id"] return result def percentile(values: list[float], share: float) -> float: ordered = sorted(values) return ordered[min(len(ordered) - 1, round(share * (len(ordered) - 1)))] def run_round(client: Client, actor: dict, chat: dict, profile: str, content: bytes, content_type: str, concurrency: int, restart: bool) -> dict: if restart: restart_api(client.base) containers = api_containers() # Прогрев: первый запрос каждой реплики открывает пулы БД/S3 warmup = png_noise(4, 4) for _ in range(2 * len(containers)): upload_once(client.base, actor["token"], chat["id"], warmup, "image/png") before = {container_name(c): process_status(c) for c in containers} samplers = [Sampler(c) for c in containers] for sampler in samplers: sampler.start() time.sleep(1) barrier = threading.Barrier(concurrency) results: list[dict] = [{} for _ in range(concurrency)] def worker(index: int) -> None: barrier.wait() results[index] = upload_once(client.base, actor["token"], chat["id"], content, content_type) started = time.monotonic() threads = [threading.Thread(target=worker, args=(i,)) for i in range(concurrency)] for thread in threads: thread.start() for thread in threads: thread.join() wall = time.monotonic() - started time.sleep(1) for sampler in samplers: sampler.stop() after = {container_name(c): process_status(c) for c in containers} ok = [r for r in results if r.get("status") == 202] sample = ok[0]["id"] if ok else None checksum_ok = None if sample: client.wait_ready(actor, {"id": sample}) downloaded = client.call("GET", f"/api/v1/attachments/{sample}/content", actor=actor)["bytes"] checksum_ok = hashlib.sha256(downloaded).digest() == hashlib.sha256(content).digest() seconds = [r["seconds"] for r in ok] mib = 1024 * 1024 replicas = {} for sampler in samplers: name = container_name(sampler.container) replicas[name] = { "uploads_served": sum(1 for r in ok if r.get("instance") == after[name].get("instance")), "rss_before_mib": round(before[name]["VmRSS"] / mib, 1), "rss_peak_process_vmhwm_mib": round(after[name]["VmHWM"] / mib, 1), "rss_peak_sampled_mib": round(sampler.rss_max / mib, 1), "cgroup_peak_sampled_mib": round(sampler.cgroup_max / mib, 1), "samples": sampler.samples, } by_instance: dict[str, int] = {} for r in ok: by_instance[r["instance"]] = by_instance.get(r["instance"], 0) + 1 return { "profile": profile, "file_bytes": len(content), "concurrency": concurrency, "replicas_restarted_before": restart, "status_counts": {str(s): sum(1 for r in results if r.get("status") == s) for s in sorted({r.get("status") for r in results}, key=str)}, "uploads_by_instance": by_instance, "wall_seconds": round(wall, 2), "latency_seconds": { "p50": round(percentile(seconds, 0.5), 3), "p95": round(percentile(seconds, 0.95), 3), "max": round(max(seconds), 3), } if seconds else None, "throughput_mib_per_second": round(len(ok) * len(content) / mib / wall, 1), "download_sha256_matches": checksum_ok, "replicas": replicas, } def environment() -> dict: info = json.loads(subprocess.run( ["docker", "info", "--format", "{{json .}}"], capture_output=True, text=True, check=True ).stdout) settings = {} for line in (ROOT / ".env").read_text().splitlines(): key = line.split("=", 1)[0] if key in {"IMAGE_INSPECT_WORKERS", "UPLOAD_SPOOL_MEMORY_BYTES", "MAX_UPLOAD_BYTES", "MAX_IMAGE_PIXELS", "API_REPLICAS", "S3_POOL_MAX_SIZE"}: settings[key] = line.split("=", 1)[1] return { "host": f"{platform.system()} {platform.machine()}", "docker": info.get("ServerVersion"), "docker_vm_cpus": info.get("NCPU"), "docker_vm_memory_gib": round(info.get("MemTotal", 0) / 2**30, 1), "api_limits": "cpus 1.0, memory 512M per replica (compose.yaml)", "s3": "chrislusf/seaweedfs:4.48", "settings": settings, } def main() -> None: parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) parser.add_argument("--base-url", default=None) parser.add_argument("--output", default="evidence/experiment-uploads.json") parser.add_argument("--concurrency", default="10,20") parser.add_argument("--profiles", default="png,jpeg") parser.add_argument("--no-restart", action="store_true") args = parser.parse_args() client = Client(args.base_url) alice, _, _, chat = client.bootstrap() payloads = {} if "png" in args.profiles: payloads["png"] = (png_noise(1750, 1750), "image/png") if "jpeg" in args.profiles: try: payloads["jpeg"] = (jpeg_worst_case(), "image/jpeg") except ImportError: print("jpeg profile skipped: Pillow is not installed (use poetry run)") report = {"environment": environment(), "rounds": []} for profile, (content, content_type) in payloads.items(): for concurrency in (int(x) for x in args.concurrency.split(",")): result = run_round(client, alice, chat, profile, content, content_type, concurrency, not args.no_restart) report["rounds"].append(result) print(json.dumps(result, ensure_ascii=False)) output = ROOT / args.output output.parent.mkdir(parents=True, exist_ok=True) output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n") print("saved:", args.output) if __name__ == "__main__": main()