forked from mtusi2026/mtusi2026lab5
318 lines
12 KiB
Python
318 lines
12 KiB
Python
#!/usr/bin/env python3
|
||
"""Эксперимент: N одновременных загрузок вложений и пик памяти API-реплик.
|
||
|
||
Гипотеза: память реплики на загрузку ограничена (spool ≤ 1 MiB RAM, потоковый
|
||
PUT в S3, проверка изображения в пуле из IMAGE_INSPECT_WORKERS потоков),
|
||
поэтому пик RSS растёт не пропорционально N × размер файла.
|
||
|
||
Профили:
|
||
png — PNG ~9 MiB из шума (почти не сжимается): нагрузка на приём тела,
|
||
spool, SHA-256 и PUT в S3. Генерируется стандартной библиотекой.
|
||
jpeg — progressive JPEG 8000×5000 = 40 Мпикс (предел): худший случай
|
||
декодера libjpeg. Нужен Pillow (poetry run python ...).
|
||
|
||
Перед каждым прогоном API-реплики перезапускаются: VmHWM (пиковый RSS
|
||
процесса) начинается заново. Память опрашивается внутри контейнеров каждые
|
||
0.2 с (cgroup memory.current и VmRSS процесса). Все N запросов стартуют по
|
||
одному барьеру. Tokens не печатаются и не сохраняются.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import io
|
||
import json
|
||
import os
|
||
import platform
|
||
import statistics
|
||
import struct
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
import zlib
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(ROOT / "tests"))
|
||
|
||
from client import Client # noqa: E402
|
||
|
||
REQUEST_TIMEOUT_SEC = 120
|
||
READY_TIMEOUT_SEC = 90
|
||
SAMPLE_INTERVAL_SEC = 0.2
|
||
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
||
SAMPLER = (
|
||
"while :; do "
|
||
"echo $(cat /sys/fs/cgroup/memory.current) "
|
||
"$(awk '/VmRSS/ {print $2}' /proc/1/status); "
|
||
f"sleep {SAMPLE_INTERVAL_SEC}; done"
|
||
)
|
||
|
||
|
||
def png_noise(width: int, height: int) -> bytes:
|
||
"""Настоящий RGB PNG из случайных байт: размер ≈ width × height × 3"""
|
||
|
||
def chunk(kind: bytes, data: bytes) -> bytes:
|
||
return (
|
||
struct.pack(">I", len(data))
|
||
+ kind
|
||
+ data
|
||
+ struct.pack(">I", zlib.crc32(kind + data))
|
||
)
|
||
|
||
raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height))
|
||
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||
return (
|
||
b"\x89PNG\r\n\x1a\n"
|
||
+ chunk(b"IHDR", header)
|
||
+ chunk(b"IDAT", zlib.compress(raw, 1))
|
||
+ chunk(b"IEND", b"")
|
||
)
|
||
|
||
|
||
def jpeg_worst_case() -> bytes:
|
||
from PIL import Image
|
||
|
||
Image.MAX_IMAGE_PIXELS = None
|
||
image = Image.effect_noise((8000, 5000), 64).convert("RGB")
|
||
buffer = io.BytesIO()
|
||
image.save(buffer, "JPEG", quality=10, progressive=True, subsampling=0)
|
||
return buffer.getvalue()
|
||
|
||
|
||
def compose(*args: str) -> str:
|
||
return subprocess.run(
|
||
[*COMPOSE, *args], cwd=ROOT, capture_output=True, text=True, check=True
|
||
).stdout
|
||
|
||
|
||
def api_containers() -> list[str]:
|
||
return compose("ps", "-q", "api").split()
|
||
|
||
|
||
def container_name(container: str) -> str:
|
||
return subprocess.run(
|
||
["docker", "inspect", "-f", "{{.Name}}", container],
|
||
capture_output=True, text=True, check=True,
|
||
).stdout.strip().lstrip("/")
|
||
|
||
|
||
def process_status(container: str) -> dict:
|
||
"""VmRSS/VmHWM процесса API (PID 1), байты, и его X-Instance-Id"""
|
||
output = subprocess.run(
|
||
["docker", "exec", container, "sh", "-c",
|
||
"grep -E 'VmRSS|VmHWM' /proc/1/status; cat /tmp/instance-id"],
|
||
capture_output=True, text=True, check=True,
|
||
).stdout.split("\n")
|
||
values: dict = {line.split(":")[0]: int(line.split()[1]) * 1024 for line in output[:2]}
|
||
values["instance"] = output[2].strip()
|
||
return values
|
||
|
||
|
||
def restart_api(base_url: str) -> None:
|
||
compose("restart", "api")
|
||
deadline = time.monotonic() + READY_TIMEOUT_SEC
|
||
while time.monotonic() < deadline:
|
||
states = [
|
||
subprocess.run(
|
||
["docker", "inspect", "-f", "{{.State.Health.Status}}", c],
|
||
capture_output=True, text=True,
|
||
).stdout.strip()
|
||
for c in api_containers()
|
||
]
|
||
if states and all(state == "healthy" for state in states):
|
||
return
|
||
time.sleep(1)
|
||
raise SystemExit("API replicas are not healthy after restart")
|
||
|
||
|
||
class Sampler(threading.Thread):
|
||
"""Опрос памяти одного контейнера изнутри одним долгим docker exec"""
|
||
|
||
def __init__(self, container: str) -> None:
|
||
super().__init__(daemon=True)
|
||
self.container = container
|
||
self.cgroup_max = 0
|
||
self.rss_max = 0
|
||
self.samples = 0
|
||
self._process = subprocess.Popen(
|
||
["docker", "exec", container, "sh", "-c", SAMPLER],
|
||
stdout=subprocess.PIPE, text=True,
|
||
)
|
||
|
||
def run(self) -> None:
|
||
assert self._process.stdout is not None
|
||
for line in self._process.stdout:
|
||
parts = line.split()
|
||
if len(parts) == 2:
|
||
self.cgroup_max = max(self.cgroup_max, int(parts[0]))
|
||
self.rss_max = max(self.rss_max, int(parts[1]) * 1024)
|
||
self.samples += 1
|
||
|
||
def stop(self) -> None:
|
||
self._process.terminate()
|
||
self._process.wait()
|
||
|
||
|
||
def upload_once(base: str, token: str, chat_id: str, content: bytes, content_type: str) -> dict:
|
||
request = urllib.request.Request(
|
||
f"{base}/api/v1/chats/{chat_id}/attachments?filename=experiment",
|
||
data=content,
|
||
method="POST",
|
||
headers={"Authorization": "Bearer " + token, "Content-Type": content_type},
|
||
)
|
||
started = time.monotonic()
|
||
try:
|
||
with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT_SEC) as response:
|
||
body, status, instance = response.read(), response.status, response.headers.get("X-Instance-Id")
|
||
except urllib.error.HTTPError as error:
|
||
body, status, instance = error.read(), error.code, error.headers.get("X-Instance-Id")
|
||
except OSError as error:
|
||
return {"status": None, "error": type(error).__name__, "seconds": time.monotonic() - started}
|
||
result = {"status": status, "instance": instance, "seconds": time.monotonic() - started}
|
||
if status == 201:
|
||
result["id"] = json.loads(body)["id"]
|
||
return result
|
||
|
||
|
||
def percentile(values: list[float], share: float) -> float:
|
||
ordered = sorted(values)
|
||
return ordered[min(len(ordered) - 1, round(share * (len(ordered) - 1)))]
|
||
|
||
|
||
def run_round(client: Client, actor: dict, chat: dict, profile: str, content: bytes,
|
||
content_type: str, concurrency: int, restart: bool) -> dict:
|
||
if restart:
|
||
restart_api(client.base)
|
||
containers = api_containers()
|
||
# Прогрев: первый запрос каждой реплики открывает пулы БД/S3
|
||
warmup = png_noise(4, 4)
|
||
for _ in range(2 * len(containers)):
|
||
upload_once(client.base, actor["token"], chat["id"], warmup, "image/png")
|
||
before = {container_name(c): process_status(c) for c in containers}
|
||
samplers = [Sampler(c) for c in containers]
|
||
for sampler in samplers:
|
||
sampler.start()
|
||
time.sleep(1)
|
||
|
||
barrier = threading.Barrier(concurrency)
|
||
results: list[dict] = [{} for _ in range(concurrency)]
|
||
|
||
def worker(index: int) -> None:
|
||
barrier.wait()
|
||
results[index] = upload_once(client.base, actor["token"], chat["id"], content, content_type)
|
||
|
||
started = time.monotonic()
|
||
threads = [threading.Thread(target=worker, args=(i,)) for i in range(concurrency)]
|
||
for thread in threads:
|
||
thread.start()
|
||
for thread in threads:
|
||
thread.join()
|
||
wall = time.monotonic() - started
|
||
time.sleep(1)
|
||
for sampler in samplers:
|
||
sampler.stop()
|
||
after = {container_name(c): process_status(c) for c in containers}
|
||
|
||
ok = [r for r in results if r.get("status") == 201]
|
||
sample = ok[0]["id"] if ok else None
|
||
checksum_ok = None
|
||
if sample:
|
||
downloaded = client.call("GET", f"/api/v1/attachments/{sample}/content", actor=actor)["bytes"]
|
||
checksum_ok = hashlib.sha256(downloaded).digest() == hashlib.sha256(content).digest()
|
||
seconds = [r["seconds"] for r in ok]
|
||
mib = 1024 * 1024
|
||
replicas = {}
|
||
for sampler in samplers:
|
||
name = container_name(sampler.container)
|
||
replicas[name] = {
|
||
"uploads_served": sum(1 for r in ok if r.get("instance") == after[name].get("instance")),
|
||
"rss_before_mib": round(before[name]["VmRSS"] / mib, 1),
|
||
"rss_peak_process_vmhwm_mib": round(after[name]["VmHWM"] / mib, 1),
|
||
"rss_peak_sampled_mib": round(sampler.rss_max / mib, 1),
|
||
"cgroup_peak_sampled_mib": round(sampler.cgroup_max / mib, 1),
|
||
"samples": sampler.samples,
|
||
}
|
||
by_instance: dict[str, int] = {}
|
||
for r in ok:
|
||
by_instance[r["instance"]] = by_instance.get(r["instance"], 0) + 1
|
||
return {
|
||
"profile": profile,
|
||
"file_bytes": len(content),
|
||
"concurrency": concurrency,
|
||
"replicas_restarted_before": restart,
|
||
"status_counts": {str(s): sum(1 for r in results if r.get("status") == s)
|
||
for s in sorted({r.get("status") for r in results}, key=str)},
|
||
"uploads_by_instance": by_instance,
|
||
"wall_seconds": round(wall, 2),
|
||
"latency_seconds": {
|
||
"p50": round(percentile(seconds, 0.5), 3),
|
||
"p95": round(percentile(seconds, 0.95), 3),
|
||
"max": round(max(seconds), 3),
|
||
} if seconds else None,
|
||
"throughput_mib_per_second": round(len(ok) * len(content) / mib / wall, 1),
|
||
"download_sha256_matches": checksum_ok,
|
||
"replicas": replicas,
|
||
}
|
||
|
||
|
||
def environment() -> dict:
|
||
info = json.loads(subprocess.run(
|
||
["docker", "info", "--format", "{{json .}}"], capture_output=True, text=True, check=True
|
||
).stdout)
|
||
settings = {}
|
||
for line in (ROOT / ".env").read_text().splitlines():
|
||
key = line.split("=", 1)[0]
|
||
if key in {"IMAGE_INSPECT_WORKERS", "UPLOAD_SPOOL_MEMORY_BYTES", "MAX_UPLOAD_BYTES",
|
||
"MAX_IMAGE_PIXELS", "API_REPLICAS", "S3_POOL_MAX_SIZE"}:
|
||
settings[key] = line.split("=", 1)[1]
|
||
return {
|
||
"host": f"{platform.system()} {platform.machine()}",
|
||
"docker": info.get("ServerVersion"),
|
||
"docker_vm_cpus": info.get("NCPU"),
|
||
"docker_vm_memory_gib": round(info.get("MemTotal", 0) / 2**30, 1),
|
||
"api_limits": "cpus 1.0, memory 512M per replica (compose.yaml)",
|
||
"s3": "chrislusf/seaweedfs:4.48",
|
||
"settings": settings,
|
||
}
|
||
|
||
|
||
def main() -> None:
|
||
parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||
parser.add_argument("--base-url", default=None)
|
||
parser.add_argument("--output", default="evidence/experiment-uploads.json")
|
||
parser.add_argument("--concurrency", default="10,20")
|
||
parser.add_argument("--profiles", default="png,jpeg")
|
||
parser.add_argument("--no-restart", action="store_true")
|
||
args = parser.parse_args()
|
||
|
||
client = Client(args.base_url)
|
||
alice, _, _, chat = client.bootstrap()
|
||
payloads = {}
|
||
if "png" in args.profiles:
|
||
payloads["png"] = (png_noise(1750, 1750), "image/png")
|
||
if "jpeg" in args.profiles:
|
||
try:
|
||
payloads["jpeg"] = (jpeg_worst_case(), "image/jpeg")
|
||
except ImportError:
|
||
print("jpeg profile skipped: Pillow is not installed (use poetry run)")
|
||
report = {"environment": environment(), "rounds": []}
|
||
for profile, (content, content_type) in payloads.items():
|
||
for concurrency in (int(x) for x in args.concurrency.split(",")):
|
||
result = run_round(client, alice, chat, profile, content, content_type,
|
||
concurrency, not args.no_restart)
|
||
report["rounds"].append(result)
|
||
print(json.dumps(result, ensure_ascii=False))
|
||
output = ROOT / args.output
|
||
output.parent.mkdir(parents=True, exist_ok=True)
|
||
output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
||
print("saved:", args.output)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|