forked from mtusi2026/mtusi2026lab4
feat: Саватеев А трек 4
This commit is contained in:
@@ -0,0 +1,317 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Эксперимент: N одновременных загрузок вложений и пик памяти API-реплик.
|
||||
|
||||
Гипотеза: память реплики на загрузку ограничена (spool ≤ 1 MiB RAM, потоковый
|
||||
PUT в S3, проверка изображения в пуле из IMAGE_INSPECT_WORKERS потоков),
|
||||
поэтому пик RSS растёт не пропорционально N × размер файла.
|
||||
|
||||
Профили:
|
||||
png — PNG ~9 MiB из шума (почти не сжимается): нагрузка на приём тела,
|
||||
spool, SHA-256 и PUT в S3. Генерируется стандартной библиотекой.
|
||||
jpeg — progressive JPEG 8000×5000 = 40 Мпикс (предел): худший случай
|
||||
декодера libjpeg. Нужен Pillow (poetry run python ...).
|
||||
|
||||
Перед каждым прогоном API-реплики перезапускаются: VmHWM (пиковый RSS
|
||||
процесса) начинается заново. Память опрашивается внутри контейнеров каждые
|
||||
0.2 с (cgroup memory.current и VmRSS процесса). Все N запросов стартуют по
|
||||
одному барьеру. Tokens не печатаются и не сохраняются.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import statistics
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client # noqa: E402
|
||||
|
||||
REQUEST_TIMEOUT_SEC = 120
|
||||
READY_TIMEOUT_SEC = 90
|
||||
SAMPLE_INTERVAL_SEC = 0.2
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
||||
SAMPLER = (
|
||||
"while :; do "
|
||||
"echo $(cat /sys/fs/cgroup/memory.current) "
|
||||
"$(awk '/VmRSS/ {print $2}' /proc/1/status); "
|
||||
f"sleep {SAMPLE_INTERVAL_SEC}; done"
|
||||
)
|
||||
|
||||
|
||||
def png_noise(width: int, height: int) -> bytes:
|
||||
"""Настоящий RGB PNG из случайных байт: размер ≈ width × height × 3"""
|
||||
|
||||
def chunk(kind: bytes, data: bytes) -> bytes:
|
||||
return (
|
||||
struct.pack(">I", len(data))
|
||||
+ kind
|
||||
+ data
|
||||
+ struct.pack(">I", zlib.crc32(kind + data))
|
||||
)
|
||||
|
||||
raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height))
|
||||
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
return (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", header)
|
||||
+ chunk(b"IDAT", zlib.compress(raw, 1))
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
|
||||
|
||||
def jpeg_worst_case() -> bytes:
|
||||
from PIL import Image
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = None
|
||||
image = Image.effect_noise((8000, 5000), 64).convert("RGB")
|
||||
buffer = io.BytesIO()
|
||||
image.save(buffer, "JPEG", quality=10, progressive=True, subsampling=0)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def compose(*args: str) -> str:
|
||||
return subprocess.run(
|
||||
[*COMPOSE, *args], cwd=ROOT, capture_output=True, text=True, check=True
|
||||
).stdout
|
||||
|
||||
|
||||
def api_containers() -> list[str]:
|
||||
return compose("ps", "-q", "api").split()
|
||||
|
||||
|
||||
def container_name(container: str) -> str:
|
||||
return subprocess.run(
|
||||
["docker", "inspect", "-f", "{{.Name}}", container],
|
||||
capture_output=True, text=True, check=True,
|
||||
).stdout.strip().lstrip("/")
|
||||
|
||||
|
||||
def process_status(container: str) -> dict:
|
||||
"""VmRSS/VmHWM процесса API (PID 1), байты, и его X-Instance-Id"""
|
||||
output = subprocess.run(
|
||||
["docker", "exec", container, "sh", "-c",
|
||||
"grep -E 'VmRSS|VmHWM' /proc/1/status; cat /tmp/instance-id"],
|
||||
capture_output=True, text=True, check=True,
|
||||
).stdout.split("\n")
|
||||
values: dict = {line.split(":")[0]: int(line.split()[1]) * 1024 for line in output[:2]}
|
||||
values["instance"] = output[2].strip()
|
||||
return values
|
||||
|
||||
|
||||
def restart_api(base_url: str) -> None:
|
||||
compose("restart", "api")
|
||||
deadline = time.monotonic() + READY_TIMEOUT_SEC
|
||||
while time.monotonic() < deadline:
|
||||
states = [
|
||||
subprocess.run(
|
||||
["docker", "inspect", "-f", "{{.State.Health.Status}}", c],
|
||||
capture_output=True, text=True,
|
||||
).stdout.strip()
|
||||
for c in api_containers()
|
||||
]
|
||||
if states and all(state == "healthy" for state in states):
|
||||
return
|
||||
time.sleep(1)
|
||||
raise SystemExit("API replicas are not healthy after restart")
|
||||
|
||||
|
||||
class Sampler(threading.Thread):
|
||||
"""Опрос памяти одного контейнера изнутри одним долгим docker exec"""
|
||||
|
||||
def __init__(self, container: str) -> None:
|
||||
super().__init__(daemon=True)
|
||||
self.container = container
|
||||
self.cgroup_max = 0
|
||||
self.rss_max = 0
|
||||
self.samples = 0
|
||||
self._process = subprocess.Popen(
|
||||
["docker", "exec", container, "sh", "-c", SAMPLER],
|
||||
stdout=subprocess.PIPE, text=True,
|
||||
)
|
||||
|
||||
def run(self) -> None:
|
||||
assert self._process.stdout is not None
|
||||
for line in self._process.stdout:
|
||||
parts = line.split()
|
||||
if len(parts) == 2:
|
||||
self.cgroup_max = max(self.cgroup_max, int(parts[0]))
|
||||
self.rss_max = max(self.rss_max, int(parts[1]) * 1024)
|
||||
self.samples += 1
|
||||
|
||||
def stop(self) -> None:
|
||||
self._process.terminate()
|
||||
self._process.wait()
|
||||
|
||||
|
||||
def upload_once(base: str, token: str, chat_id: str, content: bytes, content_type: str) -> dict:
|
||||
request = urllib.request.Request(
|
||||
f"{base}/api/v1/chats/{chat_id}/attachments?filename=experiment",
|
||||
data=content,
|
||||
method="POST",
|
||||
headers={"Authorization": "Bearer " + token, "Content-Type": content_type},
|
||||
)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT_SEC) as response:
|
||||
body, status, instance = response.read(), response.status, response.headers.get("X-Instance-Id")
|
||||
except urllib.error.HTTPError as error:
|
||||
body, status, instance = error.read(), error.code, error.headers.get("X-Instance-Id")
|
||||
except OSError as error:
|
||||
return {"status": None, "error": type(error).__name__, "seconds": time.monotonic() - started}
|
||||
result = {"status": status, "instance": instance, "seconds": time.monotonic() - started}
|
||||
if status == 201:
|
||||
result["id"] = json.loads(body)["id"]
|
||||
return result
|
||||
|
||||
|
||||
def percentile(values: list[float], share: float) -> float:
|
||||
ordered = sorted(values)
|
||||
return ordered[min(len(ordered) - 1, round(share * (len(ordered) - 1)))]
|
||||
|
||||
|
||||
def run_round(client: Client, actor: dict, chat: dict, profile: str, content: bytes,
|
||||
content_type: str, concurrency: int, restart: bool) -> dict:
|
||||
if restart:
|
||||
restart_api(client.base)
|
||||
containers = api_containers()
|
||||
# Прогрев: первый запрос каждой реплики открывает пулы БД/S3
|
||||
warmup = png_noise(4, 4)
|
||||
for _ in range(2 * len(containers)):
|
||||
upload_once(client.base, actor["token"], chat["id"], warmup, "image/png")
|
||||
before = {container_name(c): process_status(c) for c in containers}
|
||||
samplers = [Sampler(c) for c in containers]
|
||||
for sampler in samplers:
|
||||
sampler.start()
|
||||
time.sleep(1)
|
||||
|
||||
barrier = threading.Barrier(concurrency)
|
||||
results: list[dict] = [{} for _ in range(concurrency)]
|
||||
|
||||
def worker(index: int) -> None:
|
||||
barrier.wait()
|
||||
results[index] = upload_once(client.base, actor["token"], chat["id"], content, content_type)
|
||||
|
||||
started = time.monotonic()
|
||||
threads = [threading.Thread(target=worker, args=(i,)) for i in range(concurrency)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
wall = time.monotonic() - started
|
||||
time.sleep(1)
|
||||
for sampler in samplers:
|
||||
sampler.stop()
|
||||
after = {container_name(c): process_status(c) for c in containers}
|
||||
|
||||
ok = [r for r in results if r.get("status") == 201]
|
||||
sample = ok[0]["id"] if ok else None
|
||||
checksum_ok = None
|
||||
if sample:
|
||||
downloaded = client.call("GET", f"/api/v1/attachments/{sample}/content", actor=actor)["bytes"]
|
||||
checksum_ok = hashlib.sha256(downloaded).digest() == hashlib.sha256(content).digest()
|
||||
seconds = [r["seconds"] for r in ok]
|
||||
mib = 1024 * 1024
|
||||
replicas = {}
|
||||
for sampler in samplers:
|
||||
name = container_name(sampler.container)
|
||||
replicas[name] = {
|
||||
"uploads_served": sum(1 for r in ok if r.get("instance") == after[name].get("instance")),
|
||||
"rss_before_mib": round(before[name]["VmRSS"] / mib, 1),
|
||||
"rss_peak_process_vmhwm_mib": round(after[name]["VmHWM"] / mib, 1),
|
||||
"rss_peak_sampled_mib": round(sampler.rss_max / mib, 1),
|
||||
"cgroup_peak_sampled_mib": round(sampler.cgroup_max / mib, 1),
|
||||
"samples": sampler.samples,
|
||||
}
|
||||
by_instance: dict[str, int] = {}
|
||||
for r in ok:
|
||||
by_instance[r["instance"]] = by_instance.get(r["instance"], 0) + 1
|
||||
return {
|
||||
"profile": profile,
|
||||
"file_bytes": len(content),
|
||||
"concurrency": concurrency,
|
||||
"replicas_restarted_before": restart,
|
||||
"status_counts": {str(s): sum(1 for r in results if r.get("status") == s)
|
||||
for s in sorted({r.get("status") for r in results}, key=str)},
|
||||
"uploads_by_instance": by_instance,
|
||||
"wall_seconds": round(wall, 2),
|
||||
"latency_seconds": {
|
||||
"p50": round(percentile(seconds, 0.5), 3),
|
||||
"p95": round(percentile(seconds, 0.95), 3),
|
||||
"max": round(max(seconds), 3),
|
||||
} if seconds else None,
|
||||
"throughput_mib_per_second": round(len(ok) * len(content) / mib / wall, 1),
|
||||
"download_sha256_matches": checksum_ok,
|
||||
"replicas": replicas,
|
||||
}
|
||||
|
||||
|
||||
def environment() -> dict:
|
||||
info = json.loads(subprocess.run(
|
||||
["docker", "info", "--format", "{{json .}}"], capture_output=True, text=True, check=True
|
||||
).stdout)
|
||||
settings = {}
|
||||
for line in (ROOT / ".env").read_text().splitlines():
|
||||
key = line.split("=", 1)[0]
|
||||
if key in {"IMAGE_INSPECT_WORKERS", "UPLOAD_SPOOL_MEMORY_BYTES", "MAX_UPLOAD_BYTES",
|
||||
"MAX_IMAGE_PIXELS", "API_REPLICAS", "S3_POOL_MAX_SIZE"}:
|
||||
settings[key] = line.split("=", 1)[1]
|
||||
return {
|
||||
"host": f"{platform.system()} {platform.machine()}",
|
||||
"docker": info.get("ServerVersion"),
|
||||
"docker_vm_cpus": info.get("NCPU"),
|
||||
"docker_vm_memory_gib": round(info.get("MemTotal", 0) / 2**30, 1),
|
||||
"api_limits": "cpus 1.0, memory 512M per replica (compose.yaml)",
|
||||
"s3": "chrislusf/seaweedfs:4.48",
|
||||
"settings": settings,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
parser.add_argument("--base-url", default=None)
|
||||
parser.add_argument("--output", default="evidence/experiment-uploads.json")
|
||||
parser.add_argument("--concurrency", default="10,20")
|
||||
parser.add_argument("--profiles", default="png,jpeg")
|
||||
parser.add_argument("--no-restart", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
client = Client(args.base_url)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
payloads = {}
|
||||
if "png" in args.profiles:
|
||||
payloads["png"] = (png_noise(1750, 1750), "image/png")
|
||||
if "jpeg" in args.profiles:
|
||||
try:
|
||||
payloads["jpeg"] = (jpeg_worst_case(), "image/jpeg")
|
||||
except ImportError:
|
||||
print("jpeg profile skipped: Pillow is not installed (use poetry run)")
|
||||
report = {"environment": environment(), "rounds": []}
|
||||
for profile, (content, content_type) in payloads.items():
|
||||
for concurrency in (int(x) for x in args.concurrency.split(",")):
|
||||
result = run_round(client, alice, chat, profile, content, content_type,
|
||||
concurrency, not args.no_restart)
|
||||
report["rounds"].append(result)
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
output = ROOT / args.output
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
||||
print("saved:", args.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user