Files
mtusi2026lab5/scripts/upload_experiment.py
T

318 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Эксперимент: N одновременных загрузок вложений и пик памяти API-реплик.
Гипотеза: память реплики на загрузку ограничена (spool ≤ 1 MiB RAM, потоковый
PUT в S3, проверка изображения в пуле из IMAGE_INSPECT_WORKERS потоков),
поэтому пик RSS растёт не пропорционально N × размер файла.
Профили:
png — PNG ~9 MiB из шума (почти не сжимается): нагрузка на приём тела,
spool, SHA-256 и PUT в S3. Генерируется стандартной библиотекой.
jpeg — progressive JPEG 8000×5000 = 40 Мпикс (предел): худший случай
декодера libjpeg. Нужен Pillow (poetry run python ...).
Перед каждым прогоном API-реплики перезапускаются: VmHWM (пиковый RSS
процесса) начинается заново. Память опрашивается внутри контейнеров каждые
0.2 с (cgroup memory.current и VmRSS процесса). Все N запросов стартуют по
одному барьеру. Tokens не печатаются и не сохраняются.
"""
from __future__ import annotations
import argparse
import hashlib
import io
import json
import os
import platform
import statistics
import struct
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
import zlib
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client # noqa: E402
REQUEST_TIMEOUT_SEC = 120
READY_TIMEOUT_SEC = 90
SAMPLE_INTERVAL_SEC = 0.2
COMPOSE = ["docker", "compose", "--env-file", ".env"]
SAMPLER = (
"while :; do "
"echo $(cat /sys/fs/cgroup/memory.current) "
"$(awk '/VmRSS/ {print $2}' /proc/1/status); "
f"sleep {SAMPLE_INTERVAL_SEC}; done"
)
def png_noise(width: int, height: int) -> bytes:
"""Настоящий RGB PNG из случайных байт: размер ≈ width × height × 3"""
def chunk(kind: bytes, data: bytes) -> bytes:
return (
struct.pack(">I", len(data))
+ kind
+ data
+ struct.pack(">I", zlib.crc32(kind + data))
)
raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height))
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", header)
+ chunk(b"IDAT", zlib.compress(raw, 1))
+ chunk(b"IEND", b"")
)
def jpeg_worst_case() -> bytes:
from PIL import Image
Image.MAX_IMAGE_PIXELS = None
image = Image.effect_noise((8000, 5000), 64).convert("RGB")
buffer = io.BytesIO()
image.save(buffer, "JPEG", quality=10, progressive=True, subsampling=0)
return buffer.getvalue()
def compose(*args: str) -> str:
return subprocess.run(
[*COMPOSE, *args], cwd=ROOT, capture_output=True, text=True, check=True
).stdout
def api_containers() -> list[str]:
return compose("ps", "-q", "api").split()
def container_name(container: str) -> str:
return subprocess.run(
["docker", "inspect", "-f", "{{.Name}}", container],
capture_output=True, text=True, check=True,
).stdout.strip().lstrip("/")
def process_status(container: str) -> dict:
"""VmRSS/VmHWM процесса API (PID 1), байты, и его X-Instance-Id"""
output = subprocess.run(
["docker", "exec", container, "sh", "-c",
"grep -E 'VmRSS|VmHWM' /proc/1/status; cat /tmp/instance-id"],
capture_output=True, text=True, check=True,
).stdout.split("\n")
values: dict = {line.split(":")[0]: int(line.split()[1]) * 1024 for line in output[:2]}
values["instance"] = output[2].strip()
return values
def restart_api(base_url: str) -> None:
compose("restart", "api")
deadline = time.monotonic() + READY_TIMEOUT_SEC
while time.monotonic() < deadline:
states = [
subprocess.run(
["docker", "inspect", "-f", "{{.State.Health.Status}}", c],
capture_output=True, text=True,
).stdout.strip()
for c in api_containers()
]
if states and all(state == "healthy" for state in states):
return
time.sleep(1)
raise SystemExit("API replicas are not healthy after restart")
class Sampler(threading.Thread):
"""Опрос памяти одного контейнера изнутри одним долгим docker exec"""
def __init__(self, container: str) -> None:
super().__init__(daemon=True)
self.container = container
self.cgroup_max = 0
self.rss_max = 0
self.samples = 0
self._process = subprocess.Popen(
["docker", "exec", container, "sh", "-c", SAMPLER],
stdout=subprocess.PIPE, text=True,
)
def run(self) -> None:
assert self._process.stdout is not None
for line in self._process.stdout:
parts = line.split()
if len(parts) == 2:
self.cgroup_max = max(self.cgroup_max, int(parts[0]))
self.rss_max = max(self.rss_max, int(parts[1]) * 1024)
self.samples += 1
def stop(self) -> None:
self._process.terminate()
self._process.wait()
def upload_once(base: str, token: str, chat_id: str, content: bytes, content_type: str) -> dict:
request = urllib.request.Request(
f"{base}/api/v1/chats/{chat_id}/attachments?filename=experiment",
data=content,
method="POST",
headers={"Authorization": "Bearer " + token, "Content-Type": content_type},
)
started = time.monotonic()
try:
with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT_SEC) as response:
body, status, instance = response.read(), response.status, response.headers.get("X-Instance-Id")
except urllib.error.HTTPError as error:
body, status, instance = error.read(), error.code, error.headers.get("X-Instance-Id")
except OSError as error:
return {"status": None, "error": type(error).__name__, "seconds": time.monotonic() - started}
result = {"status": status, "instance": instance, "seconds": time.monotonic() - started}
if status == 201:
result["id"] = json.loads(body)["id"]
return result
def percentile(values: list[float], share: float) -> float:
ordered = sorted(values)
return ordered[min(len(ordered) - 1, round(share * (len(ordered) - 1)))]
def run_round(client: Client, actor: dict, chat: dict, profile: str, content: bytes,
content_type: str, concurrency: int, restart: bool) -> dict:
if restart:
restart_api(client.base)
containers = api_containers()
# Прогрев: первый запрос каждой реплики открывает пулы БД/S3
warmup = png_noise(4, 4)
for _ in range(2 * len(containers)):
upload_once(client.base, actor["token"], chat["id"], warmup, "image/png")
before = {container_name(c): process_status(c) for c in containers}
samplers = [Sampler(c) for c in containers]
for sampler in samplers:
sampler.start()
time.sleep(1)
barrier = threading.Barrier(concurrency)
results: list[dict] = [{} for _ in range(concurrency)]
def worker(index: int) -> None:
barrier.wait()
results[index] = upload_once(client.base, actor["token"], chat["id"], content, content_type)
started = time.monotonic()
threads = [threading.Thread(target=worker, args=(i,)) for i in range(concurrency)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
wall = time.monotonic() - started
time.sleep(1)
for sampler in samplers:
sampler.stop()
after = {container_name(c): process_status(c) for c in containers}
ok = [r for r in results if r.get("status") == 201]
sample = ok[0]["id"] if ok else None
checksum_ok = None
if sample:
downloaded = client.call("GET", f"/api/v1/attachments/{sample}/content", actor=actor)["bytes"]
checksum_ok = hashlib.sha256(downloaded).digest() == hashlib.sha256(content).digest()
seconds = [r["seconds"] for r in ok]
mib = 1024 * 1024
replicas = {}
for sampler in samplers:
name = container_name(sampler.container)
replicas[name] = {
"uploads_served": sum(1 for r in ok if r.get("instance") == after[name].get("instance")),
"rss_before_mib": round(before[name]["VmRSS"] / mib, 1),
"rss_peak_process_vmhwm_mib": round(after[name]["VmHWM"] / mib, 1),
"rss_peak_sampled_mib": round(sampler.rss_max / mib, 1),
"cgroup_peak_sampled_mib": round(sampler.cgroup_max / mib, 1),
"samples": sampler.samples,
}
by_instance: dict[str, int] = {}
for r in ok:
by_instance[r["instance"]] = by_instance.get(r["instance"], 0) + 1
return {
"profile": profile,
"file_bytes": len(content),
"concurrency": concurrency,
"replicas_restarted_before": restart,
"status_counts": {str(s): sum(1 for r in results if r.get("status") == s)
for s in sorted({r.get("status") for r in results}, key=str)},
"uploads_by_instance": by_instance,
"wall_seconds": round(wall, 2),
"latency_seconds": {
"p50": round(percentile(seconds, 0.5), 3),
"p95": round(percentile(seconds, 0.95), 3),
"max": round(max(seconds), 3),
} if seconds else None,
"throughput_mib_per_second": round(len(ok) * len(content) / mib / wall, 1),
"download_sha256_matches": checksum_ok,
"replicas": replicas,
}
def environment() -> dict:
info = json.loads(subprocess.run(
["docker", "info", "--format", "{{json .}}"], capture_output=True, text=True, check=True
).stdout)
settings = {}
for line in (ROOT / ".env").read_text().splitlines():
key = line.split("=", 1)[0]
if key in {"IMAGE_INSPECT_WORKERS", "UPLOAD_SPOOL_MEMORY_BYTES", "MAX_UPLOAD_BYTES",
"MAX_IMAGE_PIXELS", "API_REPLICAS", "S3_POOL_MAX_SIZE"}:
settings[key] = line.split("=", 1)[1]
return {
"host": f"{platform.system()} {platform.machine()}",
"docker": info.get("ServerVersion"),
"docker_vm_cpus": info.get("NCPU"),
"docker_vm_memory_gib": round(info.get("MemTotal", 0) / 2**30, 1),
"api_limits": "cpus 1.0, memory 512M per replica (compose.yaml)",
"s3": "chrislusf/seaweedfs:4.48",
"settings": settings,
}
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
parser.add_argument("--base-url", default=None)
parser.add_argument("--output", default="evidence/experiment-uploads.json")
parser.add_argument("--concurrency", default="10,20")
parser.add_argument("--profiles", default="png,jpeg")
parser.add_argument("--no-restart", action="store_true")
args = parser.parse_args()
client = Client(args.base_url)
alice, _, _, chat = client.bootstrap()
payloads = {}
if "png" in args.profiles:
payloads["png"] = (png_noise(1750, 1750), "image/png")
if "jpeg" in args.profiles:
try:
payloads["jpeg"] = (jpeg_worst_case(), "image/jpeg")
except ImportError:
print("jpeg profile skipped: Pillow is not installed (use poetry run)")
report = {"environment": environment(), "rounds": []}
for profile, (content, content_type) in payloads.items():
for concurrency in (int(x) for x in args.concurrency.split(",")):
result = run_round(client, alice, chat, profile, content, content_type,
concurrency, not args.no_restart)
report["rounds"].append(result)
print(json.dumps(result, ensure_ascii=False))
output = ROOT / args.output
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
print("saved:", args.output)
if __name__ == "__main__":
main()