Files
mtusi2026lab6-fork/student-tests/upload_memory.py

233 lines
10 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Эксперимент: N одновременных загрузок ~9.5 MiB и пик памяти API-реплик.
Гипотеза: память реплики ограничена сверху
baseline + MAX_CONCURRENT_UPLOADS x (~2 x размер файла) + разбор изображения,
и не растёт с числом одновременных клиентов: лишние ждут слот
(UPLOAD_QUEUE_TIMEOUT_SECONDS), а не читают тело в память.
Источники памяти:
всегда VmHWM процесса API (пик RSS, считает ядро), обнулённый перед
экспериментом через /proc/PID/clear_refs — главный показатель;
--docker + docker stats (опрос ~1 с) по контейнерам сервиса api
+ cgroup memory.peak (ядро >= 5.19) до и после;
--pids 123,456 + /proc/<pid>/status: VmRSS каждые 20 мс (запуск без Docker).
python student-tests/upload_memory.py --base-url http://127.0.0.1:8080 \
--concurrency 10 --docker --output evidence/upload_memory.json
Изображения синтетические (шум, PNG без сжатия). Токены не сохраняются.
"""
import argparse
import hashlib
import json
import os
import platform
import re
import statistics
import subprocess
import sys
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import common # noqa: E402
import imagegen # noqa: E402
MiB = 1024 * 1024
UNITS = {"b": 1, "kib": 1024, "mib": MiB, "gib": 1024 * MiB, "kb": 1000, "mb": 1000 ** 2, "gb": 1000 ** 3}
def parse_mem(text):
m = re.match(r"\s*([\d.]+)\s*([A-Za-z]+)", text)
return float(m.group(1)) * UNITS.get(m.group(2).lower(), 1) if m else 0.0
class Sampler(threading.Thread):
def __init__(self, args):
super().__init__(daemon=True)
self.args = args
self.stop = threading.Event()
self.samples = {} # name -> [bytes]
def add(self, name, value):
self.samples.setdefault(name, []).append(value)
def run(self):
while not self.stop.is_set():
if self.args.pids:
for pid in self.args.pids.split(","):
try:
status = Path("/proc/%s/status" % pid).read_text()
rss = int(re.search(r"VmRSS:\s+(\d+) kB", status).group(1)) * 1024
self.add("pid-" + pid, rss)
except (OSError, AttributeError):
pass
time.sleep(0.02)
else:
out = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.MemUsage}}"] + docker_api_ids(self.args),
capture_output=True, text=True).stdout
for line in out.splitlines():
name, usage = line.split("\t", 1)
self.add(name, parse_mem(usage.split("/")[0]))
def docker_api_ids(args):
out = subprocess.run(["docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + args.project,
"--filter", "label=com.docker.compose.service=api"], capture_output=True, text=True).stdout
return out.split()
def proc_hwm(args, reset=False):
"""Пик RSS процесса (VmHWM, считает ядро — без пропусков между опросами).
--docker: PID 1 контейнера api (python, exec-форма CMD); --pids: заданные PID.
reset=True обнуляет пик (echo 5 > /proc/PID/clear_refs) перед экспериментом.
"""
result = {}
if args.docker:
for cid in docker_api_ids(args):
name = subprocess.run(["docker", "inspect", "-f", "{{.Name}}", cid], capture_output=True,
text=True).stdout.strip().lstrip("/")
if reset:
subprocess.run(["docker", "exec", cid, "sh", "-c", "echo 5 > /proc/1/clear_refs"],
capture_output=True)
out = subprocess.run(["docker", "exec", cid, "cat", "/proc/1/status"], capture_output=True,
text=True).stdout
m = re.search(r"VmHWM:\s+(\d+) kB", out)
if m:
result[name] = int(m.group(1)) * 1024
else:
for pid in args.pids.split(","):
try:
if reset:
Path("/proc/%s/clear_refs" % pid).write_text("5")
m = re.search(r"VmHWM:\s+(\d+) kB", Path("/proc/%s/status" % pid).read_text())
result["pid-" + pid] = int(m.group(1)) * 1024
except (OSError, AttributeError):
pass
return result
def cgroup_peaks(args):
"""cgroup v2 memory.peak (или v1 max_usage_in_bytes) каждого контейнера api."""
peaks = {}
for cid in docker_api_ids(args):
for path in ("/sys/fs/cgroup/memory.peak", "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"):
r = subprocess.run(["docker", "exec", cid, "cat", path], capture_output=True, text=True)
if r.returncode == 0 and r.stdout.strip().isdigit():
name = subprocess.run(["docker", "inspect", "-f", "{{.Name}}", cid], capture_output=True,
text=True).stdout.strip().lstrip("/")
peaks[name] = int(r.stdout.strip())
break
return peaks
def one_upload(base, token, chat_id, data):
req = Request(base + "/api/v1/chats/%s/attachments?filename=mem.png" % chat_id, data=data, method="POST",
headers={"Authorization": "Bearer " + token, "Content-Type": "image/png"})
t0 = time.monotonic()
try:
with urlopen(req, timeout=120) as r:
body = json.loads(r.read())
ok = body["sha256"] == hashlib.sha256(data).hexdigest()
return r.status, time.monotonic() - t0, r.headers.get("X-Instance-Id"), ok
except HTTPError as e:
return e.code, time.monotonic() - t0, e.headers.get("X-Instance-Id"), False
except (URLError, OSError):
return 0, time.monotonic() - t0, None, False
def main():
p = argparse.ArgumentParser()
p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://127.0.0.1:8080"))
p.add_argument("--concurrency", type=int, default=10)
p.add_argument("--rounds", type=int, default=2, help="повторы волны одновременных загрузок")
p.add_argument("--size-mib", type=float, default=9.5)
p.add_argument("--docker", action="store_true")
p.add_argument("--project", default=os.environ.get("COMPOSE_PROJECT_NAME", "messenger-lab4"))
p.add_argument("--pids", default="")
p.add_argument("--label", default="")
p.add_argument("--output", default="")
args = p.parse_args()
if not args.docker and not args.pids:
p.error("choose --docker or --pids")
base = args.base_url.rstrip("/")
common.BASE_URL = base
owner = common.actor("mem")
chat = common.create_chat(owner)
files = [imagegen.noise_png(int(args.size_mib * MiB)) for _ in range(args.concurrency)]
assert all(len(f) <= 10 * MiB for f in files)
before = cgroup_peaks(args) if args.docker else {}
proc_hwm(args, reset=True)
sampler = Sampler(args)
sampler.start()
time.sleep(1.5) # baseline
baseline = {k: max(v) for k, v in sampler.samples.items()}
results = []
started = time.monotonic()
for _ in range(args.rounds):
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
results += list(pool.map(lambda d: one_upload(base, owner["_token"], chat["id"], d), files))
duration = time.monotonic() - started
time.sleep(1.0)
sampler.stop.set()
sampler.join(timeout=5)
after = cgroup_peaks(args) if args.docker else {}
hwm = proc_hwm(args)
statuses = {}
for status, _, _, _ in results:
statuses[str(status)] = statuses.get(str(status), 0) + 1
lat = sorted(r[1] for r in results if r[0] in (201, 202))
def pct(q):
return round(lat[min(len(lat) - 1, int(q * len(lat)))], 3) if lat else None
memory = {}
for name, values in sampler.samples.items():
memory[name] = {
"baseline_mib": round(baseline.get(name, values[0]) / MiB, 1),
"peak_mib": round(max(values) / MiB, 1),
"growth_mib": round((max(values) - baseline.get(name, values[0])) / MiB, 1),
"samples": len(values),
}
report = {
"label": args.label,
"date_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"host": "%s %s, %d CPU" % (platform.system(), platform.machine(), os.cpu_count() or 0),
"source": "docker stats + cgroup peak" if args.docker else "/proc VmRSS (20 ms)",
"concurrency": args.concurrency,
"rounds": args.rounds,
"file_mib": round(len(files[0]) / MiB, 2),
"uploads": len(results),
"statuses": statuses,
"sha256_ok": sum(1 for r in results if r[3]),
"instances": sorted({r[2] for r in results if r[2]}),
"duration_s": round(duration, 2),
"latency_201_s": {"p50": pct(0.5), "p95": pct(0.95), "max": lat[-1] if lat else None,
"mean": round(statistics.mean(lat), 3) if lat else None},
"memory": memory,
}
report["process_peak_rss_mib"] = {k: round(v / MiB, 1) for k, v in hwm.items()}
if args.docker:
report["cgroup_peak_mib_before"] = {k: round(v / MiB, 1) for k, v in before.items()}
report["cgroup_peak_mib_after"] = {k: round(v / MiB, 1) for k, v in after.items()}
text = json.dumps(report, ensure_ascii=False, indent=2)
print(text)
if args.output:
Path(args.output).write_text(text + "\n", encoding="utf-8")
if __name__ == "__main__":
main()