#!/usr/bin/env python3 """Эксперимент: N одновременных загрузок ~9.5 MiB и пик памяти API-реплик. Гипотеза: память реплики ограничена сверху baseline + MAX_CONCURRENT_UPLOADS x (~2 x размер файла) + разбор изображения, и не растёт с числом одновременных клиентов: лишние ждут слот (UPLOAD_QUEUE_TIMEOUT_SECONDS), а не читают тело в память. Источники памяти: всегда VmHWM процесса API (пик RSS, считает ядро), обнулённый перед экспериментом через /proc/PID/clear_refs — главный показатель; --docker + docker stats (опрос ~1 с) по контейнерам сервиса api + cgroup memory.peak (ядро >= 5.19) до и после; --pids 123,456 + /proc//status: VmRSS каждые 20 мс (запуск без Docker). python student-tests/upload_memory.py --base-url http://127.0.0.1:8080 \ --concurrency 10 --docker --output evidence/upload_memory.json Изображения синтетические (шум, PNG без сжатия). Токены не сохраняются. """ import argparse import hashlib import json import os import platform import re import statistics import subprocess import sys import threading import time import uuid from concurrent.futures import ThreadPoolExecutor from pathlib import Path from urllib.error import HTTPError, URLError from urllib.request import Request, urlopen HERE = Path(__file__).resolve().parent sys.path.insert(0, str(HERE)) import common # noqa: E402 import imagegen # noqa: E402 MiB = 1024 * 1024 UNITS = {"b": 1, "kib": 1024, "mib": MiB, "gib": 1024 * MiB, "kb": 1000, "mb": 1000 ** 2, "gb": 1000 ** 3} def parse_mem(text): m = re.match(r"\s*([\d.]+)\s*([A-Za-z]+)", text) return float(m.group(1)) * UNITS.get(m.group(2).lower(), 1) if m else 0.0 class Sampler(threading.Thread): def __init__(self, args): super().__init__(daemon=True) self.args = args self.stop = threading.Event() self.samples = {} # name -> [bytes] def add(self, name, value): self.samples.setdefault(name, []).append(value) def run(self): while not self.stop.is_set(): if self.args.pids: for pid in self.args.pids.split(","): try: status = Path("/proc/%s/status" % pid).read_text() rss = int(re.search(r"VmRSS:\s+(\d+) kB", status).group(1)) * 1024 self.add("pid-" + pid, rss) except (OSError, AttributeError): pass time.sleep(0.02) else: out = subprocess.run( ["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.MemUsage}}"] + docker_api_ids(self.args), capture_output=True, text=True).stdout for line in out.splitlines(): name, usage = line.split("\t", 1) self.add(name, parse_mem(usage.split("/")[0])) def docker_api_ids(args): out = subprocess.run(["docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + args.project, "--filter", "label=com.docker.compose.service=api"], capture_output=True, text=True).stdout return out.split() def proc_hwm(args, reset=False): """Пик RSS процесса (VmHWM, считает ядро — без пропусков между опросами). --docker: PID 1 контейнера api (python, exec-форма CMD); --pids: заданные PID. reset=True обнуляет пик (echo 5 > /proc/PID/clear_refs) перед экспериментом. """ result = {} if args.docker: for cid in docker_api_ids(args): name = subprocess.run(["docker", "inspect", "-f", "{{.Name}}", cid], capture_output=True, text=True).stdout.strip().lstrip("/") if reset: subprocess.run(["docker", "exec", cid, "sh", "-c", "echo 5 > /proc/1/clear_refs"], capture_output=True) out = subprocess.run(["docker", "exec", cid, "cat", "/proc/1/status"], capture_output=True, text=True).stdout m = re.search(r"VmHWM:\s+(\d+) kB", out) if m: result[name] = int(m.group(1)) * 1024 else: for pid in args.pids.split(","): try: if reset: Path("/proc/%s/clear_refs" % pid).write_text("5") m = re.search(r"VmHWM:\s+(\d+) kB", Path("/proc/%s/status" % pid).read_text()) result["pid-" + pid] = int(m.group(1)) * 1024 except (OSError, AttributeError): pass return result def cgroup_peaks(args): """cgroup v2 memory.peak (или v1 max_usage_in_bytes) каждого контейнера api.""" peaks = {} for cid in docker_api_ids(args): for path in ("/sys/fs/cgroup/memory.peak", "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"): r = subprocess.run(["docker", "exec", cid, "cat", path], capture_output=True, text=True) if r.returncode == 0 and r.stdout.strip().isdigit(): name = subprocess.run(["docker", "inspect", "-f", "{{.Name}}", cid], capture_output=True, text=True).stdout.strip().lstrip("/") peaks[name] = int(r.stdout.strip()) break return peaks def one_upload(base, token, chat_id, data): req = Request(base + "/api/v1/chats/%s/attachments?filename=mem.png" % chat_id, data=data, method="POST", headers={"Authorization": "Bearer " + token, "Content-Type": "image/png"}) t0 = time.monotonic() try: with urlopen(req, timeout=120) as r: body = json.loads(r.read()) ok = body["sha256"] == hashlib.sha256(data).hexdigest() return r.status, time.monotonic() - t0, r.headers.get("X-Instance-Id"), ok except HTTPError as e: return e.code, time.monotonic() - t0, e.headers.get("X-Instance-Id"), False except (URLError, OSError): return 0, time.monotonic() - t0, None, False def main(): p = argparse.ArgumentParser() p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://127.0.0.1:8080")) p.add_argument("--concurrency", type=int, default=10) p.add_argument("--rounds", type=int, default=2, help="повторы волны одновременных загрузок") p.add_argument("--size-mib", type=float, default=9.5) p.add_argument("--docker", action="store_true") p.add_argument("--project", default=os.environ.get("COMPOSE_PROJECT_NAME", "messenger-lab4")) p.add_argument("--pids", default="") p.add_argument("--label", default="") p.add_argument("--output", default="") args = p.parse_args() if not args.docker and not args.pids: p.error("choose --docker or --pids") base = args.base_url.rstrip("/") common.BASE_URL = base owner = common.actor("mem") chat = common.create_chat(owner) files = [imagegen.noise_png(int(args.size_mib * MiB)) for _ in range(args.concurrency)] assert all(len(f) <= 10 * MiB for f in files) before = cgroup_peaks(args) if args.docker else {} proc_hwm(args, reset=True) sampler = Sampler(args) sampler.start() time.sleep(1.5) # baseline baseline = {k: max(v) for k, v in sampler.samples.items()} results = [] started = time.monotonic() for _ in range(args.rounds): with ThreadPoolExecutor(max_workers=args.concurrency) as pool: results += list(pool.map(lambda d: one_upload(base, owner["_token"], chat["id"], d), files)) duration = time.monotonic() - started time.sleep(1.0) sampler.stop.set() sampler.join(timeout=5) after = cgroup_peaks(args) if args.docker else {} hwm = proc_hwm(args) statuses = {} for status, _, _, _ in results: statuses[str(status)] = statuses.get(str(status), 0) + 1 lat = sorted(r[1] for r in results if r[0] == 201) def pct(q): return round(lat[min(len(lat) - 1, int(q * len(lat)))], 3) if lat else None memory = {} for name, values in sampler.samples.items(): memory[name] = { "baseline_mib": round(baseline.get(name, values[0]) / MiB, 1), "peak_mib": round(max(values) / MiB, 1), "growth_mib": round((max(values) - baseline.get(name, values[0])) / MiB, 1), "samples": len(values), } report = { "label": args.label, "date_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "host": "%s %s, %d CPU" % (platform.system(), platform.machine(), os.cpu_count() or 0), "source": "docker stats + cgroup peak" if args.docker else "/proc VmRSS (20 ms)", "concurrency": args.concurrency, "rounds": args.rounds, "file_mib": round(len(files[0]) / MiB, 2), "uploads": len(results), "statuses": statuses, "sha256_ok": sum(1 for r in results if r[3]), "instances": sorted({r[2] for r in results if r[2]}), "duration_s": round(duration, 2), "latency_201_s": {"p50": pct(0.5), "p95": pct(0.95), "max": lat[-1] if lat else None, "mean": round(statistics.mean(lat), 3) if lat else None}, "memory": memory, } report["process_peak_rss_mib"] = {k: round(v / MiB, 1) for k, v in hwm.items()} if args.docker: report["cgroup_peak_mib_before"] = {k: round(v / MiB, 1) for k, v in before.items()} report["cgroup_peak_mib_after"] = {k: round(v / MiB, 1) for k, v in after.items()} text = json.dumps(report, ensure_ascii=False, indent=2) print(text) if args.output: Path(args.output).write_text(text + "\n", encoding="utf-8") if __name__ == "__main__": main()