"""Переносимые помощники экспериментов: stdin для секретов и cgroup RAM sampler.""" import json import os from pathlib import Path import struct import subprocess import threading import time import zlib from replicas import COMPOSE, ROOT def save_json(path, value): path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") def record_command(out, name, command, *, timeout=600): """Записать только команды, не содержащие credentials или полные inspect/env.""" started = time.monotonic() env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8") result = subprocess.run(command, cwd=ROOT, capture_output=True, env=env, timeout=timeout) output = (result.stdout + result.stderr).decode("utf-8", errors="replace") (out / (name + ".txt")).write_text( "$ " + " ".join(map(str, command)) + "\n" + output + f"\nexit_code={result.returncode}; elapsed_seconds={time.monotonic() - started:.3f}\n", encoding="utf-8", ) if result.returncode: raise RuntimeError(name + " failed; inspect the evidence file") def exec_json(service, code, payload=None, *, timeout=90): """JSON передаётся через stdin, поэтому Bearer не появляется в argv или evidence.""" result = subprocess.run( COMPOSE + ["exec", "-T", service, "python", "-c", code], cwd=ROOT, input=json.dumps(payload or {}).encode("utf-8"), capture_output=True, timeout=timeout, ) if result.returncode: # Не печатать stderr кода, который имеет доступ к credentials. raise RuntimeError("in-container check failed in " + service) try: return json.loads(result.stdout) except (ValueError, UnicodeError): raise RuntimeError("in-container check returned invalid JSON") from None def png_fixture(width=1024, height=683): """Настоящий RGB PNG >2 MiB, без Pillow и случайного содержимого на хосте.""" def chunk(kind, data): return struct.pack(">I", len(data)) + kind + data + struct.pack( ">I", zlib.crc32(kind + data) & 0xffffffff ) row = bytes(range(256)) * ((width * 3 + 255) // 256) pixels = b"".join(b"\x00" + row[:width * 3] for _ in range(height)) return ( b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + chunk(b"IDAT", zlib.compress(pixels, level=0)) + chunk(b"IEND", b"") ) MEMORY_SAMPLER = r''' import json, pathlib, sys, threading, time stop = threading.Event() threading.Thread(target=lambda: (sys.stdin.buffer.read(), stop.set()), daemon=True).start() root = pathlib.Path('/sys/fs/cgroup') if (root / 'memory.current').exists(): names = ('memory.current', 'memory.peak', 'memory.max') version = 2 else: root = root / 'memory' names = ('memory.usage_in_bytes', 'memory.max_usage_in_bytes', 'memory.limit_in_bytes') version = 1 started = time.monotonic() def number(name): try: value = (root / name).read_text().strip() return int(value) if value != 'max' else None except (OSError, ValueError): return None def sample(): rss = None try: for line in pathlib.Path('/proc/1/status').read_text().splitlines(): if line.startswith('VmRSS:'): rss = int(line.split()[1]) * 1024 except (OSError, ValueError): pass print(json.dumps({'seconds': round(time.monotonic() - started, 3), 'cgroup_version': version, 'memory_current_bytes': number(names[0]), 'container_lifetime_peak_bytes': number(names[1]), 'limit_bytes': number(names[2]), 'api_pid1_rss_bytes': rss}), flush=True) sample() while not stop.wait(0.2): sample() sample() ''' class MemorySampler: """Один небольшой Python-процесс в API читает RAM каждые 200 мс.""" def __init__(self, service): self.service = service self.samples = [] self.ready = threading.Event() self.process = None def start(self): self.process = subprocess.Popen( COMPOSE + ["exec", "-T", self.service, "python", "-u", "-c", MEMORY_SAMPLER], cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) def read(): for line in self.process.stdout: try: sample = json.loads(line) self.samples.append(sample) self.ready.set() except (ValueError, UnicodeError): pass self.reader = threading.Thread(target=read, daemon=True) self.reader.start() if not self.ready.wait(timeout=10): self.stop() raise RuntimeError("memory sampler did not start in " + self.service) if self.samples[0]["memory_current_bytes"] is None: self.stop() raise RuntimeError("cgroup memory counters unavailable in " + self.service) return self def stop(self): if self.process is None: return self.process.stdin.close() try: self.process.wait(timeout=10) except subprocess.TimeoutExpired: self.process.kill() self.process.wait(timeout=5) self.reader.join(timeout=5) self.process.stdout.close() self.process.stderr.close() self.process = None def report(self): current = [s["memory_current_bytes"] for s in self.samples if s["memory_current_bytes"] is not None] rss = [s["api_pid1_rss_bytes"] for s in self.samples if s["api_pid1_rss_bytes"] is not None] lifetime = [s["container_lifetime_peak_bytes"] for s in self.samples if s["container_lifetime_peak_bytes"] is not None] return { "service": self.service, "sampling_interval_seconds": 0.2, "baseline_cgroup_bytes": current[0], "sampled_cgroup_peak_bytes": max(current), "sampled_growth_bytes": max(current) - current[0], "sampled_api_pid1_rss_peak_bytes": max(rss) if rss else None, "container_lifetime_peak_bytes": max(lifetime) if lifetime else None, "limit_bytes": self.samples[0]["limit_bytes"], "samples": self.samples, "limitations": "cgroup includes file cache and sampler overhead; 200 ms samples may miss short spikes; lifetime peak may predate this experiment", }