"""Общие помощники скриптов Track 2. psql/pgbench/pg_dump запускаются внутри контейнеров (`docker compose exec` или `docker exec`), поэтому на хосте нужны только Docker и Python 3.9+. Учётные данные берутся из .env (POSTGRES_USER, POSTGRES_DB) и не печатаются. """ import datetime import os import platform import statistics import subprocess import sys from pathlib import Path ROOT = Path(__file__).resolve().parent.parent PERF_USER_ID = "11111111-1111-4111-8111-111111111111" MESSAGE_COLS = ("id, chat_id, sender_id, text, " "EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch") def load_env(): env = {} path = ROOT / ".env" if path.exists(): for line in path.read_text(encoding="utf-8").splitlines(): line = line.strip() if not line or line.startswith("#") or "=" not in line: continue key, value = line.split("=", 1) env[key.strip()] = value.strip().strip('"').strip("'") for key in ("POSTGRES_USER", "POSTGRES_DB", "SESSION_STORE"): if os.environ.get(key): env[key] = os.environ[key] missing = [k for k in ("POSTGRES_USER", "POSTGRES_DB") if not env.get(k)] if missing: raise SystemExit(f"не заданы {', '.join(missing)} в .env") return env ENV = load_env() class Pg: """Клиент к PostgreSQL в контейнере через stdin/stdout утилит.""" def __init__(self, exec_prefix, label): self.prefix = exec_prefix self.label = label @classmethod def compose(cls, service="db"): return cls(["docker", "compose", "--env-file", str(ROOT / ".env"), "exec", "-T", service], service) @classmethod def container(cls, name): return cls(["docker", "exec", "-i", name], name) def run(self, args, input=None, binary=False, check=True, stdout=None): r = subprocess.run( self.prefix + args, input=input, stdout=stdout if stdout is not None else subprocess.PIPE, stderr=subprocess.PIPE, text=not binary, cwd=ROOT, ) if check and r.returncode != 0: err = r.stderr if not binary else r.stderr.decode(errors="replace") sys.stderr.write(err) raise SystemExit(f"[{self.label}] команда {args[0]} завершилась с кодом {r.returncode}") return r def psql(self, sql, tuples=True): args = ["psql", "-U", ENV["POSTGRES_USER"], "-d", ENV["POSTGRES_DB"], "-X", "-q", "-v", "ON_ERROR_STOP=1"] if tuples: args += ["-t", "-A", "-F", "\t"] return self.run(args, input=sql).stdout def scalar(self, sql): return self.psql(sql).strip() def rows(self, sql): return [line.split("\t") for line in self.psql(sql).splitlines() if line] def median(values): return statistics.median(values) if values else float("nan") def pick_chats(db): """Выбирает чаты разного размера из засеянных perf100k.py.""" rows = db.rows(""" SELECT c.id, c.title, count(m.id) FROM chats c JOIN messages m ON m.chat_id = c.id WHERE c.title LIKE 'perf-%' GROUP BY c.id, c.title; """) chats = [(r[0], r[1], int(r[2])) for r in rows] regular = [c for c in chats if not c[1].startswith("perf-burst")] burst = [c for c in chats if c[1].startswith("perf-burst")] if sum(c[2] for c in regular) < 100_000: raise SystemExit("меньше 100 000 perf-сообщений: запустите " "python3 scripts/perf100k.py seed") if not burst: raise SystemExit("нет чата с одинаковыми created_at: запустите " "python3 scripts/perf100k.py seed-burst") def closest(target): return min(regular, key=lambda c: abs(c[2] - target)) return [ ("small", closest(50)), ("medium", closest(1000)), ("large", max(regular, key=lambda c: c[2])), ("burst", burst[0]), ] def environment_lines(db): commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, text=True).stdout.strip() dirty = subprocess.run(["git", "status", "--porcelain"], cwd=ROOT, capture_output=True, text=True).stdout.strip() try: mem_kb = int(next(l for l in Path("/proc/meminfo").read_text().splitlines() if l.startswith("MemTotal")).split()[1]) mem = f"{mem_kb / 1024 / 1024:.1f} GiB" except (OSError, StopIteration, ValueError): mem = "unknown" version = db.scalar("SHOW server_version;") return [ f"- Дата: {datetime.datetime.now().astimezone().isoformat(timespec='seconds')}", f"- Commit: `{commit or 'unknown'}`{' (есть незакоммиченные изменения)' if dirty else ''}", f"- Хост: {platform.system()} {platform.release()}, {os.cpu_count()} CPU, RAM {mem}; " "CPU/RAM-лимиты контейнеров не заданы", f"- PostgreSQL {version} (`postgres:16-alpine`), настройки по умолчанию, " f"synchronous_commit={db.scalar('SHOW synchronous_commit;')}, " f"shared_buffers={db.scalar('SHOW shared_buffers;')}", ] def dataset_lines(db): total, chats, mn, med, mx = db.rows(""" SELECT (SELECT count(*) FROM messages), count(*), min(n), percentile_disc(0.5) WITHIN GROUP (ORDER BY n), max(n) FROM (SELECT chat_id, count(*) AS n FROM messages GROUP BY chat_id) s; """)[0] size = db.scalar("SELECT pg_size_pretty(pg_table_size('messages'));") return [ f"- Сообщений: {int(total):,}".replace(",", " ") + f", чатов с сообщениями: {chats}; размер чата min/median/max = {mn}/{med}/{mx}", f"- Размер heap таблицы `messages`: {size}", ] def write_evidence(name, text): path = ROOT / "evidence" / name path.parent.mkdir(exist_ok=True) path.write_text(text, encoding="utf-8") print(f"\nзаписан {path.relative_to(ROOT)}")