Files
mtusi2026lab2/scripts/track2_common.py
2026-10-16 05:27:13 +03:00

163 lines
6.2 KiB
Python

"""Общие помощники скриптов Track 2.
psql/pgbench/pg_dump запускаются внутри контейнеров (`docker compose exec`
или `docker exec`), поэтому на хосте нужны только Docker и Python 3.9+.
Учётные данные берутся из .env (POSTGRES_USER, POSTGRES_DB) и не печатаются.
"""
import datetime
import os
import platform
import statistics
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
PERF_USER_ID = "11111111-1111-4111-8111-111111111111"
MESSAGE_COLS = ("id, chat_id, sender_id, text, "
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch")
def load_env():
env = {}
path = ROOT / ".env"
if path.exists():
for line in path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, value = line.split("=", 1)
env[key.strip()] = value.strip().strip('"').strip("'")
for key in ("POSTGRES_USER", "POSTGRES_DB", "SESSION_STORE"):
if os.environ.get(key):
env[key] = os.environ[key]
missing = [k for k in ("POSTGRES_USER", "POSTGRES_DB") if not env.get(k)]
if missing:
raise SystemExit(f"не заданы {', '.join(missing)} в .env")
return env
ENV = load_env()
class Pg:
"""Клиент к PostgreSQL в контейнере через stdin/stdout утилит."""
def __init__(self, exec_prefix, label):
self.prefix = exec_prefix
self.label = label
@classmethod
def compose(cls, service="db"):
return cls(["docker", "compose", "--env-file", str(ROOT / ".env"),
"exec", "-T", service], service)
@classmethod
def container(cls, name):
return cls(["docker", "exec", "-i", name], name)
def run(self, args, input=None, binary=False, check=True, stdout=None):
r = subprocess.run(
self.prefix + args,
input=input,
stdout=stdout if stdout is not None else subprocess.PIPE,
stderr=subprocess.PIPE,
text=not binary,
cwd=ROOT,
)
if check and r.returncode != 0:
err = r.stderr if not binary else r.stderr.decode(errors="replace")
sys.stderr.write(err)
raise SystemExit(f"[{self.label}] команда {args[0]} завершилась с кодом {r.returncode}")
return r
def psql(self, sql, tuples=True):
args = ["psql", "-U", ENV["POSTGRES_USER"], "-d", ENV["POSTGRES_DB"],
"-X", "-q", "-v", "ON_ERROR_STOP=1"]
if tuples:
args += ["-t", "-A", "-F", "\t"]
return self.run(args, input=sql).stdout
def scalar(self, sql):
return self.psql(sql).strip()
def rows(self, sql):
return [line.split("\t") for line in self.psql(sql).splitlines() if line]
def median(values):
return statistics.median(values) if values else float("nan")
def pick_chats(db):
"""Выбирает чаты разного размера из засеянных perf100k.py."""
rows = db.rows("""
SELECT c.id, c.title, count(m.id)
FROM chats c JOIN messages m ON m.chat_id = c.id
WHERE c.title LIKE 'perf-%'
GROUP BY c.id, c.title;
""")
chats = [(r[0], r[1], int(r[2])) for r in rows]
regular = [c for c in chats if not c[1].startswith("perf-burst")]
burst = [c for c in chats if c[1].startswith("perf-burst")]
if sum(c[2] for c in regular) < 100_000:
raise SystemExit("меньше 100 000 perf-сообщений: запустите "
"python3 scripts/perf100k.py seed")
if not burst:
raise SystemExit("нет чата с одинаковыми created_at: запустите "
"python3 scripts/perf100k.py seed-burst")
def closest(target):
return min(regular, key=lambda c: abs(c[2] - target))
return [
("small", closest(50)),
("medium", closest(1000)),
("large", max(regular, key=lambda c: c[2])),
("burst", burst[0]),
]
def environment_lines(db):
commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT,
capture_output=True, text=True).stdout.strip()
dirty = subprocess.run(["git", "status", "--porcelain"], cwd=ROOT,
capture_output=True, text=True).stdout.strip()
try:
mem_kb = int(next(l for l in Path("/proc/meminfo").read_text().splitlines()
if l.startswith("MemTotal")).split()[1])
mem = f"{mem_kb / 1024 / 1024:.1f} GiB"
except (OSError, StopIteration, ValueError):
mem = "unknown"
version = db.scalar("SHOW server_version;")
return [
f"- Дата: {datetime.datetime.now().astimezone().isoformat(timespec='seconds')}",
f"- Commit: `{commit or 'unknown'}`{' (есть незакоммиченные изменения)' if dirty else ''}",
f"- Хост: {platform.system()} {platform.release()}, {os.cpu_count()} CPU, RAM {mem}; "
"CPU/RAM-лимиты контейнеров не заданы",
f"- PostgreSQL {version} (`postgres:16-alpine`), настройки по умолчанию, "
f"synchronous_commit={db.scalar('SHOW synchronous_commit;')}, "
f"shared_buffers={db.scalar('SHOW shared_buffers;')}",
]
def dataset_lines(db):
total, chats, mn, med, mx = db.rows("""
SELECT (SELECT count(*) FROM messages),
count(*), min(n), percentile_disc(0.5) WITHIN GROUP (ORDER BY n), max(n)
FROM (SELECT chat_id, count(*) AS n FROM messages GROUP BY chat_id) s;
""")[0]
size = db.scalar("SELECT pg_size_pretty(pg_table_size('messages'));")
return [
f"- Сообщений: {int(total):,}".replace(",", " ") +
f", чатов с сообщениями: {chats}; размер чата min/median/max = {mn}/{med}/{mx}",
f"- Размер heap таблицы `messages`: {size}",
]
def write_evidence(name, text):
path = ROOT / "evidence" / name
path.parent.mkdir(exist_ok=True)
path.write_text(text, encoding="utf-8")
print(f"\nзаписан {path.relative_to(ROOT)}")