forked from mtusi2026/mtusi2026lab2
163 lines
6.2 KiB
Python
163 lines
6.2 KiB
Python
"""Общие помощники скриптов Track 2.
|
|
|
|
psql/pgbench/pg_dump запускаются внутри контейнеров (`docker compose exec`
|
|
или `docker exec`), поэтому на хосте нужны только Docker и Python 3.9+.
|
|
Учётные данные берутся из .env (POSTGRES_USER, POSTGRES_DB) и не печатаются.
|
|
"""
|
|
import datetime
|
|
import os
|
|
import platform
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
PERF_USER_ID = "11111111-1111-4111-8111-111111111111"
|
|
MESSAGE_COLS = ("id, chat_id, sender_id, text, "
|
|
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch")
|
|
|
|
|
|
def load_env():
|
|
env = {}
|
|
path = ROOT / ".env"
|
|
if path.exists():
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
line = line.strip()
|
|
if not line or line.startswith("#") or "=" not in line:
|
|
continue
|
|
key, value = line.split("=", 1)
|
|
env[key.strip()] = value.strip().strip('"').strip("'")
|
|
for key in ("POSTGRES_USER", "POSTGRES_DB", "SESSION_STORE"):
|
|
if os.environ.get(key):
|
|
env[key] = os.environ[key]
|
|
missing = [k for k in ("POSTGRES_USER", "POSTGRES_DB") if not env.get(k)]
|
|
if missing:
|
|
raise SystemExit(f"не заданы {', '.join(missing)} в .env")
|
|
return env
|
|
|
|
|
|
ENV = load_env()
|
|
|
|
|
|
class Pg:
|
|
"""Клиент к PostgreSQL в контейнере через stdin/stdout утилит."""
|
|
|
|
def __init__(self, exec_prefix, label):
|
|
self.prefix = exec_prefix
|
|
self.label = label
|
|
|
|
@classmethod
|
|
def compose(cls, service="db"):
|
|
return cls(["docker", "compose", "--env-file", str(ROOT / ".env"),
|
|
"exec", "-T", service], service)
|
|
|
|
@classmethod
|
|
def container(cls, name):
|
|
return cls(["docker", "exec", "-i", name], name)
|
|
|
|
def run(self, args, input=None, binary=False, check=True, stdout=None):
|
|
r = subprocess.run(
|
|
self.prefix + args,
|
|
input=input,
|
|
stdout=stdout if stdout is not None else subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
text=not binary,
|
|
cwd=ROOT,
|
|
)
|
|
if check and r.returncode != 0:
|
|
err = r.stderr if not binary else r.stderr.decode(errors="replace")
|
|
sys.stderr.write(err)
|
|
raise SystemExit(f"[{self.label}] команда {args[0]} завершилась с кодом {r.returncode}")
|
|
return r
|
|
|
|
def psql(self, sql, tuples=True):
|
|
args = ["psql", "-U", ENV["POSTGRES_USER"], "-d", ENV["POSTGRES_DB"],
|
|
"-X", "-q", "-v", "ON_ERROR_STOP=1"]
|
|
if tuples:
|
|
args += ["-t", "-A", "-F", "\t"]
|
|
return self.run(args, input=sql).stdout
|
|
|
|
def scalar(self, sql):
|
|
return self.psql(sql).strip()
|
|
|
|
def rows(self, sql):
|
|
return [line.split("\t") for line in self.psql(sql).splitlines() if line]
|
|
|
|
|
|
def median(values):
|
|
return statistics.median(values) if values else float("nan")
|
|
|
|
|
|
def pick_chats(db):
|
|
"""Выбирает чаты разного размера из засеянных perf100k.py."""
|
|
rows = db.rows("""
|
|
SELECT c.id, c.title, count(m.id)
|
|
FROM chats c JOIN messages m ON m.chat_id = c.id
|
|
WHERE c.title LIKE 'perf-%'
|
|
GROUP BY c.id, c.title;
|
|
""")
|
|
chats = [(r[0], r[1], int(r[2])) for r in rows]
|
|
regular = [c for c in chats if not c[1].startswith("perf-burst")]
|
|
burst = [c for c in chats if c[1].startswith("perf-burst")]
|
|
if sum(c[2] for c in regular) < 100_000:
|
|
raise SystemExit("меньше 100 000 perf-сообщений: запустите "
|
|
"python3 scripts/perf100k.py seed")
|
|
if not burst:
|
|
raise SystemExit("нет чата с одинаковыми created_at: запустите "
|
|
"python3 scripts/perf100k.py seed-burst")
|
|
|
|
def closest(target):
|
|
return min(regular, key=lambda c: abs(c[2] - target))
|
|
|
|
return [
|
|
("small", closest(50)),
|
|
("medium", closest(1000)),
|
|
("large", max(regular, key=lambda c: c[2])),
|
|
("burst", burst[0]),
|
|
]
|
|
|
|
|
|
def environment_lines(db):
|
|
commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT,
|
|
capture_output=True, text=True).stdout.strip()
|
|
dirty = subprocess.run(["git", "status", "--porcelain"], cwd=ROOT,
|
|
capture_output=True, text=True).stdout.strip()
|
|
try:
|
|
mem_kb = int(next(l for l in Path("/proc/meminfo").read_text().splitlines()
|
|
if l.startswith("MemTotal")).split()[1])
|
|
mem = f"{mem_kb / 1024 / 1024:.1f} GiB"
|
|
except (OSError, StopIteration, ValueError):
|
|
mem = "unknown"
|
|
version = db.scalar("SHOW server_version;")
|
|
return [
|
|
f"- Дата: {datetime.datetime.now().astimezone().isoformat(timespec='seconds')}",
|
|
f"- Commit: `{commit or 'unknown'}`{' (есть незакоммиченные изменения)' if dirty else ''}",
|
|
f"- Хост: {platform.system()} {platform.release()}, {os.cpu_count()} CPU, RAM {mem}; "
|
|
"CPU/RAM-лимиты контейнеров не заданы",
|
|
f"- PostgreSQL {version} (`postgres:16-alpine`), настройки по умолчанию, "
|
|
f"synchronous_commit={db.scalar('SHOW synchronous_commit;')}, "
|
|
f"shared_buffers={db.scalar('SHOW shared_buffers;')}",
|
|
]
|
|
|
|
|
|
def dataset_lines(db):
|
|
total, chats, mn, med, mx = db.rows("""
|
|
SELECT (SELECT count(*) FROM messages),
|
|
count(*), min(n), percentile_disc(0.5) WITHIN GROUP (ORDER BY n), max(n)
|
|
FROM (SELECT chat_id, count(*) AS n FROM messages GROUP BY chat_id) s;
|
|
""")[0]
|
|
size = db.scalar("SELECT pg_size_pretty(pg_table_size('messages'));")
|
|
return [
|
|
f"- Сообщений: {int(total):,}".replace(",", " ") +
|
|
f", чатов с сообщениями: {chats}; размер чата min/median/max = {mn}/{med}/{mx}",
|
|
f"- Размер heap таблицы `messages`: {size}",
|
|
]
|
|
|
|
|
|
def write_evidence(name, text):
|
|
path = ROOT / "evidence" / name
|
|
path.parent.mkdir(exist_ok=True)
|
|
path.write_text(text, encoding="utf-8")
|
|
print(f"\nзаписан {path.relative_to(ROOT)}")
|