140 lines
6.5 KiB
Python
140 lines
6.5 KiB
Python
"""Анализ плана запроса истории: EXPLAIN (ANALYZE, BUFFERS) на синтетических данных.
|
|
|
|
Запуск: python -m app.explain [--messages N]. Создаёт данные один раз,
|
|
затем сравнивает запрос приложения с OFFSET и с вариантом без индекса.
|
|
"""
|
|
|
|
import argparse
|
|
import asyncio
|
|
import sys
|
|
import uuid
|
|
from datetime import datetime
|
|
from typing import List, Optional, Tuple
|
|
|
|
from sqlalchemy.ext.asyncio import AsyncEngine
|
|
|
|
from .config import Settings
|
|
from .crud import history_query
|
|
from .db import create_engine
|
|
|
|
SEED_NAMESPACE = uuid.UUID("6f1c1a52-6a39-4c1e-9d0e-5ab0e0c0e2a1")
|
|
BIG_CHAT = uuid.uuid5(SEED_NAMESPACE, "chat-big")
|
|
OTHER_CHATS = [uuid.uuid5(SEED_NAMESPACE, "chat-%d" % n) for n in range(49)]
|
|
SENDERS = [uuid.uuid5(SEED_NAMESPACE, "user-a"), uuid.uuid5(SEED_NAMESPACE, "user-b")]
|
|
PAGE = 50
|
|
DEEP_OFFSET = 15000
|
|
|
|
|
|
async def seed(pg, total: int) -> None:
|
|
"""Создать синтетических пользователей, чаты и сообщения, если их ещё нет."""
|
|
if await pg.fetchval("SELECT 1 FROM chats WHERE id = $1", BIG_CHAT):
|
|
return
|
|
async with pg.transaction():
|
|
await pg.executemany(
|
|
"INSERT INTO users (id, username, display_name, password_hash)"
|
|
" VALUES ($1, $2, $3, 'disabled')",
|
|
[(SENDERS[0], "explain_seed_a", "Seed A"), (SENDERS[1], "explain_seed_b", "Seed B")],
|
|
)
|
|
chats = [BIG_CHAT] + OTHER_CHATS
|
|
await pg.executemany(
|
|
"INSERT INTO chats (id, title, created_at)"
|
|
" VALUES ($1, $2, now() - interval '30 days')",
|
|
[(chat, "explain seed %d" % n) for n, chat in enumerate(chats)],
|
|
)
|
|
await pg.executemany(
|
|
"INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)",
|
|
[(chat, sender) for chat in chats for sender in SENDERS],
|
|
)
|
|
await pg.execute(
|
|
"""
|
|
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
|
SELECT gen_random_uuid(),
|
|
CASE WHEN g % 3 = 0 THEN $1::uuid ELSE ($2::uuid[])[1 + g % 49] END,
|
|
($3::uuid[])[1 + g % 2],
|
|
'synthetic message #' || g,
|
|
now() - interval '30 days' + g * interval '10 milliseconds'
|
|
FROM generate_series(1, $4::int) AS g
|
|
""",
|
|
BIG_CHAT, OTHER_CHATS, SENDERS, total,
|
|
)
|
|
await pg.execute("ANALYZE messages")
|
|
await pg.execute("ANALYZE chats")
|
|
await pg.execute("ANALYZE chat_members")
|
|
|
|
|
|
def compiled(engine: AsyncEngine, after: Optional[Tuple[datetime, uuid.UUID]]) -> Tuple[str, List]:
|
|
"""Текст SQL запроса истории и значения его параметров."""
|
|
query = history_query(BIG_CHAT, after, PAGE).compile(dialect=engine.dialect)
|
|
return query.string, [query.params[name] for name in query.positiontup]
|
|
|
|
|
|
async def explain(pg, title: str, sql: str, params: List) -> None:
|
|
"""Напечатать запрос, параметры и план выполнения."""
|
|
print("\n" + "=" * 78)
|
|
print(title)
|
|
print("=" * 78)
|
|
print(sql)
|
|
print("-- параметры:", ", ".join(str(p) for p in params))
|
|
for row in await pg.fetch("EXPLAIN (ANALYZE, BUFFERS) " + sql, *params):
|
|
print(row[0])
|
|
|
|
|
|
async def main_async(total: int) -> int:
|
|
"""Заполнить данные и вывести планы запросов."""
|
|
engine = create_engine(Settings.from_env())
|
|
try:
|
|
async with engine.connect() as conn:
|
|
pg = (await conn.get_raw_connection()).driver_connection
|
|
await seed(pg, total)
|
|
print("PostgreSQL:", await pg.fetchval("SHOW server_version"))
|
|
print("Сообщений всего:", await pg.fetchval("SELECT count(*) FROM messages"))
|
|
print("Сообщений в большом чате:",
|
|
await pg.fetchval("SELECT count(*) FROM messages WHERE chat_id = $1", BIG_CHAT))
|
|
print("Размер таблицы messages:",
|
|
await pg.fetchval("SELECT pg_size_pretty(pg_relation_size('messages'))"))
|
|
print("Размер индекса messages_chat_created_idx:",
|
|
await pg.fetchval("SELECT pg_size_pretty(pg_relation_size('messages_chat_created_idx'))"))
|
|
cursor = await pg.fetchrow(
|
|
"SELECT created_at, id FROM messages WHERE chat_id = $1"
|
|
" ORDER BY created_at, id OFFSET $2 LIMIT 1",
|
|
BIG_CHAT, DEEP_OFFSET - 1,
|
|
)
|
|
after = (cursor["created_at"], cursor["id"])
|
|
|
|
sql, params = compiled(engine, None)
|
|
await explain(pg, "1. Первая страница (без cursor), запрос приложения", sql, params)
|
|
sql, params = compiled(engine, after)
|
|
await explain(pg, "2. Глубокая страница: cursor после %d-го сообщения (keyset)"
|
|
% DEEP_OFFSET, sql, params)
|
|
await explain(
|
|
pg, "3. Для сравнения: та же страница через OFFSET (так приложение НЕ делает)",
|
|
"SELECT * FROM messages WHERE chat_id = $1 ORDER BY created_at, id"
|
|
" OFFSET %d LIMIT %d" % (DEEP_OFFSET, PAGE + 1),
|
|
[BIG_CHAT],
|
|
)
|
|
tx = pg.transaction()
|
|
await tx.start()
|
|
try:
|
|
await pg.execute("DROP INDEX messages_chat_created_idx")
|
|
await explain(pg, "4. Для сравнения: запрос приложения БЕЗ индекса"
|
|
" (DROP INDEX в транзакции, затем ROLLBACK)", sql, params)
|
|
finally:
|
|
await tx.rollback()
|
|
print("\nИндекс messages_chat_created_idx на месте:",
|
|
bool(await pg.fetchval("SELECT to_regclass('messages_chat_created_idx')")))
|
|
finally:
|
|
await engine.dispose()
|
|
return 0
|
|
|
|
|
|
def main() -> int:
|
|
"""Разобрать аргументы командной строки и запустить анализ."""
|
|
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
parser.add_argument("--messages", type=int, default=60000,
|
|
help="сколько синтетических сообщений создать при первом запуске")
|
|
return asyncio.run(main_async(parser.parse_args().messages))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|