"""Трек 2: два индекса и OFFSET/keyset на 100 000 сообщений. Только синтетические данные в отдельной схеме. Не меняет таблицы API. EXPLAIN включает BUFFERS, планировщику не навязывается тип плана. """ import json import os import platform import random import statistics import time from datetime import datetime, timedelta, timezone from pathlib import Path from uuid import uuid4 import psycopg from psycopg import sql from psycopg.rows import dict_row from src.config import DATABASE_URL SEED = 42 SIZES = [70000, 20000, 9000, 900, 100] REPEATS = 30 def percentile(values, p): return sorted(values)[max(0, int(len(values) * p + 0.9999) - 1)] def measure(db, query, params): for _ in range(5): db.execute(query, params).fetchall() raw = [] for _ in range(REPEATS): start = time.perf_counter() db.execute(query, params).fetchall() raw.append((time.perf_counter() - start) * 1000) plan = db.execute('EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ' + query, params).fetchone()[0][0] return {'latency_ms': raw, 'p50_ms': statistics.median(raw), 'p95_ms': percentile(raw, .95), 'p99_ms': percentile(raw, .99), 'plan': plan} def main(): schema = 'research_' + uuid4().hex[:12] rng = random.Random(SEED) out = {'seed': SEED, 'schema': schema, 'message_count': sum(SIZES), 'sizes': SIZES, 'repeats': REPEATS, 'warmup': 5, 'concurrency': 1, 'platform': platform.platform(), 'cpu_count': os.cpu_count(), 'results': []} Path('evidence').mkdir(exist_ok=True) with psycopg.connect(DATABASE_URL, autocommit=True) as db: out['postgres_version'] = db.execute('SHOW server_version').fetchone()[0] out['settings'] = {k: db.execute('SHOW ' + k).fetchone()[0] for k in ['shared_buffers', 'work_mem', 'fsync', 'synchronous_commit']} db.execute(sql.SQL('CREATE SCHEMA {}').format(sql.Identifier(schema))) db.execute(sql.SQL('SET search_path TO {}').format(sql.Identifier(schema))) # Точная схема приложения; другой search_path изолирует эксперимент. db.execute(Path('migrations/001_initial.sql').read_text()) user = uuid4() db.execute('INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s)', (user, 'research_user', 'Synthetic research', 'not-a-login-account')) chats = [uuid4() for _ in SIZES] start_date = datetime(2026, 1, 1, tzinfo=timezone.utc) for chat, size in zip(chats, SIZES): db.execute('INSERT INTO chats(id,title) VALUES (%s,%s)', (chat, 'Synthetic ' + str(size))) db.execute('INSERT INTO chat_members VALUES (%s,%s)', (chat, user)) with db.cursor().copy('COPY messages(id,chat_id,sender_id,text,created_at) FROM STDIN') as copy: for n in range(size): copy.write_row((uuid4(), chat, user, 'message ' + str(n) + ' x' * (n % 80), start_date + timedelta(microseconds=n))) samples = [] for chat in chats: for fraction in [0, .5, .9]: count = db.execute('SELECT count(*) FROM messages WHERE chat_id=%s', (chat,)).fetchone()[0] offset = int((count - 1) * fraction) cursor = db.execute('SELECT created_at,id FROM messages WHERE chat_id=%s ' 'ORDER BY created_at,id OFFSET %s LIMIT 1', (chat, offset)).fetchone() samples.append((chat, offset, cursor)) db.execute('DROP INDEX messages_history_idx') # Порядок фиксирован, но обе конфигурации прогреваются и имеют 30 повторов. for name, columns in [('chat_only', 'chat_id'), ('history', 'chat_id,created_at,id')]: db.execute('CREATE INDEX research_candidate ON messages(' + columns + ')') db.execute('ANALYZE messages') index_bytes = db.execute("SELECT pg_relation_size('research_candidate')").fetchone()[0] queries = list(samples) rng.shuffle(queries) for chat, offset, cursor in queries: for method in ['offset', 'keyset']: query = 'SELECT * FROM messages WHERE chat_id=%s' if method == 'keyset': query += ' AND (created_at,id)>(%s,%s) ORDER BY created_at,id LIMIT 51' params = (chat, *cursor) else: # Cursor указывает последнюю уже прочитанную строку. query += ' ORDER BY created_at,id OFFSET %s LIMIT 51' params = (chat, offset + 1) result = measure(db, query, params) result.update(index=name, method=method, chat_size=SIZES[chats.index(chat)], offset=offset, index_bytes=index_bytes) out['results'].append(result) write_ms = [] for batch in range(5): # Цена 1000 записей + COMMIT; затем откат данных отдельной очисткой. # Исходная история остаётся 100 000 для следующего индекса. batch_ids = [uuid4() for _ in range(1000)] begin = time.perf_counter() with db.transaction(): with db.cursor().copy('COPY messages(id,chat_id,sender_id,text,created_at) FROM STDIN') as copy: for n, mid in enumerate(batch_ids): copy.write_row((mid, chats[n % len(chats)], user, 'write-cost ' + 'x' * 128, start_date + timedelta(days=1, microseconds=n))) write_ms.append((time.perf_counter() - begin) * 1000) db.execute('DELETE FROM messages WHERE id=ANY(%s)', (batch_ids,)) out.setdefault('writes', {})[name] = {'batch_rows': 1000, 'batch_ms': write_ms, 'median_ms': statistics.median(write_ms), 'index_bytes': index_bytes} db.execute('DROP INDEX research_candidate') db.execute('VACUUM ANALYZE messages') db.execute('CREATE INDEX messages_history_idx ON messages(chat_id,created_at,id)') with db.cursor(row_factory=dict_row) as cur: out['counts'] = {} for table in ['users', 'chats', 'chat_members', 'messages', 'sessions']: out['counts'][table] = db.execute('SELECT count(*) FROM ' + table).fetchone()[0] cur.execute('SELECT * FROM messages ORDER BY created_at,id LIMIT 5') out['sample_messages'] = [dict(row) for row in cur.fetchall()] Path('evidence/track2_raw.json').write_text(json.dumps(out, ensure_ascii=False, indent=2, default=str)) lines = ['# Трек 2: измерения', '', 'Исходных сообщений: 100000. Прогрев: 5; повторов: 30.', '', '| Чат | Позиция | Индекс | Запрос | p50, мс | p95, мс |', '|---:|---:|---|---|---:|---:|'] for row in sorted(out['results'], key=lambda r: (r['chat_size'], r['offset'], r['index'], r['method'])): lines.append(f"| {row['chat_size']} | {row['offset']} | {row['index']} | {row['method']} | " f"{row['p50_ms']:.3f} | {row['p95_ms']:.3f} |") lines.extend(['', 'Цена записи: 5 транзакций по 1000 строк (COPY + COMMIT).', '', '| Индекс | Медиана партии, мс | Размер, байт |', '|---|---:|---:|']) for name, row in out['writes'].items(): lines.append(f"| {name} | {row['median_ms']:.3f} | {row['index_bytes']} |") Path('evidence/track2_summary.md').write_text('\n'.join(lines) + '\n') print('PASS: measured two indexes, two queries, write cost; raw evidence saved') if __name__ == '__main__': main()