This commit is contained in:
2026-10-23 14:47:53 +03:00
parent 65003b6119
commit 84fc48d315
29 changed files with 1940 additions and 24 deletions
+69
View File
@@ -0,0 +1,69 @@
#!/usr/bin/env python3
"""Seed >=10k messages and print EXPLAIN (ANALYZE, BUFFERS) for history pagination."""
import os
import sys
import uuid
from datetime import datetime, timezone
import psycopg
def main():
url = os.environ.get('DATABASE_URL')
if not url:
print('Set DATABASE_URL', file=sys.stderr)
raise SystemExit(2)
chat_id = os.environ.get('EXPLAIN_CHAT_ID')
user_id = os.environ.get('EXPLAIN_USER_ID')
count = int(os.environ.get('EXPLAIN_MESSAGE_COUNT', '10000'))
with psycopg.connect(url) as conn:
if not chat_id or not user_id:
user_id = str(uuid.uuid4())
chat_id = str(uuid.uuid4())
now = datetime.now(timezone.utc)
conn.execute(
'INSERT INTO users (id, username, display_name, password_hash, created_at) '
'VALUES (%s, %s, %s, %s, %s)',
(user_id, 'bench_' + uuid.uuid4().hex[:8], 'Bench', 'x', now),
)
conn.execute(
'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)',
(chat_id, 'Bench', now),
)
conn.execute(
'INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, 0)',
(chat_id, user_id),
)
print(f'created chat_id={chat_id} user_id={user_id}')
existing = conn.execute(
'SELECT count(*) FROM messages WHERE chat_id = %s',
(chat_id,),
).fetchone()[0]
need = max(0, count - existing)
base = datetime.now(timezone.utc)
with conn.cursor() as cur:
for index in range(need):
cur.execute(
'INSERT INTO messages (id, chat_id, sender_id, text, created_at) '
'VALUES (%s, %s, %s, %s, %s)',
(
str(uuid.uuid4()),
chat_id,
user_id,
'm',
base,
),
)
conn.commit()
plan = conn.execute(
'EXPLAIN (ANALYZE, BUFFERS) '
'SELECT id FROM messages WHERE chat_id = %s '
'ORDER BY created_at ASC, id ASC LIMIT 50',
(chat_id,),
).fetchall()
for row in plan:
print(row[0])
if __name__ == '__main__':
main()
+3
View File
@@ -0,0 +1,3 @@
@echo off
cd /d "%~dp0.."
docker compose restart api postgres redis
+13
View File
@@ -0,0 +1,13 @@
#!/usr/bin/env python3
"""Cross-platform restart hook; on Windows use: ACTION_SCRIPT=scripts/restart.py"""
import subprocess
import sys
from pathlib import Path
root = Path(__file__).resolve().parents[1]
result = subprocess.run(
['docker', 'compose', 'restart', 'api', 'postgres', 'redis'],
cwd=root,
check=False,
)
raise SystemExit(result.returncode)
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
root="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
cd "$root"
docker compose restart api postgres redis
+108
View File
@@ -0,0 +1,108 @@
#!/usr/bin/env python3
"""pg_dump → fresh Postgres volume → verification via psql (no local psycopg)."""
import json
import os
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / 'evidence' / 'track2'
DUMP = OUT / 'backup.dump'
MAIN = ['docker', 'compose', '-f', 'compose.yaml']
RESTORE = ['docker', 'compose', '-f', 'compose.track2-restore.yaml']
PSQL = os.environ.get('TRACK2_PSQL_SERVICE', 'postgres')
RESTORE_SVC = 'postgres_restore'
def run(cmd, **kwargs):
subprocess.run(cmd, cwd=ROOT, check=True, **kwargs)
def psql(service, compose, database, sql):
out = subprocess.check_output(
compose + ['exec', '-T', service, 'psql', '-U', 'messenger', '-d', database, '-tAc', sql],
cwd=ROOT,
text=True,
)
return out.strip()
def snapshot(service, compose, database):
messages = int(psql(service, compose, database, 'SELECT count(*) FROM messages'))
users = int(psql(service, compose, database, 'SELECT count(*) FROM users'))
raw = subprocess.check_output(
compose + [
'exec', '-T', service, 'psql', '-U', 'messenger', '-d', database, '-tAc',
"SELECT id::text || '|' || chat_id::text || '|' || text FROM messages "
'ORDER BY created_at DESC LIMIT 5',
],
cwd=ROOT,
text=True,
).strip()
sample = []
if raw:
for line in raw.splitlines():
parts = line.split('|', 2)
sample.append({'id': parts[0], 'chat_id': parts[1], 'text': parts[2]})
return {'messages': messages, 'users': users, 'sample': sample}
def main():
OUT.mkdir(parents=True, exist_ok=True)
started = time.perf_counter()
run(RESTORE + ['up', '-d', '--wait'])
with DUMP.open('wb') as handle:
run(MAIN + ['exec', '-T', PSQL, 'pg_dump', '-U', 'messenger', '-Fc', 'messenger'], stdout=handle)
before = snapshot(PSQL, MAIN, 'messenger')
run(
RESTORE + ['exec', '-T', RESTORE_SVC, 'psql', '-U', 'messenger', '-d', 'postgres', '-c',
'DROP DATABASE IF EXISTS messenger_restore'],
)
run(
RESTORE + ['exec', '-T', RESTORE_SVC, 'psql', '-U', 'messenger', '-d', 'postgres', '-c',
'CREATE DATABASE messenger_restore'],
)
run(RESTORE + ['cp', str(DUMP), f'{RESTORE_SVC}:/tmp/backup.dump'])
restore = subprocess.run(
RESTORE + [
'exec', '-T', RESTORE_SVC,
'pg_restore', '-U', 'messenger', '-d', 'messenger_restore',
'--no-owner', '--no-privileges', '/tmp/backup.dump',
],
cwd=ROOT,
)
if restore.returncode not in (0, 1):
restore.check_returncode()
after = snapshot(RESTORE_SVC, RESTORE, 'messenger_restore')
elapsed = time.perf_counter() - started
ok = (
before['messages'] == after['messages']
and before['users'] == after['users']
and before['sample'] == after['sample']
)
report = {
'source': before,
'restored': after,
'counts_match': before['messages'] == after['messages'],
'sample_match': before['sample'] == after['sample'],
'restore_seconds': round(elapsed, 2),
'dump_bytes': DUMP.stat().st_size,
'data_loss_boundary': (
'Logical pg_dump is consistent at dump start; concurrent writes during dump are not '
'included. WAL/PITR not used — RPO equals changes after dump began.'
),
}
(OUT / 'restore_report.json').write_text(
json.dumps(report, indent=2, ensure_ascii=False),
encoding='utf-8',
)
print(json.dumps(report, indent=2, ensure_ascii=False))
if not ok:
raise SystemExit(1)
print('PASS: restore verification')
if __name__ == '__main__':
main()
+117
View File
@@ -0,0 +1,117 @@
#!/usr/bin/env python3
"""Compare index plans and INSERT cost; write evidence/track2/benchmark.json."""
import json
import os
import sys
import time
import uuid
from datetime import datetime, timezone
from pathlib import Path
import psycopg
ROOT = Path(__file__).resolve().parents[1]
OUT = Path(os.environ.get('TRACK2_EVIDENCE_DIR', str(ROOT / 'evidence' / 'track2')))
DATABASE_URL = os.environ.get('DATABASE_URL', '')
HOT_CHAT_TITLE = os.environ.get('TRACK2_HOT_CHAT_TITLE', 'track2_hot')
def _find_index(plan):
if plan.get('Index Name'):
return plan.get('Index Name'), plan.get('Node Type')
for child in plan.get('Plans') or []:
found = _find_index(child)
if found:
return found
return None, plan.get('Node Type')
def explain(conn, sql, params):
rows = conn.execute('EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ' + sql, params).fetchone()[0]
plan = rows[0]['Plan']
index_name, node_type = _find_index(plan)
return {
'node_type': node_type,
'index_name': index_name,
'execution_ms': rows[0]['Execution Time'],
'plan_rows': plan.get('Plan Rows'),
}
def insert_batch(conn, chat_id, user_id, n):
base = datetime.now(timezone.utc)
start = time.perf_counter()
with conn.cursor() as cur:
cur.executemany(
'INSERT INTO messages (id, chat_id, sender_id, text, created_at) '
'VALUES (%s, %s, %s, %s, %s)',
[
(str(uuid.uuid4()), chat_id, user_id, 'w', base)
for _ in range(n)
],
)
conn.commit()
elapsed = time.perf_counter() - start
return n / elapsed
def main():
if not DATABASE_URL:
print('DATABASE_URL required', file=sys.stderr)
raise SystemExit(2)
OUT.mkdir(parents=True, exist_ok=True)
with psycopg.connect(DATABASE_URL) as conn:
hot = conn.execute(
'SELECT c.id FROM chats c WHERE c.title = %s',
(HOT_CHAT_TITLE,),
).fetchone()
if not hot:
print('Run track2_seed.py first', file=sys.stderr)
raise SystemExit(2)
hot_id = str(hot[0])
user = conn.execute(
"SELECT id FROM users WHERE username = 'track2_bench'"
).fetchone()
user_id = str(user[0])
keyset_sql = (
'SELECT id FROM messages WHERE chat_id = %s '
'ORDER BY created_at ASC, id ASC LIMIT 50'
)
offset_sql = (
'SELECT id FROM messages WHERE chat_id = %s '
'ORDER BY created_at ASC, id ASC OFFSET 50000 LIMIT 50'
)
plan_keyset = explain(conn, keyset_sql, (hot_id,))
plan_offset = explain(conn, offset_sql, (hot_id,))
conn.execute('DROP INDEX IF EXISTS messages_chat_id_only_idx')
conn.execute('CREATE INDEX messages_chat_id_only_idx ON messages (chat_id)')
conn.commit()
plan_keyset_with_extra = explain(conn, keyset_sql, (hot_id,))
write_before = insert_batch(conn, hot_id, user_id, 500)
conn.execute('DROP INDEX IF EXISTS messages_chat_id_only_idx')
conn.commit()
write_after = insert_batch(conn, hot_id, user_id, 500)
result = {
'hypothesis': (
'Keyset pagination on (chat_id, created_at, id) avoids large offsets; '
'redundant (chat_id)-only index does not improve reads but slows writes.'
),
'hot_chat_id': hot_id,
'read_plans': {
'keyset_limit_50': plan_keyset,
'offset_50000_limit_50': plan_offset,
'keyset_with_extra_chat_id_index': plan_keyset_with_extra,
},
'write_messages_per_second': {
'with_extra_chat_id_index': round(write_before, 1),
'production_indexes_only': round(write_after, 1),
},
}
out_path = OUT / 'benchmark.json'
out_path.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding='utf-8')
print(json.dumps(result, indent=2, ensure_ascii=False))
print('wrote', out_path)
if __name__ == '__main__':
main()
+37
View File
@@ -0,0 +1,37 @@
#!/usr/bin/env python3
"""Run full track 2 pipeline (seed, benchmark, backup/restore)."""
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
DB = 'postgresql://messenger:messenger_dev_only@postgres:5432/messenger'
EVIDENCE = ROOT / 'evidence' / 'track2'
EVIDENCE.mkdir(parents=True, exist_ok=True)
def docker_python(script):
subprocess.run(
[
'docker', 'compose', 'run', '--rm',
'-e', 'DATABASE_URL=' + DB,
'-e', 'TRACK2_EVIDENCE_DIR=/evidence',
'-v', f'{ROOT / "scripts"}:/scripts:ro',
'-v', f'{EVIDENCE}:/evidence',
'api', 'python', f'/scripts/{script}',
],
cwd=ROOT,
check=True,
)
def main():
for name in ('track2_seed.py', 'track2_benchmark.py'):
print('===', name, '===')
docker_python(name)
print('=== track2_backup_restore.py ===')
subprocess.run([sys.executable, str(ROOT / 'scripts' / 'track2_backup_restore.py')], cwd=ROOT, check=True)
if __name__ == '__main__':
main()
+105
View File
@@ -0,0 +1,105 @@
#!/usr/bin/env python3
"""Seed >=100k messages with skewed chat sizes (track 2)."""
import os
import sys
import uuid
from datetime import datetime, timedelta, timezone
import psycopg
TARGET = int(os.environ.get('TRACK2_MESSAGE_COUNT', '100000'))
DATABASE_URL = os.environ.get('DATABASE_URL', '')
def main():
if not DATABASE_URL:
print('DATABASE_URL required', file=sys.stderr)
raise SystemExit(2)
base = datetime.now(timezone.utc)
with psycopg.connect(DATABASE_URL) as conn:
hot_exists = conn.execute(
"SELECT 1 FROM chats WHERE title = 'track2_hot'"
).fetchone()
if hot_exists:
total = conn.execute('SELECT count(*) FROM messages').fetchone()[0]
print(f'skip_seed existing track2 dataset messages_total={total}')
return
row = conn.execute(
"SELECT id FROM users WHERE username = 'track2_bench'"
).fetchone()
if row:
user_id = str(row[0])
else:
user_id = str(uuid.uuid4())
conn.execute(
'INSERT INTO users (id, username, display_name, password_hash, created_at) '
'VALUES (%s, %s, %s, %s, %s)',
(user_id, 'track2_bench', 'Track2', 'x', base),
)
plans = [
('hot', int(TARGET * 0.70)),
('medium_a', int(TARGET * 0.05)),
('medium_b', int(TARGET * 0.05)),
('medium_c', int(TARGET * 0.05)),
('medium_d', int(TARGET * 0.05)),
('medium_e', int(TARGET * 0.05)),
]
tail = TARGET - sum(count for _, count in plans)
small_chats = 50
per_small = max(1, tail // small_chats)
chat_ids = []
for name, count in plans:
chat_id = str(uuid.uuid4())
chat_ids.append((chat_id, count, name))
conn.execute(
'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)',
(chat_id, 'track2_' + name, base),
)
conn.execute(
'INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, 0)',
(chat_id, user_id),
)
for index in range(small_chats):
chat_id = str(uuid.uuid4())
chat_ids.append((chat_id, per_small, f'small_{index}'))
conn.execute(
'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)',
(chat_id, 'track2_small_' + str(index), base),
)
conn.execute(
'INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, 0)',
(chat_id, user_id),
)
total = 0
with conn.cursor() as cur:
for chat_id, count, _name in chat_ids:
rows = []
for offset in range(count):
rows.append(
(
str(uuid.uuid4()),
chat_id,
user_id,
't',
base + timedelta(microseconds=offset),
)
)
cur.executemany(
'INSERT INTO messages (id, chat_id, sender_id, text, created_at) '
'VALUES (%s, %s, %s, %s, %s)',
rows,
)
total += count
conn.commit()
actual = conn.execute('SELECT count(*) FROM messages').fetchone()[0]
hot = conn.execute(
"SELECT count(*) FROM messages m JOIN chats c ON c.id = m.chat_id WHERE c.title = 'track2_hot'"
).fetchone()[0]
print(f'seeded_messages={total} table_total={actual} hot_chat_messages={hot}')
print(f'benchmark_user_id={user_id}')
for chat_id, count, name in chat_ids[:3]:
print(f'sample_chat {name} id={chat_id} messages={count}')
if __name__ == '__main__':
main()