forked from mtusi2026/mtusi2026lab2
lab2
This commit is contained in:
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Seed >=10k messages and print EXPLAIN (ANALYZE, BUFFERS) for history pagination."""
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import psycopg
|
||||
|
||||
|
||||
def main():
|
||||
url = os.environ.get('DATABASE_URL')
|
||||
if not url:
|
||||
print('Set DATABASE_URL', file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
chat_id = os.environ.get('EXPLAIN_CHAT_ID')
|
||||
user_id = os.environ.get('EXPLAIN_USER_ID')
|
||||
count = int(os.environ.get('EXPLAIN_MESSAGE_COUNT', '10000'))
|
||||
with psycopg.connect(url) as conn:
|
||||
if not chat_id or not user_id:
|
||||
user_id = str(uuid.uuid4())
|
||||
chat_id = str(uuid.uuid4())
|
||||
now = datetime.now(timezone.utc)
|
||||
conn.execute(
|
||||
'INSERT INTO users (id, username, display_name, password_hash, created_at) '
|
||||
'VALUES (%s, %s, %s, %s, %s)',
|
||||
(user_id, 'bench_' + uuid.uuid4().hex[:8], 'Bench', 'x', now),
|
||||
)
|
||||
conn.execute(
|
||||
'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)',
|
||||
(chat_id, 'Bench', now),
|
||||
)
|
||||
conn.execute(
|
||||
'INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, 0)',
|
||||
(chat_id, user_id),
|
||||
)
|
||||
print(f'created chat_id={chat_id} user_id={user_id}')
|
||||
existing = conn.execute(
|
||||
'SELECT count(*) FROM messages WHERE chat_id = %s',
|
||||
(chat_id,),
|
||||
).fetchone()[0]
|
||||
need = max(0, count - existing)
|
||||
base = datetime.now(timezone.utc)
|
||||
with conn.cursor() as cur:
|
||||
for index in range(need):
|
||||
cur.execute(
|
||||
'INSERT INTO messages (id, chat_id, sender_id, text, created_at) '
|
||||
'VALUES (%s, %s, %s, %s, %s)',
|
||||
(
|
||||
str(uuid.uuid4()),
|
||||
chat_id,
|
||||
user_id,
|
||||
'm',
|
||||
base,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
plan = conn.execute(
|
||||
'EXPLAIN (ANALYZE, BUFFERS) '
|
||||
'SELECT id FROM messages WHERE chat_id = %s '
|
||||
'ORDER BY created_at ASC, id ASC LIMIT 50',
|
||||
(chat_id,),
|
||||
).fetchall()
|
||||
for row in plan:
|
||||
print(row[0])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
cd /d "%~dp0.."
|
||||
docker compose restart api postgres redis
|
||||
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Cross-platform restart hook; on Windows use: ACTION_SCRIPT=scripts/restart.py"""
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
result = subprocess.run(
|
||||
['docker', 'compose', 'restart', 'api', 'postgres', 'redis'],
|
||||
cwd=root,
|
||||
check=False,
|
||||
)
|
||||
raise SystemExit(result.returncode)
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
root="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
|
||||
cd "$root"
|
||||
docker compose restart api postgres redis
|
||||
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env python3
|
||||
"""pg_dump → fresh Postgres volume → verification via psql (no local psycopg)."""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = ROOT / 'evidence' / 'track2'
|
||||
DUMP = OUT / 'backup.dump'
|
||||
MAIN = ['docker', 'compose', '-f', 'compose.yaml']
|
||||
RESTORE = ['docker', 'compose', '-f', 'compose.track2-restore.yaml']
|
||||
PSQL = os.environ.get('TRACK2_PSQL_SERVICE', 'postgres')
|
||||
RESTORE_SVC = 'postgres_restore'
|
||||
|
||||
|
||||
def run(cmd, **kwargs):
|
||||
subprocess.run(cmd, cwd=ROOT, check=True, **kwargs)
|
||||
|
||||
|
||||
def psql(service, compose, database, sql):
|
||||
out = subprocess.check_output(
|
||||
compose + ['exec', '-T', service, 'psql', '-U', 'messenger', '-d', database, '-tAc', sql],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
)
|
||||
return out.strip()
|
||||
|
||||
|
||||
def snapshot(service, compose, database):
|
||||
messages = int(psql(service, compose, database, 'SELECT count(*) FROM messages'))
|
||||
users = int(psql(service, compose, database, 'SELECT count(*) FROM users'))
|
||||
raw = subprocess.check_output(
|
||||
compose + [
|
||||
'exec', '-T', service, 'psql', '-U', 'messenger', '-d', database, '-tAc',
|
||||
"SELECT id::text || '|' || chat_id::text || '|' || text FROM messages "
|
||||
'ORDER BY created_at DESC LIMIT 5',
|
||||
],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
).strip()
|
||||
sample = []
|
||||
if raw:
|
||||
for line in raw.splitlines():
|
||||
parts = line.split('|', 2)
|
||||
sample.append({'id': parts[0], 'chat_id': parts[1], 'text': parts[2]})
|
||||
return {'messages': messages, 'users': users, 'sample': sample}
|
||||
|
||||
|
||||
def main():
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
started = time.perf_counter()
|
||||
run(RESTORE + ['up', '-d', '--wait'])
|
||||
with DUMP.open('wb') as handle:
|
||||
run(MAIN + ['exec', '-T', PSQL, 'pg_dump', '-U', 'messenger', '-Fc', 'messenger'], stdout=handle)
|
||||
before = snapshot(PSQL, MAIN, 'messenger')
|
||||
run(
|
||||
RESTORE + ['exec', '-T', RESTORE_SVC, 'psql', '-U', 'messenger', '-d', 'postgres', '-c',
|
||||
'DROP DATABASE IF EXISTS messenger_restore'],
|
||||
)
|
||||
run(
|
||||
RESTORE + ['exec', '-T', RESTORE_SVC, 'psql', '-U', 'messenger', '-d', 'postgres', '-c',
|
||||
'CREATE DATABASE messenger_restore'],
|
||||
)
|
||||
run(RESTORE + ['cp', str(DUMP), f'{RESTORE_SVC}:/tmp/backup.dump'])
|
||||
restore = subprocess.run(
|
||||
RESTORE + [
|
||||
'exec', '-T', RESTORE_SVC,
|
||||
'pg_restore', '-U', 'messenger', '-d', 'messenger_restore',
|
||||
'--no-owner', '--no-privileges', '/tmp/backup.dump',
|
||||
],
|
||||
cwd=ROOT,
|
||||
)
|
||||
if restore.returncode not in (0, 1):
|
||||
restore.check_returncode()
|
||||
after = snapshot(RESTORE_SVC, RESTORE, 'messenger_restore')
|
||||
elapsed = time.perf_counter() - started
|
||||
ok = (
|
||||
before['messages'] == after['messages']
|
||||
and before['users'] == after['users']
|
||||
and before['sample'] == after['sample']
|
||||
)
|
||||
report = {
|
||||
'source': before,
|
||||
'restored': after,
|
||||
'counts_match': before['messages'] == after['messages'],
|
||||
'sample_match': before['sample'] == after['sample'],
|
||||
'restore_seconds': round(elapsed, 2),
|
||||
'dump_bytes': DUMP.stat().st_size,
|
||||
'data_loss_boundary': (
|
||||
'Logical pg_dump is consistent at dump start; concurrent writes during dump are not '
|
||||
'included. WAL/PITR not used — RPO equals changes after dump began.'
|
||||
),
|
||||
}
|
||||
(OUT / 'restore_report.json').write_text(
|
||||
json.dumps(report, indent=2, ensure_ascii=False),
|
||||
encoding='utf-8',
|
||||
)
|
||||
print(json.dumps(report, indent=2, ensure_ascii=False))
|
||||
if not ok:
|
||||
raise SystemExit(1)
|
||||
print('PASS: restore verification')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare index plans and INSERT cost; write evidence/track2/benchmark.json."""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = Path(os.environ.get('TRACK2_EVIDENCE_DIR', str(ROOT / 'evidence' / 'track2')))
|
||||
DATABASE_URL = os.environ.get('DATABASE_URL', '')
|
||||
HOT_CHAT_TITLE = os.environ.get('TRACK2_HOT_CHAT_TITLE', 'track2_hot')
|
||||
|
||||
|
||||
def _find_index(plan):
|
||||
if plan.get('Index Name'):
|
||||
return plan.get('Index Name'), plan.get('Node Type')
|
||||
for child in plan.get('Plans') or []:
|
||||
found = _find_index(child)
|
||||
if found:
|
||||
return found
|
||||
return None, plan.get('Node Type')
|
||||
|
||||
|
||||
def explain(conn, sql, params):
|
||||
rows = conn.execute('EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ' + sql, params).fetchone()[0]
|
||||
plan = rows[0]['Plan']
|
||||
index_name, node_type = _find_index(plan)
|
||||
return {
|
||||
'node_type': node_type,
|
||||
'index_name': index_name,
|
||||
'execution_ms': rows[0]['Execution Time'],
|
||||
'plan_rows': plan.get('Plan Rows'),
|
||||
}
|
||||
|
||||
|
||||
def insert_batch(conn, chat_id, user_id, n):
|
||||
base = datetime.now(timezone.utc)
|
||||
start = time.perf_counter()
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
'INSERT INTO messages (id, chat_id, sender_id, text, created_at) '
|
||||
'VALUES (%s, %s, %s, %s, %s)',
|
||||
[
|
||||
(str(uuid.uuid4()), chat_id, user_id, 'w', base)
|
||||
for _ in range(n)
|
||||
],
|
||||
)
|
||||
conn.commit()
|
||||
elapsed = time.perf_counter() - start
|
||||
return n / elapsed
|
||||
|
||||
|
||||
def main():
|
||||
if not DATABASE_URL:
|
||||
print('DATABASE_URL required', file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
with psycopg.connect(DATABASE_URL) as conn:
|
||||
hot = conn.execute(
|
||||
'SELECT c.id FROM chats c WHERE c.title = %s',
|
||||
(HOT_CHAT_TITLE,),
|
||||
).fetchone()
|
||||
if not hot:
|
||||
print('Run track2_seed.py first', file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
hot_id = str(hot[0])
|
||||
user = conn.execute(
|
||||
"SELECT id FROM users WHERE username = 'track2_bench'"
|
||||
).fetchone()
|
||||
user_id = str(user[0])
|
||||
keyset_sql = (
|
||||
'SELECT id FROM messages WHERE chat_id = %s '
|
||||
'ORDER BY created_at ASC, id ASC LIMIT 50'
|
||||
)
|
||||
offset_sql = (
|
||||
'SELECT id FROM messages WHERE chat_id = %s '
|
||||
'ORDER BY created_at ASC, id ASC OFFSET 50000 LIMIT 50'
|
||||
)
|
||||
plan_keyset = explain(conn, keyset_sql, (hot_id,))
|
||||
plan_offset = explain(conn, offset_sql, (hot_id,))
|
||||
conn.execute('DROP INDEX IF EXISTS messages_chat_id_only_idx')
|
||||
conn.execute('CREATE INDEX messages_chat_id_only_idx ON messages (chat_id)')
|
||||
conn.commit()
|
||||
plan_keyset_with_extra = explain(conn, keyset_sql, (hot_id,))
|
||||
write_before = insert_batch(conn, hot_id, user_id, 500)
|
||||
conn.execute('DROP INDEX IF EXISTS messages_chat_id_only_idx')
|
||||
conn.commit()
|
||||
write_after = insert_batch(conn, hot_id, user_id, 500)
|
||||
result = {
|
||||
'hypothesis': (
|
||||
'Keyset pagination on (chat_id, created_at, id) avoids large offsets; '
|
||||
'redundant (chat_id)-only index does not improve reads but slows writes.'
|
||||
),
|
||||
'hot_chat_id': hot_id,
|
||||
'read_plans': {
|
||||
'keyset_limit_50': plan_keyset,
|
||||
'offset_50000_limit_50': plan_offset,
|
||||
'keyset_with_extra_chat_id_index': plan_keyset_with_extra,
|
||||
},
|
||||
'write_messages_per_second': {
|
||||
'with_extra_chat_id_index': round(write_before, 1),
|
||||
'production_indexes_only': round(write_after, 1),
|
||||
},
|
||||
}
|
||||
out_path = OUT / 'benchmark.json'
|
||||
out_path.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding='utf-8')
|
||||
print(json.dumps(result, indent=2, ensure_ascii=False))
|
||||
print('wrote', out_path)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run full track 2 pipeline (seed, benchmark, backup/restore)."""
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DB = 'postgresql://messenger:messenger_dev_only@postgres:5432/messenger'
|
||||
EVIDENCE = ROOT / 'evidence' / 'track2'
|
||||
EVIDENCE.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def docker_python(script):
|
||||
subprocess.run(
|
||||
[
|
||||
'docker', 'compose', 'run', '--rm',
|
||||
'-e', 'DATABASE_URL=' + DB,
|
||||
'-e', 'TRACK2_EVIDENCE_DIR=/evidence',
|
||||
'-v', f'{ROOT / "scripts"}:/scripts:ro',
|
||||
'-v', f'{EVIDENCE}:/evidence',
|
||||
'api', 'python', f'/scripts/{script}',
|
||||
],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
for name in ('track2_seed.py', 'track2_benchmark.py'):
|
||||
print('===', name, '===')
|
||||
docker_python(name)
|
||||
print('=== track2_backup_restore.py ===')
|
||||
subprocess.run([sys.executable, str(ROOT / 'scripts' / 'track2_backup_restore.py')], cwd=ROOT, check=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Seed >=100k messages with skewed chat sizes (track 2)."""
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import psycopg
|
||||
|
||||
TARGET = int(os.environ.get('TRACK2_MESSAGE_COUNT', '100000'))
|
||||
DATABASE_URL = os.environ.get('DATABASE_URL', '')
|
||||
|
||||
|
||||
def main():
|
||||
if not DATABASE_URL:
|
||||
print('DATABASE_URL required', file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
base = datetime.now(timezone.utc)
|
||||
with psycopg.connect(DATABASE_URL) as conn:
|
||||
hot_exists = conn.execute(
|
||||
"SELECT 1 FROM chats WHERE title = 'track2_hot'"
|
||||
).fetchone()
|
||||
if hot_exists:
|
||||
total = conn.execute('SELECT count(*) FROM messages').fetchone()[0]
|
||||
print(f'skip_seed existing track2 dataset messages_total={total}')
|
||||
return
|
||||
row = conn.execute(
|
||||
"SELECT id FROM users WHERE username = 'track2_bench'"
|
||||
).fetchone()
|
||||
if row:
|
||||
user_id = str(row[0])
|
||||
else:
|
||||
user_id = str(uuid.uuid4())
|
||||
conn.execute(
|
||||
'INSERT INTO users (id, username, display_name, password_hash, created_at) '
|
||||
'VALUES (%s, %s, %s, %s, %s)',
|
||||
(user_id, 'track2_bench', 'Track2', 'x', base),
|
||||
)
|
||||
plans = [
|
||||
('hot', int(TARGET * 0.70)),
|
||||
('medium_a', int(TARGET * 0.05)),
|
||||
('medium_b', int(TARGET * 0.05)),
|
||||
('medium_c', int(TARGET * 0.05)),
|
||||
('medium_d', int(TARGET * 0.05)),
|
||||
('medium_e', int(TARGET * 0.05)),
|
||||
]
|
||||
tail = TARGET - sum(count for _, count in plans)
|
||||
small_chats = 50
|
||||
per_small = max(1, tail // small_chats)
|
||||
chat_ids = []
|
||||
for name, count in plans:
|
||||
chat_id = str(uuid.uuid4())
|
||||
chat_ids.append((chat_id, count, name))
|
||||
conn.execute(
|
||||
'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)',
|
||||
(chat_id, 'track2_' + name, base),
|
||||
)
|
||||
conn.execute(
|
||||
'INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, 0)',
|
||||
(chat_id, user_id),
|
||||
)
|
||||
for index in range(small_chats):
|
||||
chat_id = str(uuid.uuid4())
|
||||
chat_ids.append((chat_id, per_small, f'small_{index}'))
|
||||
conn.execute(
|
||||
'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)',
|
||||
(chat_id, 'track2_small_' + str(index), base),
|
||||
)
|
||||
conn.execute(
|
||||
'INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, 0)',
|
||||
(chat_id, user_id),
|
||||
)
|
||||
total = 0
|
||||
with conn.cursor() as cur:
|
||||
for chat_id, count, _name in chat_ids:
|
||||
rows = []
|
||||
for offset in range(count):
|
||||
rows.append(
|
||||
(
|
||||
str(uuid.uuid4()),
|
||||
chat_id,
|
||||
user_id,
|
||||
't',
|
||||
base + timedelta(microseconds=offset),
|
||||
)
|
||||
)
|
||||
cur.executemany(
|
||||
'INSERT INTO messages (id, chat_id, sender_id, text, created_at) '
|
||||
'VALUES (%s, %s, %s, %s, %s)',
|
||||
rows,
|
||||
)
|
||||
total += count
|
||||
conn.commit()
|
||||
actual = conn.execute('SELECT count(*) FROM messages').fetchone()[0]
|
||||
hot = conn.execute(
|
||||
"SELECT count(*) FROM messages m JOIN chats c ON c.id = m.chat_id WHERE c.title = 'track2_hot'"
|
||||
).fetchone()[0]
|
||||
print(f'seeded_messages={total} table_total={actual} hot_chat_messages={hot}')
|
||||
print(f'benchmark_user_id={user_id}')
|
||||
for chat_id, count, name in chat_ids[:3]:
|
||||
print(f'sample_chat {name} id={chat_id} messages={count}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user