Files
mtusi2026lab2/student-tests/experiment_track2.py
T
vikaandCursor c4c7a6aca2 Перенести мессенджер в PostgreSQL и сессии в Redis.
Подтверждённые сообщения и действующие сессии переживают перезапуск, история читается по индексу, копия восстанавливается в новый том.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-23 12:24:59 +03:00

518 lines
18 KiB
Python

"""Track 2: uneven history, two indexes, backup into a new Postgres volume.
The bench database is not the API database. Scratch containers are removed at the end.
"""
import json
import platform
import subprocess
import sys
import time
import uuid
from datetime import datetime, timedelta, timezone
from pathlib import Path
import psycopg
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'src'))
import db # noqa: E402
EVIDENCE = ROOT / 'evidence'
BACKUPS = ROOT / 'backups'
RESTORE_NAME = 'mtusi-lab2-restore'
RESTORE_VOLUME = 'mtusi-lab2-restore-data'
def load_env():
env = {}
for line in (ROOT / '.env').read_text(encoding='utf-8').splitlines():
line = line.strip()
if not line or line.startswith('#') or '=' not in line:
continue
key, value = line.split('=', 1)
env[key] = value
return env
def connect(url):
return psycopg.connect(url, connect_timeout=10)
def chat_sizes():
sizes = [80000] + [4000] * 4 + [300] * 10 + [40] * 20 + [10] * 20 + [1] * 100
if sum(sizes) < 100000:
raise SystemExit('fixture is smaller than 100000 messages')
return sizes
def reset_bench(admin_url):
with psycopg.connect(admin_url, autocommit=True) as conn:
conn.execute(
"""
SELECT pg_terminate_backend(pid)
FROM pg_stat_activity
WHERE datname = 'messenger_bench' AND pid <> pg_backend_pid()
"""
)
conn.execute('DROP DATABASE IF EXISTS messenger_bench')
conn.execute('CREATE DATABASE messenger_bench')
def seed(url):
users = []
with connect(url) as conn:
for name in ('bench_a', 'bench_b', 'bench_c'):
user_id = uuid.uuid4()
conn.execute(
"""
INSERT INTO users (id, username, display_name, password_hash, created_at)
VALUES (%s, %s, %s, %s, %s)
""",
(user_id, name, 'Стенд ' + name, 'bench-not-a-login-hash', datetime.now(timezone.utc)),
)
users.append(user_id)
conn.commit()
sizes = chat_sizes()
chats = []
for index, size in enumerate(sizes):
chat_id = uuid.uuid4()
conn.execute(
'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)',
(chat_id, 'чат %s' % index, datetime.now(timezone.utc)),
)
member_ids = users if size >= 4000 else users[:1]
for ordinal, member_id in enumerate(member_ids):
conn.execute(
"""
INSERT INTO chat_members (chat_id, user_id, member_ord)
VALUES (%s, %s, %s)
""",
(chat_id, member_id, ordinal),
)
chats.append((chat_id, size))
conn.commit()
samples = {}
base = datetime(2026, 1, 1, tzinfo=timezone.utc)
# Spread one chat's messages across the whole timeline. An index on
# created_at alone then has to skip other chats; (chat_id, created_at, id) does not.
chat_count = len(chats)
sender = users[0]
with conn.cursor() as cur:
with cur.copy(
'COPY messages (id, chat_id, sender_id, body, created_at) FROM STDIN'
) as copy:
for chat_index, (chat_id, size) in enumerate(chats):
for i in range(size):
message_id = uuid.uuid4()
created = base + timedelta(microseconds=i * chat_count + chat_index)
if i % 1000 == 0:
body = ('длинное-%s-' % i) + ('я' * 180)
else:
body = 'm-%s-%s' % (chat_index, i)
copy.write_row((message_id, chat_id, sender, body, created))
mark = {
'id': str(message_id),
'chat_id': str(chat_id),
'sender_id': str(sender),
'body': body,
'created_at': created.strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
}
if chat_index == 0 and i == 0:
samples['huge_first'] = mark
elif chat_index == 0 and i == 40000:
samples['huge_middle'] = mark
elif chat_index == 0 and i == size - 1:
samples['huge_last'] = mark
elif chat_index == 1 and i == 2000:
samples['medium'] = mark
elif size == 1 and 'tiny' not in samples:
samples['tiny'] = mark
if chat_index % 20 == 0:
print('seeded chat %s/%s' % (chat_index + 1, len(chats)), flush=True)
conn.commit()
conn.execute('ANALYZE users, chats, chat_members, messages')
conn.commit()
huge_id = chats[0][0]
return {
'sizes': sizes,
'samples': samples,
'huge_chat_id': str(huge_id),
'sender_id': str(sender),
'message_count': sum(sizes),
'chat_count': len(sizes),
'user_count': len(users),
}
def explain(conn, sql, params):
text = ''
for _ in range(2):
rows = conn.execute('EXPLAIN (ANALYZE, BUFFERS) ' + sql, params).fetchall()
text = '\n'.join(row[0] for row in rows)
return text
def summary_line(text, needle):
for line in text.splitlines():
if needle in line:
return line.strip()
return ''
def plan_brief(text):
scan = ''
for line in text.splitlines():
stripped = line.strip()
if 'Scan' in stripped or 'Seq Scan' in stripped:
scan = stripped
break
return {
'scan': scan,
'buffers': summary_line(text, 'Buffers:'),
'execution_time': summary_line(text, 'Execution Time:'),
'planning_time': summary_line(text, 'Planning Time:'),
}
KEYSET = """
SELECT id, chat_id, sender_id, body, created_at
FROM messages
WHERE chat_id = %s
AND (created_at, id) > (%s, %s)
ORDER BY created_at, id
LIMIT 50
"""
OFFSET = """
SELECT id, chat_id, sender_id, body, created_at
FROM messages
WHERE chat_id = %s
ORDER BY created_at, id
OFFSET 40000
LIMIT 50
"""
def _mark_params(mark):
created = datetime.strptime(mark['created_at'], '%Y-%m-%dT%H:%M:%S.%fZ').replace(tzinfo=timezone.utc)
return (uuid.UUID(mark['chat_id']), created, uuid.UUID(mark['id']))
def plans(url, seeded):
# Page inside the 80k chat. Past the smaller chats this timestamp is mostly one chat,
# so the created_at-only index is judged on the 4k chat instead, where other chats still overlap.
huge_params = _mark_params(seeded['samples']['huge_middle'])
medium_params = _mark_params(seeded['samples']['medium'])
huge = uuid.UUID(seeded['huge_chat_id'])
with connect(url) as conn:
keyset = explain(conn, KEYSET, huge_params)
offset = explain(conn, OFFSET, (huge,))
conn.execute('DROP INDEX messages_chat_history_idx')
conn.commit()
seqscan = explain(conn, KEYSET, medium_params)
conn.execute('CREATE INDEX messages_created_only_idx ON messages (created_at)')
conn.execute('ANALYZE messages')
conn.commit()
created_only = explain(conn, KEYSET, medium_params)
conn.execute('DROP INDEX messages_created_only_idx')
started = time.perf_counter()
conn.execute(
'CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id)'
)
conn.commit()
create_index_ms = (time.perf_counter() - started) * 1000
conn.execute('ANALYZE messages')
conn.commit()
return {
'keyset': keyset,
'offset': offset,
'seqscan': seqscan,
'created_only': created_only,
'create_index_ms': round(create_index_ms, 1),
}
def timed_insert(conn, chat_id, sender_id, n):
base = datetime(2027, 1, 1, tzinfo=timezone.utc)
started = time.perf_counter()
try:
with conn.cursor() as cur:
with cur.copy(
'COPY messages (id, chat_id, sender_id, body, created_at) FROM STDIN'
) as copy:
for i in range(n):
copy.write_row((
uuid.uuid4(),
chat_id,
sender_id,
'cost-%s' % i,
base + timedelta(microseconds=i),
))
return (time.perf_counter() - started) * 1000
finally:
conn.rollback()
def write_cost(url, seeded):
chat_id = uuid.UUID(seeded['huge_chat_id'])
sender_id = uuid.UUID(seeded['sender_id'])
with connect(url) as conn:
with_index = timed_insert(conn, chat_id, sender_id, 2000)
conn.execute('DROP INDEX messages_chat_history_idx')
conn.commit()
without_index = timed_insert(conn, chat_id, sender_id, 2000)
conn.execute(
'CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id)'
)
conn.commit()
return {
'rows': 2000,
'with_history_index_ms': round(with_index, 1),
'without_secondary_index_ms': round(without_index, 1),
}
def counts(url):
with connect(url) as conn:
row = conn.execute(
"""
SELECT
(SELECT count(*) FROM users),
(SELECT count(*) FROM chats),
(SELECT count(*) FROM chat_members),
(SELECT count(*) FROM messages),
(SELECT version()),
(SELECT current_setting('synchronous_commit')),
(SELECT current_setting('fsync')),
(SELECT current_setting('shared_buffers'))
"""
).fetchone()
return {
'users': row[0],
'chats': row[1],
'chat_members': row[2],
'messages': row[3],
'version': row[4],
'synchronous_commit': row[5],
'fsync': row[6],
'shared_buffers': row[7],
}
def fetch_sample(url, message_id):
with connect(url) as conn:
row = conn.execute(
"""
SELECT id::text, chat_id::text, sender_id::text, body,
to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"')
FROM messages
WHERE id = %s
""",
(uuid.UUID(message_id),),
).fetchone()
if row is None:
return None
return {
'id': row[0],
'chat_id': row[1],
'sender_id': row[2],
'body': row[3],
'created_at': row[4],
}
def dump_bench(password):
started = time.perf_counter()
# pg_dump writes the custom archive to stdout. No TTY, so the bytes stay intact.
data = subprocess.check_output(
[
'docker', 'exec', 'mtusi-lab2-postgres',
'pg_dump', '-U', 'messenger', '-d', 'messenger_bench',
'-Fc', '--no-owner', '--no-acl',
]
)
BACKUPS.mkdir(exist_ok=True)
path = BACKUPS / 'messenger_bench.dump'
path.write_bytes(data)
return path, round(time.perf_counter() - started, 3), len(data)
def image_of_source():
return subprocess.check_output(
['docker', 'inspect', '-f', '{{.Config.Image}}', 'mtusi-lab2-postgres'],
text=True,
).strip()
def restore_into_new_volume(dump_path, password, image):
subprocess.run(['docker', 'rm', '-f', RESTORE_NAME], capture_output=True)
subprocess.run(['docker', 'volume', 'rm', RESTORE_VOLUME], capture_output=True)
subprocess.run(
[
'docker', 'run', '-d', '--name', RESTORE_NAME,
'-e', 'POSTGRES_USER=messenger',
'-e', 'POSTGRES_PASSWORD=' + password,
'-e', 'POSTGRES_DB=messenger',
'-p', '127.0.0.1:5433:5432',
'-v', RESTORE_VOLUME + ':/var/lib/postgresql/data',
image,
],
check=True,
capture_output=True,
)
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
probe = subprocess.run(
['docker', 'exec', RESTORE_NAME, 'pg_isready', '-U', 'messenger', '-d', 'messenger'],
capture_output=True,
)
if probe.returncode == 0:
break
time.sleep(1)
else:
raise SystemExit('restore postgres did not become ready')
subprocess.run(
['docker', 'cp', str(dump_path), RESTORE_NAME + ':/tmp/bench.dump'],
check=True,
)
started = time.perf_counter()
subprocess.run(
[
'docker', 'exec', RESTORE_NAME,
'pg_restore', '-U', 'messenger', '-d', 'messenger',
'--no-owner', '--exit-on-error', '/tmp/bench.dump',
],
check=True,
)
return round(time.perf_counter() - started, 3)
def cleanup_restore():
subprocess.run(['docker', 'rm', '-f', RESTORE_NAME], capture_output=True)
subprocess.run(['docker', 'volume', 'rm', RESTORE_VOLUME], capture_output=True)
def drop_bench(admin_url):
with psycopg.connect(admin_url, autocommit=True) as conn:
conn.execute(
"""
SELECT pg_terminate_backend(pid)
FROM pg_stat_activity
WHERE datname = 'messenger_bench' AND pid <> pg_backend_pid()
"""
)
conn.execute('DROP DATABASE IF EXISTS messenger_bench')
def main():
env = load_env()
password = env['POSTGRES_PASSWORD']
admin_url = 'postgresql://messenger:%s@127.0.0.1:5432/messenger' % password
bench_url = 'postgresql://messenger:%s@127.0.0.1:5432/messenger_bench' % password
restored_url = 'postgresql://messenger:%s@127.0.0.1:5433/messenger' % password
EVIDENCE.mkdir(exist_ok=True)
try:
print('reset bench', flush=True)
reset_bench(admin_url)
print('migrate bench', flush=True)
applied = db.apply_migrations(bench_url)
print('applied %s' % applied, flush=True)
print('seed', flush=True)
started = time.perf_counter()
seeded = seed(bench_url)
seed_seconds = round(time.perf_counter() - started, 3)
print('seeded %s messages in %ss' % (seeded['message_count'], seed_seconds), flush=True)
print('explain', flush=True)
explained = plans(bench_url, seeded)
(EVIDENCE / 'explain-keyset.txt').write_text(explained['keyset'] + '\n', encoding='utf-8')
(EVIDENCE / 'explain-offset.txt').write_text(explained['offset'] + '\n', encoding='utf-8')
(EVIDENCE / 'explain-seqscan.txt').write_text(explained['seqscan'] + '\n', encoding='utf-8')
(EVIDENCE / 'explain-created-only.txt').write_text(explained['created_only'] + '\n', encoding='utf-8')
print('write cost', flush=True)
cost = write_cost(bench_url, seeded)
source_counts = counts(bench_url)
print('dump', flush=True)
dump_path, dump_seconds, dump_bytes = dump_bench(password)
image = image_of_source()
print('restore into %s' % image, flush=True)
restore_seconds = restore_into_new_volume(dump_path, password, image)
restored_counts = counts(restored_url)
compared = {}
for name, sample in seeded['samples'].items():
got = fetch_sample(restored_url, sample['id'])
compared[name] = got == sample
if got != sample:
print('mismatch', name, file=sys.stderr)
print('expected', sample, file=sys.stderr)
print('got', got, file=sys.stderr)
with connect(restored_url) as conn:
conn.execute('ANALYZE messages')
conn.commit()
mark = seeded['samples']['huge_middle']
created = datetime.strptime(mark['created_at'], '%Y-%m-%dT%H:%M:%S.%fZ').replace(tzinfo=timezone.utc)
restored_plan = explain(conn, KEYSET, (
uuid.UUID(mark['chat_id']), created, uuid.UUID(mark['id']),
))
(EVIDENCE / 'explain-restored.txt').write_text(restored_plan + '\n', encoding='utf-8')
indexes = []
with connect(restored_url) as conn:
indexes = [row[0] for row in conn.execute(
"SELECT indexname FROM pg_indexes WHERE tablename = 'messages' ORDER BY 1"
).fetchall()]
report = {
'platform': platform.platform(),
'processor': platform.processor(),
'python': platform.python_version(),
'seed_seconds': seed_seconds,
'chat_sizes_head': seeded['sizes'][:8],
'chat_count': seeded['chat_count'],
'smallest_chat': min(seeded['sizes']),
'largest_chat': max(seeded['sizes']),
'source': {key: source_counts[key] for key in (
'users', 'chats', 'chat_members', 'messages', 'version',
'synchronous_commit', 'fsync', 'shared_buffers',
)},
'restored': {key: restored_counts[key] for key in (
'users', 'chats', 'chat_members', 'messages',
)},
'counts_match': (
source_counts['users'] == restored_counts['users']
and source_counts['chats'] == restored_counts['chats']
and source_counts['chat_members'] == restored_counts['chat_members']
and source_counts['messages'] == restored_counts['messages']
),
'samples_match': compared,
'dump_seconds': dump_seconds,
'dump_bytes': dump_bytes,
'restore_seconds': restore_seconds,
'create_index_ms': explained['create_index_ms'],
'write_cost': cost,
'keyset': plan_brief(explained['keyset']),
'offset': plan_brief(explained['offset']),
'seqscan': plan_brief(explained['seqscan']),
'created_only': plan_brief(explained['created_only']),
'restored_keyset': plan_brief(restored_plan),
'restored_message_indexes': indexes,
'dump_file': 'backups/messenger_bench.dump',
}
(EVIDENCE / 'track2.json').write_text(
json.dumps(report, ensure_ascii=False, indent=2) + '\n',
encoding='utf-8',
)
print(json.dumps(report, ensure_ascii=False, indent=2))
if not report['counts_match'] or not all(compared.values()):
raise SystemExit('restore comparison failed')
print('PASS track2')
finally:
cleanup_restore()
try:
drop_bench(admin_url)
except Exception as exc:
print('bench cleanup: %s' % type(exc).__name__, flush=True)
if __name__ == '__main__':
main()