This commit is contained in:
2026-10-02 22:04:22 +03:00
parent 65003b6119
commit dee2dacdd3
49 changed files with 10067 additions and 24 deletions
+40
View File
@@ -0,0 +1,40 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
# Каждое восстановление использует новый project/volume, без drop основной БД.
project="lab2-restore-$(date +%s)-$$"
mkdir -p backups evidence
start=$(date +%s%N)
docker compose exec -T db pg_dump -U messenger -d messenger -Fc > backups/lab2.dump
backed=$(date +%s%N)
docker compose -p "$project" -f compose.yaml -f compose.restore.yaml up -d --wait restored
up=$(date +%s%N)
docker compose -p "$project" -f compose.yaml -f compose.restore.yaml exec -T restored \
pg_restore -U messenger -d restored --no-owner --exit-on-error < backups/lab2.dump
restored=$(date +%s%N)
# Вычисляем URL из существующего secret, не выводя пароль в команду/журнал.
docker compose -p "$project" -f compose.yaml -f compose.restore.yaml run --build --rm -T --no-deps --user "$(id -u):$(id -g)" \
-v "$PWD/evidence:/app/evidence" api python -c '
from pathlib import Path
from psycopg.conninfo import make_conninfo
from src.config import DATABASE_URL
import os, runpy
os.environ.pop("DATABASE_URL_FILE", None)
os.environ["DATABASE_URL"] = make_conninfo(DATABASE_URL, host="restored", dbname="restored")
# config уже импортирован, обновляем значение для сценария сверки.
import src.config
src.config.DATABASE_URL = os.environ["DATABASE_URL"]
runpy.run_module("scripts.verify_restore", run_name="__main__")'
python3 - "$start" "$backed" "$up" "$restored" "$project" <<'PY'
import sys, json
from pathlib import Path
start, backed, up, restored = map(int, sys.argv[1:5])
result = {'backup_seconds': (backed-start)/1e9, 'new_server_start_seconds': (up-backed)/1e9,
'restore_seconds': (restored-up)/1e9, 'restore_project': sys.argv[5],
'backup_bytes': Path('backups/lab2.dump').stat().st_size,
'stand': 'Docker Compose, PostgreSQL 16.9'}
Path('evidence/backup_restore.json').write_text(json.dumps(result, indent=2))
PY
# Сервер останавливаем, volume остаётся доступным для проверки преподавателем.
docker compose -p "$project" -f compose.yaml -f compose.restore.yaml stop restored
printf 'PASS: restored into separate PostgreSQL and volume; project=%s\n' "$project"
+141
View File
@@ -0,0 +1,141 @@
"""Трек 2: два индекса и OFFSET/keyset на 100 000 сообщений.
Только синтетические данные в отдельной схеме. Не меняет таблицы API.
EXPLAIN включает BUFFERS, планировщику не навязывается тип плана.
"""
import json
import os
import platform
import random
import statistics
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path
from uuid import uuid4
import psycopg
from psycopg import sql
from psycopg.rows import dict_row
from src.config import DATABASE_URL
SEED = 42
SIZES = [70000, 20000, 9000, 900, 100]
REPEATS = 30
def percentile(values, p):
return sorted(values)[max(0, int(len(values) * p + 0.9999) - 1)]
def measure(db, query, params):
for _ in range(5):
db.execute(query, params).fetchall()
raw = []
for _ in range(REPEATS):
start = time.perf_counter()
db.execute(query, params).fetchall()
raw.append((time.perf_counter() - start) * 1000)
plan = db.execute('EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ' + query, params).fetchone()[0][0]
return {'latency_ms': raw, 'p50_ms': statistics.median(raw),
'p95_ms': percentile(raw, .95), 'p99_ms': percentile(raw, .99), 'plan': plan}
def main():
schema = 'research_' + uuid4().hex[:12]
rng = random.Random(SEED)
out = {'seed': SEED, 'schema': schema, 'message_count': sum(SIZES), 'sizes': SIZES,
'repeats': REPEATS, 'warmup': 5, 'concurrency': 1,
'platform': platform.platform(), 'cpu_count': os.cpu_count(), 'results': []}
Path('evidence').mkdir(exist_ok=True)
with psycopg.connect(DATABASE_URL, autocommit=True) as db:
out['postgres_version'] = db.execute('SHOW server_version').fetchone()[0]
out['settings'] = {k: db.execute('SHOW ' + k).fetchone()[0] for k in
['shared_buffers', 'work_mem', 'fsync', 'synchronous_commit']}
db.execute(sql.SQL('CREATE SCHEMA {}').format(sql.Identifier(schema)))
db.execute(sql.SQL('SET search_path TO {}').format(sql.Identifier(schema)))
# Точная схема приложения; другой search_path изолирует эксперимент.
db.execute(Path('migrations/001_initial.sql').read_text())
user = uuid4()
db.execute('INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s)',
(user, 'research_user', 'Synthetic research', 'not-a-login-account'))
chats = [uuid4() for _ in SIZES]
start_date = datetime(2026, 1, 1, tzinfo=timezone.utc)
for chat, size in zip(chats, SIZES):
db.execute('INSERT INTO chats(id,title) VALUES (%s,%s)', (chat, 'Synthetic ' + str(size)))
db.execute('INSERT INTO chat_members VALUES (%s,%s)', (chat, user))
with db.cursor().copy('COPY messages(id,chat_id,sender_id,text,created_at) FROM STDIN') as copy:
for n in range(size):
copy.write_row((uuid4(), chat, user, 'message ' + str(n) + ' x' * (n % 80),
start_date + timedelta(microseconds=n)))
samples = []
for chat in chats:
for fraction in [0, .5, .9]:
count = db.execute('SELECT count(*) FROM messages WHERE chat_id=%s', (chat,)).fetchone()[0]
offset = int((count - 1) * fraction)
cursor = db.execute('SELECT created_at,id FROM messages WHERE chat_id=%s '
'ORDER BY created_at,id OFFSET %s LIMIT 1', (chat, offset)).fetchone()
samples.append((chat, offset, cursor))
db.execute('DROP INDEX messages_history_idx')
# Порядок фиксирован, но обе конфигурации прогреваются и имеют 30 повторов.
for name, columns in [('chat_only', 'chat_id'), ('history', 'chat_id,created_at,id')]:
db.execute('CREATE INDEX research_candidate ON messages(' + columns + ')')
db.execute('ANALYZE messages')
index_bytes = db.execute("SELECT pg_relation_size('research_candidate')").fetchone()[0]
queries = list(samples)
rng.shuffle(queries)
for chat, offset, cursor in queries:
for method in ['offset', 'keyset']:
query = 'SELECT * FROM messages WHERE chat_id=%s'
if method == 'keyset':
query += ' AND (created_at,id)>(%s,%s) ORDER BY created_at,id LIMIT 51'
params = (chat, *cursor)
else:
# Cursor указывает последнюю уже прочитанную строку.
query += ' ORDER BY created_at,id OFFSET %s LIMIT 51'
params = (chat, offset + 1)
result = measure(db, query, params)
result.update(index=name, method=method, chat_size=SIZES[chats.index(chat)],
offset=offset, index_bytes=index_bytes)
out['results'].append(result)
write_ms = []
for batch in range(5):
# Цена 1000 записей + COMMIT; затем откат данных отдельной очисткой.
# Исходная история остаётся 100 000 для следующего индекса.
batch_ids = [uuid4() for _ in range(1000)]
begin = time.perf_counter()
with db.transaction():
with db.cursor().copy('COPY messages(id,chat_id,sender_id,text,created_at) FROM STDIN') as copy:
for n, mid in enumerate(batch_ids):
copy.write_row((mid, chats[n % len(chats)], user, 'write-cost ' + 'x' * 128,
start_date + timedelta(days=1, microseconds=n)))
write_ms.append((time.perf_counter() - begin) * 1000)
db.execute('DELETE FROM messages WHERE id=ANY(%s)', (batch_ids,))
out.setdefault('writes', {})[name] = {'batch_rows': 1000, 'batch_ms': write_ms,
'median_ms': statistics.median(write_ms),
'index_bytes': index_bytes}
db.execute('DROP INDEX research_candidate')
db.execute('VACUUM ANALYZE messages')
db.execute('CREATE INDEX messages_history_idx ON messages(chat_id,created_at,id)')
with db.cursor(row_factory=dict_row) as cur:
out['counts'] = {}
for table in ['users', 'chats', 'chat_members', 'messages', 'sessions']:
out['counts'][table] = db.execute('SELECT count(*) FROM ' + table).fetchone()[0]
cur.execute('SELECT * FROM messages ORDER BY created_at,id LIMIT 5')
out['sample_messages'] = [dict(row) for row in cur.fetchall()]
Path('evidence/track2_raw.json').write_text(json.dumps(out, ensure_ascii=False, indent=2, default=str))
lines = ['# Трек 2: измерения', '', 'Исходных сообщений: 100000. Прогрев: 5; повторов: 30.', '',
'| Чат | Позиция | Индекс | Запрос | p50, мс | p95, мс |',
'|---:|---:|---|---|---:|---:|']
for row in sorted(out['results'], key=lambda r: (r['chat_size'], r['offset'], r['index'], r['method'])):
lines.append(f"| {row['chat_size']} | {row['offset']} | {row['index']} | {row['method']} | "
f"{row['p50_ms']:.3f} | {row['p95_ms']:.3f} |")
lines.extend(['', 'Цена записи: 5 транзакций по 1000 строк (COPY + COMMIT).', '',
'| Индекс | Медиана партии, мс | Размер, байт |', '|---|---:|---:|'])
for name, row in out['writes'].items():
lines.append(f"| {name} | {row['median_ms']:.3f} | {row['index_bytes']} |")
Path('evidence/track2_summary.md').write_text('\n'.join(lines) + '\n')
print('PASS: measured two indexes, two queries, write cost; raw evidence saved')
if __name__ == '__main__':
main()
+12
View File
@@ -0,0 +1,12 @@
#!/usr/bin/env python3
import json
import os
import urllib.request
host = "127.0.0.1"
port = os.getenv("APP_PORT", "8080")
with urllib.request.urlopen(f"http://{host}:{port}/health/ready", timeout=2) as response:
payload = json.load(response)
if response.status != 200 or payload != {"status": "ok"}:
raise SystemExit(1)
+8
View File
@@ -0,0 +1,8 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
# Остановка/запуск всех сервисов; именованные volumes остаются.
# API запускаем после готовности хранилищ, иначе миграции могут не подключиться.
docker compose stop api cache db
docker compose up -d --wait db cache
docker compose up -d --wait api
+19
View File
@@ -0,0 +1,19 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
mkdir -p .secrets
chmod 700 .secrets
python3 - <<'PY'
from pathlib import Path
import secrets
password_file = Path('.secrets/db_password')
if not password_file.exists():
password_file.write_text(secrets.token_hex(24))
password = password_file.read_text().strip()
Path('.secrets/database_url').write_text(f'postgresql://messenger:{password}@db:5432/messenger')
for file in Path('.secrets').iterdir():
file.chmod(0o644 if file.name == "database_url" else 0o600)
if not Path('.env').exists():
Path('.env').write_text('APP_PORT=8080\nSESSION_TTL_SECONDS=3600\nDB_POOL_MAX=8\n')
print('Локальные настройки подготовлены; секреты не выводятся.')
PY
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
docker compose start cache
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
docker compose start db
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
docker compose stop cache
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
docker compose stop db
+11
View File
@@ -0,0 +1,11 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
# Меняем только конфигурацию процесса; сессии и БД остаются на месте.
restore() {
docker compose up -d --no-deps --force-recreate api
}
trap restore EXIT
SESSION_TTL_SECONDS=2 docker compose up -d --no-deps --force-recreate api
python3 scripts/wait_ready.py
python3 student-tests/checks.py --expiry
+25
View File
@@ -0,0 +1,25 @@
"""Сверка восстановленной БД с counts и пятью полными строками из эксперимента."""
import json
from pathlib import Path
import psycopg
from psycopg import sql
from psycopg.rows import dict_row
from src.config import DATABASE_URL
expected = json.loads(Path('evidence/track2_raw.json').read_text())
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as db:
db.execute(sql.SQL('SET search_path TO {}').format(sql.Identifier(expected['schema'])))
counts = {}
for table, count in expected['counts'].items():
actual = db.execute(sql.SQL('SELECT count(*) AS n FROM {}').format(sql.Identifier(table))).fetchone()['n']
assert actual == count, (table, actual, count)
counts[table] = actual
actual_samples = []
for sample in expected['sample_messages']:
row = db.execute('SELECT * FROM messages WHERE id=%s', (sample['id'],)).fetchone()
actual = json.loads(json.dumps(row, default=str))
assert actual == sample, 'Restored message differs'
actual_samples.append(actual)
result = {'counts': counts, 'sample_messages_matched': len(actual_samples), 'status': 'PASS'}
Path('evidence/restore_verification.json').write_text(json.dumps(result, indent=2))
print('PASS: restored row counts and selected full messages match backup dataset')
+17
View File
@@ -0,0 +1,17 @@
"""Ожидание готовности API без сторонних библиотек."""
import os
import time
import urllib.request
url = os.getenv('BASE_URL', 'http://localhost:8080') + '/health/ready'
for attempt in range(90):
try:
with urllib.request.urlopen(url, timeout=2) as response:
if response.status == 200:
print('API ready')
break
except OSError:
pass
time.sleep(1)
else:
raise SystemExit('API не стал готов за 90 попыток')