Executable
+40
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
# Каждое восстановление использует новый project/volume, без drop основной БД.
|
||||
project="lab2-restore-$(date +%s)-$$"
|
||||
mkdir -p backups evidence
|
||||
start=$(date +%s%N)
|
||||
docker compose exec -T db pg_dump -U messenger -d messenger -Fc > backups/lab2.dump
|
||||
backed=$(date +%s%N)
|
||||
docker compose -p "$project" -f compose.yaml -f compose.restore.yaml up -d --wait restored
|
||||
up=$(date +%s%N)
|
||||
docker compose -p "$project" -f compose.yaml -f compose.restore.yaml exec -T restored \
|
||||
pg_restore -U messenger -d restored --no-owner --exit-on-error < backups/lab2.dump
|
||||
restored=$(date +%s%N)
|
||||
# Вычисляем URL из существующего secret, не выводя пароль в команду/журнал.
|
||||
docker compose -p "$project" -f compose.yaml -f compose.restore.yaml run --build --rm -T --no-deps --user "$(id -u):$(id -g)" \
|
||||
-v "$PWD/evidence:/app/evidence" api python -c '
|
||||
from pathlib import Path
|
||||
from psycopg.conninfo import make_conninfo
|
||||
from src.config import DATABASE_URL
|
||||
import os, runpy
|
||||
os.environ.pop("DATABASE_URL_FILE", None)
|
||||
os.environ["DATABASE_URL"] = make_conninfo(DATABASE_URL, host="restored", dbname="restored")
|
||||
# config уже импортирован, обновляем значение для сценария сверки.
|
||||
import src.config
|
||||
src.config.DATABASE_URL = os.environ["DATABASE_URL"]
|
||||
runpy.run_module("scripts.verify_restore", run_name="__main__")'
|
||||
python3 - "$start" "$backed" "$up" "$restored" "$project" <<'PY'
|
||||
import sys, json
|
||||
from pathlib import Path
|
||||
start, backed, up, restored = map(int, sys.argv[1:5])
|
||||
result = {'backup_seconds': (backed-start)/1e9, 'new_server_start_seconds': (up-backed)/1e9,
|
||||
'restore_seconds': (restored-up)/1e9, 'restore_project': sys.argv[5],
|
||||
'backup_bytes': Path('backups/lab2.dump').stat().st_size,
|
||||
'stand': 'Docker Compose, PostgreSQL 16.9'}
|
||||
Path('evidence/backup_restore.json').write_text(json.dumps(result, indent=2))
|
||||
PY
|
||||
# Сервер останавливаем, volume остаётся доступным для проверки преподавателем.
|
||||
docker compose -p "$project" -f compose.yaml -f compose.restore.yaml stop restored
|
||||
printf 'PASS: restored into separate PostgreSQL and volume; project=%s\n' "$project"
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Трек 2: два индекса и OFFSET/keyset на 100 000 сообщений.
|
||||
|
||||
Только синтетические данные в отдельной схеме. Не меняет таблицы API.
|
||||
EXPLAIN включает BUFFERS, планировщику не навязывается тип плана.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import random
|
||||
import statistics
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
from psycopg import sql
|
||||
from psycopg.rows import dict_row
|
||||
from src.config import DATABASE_URL
|
||||
|
||||
SEED = 42
|
||||
SIZES = [70000, 20000, 9000, 900, 100]
|
||||
REPEATS = 30
|
||||
|
||||
|
||||
def percentile(values, p):
|
||||
return sorted(values)[max(0, int(len(values) * p + 0.9999) - 1)]
|
||||
|
||||
|
||||
def measure(db, query, params):
|
||||
for _ in range(5):
|
||||
db.execute(query, params).fetchall()
|
||||
raw = []
|
||||
for _ in range(REPEATS):
|
||||
start = time.perf_counter()
|
||||
db.execute(query, params).fetchall()
|
||||
raw.append((time.perf_counter() - start) * 1000)
|
||||
plan = db.execute('EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ' + query, params).fetchone()[0][0]
|
||||
return {'latency_ms': raw, 'p50_ms': statistics.median(raw),
|
||||
'p95_ms': percentile(raw, .95), 'p99_ms': percentile(raw, .99), 'plan': plan}
|
||||
|
||||
|
||||
def main():
|
||||
schema = 'research_' + uuid4().hex[:12]
|
||||
rng = random.Random(SEED)
|
||||
out = {'seed': SEED, 'schema': schema, 'message_count': sum(SIZES), 'sizes': SIZES,
|
||||
'repeats': REPEATS, 'warmup': 5, 'concurrency': 1,
|
||||
'platform': platform.platform(), 'cpu_count': os.cpu_count(), 'results': []}
|
||||
Path('evidence').mkdir(exist_ok=True)
|
||||
with psycopg.connect(DATABASE_URL, autocommit=True) as db:
|
||||
out['postgres_version'] = db.execute('SHOW server_version').fetchone()[0]
|
||||
out['settings'] = {k: db.execute('SHOW ' + k).fetchone()[0] for k in
|
||||
['shared_buffers', 'work_mem', 'fsync', 'synchronous_commit']}
|
||||
db.execute(sql.SQL('CREATE SCHEMA {}').format(sql.Identifier(schema)))
|
||||
db.execute(sql.SQL('SET search_path TO {}').format(sql.Identifier(schema)))
|
||||
# Точная схема приложения; другой search_path изолирует эксперимент.
|
||||
db.execute(Path('migrations/001_initial.sql').read_text())
|
||||
user = uuid4()
|
||||
db.execute('INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s)',
|
||||
(user, 'research_user', 'Synthetic research', 'not-a-login-account'))
|
||||
chats = [uuid4() for _ in SIZES]
|
||||
start_date = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
||||
for chat, size in zip(chats, SIZES):
|
||||
db.execute('INSERT INTO chats(id,title) VALUES (%s,%s)', (chat, 'Synthetic ' + str(size)))
|
||||
db.execute('INSERT INTO chat_members VALUES (%s,%s)', (chat, user))
|
||||
with db.cursor().copy('COPY messages(id,chat_id,sender_id,text,created_at) FROM STDIN') as copy:
|
||||
for n in range(size):
|
||||
copy.write_row((uuid4(), chat, user, 'message ' + str(n) + ' x' * (n % 80),
|
||||
start_date + timedelta(microseconds=n)))
|
||||
samples = []
|
||||
for chat in chats:
|
||||
for fraction in [0, .5, .9]:
|
||||
count = db.execute('SELECT count(*) FROM messages WHERE chat_id=%s', (chat,)).fetchone()[0]
|
||||
offset = int((count - 1) * fraction)
|
||||
cursor = db.execute('SELECT created_at,id FROM messages WHERE chat_id=%s '
|
||||
'ORDER BY created_at,id OFFSET %s LIMIT 1', (chat, offset)).fetchone()
|
||||
samples.append((chat, offset, cursor))
|
||||
db.execute('DROP INDEX messages_history_idx')
|
||||
# Порядок фиксирован, но обе конфигурации прогреваются и имеют 30 повторов.
|
||||
for name, columns in [('chat_only', 'chat_id'), ('history', 'chat_id,created_at,id')]:
|
||||
db.execute('CREATE INDEX research_candidate ON messages(' + columns + ')')
|
||||
db.execute('ANALYZE messages')
|
||||
index_bytes = db.execute("SELECT pg_relation_size('research_candidate')").fetchone()[0]
|
||||
queries = list(samples)
|
||||
rng.shuffle(queries)
|
||||
for chat, offset, cursor in queries:
|
||||
for method in ['offset', 'keyset']:
|
||||
query = 'SELECT * FROM messages WHERE chat_id=%s'
|
||||
if method == 'keyset':
|
||||
query += ' AND (created_at,id)>(%s,%s) ORDER BY created_at,id LIMIT 51'
|
||||
params = (chat, *cursor)
|
||||
else:
|
||||
# Cursor указывает последнюю уже прочитанную строку.
|
||||
query += ' ORDER BY created_at,id OFFSET %s LIMIT 51'
|
||||
params = (chat, offset + 1)
|
||||
result = measure(db, query, params)
|
||||
result.update(index=name, method=method, chat_size=SIZES[chats.index(chat)],
|
||||
offset=offset, index_bytes=index_bytes)
|
||||
out['results'].append(result)
|
||||
write_ms = []
|
||||
for batch in range(5):
|
||||
# Цена 1000 записей + COMMIT; затем откат данных отдельной очисткой.
|
||||
# Исходная история остаётся 100 000 для следующего индекса.
|
||||
batch_ids = [uuid4() for _ in range(1000)]
|
||||
begin = time.perf_counter()
|
||||
with db.transaction():
|
||||
with db.cursor().copy('COPY messages(id,chat_id,sender_id,text,created_at) FROM STDIN') as copy:
|
||||
for n, mid in enumerate(batch_ids):
|
||||
copy.write_row((mid, chats[n % len(chats)], user, 'write-cost ' + 'x' * 128,
|
||||
start_date + timedelta(days=1, microseconds=n)))
|
||||
write_ms.append((time.perf_counter() - begin) * 1000)
|
||||
db.execute('DELETE FROM messages WHERE id=ANY(%s)', (batch_ids,))
|
||||
out.setdefault('writes', {})[name] = {'batch_rows': 1000, 'batch_ms': write_ms,
|
||||
'median_ms': statistics.median(write_ms),
|
||||
'index_bytes': index_bytes}
|
||||
db.execute('DROP INDEX research_candidate')
|
||||
db.execute('VACUUM ANALYZE messages')
|
||||
db.execute('CREATE INDEX messages_history_idx ON messages(chat_id,created_at,id)')
|
||||
with db.cursor(row_factory=dict_row) as cur:
|
||||
out['counts'] = {}
|
||||
for table in ['users', 'chats', 'chat_members', 'messages', 'sessions']:
|
||||
out['counts'][table] = db.execute('SELECT count(*) FROM ' + table).fetchone()[0]
|
||||
cur.execute('SELECT * FROM messages ORDER BY created_at,id LIMIT 5')
|
||||
out['sample_messages'] = [dict(row) for row in cur.fetchall()]
|
||||
Path('evidence/track2_raw.json').write_text(json.dumps(out, ensure_ascii=False, indent=2, default=str))
|
||||
lines = ['# Трек 2: измерения', '', 'Исходных сообщений: 100000. Прогрев: 5; повторов: 30.', '',
|
||||
'| Чат | Позиция | Индекс | Запрос | p50, мс | p95, мс |',
|
||||
'|---:|---:|---|---|---:|---:|']
|
||||
for row in sorted(out['results'], key=lambda r: (r['chat_size'], r['offset'], r['index'], r['method'])):
|
||||
lines.append(f"| {row['chat_size']} | {row['offset']} | {row['index']} | {row['method']} | "
|
||||
f"{row['p50_ms']:.3f} | {row['p95_ms']:.3f} |")
|
||||
lines.extend(['', 'Цена записи: 5 транзакций по 1000 строк (COPY + COMMIT).', '',
|
||||
'| Индекс | Медиана партии, мс | Размер, байт |', '|---|---:|---:|'])
|
||||
for name, row in out['writes'].items():
|
||||
lines.append(f"| {name} | {row['median_ms']:.3f} | {row['index_bytes']} |")
|
||||
Path('evidence/track2_summary.md').write_text('\n'.join(lines) + '\n')
|
||||
print('PASS: measured two indexes, two queries, write cost; raw evidence saved')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import os
|
||||
import urllib.request
|
||||
|
||||
|
||||
host = "127.0.0.1"
|
||||
port = os.getenv("APP_PORT", "8080")
|
||||
with urllib.request.urlopen(f"http://{host}:{port}/health/ready", timeout=2) as response:
|
||||
payload = json.load(response)
|
||||
if response.status != 200 or payload != {"status": "ok"}:
|
||||
raise SystemExit(1)
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
# Остановка/запуск всех сервисов; именованные volumes остаются.
|
||||
# API запускаем после готовности хранилищ, иначе миграции могут не подключиться.
|
||||
docker compose stop api cache db
|
||||
docker compose up -d --wait db cache
|
||||
docker compose up -d --wait api
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
mkdir -p .secrets
|
||||
chmod 700 .secrets
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
password_file = Path('.secrets/db_password')
|
||||
if not password_file.exists():
|
||||
password_file.write_text(secrets.token_hex(24))
|
||||
password = password_file.read_text().strip()
|
||||
Path('.secrets/database_url').write_text(f'postgresql://messenger:{password}@db:5432/messenger')
|
||||
for file in Path('.secrets').iterdir():
|
||||
file.chmod(0o644 if file.name == "database_url" else 0o600)
|
||||
if not Path('.env').exists():
|
||||
Path('.env').write_text('APP_PORT=8080\nSESSION_TTL_SECONDS=3600\nDB_POOL_MAX=8\n')
|
||||
print('Локальные настройки подготовлены; секреты не выводятся.')
|
||||
PY
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose start cache
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose start db
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop cache
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop db
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
# Меняем только конфигурацию процесса; сессии и БД остаются на месте.
|
||||
restore() {
|
||||
docker compose up -d --no-deps --force-recreate api
|
||||
}
|
||||
trap restore EXIT
|
||||
SESSION_TTL_SECONDS=2 docker compose up -d --no-deps --force-recreate api
|
||||
python3 scripts/wait_ready.py
|
||||
python3 student-tests/checks.py --expiry
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Сверка восстановленной БД с counts и пятью полными строками из эксперимента."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import psycopg
|
||||
from psycopg import sql
|
||||
from psycopg.rows import dict_row
|
||||
from src.config import DATABASE_URL
|
||||
|
||||
expected = json.loads(Path('evidence/track2_raw.json').read_text())
|
||||
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as db:
|
||||
db.execute(sql.SQL('SET search_path TO {}').format(sql.Identifier(expected['schema'])))
|
||||
counts = {}
|
||||
for table, count in expected['counts'].items():
|
||||
actual = db.execute(sql.SQL('SELECT count(*) AS n FROM {}').format(sql.Identifier(table))).fetchone()['n']
|
||||
assert actual == count, (table, actual, count)
|
||||
counts[table] = actual
|
||||
actual_samples = []
|
||||
for sample in expected['sample_messages']:
|
||||
row = db.execute('SELECT * FROM messages WHERE id=%s', (sample['id'],)).fetchone()
|
||||
actual = json.loads(json.dumps(row, default=str))
|
||||
assert actual == sample, 'Restored message differs'
|
||||
actual_samples.append(actual)
|
||||
result = {'counts': counts, 'sample_messages_matched': len(actual_samples), 'status': 'PASS'}
|
||||
Path('evidence/restore_verification.json').write_text(json.dumps(result, indent=2))
|
||||
print('PASS: restored row counts and selected full messages match backup dataset')
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Ожидание готовности API без сторонних библиотек."""
|
||||
import os
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
url = os.getenv('BASE_URL', 'http://localhost:8080') + '/health/ready'
|
||||
for attempt in range(90):
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=2) as response:
|
||||
if response.status == 200:
|
||||
print('API ready')
|
||||
break
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise SystemExit('API не стал готов за 90 попыток')
|
||||
Reference in New Issue
Block a user