forked from mtusi2026/mtusi2026lab4
ЛР4 - БВТ2302 - Никитина - на 5
This commit is contained in:
@@ -0,0 +1,41 @@
|
||||
-- Создаем тестового пользователя
|
||||
INSERT INTO users (id, username, display_name, password_hash)
|
||||
VALUES ('00000000-0000-0000-0000-000000000001', 'load_user', 'Load User', 'fake_hash')
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
-- Создаем большой чат (50 000 сообщений)
|
||||
INSERT INTO chats (id, title) VALUES ('00000000-0000-0000-0000-000000000002', 'Big Chat');
|
||||
INSERT INTO chat_members (chat_id, user_id) VALUES ('00000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000001');
|
||||
|
||||
-- Создаем средний чат (30 000 сообщений)
|
||||
INSERT INTO chats (id, title) VALUES ('00000000-0000-0000-0000-000000000003', 'Medium Chat');
|
||||
INSERT INTO chat_members (chat_id, user_id) VALUES ('00000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000001');
|
||||
|
||||
-- Создаем маленький чат (20 000 сообщений)
|
||||
INSERT INTO chats (id, title) VALUES ('00000000-0000-0000-0000-000000000004', 'Small Chat');
|
||||
INSERT INTO chat_members (chat_id, user_id) VALUES ('00000000-0000-0000-0000-000000000004', '00000000-0000-0000-0000-000000000001');
|
||||
|
||||
-- Генерируем сообщения
|
||||
INSERT INTO messages (chat_id, sender_id, text, created_at)
|
||||
SELECT
|
||||
'00000000-0000-0000-0000-000000000002',
|
||||
'00000000-0000-0000-0000-000000000001',
|
||||
'Big chat message ' || i,
|
||||
now() - (100000 - i) * interval '1 second'
|
||||
FROM generate_series(1, 50000) AS i;
|
||||
|
||||
INSERT INTO messages (chat_id, sender_id, text, created_at)
|
||||
SELECT
|
||||
'00000000-0000-0000-0000-000000000003',
|
||||
'00000000-0000-0000-0000-000000000001',
|
||||
'Medium chat message ' || i,
|
||||
now() - (100000 - i) * interval '1 second'
|
||||
FROM generate_series(1, 30000) AS i;
|
||||
|
||||
INSERT INTO messages (chat_id, sender_id, text, created_at)
|
||||
SELECT
|
||||
'00000000-0000-0000-0000-000000000004',
|
||||
'00000000-0000-0000-0000-000000000001',
|
||||
'Small chat message ' || i,
|
||||
now() - (100000 - i) * interval '1 second'
|
||||
FROM generate_series(1, 20000) AS i;
|
||||
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import argparse
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
import psycopg
|
||||
|
||||
DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql://messenger:messenger@db:5432/messenger")
|
||||
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://minio:9000")
|
||||
S3_ACCESS_KEY = os.environ.get("S3_ACCESS_KEY", "minioadmin")
|
||||
S3_SECRET_KEY = os.environ.get("S3_SECRET_KEY", "minioadmin")
|
||||
S3_BUCKET = os.environ.get("S3_BUCKET", "attachments")
|
||||
|
||||
s3 = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=S3_ENDPOINT,
|
||||
aws_access_key_id=S3_ACCESS_KEY,
|
||||
aws_secret_access_key=S3_SECRET_KEY,
|
||||
config=Config(signature_version="s3v4"),
|
||||
region_name="us-east-1"
|
||||
)
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Reconcile S3 objects and DB metadata.")
|
||||
parser.add_argument("--fix", action="store_true", help="Delete orphan objects from S3")
|
||||
args = parser.parse_args()
|
||||
|
||||
print("Connecting to S3 and PostgreSQL...")
|
||||
|
||||
# 1. Получаем все ключи из S3
|
||||
s3_keys = set()
|
||||
paginator = s3.get_paginator("list_objects_v2")
|
||||
for page in paginator.paginate(Bucket=S3_BUCKET):
|
||||
for obj in page.get("Contents", []):
|
||||
s3_keys.add(obj["Key"])
|
||||
|
||||
# 2. Получаем все s3_key из БД
|
||||
db_keys = set()
|
||||
with psycopg.connect(DATABASE_URL) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT s3_key FROM attachments")
|
||||
for row in cur.fetchall():
|
||||
db_keys.add(row[0])
|
||||
|
||||
# 3. Сравниваем
|
||||
orphans = s3_keys - db_keys
|
||||
missing = db_keys - s3_keys
|
||||
|
||||
print(f"\n--- RECONCILIATION REPORT ---")
|
||||
print(f"Total objects in S3: {len(s3_keys)}")
|
||||
print(f"Total records in DB: {len(db_keys)}")
|
||||
print(f"Orphan objects (in S3, not in DB): {len(orphans)}")
|
||||
print(f"Missing objects (in DB, not in S3): {len(missing)}")
|
||||
|
||||
if orphans:
|
||||
print("\n[ORPHANS] (Safe to delete, no metadata in DB):")
|
||||
for key in orphans:
|
||||
print(f" - {key}")
|
||||
|
||||
if missing:
|
||||
print("\n[MISSING] (CRITICAL! Metadata exists, but file is lost):")
|
||||
for key in missing:
|
||||
print(f" - {key}")
|
||||
|
||||
if args.fix and orphans:
|
||||
print("\n[FIX] Deleting orphan objects from S3...")
|
||||
for key in orphans:
|
||||
s3.delete_object(Bucket=S3_BUCKET, Key=key)
|
||||
print(f" Deleted {key}")
|
||||
|
||||
if not args.fix and (orphans or missing):
|
||||
print("\nHint: run with `--fix` to delete orphan objects (missing objects require manual review).")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,2 @@
|
||||
@echo off
|
||||
docker compose restart
|
||||
@@ -0,0 +1,3 @@
|
||||
Write-Host "Restarting the whole stack..."
|
||||
docker compose down
|
||||
docker compose up -d --scale api=2
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
echo "Restarting the whole stack..."
|
||||
docker compose down
|
||||
docker compose up -d --scale api=2
|
||||
@@ -0,0 +1,79 @@
|
||||
param(
|
||||
[string]$Image = "mtusi-messenger:lab1",
|
||||
[string]$Container = "mtusi-messenger-lab1",
|
||||
[string]$BaseUrl = "http://127.0.0.1:8080",
|
||||
[string]$History = "1000,5000,20000",
|
||||
[string]$Concurrency = "1,4,16",
|
||||
[int]$Duration = 15,
|
||||
[int]$Warmup = 2
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Stop-App {
|
||||
$running = docker ps -q -f "name=$Container"
|
||||
if ($running) {
|
||||
Write-Host " stopping container..."
|
||||
docker kill --signal SIGTERM $Container | Out-Null
|
||||
for ($i = 0; $i -lt 60; $i++) {
|
||||
$still = docker ps -q -f "name=$Container"
|
||||
if (-not $still) { break }
|
||||
Start-Sleep -Milliseconds 250
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Test-Ready {
|
||||
$code = (& curl.exe -s -o NUL -m 2 -w "%{http_code}" "$BaseUrl/health/ready" 2>$null)
|
||||
return ("$code".Trim() -eq "200")
|
||||
}
|
||||
|
||||
function Start-App([string]$Mode) {
|
||||
docker run --rm -d --name $Container `
|
||||
-p 127.0.0.1:8080:8080 `
|
||||
-e HOST=0.0.0.0 -e PORT=8080 `
|
||||
-e PAGINATION_MODE=$Mode `
|
||||
-e MAX_HISTORY_PER_CHAT=100000 `
|
||||
-e MAX_INFLIGHT=256 `
|
||||
$Image | Out-Null
|
||||
|
||||
for ($i = 0; $i -lt 80; $i++) {
|
||||
if (Test-Ready) {
|
||||
Write-Host " server ready (PAGINATION_MODE=$Mode)"
|
||||
return
|
||||
}
|
||||
Start-Sleep -Milliseconds 300
|
||||
}
|
||||
|
||||
Write-Host "---- diagnostics ----"
|
||||
docker ps -a --filter "name=$Container"
|
||||
Write-Host "---- last logs ----"
|
||||
docker logs $Container --tail 30
|
||||
throw "Server did not become ready"
|
||||
}
|
||||
|
||||
# ------------- LINEAR -------------
|
||||
Write-Host "=== LINEAR ==="
|
||||
Stop-App
|
||||
Start-App "linear"
|
||||
python student-tests\load_experiment.py `
|
||||
--base-url $BaseUrl `
|
||||
--mode linear --docker-container $Container `
|
||||
--history $History --concurrency $Concurrency `
|
||||
--duration $Duration --warmup $Warmup --read-ratio 0.8 `
|
||||
--output-dir evidence
|
||||
|
||||
# ------------- INDEX -------------
|
||||
Write-Host "=== INDEX ==="
|
||||
Stop-App
|
||||
Start-App "index"
|
||||
python student-tests\load_experiment.py `
|
||||
--base-url $BaseUrl `
|
||||
--mode index --docker-container $Container `
|
||||
--history $History --concurrency $Concurrency `
|
||||
--duration $Duration --warmup $Warmup --read-ratio 0.8 `
|
||||
--output-dir evidence
|
||||
|
||||
Stop-App
|
||||
Write-Host ""
|
||||
Write-Host "DONE. See evidence\linear_summary.json and evidence\index_summary.json"
|
||||
@@ -0,0 +1,43 @@
|
||||
$ErrorActionPreference = "Stop"
|
||||
$EvidDir = "evidence"
|
||||
if (-not (Test-Path $EvidDir)) { New-Item -ItemType Directory -Path $EvidDir | Out-Null }
|
||||
|
||||
Write-Host "=== TRACK 2: Gathering evidence ===" -ForegroundColor Cyan
|
||||
|
||||
# 1. EXPLAIN WITH INDEX
|
||||
Write-Host "1. Read query (WITH index)..." -ForegroundColor Yellow
|
||||
$sql_with_idx = "EXPLAIN (ANALYZE, BUFFERS) SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = '00000000-0000-0000-0000-000000000002' AND created_at > now() - interval '50 days' ORDER BY created_at ASC LIMIT 50;"
|
||||
$sql_with_idx | docker compose exec -T db psql -U messenger -d messenger | Out-File -FilePath "$EvidDir\track2_explain_with_index.txt" -Encoding utf8
|
||||
|
||||
# 2. EXPLAIN WITHOUT INDEX
|
||||
Write-Host "2. Read query (WITHOUT index)..." -ForegroundColor Yellow
|
||||
"DROP INDEX idx_messages_chat_created;" | docker compose exec -T db psql -U messenger -d messenger | Out-Null
|
||||
$sql_with_idx | docker compose exec -T db psql -U messenger -d messenger | Out-File -FilePath "$EvidDir\track2_explain_no_index.txt" -Encoding utf8
|
||||
"CREATE INDEX idx_messages_chat_created ON messages (chat_id, created_at, id);" | docker compose exec -T db psql -U messenger -d messenger | Out-Null
|
||||
|
||||
# 3. WRITE COST
|
||||
Write-Host "3. Write cost analysis..." -ForegroundColor Yellow
|
||||
$insert_script = @"
|
||||
DROP INDEX idx_messages_chat_created;
|
||||
EXPLAIN ANALYZE INSERT INTO messages (chat_id, sender_id, text) SELECT '00000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000001', 'test ' || i FROM generate_series(1, 1000) i;
|
||||
CREATE INDEX idx_messages_chat_created ON messages (chat_id, created_at, id);
|
||||
EXPLAIN ANALYZE INSERT INTO messages (chat_id, sender_id, text) SELECT '00000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000001', 'test2 ' || i FROM generate_series(1, 1000) i;
|
||||
"@
|
||||
$insert_script | docker compose exec -T db psql -U messenger -d messenger | Out-File -FilePath "$EvidDir\track2_write_cost.txt" -Encoding utf8
|
||||
|
||||
# 4. BACKUP
|
||||
Write-Host "4. Creating backup..." -ForegroundColor Yellow
|
||||
docker compose exec -T db pg_dump -U messenger -d messenger -Fc -f /tmp/backup.dump
|
||||
docker cp mtusi2026lab2-db-1:/tmp/backup.dump ./backup.dump | Out-Null
|
||||
|
||||
# 5. RESTORE
|
||||
Write-Host "5. Restoring from backup..." -ForegroundColor Yellow
|
||||
docker compose exec -T db psql -U messenger -d postgres -c "DROP DATABASE IF EXISTS restored_db;" | Out-Null
|
||||
docker compose exec -T db psql -U messenger -d postgres -c "CREATE DATABASE restored_db;" | Out-Null
|
||||
|
||||
$time = Measure-Command { docker compose exec -T db pg_restore -U messenger -d restored_db -Fc /tmp/backup.dump }
|
||||
"Restore time: $($time.TotalSeconds) seconds" | Out-File -FilePath "$EvidDir\track2_restore.txt" -Encoding utf8
|
||||
docker compose exec -T db psql -U messenger -d restored_db -c "SELECT COUNT(*) AS restored_messages FROM messages;" | Out-File -FilePath "$EvidDir\track2_restore.txt" -Encoding utf8 -Append
|
||||
|
||||
Write-Host "=== DONE! All results saved to evidence/ ===" -ForegroundColor Green
|
||||
Get-ChildItem $EvidDir
|
||||
@@ -0,0 +1,2 @@
|
||||
@echo off
|
||||
powershell -ExecutionPolicy Bypass -File scripts/stop-one.ps1
|
||||
@@ -0,0 +1,8 @@
|
||||
$containers = docker ps -q -f "name=mtusi2026lab4-api-*"
|
||||
if (-not $containers) {
|
||||
Write-Host "No API containers running"
|
||||
exit 1
|
||||
}
|
||||
$target = ($containers -split "`n")[0].Trim()
|
||||
Write-Host "Stopping API replica $target..."
|
||||
docker stop $target
|
||||
Reference in New Issue
Block a user