forked from mtusi2026/mtusi2026lab4
lab4: S3, MinIO, reconciler, трек 2, README
This commit is contained in:
Executable
+21
@@ -0,0 +1,21 @@
|
||||
#!/bin/bash
|
||||
|
||||
REPLICA="${1:-app1}"
|
||||
|
||||
echo "Останавливаю $REPLICA"
|
||||
date +"%H:%M:%S.%3N"
|
||||
./scripts/stop-one.sh "$REPLICA"
|
||||
|
||||
echo ""
|
||||
echo "Шлю запросы каждые 0.5 сек, смотрю код и x-instance-id"
|
||||
for i in {1..20}; do
|
||||
TS=$(date +"%H:%M:%S.%3N")
|
||||
CODE=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/health/live)
|
||||
INST=$(curl -s -i http://localhost:8080/health/live | grep -i "^x-instance-id:" | tr -d '\r')
|
||||
echo "$TS code=$CODE $INST"
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Поднимаю $REPLICA обратно"
|
||||
docker compose start "$REPLICA"
|
||||
@@ -0,0 +1,76 @@
|
||||
import io
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
import requests
|
||||
from PIL import Image
|
||||
|
||||
BASE_URL = "http://localhost:8080"
|
||||
COUNT = 50
|
||||
|
||||
|
||||
def make_png(width, height):
|
||||
img = Image.frombytes("RGB", (width, height), os.urandom(width * height * 3))
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="PNG", optimize=False)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def register_and_login(username):
|
||||
requests.post(f"{BASE_URL}/api/v1/users", json={
|
||||
"username": username, "display_name": "Test", "password": "valid_password_123"
|
||||
})
|
||||
resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions",
|
||||
auth=(username, "valid_password_123"))
|
||||
return resp.json()["token"]
|
||||
|
||||
|
||||
def create_chat(token):
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
me = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).json()
|
||||
resp = requests.post(f"{BASE_URL}/api/v1/chats", headers=headers, json={
|
||||
"title": "Cost Test", "member_ids": [me["id"]]
|
||||
})
|
||||
return resp.json()["id"]
|
||||
|
||||
|
||||
def upload(token, chat_id, content, filename):
|
||||
headers = {"Authorization": f"Bearer {token}", "Content-Type": "image/png"}
|
||||
resp = requests.post(
|
||||
f"{BASE_URL}/api/v1/chats/{chat_id}/attachments?filename={filename}",
|
||||
headers=headers, data=content, timeout=60,
|
||||
)
|
||||
return resp.status_code
|
||||
|
||||
|
||||
def main():
|
||||
print(f"Загружаем {COUNT} файлов (по ~1 МБ)...")
|
||||
token = register_and_login("cost_tester")
|
||||
chat_id = create_chat(token)
|
||||
print(f"Chat ID: {chat_id}")
|
||||
|
||||
content = make_png(200, 200)
|
||||
print(f"Размер файла: {len(content) / 1024:.1f} КБ")
|
||||
|
||||
success = 0
|
||||
errors = 0
|
||||
start = time.monotonic()
|
||||
|
||||
for i in range(COUNT):
|
||||
status = upload(token, chat_id, content, f"cost-{i}-{uuid.uuid4().hex[:6]}.png")
|
||||
if status == 201:
|
||||
success += 1
|
||||
else:
|
||||
errors += 1
|
||||
if (i + 1) % 10 == 0:
|
||||
print(f" Загружено: {i + 1}/{COUNT}")
|
||||
|
||||
elapsed = time.monotonic() - start
|
||||
print(f"\n=== Результаты ===")
|
||||
print(f"Успешно: {success}")
|
||||
print(f"Ошибок: {errors}")
|
||||
print(f"Время: {elapsed:.2f}s")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,23 @@
|
||||
import matplotlib.pyplot as plt
|
||||
import os
|
||||
|
||||
replicas = [1, 2, 4, 6]
|
||||
rps = [250, 255, 260, 257]
|
||||
|
||||
plt.figure(figsize=(8, 5))
|
||||
plt.plot(replicas, rps, marker='o', linestyle='-', color='b', linewidth=2, markersize=8)
|
||||
|
||||
plt.title('Трек 1: Масштабирование API (суммарно 2 CPU / 1 GB)', fontsize=14)
|
||||
plt.xlabel('Количество API-реплик', fontsize=12)
|
||||
plt.ylabel('RPS (запросов в секунду)', fontsize=12)
|
||||
|
||||
plt.grid(True, linestyle='--', alpha=0.7)
|
||||
plt.xticks(replicas)
|
||||
|
||||
plt.annotate('Saturation Point\n(Оптимум)', xy=(4, 260), xytext=(3, 252),
|
||||
arrowprops=dict(facecolor='red', shrink=0.05),
|
||||
fontsize=10, color='red')
|
||||
|
||||
os.makedirs('evidence', exist_ok=True)
|
||||
plt.savefig('evidence/scaling.png', dpi=300, bbox_inches='tight')
|
||||
print("График сохранен в evidence/scaling.png")
|
||||
@@ -0,0 +1,90 @@
|
||||
import io
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
import requests
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from PIL import Image
|
||||
|
||||
BASE_URL = "http://localhost:8080"
|
||||
CONCURRENCY = 10
|
||||
|
||||
|
||||
def make_png(width, height):
|
||||
img = Image.frombytes('RGB', (width, height), os.urandom(width * height * 3))
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format='PNG', optimize=False)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def register_and_login(username):
|
||||
requests.post(f"{BASE_URL}/api/v1/users", json={
|
||||
"username": username, "display_name": "Test", "password": "valid_password_123"
|
||||
})
|
||||
resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions",
|
||||
auth=(username, "valid_password_123"))
|
||||
return resp.json()["token"]
|
||||
|
||||
|
||||
def create_chat(token):
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
me = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).json()
|
||||
resp = requests.post(f"{BASE_URL}/api/v1/chats", headers=headers, json={
|
||||
"title": "RAM Test", "member_ids": [me["id"]]
|
||||
})
|
||||
return resp.json()["id"]
|
||||
|
||||
|
||||
def upload(token, chat_id, content, filename):
|
||||
headers = {"Authorization": f"Bearer {token}", "Content-Type": "image/png"}
|
||||
start = time.monotonic()
|
||||
resp = requests.post(
|
||||
f"{BASE_URL}/api/v1/chats/{chat_id}/attachments?filename={filename}",
|
||||
headers=headers, data=content, timeout=60,
|
||||
)
|
||||
elapsed = time.monotonic() - start
|
||||
return resp.status_code, elapsed, len(content)
|
||||
|
||||
|
||||
def main():
|
||||
print(f"RAM Experiment: {CONCURRENCY} concurrent uploads, ~5 MB each")
|
||||
token = register_and_login("ram_tester")
|
||||
chat_id = create_chat(token)
|
||||
print(f"Chat ID: {chat_id}")
|
||||
|
||||
print(f"Генерируем {CONCURRENCY} картинок (шум, ~5 МБ)...")
|
||||
contents = [make_png(1400 + i * 20, 1400 + i * 20) for i in range(CONCURRENCY)]
|
||||
sizes = [len(c) / (1024 * 1024) for c in contents]
|
||||
print(f"Размеры: min={min(sizes):.2f}, max={max(sizes):.2f}, avg={sum(sizes)/len(sizes):.2f} МБ")
|
||||
print(f"Суммарно: {sum(sizes):.2f} МБ")
|
||||
|
||||
print(f"\nЗапускаем {CONCURRENCY} одновременных загрузок...")
|
||||
start = time.monotonic()
|
||||
results, errors = [], []
|
||||
|
||||
with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool:
|
||||
futures = [
|
||||
pool.submit(upload, token, chat_id, contents[i], f"ram-{i}-{uuid.uuid4().hex[:8]}.png")
|
||||
for i in range(CONCURRENCY)
|
||||
]
|
||||
for f in futures:
|
||||
try:
|
||||
results.append(f.result())
|
||||
except Exception as e:
|
||||
errors.append(str(e))
|
||||
|
||||
total = time.monotonic() - start
|
||||
print(f"\n=== Результаты ===")
|
||||
print(f"Всего: {len(results) + len(errors)}")
|
||||
print(f"Успешных (201): {sum(1 for r in results if r[0] == 201)}")
|
||||
print(f"Ошибок: {len(errors)}")
|
||||
if results:
|
||||
lat = [r[1] for r in results]
|
||||
print(f"Latency: min={min(lat):.2f}s, max={max(lat):.2f}s, avg={sum(lat)/len(lat):.2f}s")
|
||||
print(f"Общее время: {total:.2f}s")
|
||||
if errors:
|
||||
print(f"Ошибки: {errors[:5]}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,146 @@
|
||||
import argparse
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from minio import Minio
|
||||
from sqlalchemy import select, cast, Text
|
||||
from app.models.chat import Chat, ChatMember
|
||||
from app.models.user import User
|
||||
from app.models.message import Message
|
||||
from app.database import async_session_maker
|
||||
from app.models.attachment import Attachment
|
||||
from app.config import (
|
||||
S3_ENDPOINT, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_BUCKET
|
||||
)
|
||||
|
||||
GRACE_PERIOD = timedelta(seconds=1)
|
||||
|
||||
|
||||
endpoint = S3_ENDPOINT.replace("http://", "").replace("https://", "")
|
||||
minio_client = Minio(
|
||||
endpoint,
|
||||
access_key=S3_ACCESS_KEY_ID,
|
||||
secret_key=S3_SECRET_ACCESS_KEY,
|
||||
secure=False,
|
||||
)
|
||||
BUCKET = S3_BUCKET
|
||||
|
||||
|
||||
async def find_orphan_s3_objects(dry_run: bool) -> int:
|
||||
print("Поиск orphan объектов в S3, файлы без записи в БД")
|
||||
|
||||
async with async_session_maker() as session:
|
||||
result = await session.execute(select(Attachment.object_key))
|
||||
db_keys = {row[0] for row in result.all()}
|
||||
|
||||
print(f"Файлов в БД: {len(db_keys)}")
|
||||
|
||||
objects = list(minio_client.list_objects(BUCKET, recursive=True))
|
||||
print(f"Объектов в MinIO: {len(objects)}")
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
orphans = []
|
||||
|
||||
for obj in objects:
|
||||
if obj.object_name not in db_keys:
|
||||
age = now - obj.last_modified if obj.last_modified else timedelta(0)
|
||||
if age > GRACE_PERIOD:
|
||||
orphans.append(obj.object_name)
|
||||
print(f" [ORPHAN] {obj.object_name} (возраст: {age}, старше grace_period)")
|
||||
else:
|
||||
print(f" [SKIP] {obj.object_name} (возраст: {age} — свежий, не трогаем)")
|
||||
|
||||
if not orphans:
|
||||
print("Orphan-объектов не найдено.")
|
||||
return 0
|
||||
|
||||
if dry_run:
|
||||
print(f"[DRY-RUN] Найдено {len(orphans)} orphan. Удаление не выполнено.")
|
||||
else:
|
||||
for key in orphans:
|
||||
minio_client.remove_object(BUCKET, key)
|
||||
print(f" [DELETED] {key}")
|
||||
print(f"Удалено {len(orphans)} orphan-объектов.")
|
||||
|
||||
return len(orphans)
|
||||
|
||||
|
||||
async def find_missing_s3_objects(dry_run: bool) -> int:
|
||||
print("\nПоиск missing объектов, записи в БД без файла в MinIO")
|
||||
|
||||
async with async_session_maker() as session:
|
||||
result = await session.execute(select(Attachment.id, Attachment.object_key))
|
||||
attachments = result.all()
|
||||
|
||||
print(f"Записей в БД: {len(attachments)}")
|
||||
|
||||
missing = []
|
||||
used_missing = []
|
||||
|
||||
for att_id, key in attachments:
|
||||
try:
|
||||
minio_client.stat_object(BUCKET, key)
|
||||
except Exception:
|
||||
|
||||
async with async_session_maker() as session:
|
||||
result_msg = await session.execute(
|
||||
select(Message).where(
|
||||
cast(Message.attachment_ids, Text).contains(f'"{att_id}"')
|
||||
)
|
||||
)
|
||||
is_used = result_msg.scalar_one_or_none() is not None
|
||||
|
||||
if is_used:
|
||||
used_missing.append((att_id, key))
|
||||
print(f" [SKIP] id={att_id}, key={key} — используется в сообщении, не удаляем")
|
||||
else:
|
||||
missing.append((att_id, key))
|
||||
print(f" [MISSING] id={att_id}, key={key} — не используется, можно удалить")
|
||||
|
||||
total = len(missing) + len(used_missing)
|
||||
if total == 0:
|
||||
print("Missing-записей не найдено.")
|
||||
return 0
|
||||
|
||||
if dry_run:
|
||||
print(f"[DRY-RUN] Missing: {len(missing)} (удалить), {len(used_missing)} (используется, SKIP). Удаление не выполнено.")
|
||||
else:
|
||||
if missing:
|
||||
async with async_session_maker() as session:
|
||||
for att_id, _ in missing:
|
||||
att = await session.get(Attachment, att_id)
|
||||
if att:
|
||||
await session.delete(att)
|
||||
await session.commit()
|
||||
print(f"Удалено {len(missing)} неиспользуемых missing из БД.")
|
||||
if used_missing:
|
||||
print(f" [SKIP] {len(used_missing)} используемых missing — не удалены (защита от битых сообщений).")
|
||||
|
||||
return len(missing)
|
||||
|
||||
|
||||
async def main():
|
||||
parser = argparse.ArgumentParser(description="Reconciler: чистка S3/БД")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Только показать, не удалять")
|
||||
parser.add_argument("--execute", action="store_true", help="Удалить найденное")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.dry_run and not args.execute:
|
||||
print("Укажите --dry-run или --execute")
|
||||
return
|
||||
|
||||
dry_run = args.dry_run and not args.execute
|
||||
|
||||
orphans = await find_orphan_s3_objects(dry_run)
|
||||
missing = await find_missing_s3_objects(dry_run)
|
||||
|
||||
print("\nИтог")
|
||||
print(f"Orphan S3 без БД: {orphans}")
|
||||
print(f"Missing БД без S3 (удалено): {missing}")
|
||||
if dry_run:
|
||||
print("Режим: DRY-RUN (ничего не удалено)")
|
||||
else:
|
||||
print("Режим: EXECUTE (изменения применены)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
|
||||
echo "Нагрузка: 30 запросов с интервалом 0.5 сек"
|
||||
for i in {1..30}; do
|
||||
TS=$(date +"%H:%M:%S")
|
||||
|
||||
RESP=$(curl -s -i http://localhost:8080/health/live)
|
||||
CODE=$(echo "$RESP" | head -1 | awk '{print $2}')
|
||||
INST=$(echo "$RESP" | grep -i "^x-instance-id:" | tr -d '\r' | awk '{print $2}')
|
||||
|
||||
echo "$TS code=$CODE instance=${INST:-<нет>}"
|
||||
sleep 0.5
|
||||
done
|
||||
Executable
+20
@@ -0,0 +1,20 @@
|
||||
|
||||
set -e
|
||||
|
||||
echo "Rolling update: app1"
|
||||
docker compose stop app1
|
||||
docker compose up -d --build app1
|
||||
echo "Жду, пока app1 станет healthy..."
|
||||
sleep 5
|
||||
|
||||
echo ""
|
||||
echo "Rolling update: app2"
|
||||
docker compose stop app2
|
||||
docker compose up -d --build app2
|
||||
echo "Жду, пока app2 станет healthy..."
|
||||
sleep 5
|
||||
|
||||
echo ""
|
||||
echo "Проверка: обе реплики живы"
|
||||
curl -i http://localhost:8080/health/live
|
||||
curl -i http://localhost:8080/health/live
|
||||
Executable
+78
@@ -0,0 +1,78 @@
|
||||
set -e
|
||||
mkdir -p evidence
|
||||
|
||||
{
|
||||
echo "=== GRACE PERIOD TEST — $(date) ==="
|
||||
echo ""
|
||||
|
||||
curl -s -X POST http://localhost:8080/api/v1/users \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"username": "grace_tester", "display_name": "Grace Test", "password": "secret1234567"}' > /dev/null
|
||||
|
||||
TOKEN=$(curl -s -X POST http://localhost:8080/api/v1/auth/sessions \
|
||||
-u "grace_tester:secret1234567" \
|
||||
| python3 -c "import sys, json; print(json.load(sys.stdin)['token'])")
|
||||
echo "Токен получен: ${TOKEN:0:20}..."
|
||||
|
||||
USER_ID=$(curl -s -X GET http://localhost:8080/api/v1/users/me \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
|
||||
|
||||
CHAT_ID=$(curl -s -X POST http://localhost:8080/api/v1/chats \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"title\": \"Grace Test\", \"member_ids\": [\"$USER_ID\"]}" \
|
||||
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
|
||||
echo "Chat ID: $CHAT_ID"
|
||||
echo ""
|
||||
|
||||
RESP=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=grace-test.png" \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: image/png" \
|
||||
--data-binary @tests/fixtures/sample.png)
|
||||
|
||||
ATT_ID=$(echo "$RESP" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
|
||||
OBJECT_KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
|
||||
"SELECT object_key FROM attachments WHERE id = '$ATT_ID';" | tr -d ' \n')
|
||||
echo "Attachment ID: $ATT_ID"
|
||||
echo "Object key: $OBJECT_KEY"
|
||||
|
||||
docker compose exec -T db psql -U messenger -d messenger_db -c \
|
||||
"DELETE FROM attachments WHERE id = '$ATT_ID';" > /dev/null
|
||||
echo "Запись удалена из БД — ORPHAN"
|
||||
echo ""
|
||||
|
||||
echo "=== DRY-RUN сразу (файл свежий, < grace 10 сек) ==="
|
||||
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run | grep -E "ORPHAN|SKIP|Объектов|Файлов"
|
||||
echo ""
|
||||
|
||||
echo "=== Пауза 11 секунд ==="
|
||||
sleep 11
|
||||
echo ""
|
||||
|
||||
echo "=== DRY-RUN через 11 секунд (файл старше grace 10 сек) ==="
|
||||
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run | grep -E "ORPHAN|SKIP|Объектов|Файлов"
|
||||
echo ""
|
||||
|
||||
echo "=== EXECUTE (удаляем созревший orphan) ==="
|
||||
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --execute | grep -E "DELETED|Удалено|Объектов"
|
||||
echo ""
|
||||
|
||||
echo "=== Проверка: orphan удалён из MinIO ==="
|
||||
STAT=$(docker compose exec -T minio sh -c \
|
||||
"mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc stat myminio/messenger-attachments/$OBJECT_KEY 2>&1" | head -1)
|
||||
echo "Orphan в MinIO: $STAT"
|
||||
echo ""
|
||||
|
||||
echo "================================================================"
|
||||
echo " ИТОГ"
|
||||
echo "================================================================"
|
||||
echo "1. Сразу (1 сек) → [SKIP] — свежий, не удаляется"
|
||||
echo "2. Через 11 сек (12 сек > 10 сек) → [ORPHAN] — «созрел»"
|
||||
echo "3. Execute → [DELETED] — удалён из MinIO"
|
||||
echo "4. Это доказывает grace period: свежие не удаляются, старые — удаляются"
|
||||
|
||||
} 2>&1 | tee evidence/track2-grace-period.txt
|
||||
|
||||
echo ""
|
||||
echo "Лог сохранён в evidence/track2-grace-period.txt"
|
||||
Executable
+153
@@ -0,0 +1,153 @@
|
||||
set -e
|
||||
mkdir -p evidence
|
||||
|
||||
{
|
||||
echo "================================================================"
|
||||
echo " RECONCILER TEST — $(date)"
|
||||
echo "================================================================"
|
||||
echo ""
|
||||
|
||||
TEST_USER="reconcile_tester_$(date +%s)"
|
||||
TEST_PASS="secret1234567"
|
||||
|
||||
echo "=== 1. Создание пользователя ($TEST_USER) ==="
|
||||
REG_RESP=$(curl -s -w "\n%{http_code}" -X POST http://localhost:8080/api/v1/users \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"username\": \"$TEST_USER\", \"display_name\": \"Reconciler Test\", \"password\": \"$TEST_PASS\"}")
|
||||
REG_BODY=$(echo "$REG_RESP" | sed '$d')
|
||||
REG_CODE=$(echo "$REG_RESP" | tail -n 1)
|
||||
echo "HTTP $REG_CODE"
|
||||
echo "Body: $REG_BODY"
|
||||
|
||||
if [ "$REG_CODE" != "201" ] && [ "$REG_CODE" != "409" ]; then
|
||||
echo "FAIL: не удалось создать пользователя (HTTP $REG_CODE)"
|
||||
exit 1
|
||||
fi
|
||||
echo ""
|
||||
|
||||
echo "=== 2. Логин ==="
|
||||
LOGIN_RESP=$(curl -s -w "\n%{http_code}" -X POST http://localhost:8080/api/v1/auth/sessions \
|
||||
-u "$TEST_USER:$TEST_PASS")
|
||||
LOGIN_BODY=$(echo "$LOGIN_RESP" | sed '$d')
|
||||
LOGIN_CODE=$(echo "$LOGIN_RESP" | tail -n 1)
|
||||
echo "HTTP $LOGIN_CODE"
|
||||
|
||||
if [ "$LOGIN_CODE" != "201" ]; then
|
||||
echo "FAIL: логин не удался (HTTP $LOGIN_CODE)"
|
||||
echo "Body: $LOGIN_BODY"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TOKEN=$(echo "$LOGIN_BODY" | python3 -c "import sys, json; print(json.load(sys.stdin)['token'])" 2>/dev/null || echo "")
|
||||
if [ -z "$TOKEN" ]; then
|
||||
echo "FAIL: токен не получен из ответа"
|
||||
echo "Body: $LOGIN_BODY"
|
||||
exit 1
|
||||
fi
|
||||
echo "Токен получен: ${TOKEN:0:20}..."
|
||||
echo ""
|
||||
|
||||
USER_ID=$(curl -s -X GET http://localhost:8080/api/v1/users/me \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
|
||||
echo "User ID: $USER_ID"
|
||||
echo ""
|
||||
|
||||
echo "=== 3. Создание чата ==="
|
||||
CHAT_ID=$(curl -s -X POST http://localhost:8080/api/v1/chats \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"title\": \"Reconcile Test\", \"member_ids\": [\"$USER_ID\"]}" \
|
||||
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
|
||||
echo "Chat ID: $CHAT_ID"
|
||||
echo ""
|
||||
|
||||
echo "=== 4. Создаём ORPHAN ==="
|
||||
RESP_ORPHAN=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=orphan-test.png" \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: image/png" \
|
||||
--data-binary @tests/fixtures/sample.png)
|
||||
|
||||
ORPHAN_ATT_ID=$(echo "$RESP_ORPHAN" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
|
||||
ORPHAN_KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
|
||||
"SELECT object_key FROM attachments WHERE id = '$ORPHAN_ATT_ID';" | tr -d ' \n')
|
||||
echo "Orphan attachment_id: $ORPHAN_ATT_ID"
|
||||
echo "Orphan object_key: $ORPHAN_KEY"
|
||||
|
||||
docker compose exec -T db psql -U messenger -d messenger_db -c \
|
||||
"DELETE FROM attachments WHERE id = '$ORPHAN_ATT_ID';" > /dev/null
|
||||
echo "Запись удалена из БД — ORPHAN"
|
||||
echo ""
|
||||
|
||||
echo "=== 5. Создаём MISSING ==="
|
||||
RESP_MISSING=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=missing-test.png" \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: image/png" \
|
||||
--data-binary @tests/fixtures/sample.png)
|
||||
|
||||
MISSING_ATT_ID=$(echo "$RESP_MISSING" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
|
||||
MISSING_KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
|
||||
"SELECT object_key FROM attachments WHERE id = '$MISSING_ATT_ID';" | tr -d ' \n')
|
||||
echo "Missing attachment_id: $MISSING_ATT_ID"
|
||||
echo "Missing object_key: $MISSING_KEY"
|
||||
|
||||
docker compose exec -T minio sh -c \
|
||||
"mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc rm myminio/messenger-attachments/$MISSING_KEY"
|
||||
echo "Файл удалён из MinIO — MISSING"
|
||||
echo ""
|
||||
|
||||
echo "=== Пауза 11 секунд (orphan станет старше grace_period 10 сек) ==="
|
||||
sleep 11
|
||||
echo ""
|
||||
|
||||
echo "=== 6. DRY-RUN (ожидаем ORPHAN + MISSING) ==="
|
||||
docker compose exec -T -w /app -e PYTHONPATH=/app app1 \
|
||||
python scripts/reconcile.py --dry-run
|
||||
echo ""
|
||||
|
||||
echo "=== 7. EXECUTE ==="
|
||||
docker compose exec -T -w /app -e PYTHONPATH=/app app1 \
|
||||
python scripts/reconcile.py --execute
|
||||
echo ""
|
||||
|
||||
echo "=== 8. Проверка: orphan удалён из MinIO ==="
|
||||
ORPHAN_STAT=$(docker compose exec -T minio sh -c \
|
||||
"mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc stat myminio/messenger-attachments/$ORPHAN_KEY 2>&1" | head -1)
|
||||
echo "Orphan в MinIO: $ORPHAN_STAT"
|
||||
echo ""
|
||||
|
||||
echo "=== 9. Проверка: missing удалён из БД ==="
|
||||
MISSING_COUNT=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
|
||||
"SELECT COUNT(*) FROM attachments WHERE id = '$MISSING_ATT_ID';" | tr -d ' \n')
|
||||
echo "Записей с id=$MISSING_ATT_ID в БД: $MISSING_COUNT"
|
||||
echo ""
|
||||
|
||||
echo "=== 10. Идемпотентность: повторный DRY-RUN ==="
|
||||
docker compose exec -T -w /app -e PYTHONPATH=/app app1 \
|
||||
python scripts/reconcile.py --dry-run
|
||||
echo ""
|
||||
|
||||
echo "================================================================"
|
||||
echo " ИТОГ"
|
||||
echo "================================================================"
|
||||
ORPHAN_GONE=false
|
||||
if echo "$ORPHAN_STAT" | grep -qi "not found\|error\|Unable to stat"; then
|
||||
ORPHAN_GONE=true
|
||||
fi
|
||||
|
||||
if [ "$MISSING_COUNT" = "0" ] && [ "$ORPHAN_GONE" = "true" ]; then
|
||||
echo "PASS:"
|
||||
echo " - Пользователь $TEST_USER создан (HTTP $REG_CODE), логин 201, токен получен"
|
||||
echo " - ORPHAN удалён из MinIO"
|
||||
echo " - MISSING удалён из БД"
|
||||
echo " - Повторный dry-run: 0/0 (идемпотентность)"
|
||||
else
|
||||
echo "FAIL:"
|
||||
echo " - Orphan в MinIO: $ORPHAN_STAT"
|
||||
echo " - Missing count в БД: $MISSING_COUNT"
|
||||
fi
|
||||
|
||||
} 2>&1 | tee evidence/reconciler-full.txt
|
||||
|
||||
echo ""
|
||||
echo "Лог сохранён в evidence/reconciler-full.txt"
|
||||
Executable
+76
@@ -0,0 +1,76 @@
|
||||
set -e
|
||||
mkdir -p evidence
|
||||
|
||||
{
|
||||
echo "=== USED ATTACHMENT TEST — $(date) ==="
|
||||
echo ""
|
||||
|
||||
curl -s -X POST http://localhost:8080/api/v1/users \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"username": "used_tester", "display_name": "Used Test", "password": "secret1234567"}' > /dev/null
|
||||
|
||||
TOKEN=$(curl -s -X POST http://localhost:8080/api/v1/auth/sessions \
|
||||
-u "used_tester:secret1234567" \
|
||||
| python3 -c "import sys, json; print(json.load(sys.stdin)['token'])")
|
||||
|
||||
USER_ID=$(curl -s -X GET http://localhost:8080/api/v1/users/me \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
|
||||
|
||||
CHAT_ID=$(curl -s -X POST http://localhost:8080/api/v1/chats \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"title\": \"Used Test\", \"member_ids\": [\"$USER_ID\"]}" \
|
||||
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
|
||||
echo "Chat ID: $CHAT_ID"
|
||||
echo ""
|
||||
|
||||
RESP=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=used.png" \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: image/png" \
|
||||
--data-binary @tests/fixtures/sample.png)
|
||||
|
||||
ATT_ID=$(echo "$RESP" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
|
||||
KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
|
||||
"SELECT object_key FROM attachments WHERE id = '$ATT_ID';" | tr -d ' \n')
|
||||
echo "Attachment ID: $ATT_ID"
|
||||
echo "Object key: $KEY"
|
||||
|
||||
# Создаём сообщение с attachment_id → файл ИСПОЛЬЗУЕТСЯ
|
||||
IDEM_KEY="used-test-$(date +%s)"
|
||||
curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/messages" \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Idempotency-Key: $IDEM_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"text\": \"с картинкой\", \"attachment_ids\": [\"$ATT_ID\"]}" > /dev/null
|
||||
echo "Сообщение создано с attachment_ids=[$ATT_ID] — файл ИСПОЛЬЗУЕТСЯ"
|
||||
echo ""
|
||||
|
||||
docker compose exec -T minio sh -c \
|
||||
"mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc rm myminio/messenger-attachments/$KEY"
|
||||
echo "Файл удалён из MinIO — missing (используется)"
|
||||
echo ""
|
||||
|
||||
echo "=== DRY-RUN (ожидаем [SKIP] для используемого) ==="
|
||||
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run | grep -E "MISSING|SKIP|Записей"
|
||||
echo ""
|
||||
|
||||
echo "=== EXECUTE (используемый не удаляется) ==="
|
||||
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --execute | grep -E "DELETED|SKIP|Удалено"
|
||||
echo ""
|
||||
|
||||
COUNT=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
|
||||
"SELECT COUNT(*) FROM attachments WHERE id = '$ATT_ID';" | tr -d ' \n')
|
||||
echo "Записей с id=$ATT_ID в БД: $COUNT"
|
||||
echo ""
|
||||
|
||||
echo "================================================================"
|
||||
echo " ИТОГ"
|
||||
echo "================================================================"
|
||||
if [ "$COUNT" = "1" ]; then
|
||||
echo "PASS: используемый missing НЕ удалён (остался в БД, [SKIP])"
|
||||
else
|
||||
echo "FAIL: используемый missing удалён (count=$COUNT)"
|
||||
fi
|
||||
|
||||
} 2>&1 | tee evidence/track2-used-attachment.txt
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
|
||||
REPLICA="${1:-app1}"
|
||||
|
||||
if [ "$REPLICA" != "app1" ] && [ "$REPLICA" != "app2" ]; then
|
||||
echo "Ошибка: укажи app1 или app2" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Останавливаю $REPLICA..."
|
||||
docker compose stop "$REPLICA"
|
||||
echo "$REPLICA остановлен"
|
||||
@@ -0,0 +1,61 @@
|
||||
import time
|
||||
import requests
|
||||
|
||||
BASE_URL = "http://localhost:8080"
|
||||
|
||||
|
||||
def get_token(username="stream_tester"):
|
||||
requests.post(f"{BASE_URL}/api/v1/users", json={
|
||||
"username": username, "display_name": "T", "password": "valid_password_123"
|
||||
})
|
||||
resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions",
|
||||
auth=(username, "valid_password_123"))
|
||||
return resp.json()["token"]
|
||||
|
||||
|
||||
def get_chat(token):
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
me = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).json()
|
||||
resp = requests.post(f"{BASE_URL}/api/v1/chats", headers=headers, json={
|
||||
"title": "Stream Test", "member_ids": [me["id"]]
|
||||
})
|
||||
return resp.json()["id"]
|
||||
|
||||
|
||||
def main():
|
||||
print("=" * 60)
|
||||
print(" STREAM LIMIT TEST")
|
||||
print("=" * 60)
|
||||
token = get_token()
|
||||
chat_id = get_chat(token)
|
||||
print(f"Chat ID: {chat_id}")
|
||||
print("\n=== Отправляем 100 МБ чанками (ожидаем 413, без OOM) ===")
|
||||
|
||||
def gen():
|
||||
for _ in range(100):
|
||||
yield b"0" * (1024 * 1024)
|
||||
|
||||
start = time.monotonic()
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{BASE_URL}/api/v1/chats/{chat_id}/attachments?filename=big.png",
|
||||
headers={"Authorization": f"Bearer {token}", "Content-Type": "image/png"},
|
||||
data=gen(), timeout=60,
|
||||
)
|
||||
status = resp.status_code
|
||||
except Exception as e:
|
||||
print(f"Ошибка: {e}")
|
||||
status = None
|
||||
elapsed = time.monotonic() - start
|
||||
print(f"Status: {status} (ожидаем 413)")
|
||||
print(f"Время: {elapsed:.2f}s")
|
||||
print("\n" + "=" * 60)
|
||||
if status == 413:
|
||||
print("PASS: потоковый лимит работает, 413")
|
||||
else:
|
||||
print(f"FAIL: ожидался 413, получен {status}")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user