lab4: S3, MinIO, reconciler, трек 2, README

This commit is contained in:
2026-11-12 20:14:39 +03:00
parent ec540a1a72
commit d88f77cccc
104 changed files with 204719 additions and 47 deletions
View File
+21
View File
@@ -0,0 +1,21 @@
#!/bin/bash
REPLICA="${1:-app1}"
echo "Останавливаю $REPLICA"
date +"%H:%M:%S.%3N"
./scripts/stop-one.sh "$REPLICA"
echo ""
echo "Шлю запросы каждые 0.5 сек, смотрю код и x-instance-id"
for i in {1..20}; do
TS=$(date +"%H:%M:%S.%3N")
CODE=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/health/live)
INST=$(curl -s -i http://localhost:8080/health/live | grep -i "^x-instance-id:" | tr -d '\r')
echo "$TS code=$CODE $INST"
sleep 0.5
done
echo ""
echo "Поднимаю $REPLICA обратно"
docker compose start "$REPLICA"
+76
View File
@@ -0,0 +1,76 @@
import io
import os
import time
import uuid
import requests
from PIL import Image
BASE_URL = "http://localhost:8080"
COUNT = 50
def make_png(width, height):
img = Image.frombytes("RGB", (width, height), os.urandom(width * height * 3))
buf = io.BytesIO()
img.save(buf, format="PNG", optimize=False)
return buf.getvalue()
def register_and_login(username):
requests.post(f"{BASE_URL}/api/v1/users", json={
"username": username, "display_name": "Test", "password": "valid_password_123"
})
resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions",
auth=(username, "valid_password_123"))
return resp.json()["token"]
def create_chat(token):
headers = {"Authorization": f"Bearer {token}"}
me = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).json()
resp = requests.post(f"{BASE_URL}/api/v1/chats", headers=headers, json={
"title": "Cost Test", "member_ids": [me["id"]]
})
return resp.json()["id"]
def upload(token, chat_id, content, filename):
headers = {"Authorization": f"Bearer {token}", "Content-Type": "image/png"}
resp = requests.post(
f"{BASE_URL}/api/v1/chats/{chat_id}/attachments?filename={filename}",
headers=headers, data=content, timeout=60,
)
return resp.status_code
def main():
print(f"Загружаем {COUNT} файлов (по ~1 МБ)...")
token = register_and_login("cost_tester")
chat_id = create_chat(token)
print(f"Chat ID: {chat_id}")
content = make_png(200, 200)
print(f"Размер файла: {len(content) / 1024:.1f} КБ")
success = 0
errors = 0
start = time.monotonic()
for i in range(COUNT):
status = upload(token, chat_id, content, f"cost-{i}-{uuid.uuid4().hex[:6]}.png")
if status == 201:
success += 1
else:
errors += 1
if (i + 1) % 10 == 0:
print(f" Загружено: {i + 1}/{COUNT}")
elapsed = time.monotonic() - start
print(f"\n=== Результаты ===")
print(f"Успешно: {success}")
print(f"Ошибок: {errors}")
print(f"Время: {elapsed:.2f}s")
if __name__ == "__main__":
main()
+23
View File
@@ -0,0 +1,23 @@
import matplotlib.pyplot as plt
import os
replicas = [1, 2, 4, 6]
rps = [250, 255, 260, 257]
plt.figure(figsize=(8, 5))
plt.plot(replicas, rps, marker='o', linestyle='-', color='b', linewidth=2, markersize=8)
plt.title('Трек 1: Масштабирование API (суммарно 2 CPU / 1 GB)', fontsize=14)
plt.xlabel('Количество API-реплик', fontsize=12)
plt.ylabel('RPS (запросов в секунду)', fontsize=12)
plt.grid(True, linestyle='--', alpha=0.7)
plt.xticks(replicas)
plt.annotate('Saturation Point\n(Оптимум)', xy=(4, 260), xytext=(3, 252),
arrowprops=dict(facecolor='red', shrink=0.05),
fontsize=10, color='red')
os.makedirs('evidence', exist_ok=True)
plt.savefig('evidence/scaling.png', dpi=300, bbox_inches='tight')
print("График сохранен в evidence/scaling.png")
+90
View File
@@ -0,0 +1,90 @@
import io
import os
import time
import uuid
import requests
from concurrent.futures import ThreadPoolExecutor
from PIL import Image
BASE_URL = "http://localhost:8080"
CONCURRENCY = 10
def make_png(width, height):
img = Image.frombytes('RGB', (width, height), os.urandom(width * height * 3))
buf = io.BytesIO()
img.save(buf, format='PNG', optimize=False)
return buf.getvalue()
def register_and_login(username):
requests.post(f"{BASE_URL}/api/v1/users", json={
"username": username, "display_name": "Test", "password": "valid_password_123"
})
resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions",
auth=(username, "valid_password_123"))
return resp.json()["token"]
def create_chat(token):
headers = {"Authorization": f"Bearer {token}"}
me = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).json()
resp = requests.post(f"{BASE_URL}/api/v1/chats", headers=headers, json={
"title": "RAM Test", "member_ids": [me["id"]]
})
return resp.json()["id"]
def upload(token, chat_id, content, filename):
headers = {"Authorization": f"Bearer {token}", "Content-Type": "image/png"}
start = time.monotonic()
resp = requests.post(
f"{BASE_URL}/api/v1/chats/{chat_id}/attachments?filename={filename}",
headers=headers, data=content, timeout=60,
)
elapsed = time.monotonic() - start
return resp.status_code, elapsed, len(content)
def main():
print(f"RAM Experiment: {CONCURRENCY} concurrent uploads, ~5 MB each")
token = register_and_login("ram_tester")
chat_id = create_chat(token)
print(f"Chat ID: {chat_id}")
print(f"Генерируем {CONCURRENCY} картинок (шум, ~5 МБ)...")
contents = [make_png(1400 + i * 20, 1400 + i * 20) for i in range(CONCURRENCY)]
sizes = [len(c) / (1024 * 1024) for c in contents]
print(f"Размеры: min={min(sizes):.2f}, max={max(sizes):.2f}, avg={sum(sizes)/len(sizes):.2f} МБ")
print(f"Суммарно: {sum(sizes):.2f} МБ")
print(f"\nЗапускаем {CONCURRENCY} одновременных загрузок...")
start = time.monotonic()
results, errors = [], []
with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool:
futures = [
pool.submit(upload, token, chat_id, contents[i], f"ram-{i}-{uuid.uuid4().hex[:8]}.png")
for i in range(CONCURRENCY)
]
for f in futures:
try:
results.append(f.result())
except Exception as e:
errors.append(str(e))
total = time.monotonic() - start
print(f"\n=== Результаты ===")
print(f"Всего: {len(results) + len(errors)}")
print(f"Успешных (201): {sum(1 for r in results if r[0] == 201)}")
print(f"Ошибок: {len(errors)}")
if results:
lat = [r[1] for r in results]
print(f"Latency: min={min(lat):.2f}s, max={max(lat):.2f}s, avg={sum(lat)/len(lat):.2f}s")
print(f"Общее время: {total:.2f}s")
if errors:
print(f"Ошибки: {errors[:5]}")
if __name__ == "__main__":
main()
+146
View File
@@ -0,0 +1,146 @@
import argparse
import asyncio
from datetime import datetime, timedelta, timezone
from minio import Minio
from sqlalchemy import select, cast, Text
from app.models.chat import Chat, ChatMember
from app.models.user import User
from app.models.message import Message
from app.database import async_session_maker
from app.models.attachment import Attachment
from app.config import (
S3_ENDPOINT, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_BUCKET
)
GRACE_PERIOD = timedelta(seconds=1)
endpoint = S3_ENDPOINT.replace("http://", "").replace("https://", "")
minio_client = Minio(
endpoint,
access_key=S3_ACCESS_KEY_ID,
secret_key=S3_SECRET_ACCESS_KEY,
secure=False,
)
BUCKET = S3_BUCKET
async def find_orphan_s3_objects(dry_run: bool) -> int:
print("Поиск orphan объектов в S3, файлы без записи в БД")
async with async_session_maker() as session:
result = await session.execute(select(Attachment.object_key))
db_keys = {row[0] for row in result.all()}
print(f"Файлов в БД: {len(db_keys)}")
objects = list(minio_client.list_objects(BUCKET, recursive=True))
print(f"Объектов в MinIO: {len(objects)}")
now = datetime.now(timezone.utc)
orphans = []
for obj in objects:
if obj.object_name not in db_keys:
age = now - obj.last_modified if obj.last_modified else timedelta(0)
if age > GRACE_PERIOD:
orphans.append(obj.object_name)
print(f" [ORPHAN] {obj.object_name} (возраст: {age}, старше grace_period)")
else:
print(f" [SKIP] {obj.object_name} (возраст: {age} — свежий, не трогаем)")
if not orphans:
print("Orphan-объектов не найдено.")
return 0
if dry_run:
print(f"[DRY-RUN] Найдено {len(orphans)} orphan. Удаление не выполнено.")
else:
for key in orphans:
minio_client.remove_object(BUCKET, key)
print(f" [DELETED] {key}")
print(f"Удалено {len(orphans)} orphan-объектов.")
return len(orphans)
async def find_missing_s3_objects(dry_run: bool) -> int:
print("\nПоиск missing объектов, записи в БД без файла в MinIO")
async with async_session_maker() as session:
result = await session.execute(select(Attachment.id, Attachment.object_key))
attachments = result.all()
print(f"Записей в БД: {len(attachments)}")
missing = []
used_missing = []
for att_id, key in attachments:
try:
minio_client.stat_object(BUCKET, key)
except Exception:
async with async_session_maker() as session:
result_msg = await session.execute(
select(Message).where(
cast(Message.attachment_ids, Text).contains(f'"{att_id}"')
)
)
is_used = result_msg.scalar_one_or_none() is not None
if is_used:
used_missing.append((att_id, key))
print(f" [SKIP] id={att_id}, key={key} — используется в сообщении, не удаляем")
else:
missing.append((att_id, key))
print(f" [MISSING] id={att_id}, key={key} — не используется, можно удалить")
total = len(missing) + len(used_missing)
if total == 0:
print("Missing-записей не найдено.")
return 0
if dry_run:
print(f"[DRY-RUN] Missing: {len(missing)} (удалить), {len(used_missing)} (используется, SKIP). Удаление не выполнено.")
else:
if missing:
async with async_session_maker() as session:
for att_id, _ in missing:
att = await session.get(Attachment, att_id)
if att:
await session.delete(att)
await session.commit()
print(f"Удалено {len(missing)} неиспользуемых missing из БД.")
if used_missing:
print(f" [SKIP] {len(used_missing)} используемых missing — не удалены (защита от битых сообщений).")
return len(missing)
async def main():
parser = argparse.ArgumentParser(description="Reconciler: чистка S3/БД")
parser.add_argument("--dry-run", action="store_true", help="Только показать, не удалять")
parser.add_argument("--execute", action="store_true", help="Удалить найденное")
args = parser.parse_args()
if not args.dry_run and not args.execute:
print("Укажите --dry-run или --execute")
return
dry_run = args.dry_run and not args.execute
orphans = await find_orphan_s3_objects(dry_run)
missing = await find_missing_s3_objects(dry_run)
print("\nИтог")
print(f"Orphan S3 без БД: {orphans}")
print(f"Missing БД без S3 (удалено): {missing}")
if dry_run:
print("Режим: DRY-RUN (ничего не удалено)")
else:
print("Режим: EXECUTE (изменения применены)")
if __name__ == "__main__":
asyncio.run(main())
+12
View File
@@ -0,0 +1,12 @@
echo "Нагрузка: 30 запросов с интервалом 0.5 сек"
for i in {1..30}; do
TS=$(date +"%H:%M:%S")
RESP=$(curl -s -i http://localhost:8080/health/live)
CODE=$(echo "$RESP" | head -1 | awk '{print $2}')
INST=$(echo "$RESP" | grep -i "^x-instance-id:" | tr -d '\r' | awk '{print $2}')
echo "$TS code=$CODE instance=${INST:-<нет>}"
sleep 0.5
done
+20
View File
@@ -0,0 +1,20 @@
set -e
echo "Rolling update: app1"
docker compose stop app1
docker compose up -d --build app1
echo "Жду, пока app1 станет healthy..."
sleep 5
echo ""
echo "Rolling update: app2"
docker compose stop app2
docker compose up -d --build app2
echo "Жду, пока app2 станет healthy..."
sleep 5
echo ""
echo "Проверка: обе реплики живы"
curl -i http://localhost:8080/health/live
curl -i http://localhost:8080/health/live
+78
View File
@@ -0,0 +1,78 @@
set -e
mkdir -p evidence
{
echo "=== GRACE PERIOD TEST — $(date) ==="
echo ""
curl -s -X POST http://localhost:8080/api/v1/users \
-H "Content-Type: application/json" \
-d '{"username": "grace_tester", "display_name": "Grace Test", "password": "secret1234567"}' > /dev/null
TOKEN=$(curl -s -X POST http://localhost:8080/api/v1/auth/sessions \
-u "grace_tester:secret1234567" \
| python3 -c "import sys, json; print(json.load(sys.stdin)['token'])")
echo "Токен получен: ${TOKEN:0:20}..."
USER_ID=$(curl -s -X GET http://localhost:8080/api/v1/users/me \
-H "Authorization: Bearer $TOKEN" \
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
CHAT_ID=$(curl -s -X POST http://localhost:8080/api/v1/chats \
-H "Authorization: Bearer $TOKEN" \
-H "Content-Type: application/json" \
-d "{\"title\": \"Grace Test\", \"member_ids\": [\"$USER_ID\"]}" \
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
echo "Chat ID: $CHAT_ID"
echo ""
RESP=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=grace-test.png" \
-H "Authorization: Bearer $TOKEN" \
-H "Content-Type: image/png" \
--data-binary @tests/fixtures/sample.png)
ATT_ID=$(echo "$RESP" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
OBJECT_KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
"SELECT object_key FROM attachments WHERE id = '$ATT_ID';" | tr -d ' \n')
echo "Attachment ID: $ATT_ID"
echo "Object key: $OBJECT_KEY"
docker compose exec -T db psql -U messenger -d messenger_db -c \
"DELETE FROM attachments WHERE id = '$ATT_ID';" > /dev/null
echo "Запись удалена из БД — ORPHAN"
echo ""
echo "=== DRY-RUN сразу (файл свежий, < grace 10 сек) ==="
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run | grep -E "ORPHAN|SKIP|Объектов|Файлов"
echo ""
echo "=== Пауза 11 секунд ==="
sleep 11
echo ""
echo "=== DRY-RUN через 11 секунд (файл старше grace 10 сек) ==="
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run | grep -E "ORPHAN|SKIP|Объектов|Файлов"
echo ""
echo "=== EXECUTE (удаляем созревший orphan) ==="
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --execute | grep -E "DELETED|Удалено|Объектов"
echo ""
echo "=== Проверка: orphan удалён из MinIO ==="
STAT=$(docker compose exec -T minio sh -c \
"mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc stat myminio/messenger-attachments/$OBJECT_KEY 2>&1" | head -1)
echo "Orphan в MinIO: $STAT"
echo ""
echo "================================================================"
echo " ИТОГ"
echo "================================================================"
echo "1. Сразу (1 сек) → [SKIP] — свежий, не удаляется"
echo "2. Через 11 сек (12 сек > 10 сек) → [ORPHAN] — «созрел»"
echo "3. Execute → [DELETED] — удалён из MinIO"
echo "4. Это доказывает grace period: свежие не удаляются, старые — удаляются"
} 2>&1 | tee evidence/track2-grace-period.txt
echo ""
echo "Лог сохранён в evidence/track2-grace-period.txt"
+153
View File
@@ -0,0 +1,153 @@
set -e
mkdir -p evidence
{
echo "================================================================"
echo " RECONCILER TEST — $(date)"
echo "================================================================"
echo ""
TEST_USER="reconcile_tester_$(date +%s)"
TEST_PASS="secret1234567"
echo "=== 1. Создание пользователя ($TEST_USER) ==="
REG_RESP=$(curl -s -w "\n%{http_code}" -X POST http://localhost:8080/api/v1/users \
-H "Content-Type: application/json" \
-d "{\"username\": \"$TEST_USER\", \"display_name\": \"Reconciler Test\", \"password\": \"$TEST_PASS\"}")
REG_BODY=$(echo "$REG_RESP" | sed '$d')
REG_CODE=$(echo "$REG_RESP" | tail -n 1)
echo "HTTP $REG_CODE"
echo "Body: $REG_BODY"
if [ "$REG_CODE" != "201" ] && [ "$REG_CODE" != "409" ]; then
echo "FAIL: не удалось создать пользователя (HTTP $REG_CODE)"
exit 1
fi
echo ""
echo "=== 2. Логин ==="
LOGIN_RESP=$(curl -s -w "\n%{http_code}" -X POST http://localhost:8080/api/v1/auth/sessions \
-u "$TEST_USER:$TEST_PASS")
LOGIN_BODY=$(echo "$LOGIN_RESP" | sed '$d')
LOGIN_CODE=$(echo "$LOGIN_RESP" | tail -n 1)
echo "HTTP $LOGIN_CODE"
if [ "$LOGIN_CODE" != "201" ]; then
echo "FAIL: логин не удался (HTTP $LOGIN_CODE)"
echo "Body: $LOGIN_BODY"
exit 1
fi
TOKEN=$(echo "$LOGIN_BODY" | python3 -c "import sys, json; print(json.load(sys.stdin)['token'])" 2>/dev/null || echo "")
if [ -z "$TOKEN" ]; then
echo "FAIL: токен не получен из ответа"
echo "Body: $LOGIN_BODY"
exit 1
fi
echo "Токен получен: ${TOKEN:0:20}..."
echo ""
USER_ID=$(curl -s -X GET http://localhost:8080/api/v1/users/me \
-H "Authorization: Bearer $TOKEN" \
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
echo "User ID: $USER_ID"
echo ""
echo "=== 3. Создание чата ==="
CHAT_ID=$(curl -s -X POST http://localhost:8080/api/v1/chats \
-H "Authorization: Bearer $TOKEN" \
-H "Content-Type: application/json" \
-d "{\"title\": \"Reconcile Test\", \"member_ids\": [\"$USER_ID\"]}" \
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
echo "Chat ID: $CHAT_ID"
echo ""
echo "=== 4. Создаём ORPHAN ==="
RESP_ORPHAN=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=orphan-test.png" \
-H "Authorization: Bearer $TOKEN" \
-H "Content-Type: image/png" \
--data-binary @tests/fixtures/sample.png)
ORPHAN_ATT_ID=$(echo "$RESP_ORPHAN" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
ORPHAN_KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
"SELECT object_key FROM attachments WHERE id = '$ORPHAN_ATT_ID';" | tr -d ' \n')
echo "Orphan attachment_id: $ORPHAN_ATT_ID"
echo "Orphan object_key: $ORPHAN_KEY"
docker compose exec -T db psql -U messenger -d messenger_db -c \
"DELETE FROM attachments WHERE id = '$ORPHAN_ATT_ID';" > /dev/null
echo "Запись удалена из БД — ORPHAN"
echo ""
echo "=== 5. Создаём MISSING ==="
RESP_MISSING=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=missing-test.png" \
-H "Authorization: Bearer $TOKEN" \
-H "Content-Type: image/png" \
--data-binary @tests/fixtures/sample.png)
MISSING_ATT_ID=$(echo "$RESP_MISSING" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
MISSING_KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
"SELECT object_key FROM attachments WHERE id = '$MISSING_ATT_ID';" | tr -d ' \n')
echo "Missing attachment_id: $MISSING_ATT_ID"
echo "Missing object_key: $MISSING_KEY"
docker compose exec -T minio sh -c \
"mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc rm myminio/messenger-attachments/$MISSING_KEY"
echo "Файл удалён из MinIO — MISSING"
echo ""
echo "=== Пауза 11 секунд (orphan станет старше grace_period 10 сек) ==="
sleep 11
echo ""
echo "=== 6. DRY-RUN (ожидаем ORPHAN + MISSING) ==="
docker compose exec -T -w /app -e PYTHONPATH=/app app1 \
python scripts/reconcile.py --dry-run
echo ""
echo "=== 7. EXECUTE ==="
docker compose exec -T -w /app -e PYTHONPATH=/app app1 \
python scripts/reconcile.py --execute
echo ""
echo "=== 8. Проверка: orphan удалён из MinIO ==="
ORPHAN_STAT=$(docker compose exec -T minio sh -c \
"mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc stat myminio/messenger-attachments/$ORPHAN_KEY 2>&1" | head -1)
echo "Orphan в MinIO: $ORPHAN_STAT"
echo ""
echo "=== 9. Проверка: missing удалён из БД ==="
MISSING_COUNT=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
"SELECT COUNT(*) FROM attachments WHERE id = '$MISSING_ATT_ID';" | tr -d ' \n')
echo "Записей с id=$MISSING_ATT_ID в БД: $MISSING_COUNT"
echo ""
echo "=== 10. Идемпотентность: повторный DRY-RUN ==="
docker compose exec -T -w /app -e PYTHONPATH=/app app1 \
python scripts/reconcile.py --dry-run
echo ""
echo "================================================================"
echo " ИТОГ"
echo "================================================================"
ORPHAN_GONE=false
if echo "$ORPHAN_STAT" | grep -qi "not found\|error\|Unable to stat"; then
ORPHAN_GONE=true
fi
if [ "$MISSING_COUNT" = "0" ] && [ "$ORPHAN_GONE" = "true" ]; then
echo "PASS:"
echo " - Пользователь $TEST_USER создан (HTTP $REG_CODE), логин 201, токен получен"
echo " - ORPHAN удалён из MinIO"
echo " - MISSING удалён из БД"
echo " - Повторный dry-run: 0/0 (идемпотентность)"
else
echo "FAIL:"
echo " - Orphan в MinIO: $ORPHAN_STAT"
echo " - Missing count в БД: $MISSING_COUNT"
fi
} 2>&1 | tee evidence/reconciler-full.txt
echo ""
echo "Лог сохранён в evidence/reconciler-full.txt"
+76
View File
@@ -0,0 +1,76 @@
set -e
mkdir -p evidence
{
echo "=== USED ATTACHMENT TEST — $(date) ==="
echo ""
curl -s -X POST http://localhost:8080/api/v1/users \
-H "Content-Type: application/json" \
-d '{"username": "used_tester", "display_name": "Used Test", "password": "secret1234567"}' > /dev/null
TOKEN=$(curl -s -X POST http://localhost:8080/api/v1/auth/sessions \
-u "used_tester:secret1234567" \
| python3 -c "import sys, json; print(json.load(sys.stdin)['token'])")
USER_ID=$(curl -s -X GET http://localhost:8080/api/v1/users/me \
-H "Authorization: Bearer $TOKEN" \
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
CHAT_ID=$(curl -s -X POST http://localhost:8080/api/v1/chats \
-H "Authorization: Bearer $TOKEN" \
-H "Content-Type: application/json" \
-d "{\"title\": \"Used Test\", \"member_ids\": [\"$USER_ID\"]}" \
| python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
echo "Chat ID: $CHAT_ID"
echo ""
RESP=$(curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/attachments?filename=used.png" \
-H "Authorization: Bearer $TOKEN" \
-H "Content-Type: image/png" \
--data-binary @tests/fixtures/sample.png)
ATT_ID=$(echo "$RESP" | python3 -c "import sys, json; print(json.load(sys.stdin)['id'])")
KEY=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
"SELECT object_key FROM attachments WHERE id = '$ATT_ID';" | tr -d ' \n')
echo "Attachment ID: $ATT_ID"
echo "Object key: $KEY"
# Создаём сообщение с attachment_id → файл ИСПОЛЬЗУЕТСЯ
IDEM_KEY="used-test-$(date +%s)"
curl -s -X POST "http://localhost:8080/api/v1/chats/$CHAT_ID/messages" \
-H "Authorization: Bearer $TOKEN" \
-H "Idempotency-Key: $IDEM_KEY" \
-H "Content-Type: application/json" \
-d "{\"text\": \"с картинкой\", \"attachment_ids\": [\"$ATT_ID\"]}" > /dev/null
echo "Сообщение создано с attachment_ids=[$ATT_ID] — файл ИСПОЛЬЗУЕТСЯ"
echo ""
docker compose exec -T minio sh -c \
"mc alias set myminio http://localhost:9000 minioadmin minioadmin > /dev/null && mc rm myminio/messenger-attachments/$KEY"
echo "Файл удалён из MinIO — missing (используется)"
echo ""
echo "=== DRY-RUN (ожидаем [SKIP] для используемого) ==="
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --dry-run | grep -E "MISSING|SKIP|Записей"
echo ""
echo "=== EXECUTE (используемый не удаляется) ==="
docker compose exec -T -w /app -e PYTHONPATH=/app app1 python scripts/reconcile.py --execute | grep -E "DELETED|SKIP|Удалено"
echo ""
COUNT=$(docker compose exec -T db psql -U messenger -d messenger_db -t -c \
"SELECT COUNT(*) FROM attachments WHERE id = '$ATT_ID';" | tr -d ' \n')
echo "Записей с id=$ATT_ID в БД: $COUNT"
echo ""
echo "================================================================"
echo " ИТОГ"
echo "================================================================"
if [ "$COUNT" = "1" ]; then
echo "PASS: используемый missing НЕ удалён (остался в БД, [SKIP])"
else
echo "FAIL: используемый missing удалён (count=$COUNT)"
fi
} 2>&1 | tee evidence/track2-used-attachment.txt
+12
View File
@@ -0,0 +1,12 @@
#!/bin/bash
REPLICA="${1:-app1}"
if [ "$REPLICA" != "app1" ] && [ "$REPLICA" != "app2" ]; then
echo "Ошибка: укажи app1 или app2" >&2
exit 1
fi
echo "Останавливаю $REPLICA..."
docker compose stop "$REPLICA"
echo "$REPLICA остановлен"
+61
View File
@@ -0,0 +1,61 @@
import time
import requests
BASE_URL = "http://localhost:8080"
def get_token(username="stream_tester"):
requests.post(f"{BASE_URL}/api/v1/users", json={
"username": username, "display_name": "T", "password": "valid_password_123"
})
resp = requests.post(f"{BASE_URL}/api/v1/auth/sessions",
auth=(username, "valid_password_123"))
return resp.json()["token"]
def get_chat(token):
headers = {"Authorization": f"Bearer {token}"}
me = requests.get(f"{BASE_URL}/api/v1/users/me", headers=headers).json()
resp = requests.post(f"{BASE_URL}/api/v1/chats", headers=headers, json={
"title": "Stream Test", "member_ids": [me["id"]]
})
return resp.json()["id"]
def main():
print("=" * 60)
print(" STREAM LIMIT TEST")
print("=" * 60)
token = get_token()
chat_id = get_chat(token)
print(f"Chat ID: {chat_id}")
print("\n=== Отправляем 100 МБ чанками (ожидаем 413, без OOM) ===")
def gen():
for _ in range(100):
yield b"0" * (1024 * 1024)
start = time.monotonic()
try:
resp = requests.post(
f"{BASE_URL}/api/v1/chats/{chat_id}/attachments?filename=big.png",
headers={"Authorization": f"Bearer {token}", "Content-Type": "image/png"},
data=gen(), timeout=60,
)
status = resp.status_code
except Exception as e:
print(f"Ошибка: {e}")
status = None
elapsed = time.monotonic() - start
print(f"Status: {status} (ожидаем 413)")
print(f"Время: {elapsed:.2f}s")
print("\n" + "=" * 60)
if status == 413:
print("PASS: потоковый лимит работает, 413")
else:
print(f"FAIL: ожидался 413, получен {status}")
print("=" * 60)
if __name__ == "__main__":
main()