feat: ЛР4 - S3 хранилище MinIO, загрузка вложений и защита от атак

This commit is contained in:
Ильяс
2026-11-06 15:59:44 +03:00
parent 9b574fa5e9
commit 4979731408
24 changed files with 1418 additions and 72 deletions
+540 -13
View File
@@ -693,19 +693,6 @@ async def test_strict_types_and_json_parsing(client: httpx.AsyncClient):
assert res_garbage.json()["error"]["code"] == "invalid_request"
@pytest.mark.asyncio
async def test_session_create_rejects_non_empty_body(client: httpx.AsyncClient):
"""POST /api/v1/auth/sessions с непустым телом {"unexpected":true} отклоняется с 400."""
alice = await register_and_login(client, f"no_body_{uuid.uuid4().hex[:6]}")
cred = base64.b64encode(f"{alice['username']}:{alice['password']}".encode()).decode()
res = await client.post(
"/api/v1/auth/sessions",
headers={"Authorization": f"Basic {cred}", "Content-Type": "application/json"},
content='{"unexpected": true}',
)
assert res.status_code == 400
assert res.json()["error"]["code"] == "invalid_request"
@pytest.mark.asyncio
@@ -770,3 +757,543 @@ async def test_keyset_pagination_returns_next_cursor(client: httpx.AsyncClient):
assert data_p2["items"][0]["id"] != data_p1["items"][0]["id"]
@pytest.mark.asyncio
async def test_attachment_upload_and_download_flow(client: httpx.AsyncClient):
"""Проверка полного цикла загрузки и скачивания PNG и JPEG вложений."""
import hashlib
import io
from PIL import Image
alice = await register_and_login(client, "att_a")
bob = await register_and_login(client, "att_b")
chat_res = await client.post(
"/api/v1/chats",
headers=alice["headers"],
json={"title": "Attach Chat", "member_ids": [alice["user"]["id"], bob["user"]["id"]]},
)
assert chat_res.status_code == 201
chat_id = chat_res.json()["id"]
# 1. Загрузка PNG
with open("tests/fixtures/sample.png", "rb") as f:
png_data = f.read()
png_sha = hashlib.sha256(png_data).hexdigest()
up_res = await client.post(
f"/api/v1/chats/{chat_id}/attachments?filename=sample.png",
headers={**alice["headers"], "Content-Type": "image/png"},
content=png_data,
)
assert up_res.status_code == 201, up_res.text
att_png = up_res.json()
assert att_png["chat_id"] == chat_id
assert att_png["owner_id"] == alice["user"]["id"]
assert att_png["filename"] == "sample.png"
assert att_png["content_type"] == "image/png"
assert att_png["size_bytes"] == len(png_data)
assert att_png["sha256"] == png_sha
assert att_png["status"] == "ready"
assert att_png["variants"] == []
assert att_png["error_code"] is None
# 2. Получение метаданных Бобом
meta_res = await client.get(
f"/api/v1/attachments/{att_png['id']}",
headers=bob["headers"],
)
assert meta_res.status_code == 200
assert meta_res.json() == att_png
# 3. Скачивание контента Бобом
down_res = await client.get(
f"/api/v1/attachments/{att_png['id']}/content",
headers=bob["headers"],
)
assert down_res.status_code == 200
assert down_res.headers.get("content-type") == "image/png"
assert down_res.content == png_data
# 4. Загрузка и скачивание JPEG
jpeg_img = Image.new("RGB", (64, 64), color=(255, 128, 0))
jpeg_buf = io.BytesIO()
jpeg_img.save(jpeg_buf, format="JPEG")
jpeg_data = jpeg_buf.getvalue()
up_jpeg = await client.post(
f"/api/v1/chats/{chat_id}/attachments?filename=orange.jpg",
headers={**alice["headers"], "Content-Type": "image/jpeg"},
content=jpeg_data,
)
assert up_jpeg.status_code == 201
att_jpeg = up_jpeg.json()
assert att_jpeg["content_type"] == "image/jpeg"
assert att_jpeg["size_bytes"] == len(jpeg_data)
down_jpeg = await client.get(
f"/api/v1/attachments/{att_jpeg['id']}/content",
headers=bob["headers"],
)
assert down_jpeg.status_code == 200
assert down_jpeg.headers.get("content-type") == "image/jpeg"
assert down_jpeg.content == jpeg_data
@pytest.mark.asyncio
async def test_attachment_unsupported_media_type_and_magic_bytes(client: httpx.AsyncClient):
"""Проверка отклонения невалидных MIME-типов и поврежденных сигнатур (415)."""
alice = await register_and_login(client, "mime_a")
chat_res = await client.post(
"/api/v1/chats",
headers=alice["headers"],
json={"title": "Mime Chat", "member_ids": [alice["user"]["id"]]},
)
chat_id = chat_res.json()["id"]
# 1. Неподдерживаемый Content-Type: text/plain -> 415
res1 = await client.post(
f"/api/v1/chats/{chat_id}/attachments?filename=test.txt",
headers={**alice["headers"], "Content-Type": "text/plain"},
content=b"plain text",
)
assert res1.status_code == 415
assert res1.json()["error"]["code"] == "unsupported_media_type"
# 2. Неподдерживаемый Content-Type: image/gif -> 415
res2 = await client.post(
f"/api/v1/chats/{chat_id}/attachments?filename=anim.gif",
headers={**alice["headers"], "Content-Type": "image/gif"},
content=b"GIF89a...",
)
assert res2.status_code == 415
assert res2.json()["error"]["code"] == "unsupported_media_type"
# 3. Content-Type image/png, но поврежденные магические байты -> 415
res3 = await client.post(
f"/api/v1/chats/{chat_id}/attachments?filename=fake.png",
headers={**alice["headers"], "Content-Type": "image/png"},
content=b"NOT_A_PNG_HEADER_HERE",
)
assert res3.status_code == 415
assert res3.json()["error"]["code"] == "unsupported_media_type"
# 4. Content-Type image/jpeg, но поврежденные магические байты -> 415
res4 = await client.post(
f"/api/v1/chats/{chat_id}/attachments?filename=fake.jpg",
headers={**alice["headers"], "Content-Type": "image/jpeg"},
content=b"NOT_A_JPEG_HEADER_HERE",
)
assert res4.status_code == 415
assert res4.json()["error"]["code"] == "unsupported_media_type"
# 5. Поврежденный JPEG без пиксельных данных (18 байт) -> 415
bad_jpeg_18 = bytes.fromhex("ffd8ffc00008080001000101ffda0002ffd9")
res5 = await client.post(
f"/api/v1/chats/{chat_id}/attachments?filename=bad18.jpg",
headers={**alice["headers"], "Content-Type": "image/jpeg"},
content=bad_jpeg_18,
)
assert res5.status_code == 415
assert res5.json()["error"]["code"] == "unsupported_media_type"
# 6. Обрезанный PNG без пиксельных данных (33 байта) -> 415
bad_png_33 = bytes.fromhex("89504e470d0a1a0a0000000d4948445200000001000000010802000000907753de")
res6 = await client.post(
f"/api/v1/chats/{chat_id}/attachments?filename=bad33.png",
headers={**alice["headers"], "Content-Type": "image/png"},
content=bad_png_33,
)
assert res6.status_code == 415
assert res6.json()["error"]["code"] == "unsupported_media_type"
@pytest.mark.asyncio
async def test_attachment_filename_validation(client: httpx.AsyncClient):
"""Проверка валидации имени файла (path traversal, длина, спецсимволы -> 400)."""
alice = await register_and_login(client, "fname_a")
chat_res = await client.post(
"/api/v1/chats",
headers=alice["headers"],
json={"title": "Filename Chat", "member_ids": [alice["user"]["id"]]},
)
chat_id = chat_res.json()["id"]
with open("tests/fixtures/sample.png", "rb") as f:
valid_png = f.read()
headers = {**alice["headers"], "Content-Type": "image/png"}
# 1. Path traversal со слешем /
res_slash = await client.post(
f"/api/v1/chats/{chat_id}/attachments?filename=../../etc/passwd",
headers=headers,
content=valid_png,
)
assert res_slash.status_code == 400
assert res_slash.json()["error"]["code"] == "invalid_request"
# 2. Path traversal с обратным слешем \
res_bslash = await client.post(
f"/api/v1/chats/{chat_id}/attachments?filename=..\\boot.ini",
headers=headers,
content=valid_png,
)
assert res_bslash.status_code == 400
assert res_bslash.json()["error"]["code"] == "invalid_request"
# 3. Символ перевода строки \n
res_nl = await client.post(
f"/api/v1/chats/{chat_id}/attachments?filename=file%0Aname.png",
headers=headers,
content=valid_png,
)
assert res_nl.status_code == 400
assert res_nl.json()["error"]["code"] == "invalid_request"
# 4. Превышение 255 символов
long_name = "a" * 256 + ".png"
res_long = await client.post(
f"/api/v1/chats/{chat_id}/attachments?filename={long_name}",
headers=headers,
content=valid_png,
)
assert res_long.status_code == 400
assert res_long.json()["error"]["code"] == "invalid_request"
# 5. Пустое имя файла
res_empty = await client.post(
f"/api/v1/chats/{chat_id}/attachments?filename=",
headers=headers,
content=valid_png,
)
assert res_empty.status_code == 400
@pytest.mark.asyncio
async def test_attachment_payload_too_large_and_pixel_bomb(client: httpx.AsyncClient):
"""Проверка лимита размера файла (>10 MiB) и бомбы декомпрессии (>40M px) -> 413."""
import io
from PIL import Image
alice = await register_and_login(client, "large_a")
chat_res = await client.post(
"/api/v1/chats",
headers=alice["headers"],
json={"title": "Large Chat", "member_ids": [alice["user"]["id"]]},
)
chat_id = chat_res.json()["id"]
# 1. Размер файла более 10 MiB (10 * 1024 * 1024 + 10 байт) -> 413
oversize_bytes = b"\x89PNG\r\n\x1a\n" + b"\x00" * (10 * 1024 * 1024 + 10)
res_oversize = await client.post(
f"/api/v1/chats/{chat_id}/attachments?filename=huge.png",
headers={**alice["headers"], "Content-Type": "image/png"},
content=oversize_bytes,
)
assert res_oversize.status_code == 413
assert res_oversize.json()["error"]["code"] == "payload_too_large"
# 2. Бомба декомпрессии (10000x5000 = 50 млн пикселей > 40 млн пикселей) -> 413
bomb_img = Image.new("L", (10000, 5000), color=0)
bomb_buf = io.BytesIO()
bomb_img.save(bomb_buf, format="PNG")
bomb_bytes = bomb_buf.getvalue()
res_bomb = await client.post(
f"/api/v1/chats/{chat_id}/attachments?filename=pixel_bomb.png",
headers={**alice["headers"], "Content-Type": "image/png"},
content=bomb_bytes,
)
assert res_bomb.status_code == 413
assert res_bomb.json()["error"]["code"] == "payload_too_large"
@pytest.mark.asyncio
async def test_attachment_acl_and_isolation(client: httpx.AsyncClient):
"""Проверка строгой изоляции: посторонний пользователь получает 404 на все операции."""
alice = await register_and_login(client, "acl_a")
eve = await register_and_login(client, "acl_eve")
chat_res = await client.post(
"/api/v1/chats",
headers=alice["headers"],
json={"title": "Private Chat", "member_ids": [alice["user"]["id"]]},
)
chat_id = chat_res.json()["id"]
with open("tests/fixtures/sample.png", "rb") as f:
png_data = f.read()
# 1. Посторонний пытается загрузить вложение в чужой чат -> 404
res_eve_up = await client.post(
f"/api/v1/chats/{chat_id}/attachments?filename=sample.png",
headers={**eve["headers"], "Content-Type": "image/png"},
content=png_data,
)
assert res_eve_up.status_code == 404
assert res_eve_up.json()["error"]["code"] == "not_found"
# Алиса загружает файл
res_alice_up = await client.post(
f"/api/v1/chats/{chat_id}/attachments?filename=sample.png",
headers={**alice["headers"], "Content-Type": "image/png"},
content=png_data,
)
assert res_alice_up.status_code == 201
att_id = res_alice_up.json()["id"]
# 2. Посторонний пытается запросить метаданные -> 404
res_eve_meta = await client.get(
f"/api/v1/attachments/{att_id}",
headers=eve["headers"],
)
assert res_eve_meta.status_code == 404
# 3. Посторонний пытается скачать контент -> 404
res_eve_down = await client.get(
f"/api/v1/attachments/{att_id}/content",
headers=eve["headers"],
)
assert res_eve_down.status_code == 404
# 4. Несуществующий attachment ID -> 404
random_uuid = str(uuid.uuid4())
res_not_found = await client.get(
f"/api/v1/attachments/{random_uuid}",
headers=alice["headers"],
)
assert res_not_found.status_code == 404
# 5. Невалидный UUID -> 400
res_bad_uuid = await client.get(
"/api/v1/attachments/not-a-uuid",
headers=alice["headers"],
)
assert res_bad_uuid.status_code == 400
@pytest.mark.asyncio
async def test_message_linking_attachments(client: httpx.AsyncClient):
"""Проверка связывания сообщений и вложений: пустой текст с вложением, чужие вложения, лимит 10."""
alice = await register_and_login(client, "msg_att_a")
bob = await register_and_login(client, "msg_att_b")
chat_res = await client.post(
"/api/v1/chats",
headers=alice["headers"],
json={"title": "Msg Attach Chat", "member_ids": [alice["user"]["id"], bob["user"]["id"]]},
)
chat_id = chat_res.json()["id"]
chat2_res = await client.post(
"/api/v1/chats",
headers=alice["headers"],
json={"title": "Chat 2", "member_ids": [alice["user"]["id"]]},
)
chat2_id = chat2_res.json()["id"]
with open("tests/fixtures/sample.png", "rb") as f:
png_data = f.read()
# Загружаем вложение в chat 1
up1 = await client.post(
f"/api/v1/chats/{chat_id}/attachments?filename=sample.png",
headers={**alice["headers"], "Content-Type": "image/png"},
content=png_data,
)
att1_id = up1.json()["id"]
# Загружаем вложение в chat 2
up2 = await client.post(
f"/api/v1/chats/{chat2_id}/attachments?filename=chat2.png",
headers={**alice["headers"], "Content-Type": "image/png"},
content=png_data,
)
att2_id = up2.json()["id"]
# 1. Сообщение с пустым текстом, но с вложением -> 201
msg1_res = await client.post(
f"/api/v1/chats/{chat_id}/messages",
headers={**alice["headers"], "Idempotency-Key": uuid.uuid4().hex},
json={"text": "", "attachment_ids": [att1_id]},
)
assert msg1_res.status_code == 201, msg1_res.text
msg1 = msg1_res.json()
assert msg1["text"] == ""
assert msg1["attachment_ids"] == [att1_id]
# 2. Сообщение с пустым текстом И без вложений -> 400
msg2_res = await client.post(
f"/api/v1/chats/{chat_id}/messages",
headers={**alice["headers"], "Idempotency-Key": uuid.uuid4().hex},
json={"text": "", "attachment_ids": []},
)
assert msg2_res.status_code == 400
# 3. Прикрепление вложения из другого чата -> 404
msg3_res = await client.post(
f"/api/v1/chats/{chat_id}/messages",
headers={**alice["headers"], "Idempotency-Key": uuid.uuid4().hex},
json={"text": "Hello", "attachment_ids": [att2_id]},
)
assert msg3_res.status_code == 404
# 4. Прикрепление несуществующего вложения -> 404
fake_uuid = str(uuid.uuid4())
msg4_res = await client.post(
f"/api/v1/chats/{chat_id}/messages",
headers={**alice["headers"], "Idempotency-Key": uuid.uuid4().hex},
json={"text": "Hello", "attachment_ids": [fake_uuid]},
)
assert msg4_res.status_code == 404
# 5. Более 10 вложений -> 400
eleven_uuids = [str(uuid.uuid4()) for _ in range(11)]
msg5_res = await client.post(
f"/api/v1/chats/{chat_id}/messages",
headers={**alice["headers"], "Idempotency-Key": uuid.uuid4().hex},
json={"text": "Hello", "attachment_ids": eleven_uuids},
)
assert msg5_res.status_code == 400
@pytest.mark.asyncio
async def test_attachment_orphan_compensation(client: httpx.AsyncClient):
"""
Проверка компенсации сиротских объектов S3 при сбое БД (критерий оценки 4):
1. Через docker compose exec в PostgreSQL создаём триггер, вызывающий сбой при записи файла 'orphan_sim_fail.png'.
2. Выполняем POST вложения 'orphan_sim_fail.png'.
S3 PUT проходит успешно, но при вставке в PostgreSQL возникает ошибка.
Сервер перехватывает ошибку, вызывает s3_client.delete_object и возвращает 500.
3. Проверяем, что сиротский объект удалён из S3 (через проверку через api-1/boto3).
4. Удаляем триггер из PostgreSQL.
"""
import subprocess
alice = await register_and_login(client, "orph_a")
chat_res = await client.post(
"/api/v1/chats",
headers=alice["headers"],
json={"title": "Orphan Chat", "member_ids": [alice["user"]["id"]]},
)
chat_id = chat_res.json()["id"]
setup_sql = """
CREATE OR REPLACE FUNCTION trg_sim_fail() RETURNS TRIGGER AS $$
BEGIN
IF NEW.filename = 'orphan_sim_fail.png' THEN
RAISE EXCEPTION 'Simulated DB failure for orphan compensation test';
END IF;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS check_orphan_sim ON attachments;
CREATE TRIGGER check_orphan_sim BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION trg_sim_fail();
"""
cmd = ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "postgres", "-d", "messenger", "-c", setup_sql]
sub_res = subprocess.run(cmd, capture_output=True, text=True)
assert sub_res.returncode == 0, f"Ошибка создания триггера: {sub_res.stderr}"
try:
with open("tests/fixtures/sample.png", "rb") as f:
png_data = f.read()
res = await client.post(
f"/api/v1/chats/{chat_id}/attachments?filename=orphan_sim_fail.png",
headers={**alice["headers"], "Content-Type": "image/png"},
content=png_data,
)
assert res.status_code in (500, 503)
check_py = f"""
import boto3
s3 = boto3.client('s3', endpoint_url='http://minio:9000', aws_access_key_id='minioadmin', aws_secret_access_key='minioadmin')
res = s3.list_objects_v2(Bucket='messenger-attachments', Prefix='chats/{chat_id}/')
keys = [o['Key'] for o in res.get('Contents', [])]
found = any('orphan_sim_fail.png' in k for k in keys)
print('FOUND' if found else 'NOT_FOUND')
"""
cmd_s3 = ["docker", "compose", "exec", "-T", "api-1", "python", "-c", check_py]
s3_res = subprocess.run(cmd_s3, capture_output=True, text=True)
assert s3_res.returncode == 0
assert "NOT_FOUND" in s3_res.stdout, f"Сиротский объект остался в S3! Вывод: {s3_res.stdout}"
finally:
cleanup_sql = """
DROP TRIGGER IF EXISTS check_orphan_sim ON attachments;
DROP FUNCTION IF EXISTS trg_sim_fail();
"""
cmd_clean = ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "postgres", "-d", "messenger", "-c", cleanup_sql]
subprocess.run(cmd_clean, capture_output=True, text=True)
@pytest.mark.asyncio
async def test_private_bucket_outside_api(client: httpx.AsyncClient):
"""
Проверка закрытости S3 бакета вне API (критерий оценки 4):
1. Порты MinIO (9000/9001) не проброшены на хост наружу -> прямое подключение отклоняется.
2. Балансировщик Nginx маршрутизирует только /api/ и /health/, запрос к S3 напрямую через LB возвращает 404.
3. Неаутентифицированный запрос на скачивание вложения возвращает 401 Unauthorized.
"""
direct_conn_failed = False
try:
async with httpx.AsyncClient() as raw_client:
await raw_client.get("http://localhost:9000", timeout=1.0)
except httpx.ConnectError:
direct_conn_failed = True
assert direct_conn_failed, "Порт 9000 MinIO не должен быть доступен с хоста!"
res_lb_s3 = await client.get("/messenger-attachments/")
assert res_lb_s3.status_code == 404
random_id = str(uuid.uuid4())
res_anon = await client.get(f"/api/v1/attachments/{random_id}/content")
assert res_anon.status_code == 401
@pytest.mark.asyncio
async def test_concurrent_uploads_memory_limit(client: httpx.AsyncClient):
"""
Проверка конкурентной загрузки ≥10 файлов с контролем потребления RAM (критерий оценки 4):
1. Создаем пользователя и чат.
2. Запускаем параллельную загрузку 10 изображений через asyncio.gather.
3. Измеряем память контейнеров API через docker stats.
4. Все 10 загрузок завершаются успехом (201 Created), память остаётся в стабильных границах.
"""
import subprocess
alice = await register_and_login(client, "conc_a")
chat_res = await client.post(
"/api/v1/chats",
headers=alice["headers"],
json={"title": "Concurrent Uploads Chat", "member_ids": [alice["user"]["id"]]},
)
chat_id = chat_res.json()["id"]
with open("tests/fixtures/sample.png", "rb") as f:
png_data = f.read()
async def upload_single(idx: int):
return await client.post(
f"/api/v1/chats/{chat_id}/attachments?filename=concurrent_{idx}.png",
headers={**alice["headers"], "Content-Type": "image/png"},
content=png_data,
)
tasks = [upload_single(i) for i in range(10)]
responses = await asyncio.gather(*tasks)
for resp in responses:
assert resp.status_code == 201
data = resp.json()
assert data["status"] == "ready"
assert data["size_bytes"] == len(png_data)
stats_cmd = [
"docker", "stats", "--no-stream", "--format", "{{.Name}}: {{.MemUsage}}",
"mtusi2026lab4-api-1-1", "mtusi2026lab4-api-2-1"
]
stats_res = subprocess.run(stats_cmd, capture_output=True, text=True)
assert stats_res.returncode == 0