Files
mtusi2026lab5-fork/student-tests/test_attachments.py
T

484 lines
17 KiB
Python

"""API и транзакции с реальной БД; байты S3 подменены управляемым адаптером."""
import asyncio
import hashlib
from io import BytesIO
from datetime import datetime, timedelta, timezone
from pathlib import Path
import struct
from uuid import UUID, uuid4
import zlib
import pytest
from PIL import Image
from sqlalchemy import func, select
from sqlalchemy.exc import OperationalError
from sqlalchemy.ext.asyncio import AsyncSession
from starlette.requests import ClientDisconnect
from app.core.s3 import S3Settings
from app.core.jobs import JobSettings
from app.domain.attachment_jobs import process_attachment_job
from app.models.job import AttachmentJob
from app.schemas.job import JobMessage
from app.domain import attachments as domain
from app.errors.exceptions import unavailable
from app.models.attachment import Attachment, UploadIntent
from app.storage import attachments as repository
from app.storage.s3 import ObjectInfo
from helpers import bearer, create_chat, create_user, idempotency_header
PNG = Path("tests/fixtures/sample.png").read_bytes()
class MemoryStorage:
"""Только тестовый адаптер; production использует MinioStorage."""
def __init__(self):
self.settings = S3Settings(_env_file=None)
self.objects = {}
self.fail_put = self.fail_delete = self.down = False
async def put_file(self, key, path, **metadata):
self.objects[key] = (path.read_bytes(), metadata)
if self.fail_put:
raise unavailable()
async def delete_object(self, key):
if self.fail_delete:
raise unavailable()
self.objects.pop(key, None)
async def stat_object(self, key):
if self.down:
raise unavailable()
info = None
if key in self.objects:
data, metadata = self.objects[key]
info = ObjectInfo(
key, len(data), metadata["content_type"], datetime.now(timezone.utc)
)
return info
async def open_object(self, key):
async def chunks():
yield self.objects[key][0]
return chunks()
async def check_available(self):
return not self.down
@pytest.fixture
async def context(client, app):
storage = MemoryStorage()
app.state.s3 = storage
alice, bob, eve = [await create_user(client) for _ in range(3)]
chat = await create_chat(client, alice, bob)
return storage, alice, bob, eve, chat
async def upload(
client, actor, chat, data=PNG, filename="фото.png", content_type="image/png"
):
return await client.post(
f"/api/v1/chats/{chat['id']}/attachments",
params={"filename": filename},
headers={**bearer(actor), "Content-Type": content_type},
content=data,
)
async def finish_attachment(client, storage, actor, attachment_id):
engine = client._transport.app.state.engine
async with AsyncSession(engine) as session:
job = await session.scalar(
select(AttachmentJob).where(
AttachmentJob.attachment_id == UUID(attachment_id)
)
)
message = JobMessage(
job_id=job.id,
attachment_id=job.attachment_id,
pipeline_version=job.pipeline_version,
attempt=job.attempt + 1,
created_at=job.created_at,
)
await session.rollback()
assert await process_attachment_job(
session, storage, message, JobSettings(_env_file=None)
)
result = await client.get(
f"/api/v1/attachments/{attachment_id}", headers=bearer(actor)
)
assert result.json()["status"] == "ready"
return result.json()
async def test_upload_download_and_attachment_only_message(client, context):
storage, alice, bob, eve, chat = context
response = await upload(client, alice, chat)
assert response.status_code == 202, response.text
attachment = response.json()
assert attachment["status"] == "queued"
before = await client.get(
f"/api/v1/attachments/{attachment['id']}/content", headers=bearer(alice)
)
assert before.status_code == 409
attachment = await finish_attachment(client, storage, alice, attachment["id"])
assert attachment["sha256"] == hashlib.sha256(PNG).hexdigest()
assert {v["name"] for v in attachment["variants"]} == {
"thumbnail",
"crop",
"optimized",
}
assert attachment["error_code"] is None
assert "object_key" not in attachment
id = attachment["id"]
for actor in (alice, bob):
content = await client.get(
f"/api/v1/attachments/{id}/content", headers=bearer(actor)
)
assert content.status_code == 200 and content.content == PNG
assert content.headers["content-type"] == "image/png"
key = idempotency_header()
body = {"text": "", "attachment_ids": [id]}
path = f"/api/v1/chats/{chat['id']}/messages"
message = await client.post(path, json=body, headers={**bearer(bob), **key})
assert message.status_code == 201, message.text
repeat = await client.post(path, json=body, headers={**bearer(bob), **key})
assert repeat.json() == message.json()
history = await client.get(path, headers=bearer(alice))
assert history.json()["items"][0]["attachment_ids"] == [id]
for suffix in ("", "/content"):
denied = await client.get(
f"/api/v1/attachments/{id}{suffix}", headers=bearer(eve)
)
assert denied.status_code == 404
@pytest.mark.parametrize(
"data,content_type",
[
(b"not an image", "image/png"),
(PNG[:30], "image/png"),
(PNG, "image/jpeg"),
(PNG, "application/octet-stream"),
],
ids=["not-image", "truncated-png", "mime-spoofing", "unsupported-type"],
)
async def test_invalid_image_and_mime_are_415(client, context, data, content_type):
storage, alice, _, _, chat = context
response = await upload(client, alice, chat, data, content_type=content_type)
assert response.status_code == 415, response.text
assert not storage.objects
async def test_jpeg_is_stored_without_reencoding(client, context):
storage, alice, _, _, chat = context
output = BytesIO()
Image.new("RGB", (32, 32), "blue").save(output, format="JPEG")
original = output.getvalue()
response = await upload(client, alice, chat, original, content_type="image/jpeg")
assert response.status_code == 202, response.text
id = response.json()["id"]
await finish_attachment(client, storage, alice, id)
downloaded = await client.get(
f"/api/v1/attachments/{id}/content", headers=bearer(alice)
)
assert downloaded.content == original
@pytest.mark.parametrize(
"filename", ["../a.png", "a\\b.png", "a\x00.png", "a\n.png", "x" * 256]
)
async def test_unsafe_filename_is_400(client, context, filename):
storage, alice, _, _, chat = context
response = await upload(client, alice, chat, filename=filename)
assert response.status_code == 400
assert not storage.objects
async def test_oversize_chunked_upload_is_413(client, context):
storage, alice, _, _, chat = context
async def chunks():
for _ in range(11):
yield b"x" * 1048576
response = await upload(client, alice, chat, chunks())
assert response.status_code == 413
assert not storage.objects
async def test_oversize_content_length_is_413(client, context):
_, alice, _, _, chat = context
response = await client.post(
f"/api/v1/chats/{chat['id']}/attachments?filename=a.png",
headers={
**bearer(alice),
"Content-Type": "image/png",
"Content-Length": "10485761",
},
content=PNG,
)
assert response.status_code == 413
async def test_exact_byte_limit_accepts_valid_png(client, context):
_, alice, _, _, chat = context
maximum = 10_485_760
# Валидный ancillary chunk вместо данных после IEND.
payload = bytes(maximum - len(PNG) - 12)
kind = b"npAD"
padding = (
struct.pack("!I", len(payload))
+ kind
+ payload
+ struct.pack("!I", zlib.crc32(kind + payload))
)
image = PNG[:-12] + padding + PNG[-12:]
response = await upload(client, alice, chat, image)
assert response.status_code == 202, response.text
assert response.json()["size_bytes"] == maximum
assert response.json()["sha256"] == hashlib.sha256(image).hexdigest()
async def test_exact_pixel_limit_accepts_valid_png(client, context):
_, alice, _, _, chat = context
output = BytesIO()
with Image.new("RGB", (8000, 5000), "blue") as image:
image.save(output, format="PNG")
response = await upload(client, alice, chat, output.getvalue())
assert response.status_code == 202, response.text
async def test_pixel_limit_before_decoding(client, context):
_, alice, _, _, chat = context
def chunk(kind, data):
return (
struct.pack("!I", len(data))
+ kind
+ data
+ struct.pack("!I", zlib.crc32(kind + data))
)
header = struct.pack("!IIBBBBB", 8000, 5001, 8, 2, 0, 0, 0)
image = (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", header)
+ chunk(b"IDAT", zlib.compress(b"\0"))
+ chunk(b"IEND", b"")
)
response = await upload(client, alice, chat, image)
assert response.status_code == 413, response.text
async def test_unknown_and_other_chat_attachment_are_404(client, context):
_, alice, bob, _, chat = context
attachment = (await upload(client, alice, chat)).json()
other_chat = await create_chat(client, alice, bob)
for id in (attachment["id"], str(uuid4())):
response = await client.post(
f"/api/v1/chats/{other_chat['id']}/messages",
json={"text": "", "attachment_ids": [id]},
headers={**bearer(alice), **idempotency_header()},
)
assert response.status_code == 404
async def test_duplicate_ids_are_400(client, context):
_, alice, _, _, chat = context
id = (await upload(client, alice, chat)).json()["id"]
response = await client.post(
f"/api/v1/chats/{chat['id']}/messages",
json={"text": "", "attachment_ids": [id, id]},
headers={**bearer(alice), **idempotency_header()},
)
assert response.status_code == 400
async def test_empty_ids_and_absent_ids_have_same_idempotency(client, context):
_, alice, _, _, chat = context
headers = {**bearer(alice), **idempotency_header()}
path = f"/api/v1/chats/{chat['id']}/messages"
first = await client.post(path, json={"text": "same"}, headers=headers)
second = await client.post(
path, json={"text": "same", "attachment_ids": []}, headers=headers
)
assert first.status_code == second.status_code == 201
assert first.json() == second.json()
async def test_s3_outage_is_503(client, context):
storage, alice, _, _, chat = context
id = (await upload(client, alice, chat)).json()["id"]
await finish_attachment(client, storage, alice, id)
storage.down = True
response = await client.get(
f"/api/v1/attachments/{id}/content", headers=bearer(alice)
)
assert response.status_code == 503
assert (await client.get("/health/ready")).status_code == 503
async def test_uncertain_put_is_compensated(client, context, db_engine):
storage, alice, _, _, chat = context
storage.fail_put = True
response = await upload(client, alice, chat)
assert response.status_code == 503
assert not storage.objects
async with AsyncSession(db_engine) as session:
assert await session.scalar(select(func.count()).select_from(UploadIntent)) == 0
async def test_failure_after_s3_write_compensates(client, context, monkeypatch):
storage, alice, _, _, chat = context
async def fail(*args, **kwargs):
raise OperationalError("injected", {}, None)
monkeypatch.setattr(repository, "create_attachment", fail)
response = await upload(client, alice, chat)
assert response.status_code == 503
assert not storage.objects
async def test_ambiguous_successful_commit_keeps_original(
client, context, db_engine, monkeypatch
):
storage, alice, _, _, chat = context
original = AsyncSession.commit
count = 0
async def uncertain(session):
nonlocal count
count += 1
await original(session)
if count == 2:
raise OperationalError("lost commit acknowledgement", {}, None)
monkeypatch.setattr(AsyncSession, "commit", uncertain)
response = await upload(client, alice, chat)
assert response.status_code == 503
assert len(storage.objects) == 1
async with AsyncSession(db_engine) as session:
assert await session.scalar(select(func.count()).select_from(Attachment)) == 1
assert await session.scalar(select(func.count()).select_from(UploadIntent)) == 0
async def test_failed_compensation_then_dry_run_and_cleanup(client, context, db_engine):
storage, alice, _, _, chat = context
storage.fail_put = storage.fail_delete = True
assert (await upload(client, alice, chat)).status_code == 503
assert len(storage.objects) == 1
async with AsyncSession(db_engine) as session:
intent = await session.scalar(select(UploadIntent))
intent.created_at = datetime.now(timezone.utc) - timedelta(days=2)
await session.commit()
storage.fail_put = storage.fail_delete = False
async with AsyncSession(db_engine) as session:
dry_run = await domain.reconcile_attachments(session, storage)
assert dry_run.candidates == 1 and dry_run.deleted == 0
assert storage.objects
applied = await domain.reconcile_attachments(session, storage, dry_run=False)
assert applied.deleted == 1 and not storage.objects
repeated = await domain.reconcile_attachments(session, storage, dry_run=False)
assert repeated.deleted == 0
async def test_cleanup_keeps_fresh_and_skips_locked_uploads(client, context, db_engine):
storage, _, _, _, _ = context
old_id, fresh_id = uuid4(), uuid4()
async with AsyncSession(db_engine) as session:
session.add_all(
[
UploadIntent(
id=old_id,
object_key="originals/old",
created_at=datetime.now(timezone.utc) - timedelta(days=2),
),
UploadIntent(id=fresh_id, object_key="originals/fresh"),
]
)
await session.commit()
async with AsyncSession(db_engine) as uploader, AsyncSession(db_engine) as cleaner:
await repository.lock_upload(uploader, old_id)
result = await domain.reconcile_attachments(cleaner, storage, dry_run=False)
assert result.examined == 0
await uploader.rollback()
result = await domain.reconcile_attachments(cleaner, storage, dry_run=False)
assert result.examined == 1
assert await cleaner.get(UploadIntent, fresh_id) is not None
async def test_client_disconnect_before_put_has_no_side_effects(client, context):
storage, alice, _, _, chat = context
async def interrupted():
yield PNG[:16]
raise ClientDisconnect()
response = await upload(client, alice, chat, interrupted())
assert response.status_code == 400
assert not storage.objects
async def test_upload_deadline_cancels_validation(client, context, monkeypatch):
storage, alice, _, _, chat = context
storage.settings = storage.settings.model_copy(
update={"upload_timeout_seconds": 0.02}
)
async def stalled(*args):
await asyncio.sleep(10)
monkeypatch.setattr(domain, "validate_image", stalled)
response = await upload(client, alice, chat)
assert response.status_code == 400
assert not storage.objects
async def test_cancelled_put_is_compensated(client, context, db_engine, monkeypatch):
storage, alice, _, _, chat = context
entered = asyncio.Event()
original = storage.put_file
async def interrupted(*args, **kwargs):
await original(*args, **kwargs)
entered.set()
await asyncio.sleep(10)
monkeypatch.setattr(storage, "put_file", interrupted)
task = asyncio.create_task(upload(client, alice, chat))
await asyncio.wait_for(entered.wait(), timeout=10)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert not storage.objects
async with AsyncSession(db_engine) as session:
assert await session.scalar(select(func.count()).select_from(UploadIntent)) == 0
async def test_cleanup_never_deletes_ready_attachment(client, context, db_engine):
storage, alice, _, _, chat = context
id = (await upload(client, alice, chat)).json()["id"]
await finish_attachment(client, storage, alice, id)
async with AsyncSession(db_engine) as session:
attachment = await session.get(Attachment, UUID(id))
key = attachment.object_key
# Искусственно оставляем старый журнал у уже подтверждённого объекта.
session.add(
UploadIntent(
id=uuid4(),
object_key=key,
created_at=datetime.now(timezone.utc) - timedelta(days=2),
)
)
await session.commit()
result = await domain.reconcile_attachments(session, storage, dry_run=False)
assert result.examined == 1 and result.deleted == 0
assert key in storage.objects