forked from mtusi2026/mtusi2026lab5
484 lines
17 KiB
Python
484 lines
17 KiB
Python
"""API и транзакции с реальной БД; байты S3 подменены управляемым адаптером."""
|
|
|
|
import asyncio
|
|
import hashlib
|
|
from io import BytesIO
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
import struct
|
|
from uuid import UUID, uuid4
|
|
import zlib
|
|
|
|
import pytest
|
|
from PIL import Image
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.exc import OperationalError
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
from starlette.requests import ClientDisconnect
|
|
|
|
from app.core.s3 import S3Settings
|
|
from app.core.jobs import JobSettings
|
|
from app.domain.attachment_jobs import process_attachment_job
|
|
from app.models.job import AttachmentJob
|
|
from app.schemas.job import JobMessage
|
|
from app.domain import attachments as domain
|
|
from app.errors.exceptions import unavailable
|
|
from app.models.attachment import Attachment, UploadIntent
|
|
from app.storage import attachments as repository
|
|
from app.storage.s3 import ObjectInfo
|
|
from helpers import bearer, create_chat, create_user, idempotency_header
|
|
|
|
PNG = Path("tests/fixtures/sample.png").read_bytes()
|
|
|
|
|
|
class MemoryStorage:
|
|
"""Только тестовый адаптер; production использует MinioStorage."""
|
|
|
|
def __init__(self):
|
|
self.settings = S3Settings(_env_file=None)
|
|
self.objects = {}
|
|
self.fail_put = self.fail_delete = self.down = False
|
|
|
|
async def put_file(self, key, path, **metadata):
|
|
self.objects[key] = (path.read_bytes(), metadata)
|
|
if self.fail_put:
|
|
raise unavailable()
|
|
|
|
async def delete_object(self, key):
|
|
if self.fail_delete:
|
|
raise unavailable()
|
|
self.objects.pop(key, None)
|
|
|
|
async def stat_object(self, key):
|
|
if self.down:
|
|
raise unavailable()
|
|
info = None
|
|
if key in self.objects:
|
|
data, metadata = self.objects[key]
|
|
info = ObjectInfo(
|
|
key, len(data), metadata["content_type"], datetime.now(timezone.utc)
|
|
)
|
|
return info
|
|
|
|
async def open_object(self, key):
|
|
async def chunks():
|
|
yield self.objects[key][0]
|
|
|
|
return chunks()
|
|
|
|
async def check_available(self):
|
|
return not self.down
|
|
|
|
|
|
@pytest.fixture
|
|
async def context(client, app):
|
|
storage = MemoryStorage()
|
|
app.state.s3 = storage
|
|
alice, bob, eve = [await create_user(client) for _ in range(3)]
|
|
chat = await create_chat(client, alice, bob)
|
|
return storage, alice, bob, eve, chat
|
|
|
|
|
|
async def upload(
|
|
client, actor, chat, data=PNG, filename="фото.png", content_type="image/png"
|
|
):
|
|
return await client.post(
|
|
f"/api/v1/chats/{chat['id']}/attachments",
|
|
params={"filename": filename},
|
|
headers={**bearer(actor), "Content-Type": content_type},
|
|
content=data,
|
|
)
|
|
|
|
|
|
async def finish_attachment(client, storage, actor, attachment_id):
|
|
engine = client._transport.app.state.engine
|
|
async with AsyncSession(engine) as session:
|
|
job = await session.scalar(
|
|
select(AttachmentJob).where(
|
|
AttachmentJob.attachment_id == UUID(attachment_id)
|
|
)
|
|
)
|
|
message = JobMessage(
|
|
job_id=job.id,
|
|
attachment_id=job.attachment_id,
|
|
pipeline_version=job.pipeline_version,
|
|
attempt=job.attempt + 1,
|
|
created_at=job.created_at,
|
|
)
|
|
await session.rollback()
|
|
assert await process_attachment_job(
|
|
session, storage, message, JobSettings(_env_file=None)
|
|
)
|
|
result = await client.get(
|
|
f"/api/v1/attachments/{attachment_id}", headers=bearer(actor)
|
|
)
|
|
assert result.json()["status"] == "ready"
|
|
return result.json()
|
|
|
|
|
|
async def test_upload_download_and_attachment_only_message(client, context):
|
|
storage, alice, bob, eve, chat = context
|
|
response = await upload(client, alice, chat)
|
|
assert response.status_code == 202, response.text
|
|
attachment = response.json()
|
|
assert attachment["status"] == "queued"
|
|
before = await client.get(
|
|
f"/api/v1/attachments/{attachment['id']}/content", headers=bearer(alice)
|
|
)
|
|
assert before.status_code == 409
|
|
attachment = await finish_attachment(client, storage, alice, attachment["id"])
|
|
assert attachment["sha256"] == hashlib.sha256(PNG).hexdigest()
|
|
assert {v["name"] for v in attachment["variants"]} == {
|
|
"thumbnail",
|
|
"crop",
|
|
"optimized",
|
|
}
|
|
assert attachment["error_code"] is None
|
|
assert "object_key" not in attachment
|
|
id = attachment["id"]
|
|
for actor in (alice, bob):
|
|
content = await client.get(
|
|
f"/api/v1/attachments/{id}/content", headers=bearer(actor)
|
|
)
|
|
assert content.status_code == 200 and content.content == PNG
|
|
assert content.headers["content-type"] == "image/png"
|
|
key = idempotency_header()
|
|
body = {"text": "", "attachment_ids": [id]}
|
|
path = f"/api/v1/chats/{chat['id']}/messages"
|
|
message = await client.post(path, json=body, headers={**bearer(bob), **key})
|
|
assert message.status_code == 201, message.text
|
|
repeat = await client.post(path, json=body, headers={**bearer(bob), **key})
|
|
assert repeat.json() == message.json()
|
|
history = await client.get(path, headers=bearer(alice))
|
|
assert history.json()["items"][0]["attachment_ids"] == [id]
|
|
for suffix in ("", "/content"):
|
|
denied = await client.get(
|
|
f"/api/v1/attachments/{id}{suffix}", headers=bearer(eve)
|
|
)
|
|
assert denied.status_code == 404
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"data,content_type",
|
|
[
|
|
(b"not an image", "image/png"),
|
|
(PNG[:30], "image/png"),
|
|
(PNG, "image/jpeg"),
|
|
(PNG, "application/octet-stream"),
|
|
],
|
|
ids=["not-image", "truncated-png", "mime-spoofing", "unsupported-type"],
|
|
)
|
|
async def test_invalid_image_and_mime_are_415(client, context, data, content_type):
|
|
storage, alice, _, _, chat = context
|
|
response = await upload(client, alice, chat, data, content_type=content_type)
|
|
assert response.status_code == 415, response.text
|
|
assert not storage.objects
|
|
|
|
|
|
async def test_jpeg_is_stored_without_reencoding(client, context):
|
|
storage, alice, _, _, chat = context
|
|
output = BytesIO()
|
|
Image.new("RGB", (32, 32), "blue").save(output, format="JPEG")
|
|
original = output.getvalue()
|
|
response = await upload(client, alice, chat, original, content_type="image/jpeg")
|
|
assert response.status_code == 202, response.text
|
|
id = response.json()["id"]
|
|
await finish_attachment(client, storage, alice, id)
|
|
downloaded = await client.get(
|
|
f"/api/v1/attachments/{id}/content", headers=bearer(alice)
|
|
)
|
|
assert downloaded.content == original
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"filename", ["../a.png", "a\\b.png", "a\x00.png", "a\n.png", "x" * 256]
|
|
)
|
|
async def test_unsafe_filename_is_400(client, context, filename):
|
|
storage, alice, _, _, chat = context
|
|
response = await upload(client, alice, chat, filename=filename)
|
|
assert response.status_code == 400
|
|
assert not storage.objects
|
|
|
|
|
|
async def test_oversize_chunked_upload_is_413(client, context):
|
|
storage, alice, _, _, chat = context
|
|
|
|
async def chunks():
|
|
for _ in range(11):
|
|
yield b"x" * 1048576
|
|
|
|
response = await upload(client, alice, chat, chunks())
|
|
assert response.status_code == 413
|
|
assert not storage.objects
|
|
|
|
|
|
async def test_oversize_content_length_is_413(client, context):
|
|
_, alice, _, _, chat = context
|
|
response = await client.post(
|
|
f"/api/v1/chats/{chat['id']}/attachments?filename=a.png",
|
|
headers={
|
|
**bearer(alice),
|
|
"Content-Type": "image/png",
|
|
"Content-Length": "10485761",
|
|
},
|
|
content=PNG,
|
|
)
|
|
assert response.status_code == 413
|
|
|
|
|
|
async def test_exact_byte_limit_accepts_valid_png(client, context):
|
|
_, alice, _, _, chat = context
|
|
maximum = 10_485_760
|
|
# Валидный ancillary chunk вместо данных после IEND.
|
|
payload = bytes(maximum - len(PNG) - 12)
|
|
kind = b"npAD"
|
|
padding = (
|
|
struct.pack("!I", len(payload))
|
|
+ kind
|
|
+ payload
|
|
+ struct.pack("!I", zlib.crc32(kind + payload))
|
|
)
|
|
image = PNG[:-12] + padding + PNG[-12:]
|
|
response = await upload(client, alice, chat, image)
|
|
assert response.status_code == 202, response.text
|
|
assert response.json()["size_bytes"] == maximum
|
|
assert response.json()["sha256"] == hashlib.sha256(image).hexdigest()
|
|
|
|
|
|
async def test_exact_pixel_limit_accepts_valid_png(client, context):
|
|
_, alice, _, _, chat = context
|
|
output = BytesIO()
|
|
with Image.new("RGB", (8000, 5000), "blue") as image:
|
|
image.save(output, format="PNG")
|
|
response = await upload(client, alice, chat, output.getvalue())
|
|
assert response.status_code == 202, response.text
|
|
|
|
|
|
async def test_pixel_limit_before_decoding(client, context):
|
|
_, alice, _, _, chat = context
|
|
|
|
def chunk(kind, data):
|
|
return (
|
|
struct.pack("!I", len(data))
|
|
+ kind
|
|
+ data
|
|
+ struct.pack("!I", zlib.crc32(kind + data))
|
|
)
|
|
|
|
header = struct.pack("!IIBBBBB", 8000, 5001, 8, 2, 0, 0, 0)
|
|
image = (
|
|
b"\x89PNG\r\n\x1a\n"
|
|
+ chunk(b"IHDR", header)
|
|
+ chunk(b"IDAT", zlib.compress(b"\0"))
|
|
+ chunk(b"IEND", b"")
|
|
)
|
|
response = await upload(client, alice, chat, image)
|
|
assert response.status_code == 413, response.text
|
|
|
|
|
|
async def test_unknown_and_other_chat_attachment_are_404(client, context):
|
|
_, alice, bob, _, chat = context
|
|
attachment = (await upload(client, alice, chat)).json()
|
|
other_chat = await create_chat(client, alice, bob)
|
|
for id in (attachment["id"], str(uuid4())):
|
|
response = await client.post(
|
|
f"/api/v1/chats/{other_chat['id']}/messages",
|
|
json={"text": "", "attachment_ids": [id]},
|
|
headers={**bearer(alice), **idempotency_header()},
|
|
)
|
|
assert response.status_code == 404
|
|
|
|
|
|
async def test_duplicate_ids_are_400(client, context):
|
|
_, alice, _, _, chat = context
|
|
id = (await upload(client, alice, chat)).json()["id"]
|
|
response = await client.post(
|
|
f"/api/v1/chats/{chat['id']}/messages",
|
|
json={"text": "", "attachment_ids": [id, id]},
|
|
headers={**bearer(alice), **idempotency_header()},
|
|
)
|
|
assert response.status_code == 400
|
|
|
|
|
|
async def test_empty_ids_and_absent_ids_have_same_idempotency(client, context):
|
|
_, alice, _, _, chat = context
|
|
headers = {**bearer(alice), **idempotency_header()}
|
|
path = f"/api/v1/chats/{chat['id']}/messages"
|
|
first = await client.post(path, json={"text": "same"}, headers=headers)
|
|
second = await client.post(
|
|
path, json={"text": "same", "attachment_ids": []}, headers=headers
|
|
)
|
|
assert first.status_code == second.status_code == 201
|
|
assert first.json() == second.json()
|
|
|
|
|
|
async def test_s3_outage_is_503(client, context):
|
|
storage, alice, _, _, chat = context
|
|
id = (await upload(client, alice, chat)).json()["id"]
|
|
await finish_attachment(client, storage, alice, id)
|
|
storage.down = True
|
|
response = await client.get(
|
|
f"/api/v1/attachments/{id}/content", headers=bearer(alice)
|
|
)
|
|
assert response.status_code == 503
|
|
assert (await client.get("/health/ready")).status_code == 503
|
|
|
|
|
|
async def test_uncertain_put_is_compensated(client, context, db_engine):
|
|
storage, alice, _, _, chat = context
|
|
storage.fail_put = True
|
|
response = await upload(client, alice, chat)
|
|
assert response.status_code == 503
|
|
assert not storage.objects
|
|
async with AsyncSession(db_engine) as session:
|
|
assert await session.scalar(select(func.count()).select_from(UploadIntent)) == 0
|
|
|
|
|
|
async def test_failure_after_s3_write_compensates(client, context, monkeypatch):
|
|
storage, alice, _, _, chat = context
|
|
|
|
async def fail(*args, **kwargs):
|
|
raise OperationalError("injected", {}, None)
|
|
|
|
monkeypatch.setattr(repository, "create_attachment", fail)
|
|
response = await upload(client, alice, chat)
|
|
assert response.status_code == 503
|
|
assert not storage.objects
|
|
|
|
|
|
async def test_ambiguous_successful_commit_keeps_original(
|
|
client, context, db_engine, monkeypatch
|
|
):
|
|
storage, alice, _, _, chat = context
|
|
original = AsyncSession.commit
|
|
count = 0
|
|
|
|
async def uncertain(session):
|
|
nonlocal count
|
|
count += 1
|
|
await original(session)
|
|
if count == 2:
|
|
raise OperationalError("lost commit acknowledgement", {}, None)
|
|
|
|
monkeypatch.setattr(AsyncSession, "commit", uncertain)
|
|
response = await upload(client, alice, chat)
|
|
assert response.status_code == 503
|
|
assert len(storage.objects) == 1
|
|
async with AsyncSession(db_engine) as session:
|
|
assert await session.scalar(select(func.count()).select_from(Attachment)) == 1
|
|
assert await session.scalar(select(func.count()).select_from(UploadIntent)) == 0
|
|
|
|
|
|
async def test_failed_compensation_then_dry_run_and_cleanup(client, context, db_engine):
|
|
storage, alice, _, _, chat = context
|
|
storage.fail_put = storage.fail_delete = True
|
|
assert (await upload(client, alice, chat)).status_code == 503
|
|
assert len(storage.objects) == 1
|
|
async with AsyncSession(db_engine) as session:
|
|
intent = await session.scalar(select(UploadIntent))
|
|
intent.created_at = datetime.now(timezone.utc) - timedelta(days=2)
|
|
await session.commit()
|
|
storage.fail_put = storage.fail_delete = False
|
|
async with AsyncSession(db_engine) as session:
|
|
dry_run = await domain.reconcile_attachments(session, storage)
|
|
assert dry_run.candidates == 1 and dry_run.deleted == 0
|
|
assert storage.objects
|
|
applied = await domain.reconcile_attachments(session, storage, dry_run=False)
|
|
assert applied.deleted == 1 and not storage.objects
|
|
repeated = await domain.reconcile_attachments(session, storage, dry_run=False)
|
|
assert repeated.deleted == 0
|
|
|
|
|
|
async def test_cleanup_keeps_fresh_and_skips_locked_uploads(client, context, db_engine):
|
|
storage, _, _, _, _ = context
|
|
old_id, fresh_id = uuid4(), uuid4()
|
|
async with AsyncSession(db_engine) as session:
|
|
session.add_all(
|
|
[
|
|
UploadIntent(
|
|
id=old_id,
|
|
object_key="originals/old",
|
|
created_at=datetime.now(timezone.utc) - timedelta(days=2),
|
|
),
|
|
UploadIntent(id=fresh_id, object_key="originals/fresh"),
|
|
]
|
|
)
|
|
await session.commit()
|
|
async with AsyncSession(db_engine) as uploader, AsyncSession(db_engine) as cleaner:
|
|
await repository.lock_upload(uploader, old_id)
|
|
result = await domain.reconcile_attachments(cleaner, storage, dry_run=False)
|
|
assert result.examined == 0
|
|
await uploader.rollback()
|
|
result = await domain.reconcile_attachments(cleaner, storage, dry_run=False)
|
|
assert result.examined == 1
|
|
assert await cleaner.get(UploadIntent, fresh_id) is not None
|
|
|
|
|
|
async def test_client_disconnect_before_put_has_no_side_effects(client, context):
|
|
storage, alice, _, _, chat = context
|
|
|
|
async def interrupted():
|
|
yield PNG[:16]
|
|
raise ClientDisconnect()
|
|
|
|
response = await upload(client, alice, chat, interrupted())
|
|
assert response.status_code == 400
|
|
assert not storage.objects
|
|
|
|
|
|
async def test_upload_deadline_cancels_validation(client, context, monkeypatch):
|
|
storage, alice, _, _, chat = context
|
|
storage.settings = storage.settings.model_copy(
|
|
update={"upload_timeout_seconds": 0.02}
|
|
)
|
|
|
|
async def stalled(*args):
|
|
await asyncio.sleep(10)
|
|
|
|
monkeypatch.setattr(domain, "validate_image", stalled)
|
|
response = await upload(client, alice, chat)
|
|
assert response.status_code == 400
|
|
assert not storage.objects
|
|
|
|
|
|
async def test_cancelled_put_is_compensated(client, context, db_engine, monkeypatch):
|
|
storage, alice, _, _, chat = context
|
|
entered = asyncio.Event()
|
|
original = storage.put_file
|
|
|
|
async def interrupted(*args, **kwargs):
|
|
await original(*args, **kwargs)
|
|
entered.set()
|
|
await asyncio.sleep(10)
|
|
|
|
monkeypatch.setattr(storage, "put_file", interrupted)
|
|
task = asyncio.create_task(upload(client, alice, chat))
|
|
await asyncio.wait_for(entered.wait(), timeout=10)
|
|
task.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
assert not storage.objects
|
|
async with AsyncSession(db_engine) as session:
|
|
assert await session.scalar(select(func.count()).select_from(UploadIntent)) == 0
|
|
|
|
|
|
async def test_cleanup_never_deletes_ready_attachment(client, context, db_engine):
|
|
storage, alice, _, _, chat = context
|
|
id = (await upload(client, alice, chat)).json()["id"]
|
|
await finish_attachment(client, storage, alice, id)
|
|
async with AsyncSession(db_engine) as session:
|
|
attachment = await session.get(Attachment, UUID(id))
|
|
key = attachment.object_key
|
|
# Искусственно оставляем старый журнал у уже подтверждённого объекта.
|
|
session.add(
|
|
UploadIntent(
|
|
id=uuid4(),
|
|
object_key=key,
|
|
created_at=datetime.now(timezone.utc) - timedelta(days=2),
|
|
)
|
|
)
|
|
await session.commit()
|
|
result = await domain.reconcile_attachments(session, storage, dry_run=False)
|
|
assert result.examined == 1 and result.deleted == 0
|
|
assert key in storage.objects
|