forked from mtusi2026/mtusi2026lab5
390 lines
17 KiB
Python
390 lines
17 KiB
Python
"""API и транзакции с реальной БД; байты S3 подменены управляемым адаптером."""
|
|
|
|
import asyncio
|
|
import hashlib
|
|
from io import BytesIO
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
import struct
|
|
from uuid import UUID, uuid4
|
|
import zlib
|
|
|
|
import pytest
|
|
from PIL import Image
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.exc import OperationalError
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
from starlette.requests import ClientDisconnect
|
|
|
|
from app.core.s3 import S3Settings
|
|
from app.core.jobs import JobSettings
|
|
from app.domain.attachment_jobs import process_attachment_job
|
|
from app.models.job import AttachmentJob
|
|
from app.schemas.job import JobMessage
|
|
from app.domain import attachments as domain
|
|
from app.errors.exceptions import unavailable
|
|
from app.models.attachment import Attachment, UploadIntent
|
|
from app.storage import attachments as repository
|
|
from app.storage.s3 import ObjectInfo
|
|
from helpers import bearer, create_chat, create_user, idempotency_header
|
|
|
|
PNG = Path('tests/fixtures/sample.png').read_bytes()
|
|
|
|
|
|
class MemoryStorage:
|
|
"""Только тестовый адаптер; production использует MinioStorage."""
|
|
|
|
def __init__(self):
|
|
self.settings = S3Settings(_env_file=None)
|
|
self.objects = {}
|
|
self.fail_put = self.fail_delete = self.down = False
|
|
|
|
async def put_file(self, key, path, **metadata):
|
|
self.objects[key] = (path.read_bytes(), metadata)
|
|
if self.fail_put:
|
|
raise unavailable()
|
|
|
|
async def delete_object(self, key):
|
|
if self.fail_delete:
|
|
raise unavailable()
|
|
self.objects.pop(key, None)
|
|
|
|
async def stat_object(self, key):
|
|
if self.down:
|
|
raise unavailable()
|
|
info = None
|
|
if key in self.objects:
|
|
data, metadata = self.objects[key]
|
|
info = ObjectInfo(key, len(data), metadata['content_type'], datetime.now(timezone.utc))
|
|
return info
|
|
|
|
async def open_object(self, key):
|
|
async def chunks():
|
|
yield self.objects[key][0]
|
|
return chunks()
|
|
|
|
async def check_available(self):
|
|
return not self.down
|
|
|
|
|
|
@pytest.fixture
|
|
async def context(client, app):
|
|
storage = MemoryStorage()
|
|
app.state.s3 = storage
|
|
alice, bob, eve = [await create_user(client) for _ in range(3)]
|
|
chat = await create_chat(client, alice, bob)
|
|
return storage, alice, bob, eve, chat
|
|
|
|
|
|
async def upload(client, actor, chat, data=PNG, filename='фото.png', content_type='image/png'):
|
|
return await client.post(
|
|
f"/api/v1/chats/{chat['id']}/attachments", params={'filename': filename},
|
|
headers={**bearer(actor), 'Content-Type': content_type}, content=data,
|
|
)
|
|
|
|
|
|
async def finish_attachment(client, storage, actor, attachment_id):
|
|
engine = client._transport.app.state.engine
|
|
async with AsyncSession(engine) as session:
|
|
job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id)))
|
|
message = JobMessage(job_id=job.id, attachment_id=job.attachment_id,
|
|
pipeline_version=job.pipeline_version, attempt=job.attempt + 1, created_at=job.created_at)
|
|
await session.rollback()
|
|
assert await process_attachment_job(session, storage, message, JobSettings(_env_file=None))
|
|
result = await client.get(f'/api/v1/attachments/{attachment_id}', headers=bearer(actor))
|
|
assert result.json()['status'] == 'ready'
|
|
return result.json()
|
|
|
|
|
|
async def test_upload_download_and_attachment_only_message(client, context):
|
|
storage, alice, bob, eve, chat = context
|
|
response = await upload(client, alice, chat)
|
|
assert response.status_code == 202, response.text
|
|
attachment = response.json()
|
|
assert attachment['status'] == 'queued'
|
|
before = await client.get(f"/api/v1/attachments/{attachment['id']}/content", headers=bearer(alice))
|
|
assert before.status_code == 409
|
|
attachment = await finish_attachment(client, storage, alice, attachment['id'])
|
|
assert attachment['sha256'] == hashlib.sha256(PNG).hexdigest()
|
|
assert {v['name'] for v in attachment['variants']} == {'thumbnail', 'crop', 'optimized'}
|
|
assert attachment['error_code'] is None
|
|
assert 'object_key' not in attachment
|
|
id = attachment['id']
|
|
for actor in (alice, bob):
|
|
content = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(actor))
|
|
assert content.status_code == 200 and content.content == PNG
|
|
assert content.headers['content-type'] == 'image/png'
|
|
key = idempotency_header()
|
|
body = {'text': '', 'attachment_ids': [id]}
|
|
path = f"/api/v1/chats/{chat['id']}/messages"
|
|
message = await client.post(path, json=body, headers={**bearer(bob), **key})
|
|
assert message.status_code == 201, message.text
|
|
repeat = await client.post(path, json=body, headers={**bearer(bob), **key})
|
|
assert repeat.json() == message.json()
|
|
history = await client.get(path, headers=bearer(alice))
|
|
assert history.json()['items'][0]['attachment_ids'] == [id]
|
|
for suffix in ('', '/content'):
|
|
denied = await client.get(f'/api/v1/attachments/{id}{suffix}', headers=bearer(eve))
|
|
assert denied.status_code == 404
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
'data,content_type',
|
|
[(b'not an image', 'image/png'), (PNG[:30], 'image/png'), (PNG, 'image/jpeg'), (PNG, 'application/octet-stream')],
|
|
ids=['not-image', 'truncated-png', 'mime-spoofing', 'unsupported-type'],
|
|
)
|
|
async def test_invalid_image_and_mime_are_415(client, context, data, content_type):
|
|
storage, alice, _, _, chat = context
|
|
response = await upload(client, alice, chat, data, content_type=content_type)
|
|
assert response.status_code == 415, response.text
|
|
assert not storage.objects
|
|
|
|
|
|
async def test_jpeg_is_stored_without_reencoding(client, context):
|
|
storage, alice, _, _, chat = context
|
|
output = BytesIO()
|
|
Image.new('RGB', (32, 32), 'blue').save(output, format='JPEG')
|
|
original = output.getvalue()
|
|
response = await upload(client, alice, chat, original, content_type='image/jpeg')
|
|
assert response.status_code == 202, response.text
|
|
id = response.json()['id']
|
|
await finish_attachment(client, storage, alice, id)
|
|
downloaded = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice))
|
|
assert downloaded.content == original
|
|
|
|
|
|
@pytest.mark.parametrize('filename', ['../a.png', 'a\\b.png', 'a\x00.png', 'a\n.png', 'x' * 256])
|
|
async def test_unsafe_filename_is_400(client, context, filename):
|
|
storage, alice, _, _, chat = context
|
|
response = await upload(client, alice, chat, filename=filename)
|
|
assert response.status_code == 400
|
|
assert not storage.objects
|
|
|
|
|
|
async def test_oversize_chunked_upload_is_413(client, context):
|
|
storage, alice, _, _, chat = context
|
|
async def chunks():
|
|
for _ in range(11):
|
|
yield b'x' * 1048576
|
|
response = await upload(client, alice, chat, chunks())
|
|
assert response.status_code == 413
|
|
assert not storage.objects
|
|
|
|
|
|
async def test_oversize_content_length_is_413(client, context):
|
|
_, alice, _, _, chat = context
|
|
response = await client.post(
|
|
f"/api/v1/chats/{chat['id']}/attachments?filename=a.png",
|
|
headers={**bearer(alice), 'Content-Type': 'image/png', 'Content-Length': '10485761'}, content=PNG,
|
|
)
|
|
assert response.status_code == 413
|
|
|
|
|
|
async def test_exact_byte_limit_accepts_valid_png(client, context):
|
|
_, alice, _, _, chat = context
|
|
maximum = 10_485_760
|
|
# Валидный ancillary chunk вместо данных после IEND.
|
|
payload = bytes(maximum - len(PNG) - 12)
|
|
kind = b'npAD'
|
|
padding = struct.pack('!I', len(payload)) + kind + payload + struct.pack('!I', zlib.crc32(kind + payload))
|
|
image = PNG[:-12] + padding + PNG[-12:]
|
|
response = await upload(client, alice, chat, image)
|
|
assert response.status_code == 202, response.text
|
|
assert response.json()['size_bytes'] == maximum
|
|
assert response.json()['sha256'] == hashlib.sha256(image).hexdigest()
|
|
|
|
|
|
async def test_exact_pixel_limit_accepts_valid_png(client, context):
|
|
_, alice, _, _, chat = context
|
|
output = BytesIO()
|
|
with Image.new('RGB', (8000, 5000), 'blue') as image:
|
|
image.save(output, format='PNG')
|
|
response = await upload(client, alice, chat, output.getvalue())
|
|
assert response.status_code == 202, response.text
|
|
|
|
|
|
async def test_pixel_limit_before_decoding(client, context):
|
|
_, alice, _, _, chat = context
|
|
def chunk(kind, data):
|
|
return struct.pack('!I', len(data)) + kind + data + struct.pack('!I', zlib.crc32(kind + data))
|
|
header = struct.pack('!IIBBBBB', 8000, 5001, 8, 2, 0, 0, 0)
|
|
image = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(b'\0')) + chunk(b'IEND', b'')
|
|
response = await upload(client, alice, chat, image)
|
|
assert response.status_code == 413, response.text
|
|
|
|
|
|
async def test_unknown_and_other_chat_attachment_are_404(client, context):
|
|
_, alice, bob, _, chat = context
|
|
attachment = (await upload(client, alice, chat)).json()
|
|
other_chat = await create_chat(client, alice, bob)
|
|
for id in (attachment['id'], str(uuid4())):
|
|
response = await client.post(
|
|
f"/api/v1/chats/{other_chat['id']}/messages",
|
|
json={'text': '', 'attachment_ids': [id]},
|
|
headers={**bearer(alice), **idempotency_header()},
|
|
)
|
|
assert response.status_code == 404
|
|
|
|
|
|
async def test_duplicate_ids_are_400(client, context):
|
|
_, alice, _, _, chat = context
|
|
id = (await upload(client, alice, chat)).json()['id']
|
|
response = await client.post(f"/api/v1/chats/{chat['id']}/messages",
|
|
json={'text': '', 'attachment_ids': [id, id]},
|
|
headers={**bearer(alice), **idempotency_header()})
|
|
assert response.status_code == 400
|
|
|
|
|
|
async def test_empty_ids_and_absent_ids_have_same_idempotency(client, context):
|
|
_, alice, _, _, chat = context
|
|
headers = {**bearer(alice), **idempotency_header()}
|
|
path = f"/api/v1/chats/{chat['id']}/messages"
|
|
first = await client.post(path, json={'text': 'same'}, headers=headers)
|
|
second = await client.post(path, json={'text': 'same', 'attachment_ids': []}, headers=headers)
|
|
assert first.status_code == second.status_code == 201
|
|
assert first.json() == second.json()
|
|
|
|
|
|
async def test_s3_outage_is_503(client, context):
|
|
storage, alice, _, _, chat = context
|
|
id = (await upload(client, alice, chat)).json()['id']
|
|
await finish_attachment(client, storage, alice, id)
|
|
storage.down = True
|
|
response = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice))
|
|
assert response.status_code == 503
|
|
assert (await client.get('/health/ready')).status_code == 503
|
|
|
|
|
|
async def test_uncertain_put_is_compensated(client, context, db_engine):
|
|
storage, alice, _, _, chat = context
|
|
storage.fail_put = True
|
|
response = await upload(client, alice, chat)
|
|
assert response.status_code == 503
|
|
assert not storage.objects
|
|
async with AsyncSession(db_engine) as session:
|
|
assert await session.scalar(select(func.count()).select_from(UploadIntent)) == 0
|
|
|
|
|
|
async def test_failure_after_s3_write_compensates(client, context, monkeypatch):
|
|
storage, alice, _, _, chat = context
|
|
async def fail(*args, **kwargs):
|
|
raise OperationalError('injected', {}, None)
|
|
monkeypatch.setattr(repository, 'create_attachment', fail)
|
|
response = await upload(client, alice, chat)
|
|
assert response.status_code == 503
|
|
assert not storage.objects
|
|
|
|
|
|
async def test_ambiguous_successful_commit_keeps_original(client, context, db_engine, monkeypatch):
|
|
storage, alice, _, _, chat = context
|
|
original = AsyncSession.commit
|
|
count = 0
|
|
async def uncertain(session):
|
|
nonlocal count
|
|
count += 1
|
|
await original(session)
|
|
if count == 2:
|
|
raise OperationalError('lost commit acknowledgement', {}, None)
|
|
monkeypatch.setattr(AsyncSession, 'commit', uncertain)
|
|
response = await upload(client, alice, chat)
|
|
assert response.status_code == 503
|
|
assert len(storage.objects) == 1
|
|
async with AsyncSession(db_engine) as session:
|
|
assert await session.scalar(select(func.count()).select_from(Attachment)) == 1
|
|
assert await session.scalar(select(func.count()).select_from(UploadIntent)) == 0
|
|
|
|
|
|
async def test_failed_compensation_then_dry_run_and_cleanup(client, context, db_engine):
|
|
storage, alice, _, _, chat = context
|
|
storage.fail_put = storage.fail_delete = True
|
|
assert (await upload(client, alice, chat)).status_code == 503
|
|
assert len(storage.objects) == 1
|
|
async with AsyncSession(db_engine) as session:
|
|
intent = await session.scalar(select(UploadIntent))
|
|
intent.created_at = datetime.now(timezone.utc) - timedelta(days=2)
|
|
await session.commit()
|
|
storage.fail_put = storage.fail_delete = False
|
|
async with AsyncSession(db_engine) as session:
|
|
dry_run = await domain.reconcile_attachments(session, storage)
|
|
assert dry_run.candidates == 1 and dry_run.deleted == 0
|
|
assert storage.objects
|
|
applied = await domain.reconcile_attachments(session, storage, dry_run=False)
|
|
assert applied.deleted == 1 and not storage.objects
|
|
repeated = await domain.reconcile_attachments(session, storage, dry_run=False)
|
|
assert repeated.deleted == 0
|
|
|
|
|
|
async def test_cleanup_keeps_fresh_and_skips_locked_uploads(client, context, db_engine):
|
|
storage, _, _, _, _ = context
|
|
old_id, fresh_id = uuid4(), uuid4()
|
|
async with AsyncSession(db_engine) as session:
|
|
session.add_all([
|
|
UploadIntent(id=old_id, object_key='originals/old', created_at=datetime.now(timezone.utc) - timedelta(days=2)),
|
|
UploadIntent(id=fresh_id, object_key='originals/fresh'),
|
|
])
|
|
await session.commit()
|
|
async with AsyncSession(db_engine) as uploader, AsyncSession(db_engine) as cleaner:
|
|
await repository.lock_upload(uploader, old_id)
|
|
result = await domain.reconcile_attachments(cleaner, storage, dry_run=False)
|
|
assert result.examined == 0
|
|
await uploader.rollback()
|
|
result = await domain.reconcile_attachments(cleaner, storage, dry_run=False)
|
|
assert result.examined == 1
|
|
assert await cleaner.get(UploadIntent, fresh_id) is not None
|
|
|
|
|
|
async def test_client_disconnect_before_put_has_no_side_effects(client, context):
|
|
storage, alice, _, _, chat = context
|
|
async def interrupted():
|
|
yield PNG[:16]
|
|
raise ClientDisconnect()
|
|
response = await upload(client, alice, chat, interrupted())
|
|
assert response.status_code == 400
|
|
assert not storage.objects
|
|
|
|
|
|
async def test_upload_deadline_cancels_validation(client, context, monkeypatch):
|
|
storage, alice, _, _, chat = context
|
|
storage.settings = storage.settings.model_copy(update={'upload_timeout_seconds': 0.02})
|
|
async def stalled(*args):
|
|
await asyncio.sleep(10)
|
|
monkeypatch.setattr(domain, 'validate_image', stalled)
|
|
response = await upload(client, alice, chat)
|
|
assert response.status_code == 400
|
|
assert not storage.objects
|
|
|
|
|
|
async def test_cancelled_put_is_compensated(client, context, db_engine, monkeypatch):
|
|
storage, alice, _, _, chat = context
|
|
entered = asyncio.Event()
|
|
original = storage.put_file
|
|
async def interrupted(*args, **kwargs):
|
|
await original(*args, **kwargs)
|
|
entered.set()
|
|
await asyncio.sleep(10)
|
|
monkeypatch.setattr(storage, 'put_file', interrupted)
|
|
task = asyncio.create_task(upload(client, alice, chat))
|
|
await asyncio.wait_for(entered.wait(), timeout=10)
|
|
task.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
assert not storage.objects
|
|
async with AsyncSession(db_engine) as session:
|
|
assert await session.scalar(select(func.count()).select_from(UploadIntent)) == 0
|
|
|
|
|
|
async def test_cleanup_never_deletes_ready_attachment(client, context, db_engine):
|
|
storage, alice, _, _, chat = context
|
|
id = (await upload(client, alice, chat)).json()['id']
|
|
await finish_attachment(client, storage, alice, id)
|
|
async with AsyncSession(db_engine) as session:
|
|
attachment = await session.get(Attachment, UUID(id))
|
|
key = attachment.object_key
|
|
# Искусственно оставляем старый журнал у уже подтверждённого объекта.
|
|
session.add(UploadIntent(id=uuid4(), object_key=key,
|
|
created_at=datetime.now(timezone.utc) - timedelta(days=2)))
|
|
await session.commit()
|
|
result = await domain.reconcile_attachments(session, storage, dry_run=False)
|
|
assert result.examined == 1 and result.deleted == 0
|
|
assert key in storage.objects
|