formatted project by python-black
This commit is contained in:
@@ -28,7 +28,7 @@ from app.storage import attachments as repository
|
||||
from app.storage.s3 import ObjectInfo
|
||||
from helpers import bearer, create_chat, create_user, idempotency_header
|
||||
|
||||
PNG = Path('tests/fixtures/sample.png').read_bytes()
|
||||
PNG = Path("tests/fixtures/sample.png").read_bytes()
|
||||
|
||||
|
||||
class MemoryStorage:
|
||||
@@ -55,12 +55,15 @@ class MemoryStorage:
|
||||
info = None
|
||||
if key in self.objects:
|
||||
data, metadata = self.objects[key]
|
||||
info = ObjectInfo(key, len(data), metadata['content_type'], datetime.now(timezone.utc))
|
||||
info = ObjectInfo(
|
||||
key, len(data), metadata["content_type"], datetime.now(timezone.utc)
|
||||
)
|
||||
return info
|
||||
|
||||
async def open_object(self, key):
|
||||
async def chunks():
|
||||
yield self.objects[key][0]
|
||||
|
||||
return chunks()
|
||||
|
||||
async def check_available(self):
|
||||
@@ -76,23 +79,40 @@ async def context(client, app):
|
||||
return storage, alice, bob, eve, chat
|
||||
|
||||
|
||||
async def upload(client, actor, chat, data=PNG, filename='фото.png', content_type='image/png'):
|
||||
async def upload(
|
||||
client, actor, chat, data=PNG, filename="фото.png", content_type="image/png"
|
||||
):
|
||||
return await client.post(
|
||||
f"/api/v1/chats/{chat['id']}/attachments", params={'filename': filename},
|
||||
headers={**bearer(actor), 'Content-Type': content_type}, content=data,
|
||||
f"/api/v1/chats/{chat['id']}/attachments",
|
||||
params={"filename": filename},
|
||||
headers={**bearer(actor), "Content-Type": content_type},
|
||||
content=data,
|
||||
)
|
||||
|
||||
|
||||
async def finish_attachment(client, storage, actor, attachment_id):
|
||||
engine = client._transport.app.state.engine
|
||||
async with AsyncSession(engine) as session:
|
||||
job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id)))
|
||||
message = JobMessage(job_id=job.id, attachment_id=job.attachment_id,
|
||||
pipeline_version=job.pipeline_version, attempt=job.attempt + 1, created_at=job.created_at)
|
||||
job = await session.scalar(
|
||||
select(AttachmentJob).where(
|
||||
AttachmentJob.attachment_id == UUID(attachment_id)
|
||||
)
|
||||
)
|
||||
message = JobMessage(
|
||||
job_id=job.id,
|
||||
attachment_id=job.attachment_id,
|
||||
pipeline_version=job.pipeline_version,
|
||||
attempt=job.attempt + 1,
|
||||
created_at=job.created_at,
|
||||
)
|
||||
await session.rollback()
|
||||
assert await process_attachment_job(session, storage, message, JobSettings(_env_file=None))
|
||||
result = await client.get(f'/api/v1/attachments/{attachment_id}', headers=bearer(actor))
|
||||
assert result.json()['status'] == 'ready'
|
||||
assert await process_attachment_job(
|
||||
session, storage, message, JobSettings(_env_file=None)
|
||||
)
|
||||
result = await client.get(
|
||||
f"/api/v1/attachments/{attachment_id}", headers=bearer(actor)
|
||||
)
|
||||
assert result.json()["status"] == "ready"
|
||||
return result.json()
|
||||
|
||||
|
||||
@@ -101,37 +121,52 @@ async def test_upload_download_and_attachment_only_message(client, context):
|
||||
response = await upload(client, alice, chat)
|
||||
assert response.status_code == 202, response.text
|
||||
attachment = response.json()
|
||||
assert attachment['status'] == 'queued'
|
||||
before = await client.get(f"/api/v1/attachments/{attachment['id']}/content", headers=bearer(alice))
|
||||
assert attachment["status"] == "queued"
|
||||
before = await client.get(
|
||||
f"/api/v1/attachments/{attachment['id']}/content", headers=bearer(alice)
|
||||
)
|
||||
assert before.status_code == 409
|
||||
attachment = await finish_attachment(client, storage, alice, attachment['id'])
|
||||
assert attachment['sha256'] == hashlib.sha256(PNG).hexdigest()
|
||||
assert {v['name'] for v in attachment['variants']} == {'thumbnail', 'crop', 'optimized'}
|
||||
assert attachment['error_code'] is None
|
||||
assert 'object_key' not in attachment
|
||||
id = attachment['id']
|
||||
attachment = await finish_attachment(client, storage, alice, attachment["id"])
|
||||
assert attachment["sha256"] == hashlib.sha256(PNG).hexdigest()
|
||||
assert {v["name"] for v in attachment["variants"]} == {
|
||||
"thumbnail",
|
||||
"crop",
|
||||
"optimized",
|
||||
}
|
||||
assert attachment["error_code"] is None
|
||||
assert "object_key" not in attachment
|
||||
id = attachment["id"]
|
||||
for actor in (alice, bob):
|
||||
content = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(actor))
|
||||
content = await client.get(
|
||||
f"/api/v1/attachments/{id}/content", headers=bearer(actor)
|
||||
)
|
||||
assert content.status_code == 200 and content.content == PNG
|
||||
assert content.headers['content-type'] == 'image/png'
|
||||
assert content.headers["content-type"] == "image/png"
|
||||
key = idempotency_header()
|
||||
body = {'text': '', 'attachment_ids': [id]}
|
||||
body = {"text": "", "attachment_ids": [id]}
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
message = await client.post(path, json=body, headers={**bearer(bob), **key})
|
||||
assert message.status_code == 201, message.text
|
||||
repeat = await client.post(path, json=body, headers={**bearer(bob), **key})
|
||||
assert repeat.json() == message.json()
|
||||
history = await client.get(path, headers=bearer(alice))
|
||||
assert history.json()['items'][0]['attachment_ids'] == [id]
|
||||
for suffix in ('', '/content'):
|
||||
denied = await client.get(f'/api/v1/attachments/{id}{suffix}', headers=bearer(eve))
|
||||
assert history.json()["items"][0]["attachment_ids"] == [id]
|
||||
for suffix in ("", "/content"):
|
||||
denied = await client.get(
|
||||
f"/api/v1/attachments/{id}{suffix}", headers=bearer(eve)
|
||||
)
|
||||
assert denied.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'data,content_type',
|
||||
[(b'not an image', 'image/png'), (PNG[:30], 'image/png'), (PNG, 'image/jpeg'), (PNG, 'application/octet-stream')],
|
||||
ids=['not-image', 'truncated-png', 'mime-spoofing', 'unsupported-type'],
|
||||
"data,content_type",
|
||||
[
|
||||
(b"not an image", "image/png"),
|
||||
(PNG[:30], "image/png"),
|
||||
(PNG, "image/jpeg"),
|
||||
(PNG, "application/octet-stream"),
|
||||
],
|
||||
ids=["not-image", "truncated-png", "mime-spoofing", "unsupported-type"],
|
||||
)
|
||||
async def test_invalid_image_and_mime_are_415(client, context, data, content_type):
|
||||
storage, alice, _, _, chat = context
|
||||
@@ -143,17 +178,21 @@ async def test_invalid_image_and_mime_are_415(client, context, data, content_typ
|
||||
async def test_jpeg_is_stored_without_reencoding(client, context):
|
||||
storage, alice, _, _, chat = context
|
||||
output = BytesIO()
|
||||
Image.new('RGB', (32, 32), 'blue').save(output, format='JPEG')
|
||||
Image.new("RGB", (32, 32), "blue").save(output, format="JPEG")
|
||||
original = output.getvalue()
|
||||
response = await upload(client, alice, chat, original, content_type='image/jpeg')
|
||||
response = await upload(client, alice, chat, original, content_type="image/jpeg")
|
||||
assert response.status_code == 202, response.text
|
||||
id = response.json()['id']
|
||||
id = response.json()["id"]
|
||||
await finish_attachment(client, storage, alice, id)
|
||||
downloaded = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice))
|
||||
downloaded = await client.get(
|
||||
f"/api/v1/attachments/{id}/content", headers=bearer(alice)
|
||||
)
|
||||
assert downloaded.content == original
|
||||
|
||||
|
||||
@pytest.mark.parametrize('filename', ['../a.png', 'a\\b.png', 'a\x00.png', 'a\n.png', 'x' * 256])
|
||||
@pytest.mark.parametrize(
|
||||
"filename", ["../a.png", "a\\b.png", "a\x00.png", "a\n.png", "x" * 256]
|
||||
)
|
||||
async def test_unsafe_filename_is_400(client, context, filename):
|
||||
storage, alice, _, _, chat = context
|
||||
response = await upload(client, alice, chat, filename=filename)
|
||||
@@ -163,9 +202,11 @@ async def test_unsafe_filename_is_400(client, context, filename):
|
||||
|
||||
async def test_oversize_chunked_upload_is_413(client, context):
|
||||
storage, alice, _, _, chat = context
|
||||
|
||||
async def chunks():
|
||||
for _ in range(11):
|
||||
yield b'x' * 1048576
|
||||
yield b"x" * 1048576
|
||||
|
||||
response = await upload(client, alice, chat, chunks())
|
||||
assert response.status_code == 413
|
||||
assert not storage.objects
|
||||
@@ -175,7 +216,12 @@ async def test_oversize_content_length_is_413(client, context):
|
||||
_, alice, _, _, chat = context
|
||||
response = await client.post(
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename=a.png",
|
||||
headers={**bearer(alice), 'Content-Type': 'image/png', 'Content-Length': '10485761'}, content=PNG,
|
||||
headers={
|
||||
**bearer(alice),
|
||||
"Content-Type": "image/png",
|
||||
"Content-Length": "10485761",
|
||||
},
|
||||
content=PNG,
|
||||
)
|
||||
assert response.status_code == 413
|
||||
|
||||
@@ -185,30 +231,47 @@ async def test_exact_byte_limit_accepts_valid_png(client, context):
|
||||
maximum = 10_485_760
|
||||
# Валидный ancillary chunk вместо данных после IEND.
|
||||
payload = bytes(maximum - len(PNG) - 12)
|
||||
kind = b'npAD'
|
||||
padding = struct.pack('!I', len(payload)) + kind + payload + struct.pack('!I', zlib.crc32(kind + payload))
|
||||
kind = b"npAD"
|
||||
padding = (
|
||||
struct.pack("!I", len(payload))
|
||||
+ kind
|
||||
+ payload
|
||||
+ struct.pack("!I", zlib.crc32(kind + payload))
|
||||
)
|
||||
image = PNG[:-12] + padding + PNG[-12:]
|
||||
response = await upload(client, alice, chat, image)
|
||||
assert response.status_code == 202, response.text
|
||||
assert response.json()['size_bytes'] == maximum
|
||||
assert response.json()['sha256'] == hashlib.sha256(image).hexdigest()
|
||||
assert response.json()["size_bytes"] == maximum
|
||||
assert response.json()["sha256"] == hashlib.sha256(image).hexdigest()
|
||||
|
||||
|
||||
async def test_exact_pixel_limit_accepts_valid_png(client, context):
|
||||
_, alice, _, _, chat = context
|
||||
output = BytesIO()
|
||||
with Image.new('RGB', (8000, 5000), 'blue') as image:
|
||||
image.save(output, format='PNG')
|
||||
with Image.new("RGB", (8000, 5000), "blue") as image:
|
||||
image.save(output, format="PNG")
|
||||
response = await upload(client, alice, chat, output.getvalue())
|
||||
assert response.status_code == 202, response.text
|
||||
|
||||
|
||||
async def test_pixel_limit_before_decoding(client, context):
|
||||
_, alice, _, _, chat = context
|
||||
|
||||
def chunk(kind, data):
|
||||
return struct.pack('!I', len(data)) + kind + data + struct.pack('!I', zlib.crc32(kind + data))
|
||||
header = struct.pack('!IIBBBBB', 8000, 5001, 8, 2, 0, 0, 0)
|
||||
image = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(b'\0')) + chunk(b'IEND', b'')
|
||||
return (
|
||||
struct.pack("!I", len(data))
|
||||
+ kind
|
||||
+ data
|
||||
+ struct.pack("!I", zlib.crc32(kind + data))
|
||||
)
|
||||
|
||||
header = struct.pack("!IIBBBBB", 8000, 5001, 8, 2, 0, 0, 0)
|
||||
image = (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", header)
|
||||
+ chunk(b"IDAT", zlib.compress(b"\0"))
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
response = await upload(client, alice, chat, image)
|
||||
assert response.status_code == 413, response.text
|
||||
|
||||
@@ -217,10 +280,10 @@ async def test_unknown_and_other_chat_attachment_are_404(client, context):
|
||||
_, alice, bob, _, chat = context
|
||||
attachment = (await upload(client, alice, chat)).json()
|
||||
other_chat = await create_chat(client, alice, bob)
|
||||
for id in (attachment['id'], str(uuid4())):
|
||||
for id in (attachment["id"], str(uuid4())):
|
||||
response = await client.post(
|
||||
f"/api/v1/chats/{other_chat['id']}/messages",
|
||||
json={'text': '', 'attachment_ids': [id]},
|
||||
json={"text": "", "attachment_ids": [id]},
|
||||
headers={**bearer(alice), **idempotency_header()},
|
||||
)
|
||||
assert response.status_code == 404
|
||||
@@ -228,10 +291,12 @@ async def test_unknown_and_other_chat_attachment_are_404(client, context):
|
||||
|
||||
async def test_duplicate_ids_are_400(client, context):
|
||||
_, alice, _, _, chat = context
|
||||
id = (await upload(client, alice, chat)).json()['id']
|
||||
response = await client.post(f"/api/v1/chats/{chat['id']}/messages",
|
||||
json={'text': '', 'attachment_ids': [id, id]},
|
||||
headers={**bearer(alice), **idempotency_header()})
|
||||
id = (await upload(client, alice, chat)).json()["id"]
|
||||
response = await client.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
json={"text": "", "attachment_ids": [id, id]},
|
||||
headers={**bearer(alice), **idempotency_header()},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
@@ -239,20 +304,24 @@ async def test_empty_ids_and_absent_ids_have_same_idempotency(client, context):
|
||||
_, alice, _, _, chat = context
|
||||
headers = {**bearer(alice), **idempotency_header()}
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
first = await client.post(path, json={'text': 'same'}, headers=headers)
|
||||
second = await client.post(path, json={'text': 'same', 'attachment_ids': []}, headers=headers)
|
||||
first = await client.post(path, json={"text": "same"}, headers=headers)
|
||||
second = await client.post(
|
||||
path, json={"text": "same", "attachment_ids": []}, headers=headers
|
||||
)
|
||||
assert first.status_code == second.status_code == 201
|
||||
assert first.json() == second.json()
|
||||
|
||||
|
||||
async def test_s3_outage_is_503(client, context):
|
||||
storage, alice, _, _, chat = context
|
||||
id = (await upload(client, alice, chat)).json()['id']
|
||||
id = (await upload(client, alice, chat)).json()["id"]
|
||||
await finish_attachment(client, storage, alice, id)
|
||||
storage.down = True
|
||||
response = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice))
|
||||
response = await client.get(
|
||||
f"/api/v1/attachments/{id}/content", headers=bearer(alice)
|
||||
)
|
||||
assert response.status_code == 503
|
||||
assert (await client.get('/health/ready')).status_code == 503
|
||||
assert (await client.get("/health/ready")).status_code == 503
|
||||
|
||||
|
||||
async def test_uncertain_put_is_compensated(client, context, db_engine):
|
||||
@@ -267,25 +336,31 @@ async def test_uncertain_put_is_compensated(client, context, db_engine):
|
||||
|
||||
async def test_failure_after_s3_write_compensates(client, context, monkeypatch):
|
||||
storage, alice, _, _, chat = context
|
||||
|
||||
async def fail(*args, **kwargs):
|
||||
raise OperationalError('injected', {}, None)
|
||||
monkeypatch.setattr(repository, 'create_attachment', fail)
|
||||
raise OperationalError("injected", {}, None)
|
||||
|
||||
monkeypatch.setattr(repository, "create_attachment", fail)
|
||||
response = await upload(client, alice, chat)
|
||||
assert response.status_code == 503
|
||||
assert not storage.objects
|
||||
|
||||
|
||||
async def test_ambiguous_successful_commit_keeps_original(client, context, db_engine, monkeypatch):
|
||||
async def test_ambiguous_successful_commit_keeps_original(
|
||||
client, context, db_engine, monkeypatch
|
||||
):
|
||||
storage, alice, _, _, chat = context
|
||||
original = AsyncSession.commit
|
||||
count = 0
|
||||
|
||||
async def uncertain(session):
|
||||
nonlocal count
|
||||
count += 1
|
||||
await original(session)
|
||||
if count == 2:
|
||||
raise OperationalError('lost commit acknowledgement', {}, None)
|
||||
monkeypatch.setattr(AsyncSession, 'commit', uncertain)
|
||||
raise OperationalError("lost commit acknowledgement", {}, None)
|
||||
|
||||
monkeypatch.setattr(AsyncSession, "commit", uncertain)
|
||||
response = await upload(client, alice, chat)
|
||||
assert response.status_code == 503
|
||||
assert len(storage.objects) == 1
|
||||
@@ -318,10 +393,16 @@ async def test_cleanup_keeps_fresh_and_skips_locked_uploads(client, context, db_
|
||||
storage, _, _, _, _ = context
|
||||
old_id, fresh_id = uuid4(), uuid4()
|
||||
async with AsyncSession(db_engine) as session:
|
||||
session.add_all([
|
||||
UploadIntent(id=old_id, object_key='originals/old', created_at=datetime.now(timezone.utc) - timedelta(days=2)),
|
||||
UploadIntent(id=fresh_id, object_key='originals/fresh'),
|
||||
])
|
||||
session.add_all(
|
||||
[
|
||||
UploadIntent(
|
||||
id=old_id,
|
||||
object_key="originals/old",
|
||||
created_at=datetime.now(timezone.utc) - timedelta(days=2),
|
||||
),
|
||||
UploadIntent(id=fresh_id, object_key="originals/fresh"),
|
||||
]
|
||||
)
|
||||
await session.commit()
|
||||
async with AsyncSession(db_engine) as uploader, AsyncSession(db_engine) as cleaner:
|
||||
await repository.lock_upload(uploader, old_id)
|
||||
@@ -335,9 +416,11 @@ async def test_cleanup_keeps_fresh_and_skips_locked_uploads(client, context, db_
|
||||
|
||||
async def test_client_disconnect_before_put_has_no_side_effects(client, context):
|
||||
storage, alice, _, _, chat = context
|
||||
|
||||
async def interrupted():
|
||||
yield PNG[:16]
|
||||
raise ClientDisconnect()
|
||||
|
||||
response = await upload(client, alice, chat, interrupted())
|
||||
assert response.status_code == 400
|
||||
assert not storage.objects
|
||||
@@ -345,10 +428,14 @@ async def test_client_disconnect_before_put_has_no_side_effects(client, context)
|
||||
|
||||
async def test_upload_deadline_cancels_validation(client, context, monkeypatch):
|
||||
storage, alice, _, _, chat = context
|
||||
storage.settings = storage.settings.model_copy(update={'upload_timeout_seconds': 0.02})
|
||||
storage.settings = storage.settings.model_copy(
|
||||
update={"upload_timeout_seconds": 0.02}
|
||||
)
|
||||
|
||||
async def stalled(*args):
|
||||
await asyncio.sleep(10)
|
||||
monkeypatch.setattr(domain, 'validate_image', stalled)
|
||||
|
||||
monkeypatch.setattr(domain, "validate_image", stalled)
|
||||
response = await upload(client, alice, chat)
|
||||
assert response.status_code == 400
|
||||
assert not storage.objects
|
||||
@@ -358,11 +445,13 @@ async def test_cancelled_put_is_compensated(client, context, db_engine, monkeypa
|
||||
storage, alice, _, _, chat = context
|
||||
entered = asyncio.Event()
|
||||
original = storage.put_file
|
||||
|
||||
async def interrupted(*args, **kwargs):
|
||||
await original(*args, **kwargs)
|
||||
entered.set()
|
||||
await asyncio.sleep(10)
|
||||
monkeypatch.setattr(storage, 'put_file', interrupted)
|
||||
|
||||
monkeypatch.setattr(storage, "put_file", interrupted)
|
||||
task = asyncio.create_task(upload(client, alice, chat))
|
||||
await asyncio.wait_for(entered.wait(), timeout=10)
|
||||
task.cancel()
|
||||
@@ -375,14 +464,19 @@ async def test_cancelled_put_is_compensated(client, context, db_engine, monkeypa
|
||||
|
||||
async def test_cleanup_never_deletes_ready_attachment(client, context, db_engine):
|
||||
storage, alice, _, _, chat = context
|
||||
id = (await upload(client, alice, chat)).json()['id']
|
||||
id = (await upload(client, alice, chat)).json()["id"]
|
||||
await finish_attachment(client, storage, alice, id)
|
||||
async with AsyncSession(db_engine) as session:
|
||||
attachment = await session.get(Attachment, UUID(id))
|
||||
key = attachment.object_key
|
||||
# Искусственно оставляем старый журнал у уже подтверждённого объекта.
|
||||
session.add(UploadIntent(id=uuid4(), object_key=key,
|
||||
created_at=datetime.now(timezone.utc) - timedelta(days=2)))
|
||||
session.add(
|
||||
UploadIntent(
|
||||
id=uuid4(),
|
||||
object_key=key,
|
||||
created_at=datetime.now(timezone.utc) - timedelta(days=2),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
result = await domain.reconcile_attachments(session, storage, dry_run=False)
|
||||
assert result.examined == 1 and result.deleted == 0
|
||||
|
||||
@@ -8,7 +8,14 @@ from sqlalchemy import text
|
||||
|
||||
from app.main import create_app
|
||||
from conftest import close_app, make_client
|
||||
from helpers import PASSWORD, basic_header, bearer, create_user, idempotency_header, register
|
||||
from helpers import (
|
||||
PASSWORD,
|
||||
basic_header,
|
||||
bearer,
|
||||
create_user,
|
||||
idempotency_header,
|
||||
register,
|
||||
)
|
||||
|
||||
|
||||
async def test_parallel_registration_of_same_username_creates_exactly_one_user(
|
||||
|
||||
@@ -29,14 +29,12 @@ async def seed_messages(
|
||||
) -> None:
|
||||
async with engine.begin() as connection:
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
text("""
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), :chat_id, :sender_id, 'message ' || g,
|
||||
now() - make_interval(secs => :count - g)
|
||||
FROM generate_series(1, :count) AS g
|
||||
"""
|
||||
),
|
||||
"""),
|
||||
{"chat_id": chat_id, "sender_id": sender_id, "count": count},
|
||||
)
|
||||
await connection.execute(text("ANALYZE messages"))
|
||||
|
||||
@@ -16,10 +16,15 @@ from helpers import bearer, create_chat, create_user, idempotency_header, send_m
|
||||
@pytest.fixture
|
||||
async def replicas(request, make_settings):
|
||||
overrides = getattr(request, "param", {})
|
||||
apps = [create_app(make_settings(instance_id=name, **overrides)) for name in ("api-a", "api-b")]
|
||||
apps = [
|
||||
create_app(make_settings(instance_id=name, **overrides))
|
||||
for name in ("api-a", "api-b")
|
||||
]
|
||||
try:
|
||||
async with AsyncExitStack() as stack:
|
||||
clients = [await stack.enter_async_context(make_client(app)) for app in apps]
|
||||
clients = [
|
||||
await stack.enter_async_context(make_client(app)) for app in apps
|
||||
]
|
||||
yield clients
|
||||
finally:
|
||||
for app in apps:
|
||||
@@ -54,7 +59,11 @@ async def test_replay_and_conflict_across_replicas(replicas, db_engine):
|
||||
original = await send_message(first, user, chat, "Привет", "same-key")
|
||||
repeated = await second.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
headers={**bearer(user), **idempotency_header("same-key"), "Content-Type": "application/json"},
|
||||
headers={
|
||||
**bearer(user),
|
||||
**idempotency_header("same-key"),
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
content='{ "text" : "\\u041f\\u0440\\u0438\\u0432\\u0435\\u0442" }',
|
||||
)
|
||||
conflict = await send_message(second, user, chat, "другой текст", "same-key")
|
||||
@@ -63,23 +72,38 @@ async def test_replay_and_conflict_across_replicas(replicas, db_engine):
|
||||
assert original.headers["X-Instance-Id"] != repeated.headers["X-Instance-Id"]
|
||||
assert conflict.status_code == 409
|
||||
async with db_engine.connect() as connection:
|
||||
assert (await connection.execute(text("SELECT count(*) FROM messages"))).scalar_one() == 1
|
||||
assert (await connection.execute(text("SELECT value FROM counters WHERE name='messages'"))).scalar_one() == 1
|
||||
assert (
|
||||
await connection.execute(text("SELECT count(*) FROM messages"))
|
||||
).scalar_one() == 1
|
||||
assert (
|
||||
await connection.execute(
|
||||
text("SELECT value FROM counters WHERE name='messages'")
|
||||
)
|
||||
).scalar_one() == 1
|
||||
|
||||
|
||||
async def test_parallel_replays_create_one_message(replicas, db_engine):
|
||||
user = await create_user(replicas[0])
|
||||
chat = await create_chat(replicas[0], user)
|
||||
responses = await asyncio.gather(*(
|
||||
send_message(replicas[index % 2], user, chat, "parallel", "one-key")
|
||||
for index in range(24)
|
||||
))
|
||||
responses = await asyncio.gather(
|
||||
*(
|
||||
send_message(replicas[index % 2], user, chat, "parallel", "one-key")
|
||||
for index in range(24)
|
||||
)
|
||||
)
|
||||
assert all(response.status_code == 201 for response in responses)
|
||||
assert len({response.json()["id"] for response in responses}) == 1
|
||||
assert {response.headers["X-Instance-Id"] for response in responses} == {"api-a", "api-b"}
|
||||
assert {response.headers["X-Instance-Id"] for response in responses} == {
|
||||
"api-a",
|
||||
"api-b",
|
||||
}
|
||||
async with db_engine.connect() as connection:
|
||||
assert (await connection.execute(text("SELECT count(*) FROM messages"))).scalar_one() == 1
|
||||
assert (await connection.execute(text("SELECT count(*) FROM idempotency_keys"))).scalar_one() == 1
|
||||
assert (
|
||||
await connection.execute(text("SELECT count(*) FROM messages"))
|
||||
).scalar_one() == 1
|
||||
assert (
|
||||
await connection.execute(text("SELECT count(*) FROM idempotency_keys"))
|
||||
).scalar_one() == 1
|
||||
|
||||
|
||||
async def test_key_scope_separates_users_and_chats(client):
|
||||
@@ -101,9 +125,11 @@ async def test_expired_key_can_be_reused_without_removing_message(client, db_eng
|
||||
chat = await create_chat(client, user)
|
||||
first = await send_message(client, user, chat, "old", "expiring-key")
|
||||
async with db_engine.begin() as connection:
|
||||
await connection.execute(text(
|
||||
"UPDATE idempotency_keys SET created_at = clock_timestamp() - interval '25 hours'"
|
||||
))
|
||||
await connection.execute(
|
||||
text(
|
||||
"UPDATE idempotency_keys SET created_at = clock_timestamp() - interval '25 hours'"
|
||||
)
|
||||
)
|
||||
second = await send_message(client, user, chat, "new", "expiring-key")
|
||||
assert first.status_code == second.status_code == 201
|
||||
assert first.json()["id"] != second.json()["id"]
|
||||
@@ -127,7 +153,9 @@ async def test_failed_transaction_does_not_poison_key(client, db_engine, monkeyp
|
||||
assert failed.status_code == 503
|
||||
async with db_engine.connect() as connection:
|
||||
for table in ("messages", "idempotency_keys", "counters"):
|
||||
assert (await connection.execute(text(f"SELECT count(*) FROM {table}"))).scalar_one() == 0
|
||||
assert (
|
||||
await connection.execute(text(f"SELECT count(*) FROM {table}"))
|
||||
).scalar_one() == 0
|
||||
monkeypatch.setattr(key_storage, "save_result", original)
|
||||
recovered = await send_message(client, user, chat, "retry", "retry-key")
|
||||
assert recovered.status_code == 201
|
||||
@@ -148,9 +176,15 @@ async def test_replay_works_when_message_limit_is_full(replicas):
|
||||
async def test_replay_checks_session_and_membership(client, db_engine):
|
||||
user = await create_user(client)
|
||||
chat = await create_chat(client, user)
|
||||
assert (await send_message(client, user, chat, "private", "private-key")).status_code == 201
|
||||
assert (
|
||||
await send_message(client, user, chat, "private", "private-key")
|
||||
).status_code == 201
|
||||
async with db_engine.begin() as connection:
|
||||
await connection.execute(text("DELETE FROM chat_members"))
|
||||
assert (await send_message(client, user, chat, "private", "private-key")).status_code == 404
|
||||
assert (
|
||||
await send_message(client, user, chat, "private", "private-key")
|
||||
).status_code == 404
|
||||
await client.delete("/api/v1/auth/sessions/current", headers=bearer(user))
|
||||
assert (await send_message(client, user, chat, "private", "private-key")).status_code == 401
|
||||
assert (
|
||||
await send_message(client, user, chat, "private", "private-key")
|
||||
).status_code == 401
|
||||
|
||||
@@ -18,54 +18,96 @@ from app.storage.jobs import JobWork
|
||||
from test_attachments import MemoryStorage
|
||||
|
||||
|
||||
@pytest.mark.parametrize('size,transparent', [((1600,800),False),((24,12),True)])
|
||||
async def test_variant_geometry_metadata_and_alpha(tmp_path,size,transparent):
|
||||
source=tmp_path/'source.png'
|
||||
with Image.new('RGBA',size,(255,0,0,0 if transparent else 255)) as image:
|
||||
exif=Image.Exif();exif[270]='private description'
|
||||
image.save(source,format='PNG',exif=exif)
|
||||
result=create_variants(source,tmp_path,'image/png',40_000_000,ImageSettings(_env_file=None))
|
||||
variants={v.name:v for v in result.variants}
|
||||
assert (variants['crop'].width,variants['crop'].height)==(128,128)
|
||||
assert (variants['thumbnail'].width,variants['thumbnail'].height)==((256,128) if size[0]>256 else size)
|
||||
assert (variants['optimized'].width,variants['optimized'].height)==((1280,640) if size[0]>1280 else size)
|
||||
for name in ('thumbnail.jpg','crop.jpg','optimized.webp'):
|
||||
with Image.open(tmp_path/name) as image:
|
||||
assert not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp')
|
||||
@pytest.mark.parametrize("size,transparent", [((1600, 800), False), ((24, 12), True)])
|
||||
async def test_variant_geometry_metadata_and_alpha(tmp_path, size, transparent):
|
||||
source = tmp_path / "source.png"
|
||||
with Image.new("RGBA", size, (255, 0, 0, 0 if transparent else 255)) as image:
|
||||
exif = Image.Exif()
|
||||
exif[270] = "private description"
|
||||
image.save(source, format="PNG", exif=exif)
|
||||
result = create_variants(
|
||||
source, tmp_path, "image/png", 40_000_000, ImageSettings(_env_file=None)
|
||||
)
|
||||
variants = {v.name: v for v in result.variants}
|
||||
assert (variants["crop"].width, variants["crop"].height) == (128, 128)
|
||||
assert (variants["thumbnail"].width, variants["thumbnail"].height) == (
|
||||
(256, 128) if size[0] > 256 else size
|
||||
)
|
||||
assert (variants["optimized"].width, variants["optimized"].height) == (
|
||||
(1280, 640) if size[0] > 1280 else size
|
||||
)
|
||||
for name in ("thumbnail.jpg", "crop.jpg", "optimized.webp"):
|
||||
with Image.open(tmp_path / name) as image:
|
||||
assert (
|
||||
not image.getexif()
|
||||
and not image.info.get("icc_profile")
|
||||
and not image.info.get("xmp")
|
||||
)
|
||||
if transparent:
|
||||
if name.endswith('webp'):assert image.getpixel((0,0))[3]==0
|
||||
else:assert min(image.getpixel((0,0)))>=250
|
||||
if name.endswith("webp"):
|
||||
assert image.getpixel((0, 0))[3] == 0
|
||||
else:
|
||||
assert min(image.getpixel((0, 0))) >= 250
|
||||
|
||||
|
||||
async def test_exif_orientation_applied(tmp_path):
|
||||
source=tmp_path/'source.jpg'
|
||||
with Image.new('RGB',(400,200),'blue') as image:
|
||||
exif=Image.Exif();exif[274]=6
|
||||
image.save(source,exif=exif)
|
||||
result=create_variants(source,tmp_path,'image/jpeg',40_000_000,ImageSettings(_env_file=None))
|
||||
assert (result.variants[0].width,result.variants[0].height)==(128,256)
|
||||
source = tmp_path / "source.jpg"
|
||||
with Image.new("RGB", (400, 200), "blue") as image:
|
||||
exif = Image.Exif()
|
||||
exif[274] = 6
|
||||
image.save(source, exif=exif)
|
||||
result = create_variants(
|
||||
source, tmp_path, "image/jpeg", 40_000_000, ImageSettings(_env_file=None)
|
||||
)
|
||||
assert (result.variants[0].width, result.variants[0].height) == (128, 256)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('failure', ['memory','cpu'])
|
||||
async def test_worker_os_limits(failure,tmp_path):
|
||||
body='return bytearray(2 * 1024 ** 3)' if failure=='memory' else 'while True: pass'
|
||||
code="from app.core import thumbnail_worker as w\ndef fail(*args):\n "+body+"\nw.create_variants=fail\nraise SystemExit(w.main())"
|
||||
result=await asyncio.to_thread(subprocess.run,[sys.executable,'-c',code,'source',str(tmp_path),str(tmp_path/'metadata'),'image/png','40000000'],
|
||||
env={**os.environ,'IMAGE_MEMORY_MIB':'256','IMAGE_CPU_SECONDS':'1'},capture_output=True,timeout=15)
|
||||
assert result.returncode==(3 if failure=='memory' else -9)
|
||||
assert not (tmp_path/'metadata').exists()
|
||||
@pytest.mark.parametrize("failure", ["memory", "cpu"])
|
||||
async def test_worker_os_limits(failure, tmp_path):
|
||||
body = (
|
||||
"return bytearray(2 * 1024 ** 3)" if failure == "memory" else "while True: pass"
|
||||
)
|
||||
code = (
|
||||
"from app.core import thumbnail_worker as w\ndef fail(*args):\n "
|
||||
+ body
|
||||
+ "\nw.create_variants=fail\nraise SystemExit(w.main())"
|
||||
)
|
||||
result = await asyncio.to_thread(
|
||||
subprocess.run,
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
code,
|
||||
"source",
|
||||
str(tmp_path),
|
||||
str(tmp_path / "metadata"),
|
||||
"image/png",
|
||||
"40000000",
|
||||
],
|
||||
env={**os.environ, "IMAGE_MEMORY_MIB": "256", "IMAGE_CPU_SECONDS": "1"},
|
||||
capture_output=True,
|
||||
timeout=15,
|
||||
)
|
||||
assert result.returncode == (3 if failure == "memory" else -9)
|
||||
assert not (tmp_path / "metadata").exists()
|
||||
|
||||
|
||||
async def test_render_timeout_kills_child(monkeypatch,tmp_path):
|
||||
create=asyncio.create_subprocess_exec
|
||||
children=[]
|
||||
async def slow_child(*args,**kwargs):
|
||||
child=await create(sys.executable,'-c','import time;time.sleep(60)',**kwargs)
|
||||
async def test_render_timeout_kills_child(monkeypatch, tmp_path):
|
||||
create = asyncio.create_subprocess_exec
|
||||
children = []
|
||||
|
||||
async def slow_child(*args, **kwargs):
|
||||
child = await create(
|
||||
sys.executable, "-c", "import time;time.sleep(60)", **kwargs
|
||||
)
|
||||
children.append(child)
|
||||
return child
|
||||
monkeypatch.setattr(asyncio,'create_subprocess_exec',slow_child)
|
||||
work=JobWork(uuid4(),uuid4(),uuid4(),1,'staging/test','image/png',1,'0'*64)
|
||||
|
||||
monkeypatch.setattr(asyncio, "create_subprocess_exec", slow_child)
|
||||
work = JobWork(
|
||||
uuid4(), uuid4(), uuid4(), 1, "staging/test", "image/png", 1, "0" * 64
|
||||
)
|
||||
with pytest.raises(TimeoutError):
|
||||
async with asyncio.timeout(.2):
|
||||
await render_variants(MemoryStorage(),work,tmp_path/'source',tmp_path)
|
||||
assert len(children)==1 and children[0].returncode is not None
|
||||
async with asyncio.timeout(0.2):
|
||||
await render_variants(MemoryStorage(), work, tmp_path / "source", tmp_path)
|
||||
assert len(children) == 1 and children[0].returncode is not None
|
||||
|
||||
@@ -92,7 +92,9 @@ async def test_instance_header_is_present_on_success_and_errors(
|
||||
assert response.headers["X-Instance-Id"] == "api-test"
|
||||
assert response.headers["X-Request-Id"]
|
||||
if status >= 400:
|
||||
assert response.json()["error"]["request_id"] == response.headers["X-Request-Id"]
|
||||
assert (
|
||||
response.json()["error"]["request_id"] == response.headers["X-Request-Id"]
|
||||
)
|
||||
|
||||
|
||||
async def test_instance_id_is_stable_while_request_ids_are_distinct(instance_client):
|
||||
@@ -100,7 +102,9 @@ async def test_instance_id_is_stable_while_request_ids_are_distinct(instance_cli
|
||||
second = await instance_client.get("/__test__/request-id")
|
||||
|
||||
assert first.status_code == second.status_code == 200
|
||||
assert first.headers["X-Instance-Id"] == second.headers["X-Instance-Id"] == "api-test"
|
||||
assert (
|
||||
first.headers["X-Instance-Id"] == second.headers["X-Instance-Id"] == "api-test"
|
||||
)
|
||||
assert first.headers["X-Request-Id"] != second.headers["X-Request-Id"]
|
||||
for response in (first, second):
|
||||
assert response.json()["request_id"] == response.headers["X-Request-Id"]
|
||||
|
||||
+137
-48
@@ -13,7 +13,12 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.jobs import JobSettings
|
||||
from app.domain import attachment_jobs
|
||||
from app.domain.attachment_jobs import claim_job, process_attachment_job, cleanup_staging, build_result
|
||||
from app.domain.attachment_jobs import (
|
||||
claim_job,
|
||||
process_attachment_job,
|
||||
cleanup_staging,
|
||||
build_result,
|
||||
)
|
||||
from app.domain.job_admin import replay_failed_job
|
||||
from app.domain.job_retry import classify_failure, retry_delay
|
||||
from app.errors.exceptions import JobOperationError, ProcessingError
|
||||
@@ -26,50 +31,85 @@ from test_attachments import context, upload, finish_attachment
|
||||
|
||||
|
||||
async def envelope(session, attachment_id):
|
||||
job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id)))
|
||||
return JobMessage(job_id=job.id, attachment_id=job.attachment_id, pipeline_version=1,
|
||||
attempt=job.attempt + 1, created_at=job.created_at)
|
||||
job = await session.scalar(
|
||||
select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id))
|
||||
)
|
||||
return JobMessage(
|
||||
job_id=job.id,
|
||||
attachment_id=job.attachment_id,
|
||||
pipeline_version=1,
|
||||
attempt=job.attempt + 1,
|
||||
created_at=job.created_at,
|
||||
)
|
||||
|
||||
|
||||
async def test_retry_budget_backoff_replay_and_duplicate(client, context, db_engine):
|
||||
storage, alice, _, _, chat = context
|
||||
attachment = (await upload(client, alice, chat)).json()
|
||||
settings = JobSettings(_env_file=None, job_max_attempts=3)
|
||||
|
||||
def fail(_):
|
||||
raise TimeoutError
|
||||
|
||||
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
||||
first = await envelope(session, attachment['id'])
|
||||
first = await envelope(session, attachment["id"])
|
||||
for attempt in range(1, 4):
|
||||
message = await envelope(session, attachment['id'])
|
||||
assert await process_attachment_job(session, storage, message, settings, before_processing=fail)
|
||||
message = await envelope(session, attachment["id"])
|
||||
assert await process_attachment_job(
|
||||
session, storage, message, settings, before_processing=fail
|
||||
)
|
||||
job, item = await jobs.lock_job(session, first.job_id)
|
||||
assert job.attempt == attempt
|
||||
delay = (job.next_attempt_at - item.updated_at).total_seconds()
|
||||
assert 5 * 2 ** (attempt - 1) / 2 <= delay <= 5 * 2 ** (attempt - 1)
|
||||
assert item.status == ('failed' if attempt == 3 else 'queued')
|
||||
assert item.status == ("failed" if attempt == 3 else "queued")
|
||||
assert job.last_error_permanent is False
|
||||
job.next_attempt_at = await jobs.database_time(session) - timedelta(seconds=1)
|
||||
job.next_attempt_at = await jobs.database_time(session) - timedelta(
|
||||
seconds=1
|
||||
)
|
||||
await session.commit()
|
||||
assert item.error_code == 'retry_exhausted'
|
||||
assert item.error_code == "retry_exhausted"
|
||||
request_id = uuid4()
|
||||
result = await replay_failed_job(session, storage, settings, first.job_id, request_id=request_id,
|
||||
actor='test', reason='dependency_restored')
|
||||
assert result['attempt_limit'] == 6
|
||||
result = await replay_failed_job(
|
||||
session,
|
||||
storage,
|
||||
settings,
|
||||
first.job_id,
|
||||
request_id=request_id,
|
||||
actor="test",
|
||||
reason="dependency_restored",
|
||||
)
|
||||
assert result["attempt_limit"] == 6
|
||||
# Old envelope cannot consume the fresh replay budget.
|
||||
assert await process_attachment_job(session, storage, first, settings)
|
||||
job, item = await jobs.lock_job(session, first.job_id)
|
||||
assert job.attempt == 3 and item.status == 'queued'
|
||||
assert job.attempt == 3 and item.status == "queued"
|
||||
await session.commit()
|
||||
message = await envelope(session, attachment['id'])
|
||||
message = await envelope(session, attachment["id"])
|
||||
assert await process_attachment_job(session, storage, message, settings)
|
||||
before = dict(storage.objects)
|
||||
assert await process_attachment_job(session, storage, message, settings)
|
||||
assert storage.objects == before
|
||||
repeat = await replay_failed_job(session, storage, settings, first.job_id, request_id=request_id,
|
||||
actor='test', reason='dependency_restored')
|
||||
assert repeat['already_applied']
|
||||
with pytest.raises(JobOperationError, match='job_not_failed'):
|
||||
await replay_failed_job(session, storage, settings, first.job_id, request_id=uuid4(), actor='test', reason='manual_review')
|
||||
repeat = await replay_failed_job(
|
||||
session,
|
||||
storage,
|
||||
settings,
|
||||
first.job_id,
|
||||
request_id=request_id,
|
||||
actor="test",
|
||||
reason="dependency_restored",
|
||||
)
|
||||
assert repeat["already_applied"]
|
||||
with pytest.raises(JobOperationError, match="job_not_failed"):
|
||||
await replay_failed_job(
|
||||
session,
|
||||
storage,
|
||||
settings,
|
||||
first.job_id,
|
||||
request_id=uuid4(),
|
||||
actor="test",
|
||||
reason="manual_review",
|
||||
)
|
||||
await session.rollback()
|
||||
assert len((await session.scalars(select(JobReplay))).all()) == 1
|
||||
|
||||
@@ -77,26 +117,40 @@ async def test_retry_budget_backoff_replay_and_duplicate(client, context, db_eng
|
||||
async def test_permanent_failure_and_missing_source_replay(client, context, db_engine):
|
||||
storage, alice, _, _, chat = context
|
||||
attachment = (await upload(client, alice, chat)).json()
|
||||
|
||||
def fail(_):
|
||||
raise ProcessingError('image_limits')
|
||||
raise ProcessingError("image_limits")
|
||||
|
||||
settings = JobSettings(_env_file=None)
|
||||
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
||||
message = await envelope(session, attachment['id'])
|
||||
await process_attachment_job(session, storage, message, settings, before_processing=fail)
|
||||
message = await envelope(session, attachment["id"])
|
||||
await process_attachment_job(
|
||||
session, storage, message, settings, before_processing=fail
|
||||
)
|
||||
job, item = await jobs.lock_job(session, message.job_id)
|
||||
assert item.status == 'failed' and job.attempt == 1 and job.last_error_permanent
|
||||
assert item.status == "failed" and job.attempt == 1 and job.last_error_permanent
|
||||
job.staging_key = None
|
||||
await session.commit()
|
||||
with pytest.raises(JobOperationError, match='source_expired'):
|
||||
await replay_failed_job(session, storage, settings, job.id, request_id=uuid4(), actor='test', reason='manual_review')
|
||||
with pytest.raises(JobOperationError, match="source_expired"):
|
||||
await replay_failed_job(
|
||||
session,
|
||||
storage,
|
||||
settings,
|
||||
job.id,
|
||||
request_id=uuid4(),
|
||||
actor="test",
|
||||
reason="manual_review",
|
||||
)
|
||||
|
||||
|
||||
async def test_stale_worker_cannot_write_and_cleanup_keeps_active(client, context, db_engine):
|
||||
async def test_stale_worker_cannot_write_and_cleanup_keeps_active(
|
||||
client, context, db_engine
|
||||
):
|
||||
storage, alice, _, _, chat = context
|
||||
attachment = (await upload(client, alice, chat)).json()
|
||||
settings = JobSettings(_env_file=None)
|
||||
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
||||
message = await envelope(session, attachment['id'])
|
||||
message = await envelope(session, attachment["id"])
|
||||
claim = await claim_job(session, message, settings)
|
||||
job, item = await jobs.lock_job(session, message.job_id)
|
||||
job.lease_token = uuid4()
|
||||
@@ -112,18 +166,33 @@ async def test_stale_worker_cannot_write_and_cleanup_keeps_active(client, contex
|
||||
async def test_cleanup_ready_preserves_final_objects(client, context, db_engine):
|
||||
storage, alice, _, _, chat = context
|
||||
attachment = (await upload(client, alice, chat)).json()
|
||||
await finish_attachment(client, storage, alice, attachment['id'])
|
||||
await finish_attachment(client, storage, alice, attachment["id"])
|
||||
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
||||
message = await envelope(session, attachment['id'])
|
||||
message = await envelope(session, attachment["id"])
|
||||
job, item = await jobs.lock_job(session, message.job_id)
|
||||
staging = job.staging_key
|
||||
job.finished_at = await jobs.database_time(session) - timedelta(days=2)
|
||||
await session.commit()
|
||||
assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=False) == 1
|
||||
assert (
|
||||
await cleanup_staging(
|
||||
session, storage, JobSettings(_env_file=None), apply=False
|
||||
)
|
||||
== 1
|
||||
)
|
||||
assert staging in storage.objects
|
||||
assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=True) == 1
|
||||
assert (
|
||||
await cleanup_staging(
|
||||
session, storage, JobSettings(_env_file=None), apply=True
|
||||
)
|
||||
== 1
|
||||
)
|
||||
assert staging not in storage.objects and len(storage.objects) == 4
|
||||
assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=True) == 0
|
||||
assert (
|
||||
await cleanup_staging(
|
||||
session, storage, JobSettings(_env_file=None), apply=True
|
||||
)
|
||||
== 0
|
||||
)
|
||||
|
||||
|
||||
async def test_retry_classification_and_cap():
|
||||
@@ -131,44 +200,64 @@ async def test_retry_classification_and_cap():
|
||||
for attempt in (1, 2, 6, 100):
|
||||
cap = min(120, 5 * 2 ** min(30, attempt - 1))
|
||||
assert all(cap / 2 <= retry_delay(settings, attempt) <= cap for _ in range(100))
|
||||
assert classify_failure(TimeoutError()) == ('processing_timeout', False)
|
||||
assert classify_failure(OSError(errno.EACCES, 'denied'))[1]
|
||||
assert not classify_failure(OSError(errno.ENOSPC, 'full'))[1]
|
||||
assert classify_failure(TimeoutError()) == ("processing_timeout", False)
|
||||
assert classify_failure(OSError(errno.EACCES, "denied"))[1]
|
||||
assert not classify_failure(OSError(errno.ENOSPC, "full"))[1]
|
||||
|
||||
|
||||
async def test_timeout_records_retry_and_clears_lease(client, context, db_engine, monkeypatch):
|
||||
async def test_timeout_records_retry_and_clears_lease(
|
||||
client, context, db_engine, monkeypatch
|
||||
):
|
||||
storage, alice, _, _, chat = context
|
||||
attachment = (await upload(client, alice, chat)).json()
|
||||
|
||||
async def stalled(*args, **kwargs):
|
||||
await asyncio.sleep(60)
|
||||
monkeypatch.setattr(attachment_jobs, 'build_result', stalled)
|
||||
settings = JobSettings(_env_file=None).model_copy(update={'job_timeout_seconds': .05})
|
||||
|
||||
monkeypatch.setattr(attachment_jobs, "build_result", stalled)
|
||||
settings = JobSettings(_env_file=None).model_copy(
|
||||
update={"job_timeout_seconds": 0.05}
|
||||
)
|
||||
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
||||
message = await envelope(session, attachment['id'])
|
||||
message = await envelope(session, attachment["id"])
|
||||
assert await process_attachment_job(session, storage, message, settings)
|
||||
job, item = await jobs.lock_job(session, message.job_id)
|
||||
assert item.status == 'queued' and job.last_error_code == 'processing_timeout'
|
||||
assert item.status == "queued" and job.last_error_code == "processing_timeout"
|
||||
assert job.lease_token is None and job.lease_until is None
|
||||
|
||||
|
||||
async def test_failed_retention_and_source_checksum_protect_replay(client, context, db_engine):
|
||||
async def test_failed_retention_and_source_checksum_protect_replay(
|
||||
client, context, db_engine
|
||||
):
|
||||
storage, alice, _, _, chat = context
|
||||
attachment = (await upload(client, alice, chat)).json()
|
||||
|
||||
def fail(_):
|
||||
raise ProcessingError('invalid_image')
|
||||
raise ProcessingError("invalid_image")
|
||||
|
||||
settings = JobSettings(_env_file=None)
|
||||
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
||||
message = await envelope(session, attachment['id'])
|
||||
await process_attachment_job(session, storage, message, settings, before_processing=fail)
|
||||
message = await envelope(session, attachment["id"])
|
||||
await process_attachment_job(
|
||||
session, storage, message, settings, before_processing=fail
|
||||
)
|
||||
job, item = await jobs.lock_job(session, message.job_id)
|
||||
key = job.staging_key
|
||||
job.finished_at = await jobs.database_time(session) - timedelta(days=2)
|
||||
await session.commit()
|
||||
assert await cleanup_staging(session, storage, settings, apply=True) == 0
|
||||
data, metadata = storage.objects[key]
|
||||
storage.objects[key] = (b'x' * len(data), metadata)
|
||||
with pytest.raises(JobOperationError, match='source_mismatch'):
|
||||
await replay_failed_job(session, storage, settings, message.job_id, request_id=uuid4(), actor='test', reason='manual_review')
|
||||
storage.objects[key] = (b"x" * len(data), metadata)
|
||||
with pytest.raises(JobOperationError, match="source_mismatch"):
|
||||
await replay_failed_job(
|
||||
session,
|
||||
storage,
|
||||
settings,
|
||||
message.job_id,
|
||||
request_id=uuid4(),
|
||||
actor="test",
|
||||
reason="manual_review",
|
||||
)
|
||||
await session.rollback()
|
||||
assert not (await session.scalars(select(JobReplay))).all()
|
||||
job, item = await jobs.lock_job(session, message.job_id)
|
||||
|
||||
@@ -20,9 +20,9 @@ async def test_draining_keeps_liveness_but_rejects_new_work(make_settings):
|
||||
app.state.lifecycle.start_draining()
|
||||
try:
|
||||
async with make_client(app) as client:
|
||||
assert (await client.get('/health/live')).status_code == 200
|
||||
assert (await client.get('/health/ready')).status_code == 503
|
||||
assert (await client.get('/api/v1/users/me')).status_code == 503
|
||||
assert (await client.get("/health/live")).status_code == 200
|
||||
assert (await client.get("/health/ready")).status_code == 503
|
||||
assert (await client.get("/api/v1/users/me")).status_code == 503
|
||||
finally:
|
||||
await close_app(app)
|
||||
|
||||
@@ -36,13 +36,13 @@ async def test_readiness_timeout_cancels_dependency_checks(make_settings, monkey
|
||||
finally:
|
||||
cancelled.append(True)
|
||||
|
||||
monkeypatch.setattr(health, 'check_database', stalled)
|
||||
monkeypatch.setattr(health, 'check_redis', stalled)
|
||||
monkeypatch.setattr(health, "check_database", stalled)
|
||||
monkeypatch.setattr(health, "check_redis", stalled)
|
||||
app = create_app(make_settings(readiness_timeout_seconds=0.03))
|
||||
try:
|
||||
async with make_client(app) as client:
|
||||
started = time.monotonic()
|
||||
response = await client.get('/health/ready')
|
||||
response = await client.get("/health/ready")
|
||||
assert response.status_code == 503
|
||||
assert time.monotonic() - started < 1
|
||||
assert len(cancelled) == 2
|
||||
|
||||
@@ -6,7 +6,13 @@ import httpx
|
||||
|
||||
from app.main import create_app
|
||||
from conftest import close_app, make_client
|
||||
from helpers import assert_request_id, bearer, create_chat, create_user, idempotency_header
|
||||
from helpers import (
|
||||
assert_request_id,
|
||||
bearer,
|
||||
create_chat,
|
||||
create_user,
|
||||
idempotency_header,
|
||||
)
|
||||
|
||||
|
||||
def _assert_error(response: httpx.Response, status: int, code: str) -> None:
|
||||
@@ -25,12 +31,16 @@ async def test_message_limit_rejects_without_deleting_confirmed_messages(make_se
|
||||
|
||||
for index in range(2):
|
||||
ok = await client.post(
|
||||
url, headers={**bearer(user), **idempotency_header()}, json={"text": f"confirmed-{index}"}
|
||||
url,
|
||||
headers={**bearer(user), **idempotency_header()},
|
||||
json={"text": f"confirmed-{index}"},
|
||||
)
|
||||
assert ok.status_code == 201
|
||||
|
||||
rejected = await client.post(
|
||||
url, headers={**bearer(user), **idempotency_header()}, json={"text": "rejected"}
|
||||
url,
|
||||
headers={**bearer(user), **idempotency_header()},
|
||||
json={"text": "rejected"},
|
||||
)
|
||||
_assert_error(rejected, 503, "unavailable")
|
||||
|
||||
@@ -65,12 +75,18 @@ async def test_in_flight_limit_returns_429_and_recovers(make_settings, monkeypat
|
||||
user = await create_user(client)
|
||||
chat = await create_chat(client, user)
|
||||
|
||||
monkeypatch.setattr(message_endpoint, "send_message_idempotent", blocking_send_message)
|
||||
monkeypatch.setattr(
|
||||
message_endpoint, "send_message_idempotent", blocking_send_message
|
||||
)
|
||||
url = f"/api/v1/chats/{chat['id']}/messages"
|
||||
|
||||
held = [
|
||||
asyncio.create_task(
|
||||
client.post(url, headers={**bearer(user), **idempotency_header()}, json={"text": f"held-{i}"})
|
||||
client.post(
|
||||
url,
|
||||
headers={**bearer(user), **idempotency_header()},
|
||||
json={"text": f"held-{i}"},
|
||||
)
|
||||
)
|
||||
for i in range(2)
|
||||
]
|
||||
@@ -79,7 +95,11 @@ async def test_in_flight_limit_returns_429_and_recovers(make_settings, monkeypat
|
||||
|
||||
rejected = await asyncio.gather(
|
||||
*(
|
||||
client.post(url, headers={**bearer(user), **idempotency_header()}, json={"text": f"rejected-{i}"})
|
||||
client.post(
|
||||
url,
|
||||
headers={**bearer(user), **idempotency_header()},
|
||||
json={"text": f"rejected-{i}"},
|
||||
)
|
||||
for i in range(4)
|
||||
)
|
||||
)
|
||||
@@ -92,7 +112,11 @@ async def test_in_flight_limit_returns_429_and_recovers(make_settings, monkeypat
|
||||
release.set()
|
||||
assert all(r.status_code == 201 for r in await asyncio.gather(*held))
|
||||
|
||||
recovered = await client.post(url, headers={**bearer(user), **idempotency_header()}, json={"text": "after"})
|
||||
recovered = await client.post(
|
||||
url,
|
||||
headers={**bearer(user), **idempotency_header()},
|
||||
json={"text": "after"},
|
||||
)
|
||||
assert recovered.status_code == 201
|
||||
assert_request_id(recovered)
|
||||
await close_app(app)
|
||||
|
||||
@@ -8,7 +8,19 @@ from sqlalchemy import inspect, text
|
||||
from app.models import Base
|
||||
from conftest import reset_schema, upgrade_head
|
||||
|
||||
EXPECTED_TABLES = {"users", "chats", "chat_members", "messages", "counters", "idempotency_keys", "attachments", "message_attachments", "upload_intents", "attachment_jobs", "job_replays"}
|
||||
EXPECTED_TABLES = {
|
||||
"users",
|
||||
"chats",
|
||||
"chat_members",
|
||||
"messages",
|
||||
"counters",
|
||||
"idempotency_keys",
|
||||
"attachments",
|
||||
"message_attachments",
|
||||
"upload_intents",
|
||||
"attachment_jobs",
|
||||
"job_replays",
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -62,6 +74,6 @@ async def test_models_and_migrations_are_in_sync(migrated_engine):
|
||||
async with migrated_engine.connect() as connection:
|
||||
diff = await connection.run_sync(compare)
|
||||
|
||||
assert diff == [], (
|
||||
f"Создайте миграцию: alembic revision --autogenerate. Diff: {diff}"
|
||||
)
|
||||
assert (
|
||||
diff == []
|
||||
), f"Создайте миграцию: alembic revision --autogenerate. Diff: {diff}"
|
||||
|
||||
@@ -10,10 +10,12 @@ from app.storage.s3 import ObjectStream, run_sync
|
||||
|
||||
async def test_sdk_thread_finishes_before_cancellation_returns():
|
||||
entered, release, finished = Event(), Event(), Event()
|
||||
|
||||
def operation():
|
||||
entered.set()
|
||||
release.wait(timeout=3)
|
||||
finished.set()
|
||||
|
||||
task = asyncio.create_task(run_sync(operation))
|
||||
await asyncio.to_thread(entered.wait, 2)
|
||||
task.cancel()
|
||||
@@ -31,10 +33,12 @@ async def test_response_discarded_when_get_is_cancelled():
|
||||
entered, release = Event(), Event()
|
||||
discarded = []
|
||||
response = object()
|
||||
|
||||
def operation():
|
||||
entered.set()
|
||||
release.wait(timeout=3)
|
||||
return response
|
||||
|
||||
task = asyncio.create_task(run_sync(operation, discard_result=discarded.append))
|
||||
await asyncio.to_thread(entered.wait, 2)
|
||||
task.cancel()
|
||||
@@ -48,9 +52,12 @@ async def test_response_discarded_when_get_is_cancelled():
|
||||
|
||||
async def test_stream_closes_even_before_first_iteration():
|
||||
calls = []
|
||||
response = SimpleNamespace(close=lambda: calls.append('close'), release_conn=lambda: calls.append('release'))
|
||||
response = SimpleNamespace(
|
||||
close=lambda: calls.append("close"),
|
||||
release_conn=lambda: calls.append("release"),
|
||||
)
|
||||
storage = SimpleNamespace(settings=S3Settings(_env_file=None))
|
||||
stream = ObjectStream(storage, response)
|
||||
await stream.aclose()
|
||||
await stream.aclose()
|
||||
assert calls == ['close', 'release']
|
||||
assert calls == ["close", "release"]
|
||||
|
||||
Reference in New Issue
Block a user