formatted project by python-black

This commit is contained in:
2026-11-10 09:52:42 +03:00
parent 7cd02da51d
commit 9899ee4676
66 changed files with 2517 additions and 944 deletions
+164 -70
View File
@@ -28,7 +28,7 @@ from app.storage import attachments as repository
from app.storage.s3 import ObjectInfo
from helpers import bearer, create_chat, create_user, idempotency_header
PNG = Path('tests/fixtures/sample.png').read_bytes()
PNG = Path("tests/fixtures/sample.png").read_bytes()
class MemoryStorage:
@@ -55,12 +55,15 @@ class MemoryStorage:
info = None
if key in self.objects:
data, metadata = self.objects[key]
info = ObjectInfo(key, len(data), metadata['content_type'], datetime.now(timezone.utc))
info = ObjectInfo(
key, len(data), metadata["content_type"], datetime.now(timezone.utc)
)
return info
async def open_object(self, key):
async def chunks():
yield self.objects[key][0]
return chunks()
async def check_available(self):
@@ -76,23 +79,40 @@ async def context(client, app):
return storage, alice, bob, eve, chat
async def upload(client, actor, chat, data=PNG, filename='фото.png', content_type='image/png'):
async def upload(
client, actor, chat, data=PNG, filename="фото.png", content_type="image/png"
):
return await client.post(
f"/api/v1/chats/{chat['id']}/attachments", params={'filename': filename},
headers={**bearer(actor), 'Content-Type': content_type}, content=data,
f"/api/v1/chats/{chat['id']}/attachments",
params={"filename": filename},
headers={**bearer(actor), "Content-Type": content_type},
content=data,
)
async def finish_attachment(client, storage, actor, attachment_id):
engine = client._transport.app.state.engine
async with AsyncSession(engine) as session:
job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id)))
message = JobMessage(job_id=job.id, attachment_id=job.attachment_id,
pipeline_version=job.pipeline_version, attempt=job.attempt + 1, created_at=job.created_at)
job = await session.scalar(
select(AttachmentJob).where(
AttachmentJob.attachment_id == UUID(attachment_id)
)
)
message = JobMessage(
job_id=job.id,
attachment_id=job.attachment_id,
pipeline_version=job.pipeline_version,
attempt=job.attempt + 1,
created_at=job.created_at,
)
await session.rollback()
assert await process_attachment_job(session, storage, message, JobSettings(_env_file=None))
result = await client.get(f'/api/v1/attachments/{attachment_id}', headers=bearer(actor))
assert result.json()['status'] == 'ready'
assert await process_attachment_job(
session, storage, message, JobSettings(_env_file=None)
)
result = await client.get(
f"/api/v1/attachments/{attachment_id}", headers=bearer(actor)
)
assert result.json()["status"] == "ready"
return result.json()
@@ -101,37 +121,52 @@ async def test_upload_download_and_attachment_only_message(client, context):
response = await upload(client, alice, chat)
assert response.status_code == 202, response.text
attachment = response.json()
assert attachment['status'] == 'queued'
before = await client.get(f"/api/v1/attachments/{attachment['id']}/content", headers=bearer(alice))
assert attachment["status"] == "queued"
before = await client.get(
f"/api/v1/attachments/{attachment['id']}/content", headers=bearer(alice)
)
assert before.status_code == 409
attachment = await finish_attachment(client, storage, alice, attachment['id'])
assert attachment['sha256'] == hashlib.sha256(PNG).hexdigest()
assert {v['name'] for v in attachment['variants']} == {'thumbnail', 'crop', 'optimized'}
assert attachment['error_code'] is None
assert 'object_key' not in attachment
id = attachment['id']
attachment = await finish_attachment(client, storage, alice, attachment["id"])
assert attachment["sha256"] == hashlib.sha256(PNG).hexdigest()
assert {v["name"] for v in attachment["variants"]} == {
"thumbnail",
"crop",
"optimized",
}
assert attachment["error_code"] is None
assert "object_key" not in attachment
id = attachment["id"]
for actor in (alice, bob):
content = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(actor))
content = await client.get(
f"/api/v1/attachments/{id}/content", headers=bearer(actor)
)
assert content.status_code == 200 and content.content == PNG
assert content.headers['content-type'] == 'image/png'
assert content.headers["content-type"] == "image/png"
key = idempotency_header()
body = {'text': '', 'attachment_ids': [id]}
body = {"text": "", "attachment_ids": [id]}
path = f"/api/v1/chats/{chat['id']}/messages"
message = await client.post(path, json=body, headers={**bearer(bob), **key})
assert message.status_code == 201, message.text
repeat = await client.post(path, json=body, headers={**bearer(bob), **key})
assert repeat.json() == message.json()
history = await client.get(path, headers=bearer(alice))
assert history.json()['items'][0]['attachment_ids'] == [id]
for suffix in ('', '/content'):
denied = await client.get(f'/api/v1/attachments/{id}{suffix}', headers=bearer(eve))
assert history.json()["items"][0]["attachment_ids"] == [id]
for suffix in ("", "/content"):
denied = await client.get(
f"/api/v1/attachments/{id}{suffix}", headers=bearer(eve)
)
assert denied.status_code == 404
@pytest.mark.parametrize(
'data,content_type',
[(b'not an image', 'image/png'), (PNG[:30], 'image/png'), (PNG, 'image/jpeg'), (PNG, 'application/octet-stream')],
ids=['not-image', 'truncated-png', 'mime-spoofing', 'unsupported-type'],
"data,content_type",
[
(b"not an image", "image/png"),
(PNG[:30], "image/png"),
(PNG, "image/jpeg"),
(PNG, "application/octet-stream"),
],
ids=["not-image", "truncated-png", "mime-spoofing", "unsupported-type"],
)
async def test_invalid_image_and_mime_are_415(client, context, data, content_type):
storage, alice, _, _, chat = context
@@ -143,17 +178,21 @@ async def test_invalid_image_and_mime_are_415(client, context, data, content_typ
async def test_jpeg_is_stored_without_reencoding(client, context):
storage, alice, _, _, chat = context
output = BytesIO()
Image.new('RGB', (32, 32), 'blue').save(output, format='JPEG')
Image.new("RGB", (32, 32), "blue").save(output, format="JPEG")
original = output.getvalue()
response = await upload(client, alice, chat, original, content_type='image/jpeg')
response = await upload(client, alice, chat, original, content_type="image/jpeg")
assert response.status_code == 202, response.text
id = response.json()['id']
id = response.json()["id"]
await finish_attachment(client, storage, alice, id)
downloaded = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice))
downloaded = await client.get(
f"/api/v1/attachments/{id}/content", headers=bearer(alice)
)
assert downloaded.content == original
@pytest.mark.parametrize('filename', ['../a.png', 'a\\b.png', 'a\x00.png', 'a\n.png', 'x' * 256])
@pytest.mark.parametrize(
"filename", ["../a.png", "a\\b.png", "a\x00.png", "a\n.png", "x" * 256]
)
async def test_unsafe_filename_is_400(client, context, filename):
storage, alice, _, _, chat = context
response = await upload(client, alice, chat, filename=filename)
@@ -163,9 +202,11 @@ async def test_unsafe_filename_is_400(client, context, filename):
async def test_oversize_chunked_upload_is_413(client, context):
storage, alice, _, _, chat = context
async def chunks():
for _ in range(11):
yield b'x' * 1048576
yield b"x" * 1048576
response = await upload(client, alice, chat, chunks())
assert response.status_code == 413
assert not storage.objects
@@ -175,7 +216,12 @@ async def test_oversize_content_length_is_413(client, context):
_, alice, _, _, chat = context
response = await client.post(
f"/api/v1/chats/{chat['id']}/attachments?filename=a.png",
headers={**bearer(alice), 'Content-Type': 'image/png', 'Content-Length': '10485761'}, content=PNG,
headers={
**bearer(alice),
"Content-Type": "image/png",
"Content-Length": "10485761",
},
content=PNG,
)
assert response.status_code == 413
@@ -185,30 +231,47 @@ async def test_exact_byte_limit_accepts_valid_png(client, context):
maximum = 10_485_760
# Валидный ancillary chunk вместо данных после IEND.
payload = bytes(maximum - len(PNG) - 12)
kind = b'npAD'
padding = struct.pack('!I', len(payload)) + kind + payload + struct.pack('!I', zlib.crc32(kind + payload))
kind = b"npAD"
padding = (
struct.pack("!I", len(payload))
+ kind
+ payload
+ struct.pack("!I", zlib.crc32(kind + payload))
)
image = PNG[:-12] + padding + PNG[-12:]
response = await upload(client, alice, chat, image)
assert response.status_code == 202, response.text
assert response.json()['size_bytes'] == maximum
assert response.json()['sha256'] == hashlib.sha256(image).hexdigest()
assert response.json()["size_bytes"] == maximum
assert response.json()["sha256"] == hashlib.sha256(image).hexdigest()
async def test_exact_pixel_limit_accepts_valid_png(client, context):
_, alice, _, _, chat = context
output = BytesIO()
with Image.new('RGB', (8000, 5000), 'blue') as image:
image.save(output, format='PNG')
with Image.new("RGB", (8000, 5000), "blue") as image:
image.save(output, format="PNG")
response = await upload(client, alice, chat, output.getvalue())
assert response.status_code == 202, response.text
async def test_pixel_limit_before_decoding(client, context):
_, alice, _, _, chat = context
def chunk(kind, data):
return struct.pack('!I', len(data)) + kind + data + struct.pack('!I', zlib.crc32(kind + data))
header = struct.pack('!IIBBBBB', 8000, 5001, 8, 2, 0, 0, 0)
image = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(b'\0')) + chunk(b'IEND', b'')
return (
struct.pack("!I", len(data))
+ kind
+ data
+ struct.pack("!I", zlib.crc32(kind + data))
)
header = struct.pack("!IIBBBBB", 8000, 5001, 8, 2, 0, 0, 0)
image = (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", header)
+ chunk(b"IDAT", zlib.compress(b"\0"))
+ chunk(b"IEND", b"")
)
response = await upload(client, alice, chat, image)
assert response.status_code == 413, response.text
@@ -217,10 +280,10 @@ async def test_unknown_and_other_chat_attachment_are_404(client, context):
_, alice, bob, _, chat = context
attachment = (await upload(client, alice, chat)).json()
other_chat = await create_chat(client, alice, bob)
for id in (attachment['id'], str(uuid4())):
for id in (attachment["id"], str(uuid4())):
response = await client.post(
f"/api/v1/chats/{other_chat['id']}/messages",
json={'text': '', 'attachment_ids': [id]},
json={"text": "", "attachment_ids": [id]},
headers={**bearer(alice), **idempotency_header()},
)
assert response.status_code == 404
@@ -228,10 +291,12 @@ async def test_unknown_and_other_chat_attachment_are_404(client, context):
async def test_duplicate_ids_are_400(client, context):
_, alice, _, _, chat = context
id = (await upload(client, alice, chat)).json()['id']
response = await client.post(f"/api/v1/chats/{chat['id']}/messages",
json={'text': '', 'attachment_ids': [id, id]},
headers={**bearer(alice), **idempotency_header()})
id = (await upload(client, alice, chat)).json()["id"]
response = await client.post(
f"/api/v1/chats/{chat['id']}/messages",
json={"text": "", "attachment_ids": [id, id]},
headers={**bearer(alice), **idempotency_header()},
)
assert response.status_code == 400
@@ -239,20 +304,24 @@ async def test_empty_ids_and_absent_ids_have_same_idempotency(client, context):
_, alice, _, _, chat = context
headers = {**bearer(alice), **idempotency_header()}
path = f"/api/v1/chats/{chat['id']}/messages"
first = await client.post(path, json={'text': 'same'}, headers=headers)
second = await client.post(path, json={'text': 'same', 'attachment_ids': []}, headers=headers)
first = await client.post(path, json={"text": "same"}, headers=headers)
second = await client.post(
path, json={"text": "same", "attachment_ids": []}, headers=headers
)
assert first.status_code == second.status_code == 201
assert first.json() == second.json()
async def test_s3_outage_is_503(client, context):
storage, alice, _, _, chat = context
id = (await upload(client, alice, chat)).json()['id']
id = (await upload(client, alice, chat)).json()["id"]
await finish_attachment(client, storage, alice, id)
storage.down = True
response = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice))
response = await client.get(
f"/api/v1/attachments/{id}/content", headers=bearer(alice)
)
assert response.status_code == 503
assert (await client.get('/health/ready')).status_code == 503
assert (await client.get("/health/ready")).status_code == 503
async def test_uncertain_put_is_compensated(client, context, db_engine):
@@ -267,25 +336,31 @@ async def test_uncertain_put_is_compensated(client, context, db_engine):
async def test_failure_after_s3_write_compensates(client, context, monkeypatch):
storage, alice, _, _, chat = context
async def fail(*args, **kwargs):
raise OperationalError('injected', {}, None)
monkeypatch.setattr(repository, 'create_attachment', fail)
raise OperationalError("injected", {}, None)
monkeypatch.setattr(repository, "create_attachment", fail)
response = await upload(client, alice, chat)
assert response.status_code == 503
assert not storage.objects
async def test_ambiguous_successful_commit_keeps_original(client, context, db_engine, monkeypatch):
async def test_ambiguous_successful_commit_keeps_original(
client, context, db_engine, monkeypatch
):
storage, alice, _, _, chat = context
original = AsyncSession.commit
count = 0
async def uncertain(session):
nonlocal count
count += 1
await original(session)
if count == 2:
raise OperationalError('lost commit acknowledgement', {}, None)
monkeypatch.setattr(AsyncSession, 'commit', uncertain)
raise OperationalError("lost commit acknowledgement", {}, None)
monkeypatch.setattr(AsyncSession, "commit", uncertain)
response = await upload(client, alice, chat)
assert response.status_code == 503
assert len(storage.objects) == 1
@@ -318,10 +393,16 @@ async def test_cleanup_keeps_fresh_and_skips_locked_uploads(client, context, db_
storage, _, _, _, _ = context
old_id, fresh_id = uuid4(), uuid4()
async with AsyncSession(db_engine) as session:
session.add_all([
UploadIntent(id=old_id, object_key='originals/old', created_at=datetime.now(timezone.utc) - timedelta(days=2)),
UploadIntent(id=fresh_id, object_key='originals/fresh'),
])
session.add_all(
[
UploadIntent(
id=old_id,
object_key="originals/old",
created_at=datetime.now(timezone.utc) - timedelta(days=2),
),
UploadIntent(id=fresh_id, object_key="originals/fresh"),
]
)
await session.commit()
async with AsyncSession(db_engine) as uploader, AsyncSession(db_engine) as cleaner:
await repository.lock_upload(uploader, old_id)
@@ -335,9 +416,11 @@ async def test_cleanup_keeps_fresh_and_skips_locked_uploads(client, context, db_
async def test_client_disconnect_before_put_has_no_side_effects(client, context):
storage, alice, _, _, chat = context
async def interrupted():
yield PNG[:16]
raise ClientDisconnect()
response = await upload(client, alice, chat, interrupted())
assert response.status_code == 400
assert not storage.objects
@@ -345,10 +428,14 @@ async def test_client_disconnect_before_put_has_no_side_effects(client, context)
async def test_upload_deadline_cancels_validation(client, context, monkeypatch):
storage, alice, _, _, chat = context
storage.settings = storage.settings.model_copy(update={'upload_timeout_seconds': 0.02})
storage.settings = storage.settings.model_copy(
update={"upload_timeout_seconds": 0.02}
)
async def stalled(*args):
await asyncio.sleep(10)
monkeypatch.setattr(domain, 'validate_image', stalled)
monkeypatch.setattr(domain, "validate_image", stalled)
response = await upload(client, alice, chat)
assert response.status_code == 400
assert not storage.objects
@@ -358,11 +445,13 @@ async def test_cancelled_put_is_compensated(client, context, db_engine, monkeypa
storage, alice, _, _, chat = context
entered = asyncio.Event()
original = storage.put_file
async def interrupted(*args, **kwargs):
await original(*args, **kwargs)
entered.set()
await asyncio.sleep(10)
monkeypatch.setattr(storage, 'put_file', interrupted)
monkeypatch.setattr(storage, "put_file", interrupted)
task = asyncio.create_task(upload(client, alice, chat))
await asyncio.wait_for(entered.wait(), timeout=10)
task.cancel()
@@ -375,14 +464,19 @@ async def test_cancelled_put_is_compensated(client, context, db_engine, monkeypa
async def test_cleanup_never_deletes_ready_attachment(client, context, db_engine):
storage, alice, _, _, chat = context
id = (await upload(client, alice, chat)).json()['id']
id = (await upload(client, alice, chat)).json()["id"]
await finish_attachment(client, storage, alice, id)
async with AsyncSession(db_engine) as session:
attachment = await session.get(Attachment, UUID(id))
key = attachment.object_key
# Искусственно оставляем старый журнал у уже подтверждённого объекта.
session.add(UploadIntent(id=uuid4(), object_key=key,
created_at=datetime.now(timezone.utc) - timedelta(days=2)))
session.add(
UploadIntent(
id=uuid4(),
object_key=key,
created_at=datetime.now(timezone.utc) - timedelta(days=2),
)
)
await session.commit()
result = await domain.reconcile_attachments(session, storage, dry_run=False)
assert result.examined == 1 and result.deleted == 0
+8 -1
View File
@@ -8,7 +8,14 @@ from sqlalchemy import text
from app.main import create_app
from conftest import close_app, make_client
from helpers import PASSWORD, basic_header, bearer, create_user, idempotency_header, register
from helpers import (
PASSWORD,
basic_header,
bearer,
create_user,
idempotency_header,
register,
)
async def test_parallel_registration_of_same_username_creates_exactly_one_user(
+2 -4
View File
@@ -29,14 +29,12 @@ async def seed_messages(
) -> None:
async with engine.begin() as connection:
await connection.execute(
text(
"""
text("""
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), :chat_id, :sender_id, 'message ' || g,
now() - make_interval(secs => :count - g)
FROM generate_series(1, :count) AS g
"""
),
"""),
{"chat_id": chat_id, "sender_id": sender_id, "count": count},
)
await connection.execute(text("ANALYZE messages"))
+53 -19
View File
@@ -16,10 +16,15 @@ from helpers import bearer, create_chat, create_user, idempotency_header, send_m
@pytest.fixture
async def replicas(request, make_settings):
overrides = getattr(request, "param", {})
apps = [create_app(make_settings(instance_id=name, **overrides)) for name in ("api-a", "api-b")]
apps = [
create_app(make_settings(instance_id=name, **overrides))
for name in ("api-a", "api-b")
]
try:
async with AsyncExitStack() as stack:
clients = [await stack.enter_async_context(make_client(app)) for app in apps]
clients = [
await stack.enter_async_context(make_client(app)) for app in apps
]
yield clients
finally:
for app in apps:
@@ -54,7 +59,11 @@ async def test_replay_and_conflict_across_replicas(replicas, db_engine):
original = await send_message(first, user, chat, "Привет", "same-key")
repeated = await second.post(
f"/api/v1/chats/{chat['id']}/messages",
headers={**bearer(user), **idempotency_header("same-key"), "Content-Type": "application/json"},
headers={
**bearer(user),
**idempotency_header("same-key"),
"Content-Type": "application/json",
},
content='{ "text" : "\\u041f\\u0440\\u0438\\u0432\\u0435\\u0442" }',
)
conflict = await send_message(second, user, chat, "другой текст", "same-key")
@@ -63,23 +72,38 @@ async def test_replay_and_conflict_across_replicas(replicas, db_engine):
assert original.headers["X-Instance-Id"] != repeated.headers["X-Instance-Id"]
assert conflict.status_code == 409
async with db_engine.connect() as connection:
assert (await connection.execute(text("SELECT count(*) FROM messages"))).scalar_one() == 1
assert (await connection.execute(text("SELECT value FROM counters WHERE name='messages'"))).scalar_one() == 1
assert (
await connection.execute(text("SELECT count(*) FROM messages"))
).scalar_one() == 1
assert (
await connection.execute(
text("SELECT value FROM counters WHERE name='messages'")
)
).scalar_one() == 1
async def test_parallel_replays_create_one_message(replicas, db_engine):
user = await create_user(replicas[0])
chat = await create_chat(replicas[0], user)
responses = await asyncio.gather(*(
send_message(replicas[index % 2], user, chat, "parallel", "one-key")
for index in range(24)
))
responses = await asyncio.gather(
*(
send_message(replicas[index % 2], user, chat, "parallel", "one-key")
for index in range(24)
)
)
assert all(response.status_code == 201 for response in responses)
assert len({response.json()["id"] for response in responses}) == 1
assert {response.headers["X-Instance-Id"] for response in responses} == {"api-a", "api-b"}
assert {response.headers["X-Instance-Id"] for response in responses} == {
"api-a",
"api-b",
}
async with db_engine.connect() as connection:
assert (await connection.execute(text("SELECT count(*) FROM messages"))).scalar_one() == 1
assert (await connection.execute(text("SELECT count(*) FROM idempotency_keys"))).scalar_one() == 1
assert (
await connection.execute(text("SELECT count(*) FROM messages"))
).scalar_one() == 1
assert (
await connection.execute(text("SELECT count(*) FROM idempotency_keys"))
).scalar_one() == 1
async def test_key_scope_separates_users_and_chats(client):
@@ -101,9 +125,11 @@ async def test_expired_key_can_be_reused_without_removing_message(client, db_eng
chat = await create_chat(client, user)
first = await send_message(client, user, chat, "old", "expiring-key")
async with db_engine.begin() as connection:
await connection.execute(text(
"UPDATE idempotency_keys SET created_at = clock_timestamp() - interval '25 hours'"
))
await connection.execute(
text(
"UPDATE idempotency_keys SET created_at = clock_timestamp() - interval '25 hours'"
)
)
second = await send_message(client, user, chat, "new", "expiring-key")
assert first.status_code == second.status_code == 201
assert first.json()["id"] != second.json()["id"]
@@ -127,7 +153,9 @@ async def test_failed_transaction_does_not_poison_key(client, db_engine, monkeyp
assert failed.status_code == 503
async with db_engine.connect() as connection:
for table in ("messages", "idempotency_keys", "counters"):
assert (await connection.execute(text(f"SELECT count(*) FROM {table}"))).scalar_one() == 0
assert (
await connection.execute(text(f"SELECT count(*) FROM {table}"))
).scalar_one() == 0
monkeypatch.setattr(key_storage, "save_result", original)
recovered = await send_message(client, user, chat, "retry", "retry-key")
assert recovered.status_code == 201
@@ -148,9 +176,15 @@ async def test_replay_works_when_message_limit_is_full(replicas):
async def test_replay_checks_session_and_membership(client, db_engine):
user = await create_user(client)
chat = await create_chat(client, user)
assert (await send_message(client, user, chat, "private", "private-key")).status_code == 201
assert (
await send_message(client, user, chat, "private", "private-key")
).status_code == 201
async with db_engine.begin() as connection:
await connection.execute(text("DELETE FROM chat_members"))
assert (await send_message(client, user, chat, "private", "private-key")).status_code == 404
assert (
await send_message(client, user, chat, "private", "private-key")
).status_code == 404
await client.delete("/api/v1/auth/sessions/current", headers=bearer(user))
assert (await send_message(client, user, chat, "private", "private-key")).status_code == 401
assert (
await send_message(client, user, chat, "private", "private-key")
).status_code == 401
+82 -40
View File
@@ -18,54 +18,96 @@ from app.storage.jobs import JobWork
from test_attachments import MemoryStorage
@pytest.mark.parametrize('size,transparent', [((1600,800),False),((24,12),True)])
async def test_variant_geometry_metadata_and_alpha(tmp_path,size,transparent):
source=tmp_path/'source.png'
with Image.new('RGBA',size,(255,0,0,0 if transparent else 255)) as image:
exif=Image.Exif();exif[270]='private description'
image.save(source,format='PNG',exif=exif)
result=create_variants(source,tmp_path,'image/png',40_000_000,ImageSettings(_env_file=None))
variants={v.name:v for v in result.variants}
assert (variants['crop'].width,variants['crop'].height)==(128,128)
assert (variants['thumbnail'].width,variants['thumbnail'].height)==((256,128) if size[0]>256 else size)
assert (variants['optimized'].width,variants['optimized'].height)==((1280,640) if size[0]>1280 else size)
for name in ('thumbnail.jpg','crop.jpg','optimized.webp'):
with Image.open(tmp_path/name) as image:
assert not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp')
@pytest.mark.parametrize("size,transparent", [((1600, 800), False), ((24, 12), True)])
async def test_variant_geometry_metadata_and_alpha(tmp_path, size, transparent):
source = tmp_path / "source.png"
with Image.new("RGBA", size, (255, 0, 0, 0 if transparent else 255)) as image:
exif = Image.Exif()
exif[270] = "private description"
image.save(source, format="PNG", exif=exif)
result = create_variants(
source, tmp_path, "image/png", 40_000_000, ImageSettings(_env_file=None)
)
variants = {v.name: v for v in result.variants}
assert (variants["crop"].width, variants["crop"].height) == (128, 128)
assert (variants["thumbnail"].width, variants["thumbnail"].height) == (
(256, 128) if size[0] > 256 else size
)
assert (variants["optimized"].width, variants["optimized"].height) == (
(1280, 640) if size[0] > 1280 else size
)
for name in ("thumbnail.jpg", "crop.jpg", "optimized.webp"):
with Image.open(tmp_path / name) as image:
assert (
not image.getexif()
and not image.info.get("icc_profile")
and not image.info.get("xmp")
)
if transparent:
if name.endswith('webp'):assert image.getpixel((0,0))[3]==0
else:assert min(image.getpixel((0,0)))>=250
if name.endswith("webp"):
assert image.getpixel((0, 0))[3] == 0
else:
assert min(image.getpixel((0, 0))) >= 250
async def test_exif_orientation_applied(tmp_path):
source=tmp_path/'source.jpg'
with Image.new('RGB',(400,200),'blue') as image:
exif=Image.Exif();exif[274]=6
image.save(source,exif=exif)
result=create_variants(source,tmp_path,'image/jpeg',40_000_000,ImageSettings(_env_file=None))
assert (result.variants[0].width,result.variants[0].height)==(128,256)
source = tmp_path / "source.jpg"
with Image.new("RGB", (400, 200), "blue") as image:
exif = Image.Exif()
exif[274] = 6
image.save(source, exif=exif)
result = create_variants(
source, tmp_path, "image/jpeg", 40_000_000, ImageSettings(_env_file=None)
)
assert (result.variants[0].width, result.variants[0].height) == (128, 256)
@pytest.mark.parametrize('failure', ['memory','cpu'])
async def test_worker_os_limits(failure,tmp_path):
body='return bytearray(2 * 1024 ** 3)' if failure=='memory' else 'while True: pass'
code="from app.core import thumbnail_worker as w\ndef fail(*args):\n "+body+"\nw.create_variants=fail\nraise SystemExit(w.main())"
result=await asyncio.to_thread(subprocess.run,[sys.executable,'-c',code,'source',str(tmp_path),str(tmp_path/'metadata'),'image/png','40000000'],
env={**os.environ,'IMAGE_MEMORY_MIB':'256','IMAGE_CPU_SECONDS':'1'},capture_output=True,timeout=15)
assert result.returncode==(3 if failure=='memory' else -9)
assert not (tmp_path/'metadata').exists()
@pytest.mark.parametrize("failure", ["memory", "cpu"])
async def test_worker_os_limits(failure, tmp_path):
body = (
"return bytearray(2 * 1024 ** 3)" if failure == "memory" else "while True: pass"
)
code = (
"from app.core import thumbnail_worker as w\ndef fail(*args):\n "
+ body
+ "\nw.create_variants=fail\nraise SystemExit(w.main())"
)
result = await asyncio.to_thread(
subprocess.run,
[
sys.executable,
"-c",
code,
"source",
str(tmp_path),
str(tmp_path / "metadata"),
"image/png",
"40000000",
],
env={**os.environ, "IMAGE_MEMORY_MIB": "256", "IMAGE_CPU_SECONDS": "1"},
capture_output=True,
timeout=15,
)
assert result.returncode == (3 if failure == "memory" else -9)
assert not (tmp_path / "metadata").exists()
async def test_render_timeout_kills_child(monkeypatch,tmp_path):
create=asyncio.create_subprocess_exec
children=[]
async def slow_child(*args,**kwargs):
child=await create(sys.executable,'-c','import time;time.sleep(60)',**kwargs)
async def test_render_timeout_kills_child(monkeypatch, tmp_path):
create = asyncio.create_subprocess_exec
children = []
async def slow_child(*args, **kwargs):
child = await create(
sys.executable, "-c", "import time;time.sleep(60)", **kwargs
)
children.append(child)
return child
monkeypatch.setattr(asyncio,'create_subprocess_exec',slow_child)
work=JobWork(uuid4(),uuid4(),uuid4(),1,'staging/test','image/png',1,'0'*64)
monkeypatch.setattr(asyncio, "create_subprocess_exec", slow_child)
work = JobWork(
uuid4(), uuid4(), uuid4(), 1, "staging/test", "image/png", 1, "0" * 64
)
with pytest.raises(TimeoutError):
async with asyncio.timeout(.2):
await render_variants(MemoryStorage(),work,tmp_path/'source',tmp_path)
assert len(children)==1 and children[0].returncode is not None
async with asyncio.timeout(0.2):
await render_variants(MemoryStorage(), work, tmp_path / "source", tmp_path)
assert len(children) == 1 and children[0].returncode is not None
+6 -2
View File
@@ -92,7 +92,9 @@ async def test_instance_header_is_present_on_success_and_errors(
assert response.headers["X-Instance-Id"] == "api-test"
assert response.headers["X-Request-Id"]
if status >= 400:
assert response.json()["error"]["request_id"] == response.headers["X-Request-Id"]
assert (
response.json()["error"]["request_id"] == response.headers["X-Request-Id"]
)
async def test_instance_id_is_stable_while_request_ids_are_distinct(instance_client):
@@ -100,7 +102,9 @@ async def test_instance_id_is_stable_while_request_ids_are_distinct(instance_cli
second = await instance_client.get("/__test__/request-id")
assert first.status_code == second.status_code == 200
assert first.headers["X-Instance-Id"] == second.headers["X-Instance-Id"] == "api-test"
assert (
first.headers["X-Instance-Id"] == second.headers["X-Instance-Id"] == "api-test"
)
assert first.headers["X-Request-Id"] != second.headers["X-Request-Id"]
for response in (first, second):
assert response.json()["request_id"] == response.headers["X-Request-Id"]
+137 -48
View File
@@ -13,7 +13,12 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.core.jobs import JobSettings
from app.domain import attachment_jobs
from app.domain.attachment_jobs import claim_job, process_attachment_job, cleanup_staging, build_result
from app.domain.attachment_jobs import (
claim_job,
process_attachment_job,
cleanup_staging,
build_result,
)
from app.domain.job_admin import replay_failed_job
from app.domain.job_retry import classify_failure, retry_delay
from app.errors.exceptions import JobOperationError, ProcessingError
@@ -26,50 +31,85 @@ from test_attachments import context, upload, finish_attachment
async def envelope(session, attachment_id):
job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id)))
return JobMessage(job_id=job.id, attachment_id=job.attachment_id, pipeline_version=1,
attempt=job.attempt + 1, created_at=job.created_at)
job = await session.scalar(
select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id))
)
return JobMessage(
job_id=job.id,
attachment_id=job.attachment_id,
pipeline_version=1,
attempt=job.attempt + 1,
created_at=job.created_at,
)
async def test_retry_budget_backoff_replay_and_duplicate(client, context, db_engine):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
settings = JobSettings(_env_file=None, job_max_attempts=3)
def fail(_):
raise TimeoutError
async with AsyncSession(db_engine, expire_on_commit=False) as session:
first = await envelope(session, attachment['id'])
first = await envelope(session, attachment["id"])
for attempt in range(1, 4):
message = await envelope(session, attachment['id'])
assert await process_attachment_job(session, storage, message, settings, before_processing=fail)
message = await envelope(session, attachment["id"])
assert await process_attachment_job(
session, storage, message, settings, before_processing=fail
)
job, item = await jobs.lock_job(session, first.job_id)
assert job.attempt == attempt
delay = (job.next_attempt_at - item.updated_at).total_seconds()
assert 5 * 2 ** (attempt - 1) / 2 <= delay <= 5 * 2 ** (attempt - 1)
assert item.status == ('failed' if attempt == 3 else 'queued')
assert item.status == ("failed" if attempt == 3 else "queued")
assert job.last_error_permanent is False
job.next_attempt_at = await jobs.database_time(session) - timedelta(seconds=1)
job.next_attempt_at = await jobs.database_time(session) - timedelta(
seconds=1
)
await session.commit()
assert item.error_code == 'retry_exhausted'
assert item.error_code == "retry_exhausted"
request_id = uuid4()
result = await replay_failed_job(session, storage, settings, first.job_id, request_id=request_id,
actor='test', reason='dependency_restored')
assert result['attempt_limit'] == 6
result = await replay_failed_job(
session,
storage,
settings,
first.job_id,
request_id=request_id,
actor="test",
reason="dependency_restored",
)
assert result["attempt_limit"] == 6
# Old envelope cannot consume the fresh replay budget.
assert await process_attachment_job(session, storage, first, settings)
job, item = await jobs.lock_job(session, first.job_id)
assert job.attempt == 3 and item.status == 'queued'
assert job.attempt == 3 and item.status == "queued"
await session.commit()
message = await envelope(session, attachment['id'])
message = await envelope(session, attachment["id"])
assert await process_attachment_job(session, storage, message, settings)
before = dict(storage.objects)
assert await process_attachment_job(session, storage, message, settings)
assert storage.objects == before
repeat = await replay_failed_job(session, storage, settings, first.job_id, request_id=request_id,
actor='test', reason='dependency_restored')
assert repeat['already_applied']
with pytest.raises(JobOperationError, match='job_not_failed'):
await replay_failed_job(session, storage, settings, first.job_id, request_id=uuid4(), actor='test', reason='manual_review')
repeat = await replay_failed_job(
session,
storage,
settings,
first.job_id,
request_id=request_id,
actor="test",
reason="dependency_restored",
)
assert repeat["already_applied"]
with pytest.raises(JobOperationError, match="job_not_failed"):
await replay_failed_job(
session,
storage,
settings,
first.job_id,
request_id=uuid4(),
actor="test",
reason="manual_review",
)
await session.rollback()
assert len((await session.scalars(select(JobReplay))).all()) == 1
@@ -77,26 +117,40 @@ async def test_retry_budget_backoff_replay_and_duplicate(client, context, db_eng
async def test_permanent_failure_and_missing_source_replay(client, context, db_engine):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
def fail(_):
raise ProcessingError('image_limits')
raise ProcessingError("image_limits")
settings = JobSettings(_env_file=None)
async with AsyncSession(db_engine, expire_on_commit=False) as session:
message = await envelope(session, attachment['id'])
await process_attachment_job(session, storage, message, settings, before_processing=fail)
message = await envelope(session, attachment["id"])
await process_attachment_job(
session, storage, message, settings, before_processing=fail
)
job, item = await jobs.lock_job(session, message.job_id)
assert item.status == 'failed' and job.attempt == 1 and job.last_error_permanent
assert item.status == "failed" and job.attempt == 1 and job.last_error_permanent
job.staging_key = None
await session.commit()
with pytest.raises(JobOperationError, match='source_expired'):
await replay_failed_job(session, storage, settings, job.id, request_id=uuid4(), actor='test', reason='manual_review')
with pytest.raises(JobOperationError, match="source_expired"):
await replay_failed_job(
session,
storage,
settings,
job.id,
request_id=uuid4(),
actor="test",
reason="manual_review",
)
async def test_stale_worker_cannot_write_and_cleanup_keeps_active(client, context, db_engine):
async def test_stale_worker_cannot_write_and_cleanup_keeps_active(
client, context, db_engine
):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
settings = JobSettings(_env_file=None)
async with AsyncSession(db_engine, expire_on_commit=False) as session:
message = await envelope(session, attachment['id'])
message = await envelope(session, attachment["id"])
claim = await claim_job(session, message, settings)
job, item = await jobs.lock_job(session, message.job_id)
job.lease_token = uuid4()
@@ -112,18 +166,33 @@ async def test_stale_worker_cannot_write_and_cleanup_keeps_active(client, contex
async def test_cleanup_ready_preserves_final_objects(client, context, db_engine):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
await finish_attachment(client, storage, alice, attachment['id'])
await finish_attachment(client, storage, alice, attachment["id"])
async with AsyncSession(db_engine, expire_on_commit=False) as session:
message = await envelope(session, attachment['id'])
message = await envelope(session, attachment["id"])
job, item = await jobs.lock_job(session, message.job_id)
staging = job.staging_key
job.finished_at = await jobs.database_time(session) - timedelta(days=2)
await session.commit()
assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=False) == 1
assert (
await cleanup_staging(
session, storage, JobSettings(_env_file=None), apply=False
)
== 1
)
assert staging in storage.objects
assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=True) == 1
assert (
await cleanup_staging(
session, storage, JobSettings(_env_file=None), apply=True
)
== 1
)
assert staging not in storage.objects and len(storage.objects) == 4
assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=True) == 0
assert (
await cleanup_staging(
session, storage, JobSettings(_env_file=None), apply=True
)
== 0
)
async def test_retry_classification_and_cap():
@@ -131,44 +200,64 @@ async def test_retry_classification_and_cap():
for attempt in (1, 2, 6, 100):
cap = min(120, 5 * 2 ** min(30, attempt - 1))
assert all(cap / 2 <= retry_delay(settings, attempt) <= cap for _ in range(100))
assert classify_failure(TimeoutError()) == ('processing_timeout', False)
assert classify_failure(OSError(errno.EACCES, 'denied'))[1]
assert not classify_failure(OSError(errno.ENOSPC, 'full'))[1]
assert classify_failure(TimeoutError()) == ("processing_timeout", False)
assert classify_failure(OSError(errno.EACCES, "denied"))[1]
assert not classify_failure(OSError(errno.ENOSPC, "full"))[1]
async def test_timeout_records_retry_and_clears_lease(client, context, db_engine, monkeypatch):
async def test_timeout_records_retry_and_clears_lease(
client, context, db_engine, monkeypatch
):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
async def stalled(*args, **kwargs):
await asyncio.sleep(60)
monkeypatch.setattr(attachment_jobs, 'build_result', stalled)
settings = JobSettings(_env_file=None).model_copy(update={'job_timeout_seconds': .05})
monkeypatch.setattr(attachment_jobs, "build_result", stalled)
settings = JobSettings(_env_file=None).model_copy(
update={"job_timeout_seconds": 0.05}
)
async with AsyncSession(db_engine, expire_on_commit=False) as session:
message = await envelope(session, attachment['id'])
message = await envelope(session, attachment["id"])
assert await process_attachment_job(session, storage, message, settings)
job, item = await jobs.lock_job(session, message.job_id)
assert item.status == 'queued' and job.last_error_code == 'processing_timeout'
assert item.status == "queued" and job.last_error_code == "processing_timeout"
assert job.lease_token is None and job.lease_until is None
async def test_failed_retention_and_source_checksum_protect_replay(client, context, db_engine):
async def test_failed_retention_and_source_checksum_protect_replay(
client, context, db_engine
):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
def fail(_):
raise ProcessingError('invalid_image')
raise ProcessingError("invalid_image")
settings = JobSettings(_env_file=None)
async with AsyncSession(db_engine, expire_on_commit=False) as session:
message = await envelope(session, attachment['id'])
await process_attachment_job(session, storage, message, settings, before_processing=fail)
message = await envelope(session, attachment["id"])
await process_attachment_job(
session, storage, message, settings, before_processing=fail
)
job, item = await jobs.lock_job(session, message.job_id)
key = job.staging_key
job.finished_at = await jobs.database_time(session) - timedelta(days=2)
await session.commit()
assert await cleanup_staging(session, storage, settings, apply=True) == 0
data, metadata = storage.objects[key]
storage.objects[key] = (b'x' * len(data), metadata)
with pytest.raises(JobOperationError, match='source_mismatch'):
await replay_failed_job(session, storage, settings, message.job_id, request_id=uuid4(), actor='test', reason='manual_review')
storage.objects[key] = (b"x" * len(data), metadata)
with pytest.raises(JobOperationError, match="source_mismatch"):
await replay_failed_job(
session,
storage,
settings,
message.job_id,
request_id=uuid4(),
actor="test",
reason="manual_review",
)
await session.rollback()
assert not (await session.scalars(select(JobReplay))).all()
job, item = await jobs.lock_job(session, message.job_id)
+6 -6
View File
@@ -20,9 +20,9 @@ async def test_draining_keeps_liveness_but_rejects_new_work(make_settings):
app.state.lifecycle.start_draining()
try:
async with make_client(app) as client:
assert (await client.get('/health/live')).status_code == 200
assert (await client.get('/health/ready')).status_code == 503
assert (await client.get('/api/v1/users/me')).status_code == 503
assert (await client.get("/health/live")).status_code == 200
assert (await client.get("/health/ready")).status_code == 503
assert (await client.get("/api/v1/users/me")).status_code == 503
finally:
await close_app(app)
@@ -36,13 +36,13 @@ async def test_readiness_timeout_cancels_dependency_checks(make_settings, monkey
finally:
cancelled.append(True)
monkeypatch.setattr(health, 'check_database', stalled)
monkeypatch.setattr(health, 'check_redis', stalled)
monkeypatch.setattr(health, "check_database", stalled)
monkeypatch.setattr(health, "check_redis", stalled)
app = create_app(make_settings(readiness_timeout_seconds=0.03))
try:
async with make_client(app) as client:
started = time.monotonic()
response = await client.get('/health/ready')
response = await client.get("/health/ready")
assert response.status_code == 503
assert time.monotonic() - started < 1
assert len(cancelled) == 2
+31 -7
View File
@@ -6,7 +6,13 @@ import httpx
from app.main import create_app
from conftest import close_app, make_client
from helpers import assert_request_id, bearer, create_chat, create_user, idempotency_header
from helpers import (
assert_request_id,
bearer,
create_chat,
create_user,
idempotency_header,
)
def _assert_error(response: httpx.Response, status: int, code: str) -> None:
@@ -25,12 +31,16 @@ async def test_message_limit_rejects_without_deleting_confirmed_messages(make_se
for index in range(2):
ok = await client.post(
url, headers={**bearer(user), **idempotency_header()}, json={"text": f"confirmed-{index}"}
url,
headers={**bearer(user), **idempotency_header()},
json={"text": f"confirmed-{index}"},
)
assert ok.status_code == 201
rejected = await client.post(
url, headers={**bearer(user), **idempotency_header()}, json={"text": "rejected"}
url,
headers={**bearer(user), **idempotency_header()},
json={"text": "rejected"},
)
_assert_error(rejected, 503, "unavailable")
@@ -65,12 +75,18 @@ async def test_in_flight_limit_returns_429_and_recovers(make_settings, monkeypat
user = await create_user(client)
chat = await create_chat(client, user)
monkeypatch.setattr(message_endpoint, "send_message_idempotent", blocking_send_message)
monkeypatch.setattr(
message_endpoint, "send_message_idempotent", blocking_send_message
)
url = f"/api/v1/chats/{chat['id']}/messages"
held = [
asyncio.create_task(
client.post(url, headers={**bearer(user), **idempotency_header()}, json={"text": f"held-{i}"})
client.post(
url,
headers={**bearer(user), **idempotency_header()},
json={"text": f"held-{i}"},
)
)
for i in range(2)
]
@@ -79,7 +95,11 @@ async def test_in_flight_limit_returns_429_and_recovers(make_settings, monkeypat
rejected = await asyncio.gather(
*(
client.post(url, headers={**bearer(user), **idempotency_header()}, json={"text": f"rejected-{i}"})
client.post(
url,
headers={**bearer(user), **idempotency_header()},
json={"text": f"rejected-{i}"},
)
for i in range(4)
)
)
@@ -92,7 +112,11 @@ async def test_in_flight_limit_returns_429_and_recovers(make_settings, monkeypat
release.set()
assert all(r.status_code == 201 for r in await asyncio.gather(*held))
recovered = await client.post(url, headers={**bearer(user), **idempotency_header()}, json={"text": "after"})
recovered = await client.post(
url,
headers={**bearer(user), **idempotency_header()},
json={"text": "after"},
)
assert recovered.status_code == 201
assert_request_id(recovered)
await close_app(app)
+16 -4
View File
@@ -8,7 +8,19 @@ from sqlalchemy import inspect, text
from app.models import Base
from conftest import reset_schema, upgrade_head
EXPECTED_TABLES = {"users", "chats", "chat_members", "messages", "counters", "idempotency_keys", "attachments", "message_attachments", "upload_intents", "attachment_jobs", "job_replays"}
EXPECTED_TABLES = {
"users",
"chats",
"chat_members",
"messages",
"counters",
"idempotency_keys",
"attachments",
"message_attachments",
"upload_intents",
"attachment_jobs",
"job_replays",
}
@pytest.fixture
@@ -62,6 +74,6 @@ async def test_models_and_migrations_are_in_sync(migrated_engine):
async with migrated_engine.connect() as connection:
diff = await connection.run_sync(compare)
assert diff == [], (
f"Создайте миграцию: alembic revision --autogenerate. Diff: {diff}"
)
assert (
diff == []
), f"Создайте миграцию: alembic revision --autogenerate. Diff: {diff}"
+9 -2
View File
@@ -10,10 +10,12 @@ from app.storage.s3 import ObjectStream, run_sync
async def test_sdk_thread_finishes_before_cancellation_returns():
entered, release, finished = Event(), Event(), Event()
def operation():
entered.set()
release.wait(timeout=3)
finished.set()
task = asyncio.create_task(run_sync(operation))
await asyncio.to_thread(entered.wait, 2)
task.cancel()
@@ -31,10 +33,12 @@ async def test_response_discarded_when_get_is_cancelled():
entered, release = Event(), Event()
discarded = []
response = object()
def operation():
entered.set()
release.wait(timeout=3)
return response
task = asyncio.create_task(run_sync(operation, discard_result=discarded.append))
await asyncio.to_thread(entered.wait, 2)
task.cancel()
@@ -48,9 +52,12 @@ async def test_response_discarded_when_get_is_cancelled():
async def test_stream_closes_even_before_first_iteration():
calls = []
response = SimpleNamespace(close=lambda: calls.append('close'), release_conn=lambda: calls.append('release'))
response = SimpleNamespace(
close=lambda: calls.append("close"),
release_conn=lambda: calls.append("release"),
)
storage = SimpleNamespace(settings=S3Settings(_env_file=None))
stream = ObjectStream(storage, response)
await stream.aclose()
await stream.aclose()
assert calls == ['close', 'release']
assert calls == ["close", "release"]