From e8becaf761c7a6487e85cc93974961e53daa0153 Mon Sep 17 00:00:00 2001 From: thefankor Date: Tue, 17 Nov 2026 18:21:09 +0300 Subject: [PATCH] =?UTF-8?q?=D0=A0=D0=B5=D0=B0=D0=BB=D0=B8=D0=B7=D0=BE?= =?UTF-8?q?=D0=B2=D0=B0=D1=82=D1=8C=20=D0=B0=D1=81=D0=B8=D0=BD=D1=85=D1=80?= =?UTF-8?q?=D0=BE=D0=BD=D0=BD=D1=83=D1=8E=20=D0=BE=D0=B1=D1=80=D0=B0=D0=B1?= =?UTF-8?q?=D0=BE=D1=82=D0=BA=D1=83=20=D0=B2=D0=BB=D0=BE=D0=B6=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 10 + Makefile | 57 ++- .../20261116_0004_async_attachments.py | 126 ++++++ app/api/routers/attachments.py | 14 +- app/api/schemas.py | 23 +- app/config/settings.py | 31 ++ app/domain/models.py | 41 +- app/jobs/__init__.py | 1 + app/jobs/broker.py | 56 +++ app/jobs/pipeline.py | 98 +++++ app/jobs/publisher.py | 64 ++++ app/jobs/runtime.py | 30 ++ app/jobs/worker.py | 274 ++++++++++++++ app/main.py | 9 +- app/models/__init__.py | 7 +- app/models/attachment.py | 25 +- app/models/job.py | 70 ++++ app/repositories/__init__.py | 6 +- app/repositories/attachments.py | 102 ++++- app/repositories/jobs.py | 358 ++++++++++++++++++ app/repositories/messages.py | 4 +- app/services/attachments.py | 42 +- app/storage/s3.py | 14 + app/tools/cleanup_orphans.py | 6 +- app/tools/cleanup_staging.py | 68 ++++ app/tools/reconcile_attachments.py | 6 +- app/tools/replay_failed.py | 45 +++ app/tools/seed_reconciler_benchmark.py | 6 +- compose.tests.yaml | 78 ++++ compose.yaml | 92 +++++ requirements.txt | 1 + scripts/crash-publisher-after-confirm.sh | 16 + scripts/crash-worker.sh | 26 ++ scripts/replay-attachment.sh | 11 + scripts/restart-api.sh | 11 + scripts/restart-broker.sh | 6 + scripts/restart.sh | 8 +- scripts/start-publisher.sh | 6 + scripts/start-worker.sh | 8 + scripts/stop-publisher.sh | 6 + scripts/stop-worker.sh | 8 + tests/attachment_failover.py | 5 +- tests/broker_persistence.py | 47 +++ tests/failed_replay.py | 58 +++ tests/outbox_recovery.py | 36 ++ tests/pipeline_smoke.py | 45 +++ tests/publisher_confirm_crash.py | 36 ++ tests/test_attachments.py | 4 +- tests/test_pipeline.py | 89 +++++ tests/worker_crash.py | 41 ++ tests/worker_outage.py | 61 +++ 51 files changed, 2234 insertions(+), 58 deletions(-) create mode 100644 alembic/versions/20261116_0004_async_attachments.py create mode 100644 app/jobs/__init__.py create mode 100644 app/jobs/broker.py create mode 100644 app/jobs/pipeline.py create mode 100644 app/jobs/publisher.py create mode 100644 app/jobs/runtime.py create mode 100644 app/jobs/worker.py create mode 100644 app/models/job.py create mode 100644 app/repositories/jobs.py create mode 100644 app/tools/cleanup_staging.py create mode 100644 app/tools/replay_failed.py create mode 100755 scripts/crash-publisher-after-confirm.sh create mode 100755 scripts/crash-worker.sh create mode 100755 scripts/replay-attachment.sh create mode 100755 scripts/restart-api.sh create mode 100755 scripts/restart-broker.sh create mode 100755 scripts/start-publisher.sh create mode 100755 scripts/start-worker.sh create mode 100755 scripts/stop-publisher.sh create mode 100755 scripts/stop-worker.sh create mode 100755 tests/broker_persistence.py create mode 100755 tests/failed_replay.py create mode 100755 tests/outbox_recovery.py create mode 100755 tests/pipeline_smoke.py create mode 100755 tests/publisher_confirm_crash.py create mode 100644 tests/test_pipeline.py create mode 100755 tests/worker_crash.py create mode 100755 tests/worker_outage.py diff --git a/.env.example b/.env.example index 926229c8..2d7a414a 100644 --- a/.env.example +++ b/.env.example @@ -20,15 +20,25 @@ MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 BROKER_URL_FILE=/run/secrets/broker_url WORKER_CONCURRENCY=2 +WORKER_PREFETCH=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 +JOB_LEASE_SECONDS=90 +RETRY_BASE_SECONDS=2 +RETRY_MAX_SECONDS=60 STAGING_RETENTION_SECONDS=86400 +THUMBNAIL_JPEG_QUALITY=85 +CROP_JPEG_QUALITY=85 +OPTIMIZED_WEBP_QUALITY=82 POSTGRES_USER=messenger POSTGRES_PASSWORD=change-me-local POSTGRES_DB=messenger DATABASE_URL=postgresql+asyncpg://messenger:change-me-local@postgres:5432/messenger REDIS_URL=redis://redis:6379/0 +BROKER_USER=messenger +BROKER_PASSWORD=change-me-rabbitmq +BROKER_URL=amqp://messenger:change-me-rabbitmq@broker:5672/ MINIO_ROOT_USER=local-minio-admin MINIO_ROOT_PASSWORD=change-me-minio-root diff --git a/Makefile b/Makefile index 5d86d171..412d0601 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test test-attachments test-pipeline test-pipeline-smoke test-outbox test-publisher-confirm-crash test-worker-crash test-failed-replay test-worker-outage test-broker-persistence test-docker load test-persistence test-failover replay-failed cleanup-staging help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -18,6 +18,54 @@ validate: test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) +test-attachments: + $(PYTHON) -m pytest -q tests/test_attachments.py + +test-pipeline: + $(PYTHON) -m pytest -q tests/test_pipeline.py + +test-pipeline-smoke: + $(PYTHON) tests/pipeline_smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-outbox: + $(PYTHON) tests/outbox_recovery.py --base-url "$(BASE_URL)" \ + --stop-publisher-script scripts/stop-publisher.sh \ + --start-publisher-script scripts/start-publisher.sh $(TLS_ARGS) + +test-publisher-confirm-crash: + $(PYTHON) tests/publisher_confirm_crash.py --base-url "$(BASE_URL)" \ + --stop-publisher-script scripts/stop-publisher.sh \ + --crash-publisher-script scripts/crash-publisher-after-confirm.sh $(TLS_ARGS) + +test-worker-crash: + $(PYTHON) tests/worker_crash.py --base-url "$(BASE_URL)" \ + --stop-worker-script scripts/stop-worker.sh \ + --crash-worker-script scripts/crash-worker.sh --mode objects $(TLS_ARGS) + $(PYTHON) tests/worker_crash.py --base-url "$(BASE_URL)" \ + --stop-worker-script scripts/stop-worker.sh \ + --crash-worker-script scripts/crash-worker.sh --mode commit $(TLS_ARGS) + +test-failed-replay: + $(PYTHON) tests/failed_replay.py --base-url "$(BASE_URL)" \ + --stop-worker-script scripts/stop-worker.sh \ + --crash-worker-script scripts/crash-worker.sh \ + --replay-script scripts/replay-attachment.sh $(TLS_ARGS) + +test-worker-outage: + $(PYTHON) tests/worker_outage.py --base-url "$(BASE_URL)" \ + --stop-worker-script scripts/stop-worker.sh \ + --restart-api-script scripts/restart-api.sh \ + --start-worker-script scripts/start-worker.sh $(TLS_ARGS) + +test-broker-persistence: + $(PYTHON) tests/broker_persistence.py --base-url "$(BASE_URL)" \ + --stop-worker-script scripts/stop-worker.sh \ + --restart-broker-script scripts/restart-broker.sh \ + --start-worker-script scripts/start-worker.sh $(TLS_ARGS) + +test-docker: + docker compose -f compose.tests.yaml run --rm tests + load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) @@ -28,3 +76,10 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +replay-failed: + @test -n "$(JOB_ID)" || (echo "Set JOB_ID"; exit 2) + docker compose --env-file .env run --rm publisher python -m app.tools.replay_failed --job-id "$(JOB_ID)" + +cleanup-staging: + docker compose --env-file .env run --rm worker python -m app.tools.cleanup_staging --dry-run diff --git a/alembic/versions/20261116_0004_async_attachments.py b/alembic/versions/20261116_0004_async_attachments.py new file mode 100644 index 00000000..0e37c9c4 --- /dev/null +++ b/alembic/versions/20261116_0004_async_attachments.py @@ -0,0 +1,126 @@ +"""Add asynchronous attachment jobs, variants and transactional outbox. + +Revision ID: 20261116_0004 +Revises: 20261030_0003 +Create Date: 2026-11-16 +""" +from alembic import op +import sqlalchemy as sa + + +revision = '20261116_0004' +down_revision = '20261030_0003' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.alter_column('attachments', 'object_key', new_column_name='original_key') + op.alter_column('attachments', 'original_key', nullable=True) + op.add_column('attachments', sa.Column('staging_key', sa.String(length=255), nullable=True)) + op.add_column( + 'attachments', + sa.Column('pipeline_version', sa.Integer(), nullable=False, server_default='1'), + ) + op.create_unique_constraint('attachments_staging_key_key', 'attachments', ['staging_key']) + + op.create_table( + 'attachment_variants', + sa.Column('attachment_id', sa.Uuid(), nullable=False), + sa.Column('pipeline_version', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=16), nullable=False), + sa.Column('object_key', sa.String(length=255), nullable=False), + sa.Column('content_type', sa.String(length=32), nullable=False), + sa.Column('size_bytes', sa.BigInteger(), nullable=False), + sa.Column('width', sa.Integer(), nullable=False), + sa.Column('height', sa.Integer(), nullable=False), + sa.Column('sha256', sa.String(length=64), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.CheckConstraint("name IN ('thumbnail', 'crop', 'optimized')", name='variant_name_check'), + sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('attachment_id', 'pipeline_version', 'name'), + sa.UniqueConstraint('object_key', name='attachment_variants_object_key_key'), + ) + + op.create_table( + 'attachment_jobs', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('attachment_id', sa.Uuid(), nullable=False), + sa.Column('pipeline_version', sa.Integer(), nullable=False), + sa.Column('state', sa.String(length=16), nullable=False), + sa.Column('attempt', sa.Integer(), nullable=False), + sa.Column('max_attempts', sa.Integer(), nullable=False), + sa.Column('next_attempt_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('lease_owner', sa.String(length=128), nullable=True), + sa.Column('lease_until', sa.DateTime(timezone=True), nullable=True), + sa.Column('last_error_code', sa.String(length=64), nullable=True), + sa.Column('replay_count', sa.Integer(), nullable=False, server_default='0'), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.CheckConstraint( + "state IN ('queued', 'processing', 'ready', 'failed')", + name='attachment_job_state_check', + ), + sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint( + 'attachment_id', 'pipeline_version', name='attachment_job_attachment_version_key' + ), + ) + op.create_index( + 'attachment_jobs_lease_idx', + 'attachment_jobs', + ['state', 'lease_until'], + ) + + op.create_table( + 'outbox_events', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('event_type', sa.String(length=64), nullable=False), + sa.Column('job_id', sa.Uuid(), nullable=False), + sa.Column('payload', sa.JSON(), nullable=False), + sa.Column('available_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('published_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('claim_owner', sa.String(length=128), nullable=True), + sa.Column('claim_until', sa.DateTime(timezone=True), nullable=True), + sa.Column('publish_attempts', sa.Integer(), nullable=False, server_default='0'), + sa.Column('last_error_code', sa.String(length=64), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['job_id'], ['attachment_jobs.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + ) + op.create_index( + 'outbox_events_pending_idx', + 'outbox_events', + ['published_at', 'available_at'], + ) + + op.create_table( + 'job_events', + sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column('job_id', sa.Uuid(), nullable=False), + sa.Column('kind', sa.String(length=32), nullable=False), + sa.Column('attempt', sa.Integer(), nullable=False), + sa.Column('error_code', sa.String(length=64), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['job_id'], ['attachment_jobs.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + ) + op.create_index('job_events_job_created_idx', 'job_events', ['job_id', 'created_at']) + + +def downgrade() -> None: + op.drop_index('job_events_job_created_idx', table_name='job_events') + op.drop_table('job_events') + op.drop_index('outbox_events_pending_idx', table_name='outbox_events') + op.drop_table('outbox_events') + op.drop_index('attachment_jobs_lease_idx', table_name='attachment_jobs') + op.drop_table('attachment_jobs') + op.drop_table('attachment_variants') + op.drop_constraint('attachments_staging_key_key', 'attachments', type_='unique') + op.drop_column('attachments', 'pipeline_version') + op.drop_column('attachments', 'staging_key') + op.alter_column('attachments', 'original_key', nullable=False) + op.alter_column('attachments', 'original_key', new_column_name='object_key') diff --git a/app/api/routers/attachments.py b/app/api/routers/attachments.py index e799f5cf..d77754de 100644 --- a/app/api/routers/attachments.py +++ b/app/api/routers/attachments.py @@ -14,7 +14,7 @@ router = APIRouter(tags=['attachments']) @router.post( '/api/v1/chats/{chat_id}/attachments', - status_code=201, + status_code=202, response_model=AttachmentOut, ) async def upload( @@ -51,16 +51,14 @@ async def content( actor: User = Depends(current_user), attachments: AttachmentService = Depends(attachment_service), ) -> StreamingResponse: - if variant != 'original': - from ...domain.errors import NotFound - - raise NotFound('вариант вложения не найден') - attachment, stream = await attachments.content(actor, attachment_id) + attachment, stream, content_type, size_bytes = await attachments.content( + actor, attachment_id, variant + ) return StreamingResponse( stream.chunks(), - media_type=attachment.content_type, + media_type=content_type, headers={ - 'Content-Length': str(attachment.size_bytes), + 'Content-Length': str(size_bytes), 'Content-Disposition': f'inline; filename="attachment-{attachment.id}"', }, ) diff --git a/app/api/schemas.py b/app/api/schemas.py index acdc2214..9a649deb 100644 --- a/app/api/schemas.py +++ b/app/api/schemas.py @@ -134,6 +134,15 @@ class MessageOut(Strict): ) +class VariantOut(Strict): + name: str + content_type: str + size_bytes: int + width: int + height: int + sha256: str + + class AttachmentOut(Strict): id: UUID chat_id: UUID @@ -143,7 +152,7 @@ class AttachmentOut(Strict): size_bytes: int sha256: str status: str - variants: list = Field(default_factory=list) + variants: list[VariantOut] = Field(default_factory=list) error_code: str | None created_at: str updated_at: str @@ -159,7 +168,17 @@ class AttachmentOut(Strict): size_bytes=attachment.size_bytes, sha256=attachment.sha256, status=attachment.status, - variants=[], + variants=[ + VariantOut( + name=item.name, + content_type=item.content_type, + size_bytes=item.size_bytes, + width=item.width, + height=item.height, + sha256=item.sha256, + ) + for item in attachment.variants + ], error_code=attachment.error_code, created_at=to_rfc3339(attachment.created_at), updated_at=to_rfc3339(attachment.updated_at), diff --git a/app/config/settings.py b/app/config/settings.py index 92aa9469..49e80ef5 100644 --- a/app/config/settings.py +++ b/app/config/settings.py @@ -27,6 +27,18 @@ class Settings: s3_secret_access_key: str max_upload_bytes: int max_image_pixels: int + broker_url: str + worker_concurrency: int + worker_prefetch: int + job_max_attempts: int + job_timeout_seconds: int + job_lease_seconds: int + retry_base_seconds: int + retry_max_seconds: int + staging_retention_seconds: int + thumbnail_jpeg_quality: int + crop_jpeg_quality: int + optimized_webp_quality: int def _positive_int(name: str, default: str) -> int: @@ -39,6 +51,13 @@ def _positive_int(name: str, default: str) -> int: return value +def _quality(name: str, default: str) -> int: + value = _positive_int(name, default) + if value > 100: + raise RuntimeError(f'{name} должен быть не больше 100') + return value + + def load_settings() -> Settings: return Settings( database_url=_read_value('DATABASE_URL', 'DATABASE_URL_FILE'), @@ -51,4 +70,16 @@ def load_settings() -> Settings: s3_secret_access_key=_read_value('S3_SECRET_ACCESS_KEY', 'S3_SECRET_ACCESS_KEY_FILE'), max_upload_bytes=_positive_int('MAX_UPLOAD_BYTES', '10485760'), max_image_pixels=_positive_int('MAX_IMAGE_PIXELS', '40000000'), + broker_url=_read_value('BROKER_URL', 'BROKER_URL_FILE'), + worker_concurrency=_positive_int('WORKER_CONCURRENCY', '2'), + worker_prefetch=_positive_int('WORKER_PREFETCH', '2'), + job_max_attempts=_positive_int('JOB_MAX_ATTEMPTS', '5'), + job_timeout_seconds=_positive_int('JOB_TIMEOUT_SECONDS', '60'), + job_lease_seconds=_positive_int('JOB_LEASE_SECONDS', '90'), + retry_base_seconds=_positive_int('RETRY_BASE_SECONDS', '2'), + retry_max_seconds=_positive_int('RETRY_MAX_SECONDS', '60'), + staging_retention_seconds=_positive_int('STAGING_RETENTION_SECONDS', '86400'), + thumbnail_jpeg_quality=_quality('THUMBNAIL_JPEG_QUALITY', '85'), + crop_jpeg_quality=_quality('CROP_JPEG_QUALITY', '85'), + optimized_webp_quality=_quality('OPTIMIZED_WEBP_QUALITY', '82'), ) diff --git a/app/domain/models.py b/app/domain/models.py index 41df1fdb..8f737405 100644 --- a/app/domain/models.py +++ b/app/domain/models.py @@ -39,7 +39,9 @@ class Attachment: chat_id: UUID owner_id: UUID filename: str - object_key: str + original_key: str | None + staging_key: str | None + pipeline_version: int content_type: str size_bytes: int sha256: str @@ -49,6 +51,43 @@ class Attachment: error_code: str | None created_at: datetime updated_at: datetime + variants: tuple['AttachmentVariant', ...] = () + + +@dataclass(frozen=True, slots=True) +class AttachmentVariant: + name: str + object_key: str + content_type: str + size_bytes: int + width: int + height: int + sha256: str + + +@dataclass(frozen=True, slots=True) +class AttachmentJob: + id: UUID + attachment_id: UUID + pipeline_version: int + state: str + attempt: int + max_attempts: int + next_attempt_at: datetime + lease_owner: str | None + lease_until: datetime | None + last_error_code: str | None + replay_count: int + created_at: datetime + updated_at: datetime + + +@dataclass(frozen=True, slots=True) +class OutboxEvent: + id: UUID + job_id: UUID + payload: dict + available_at: datetime @dataclass(frozen=True, slots=True) diff --git a/app/jobs/__init__.py b/app/jobs/__init__.py new file mode 100644 index 00000000..160c2830 --- /dev/null +++ b/app/jobs/__init__.py @@ -0,0 +1 @@ +"""Durable attachment job publisher and worker.""" diff --git a/app/jobs/broker.py b/app/jobs/broker.py new file mode 100644 index 00000000..4a2f5d67 --- /dev/null +++ b/app/jobs/broker.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import asyncio +import logging + +import aio_pika +from aio_pika import DeliveryMode, ExchangeType + +MAIN_EXCHANGE = 'attachments' +MAIN_QUEUE = 'attachments.process' +ROUTING_KEY = 'attachment.process.v1' +DEAD_EXCHANGE = 'attachments.dead' +DEAD_QUEUE = 'attachments.failed' +logger = logging.getLogger('app.broker') + + +async def connect(url: str) -> aio_pika.RobustConnection: + delay = 0.5 + while True: + try: + return await aio_pika.connect_robust(url, timeout=5) + except (OSError, aio_pika.exceptions.AMQPException): + logger.warning('broker is not ready; reconnecting in %.1fs', delay) + await asyncio.sleep(delay) + delay = min(delay * 2, 5.0) + + +async def topology(channel: aio_pika.abc.AbstractChannel): + dead_exchange = await channel.declare_exchange( + DEAD_EXCHANGE, ExchangeType.DIRECT, durable=True + ) + dead_queue = await channel.declare_queue(DEAD_QUEUE, durable=True) + await dead_queue.bind(dead_exchange, ROUTING_KEY) + exchange = await channel.declare_exchange( + MAIN_EXCHANGE, ExchangeType.DIRECT, durable=True + ) + queue = await channel.declare_queue( + MAIN_QUEUE, + durable=True, + arguments={ + 'x-dead-letter-exchange': DEAD_EXCHANGE, + 'x-dead-letter-routing-key': ROUTING_KEY, + }, + ) + await queue.bind(exchange, ROUTING_KEY) + return exchange, queue + + +def message(body: bytes, event_id: str) -> aio_pika.Message: + return aio_pika.Message( + body=body, + content_type='application/json', + delivery_mode=DeliveryMode.PERSISTENT, + message_id=event_id, + type=ROUTING_KEY, + ) diff --git a/app/jobs/pipeline.py b/app/jobs/pipeline.py new file mode 100644 index 00000000..ed8bc5f2 --- /dev/null +++ b/app/jobs/pipeline.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from io import BytesIO + +from PIL import Image, ImageOps, UnidentifiedImageError + + +class PermanentImageError(Exception): + def __init__(self, code: str) -> None: + super().__init__(code) + self.code = code + + +@dataclass(frozen=True, slots=True) +class GeneratedVariant: + name: str + content_type: str + width: int + height: int + sha256: str + content: bytes + + +def _jpeg_image(image: Image.Image) -> Image.Image: + if image.mode in {'RGBA', 'LA'} or 'transparency' in image.info: + rgba = image.convert('RGBA') + background = Image.new('RGBA', rgba.size, 'white') + background.alpha_composite(rgba) + return background.convert('RGB') + return image.convert('RGB') + + +def _encode(image: Image.Image, fmt: str, **options) -> bytes: + output = BytesIO() + image.save(output, format=fmt, **options) + return output.getvalue() + + +def _variant(name: str, content_type: str, image: Image.Image, content: bytes) -> GeneratedVariant: + return GeneratedVariant( + name=name, + content_type=content_type, + width=image.width, + height=image.height, + sha256=hashlib.sha256(content).hexdigest(), + content=content, + ) + + +def process_image( + source: bytes, + declared_type: str, + expected_sha256: str, + max_pixels: int, + thumbnail_quality: int, + crop_quality: int, + webp_quality: int, +) -> tuple[GeneratedVariant, ...]: + if hashlib.sha256(source).hexdigest() != expected_sha256: + raise PermanentImageError('checksum_mismatch') + try: + with Image.open(BytesIO(source)) as opened: + actual = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}.get(opened.format or '') + if actual != declared_type: + raise PermanentImageError('unsupported_media_type') + if opened.width * opened.height > max_pixels: + raise PermanentImageError('image_too_large') + opened.load() + oriented = ImageOps.exif_transpose(opened) + base = oriented.copy() + except PermanentImageError: + raise + except (Image.DecompressionBombError, UnidentifiedImageError, OSError, SyntaxError) as error: + raise PermanentImageError('invalid_image') from error + + thumbnail = _jpeg_image(base.copy()) + thumbnail.thumbnail((256, 256), Image.Resampling.LANCZOS, reducing_gap=3.0) + thumbnail_bytes = _encode( + thumbnail, 'JPEG', quality=thumbnail_quality, optimize=True, progressive=True + ) + + crop_source = _jpeg_image(base.copy()) + crop = ImageOps.fit( + crop_source, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5) + ) + crop_bytes = _encode(crop, 'JPEG', quality=crop_quality, optimize=True, progressive=True) + + optimized = base.copy() + optimized.thumbnail((1280, 1280), Image.Resampling.LANCZOS, reducing_gap=3.0) + optimized_bytes = _encode(optimized, 'WEBP', quality=webp_quality, method=6) + + return ( + _variant('thumbnail', 'image/jpeg', thumbnail, thumbnail_bytes), + _variant('crop', 'image/jpeg', crop, crop_bytes), + _variant('optimized', 'image/webp', optimized, optimized_bytes), + ) diff --git a/app/jobs/publisher.py b/app/jobs/publisher.py new file mode 100644 index 00000000..64abb349 --- /dev/null +++ b/app/jobs/publisher.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import os +import socket +from datetime import datetime, timezone + +from ..config import load_settings +from ..repositories import JobRepository, OutboxRepository +from .broker import ROUTING_KEY, connect, message, topology +from .runtime import configure_logging, dependencies + +logger = logging.getLogger('app.publisher') + + +async def run(once: bool = False) -> None: + settings = load_settings() + engine, sessions, _storage = dependencies(settings) + outbox = OutboxRepository(sessions) + jobs = JobRepository(sessions) + owner = f'{socket.gethostname()}:{os.getpid()}' + connection = await connect(settings.broker_url) + try: + channel = await connection.channel(publisher_confirms=True) + exchange, _queue = await topology(channel) + while True: + now = datetime.now(timezone.utc) + await jobs.recover_expired(now) + events = await outbox.claim(owner, now, lease_seconds=30) + for event in events: + try: + body = json.dumps(event.payload, separators=(',', ':')).encode() + await exchange.publish( + message(body, str(event.id)), routing_key=ROUTING_KEY, mandatory=True + ) + if os.getenv('FAULT_PUBLISHER_AFTER_CONFIRM') == '1': + os._exit(91) + await outbox.published(event.id, owner, datetime.now(timezone.utc)) + except Exception: + logger.exception('outbox publish failed event_id=%s', event.id) + await outbox.failed( + event.id, owner, 'broker_unavailable', datetime.now(timezone.utc) + ) + if once: + return + await asyncio.sleep(0.5 if events else 1.0) + finally: + await connection.close() + await engine.dispose() + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--once', action='store_true') + args = parser.parse_args() + configure_logging() + asyncio.run(run(args.once)) + + +if __name__ == '__main__': + main() diff --git a/app/jobs/runtime.py b/app/jobs/runtime.py new file mode 100644 index 00000000..9d35cce9 --- /dev/null +++ b/app/jobs/runtime.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import logging +import sys + +from ..config import Settings +from ..db import create_engine, create_session_factory +from ..storage import S3Storage + + +def configure_logging() -> None: + logging.basicConfig( + level=logging.INFO, + stream=sys.stdout, + format='%(asctime)s %(levelname)s %(name)s %(message)s', + force=True, + ) + + +def dependencies(settings: Settings): + engine = create_engine(settings.database_url) + sessions = create_session_factory(engine) + storage = S3Storage( + endpoint=settings.s3_endpoint, + access_key=settings.s3_access_key_id, + secret_key=settings.s3_secret_access_key, + bucket=settings.s3_bucket, + region=settings.s3_region, + ) + return engine, sessions, storage diff --git a/app/jobs/worker.py b/app/jobs/worker.py new file mode 100644 index 00000000..2aa94332 --- /dev/null +++ b/app/jobs/worker.py @@ -0,0 +1,274 @@ +from __future__ import annotations + +import asyncio +import json +import logging +import multiprocessing +import os +import random +import socket +from datetime import datetime, timedelta, timezone +from uuid import UUID + +import aio_pika + +from ..config import Settings, load_settings +from ..domain.errors import Unavailable +from ..domain.models import AttachmentVariant +from ..repositories import JobRepository +from .broker import connect, topology +from .pipeline import GeneratedVariant, PermanentImageError, process_image +from .runtime import configure_logging, dependencies + +logger = logging.getLogger('app.worker') + + +class ProcessingTimeout(Exception): + pass + + +def _pipeline_child(connection, arguments: tuple, memory_bytes: int) -> None: + try: + if memory_bytes: + try: + import resource + + resource.setrlimit(resource.RLIMIT_AS, (memory_bytes, memory_bytes)) + except (ImportError, OSError, ValueError): + pass + connection.send(('ok', process_image(*arguments))) + except PermanentImageError as error: + connection.send(('permanent', error.code)) + except MemoryError: + connection.send(('temporary', 'processing_memory_limit')) + except BaseException: + connection.send(('temporary', 'processing_failed')) + finally: + connection.close() + + +def _run_isolated(arguments: tuple, timeout: int, memory_bytes: int) -> tuple[GeneratedVariant, ...]: + context = multiprocessing.get_context('spawn') + receiver, sender = context.Pipe(duplex=False) + process = context.Process(target=_pipeline_child, args=(sender, arguments, memory_bytes)) + process.start() + sender.close() + process.join(timeout) + if process.is_alive(): + process.terminate() + process.join(5) + raise ProcessingTimeout('processing_timeout') + if not receiver.poll(): + raise ProcessingTimeout('processing_failed') + kind, result = receiver.recv() + receiver.close() + if kind == 'ok': + return result + if kind == 'permanent': + raise PermanentImageError(result) + raise ProcessingTimeout(result) + + +class Worker: + def __init__(self, settings: Settings, jobs: JobRepository, storage) -> None: + self.settings = settings + self.jobs = jobs + self.storage = storage + self.worker_id = f'{socket.gethostname()}:{os.getpid()}' + self.semaphore = asyncio.Semaphore(settings.worker_concurrency) + + def _delay(self, attempt: int) -> float: + base = min( + self.settings.retry_max_seconds, + self.settings.retry_base_seconds * (2 ** max(0, attempt - 1)), + ) + return base + random.uniform(0, base) + + async def _handle_failure( + self, + incoming: aio_pika.IncomingMessage, + job_id: UUID, + attempt: int, + code: str, + permanent: bool, + ) -> None: + now = datetime.now(timezone.utc) + available_at = now + timedelta(seconds=self._delay(attempt)) + try: + state = await self.jobs.retry_or_fail( + job_id, + self.worker_id, + code, + permanent, + available_at, + now, + ) + except Exception: + logger.exception('failed to persist job failure job_id=%s', job_id) + await incoming.nack(requeue=True) + return + if state == 'failed': + await incoming.reject(requeue=False) + else: + await incoming.ack() + + async def handle(self, incoming: aio_pika.IncomingMessage) -> None: + async with self.semaphore: + try: + payload = json.loads(incoming.body) + if payload.get('schema_version') != 1 or payload.get('type') != 'attachment.process.v1': + await incoming.reject(requeue=False) + return + job_id = UUID(payload['job_id']) + attachment_id = UUID(payload['attachment_id']) + attempt = int(payload['attempt']) + pipeline_version = int(payload['pipeline_version']) + except (ValueError, TypeError, KeyError, json.JSONDecodeError): + await incoming.reject(requeue=False) + return + + current = await self.jobs.state(job_id) + if current is None: + await incoming.reject(requeue=False) + return + if current.attachment_id != attachment_id or current.pipeline_version != pipeline_version: + await incoming.reject(requeue=False) + return + if current.state == 'ready': + if await self.jobs.ready_effect_complete(job_id): + await incoming.ack() + else: + await incoming.reject(requeue=False) + return + if current.state == 'failed': + await incoming.reject(requeue=False) + return + + now = datetime.now(timezone.utc) + claimed = await self.jobs.claim( + job_id, attempt, self.worker_id, now, self.settings.job_lease_seconds + ) + if claimed is None: + await incoming.ack() + return + job, attachment = claimed + if os.getenv('FAULT_WORKER_PERMANENT') == '1': + await self._handle_failure( + incoming, job.id, job.attempt, 'fault_injected', True + ) + os._exit(94) + if not attachment.staging_key: + await self._handle_failure( + incoming, job.id, job.attempt, 'staging_missing', True + ) + return + + try: + source = await self.storage.read( + attachment.staging_key, self.settings.max_upload_bytes + ) + arguments = ( + source, + attachment.content_type, + attachment.sha256, + self.settings.max_image_pixels, + self.settings.thumbnail_jpeg_quality, + self.settings.crop_jpeg_quality, + self.settings.optimized_webp_quality, + ) + generated = await asyncio.to_thread( + _run_isolated, + arguments, + self.settings.job_timeout_seconds, + 512 * 1024 * 1024, + ) + original_key = f'originals/{attachment.id}/v{job.pipeline_version}/original' + await self.storage.put_bytes(original_key, source, attachment.content_type) + variants: list[AttachmentVariant] = [] + suffixes = {'thumbnail': 'jpg', 'crop': 'jpg', 'optimized': 'webp'} + for item in generated: + key = ( + f'variants/{attachment.id}/v{job.pipeline_version}/' + f'{item.name}.{suffixes[item.name]}' + ) + await self.storage.put_bytes(key, item.content, item.content_type) + variants.append( + AttachmentVariant( + name=item.name, + object_key=key, + content_type=item.content_type, + size_bytes=len(item.content), + width=item.width, + height=item.height, + sha256=item.sha256, + ) + ) + if os.getenv('FAULT_WORKER_AFTER_OBJECTS') == '1': + os._exit(92) + completed = await self.jobs.complete( + job.id, + self.worker_id, + original_key, + tuple(variants), + datetime.now(timezone.utc), + ) + if not completed: + await incoming.nack(requeue=True) + return + if os.getenv('FAULT_WORKER_AFTER_COMMIT') == '1': + os._exit(93) + try: + await self.storage.remove(attachment.staging_key) + except Unavailable: + logger.warning('staging cleanup deferred attachment_id=%s', attachment.id) + await incoming.ack() + except PermanentImageError as error: + await self._handle_failure( + incoming, job.id, job.attempt, error.code, True + ) + except ProcessingTimeout as error: + await self._handle_failure( + incoming, job.id, job.attempt, str(error), False + ) + except (Unavailable, OSError, ValueError): + logger.exception('temporary processing failure job_id=%s', job.id) + await self._handle_failure( + incoming, job.id, job.attempt, 'storage_unavailable', False + ) + except Exception: + logger.exception('unexpected processing failure job_id=%s', job.id) + await self._handle_failure( + incoming, job.id, job.attempt, 'processing_failed', False + ) + + +async def run() -> None: + settings = load_settings() + engine, sessions, storage = dependencies(settings) + jobs = JobRepository(sessions) + worker = Worker(settings, jobs, storage) + connection = await connect(settings.broker_url) + try: + channel = await connection.channel() + await channel.set_qos(prefetch_count=settings.worker_prefetch) + _exchange, queue = await topology(channel) + await queue.consume(worker.handle, no_ack=False) + logger.info( + 'worker started id=%s concurrency=%d prefetch=%d', + worker.worker_id, + settings.worker_concurrency, + settings.worker_prefetch, + ) + await asyncio.Future() + finally: + await connection.close() + await engine.dispose() + + +def main() -> None: + configure_logging() + asyncio.run(run()) + + +if __name__ == '__main__': + main() diff --git a/app/main.py b/app/main.py index 66d9c98f..b5b40748 100644 --- a/app/main.py +++ b/app/main.py @@ -24,7 +24,7 @@ from .storage import S3Storage logger = logging.getLogger('app') -DESCRIPTION = 'PostgreSQL хранит метаданные, Redis — сессии, приватный S3 — оригиналы вложений.' +DESCRIPTION = 'PostgreSQL хранит задания, RabbitMQ доставляет их worker, S3 хранит изображения.' def configure_logging() -> None: @@ -69,6 +69,7 @@ def create_app() -> FastAPI: storage, settings.max_upload_bytes, settings.max_image_pixels, + settings.job_max_attempts, ), users=UserService(user_repository), chats=ChatService(chat_repository), @@ -79,7 +80,7 @@ def create_app() -> FastAPI: application.state.redis = redis application.state.storage = storage application.state.ready = True - logger.info('startup storage=postgres+s3 redis_sessions=true') + logger.info('startup storage=postgres+s3 redis_sessions=true async_attachments=true') yield finally: application.state.ready = False @@ -88,8 +89,8 @@ def create_app() -> FastAPI: logger.info('shutdown complete') api = FastAPI( - title='MTUSI Messenger. LR 4', - version='1.4.0', + title='MTUSI Messenger. LR 5', + version='1.5.0', description=DESCRIPTION, lifespan=lifespan, ) diff --git a/app/models/__init__.py b/app/models/__init__.py index 9d34b52b..f26d6933 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -1,13 +1,18 @@ -from .attachment import AttachmentRow, MessageAttachmentRow +from .attachment import AttachmentRow, AttachmentVariantRow, MessageAttachmentRow from .chat import ChatMemberRow, ChatRow from .message import MessageRow from .user import UserRow +from .job import AttachmentJobRow, JobEventRow, OutboxEventRow __all__ = [ 'AttachmentRow', + 'AttachmentVariantRow', + 'AttachmentJobRow', 'ChatMemberRow', 'ChatRow', 'MessageAttachmentRow', 'MessageRow', 'UserRow', + 'JobEventRow', + 'OutboxEventRow', ] diff --git a/app/models/attachment.py b/app/models/attachment.py index 4f89286f..c1444e3a 100644 --- a/app/models/attachment.py +++ b/app/models/attachment.py @@ -14,14 +14,17 @@ class AttachmentRow(Base): __tablename__ = 'attachments' __table_args__ = ( Index('attachments_chat_created_idx', 'chat_id', 'created_at', 'id'), - UniqueConstraint('object_key', name='attachments_object_key_key'), + UniqueConstraint('original_key', name='attachments_object_key_key'), + UniqueConstraint('staging_key', name='attachments_staging_key_key'), ) id: Mapped[UUID] = mapped_column(Uuid, primary_key=True) chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), nullable=False) owner_id: Mapped[UUID] = mapped_column(ForeignKey('users.id'), nullable=False) filename: Mapped[str] = mapped_column(String(255), nullable=False) - object_key: Mapped[str] = mapped_column(String(255), nullable=False) + original_key: Mapped[str | None] = mapped_column(String(255), nullable=True) + staging_key: Mapped[str | None] = mapped_column(String(255), nullable=True) + pipeline_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1) content_type: Mapped[str] = mapped_column(String(32), nullable=False) size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False) sha256: Mapped[str] = mapped_column(String(64), nullable=False) @@ -33,6 +36,24 @@ class AttachmentRow(Base): updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) +class AttachmentVariantRow(Base): + __tablename__ = 'attachment_variants' + + attachment_id: Mapped[UUID] = mapped_column( + ForeignKey('attachments.id', ondelete='CASCADE'), primary_key=True + ) + pipeline_version: Mapped[int] = mapped_column(Integer, primary_key=True) + name: Mapped[str] = mapped_column(String(16), primary_key=True) + object_key: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + content_type: Mapped[str] = mapped_column(String(32), nullable=False) + size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False) + width: Mapped[int] = mapped_column(Integer, nullable=False) + height: Mapped[int] = mapped_column(Integer, nullable=False) + sha256: Mapped[str] = mapped_column(String(64), nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + class MessageAttachmentRow(Base): __tablename__ = 'message_attachments' __table_args__ = ( diff --git a/app/models/job.py b/app/models/job.py new file mode 100644 index 00000000..4c7344bd --- /dev/null +++ b/app/models/job.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer, JSON, String, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.types import Uuid + +from ..db.postgres import Base + + +class AttachmentJobRow(Base): + __tablename__ = 'attachment_jobs' + __table_args__ = ( + UniqueConstraint( + 'attachment_id', 'pipeline_version', name='attachment_job_attachment_version_key' + ), + Index('attachment_jobs_lease_idx', 'state', 'lease_until'), + ) + + id: Mapped[UUID] = mapped_column(Uuid, primary_key=True) + attachment_id: Mapped[UUID] = mapped_column( + ForeignKey('attachments.id', ondelete='CASCADE'), nullable=False + ) + pipeline_version: Mapped[int] = mapped_column(Integer, nullable=False) + state: Mapped[str] = mapped_column(String(16), nullable=False) + attempt: Mapped[int] = mapped_column(Integer, nullable=False) + max_attempts: Mapped[int] = mapped_column(Integer, nullable=False) + next_attempt_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True) + lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True) + replay_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class OutboxEventRow(Base): + __tablename__ = 'outbox_events' + __table_args__ = (Index('outbox_events_pending_idx', 'published_at', 'available_at'),) + + id: Mapped[UUID] = mapped_column(Uuid, primary_key=True) + event_type: Mapped[str] = mapped_column(String(64), nullable=False) + job_id: Mapped[UUID] = mapped_column( + ForeignKey('attachment_jobs.id', ondelete='CASCADE'), nullable=False + ) + payload: Mapped[dict] = mapped_column(JSON, nullable=False) + available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + claim_owner: Mapped[str | None] = mapped_column(String(128), nullable=True) + claim_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + publish_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class JobEventRow(Base): + __tablename__ = 'job_events' + __table_args__ = (Index('job_events_job_created_idx', 'job_id', 'created_at'),) + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) + job_id: Mapped[UUID] = mapped_column( + ForeignKey('attachment_jobs.id', ondelete='CASCADE'), nullable=False + ) + kind: Mapped[str] = mapped_column(String(32), nullable=False) + attempt: Mapped[int] = mapped_column(Integer, nullable=False) + error_code: Mapped[str | None] = mapped_column(String(64), nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) diff --git a/app/repositories/__init__.py b/app/repositories/__init__.py index a80de674..1e0e7bf7 100644 --- a/app/repositories/__init__.py +++ b/app/repositories/__init__.py @@ -2,5 +2,9 @@ from .attachments import AttachmentRepository from .chats import ChatRepository from .messages import MessageRepository from .users import UserRepository +from .jobs import JobRepository, OutboxRepository -__all__ = ['AttachmentRepository', 'ChatRepository', 'MessageRepository', 'UserRepository'] +__all__ = [ + 'AttachmentRepository', 'ChatRepository', 'JobRepository', 'MessageRepository', + 'OutboxRepository', 'UserRepository', +] diff --git a/app/repositories/attachments.py b/app/repositories/attachments.py index d508f651..c4ccf7bd 100644 --- a/app/repositories/attachments.py +++ b/app/repositories/attachments.py @@ -1,14 +1,27 @@ from __future__ import annotations from datetime import datetime -from uuid import UUID +from uuid import UUID, uuid4 from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker -from ..domain.models import Attachment -from ..models.attachment import AttachmentRow +from ..domain.models import Attachment, AttachmentVariant +from ..models.attachment import AttachmentRow, AttachmentVariantRow from ..models.chat import ChatMemberRow, ChatRow +from ..models.job import AttachmentJobRow, OutboxEventRow + + +def job_payload(job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int, now: datetime) -> dict: + return { + 'schema_version': 1, + 'job_id': str(job_id), + 'type': 'attachment.process.v1', + 'attachment_id': str(attachment_id), + 'pipeline_version': pipeline_version, + 'attempt': attempt, + 'created_at': now.isoformat().replace('+00:00', 'Z'), + } class AttachmentRepository: @@ -16,13 +29,29 @@ class AttachmentRepository: self._sessions = sessions @staticmethod - def _to_domain(row: AttachmentRow) -> Attachment: + def _variant(row: AttachmentVariantRow) -> AttachmentVariant: + return AttachmentVariant( + name=row.name, + object_key=row.object_key, + content_type=row.content_type, + size_bytes=row.size_bytes, + width=row.width, + height=row.height, + sha256=row.sha256, + ) + + @classmethod + def _to_domain( + cls, row: AttachmentRow, variants: tuple[AttachmentVariantRow, ...] = () + ) -> Attachment: return Attachment( id=row.id, chat_id=row.chat_id, owner_id=row.owner_id, filename=row.filename, - object_key=row.object_key, + original_key=row.original_key, + staging_key=row.staging_key, + pipeline_version=row.pipeline_version, content_type=row.content_type, size_bytes=row.size_bytes, sha256=row.sha256, @@ -32,6 +61,7 @@ class AttachmentRepository: error_code=row.error_code, created_at=row.created_at, updated_at=row.updated_at, + variants=tuple(cls._variant(item) for item in variants), ) async def chat_exists_for_member(self, chat_id: UUID, user_id: UUID) -> bool: @@ -43,19 +73,21 @@ class AttachmentRepository: ) return found is not None - async def create( + async def create_queued( self, *, attachment_id: UUID, + job_id: UUID, chat_id: UUID, owner_id: UUID, filename: str, - object_key: str, + staging_key: str, content_type: str, size_bytes: int, sha256: str, width: int, height: int, + max_attempts: int, now: datetime, ) -> Attachment: row = AttachmentRow( @@ -63,23 +95,67 @@ class AttachmentRepository: chat_id=chat_id, owner_id=owner_id, filename=filename, - object_key=object_key, + original_key=None, + staging_key=staging_key, + pipeline_version=1, content_type=content_type, size_bytes=size_bytes, sha256=sha256, width=width, height=height, - status='ready', + status='queued', error_code=None, created_at=now, updated_at=now, ) + job = AttachmentJobRow( + id=job_id, + attachment_id=attachment_id, + pipeline_version=1, + state='queued', + attempt=1, + max_attempts=max_attempts, + next_attempt_at=now, + lease_owner=None, + lease_until=None, + last_error_code=None, + replay_count=0, + created_at=now, + updated_at=now, + ) + outbox = OutboxEventRow( + id=uuid4(), + event_type='attachment.process.v1', + job_id=job_id, + payload=job_payload(job_id, attachment_id, 1, 1, now), + available_at=now, + published_at=None, + claim_owner=None, + claim_until=None, + publish_attempts=0, + last_error_code=None, + created_at=now, + updated_at=now, + ) async with self._sessions() as session: async with session.begin(): - session.add(row) + session.add_all((row, job, outbox)) await session.flush() return self._to_domain(row) + async def _variants( + self, session: AsyncSession, attachment_id: UUID, pipeline_version: int + ) -> tuple[AttachmentVariantRow, ...]: + rows = await session.scalars( + select(AttachmentVariantRow) + .where( + AttachmentVariantRow.attachment_id == attachment_id, + AttachmentVariantRow.pipeline_version == pipeline_version, + ) + .order_by(AttachmentVariantRow.name) + ) + return tuple(rows) + async def get_for_member(self, attachment_id: UUID, user_id: UUID) -> Attachment | None: async with self._sessions() as session: row = await session.scalar( @@ -90,4 +166,8 @@ class AttachmentRepository: ChatMemberRow.user_id == user_id, ) ) - return self._to_domain(row) if row is not None else None + if row is None: + return None + return self._to_domain( + row, await self._variants(session, row.id, row.pipeline_version) + ) diff --git a/app/repositories/jobs.py b/app/repositories/jobs.py new file mode 100644 index 00000000..5148678a --- /dev/null +++ b/app/repositories/jobs.py @@ -0,0 +1,358 @@ +from __future__ import annotations + +from datetime import datetime, timedelta +from uuid import UUID, uuid4 + +from sqlalchemy import or_, select, update +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from ..domain.models import Attachment, AttachmentJob, AttachmentVariant, OutboxEvent +from ..models.attachment import AttachmentRow, AttachmentVariantRow +from ..models.job import AttachmentJobRow, JobEventRow, OutboxEventRow +from .attachments import AttachmentRepository, job_payload + + +class JobRepository: + def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None: + self._sessions = sessions + + @staticmethod + def _job(row: AttachmentJobRow) -> AttachmentJob: + return AttachmentJob( + id=row.id, + attachment_id=row.attachment_id, + pipeline_version=row.pipeline_version, + state=row.state, + attempt=row.attempt, + max_attempts=row.max_attempts, + next_attempt_at=row.next_attempt_at, + lease_owner=row.lease_owner, + lease_until=row.lease_until, + last_error_code=row.last_error_code, + replay_count=row.replay_count, + created_at=row.created_at, + updated_at=row.updated_at, + ) + + async def claim( + self, + job_id: UUID, + attempt: int, + worker_id: str, + now: datetime, + lease_seconds: int, + ) -> tuple[AttachmentJob, Attachment] | None: + async with self._sessions() as session: + async with session.begin(): + row = await session.scalar( + select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update() + ) + if row is None or row.state != 'queued' or row.attempt != attempt: + return None + attachment_row = await session.get(AttachmentRow, row.attachment_id) + if attachment_row is None: + return None + row.state = 'processing' + row.lease_owner = worker_id + row.lease_until = now + timedelta(seconds=lease_seconds) + row.updated_at = now + attachment_row.status = 'processing' + attachment_row.updated_at = now + session.add( + JobEventRow( + job_id=row.id, + kind='claimed', + attempt=row.attempt, + error_code=None, + created_at=now, + ) + ) + await session.flush() + return self._job(row), AttachmentRepository._to_domain(attachment_row) + + async def state(self, job_id: UUID) -> AttachmentJob | None: + async with self._sessions() as session: + row = await session.get(AttachmentJobRow, job_id) + return self._job(row) if row is not None else None + + async def id_for_attachment(self, attachment_id: UUID) -> UUID | None: + async with self._sessions() as session: + return await session.scalar( + select(AttachmentJobRow.id).where( + AttachmentJobRow.attachment_id == attachment_id + ) + ) + + async def ready_effect_complete(self, job_id: UUID) -> bool: + async with self._sessions() as session: + job = await session.get(AttachmentJobRow, job_id) + if job is None or job.state != 'ready': + return False + attachment = await session.get(AttachmentRow, job.attachment_id) + if attachment is None or not attachment.original_key: + return False + names = set( + await session.scalars( + select(AttachmentVariantRow.name).where( + AttachmentVariantRow.attachment_id == job.attachment_id, + AttachmentVariantRow.pipeline_version == job.pipeline_version, + ) + ) + ) + return names == {'thumbnail', 'crop', 'optimized'} + + async def complete( + self, + job_id: UUID, + worker_id: str, + original_key: str, + variants: tuple[AttachmentVariant, ...], + now: datetime, + ) -> bool: + async with self._sessions() as session: + async with session.begin(): + job = await session.scalar( + select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update() + ) + if job is None or job.state == 'ready': + return job is not None + if job.state != 'processing' or job.lease_owner != worker_id: + return False + attachment = await session.get(AttachmentRow, job.attachment_id) + if attachment is None: + return False + for variant in variants: + statement = ( + insert(AttachmentVariantRow) + .values( + attachment_id=attachment.id, + pipeline_version=job.pipeline_version, + name=variant.name, + object_key=variant.object_key, + content_type=variant.content_type, + size_bytes=variant.size_bytes, + width=variant.width, + height=variant.height, + sha256=variant.sha256, + created_at=now, + updated_at=now, + ) + .on_conflict_do_update( + index_elements=[ + AttachmentVariantRow.attachment_id, + AttachmentVariantRow.pipeline_version, + AttachmentVariantRow.name, + ], + set_={ + 'object_key': variant.object_key, + 'content_type': variant.content_type, + 'size_bytes': variant.size_bytes, + 'width': variant.width, + 'height': variant.height, + 'sha256': variant.sha256, + 'updated_at': now, + }, + ) + ) + await session.execute(statement) + attachment.original_key = original_key + attachment.status = 'ready' + attachment.error_code = None + attachment.updated_at = now + job.state = 'ready' + job.lease_owner = None + job.lease_until = None + job.last_error_code = None + job.updated_at = now + session.add( + JobEventRow( + job_id=job.id, + kind='ready', + attempt=job.attempt, + error_code=None, + created_at=now, + ) + ) + return True + + async def retry_or_fail( + self, + job_id: UUID, + worker_id: str | None, + error_code: str, + permanent: bool, + available_at: datetime, + now: datetime, + ) -> str: + async with self._sessions() as session: + async with session.begin(): + job = await session.scalar( + select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update() + ) + if job is None or job.state in {'ready', 'failed'}: + return job.state if job is not None else 'missing' + if job.state != 'processing': + return job.state + if worker_id is not None: + if job.lease_owner != worker_id: + return job.state + elif job.lease_until is None or job.lease_until >= now: + return job.state + attachment = await session.get(AttachmentRow, job.attachment_id) + exhausted = job.attempt >= job.max_attempts + if permanent or exhausted: + job.state = 'failed' + attachment.status = 'failed' + attachment.error_code = error_code if permanent else 'retry_exhausted' + kind = 'failed' + else: + job.attempt += 1 + job.state = 'queued' + job.next_attempt_at = available_at + attachment.status = 'queued' + attachment.error_code = None + session.add( + OutboxEventRow( + id=uuid4(), + event_type='attachment.process.v1', + job_id=job.id, + payload=job_payload( + job.id, + job.attachment_id, + job.pipeline_version, + job.attempt, + now, + ), + available_at=available_at, + published_at=None, + claim_owner=None, + claim_until=None, + publish_attempts=0, + last_error_code=None, + created_at=now, + updated_at=now, + ) + ) + kind = 'retry_scheduled' + job.last_error_code = error_code + job.lease_owner = None + job.lease_until = None + job.updated_at = now + attachment.updated_at = now + session.add( + JobEventRow( + job_id=job.id, + kind=kind, + attempt=job.attempt, + error_code=error_code, + created_at=now, + ) + ) + await session.flush() + return job.state + + async def recover_expired(self, now: datetime) -> int: + async with self._sessions() as session: + ids = tuple( + await session.scalars( + select(AttachmentJobRow.id).where( + AttachmentJobRow.state == 'processing', + AttachmentJobRow.lease_until < now, + ) + ) + ) + recovered = 0 + for job_id in ids: + state = await self.retry_or_fail( + job_id, None, 'lease_expired', False, now, now + ) + recovered += state in {'queued', 'failed'} + return recovered + + async def replay(self, job_id: UUID, now: datetime, dry_run: bool = False) -> bool: + async with self._sessions() as session: + async with session.begin(): + job = await session.scalar( + select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update() + ) + if job is None or job.state != 'failed': + return False + attachment = await session.get(AttachmentRow, job.attachment_id) + if attachment is None or not attachment.staging_key: + return False + if dry_run: + return True + job.state = 'queued' + job.attempt += 1 + job.max_attempts = job.attempt + 4 + job.replay_count += 1 + job.last_error_code = None + job.next_attempt_at = now + job.updated_at = now + attachment.status = 'queued' + attachment.error_code = None + attachment.updated_at = now + session.add_all( + ( + OutboxEventRow( + id=uuid4(), event_type='attachment.process.v1', job_id=job.id, + payload=job_payload(job.id, job.attachment_id, job.pipeline_version, job.attempt, now), + available_at=now, published_at=None, claim_owner=None, claim_until=None, + publish_attempts=0, last_error_code=None, created_at=now, updated_at=now, + ), + JobEventRow(job_id=job.id, kind='replayed', attempt=job.attempt, error_code=None, created_at=now), + ) + ) + return True + + +class OutboxRepository: + def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None: + self._sessions = sessions + + async def claim(self, owner: str, now: datetime, lease_seconds: int, limit: int = 20) -> tuple[OutboxEvent, ...]: + async with self._sessions() as session: + async with session.begin(): + rows = tuple( + await session.scalars( + select(OutboxEventRow) + .where( + OutboxEventRow.published_at.is_(None), + OutboxEventRow.available_at <= now, + or_(OutboxEventRow.claim_until.is_(None), OutboxEventRow.claim_until < now), + ) + .order_by(OutboxEventRow.available_at, OutboxEventRow.created_at) + .limit(limit) + .with_for_update(skip_locked=True) + ) + ) + until = now + timedelta(seconds=lease_seconds) + for row in rows: + row.claim_owner = owner + row.claim_until = until + row.publish_attempts += 1 + row.updated_at = now + return tuple(OutboxEvent(row.id, row.job_id, row.payload, row.available_at) for row in rows) + + async def published(self, event_id: UUID, owner: str, now: datetime) -> None: + async with self._sessions() as session: + async with session.begin(): + await session.execute( + update(OutboxEventRow) + .where( + OutboxEventRow.id == event_id, + OutboxEventRow.claim_owner == owner, + OutboxEventRow.published_at.is_(None), + ) + .values(published_at=now, claim_owner=None, claim_until=None, last_error_code=None, updated_at=now) + ) + + async def failed(self, event_id: UUID, owner: str, code: str, now: datetime) -> None: + async with self._sessions() as session: + async with session.begin(): + await session.execute( + update(OutboxEventRow) + .where(OutboxEventRow.id == event_id, OutboxEventRow.claim_owner == owner) + .values(claim_owner=None, claim_until=None, last_error_code=code, updated_at=now) + ) diff --git a/app/repositories/messages.py b/app/repositories/messages.py index 93736b8e..2df72bfb 100644 --- a/app/repositories/messages.py +++ b/app/repositories/messages.py @@ -77,8 +77,8 @@ class MessageRepository: } if set(found) != set(attachment_ids): raise NotFound('вложение не найдено') - if any(status != 'ready' for status in found.values()): - raise Conflict('оригинал вложения недоступен') + if any(status == 'failed' for status in found.values()): + raise Conflict('неуспешное вложение нельзя прикрепить') statement = ( insert(MessageRow) diff --git a/app/services/attachments.py b/app/services/attachments.py index 105c6bf6..e94b53c2 100644 --- a/app/services/attachments.py +++ b/app/services/attachments.py @@ -67,11 +67,13 @@ class AttachmentService: storage: S3Storage, max_upload_bytes: int, max_image_pixels: int, + job_max_attempts: int, ) -> None: self._repository = repository self._storage = storage self._max_upload_bytes = max_upload_bytes self._max_image_pixels = max_image_pixels + self._job_max_attempts = job_max_attempts async def upload( self, @@ -106,28 +108,31 @@ class AttachmentService: self._max_image_pixels, ) attachment_id = uuid4() - object_key = f'originals/{attachment_id}' - await self._storage.put(object_key, source, size, actual_type) + job_id = uuid4() + staging_key = f'staging/{attachment_id}/original' + await self._storage.put(staging_key, source, size, actual_type) now = datetime.now(timezone.utc) try: - return await self._repository.create( + return await self._repository.create_queued( attachment_id=attachment_id, + job_id=job_id, chat_id=chat_id, owner_id=actor.id, filename=filename, - object_key=object_key, + staging_key=staging_key, content_type=actual_type, size_bytes=size, sha256=digest.hexdigest(), width=width, height=height, + max_attempts=self._job_max_attempts, now=now, ) except Exception: try: - await self._storage.remove(object_key) + await self._storage.remove(staging_key) except Unavailable: - logger.warning('failed to compensate orphan object key=%s', object_key) + logger.warning('failed to compensate staging object key=%s', staging_key) raise async def get(self, actor: User, attachment_id: UUID) -> Attachment: @@ -136,12 +141,27 @@ class AttachmentService: raise NotFound('вложение не найдено') return attachment - async def content(self, actor: User, attachment_id: UUID) -> tuple[Attachment, ObjectStream]: + async def content( + self, actor: User, attachment_id: UUID, variant: str + ) -> tuple[Attachment, ObjectStream, str, int]: attachment = await self.get(actor, attachment_id) if attachment.status != 'ready': - raise Conflict('оригинал вложения недоступен') + raise Conflict('вложение ещё не готово') + if variant == 'original': + key = attachment.original_key + content_type = attachment.content_type + size_bytes = attachment.size_bytes + else: + found = next((item for item in attachment.variants if item.name == variant), None) + if found is None: + raise NotFound('вариант вложения не найден') + key = found.object_key + content_type = found.content_type + size_bytes = found.size_bytes + if key is None: + raise Unavailable('объект вложения отсутствует в метаданных') try: - stream = await self._storage.get(attachment.object_key) + stream = await self._storage.get(key) except FileNotFoundError as error: - raise Unavailable('оригинал вложения отсутствует в хранилище') from error - return attachment, stream + raise Unavailable('объект вложения отсутствует в хранилище') from error + return attachment, stream, content_type, size_bytes diff --git a/app/storage/s3.py b/app/storage/s3.py index dcc97ab2..995b75ea 100644 --- a/app/storage/s3.py +++ b/app/storage/s3.py @@ -5,6 +5,7 @@ import certifi from dataclasses import dataclass from datetime import datetime from typing import BinaryIO, Iterator +from io import BytesIO from urllib.parse import urlparse from minio import Minio @@ -57,6 +58,7 @@ class S3Storage: secure=parsed.scheme == 'https', region=region, http_client=PoolManager( + maxsize=8, timeout=Timeout(connect=2, read=5), retries=Retry(total=1, connect=1, read=1, backoff_factor=0.1), cert_reqs='CERT_REQUIRED', @@ -102,6 +104,18 @@ class S3Storage: self._translate(error) return ObjectStream(response) + async def read(self, key: str, max_bytes: int) -> bytes: + stream = await self.get(key) + result = bytearray() + for chunk in stream.chunks(): + result.extend(chunk) + if len(result) > max_bytes: + raise ValueError('stored object exceeds configured byte limit') + return bytes(result) + + async def put_bytes(self, key: str, content: bytes, content_type: str) -> None: + await self.put(key, BytesIO(content), len(content), content_type) + async def exists(self, key: str) -> bool: try: await asyncio.to_thread(self._client.stat_object, self.bucket, key) diff --git a/app/tools/cleanup_orphans.py b/app/tools/cleanup_orphans.py index 65740899..ae5eb14f 100644 --- a/app/tools/cleanup_orphans.py +++ b/app/tools/cleanup_orphans.py @@ -25,7 +25,11 @@ async def cleanup(apply: bool, grace_seconds: int) -> int: try: sessions = create_session_factory(engine) async with sessions() as session: - known = set(await session.scalars(select(AttachmentRow.object_key))) + known = set( + key + for key in await session.scalars(select(AttachmentRow.original_key)) + if key is not None + ) cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds) removed = 0 for item in await storage.list('originals/'): diff --git a/app/tools/cleanup_staging.py b/app/tools/cleanup_staging.py new file mode 100644 index 00000000..301e6c5d --- /dev/null +++ b/app/tools/cleanup_staging.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import argparse +import asyncio +from datetime import datetime, timedelta, timezone + +from sqlalchemy import select + +from ..config import load_settings +from ..db import create_engine, create_session_factory +from ..models.attachment import AttachmentRow +from ..storage import S3Storage + + +async def cleanup(apply: bool, retention_seconds: int) -> int: + settings = load_settings() + engine = create_engine(settings.database_url) + sessions = create_session_factory(engine) + storage = S3Storage( + settings.s3_endpoint, + settings.s3_access_key_id, + settings.s3_secret_access_key, + settings.s3_bucket, + settings.s3_region, + ) + cutoff = datetime.now(timezone.utc) - timedelta(seconds=retention_seconds) + try: + async with sessions() as session: + active = set( + key + for key in await session.scalars( + select(AttachmentRow.staging_key).where( + AttachmentRow.staging_key.is_not(None), + AttachmentRow.status.in_(('queued', 'processing')), + ) + ) + if key is not None + ) + removed = 0 + for item in await storage.list('staging/'): + if item.key in active or item.last_modified > cutoff: + continue + print(f'{"DELETE" if apply else "WOULD_DELETE"} key={item.key}') + if apply: + await storage.remove(item.key) + removed += 1 + print(f'summary active={len(active)} removed={removed} mode={"apply" if apply else "dry-run"}') + return 0 + finally: + await engine.dispose() + + +def main() -> None: + parser = argparse.ArgumentParser(description='Clean inactive durable staging objects') + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument('--dry-run', action='store_true') + mode.add_argument('--apply', action='store_true') + parser.add_argument('--retention-seconds', type=int) + args = parser.parse_args() + settings = load_settings() + retention = args.retention_seconds or settings.staging_retention_seconds + if retention < 60: + parser.error('retention must be at least 60 seconds') + raise SystemExit(asyncio.run(cleanup(args.apply, retention))) + + +if __name__ == '__main__': + main() diff --git a/app/tools/reconcile_attachments.py b/app/tools/reconcile_attachments.py index 7553e7bc..39e1ec3b 100644 --- a/app/tools/reconcile_attachments.py +++ b/app/tools/reconcile_attachments.py @@ -74,7 +74,7 @@ async def _load_attachments( await session.execute( select( AttachmentRow.id, - AttachmentRow.object_key, + AttachmentRow.original_key, AttachmentRow.status, AttachmentRow.updated_at, ) @@ -88,7 +88,7 @@ async def _metadata_key_exists( ) -> bool: async with sessions() as session: attachment_id = await session.scalar( - select(AttachmentRow.id).where(AttachmentRow.object_key == object_key).limit(1) + select(AttachmentRow.id).where(AttachmentRow.original_key == object_key).limit(1) ) return attachment_id is not None @@ -165,7 +165,7 @@ async def reconcile( update(AttachmentRow) .where( AttachmentRow.id == attachment.id, - AttachmentRow.object_key == attachment.object_key, + AttachmentRow.original_key == attachment.object_key, AttachmentRow.status == 'ready', AttachmentRow.updated_at == attachment.updated_at, ) diff --git a/app/tools/replay_failed.py b/app/tools/replay_failed.py new file mode 100644 index 00000000..e6bd814b --- /dev/null +++ b/app/tools/replay_failed.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import argparse +import asyncio +from datetime import datetime, timezone +from uuid import UUID + +from ..config import load_settings +from ..db import create_engine, create_session_factory +from ..repositories import JobRepository + + +async def replay(job_id: UUID | None, attachment_id: UUID | None, dry_run: bool) -> int: + settings = load_settings() + engine = create_engine(settings.database_url) + try: + repository = JobRepository(create_session_factory(engine)) + if job_id is None: + assert attachment_id is not None + job_id = await repository.id_for_attachment(attachment_id) + if job_id is None: + print(f'REJECTED attachment_id={attachment_id} reason=job_not_found') + return 2 + accepted = await repository.replay(job_id, datetime.now(timezone.utc), dry_run) + if not accepted: + print(f'REJECTED job_id={job_id} reason=not_failed_or_staging_missing') + return 2 + print(f'{"WOULD_REPLAY" if dry_run else "REPLAYED"} job_id={job_id}') + return 0 + finally: + await engine.dispose() + + +def main() -> None: + parser = argparse.ArgumentParser(description='Safely replay one failed attachment job') + target = parser.add_mutually_exclusive_group(required=True) + target.add_argument('--job-id', type=UUID) + target.add_argument('--attachment-id', type=UUID) + parser.add_argument('--dry-run', action='store_true') + args = parser.parse_args() + raise SystemExit(asyncio.run(replay(args.job_id, args.attachment_id, args.dry_run))) + + +if __name__ == '__main__': + main() diff --git a/app/tools/seed_reconciler_benchmark.py b/app/tools/seed_reconciler_benchmark.py index 7d98ed80..cfc10637 100644 --- a/app/tools/seed_reconciler_benchmark.py +++ b/app/tools/seed_reconciler_benchmark.py @@ -50,7 +50,9 @@ async def prepare(size: int) -> None: 'chat_id': CHAT_ID, 'owner_id': USER_ID, 'filename': f'benchmark-{index}.png', - 'object_key': f'{OBJECT_PREFIX}{index}', + 'original_key': f'{OBJECT_PREFIX}{index}', + 'staging_key': None, + 'pipeline_version': 1, 'content_type': 'image/png', 'size_bytes': 1, 'sha256': '0' * 64, @@ -80,7 +82,7 @@ async def prepare(size: int) -> None: for offset in range(0, size, 500): await session.execute(insert(AttachmentRow), rows[offset : offset + 500]) - await _put_objects(storage, [row['object_key'] for row in rows]) + await _put_objects(storage, [row['original_key'] for row in rows]) print(f'prepared metadata={size} objects={size}') finally: await engine.dispose() diff --git a/compose.tests.yaml b/compose.tests.yaml index 839a5007..232faeb0 100644 --- a/compose.tests.yaml +++ b/compose.tests.yaml @@ -20,6 +20,17 @@ services: timeout: 2s retries: 30 + broker: + image: rabbitmq:4.1.4-alpine + environment: + RABBITMQ_DEFAULT_USER: messenger_test + RABBITMQ_DEFAULT_PASS: messenger_test + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 2s + timeout: 5s + retries: 30 + s3: image: cgr.dev/chainguard/minio@sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597 user: "0" @@ -69,6 +80,7 @@ services: SESSION_TTL_SECONDS: 3600 S3_ACCESS_KEY_ID: messenger-api S3_SECRET_ACCESS_KEY: test-minio-api-password + BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/ depends_on: postgres: condition: service_healthy @@ -76,6 +88,8 @@ services: api: build: . + ports: + - "127.0.0.1:18080:8080" environment: DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test REDIS_URL: redis://redis:6379/0 @@ -87,6 +101,8 @@ services: S3_SECRET_ACCESS_KEY: test-minio-api-password MAX_UPLOAD_BYTES: 10485760 MAX_IMAGE_PIXELS: 40000000 + BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/ + JOB_MAX_ATTEMPTS: 5 depends_on: migrate: condition: service_completed_successfully @@ -100,6 +116,67 @@ services: timeout: 2s retries: 30 + publisher: + build: . + command: ["python", "-m", "app.jobs.publisher"] + environment: + DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test + REDIS_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: 3600 + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID: messenger-api + S3_SECRET_ACCESS_KEY: test-minio-api-password + BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/ + depends_on: + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"] + interval: 5s + timeout: 3s + retries: 10 + + worker: + build: . + command: ["python", "-m", "app.jobs.worker"] + environment: + DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test + REDIS_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: 3600 + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID: messenger-api + S3_SECRET_ACCESS_KEY: test-minio-api-password + BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/ + WORKER_CONCURRENCY: 2 + WORKER_PREFETCH: 2 + JOB_MAX_ATTEMPTS: 5 + JOB_TIMEOUT_SECONDS: 60 + JOB_LEASE_SECONDS: 90 + RETRY_BASE_SECONDS: 1 + RETRY_MAX_SECONDS: 4 + STAGING_RETENTION_SECONDS: 86400 + THUMBNAIL_JPEG_QUALITY: 85 + CROP_JPEG_QUALITY: 85 + OPTIMIZED_WEBP_QUALITY: 82 + depends_on: + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"] + interval: 5s + timeout: 3s + retries: 10 + tests: build: context: . @@ -114,6 +191,7 @@ services: S3_BUCKET: messenger-attachments S3_ACCESS_KEY_ID: messenger-api S3_SECRET_ACCESS_KEY: test-minio-api-password + BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/ depends_on: api: condition: service_healthy diff --git a/compose.yaml b/compose.yaml index 8e1f5878..e77fd565 100644 --- a/compose.yaml +++ b/compose.yaml @@ -28,6 +28,21 @@ services: timeout: 3s retries: 20 + broker: + image: rabbitmq:4.1.4-alpine + cpus: "0.5" + mem_limit: 384m + environment: + RABBITMQ_DEFAULT_USER: ${BROKER_USER:-messenger} + RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:-change-me-rabbitmq} + volumes: + - broker_data:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 5s + retries: 20 + s3: image: cgr.dev/chainguard/minio@sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597 user: "0" @@ -85,6 +100,7 @@ services: SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api} S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api} + BROKER_URL: ${BROKER_URL} depends_on: postgres: condition: service_healthy @@ -106,6 +122,8 @@ services: S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api} MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + BROKER_URL: ${BROKER_URL} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} expose: - "8080" deploy: @@ -124,6 +142,79 @@ services: retries: 5 stop_grace_period: 10s + publisher: + build: . + command: ["python", "-m", "app.jobs.publisher"] + cpus: "0.25" + mem_limit: 192m + environment: + DATABASE_URL: ${DATABASE_URL} + REDIS_URL: ${REDIS_URL} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api} + BROKER_URL: ${BROKER_URL} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90} + depends_on: + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"] + interval: 10s + timeout: 3s + retries: 3 + restart: unless-stopped + stop_grace_period: 10s + + worker: + build: . + command: ["python", "-m", "app.jobs.worker"] + cpus: "1.0" + mem_limit: 768m + environment: + DATABASE_URL: ${DATABASE_URL} + REDIS_URL: ${REDIS_URL} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + BROKER_URL: ${BROKER_URL} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_PREFETCH: ${WORKER_PREFETCH:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90} + RETRY_BASE_SECONDS: ${RETRY_BASE_SECONDS:-2} + RETRY_MAX_SECONDS: ${RETRY_MAX_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + THUMBNAIL_JPEG_QUALITY: ${THUMBNAIL_JPEG_QUALITY:-85} + CROP_JPEG_QUALITY: ${CROP_JPEG_QUALITY:-85} + OPTIMIZED_WEBP_QUALITY: ${OPTIMIZED_WEBP_QUALITY:-82} + depends_on: + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"] + interval: 10s + timeout: 3s + retries: 3 + restart: unless-stopped + stop_grace_period: 70s + lb: image: haproxy:3.2.26-alpine cpus: "0.25" @@ -146,3 +237,4 @@ volumes: postgres_data: redis_data: s3_data: + broker_data: diff --git a/requirements.txt b/requirements.txt index 20bb81f7..259b5e73 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ redis==5.2.1 argon2-cffi==25.1.0 minio==7.2.20 Pillow==13.0.0 +aio-pika==9.5.5 diff --git a/scripts/crash-publisher-after-confirm.sh b/scripts/crash-publisher-after-confirm.sh new file mode 100755 index 00000000..eca9541a --- /dev/null +++ b/scripts/crash-publisher-after-confirm.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +cd "$(dirname -- "$script_dir")" +compose_file=${COMPOSE_FILE_PATH:-compose.yaml} +set +e +docker compose -f "$compose_file" --env-file .env run --rm \ + -e FAULT_PUBLISHER_AFTER_CONFIRM=1 \ + publisher python -m app.jobs.publisher --once >/dev/null +status=$? +set -e +if [ "$status" -ne 91 ]; then + printf '%s\n' "publisher fault expected exit 91, got $status" >&2 + exit 1 +fi +docker compose -f "$compose_file" --env-file .env up -d publisher >/dev/null diff --git a/scripts/crash-worker.sh b/scripts/crash-worker.sh new file mode 100755 index 00000000..58109cba --- /dev/null +++ b/scripts/crash-worker.sh @@ -0,0 +1,26 @@ +#!/bin/sh +set -eu +if [ "$#" -ne 1 ]; then + printf '%s\n' 'usage: crash-worker.sh objects|commit|permanent' >&2 + exit 2 +fi +case "$1" in + objects) variable=FAULT_WORKER_AFTER_OBJECTS; expected=92 ;; + commit) variable=FAULT_WORKER_AFTER_COMMIT; expected=93 ;; + permanent) variable=FAULT_WORKER_PERMANENT; expected=94 ;; + *) printf '%s\n' 'mode must be objects, commit or permanent' >&2; exit 2 ;; +esac +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +cd "$(dirname -- "$script_dir")" +compose_file=${COMPOSE_FILE_PATH:-compose.yaml} +set +e +docker compose -f "$compose_file" --env-file .env run --rm \ + -e "$variable=1" -e JOB_LEASE_SECONDS=3 \ + worker python -m app.jobs.worker >/dev/null +status=$? +set -e +if [ "$status" -ne "$expected" ]; then + printf '%s\n' "worker fault expected exit $expected, got $status" >&2 + exit 1 +fi +docker compose -f "$compose_file" --env-file .env up -d worker >/dev/null diff --git a/scripts/replay-attachment.sh b/scripts/replay-attachment.sh new file mode 100755 index 00000000..3e8254c8 --- /dev/null +++ b/scripts/replay-attachment.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +if [ "$#" -ne 1 ]; then + printf '%s\n' 'usage: replay-attachment.sh ATTACHMENT_ID' >&2 + exit 2 +fi +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +cd "$(dirname -- "$script_dir")" +compose_file=${COMPOSE_FILE_PATH:-compose.yaml} +docker compose -f "$compose_file" --env-file .env run --rm \ + publisher python -m app.tools.replay_failed --attachment-id "$1" diff --git a/scripts/restart-api.sh b/scripts/restart-api.sh new file mode 100755 index 00000000..2354d348 --- /dev/null +++ b/scripts/restart-api.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +cd "$(dirname -- "$script_dir")" +compose_file=${COMPOSE_FILE_PATH:-compose.yaml} +docker compose -f "$compose_file" --env-file .env restart api >/dev/null +if [ "$compose_file" = "compose.yaml" ]; then + docker compose -f "$compose_file" --env-file .env up -d --wait --wait-timeout 60 api lb >/dev/null +else + docker compose -f "$compose_file" --env-file .env up -d api >/dev/null +fi diff --git a/scripts/restart-broker.sh b/scripts/restart-broker.sh new file mode 100755 index 00000000..dbf39a4b --- /dev/null +++ b/scripts/restart-broker.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +cd "$(dirname -- "$script_dir")" +compose_file=${COMPOSE_FILE_PATH:-compose.yaml} +docker compose -f "$compose_file" --env-file .env restart broker >/dev/null diff --git a/scripts/restart.sh b/scripts/restart.sh index 39e05f7a..1e3c8b6f 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -6,7 +6,7 @@ project_dir=$(dirname -- "$script_dir") cd "$project_dir" # Останавливаем публичный маршрут до перезапуска общих хранилищ. -docker compose --env-file .env stop lb api >/dev/null -docker compose --env-file .env restart postgres redis s3 >/dev/null -docker compose --env-file .env up -d --wait --wait-timeout 60 postgres redis s3 >/dev/null -docker compose --env-file .env up -d --wait --wait-timeout 60 api lb >/dev/null +docker compose --env-file .env stop lb api publisher worker >/dev/null +docker compose --env-file .env restart postgres redis s3 broker >/dev/null +docker compose --env-file .env up -d --wait --wait-timeout 60 postgres redis s3 broker >/dev/null +docker compose --env-file .env up -d --wait --wait-timeout 60 publisher worker api lb >/dev/null diff --git a/scripts/start-publisher.sh b/scripts/start-publisher.sh new file mode 100755 index 00000000..4c949edc --- /dev/null +++ b/scripts/start-publisher.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +cd "$(dirname -- "$script_dir")" +compose_file=${COMPOSE_FILE_PATH:-compose.yaml} +docker compose -f "$compose_file" --env-file .env up -d publisher >/dev/null diff --git a/scripts/start-worker.sh b/scripts/start-worker.sh new file mode 100755 index 00000000..74a59fc7 --- /dev/null +++ b/scripts/start-worker.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +project_dir=$(dirname -- "$script_dir") +cd "$project_dir" +compose_file=${COMPOSE_FILE_PATH:-compose.yaml} +docker compose -f "$compose_file" --env-file .env up -d worker >/dev/null diff --git a/scripts/stop-publisher.sh b/scripts/stop-publisher.sh new file mode 100755 index 00000000..8ace459b --- /dev/null +++ b/scripts/stop-publisher.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +cd "$(dirname -- "$script_dir")" +compose_file=${COMPOSE_FILE_PATH:-compose.yaml} +docker compose -f "$compose_file" --env-file .env stop publisher >/dev/null diff --git a/scripts/stop-worker.sh b/scripts/stop-worker.sh new file mode 100755 index 00000000..216f0700 --- /dev/null +++ b/scripts/stop-worker.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +project_dir=$(dirname -- "$script_dir") +cd "$project_dir" +compose_file=${COMPOSE_FILE_PATH:-compose.yaml} +docker compose -f "$compose_file" --env-file .env stop worker >/dev/null diff --git a/tests/attachment_failover.py b/tests/attachment_failover.py index 10e376d4..0a977938 100644 --- a/tests/attachment_failover.py +++ b/tests/attachment_failover.py @@ -22,12 +22,13 @@ def main(): uploaded = client.call( 'POST', f'/api/v1/chats/{chat["id"]}/attachments?filename=failover.png', - 201, + 202, actor=alice, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment', ) + ready = client.wait_ready(bob, uploaded['json']) target = uploaded['headers']['X-Instance-Id'] env = dict(os.environ) env['TARGET_INSTANCE_ID'] = target @@ -40,7 +41,7 @@ def main(): try: downloaded = client.call( 'GET', - f'/api/v1/attachments/{uploaded["json"]["id"]}/content', + f'/api/v1/attachments/{ready["id"]}/content', actor=bob, ) if downloaded['headers']['X-Instance-Id'] != target: diff --git a/tests/broker_persistence.py b/tests/broker_persistence.py new file mode 100755 index 00000000..747bd298 --- /dev/null +++ b/tests/broker_persistence.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""A persistent broker volume keeps a confirmed job across broker restart.""" +import os +from pathlib import Path +import subprocess +import time + +from client import Client, arguments, check, run + + +def executable(value): + path = Path(value).resolve() + check(path.is_file() and os.access(path, os.X_OK), str(path) + ': not executable') + return path + + +def main(): + def configure(parser): + parser.add_argument('--stop-worker-script', required=True) + parser.add_argument('--restart-broker-script', required=True) + parser.add_argument('--start-worker-script', required=True) + + args = arguments('RabbitMQ persistent delivery after restart', configure) + stop_worker = executable(args.stop_worker_script) + restart_broker = executable(args.restart_broker_script) + start_worker = executable(args.start_worker_script) + client = Client(args.base_url, args.ca_file) + alice, bob, _eve, chat = client.bootstrap() + subprocess.run([str(stop_worker)], check=True) + attachment, original = client.attachment(alice, chat) + time.sleep(2) + queued = client.call( + 'GET', f'/api/v1/attachments/{attachment["id"]}', actor=bob, schema='Attachment' + )['json'] + check(queued['status'] == 'queued', 'worker processed job while expected stopped') + subprocess.run([str(restart_broker)], check=True) + subprocess.run([str(start_worker)], check=True) + client.wait_ready(bob, attachment) + downloaded = client.call( + 'GET', f'/api/v1/attachments/{attachment["id"]}/content', actor=bob + )['bytes'] + check(downloaded == original, 'job/original lost across broker restart') + print('PASS: confirmed durable message survived broker restart with its volume') + + +if __name__ == '__main__': + run(main) diff --git a/tests/failed_replay.py b/tests/failed_replay.py new file mode 100755 index 00000000..33fd3323 --- /dev/null +++ b/tests/failed_replay.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Prove permanent failure, DLQ routing and guarded operator replay.""" +import os +from pathlib import Path +import subprocess +import time + +from client import Client, arguments, check, run + + +def executable(value): + path = Path(value).resolve() + check(path.is_file() and os.access(path, os.X_OK), str(path) + ': not executable') + return path + + +def main(): + def configure(parser): + parser.add_argument('--stop-worker-script', required=True) + parser.add_argument('--crash-worker-script', required=True) + parser.add_argument('--replay-script', required=True) + + args = arguments('Failed registry, DLQ and guarded replay', configure) + stop = executable(args.stop_worker_script) + crash = executable(args.crash_worker_script) + replay = executable(args.replay_script) + client = Client(args.base_url, args.ca_file) + alice, bob, _eve, chat = client.bootstrap() + subprocess.run([str(stop)], check=True) + attachment, original = client.attachment(alice, chat) + subprocess.run([str(crash), 'permanent'], check=True) + + deadline = time.monotonic() + 20 + while True: + failed = client.call( + 'GET', f'/api/v1/attachments/{attachment["id"]}', actor=bob, + schema='Attachment', + )['json'] + if failed['status'] == 'failed': + break + check(time.monotonic() < deadline, 'attachment did not reach failed') + time.sleep(0.2) + check(failed['error_code'] == 'fault_injected', 'unsafe or unexpected error code') + + subprocess.run([str(replay), attachment['id']], check=True) + ready = client.wait_ready(bob, attachment) + check(len(ready['variants']) == 3, 'replay produced incomplete variants') + content = client.call( + 'GET', f'/api/v1/attachments/{attachment["id"]}/content', actor=bob + )['bytes'] + check(content == original, 'replay changed original') + duplicate = subprocess.run([str(replay), attachment['id']], check=False) + check(duplicate.returncode != 0, 'ready job was replayed again') + print('PASS: permanent failure reached failed/DLQ and guarded replay reached ready') + + +if __name__ == '__main__': + run(main) diff --git a/tests/outbox_recovery.py b/tests/outbox_recovery.py new file mode 100755 index 00000000..56d81bea --- /dev/null +++ b/tests/outbox_recovery.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Prove that a committed outbox survives publisher downtime.""" +import os +from pathlib import Path +import subprocess +import time + +from client import Client, arguments, check, run + + +def main(): + def configure(parser): + parser.add_argument('--stop-publisher-script', required=True) + parser.add_argument('--start-publisher-script', required=True) + + args = arguments('Transactional outbox recovery', configure) + stop = Path(args.stop_publisher_script).resolve() + start = Path(args.start_publisher_script).resolve() + check(all(path.is_file() and os.access(path, os.X_OK) for path in (stop, start)), 'scripts must be executable') + client = Client(args.base_url, args.ca_file) + alice, bob, _eve, chat = client.bootstrap() + subprocess.run([str(stop)], check=True) + attachment, _content = client.attachment(alice, chat) + time.sleep(1) + queued = client.call( + 'GET', f'/api/v1/attachments/{attachment["id"]}', actor=bob, schema='Attachment' + )['json'] + check(queued['status'] == 'queued', 'job escaped outbox while publisher was stopped') + subprocess.run([str(start)], check=True) + ready = client.wait_ready(bob, attachment) + check(ready['status'] == 'ready', 'outbox did not recover committed job') + print('PASS: COMMIT-before-publish recovered without a new upload') + + +if __name__ == '__main__': + run(main) diff --git a/tests/pipeline_smoke.py b/tests/pipeline_smoke.py new file mode 100755 index 00000000..e8122418 --- /dev/null +++ b/tests/pipeline_smoke.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Grade-4 black-box check for every promised image variant.""" +import hashlib + +from client import Client, arguments, check, run +from image_probe import jpeg_size + + +def main(): + args = arguments('Verify thumbnail, crop and optimized variants') + client = Client(args.base_url, args.ca_file) + alice, bob, _eve, chat = client.bootstrap() + attachment, original = client.attachment(alice, chat) + ready = client.wait_ready(bob, attachment) + variants = {item['name']: item for item in ready['variants']} + check(set(variants) == {'thumbnail', 'crop', 'optimized'}, 'grade-4 variants missing') + expected = { + 'thumbnail': ('image/jpeg', (256, 160)), + 'crop': ('image/jpeg', (128, 128)), + 'optimized': ('image/webp', (320, 200)), + } + for name, (content_type, dimensions) in expected.items(): + response = client.call( + 'GET', + f'/api/v1/attachments/{ready["id"]}/content?variant={name}', + actor=bob, + ) + metadata = variants[name] + check(response['headers'].get_content_type() == content_type, name + ': MIME mismatch') + check((metadata['width'], metadata['height']) == dimensions, name + ': metadata dimensions') + check(len(response['bytes']) == metadata['size_bytes'], name + ': size mismatch') + check(hashlib.sha256(response['bytes']).hexdigest() == metadata['sha256'], name + ': hash mismatch') + if content_type == 'image/jpeg': + check(jpeg_size(response['bytes']) == dimensions, name + ': actual dimensions') + else: + check(response['bytes'][:4] == b'RIFF' and response['bytes'][8:12] == b'WEBP', 'optimized is not WebP') + downloaded = client.call( + 'GET', f'/api/v1/attachments/{ready["id"]}/content', actor=bob + )['bytes'] + check(downloaded == original, 'original bytes changed') + print('PASS: original plus thumbnail/crop/optimized variants are valid') + + +if __name__ == '__main__': + run(main) diff --git a/tests/publisher_confirm_crash.py b/tests/publisher_confirm_crash.py new file mode 100755 index 00000000..8ee2920d --- /dev/null +++ b/tests/publisher_confirm_crash.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Crash after publisher confirm creates a harmless duplicate delivery.""" +import os +from pathlib import Path +import subprocess + +from client import Client, arguments, check, run + + +def executable(value): + path = Path(value).resolve() + check(path.is_file() and os.access(path, os.X_OK), str(path) + ': not executable') + return path + + +def main(): + def configure(parser): + parser.add_argument('--stop-publisher-script', required=True) + parser.add_argument('--crash-publisher-script', required=True) + + args = arguments('Crash after publisher confirm', configure) + stop = executable(args.stop_publisher_script) + crash = executable(args.crash_publisher_script) + client = Client(args.base_url, args.ca_file) + alice, bob, _eve, chat = client.bootstrap() + subprocess.run([str(stop)], check=True) + attachment, _original = client.attachment(alice, chat) + subprocess.run([str(crash)], check=True) + ready = client.wait_ready(bob, attachment) + names = [item['name'] for item in ready['variants']] + check(sorted(names) == ['crop', 'optimized', 'thumbnail'], 'duplicate or missing variants') + print('PASS: confirm-before-outbox-mark duplicate produced one logical result') + + +if __name__ == '__main__': + run(main) diff --git a/tests/test_attachments.py b/tests/test_attachments.py index 0a34c667..32d2beb2 100644 --- a/tests/test_attachments.py +++ b/tests/test_attachments.py @@ -16,7 +16,7 @@ def scenario(): return client, alice, bob, eve, chat -def upload(client, actor, chat, content, *, filename='sample.png', content_type='image/png', expected=201): +def upload(client, actor, chat, content, *, filename='sample.png', content_type='image/png', expected=202): return client.call( 'POST', f'/api/v1/chats/{chat["id"]}/attachments?filename={quote(filename, safe="")}', @@ -24,7 +24,7 @@ def upload(client, actor, chat, content, *, filename='sample.png', content_type= actor=actor, raw=content, headers={'Content-Type': content_type}, - schema='Attachment' if expected == 201 else None, + schema='Attachment' if expected == 202 else None, ) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py new file mode 100644 index 00000000..ade9492b --- /dev/null +++ b/tests/test_pipeline.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import hashlib +from io import BytesIO + +import pytest +from PIL import Image + +from app.jobs.pipeline import PermanentImageError, process_image + + +def encoded(width: int, height: int, mode: str = 'RGBA') -> bytes: + image = Image.new(mode, (width, height), (20, 80, 160, 128) if mode == 'RGBA' else 'navy') + output = BytesIO() + image.save(output, 'PNG', pnginfo=None) + return output.getvalue() + + +def run(source: bytes): + return process_image( + source, + 'image/png', + hashlib.sha256(source).hexdigest(), + 40_000_000, + 85, + 85, + 82, + ) + + +def test_pipeline_creates_all_grade4_variants() -> None: + """Create every grade-4 variant with the required format, size and checksum.""" + variants = {item.name: item for item in run(encoded(320, 200))} + assert set(variants) == {'thumbnail', 'crop', 'optimized'} + assert (variants['thumbnail'].width, variants['thumbnail'].height) == (256, 160) + assert variants['thumbnail'].content_type == 'image/jpeg' + assert (variants['crop'].width, variants['crop'].height) == (128, 128) + assert variants['crop'].content_type == 'image/jpeg' + assert (variants['optimized'].width, variants['optimized'].height) == (320, 200) + assert variants['optimized'].content_type == 'image/webp' + for item in variants.values(): + assert hashlib.sha256(item.content).hexdigest() == item.sha256 + + +def test_thumbnail_and_optimized_do_not_upscale_small_image() -> None: + """Keep small images at source size except for the required 128x128 crop.""" + variants = {item.name: item for item in run(encoded(40, 20, 'RGB'))} + assert (variants['thumbnail'].width, variants['thumbnail'].height) == (40, 20) + assert (variants['optimized'].width, variants['optimized'].height) == (40, 20) + assert (variants['crop'].width, variants['crop'].height) == (128, 128) + + +def test_jpeg_variants_composite_alpha_on_white() -> None: + """Flatten transparent input before encoding JPEG variants without an alpha channel.""" + variants = {item.name: item for item in run(encoded(2, 2))} + with Image.open(BytesIO(variants['thumbnail'].content)) as image: + assert image.mode == 'RGB' + + +def test_checksum_mismatch_is_permanent() -> None: + """Reject changed staging bytes as a permanent checksum error.""" + source = encoded(20, 20) + with pytest.raises(PermanentImageError, match='checksum_mismatch'): + process_image(source, 'image/png', '0' * 64, 40_000_000, 85, 85, 82) + + +def test_exif_orientation_is_applied_and_removed() -> None: + """Apply EXIF orientation before resizing and omit metadata from generated output.""" + image = Image.new('RGB', (320, 200), 'red') + exif = Image.Exif() + exif[274] = 6 + output = BytesIO() + image.save(output, 'JPEG', exif=exif) + source = output.getvalue() + variants = { + item.name: item + for item in process_image( + source, + 'image/jpeg', + hashlib.sha256(source).hexdigest(), + 40_000_000, + 85, + 85, + 82, + ) + } + assert (variants['thumbnail'].width, variants['thumbnail'].height) == (160, 256) + with Image.open(BytesIO(variants['thumbnail'].content)) as transformed: + assert not transformed.getexif() diff --git a/tests/worker_crash.py b/tests/worker_crash.py new file mode 100755 index 00000000..9e81ac0b --- /dev/null +++ b/tests/worker_crash.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Fault injection around S3 effects, DB commit and broker ack.""" +import os +from pathlib import Path +import subprocess + +from client import Client, arguments, check, run + + +def executable(value): + path = Path(value).resolve() + check(path.is_file() and os.access(path, os.X_OK), str(path) + ': not executable') + return path + + +def main(): + def configure(parser): + parser.add_argument('--stop-worker-script', required=True) + parser.add_argument('--crash-worker-script', required=True) + parser.add_argument('--mode', choices=('objects', 'commit'), required=True) + + args = arguments('Worker crash around durable effects', configure) + stop = executable(args.stop_worker_script) + crash = executable(args.crash_worker_script) + client = Client(args.base_url, args.ca_file) + alice, bob, _eve, chat = client.bootstrap() + subprocess.run([str(stop)], check=True) + attachment, original = client.attachment(alice, chat) + subprocess.run([str(crash), args.mode], check=True) + ready = client.wait_ready(bob, attachment) + names = sorted(item['name'] for item in ready['variants']) + check(names == ['crop', 'optimized', 'thumbnail'], 'duplicate or missing variants') + downloaded = client.call( + 'GET', f'/api/v1/attachments/{attachment["id"]}/content', actor=bob + )['bytes'] + check(downloaded == original, 'original changed after crash recovery') + print('PASS: worker crash mode=%s recovered one logical result' % args.mode) + + +if __name__ == '__main__': + run(main) diff --git a/tests/worker_outage.py b/tests/worker_outage.py new file mode 100755 index 00000000..4b802dcb --- /dev/null +++ b/tests/worker_outage.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Required scenario: accepted work survives worker and API downtime.""" +import os +from pathlib import Path +import subprocess +import time + +from client import Client, arguments, check, run + + +def executable(value): + path = Path(value).expanduser().resolve() + check(path.is_file() and os.access(path, os.X_OK), str(path) + ': not executable') + return path + + +def main(): + def configure(parser): + parser.add_argument('--stop-worker-script', required=True) + parser.add_argument('--restart-api-script', required=True) + parser.add_argument('--start-worker-script', required=True) + + args = arguments('Worker outage and durable accepted job', configure) + stop_worker = executable(args.stop_worker_script) + restart_api = executable(args.restart_api_script) + start_worker = executable(args.start_worker_script) + client = Client(args.base_url, args.ca_file) + alice, bob, _eve, chat = client.bootstrap() + subprocess.run([str(stop_worker)], check=True) + attachment, original = client.attachment(alice, chat) + message = client.message(alice, chat, '', attachments=[attachment['id']])['json'] + check(message['attachment_ids'] == [attachment['id']], 'queued attachment was not linkable') + time.sleep(1) + client.call( + 'GET', f'/api/v1/attachments/{attachment["id"]}/content', 409, actor=bob + ) + subprocess.run([str(restart_api)], check=True) + deadline = time.monotonic() + 60 + while True: + try: + client.call('GET', '/health/ready', schema='Health') + break + except Exception: + check(time.monotonic() < deadline, 'API did not recover after restart') + time.sleep(0.3) + persisted = client.call( + 'GET', f'/api/v1/attachments/{attachment["id"]}', actor=bob, schema='Attachment' + )['json'] + check(persisted['status'] in {'queued', 'processing'}, 'accepted job disappeared after API restart') + subprocess.run([str(start_worker)], check=True) + ready = client.wait_ready(bob, attachment) + downloaded = client.call( + 'GET', f'/api/v1/attachments/{attachment["id"]}/content', actor=bob + )['bytes'] + check(downloaded == original, 'original changed after worker recovery') + check(ready['status'] == 'ready', 'job did not become ready') + print('PASS: worker stopped -> 202 -> message -> API restart -> worker -> ready') + + +if __name__ == '__main__': + run(main)