forked from mtusi2026/mtusi2026lab5
Реализовать асинхронную обработку вложений
This commit is contained in:
@@ -20,15 +20,25 @@ MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
WORKER_PREFETCH=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
JOB_LEASE_SECONDS=90
|
||||
RETRY_BASE_SECONDS=2
|
||||
RETRY_MAX_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
THUMBNAIL_JPEG_QUALITY=85
|
||||
CROP_JPEG_QUALITY=85
|
||||
OPTIMIZED_WEBP_QUALITY=82
|
||||
|
||||
POSTGRES_USER=messenger
|
||||
POSTGRES_PASSWORD=change-me-local
|
||||
POSTGRES_DB=messenger
|
||||
DATABASE_URL=postgresql+asyncpg://messenger:change-me-local@postgres:5432/messenger
|
||||
REDIS_URL=redis://redis:6379/0
|
||||
BROKER_USER=messenger
|
||||
BROKER_PASSWORD=change-me-rabbitmq
|
||||
BROKER_URL=amqp://messenger:change-me-rabbitmq@broker:5672/
|
||||
|
||||
MINIO_ROOT_USER=local-minio-admin
|
||||
MINIO_ROOT_PASSWORD=change-me-minio-root
|
||||
|
||||
@@ -4,7 +4,7 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help check validate test test-attachments test-pipeline test-pipeline-smoke test-outbox test-publisher-confirm-crash test-worker-crash test-failed-replay test-worker-outage test-broker-persistence test-docker load test-persistence test-failover replay-failed cleanup-staging
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -18,6 +18,54 @@ validate:
|
||||
test:
|
||||
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-attachments:
|
||||
$(PYTHON) -m pytest -q tests/test_attachments.py
|
||||
|
||||
test-pipeline:
|
||||
$(PYTHON) -m pytest -q tests/test_pipeline.py
|
||||
|
||||
test-pipeline-smoke:
|
||||
$(PYTHON) tests/pipeline_smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-outbox:
|
||||
$(PYTHON) tests/outbox_recovery.py --base-url "$(BASE_URL)" \
|
||||
--stop-publisher-script scripts/stop-publisher.sh \
|
||||
--start-publisher-script scripts/start-publisher.sh $(TLS_ARGS)
|
||||
|
||||
test-publisher-confirm-crash:
|
||||
$(PYTHON) tests/publisher_confirm_crash.py --base-url "$(BASE_URL)" \
|
||||
--stop-publisher-script scripts/stop-publisher.sh \
|
||||
--crash-publisher-script scripts/crash-publisher-after-confirm.sh $(TLS_ARGS)
|
||||
|
||||
test-worker-crash:
|
||||
$(PYTHON) tests/worker_crash.py --base-url "$(BASE_URL)" \
|
||||
--stop-worker-script scripts/stop-worker.sh \
|
||||
--crash-worker-script scripts/crash-worker.sh --mode objects $(TLS_ARGS)
|
||||
$(PYTHON) tests/worker_crash.py --base-url "$(BASE_URL)" \
|
||||
--stop-worker-script scripts/stop-worker.sh \
|
||||
--crash-worker-script scripts/crash-worker.sh --mode commit $(TLS_ARGS)
|
||||
|
||||
test-failed-replay:
|
||||
$(PYTHON) tests/failed_replay.py --base-url "$(BASE_URL)" \
|
||||
--stop-worker-script scripts/stop-worker.sh \
|
||||
--crash-worker-script scripts/crash-worker.sh \
|
||||
--replay-script scripts/replay-attachment.sh $(TLS_ARGS)
|
||||
|
||||
test-worker-outage:
|
||||
$(PYTHON) tests/worker_outage.py --base-url "$(BASE_URL)" \
|
||||
--stop-worker-script scripts/stop-worker.sh \
|
||||
--restart-api-script scripts/restart-api.sh \
|
||||
--start-worker-script scripts/start-worker.sh $(TLS_ARGS)
|
||||
|
||||
test-broker-persistence:
|
||||
$(PYTHON) tests/broker_persistence.py --base-url "$(BASE_URL)" \
|
||||
--stop-worker-script scripts/stop-worker.sh \
|
||||
--restart-broker-script scripts/restart-broker.sh \
|
||||
--start-worker-script scripts/start-worker.sh $(TLS_ARGS)
|
||||
|
||||
test-docker:
|
||||
docker compose -f compose.tests.yaml run --rm tests
|
||||
|
||||
load:
|
||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
@@ -28,3 +76,10 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
replay-failed:
|
||||
@test -n "$(JOB_ID)" || (echo "Set JOB_ID"; exit 2)
|
||||
docker compose --env-file .env run --rm publisher python -m app.tools.replay_failed --job-id "$(JOB_ID)"
|
||||
|
||||
cleanup-staging:
|
||||
docker compose --env-file .env run --rm worker python -m app.tools.cleanup_staging --dry-run
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Add asynchronous attachment jobs, variants and transactional outbox.
|
||||
|
||||
Revision ID: 20261116_0004
|
||||
Revises: 20261030_0003
|
||||
Create Date: 2026-11-16
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = '20261116_0004'
|
||||
down_revision = '20261030_0003'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.alter_column('attachments', 'object_key', new_column_name='original_key')
|
||||
op.alter_column('attachments', 'original_key', nullable=True)
|
||||
op.add_column('attachments', sa.Column('staging_key', sa.String(length=255), nullable=True))
|
||||
op.add_column(
|
||||
'attachments',
|
||||
sa.Column('pipeline_version', sa.Integer(), nullable=False, server_default='1'),
|
||||
)
|
||||
op.create_unique_constraint('attachments_staging_key_key', 'attachments', ['staging_key'])
|
||||
|
||||
op.create_table(
|
||||
'attachment_variants',
|
||||
sa.Column('attachment_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('pipeline_version', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=16), nullable=False),
|
||||
sa.Column('object_key', sa.String(length=255), nullable=False),
|
||||
sa.Column('content_type', sa.String(length=32), nullable=False),
|
||||
sa.Column('size_bytes', sa.BigInteger(), nullable=False),
|
||||
sa.Column('width', sa.Integer(), nullable=False),
|
||||
sa.Column('height', sa.Integer(), nullable=False),
|
||||
sa.Column('sha256', sa.String(length=64), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.CheckConstraint("name IN ('thumbnail', 'crop', 'optimized')", name='variant_name_check'),
|
||||
sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('attachment_id', 'pipeline_version', 'name'),
|
||||
sa.UniqueConstraint('object_key', name='attachment_variants_object_key_key'),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
'attachment_jobs',
|
||||
sa.Column('id', sa.Uuid(), nullable=False),
|
||||
sa.Column('attachment_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('pipeline_version', sa.Integer(), nullable=False),
|
||||
sa.Column('state', sa.String(length=16), nullable=False),
|
||||
sa.Column('attempt', sa.Integer(), nullable=False),
|
||||
sa.Column('max_attempts', sa.Integer(), nullable=False),
|
||||
sa.Column('next_attempt_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('lease_owner', sa.String(length=128), nullable=True),
|
||||
sa.Column('lease_until', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('last_error_code', sa.String(length=64), nullable=True),
|
||||
sa.Column('replay_count', sa.Integer(), nullable=False, server_default='0'),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.CheckConstraint(
|
||||
"state IN ('queued', 'processing', 'ready', 'failed')",
|
||||
name='attachment_job_state_check',
|
||||
),
|
||||
sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint(
|
||||
'attachment_id', 'pipeline_version', name='attachment_job_attachment_version_key'
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
'attachment_jobs_lease_idx',
|
||||
'attachment_jobs',
|
||||
['state', 'lease_until'],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
'outbox_events',
|
||||
sa.Column('id', sa.Uuid(), nullable=False),
|
||||
sa.Column('event_type', sa.String(length=64), nullable=False),
|
||||
sa.Column('job_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('payload', sa.JSON(), nullable=False),
|
||||
sa.Column('available_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('published_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('claim_owner', sa.String(length=128), nullable=True),
|
||||
sa.Column('claim_until', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('publish_attempts', sa.Integer(), nullable=False, server_default='0'),
|
||||
sa.Column('last_error_code', sa.String(length=64), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['job_id'], ['attachment_jobs.id'], ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
)
|
||||
op.create_index(
|
||||
'outbox_events_pending_idx',
|
||||
'outbox_events',
|
||||
['published_at', 'available_at'],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
'job_events',
|
||||
sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False),
|
||||
sa.Column('job_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('kind', sa.String(length=32), nullable=False),
|
||||
sa.Column('attempt', sa.Integer(), nullable=False),
|
||||
sa.Column('error_code', sa.String(length=64), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['job_id'], ['attachment_jobs.id'], ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
)
|
||||
op.create_index('job_events_job_created_idx', 'job_events', ['job_id', 'created_at'])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index('job_events_job_created_idx', table_name='job_events')
|
||||
op.drop_table('job_events')
|
||||
op.drop_index('outbox_events_pending_idx', table_name='outbox_events')
|
||||
op.drop_table('outbox_events')
|
||||
op.drop_index('attachment_jobs_lease_idx', table_name='attachment_jobs')
|
||||
op.drop_table('attachment_jobs')
|
||||
op.drop_table('attachment_variants')
|
||||
op.drop_constraint('attachments_staging_key_key', 'attachments', type_='unique')
|
||||
op.drop_column('attachments', 'pipeline_version')
|
||||
op.drop_column('attachments', 'staging_key')
|
||||
op.alter_column('attachments', 'original_key', nullable=False)
|
||||
op.alter_column('attachments', 'original_key', new_column_name='object_key')
|
||||
@@ -14,7 +14,7 @@ router = APIRouter(tags=['attachments'])
|
||||
|
||||
@router.post(
|
||||
'/api/v1/chats/{chat_id}/attachments',
|
||||
status_code=201,
|
||||
status_code=202,
|
||||
response_model=AttachmentOut,
|
||||
)
|
||||
async def upload(
|
||||
@@ -51,16 +51,14 @@ async def content(
|
||||
actor: User = Depends(current_user),
|
||||
attachments: AttachmentService = Depends(attachment_service),
|
||||
) -> StreamingResponse:
|
||||
if variant != 'original':
|
||||
from ...domain.errors import NotFound
|
||||
|
||||
raise NotFound('вариант вложения не найден')
|
||||
attachment, stream = await attachments.content(actor, attachment_id)
|
||||
attachment, stream, content_type, size_bytes = await attachments.content(
|
||||
actor, attachment_id, variant
|
||||
)
|
||||
return StreamingResponse(
|
||||
stream.chunks(),
|
||||
media_type=attachment.content_type,
|
||||
media_type=content_type,
|
||||
headers={
|
||||
'Content-Length': str(attachment.size_bytes),
|
||||
'Content-Length': str(size_bytes),
|
||||
'Content-Disposition': f'inline; filename="attachment-{attachment.id}"',
|
||||
},
|
||||
)
|
||||
|
||||
+21
-2
@@ -134,6 +134,15 @@ class MessageOut(Strict):
|
||||
)
|
||||
|
||||
|
||||
class VariantOut(Strict):
|
||||
name: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
width: int
|
||||
height: int
|
||||
sha256: str
|
||||
|
||||
|
||||
class AttachmentOut(Strict):
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
@@ -143,7 +152,7 @@ class AttachmentOut(Strict):
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
status: str
|
||||
variants: list = Field(default_factory=list)
|
||||
variants: list[VariantOut] = Field(default_factory=list)
|
||||
error_code: str | None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
@@ -159,7 +168,17 @@ class AttachmentOut(Strict):
|
||||
size_bytes=attachment.size_bytes,
|
||||
sha256=attachment.sha256,
|
||||
status=attachment.status,
|
||||
variants=[],
|
||||
variants=[
|
||||
VariantOut(
|
||||
name=item.name,
|
||||
content_type=item.content_type,
|
||||
size_bytes=item.size_bytes,
|
||||
width=item.width,
|
||||
height=item.height,
|
||||
sha256=item.sha256,
|
||||
)
|
||||
for item in attachment.variants
|
||||
],
|
||||
error_code=attachment.error_code,
|
||||
created_at=to_rfc3339(attachment.created_at),
|
||||
updated_at=to_rfc3339(attachment.updated_at),
|
||||
|
||||
@@ -27,6 +27,18 @@ class Settings:
|
||||
s3_secret_access_key: str
|
||||
max_upload_bytes: int
|
||||
max_image_pixels: int
|
||||
broker_url: str
|
||||
worker_concurrency: int
|
||||
worker_prefetch: int
|
||||
job_max_attempts: int
|
||||
job_timeout_seconds: int
|
||||
job_lease_seconds: int
|
||||
retry_base_seconds: int
|
||||
retry_max_seconds: int
|
||||
staging_retention_seconds: int
|
||||
thumbnail_jpeg_quality: int
|
||||
crop_jpeg_quality: int
|
||||
optimized_webp_quality: int
|
||||
|
||||
|
||||
def _positive_int(name: str, default: str) -> int:
|
||||
@@ -39,6 +51,13 @@ def _positive_int(name: str, default: str) -> int:
|
||||
return value
|
||||
|
||||
|
||||
def _quality(name: str, default: str) -> int:
|
||||
value = _positive_int(name, default)
|
||||
if value > 100:
|
||||
raise RuntimeError(f'{name} должен быть не больше 100')
|
||||
return value
|
||||
|
||||
|
||||
def load_settings() -> Settings:
|
||||
return Settings(
|
||||
database_url=_read_value('DATABASE_URL', 'DATABASE_URL_FILE'),
|
||||
@@ -51,4 +70,16 @@ def load_settings() -> Settings:
|
||||
s3_secret_access_key=_read_value('S3_SECRET_ACCESS_KEY', 'S3_SECRET_ACCESS_KEY_FILE'),
|
||||
max_upload_bytes=_positive_int('MAX_UPLOAD_BYTES', '10485760'),
|
||||
max_image_pixels=_positive_int('MAX_IMAGE_PIXELS', '40000000'),
|
||||
broker_url=_read_value('BROKER_URL', 'BROKER_URL_FILE'),
|
||||
worker_concurrency=_positive_int('WORKER_CONCURRENCY', '2'),
|
||||
worker_prefetch=_positive_int('WORKER_PREFETCH', '2'),
|
||||
job_max_attempts=_positive_int('JOB_MAX_ATTEMPTS', '5'),
|
||||
job_timeout_seconds=_positive_int('JOB_TIMEOUT_SECONDS', '60'),
|
||||
job_lease_seconds=_positive_int('JOB_LEASE_SECONDS', '90'),
|
||||
retry_base_seconds=_positive_int('RETRY_BASE_SECONDS', '2'),
|
||||
retry_max_seconds=_positive_int('RETRY_MAX_SECONDS', '60'),
|
||||
staging_retention_seconds=_positive_int('STAGING_RETENTION_SECONDS', '86400'),
|
||||
thumbnail_jpeg_quality=_quality('THUMBNAIL_JPEG_QUALITY', '85'),
|
||||
crop_jpeg_quality=_quality('CROP_JPEG_QUALITY', '85'),
|
||||
optimized_webp_quality=_quality('OPTIMIZED_WEBP_QUALITY', '82'),
|
||||
)
|
||||
|
||||
+40
-1
@@ -39,7 +39,9 @@ class Attachment:
|
||||
chat_id: UUID
|
||||
owner_id: UUID
|
||||
filename: str
|
||||
object_key: str
|
||||
original_key: str | None
|
||||
staging_key: str | None
|
||||
pipeline_version: int
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
@@ -49,6 +51,43 @@ class Attachment:
|
||||
error_code: str | None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
variants: tuple['AttachmentVariant', ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AttachmentVariant:
|
||||
name: str
|
||||
object_key: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
width: int
|
||||
height: int
|
||||
sha256: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AttachmentJob:
|
||||
id: UUID
|
||||
attachment_id: UUID
|
||||
pipeline_version: int
|
||||
state: str
|
||||
attempt: int
|
||||
max_attempts: int
|
||||
next_attempt_at: datetime
|
||||
lease_owner: str | None
|
||||
lease_until: datetime | None
|
||||
last_error_code: str | None
|
||||
replay_count: int
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class OutboxEvent:
|
||||
id: UUID
|
||||
job_id: UUID
|
||||
payload: dict
|
||||
available_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Durable attachment job publisher and worker."""
|
||||
@@ -0,0 +1,56 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import aio_pika
|
||||
from aio_pika import DeliveryMode, ExchangeType
|
||||
|
||||
MAIN_EXCHANGE = 'attachments'
|
||||
MAIN_QUEUE = 'attachments.process'
|
||||
ROUTING_KEY = 'attachment.process.v1'
|
||||
DEAD_EXCHANGE = 'attachments.dead'
|
||||
DEAD_QUEUE = 'attachments.failed'
|
||||
logger = logging.getLogger('app.broker')
|
||||
|
||||
|
||||
async def connect(url: str) -> aio_pika.RobustConnection:
|
||||
delay = 0.5
|
||||
while True:
|
||||
try:
|
||||
return await aio_pika.connect_robust(url, timeout=5)
|
||||
except (OSError, aio_pika.exceptions.AMQPException):
|
||||
logger.warning('broker is not ready; reconnecting in %.1fs', delay)
|
||||
await asyncio.sleep(delay)
|
||||
delay = min(delay * 2, 5.0)
|
||||
|
||||
|
||||
async def topology(channel: aio_pika.abc.AbstractChannel):
|
||||
dead_exchange = await channel.declare_exchange(
|
||||
DEAD_EXCHANGE, ExchangeType.DIRECT, durable=True
|
||||
)
|
||||
dead_queue = await channel.declare_queue(DEAD_QUEUE, durable=True)
|
||||
await dead_queue.bind(dead_exchange, ROUTING_KEY)
|
||||
exchange = await channel.declare_exchange(
|
||||
MAIN_EXCHANGE, ExchangeType.DIRECT, durable=True
|
||||
)
|
||||
queue = await channel.declare_queue(
|
||||
MAIN_QUEUE,
|
||||
durable=True,
|
||||
arguments={
|
||||
'x-dead-letter-exchange': DEAD_EXCHANGE,
|
||||
'x-dead-letter-routing-key': ROUTING_KEY,
|
||||
},
|
||||
)
|
||||
await queue.bind(exchange, ROUTING_KEY)
|
||||
return exchange, queue
|
||||
|
||||
|
||||
def message(body: bytes, event_id: str) -> aio_pika.Message:
|
||||
return aio_pika.Message(
|
||||
body=body,
|
||||
content_type='application/json',
|
||||
delivery_mode=DeliveryMode.PERSISTENT,
|
||||
message_id=event_id,
|
||||
type=ROUTING_KEY,
|
||||
)
|
||||
@@ -0,0 +1,98 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from dataclasses import dataclass
|
||||
from io import BytesIO
|
||||
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
|
||||
|
||||
class PermanentImageError(Exception):
|
||||
def __init__(self, code: str) -> None:
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GeneratedVariant:
|
||||
name: str
|
||||
content_type: str
|
||||
width: int
|
||||
height: int
|
||||
sha256: str
|
||||
content: bytes
|
||||
|
||||
|
||||
def _jpeg_image(image: Image.Image) -> Image.Image:
|
||||
if image.mode in {'RGBA', 'LA'} or 'transparency' in image.info:
|
||||
rgba = image.convert('RGBA')
|
||||
background = Image.new('RGBA', rgba.size, 'white')
|
||||
background.alpha_composite(rgba)
|
||||
return background.convert('RGB')
|
||||
return image.convert('RGB')
|
||||
|
||||
|
||||
def _encode(image: Image.Image, fmt: str, **options) -> bytes:
|
||||
output = BytesIO()
|
||||
image.save(output, format=fmt, **options)
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
def _variant(name: str, content_type: str, image: Image.Image, content: bytes) -> GeneratedVariant:
|
||||
return GeneratedVariant(
|
||||
name=name,
|
||||
content_type=content_type,
|
||||
width=image.width,
|
||||
height=image.height,
|
||||
sha256=hashlib.sha256(content).hexdigest(),
|
||||
content=content,
|
||||
)
|
||||
|
||||
|
||||
def process_image(
|
||||
source: bytes,
|
||||
declared_type: str,
|
||||
expected_sha256: str,
|
||||
max_pixels: int,
|
||||
thumbnail_quality: int,
|
||||
crop_quality: int,
|
||||
webp_quality: int,
|
||||
) -> tuple[GeneratedVariant, ...]:
|
||||
if hashlib.sha256(source).hexdigest() != expected_sha256:
|
||||
raise PermanentImageError('checksum_mismatch')
|
||||
try:
|
||||
with Image.open(BytesIO(source)) as opened:
|
||||
actual = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}.get(opened.format or '')
|
||||
if actual != declared_type:
|
||||
raise PermanentImageError('unsupported_media_type')
|
||||
if opened.width * opened.height > max_pixels:
|
||||
raise PermanentImageError('image_too_large')
|
||||
opened.load()
|
||||
oriented = ImageOps.exif_transpose(opened)
|
||||
base = oriented.copy()
|
||||
except PermanentImageError:
|
||||
raise
|
||||
except (Image.DecompressionBombError, UnidentifiedImageError, OSError, SyntaxError) as error:
|
||||
raise PermanentImageError('invalid_image') from error
|
||||
|
||||
thumbnail = _jpeg_image(base.copy())
|
||||
thumbnail.thumbnail((256, 256), Image.Resampling.LANCZOS, reducing_gap=3.0)
|
||||
thumbnail_bytes = _encode(
|
||||
thumbnail, 'JPEG', quality=thumbnail_quality, optimize=True, progressive=True
|
||||
)
|
||||
|
||||
crop_source = _jpeg_image(base.copy())
|
||||
crop = ImageOps.fit(
|
||||
crop_source, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)
|
||||
)
|
||||
crop_bytes = _encode(crop, 'JPEG', quality=crop_quality, optimize=True, progressive=True)
|
||||
|
||||
optimized = base.copy()
|
||||
optimized.thumbnail((1280, 1280), Image.Resampling.LANCZOS, reducing_gap=3.0)
|
||||
optimized_bytes = _encode(optimized, 'WEBP', quality=webp_quality, method=6)
|
||||
|
||||
return (
|
||||
_variant('thumbnail', 'image/jpeg', thumbnail, thumbnail_bytes),
|
||||
_variant('crop', 'image/jpeg', crop, crop_bytes),
|
||||
_variant('optimized', 'image/webp', optimized, optimized_bytes),
|
||||
)
|
||||
@@ -0,0 +1,64 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from ..config import load_settings
|
||||
from ..repositories import JobRepository, OutboxRepository
|
||||
from .broker import ROUTING_KEY, connect, message, topology
|
||||
from .runtime import configure_logging, dependencies
|
||||
|
||||
logger = logging.getLogger('app.publisher')
|
||||
|
||||
|
||||
async def run(once: bool = False) -> None:
|
||||
settings = load_settings()
|
||||
engine, sessions, _storage = dependencies(settings)
|
||||
outbox = OutboxRepository(sessions)
|
||||
jobs = JobRepository(sessions)
|
||||
owner = f'{socket.gethostname()}:{os.getpid()}'
|
||||
connection = await connect(settings.broker_url)
|
||||
try:
|
||||
channel = await connection.channel(publisher_confirms=True)
|
||||
exchange, _queue = await topology(channel)
|
||||
while True:
|
||||
now = datetime.now(timezone.utc)
|
||||
await jobs.recover_expired(now)
|
||||
events = await outbox.claim(owner, now, lease_seconds=30)
|
||||
for event in events:
|
||||
try:
|
||||
body = json.dumps(event.payload, separators=(',', ':')).encode()
|
||||
await exchange.publish(
|
||||
message(body, str(event.id)), routing_key=ROUTING_KEY, mandatory=True
|
||||
)
|
||||
if os.getenv('FAULT_PUBLISHER_AFTER_CONFIRM') == '1':
|
||||
os._exit(91)
|
||||
await outbox.published(event.id, owner, datetime.now(timezone.utc))
|
||||
except Exception:
|
||||
logger.exception('outbox publish failed event_id=%s', event.id)
|
||||
await outbox.failed(
|
||||
event.id, owner, 'broker_unavailable', datetime.now(timezone.utc)
|
||||
)
|
||||
if once:
|
||||
return
|
||||
await asyncio.sleep(0.5 if events else 1.0)
|
||||
finally:
|
||||
await connection.close()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--once', action='store_true')
|
||||
args = parser.parse_args()
|
||||
configure_logging()
|
||||
asyncio.run(run(args.once))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,30 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from ..config import Settings
|
||||
from ..db import create_engine, create_session_factory
|
||||
from ..storage import S3Storage
|
||||
|
||||
|
||||
def configure_logging() -> None:
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
stream=sys.stdout,
|
||||
format='%(asctime)s %(levelname)s %(name)s %(message)s',
|
||||
force=True,
|
||||
)
|
||||
|
||||
|
||||
def dependencies(settings: Settings):
|
||||
engine = create_engine(settings.database_url)
|
||||
sessions = create_session_factory(engine)
|
||||
storage = S3Storage(
|
||||
endpoint=settings.s3_endpoint,
|
||||
access_key=settings.s3_access_key_id,
|
||||
secret_key=settings.s3_secret_access_key,
|
||||
bucket=settings.s3_bucket,
|
||||
region=settings.s3_region,
|
||||
)
|
||||
return engine, sessions, storage
|
||||
@@ -0,0 +1,274 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
import random
|
||||
import socket
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import UUID
|
||||
|
||||
import aio_pika
|
||||
|
||||
from ..config import Settings, load_settings
|
||||
from ..domain.errors import Unavailable
|
||||
from ..domain.models import AttachmentVariant
|
||||
from ..repositories import JobRepository
|
||||
from .broker import connect, topology
|
||||
from .pipeline import GeneratedVariant, PermanentImageError, process_image
|
||||
from .runtime import configure_logging, dependencies
|
||||
|
||||
logger = logging.getLogger('app.worker')
|
||||
|
||||
|
||||
class ProcessingTimeout(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _pipeline_child(connection, arguments: tuple, memory_bytes: int) -> None:
|
||||
try:
|
||||
if memory_bytes:
|
||||
try:
|
||||
import resource
|
||||
|
||||
resource.setrlimit(resource.RLIMIT_AS, (memory_bytes, memory_bytes))
|
||||
except (ImportError, OSError, ValueError):
|
||||
pass
|
||||
connection.send(('ok', process_image(*arguments)))
|
||||
except PermanentImageError as error:
|
||||
connection.send(('permanent', error.code))
|
||||
except MemoryError:
|
||||
connection.send(('temporary', 'processing_memory_limit'))
|
||||
except BaseException:
|
||||
connection.send(('temporary', 'processing_failed'))
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def _run_isolated(arguments: tuple, timeout: int, memory_bytes: int) -> tuple[GeneratedVariant, ...]:
|
||||
context = multiprocessing.get_context('spawn')
|
||||
receiver, sender = context.Pipe(duplex=False)
|
||||
process = context.Process(target=_pipeline_child, args=(sender, arguments, memory_bytes))
|
||||
process.start()
|
||||
sender.close()
|
||||
process.join(timeout)
|
||||
if process.is_alive():
|
||||
process.terminate()
|
||||
process.join(5)
|
||||
raise ProcessingTimeout('processing_timeout')
|
||||
if not receiver.poll():
|
||||
raise ProcessingTimeout('processing_failed')
|
||||
kind, result = receiver.recv()
|
||||
receiver.close()
|
||||
if kind == 'ok':
|
||||
return result
|
||||
if kind == 'permanent':
|
||||
raise PermanentImageError(result)
|
||||
raise ProcessingTimeout(result)
|
||||
|
||||
|
||||
class Worker:
|
||||
def __init__(self, settings: Settings, jobs: JobRepository, storage) -> None:
|
||||
self.settings = settings
|
||||
self.jobs = jobs
|
||||
self.storage = storage
|
||||
self.worker_id = f'{socket.gethostname()}:{os.getpid()}'
|
||||
self.semaphore = asyncio.Semaphore(settings.worker_concurrency)
|
||||
|
||||
def _delay(self, attempt: int) -> float:
|
||||
base = min(
|
||||
self.settings.retry_max_seconds,
|
||||
self.settings.retry_base_seconds * (2 ** max(0, attempt - 1)),
|
||||
)
|
||||
return base + random.uniform(0, base)
|
||||
|
||||
async def _handle_failure(
|
||||
self,
|
||||
incoming: aio_pika.IncomingMessage,
|
||||
job_id: UUID,
|
||||
attempt: int,
|
||||
code: str,
|
||||
permanent: bool,
|
||||
) -> None:
|
||||
now = datetime.now(timezone.utc)
|
||||
available_at = now + timedelta(seconds=self._delay(attempt))
|
||||
try:
|
||||
state = await self.jobs.retry_or_fail(
|
||||
job_id,
|
||||
self.worker_id,
|
||||
code,
|
||||
permanent,
|
||||
available_at,
|
||||
now,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception('failed to persist job failure job_id=%s', job_id)
|
||||
await incoming.nack(requeue=True)
|
||||
return
|
||||
if state == 'failed':
|
||||
await incoming.reject(requeue=False)
|
||||
else:
|
||||
await incoming.ack()
|
||||
|
||||
async def handle(self, incoming: aio_pika.IncomingMessage) -> None:
|
||||
async with self.semaphore:
|
||||
try:
|
||||
payload = json.loads(incoming.body)
|
||||
if payload.get('schema_version') != 1 or payload.get('type') != 'attachment.process.v1':
|
||||
await incoming.reject(requeue=False)
|
||||
return
|
||||
job_id = UUID(payload['job_id'])
|
||||
attachment_id = UUID(payload['attachment_id'])
|
||||
attempt = int(payload['attempt'])
|
||||
pipeline_version = int(payload['pipeline_version'])
|
||||
except (ValueError, TypeError, KeyError, json.JSONDecodeError):
|
||||
await incoming.reject(requeue=False)
|
||||
return
|
||||
|
||||
current = await self.jobs.state(job_id)
|
||||
if current is None:
|
||||
await incoming.reject(requeue=False)
|
||||
return
|
||||
if current.attachment_id != attachment_id or current.pipeline_version != pipeline_version:
|
||||
await incoming.reject(requeue=False)
|
||||
return
|
||||
if current.state == 'ready':
|
||||
if await self.jobs.ready_effect_complete(job_id):
|
||||
await incoming.ack()
|
||||
else:
|
||||
await incoming.reject(requeue=False)
|
||||
return
|
||||
if current.state == 'failed':
|
||||
await incoming.reject(requeue=False)
|
||||
return
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
claimed = await self.jobs.claim(
|
||||
job_id, attempt, self.worker_id, now, self.settings.job_lease_seconds
|
||||
)
|
||||
if claimed is None:
|
||||
await incoming.ack()
|
||||
return
|
||||
job, attachment = claimed
|
||||
if os.getenv('FAULT_WORKER_PERMANENT') == '1':
|
||||
await self._handle_failure(
|
||||
incoming, job.id, job.attempt, 'fault_injected', True
|
||||
)
|
||||
os._exit(94)
|
||||
if not attachment.staging_key:
|
||||
await self._handle_failure(
|
||||
incoming, job.id, job.attempt, 'staging_missing', True
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
source = await self.storage.read(
|
||||
attachment.staging_key, self.settings.max_upload_bytes
|
||||
)
|
||||
arguments = (
|
||||
source,
|
||||
attachment.content_type,
|
||||
attachment.sha256,
|
||||
self.settings.max_image_pixels,
|
||||
self.settings.thumbnail_jpeg_quality,
|
||||
self.settings.crop_jpeg_quality,
|
||||
self.settings.optimized_webp_quality,
|
||||
)
|
||||
generated = await asyncio.to_thread(
|
||||
_run_isolated,
|
||||
arguments,
|
||||
self.settings.job_timeout_seconds,
|
||||
512 * 1024 * 1024,
|
||||
)
|
||||
original_key = f'originals/{attachment.id}/v{job.pipeline_version}/original'
|
||||
await self.storage.put_bytes(original_key, source, attachment.content_type)
|
||||
variants: list[AttachmentVariant] = []
|
||||
suffixes = {'thumbnail': 'jpg', 'crop': 'jpg', 'optimized': 'webp'}
|
||||
for item in generated:
|
||||
key = (
|
||||
f'variants/{attachment.id}/v{job.pipeline_version}/'
|
||||
f'{item.name}.{suffixes[item.name]}'
|
||||
)
|
||||
await self.storage.put_bytes(key, item.content, item.content_type)
|
||||
variants.append(
|
||||
AttachmentVariant(
|
||||
name=item.name,
|
||||
object_key=key,
|
||||
content_type=item.content_type,
|
||||
size_bytes=len(item.content),
|
||||
width=item.width,
|
||||
height=item.height,
|
||||
sha256=item.sha256,
|
||||
)
|
||||
)
|
||||
if os.getenv('FAULT_WORKER_AFTER_OBJECTS') == '1':
|
||||
os._exit(92)
|
||||
completed = await self.jobs.complete(
|
||||
job.id,
|
||||
self.worker_id,
|
||||
original_key,
|
||||
tuple(variants),
|
||||
datetime.now(timezone.utc),
|
||||
)
|
||||
if not completed:
|
||||
await incoming.nack(requeue=True)
|
||||
return
|
||||
if os.getenv('FAULT_WORKER_AFTER_COMMIT') == '1':
|
||||
os._exit(93)
|
||||
try:
|
||||
await self.storage.remove(attachment.staging_key)
|
||||
except Unavailable:
|
||||
logger.warning('staging cleanup deferred attachment_id=%s', attachment.id)
|
||||
await incoming.ack()
|
||||
except PermanentImageError as error:
|
||||
await self._handle_failure(
|
||||
incoming, job.id, job.attempt, error.code, True
|
||||
)
|
||||
except ProcessingTimeout as error:
|
||||
await self._handle_failure(
|
||||
incoming, job.id, job.attempt, str(error), False
|
||||
)
|
||||
except (Unavailable, OSError, ValueError):
|
||||
logger.exception('temporary processing failure job_id=%s', job.id)
|
||||
await self._handle_failure(
|
||||
incoming, job.id, job.attempt, 'storage_unavailable', False
|
||||
)
|
||||
except Exception:
|
||||
logger.exception('unexpected processing failure job_id=%s', job.id)
|
||||
await self._handle_failure(
|
||||
incoming, job.id, job.attempt, 'processing_failed', False
|
||||
)
|
||||
|
||||
|
||||
async def run() -> None:
|
||||
settings = load_settings()
|
||||
engine, sessions, storage = dependencies(settings)
|
||||
jobs = JobRepository(sessions)
|
||||
worker = Worker(settings, jobs, storage)
|
||||
connection = await connect(settings.broker_url)
|
||||
try:
|
||||
channel = await connection.channel()
|
||||
await channel.set_qos(prefetch_count=settings.worker_prefetch)
|
||||
_exchange, queue = await topology(channel)
|
||||
await queue.consume(worker.handle, no_ack=False)
|
||||
logger.info(
|
||||
'worker started id=%s concurrency=%d prefetch=%d',
|
||||
worker.worker_id,
|
||||
settings.worker_concurrency,
|
||||
settings.worker_prefetch,
|
||||
)
|
||||
await asyncio.Future()
|
||||
finally:
|
||||
await connection.close()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
configure_logging()
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+5
-4
@@ -24,7 +24,7 @@ from .storage import S3Storage
|
||||
|
||||
logger = logging.getLogger('app')
|
||||
|
||||
DESCRIPTION = 'PostgreSQL хранит метаданные, Redis — сессии, приватный S3 — оригиналы вложений.'
|
||||
DESCRIPTION = 'PostgreSQL хранит задания, RabbitMQ доставляет их worker, S3 хранит изображения.'
|
||||
|
||||
|
||||
def configure_logging() -> None:
|
||||
@@ -69,6 +69,7 @@ def create_app() -> FastAPI:
|
||||
storage,
|
||||
settings.max_upload_bytes,
|
||||
settings.max_image_pixels,
|
||||
settings.job_max_attempts,
|
||||
),
|
||||
users=UserService(user_repository),
|
||||
chats=ChatService(chat_repository),
|
||||
@@ -79,7 +80,7 @@ def create_app() -> FastAPI:
|
||||
application.state.redis = redis
|
||||
application.state.storage = storage
|
||||
application.state.ready = True
|
||||
logger.info('startup storage=postgres+s3 redis_sessions=true')
|
||||
logger.info('startup storage=postgres+s3 redis_sessions=true async_attachments=true')
|
||||
yield
|
||||
finally:
|
||||
application.state.ready = False
|
||||
@@ -88,8 +89,8 @@ def create_app() -> FastAPI:
|
||||
logger.info('shutdown complete')
|
||||
|
||||
api = FastAPI(
|
||||
title='MTUSI Messenger. LR 4',
|
||||
version='1.4.0',
|
||||
title='MTUSI Messenger. LR 5',
|
||||
version='1.5.0',
|
||||
description=DESCRIPTION,
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
@@ -1,13 +1,18 @@
|
||||
from .attachment import AttachmentRow, MessageAttachmentRow
|
||||
from .attachment import AttachmentRow, AttachmentVariantRow, MessageAttachmentRow
|
||||
from .chat import ChatMemberRow, ChatRow
|
||||
from .message import MessageRow
|
||||
from .user import UserRow
|
||||
from .job import AttachmentJobRow, JobEventRow, OutboxEventRow
|
||||
|
||||
__all__ = [
|
||||
'AttachmentRow',
|
||||
'AttachmentVariantRow',
|
||||
'AttachmentJobRow',
|
||||
'ChatMemberRow',
|
||||
'ChatRow',
|
||||
'MessageAttachmentRow',
|
||||
'MessageRow',
|
||||
'UserRow',
|
||||
'JobEventRow',
|
||||
'OutboxEventRow',
|
||||
]
|
||||
|
||||
@@ -14,14 +14,17 @@ class AttachmentRow(Base):
|
||||
__tablename__ = 'attachments'
|
||||
__table_args__ = (
|
||||
Index('attachments_chat_created_idx', 'chat_id', 'created_at', 'id'),
|
||||
UniqueConstraint('object_key', name='attachments_object_key_key'),
|
||||
UniqueConstraint('original_key', name='attachments_object_key_key'),
|
||||
UniqueConstraint('staging_key', name='attachments_staging_key_key'),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
|
||||
chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), nullable=False)
|
||||
owner_id: Mapped[UUID] = mapped_column(ForeignKey('users.id'), nullable=False)
|
||||
filename: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
object_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
original_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
staging_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
pipeline_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||
content_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||
sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
@@ -33,6 +36,24 @@ class AttachmentRow(Base):
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class AttachmentVariantRow(Base):
|
||||
__tablename__ = 'attachment_variants'
|
||||
|
||||
attachment_id: Mapped[UUID] = mapped_column(
|
||||
ForeignKey('attachments.id', ondelete='CASCADE'), primary_key=True
|
||||
)
|
||||
pipeline_version: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(16), primary_key=True)
|
||||
object_key: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||
content_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||
width: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
height: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class MessageAttachmentRow(Base):
|
||||
__tablename__ = 'message_attachments'
|
||||
__table_args__ = (
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer, JSON, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.types import Uuid
|
||||
|
||||
from ..db.postgres import Base
|
||||
|
||||
|
||||
class AttachmentJobRow(Base):
|
||||
__tablename__ = 'attachment_jobs'
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
'attachment_id', 'pipeline_version', name='attachment_job_attachment_version_key'
|
||||
),
|
||||
Index('attachment_jobs_lease_idx', 'state', 'lease_until'),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
|
||||
attachment_id: Mapped[UUID] = mapped_column(
|
||||
ForeignKey('attachments.id', ondelete='CASCADE'), nullable=False
|
||||
)
|
||||
pipeline_version: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
state: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
attempt: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
next_attempt_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
replay_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class OutboxEventRow(Base):
|
||||
__tablename__ = 'outbox_events'
|
||||
__table_args__ = (Index('outbox_events_pending_idx', 'published_at', 'available_at'),)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
|
||||
event_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
job_id: Mapped[UUID] = mapped_column(
|
||||
ForeignKey('attachment_jobs.id', ondelete='CASCADE'), nullable=False
|
||||
)
|
||||
payload: Mapped[dict] = mapped_column(JSON, nullable=False)
|
||||
available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
claim_owner: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
claim_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
publish_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class JobEventRow(Base):
|
||||
__tablename__ = 'job_events'
|
||||
__table_args__ = (Index('job_events_job_created_idx', 'job_id', 'created_at'),)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True)
|
||||
job_id: Mapped[UUID] = mapped_column(
|
||||
ForeignKey('attachment_jobs.id', ondelete='CASCADE'), nullable=False
|
||||
)
|
||||
kind: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
attempt: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
@@ -2,5 +2,9 @@ from .attachments import AttachmentRepository
|
||||
from .chats import ChatRepository
|
||||
from .messages import MessageRepository
|
||||
from .users import UserRepository
|
||||
from .jobs import JobRepository, OutboxRepository
|
||||
|
||||
__all__ = ['AttachmentRepository', 'ChatRepository', 'MessageRepository', 'UserRepository']
|
||||
__all__ = [
|
||||
'AttachmentRepository', 'ChatRepository', 'JobRepository', 'MessageRepository',
|
||||
'OutboxRepository', 'UserRepository',
|
||||
]
|
||||
|
||||
@@ -1,14 +1,27 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from ..domain.models import Attachment
|
||||
from ..models.attachment import AttachmentRow
|
||||
from ..domain.models import Attachment, AttachmentVariant
|
||||
from ..models.attachment import AttachmentRow, AttachmentVariantRow
|
||||
from ..models.chat import ChatMemberRow, ChatRow
|
||||
from ..models.job import AttachmentJobRow, OutboxEventRow
|
||||
|
||||
|
||||
def job_payload(job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int, now: datetime) -> dict:
|
||||
return {
|
||||
'schema_version': 1,
|
||||
'job_id': str(job_id),
|
||||
'type': 'attachment.process.v1',
|
||||
'attachment_id': str(attachment_id),
|
||||
'pipeline_version': pipeline_version,
|
||||
'attempt': attempt,
|
||||
'created_at': now.isoformat().replace('+00:00', 'Z'),
|
||||
}
|
||||
|
||||
|
||||
class AttachmentRepository:
|
||||
@@ -16,13 +29,29 @@ class AttachmentRepository:
|
||||
self._sessions = sessions
|
||||
|
||||
@staticmethod
|
||||
def _to_domain(row: AttachmentRow) -> Attachment:
|
||||
def _variant(row: AttachmentVariantRow) -> AttachmentVariant:
|
||||
return AttachmentVariant(
|
||||
name=row.name,
|
||||
object_key=row.object_key,
|
||||
content_type=row.content_type,
|
||||
size_bytes=row.size_bytes,
|
||||
width=row.width,
|
||||
height=row.height,
|
||||
sha256=row.sha256,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _to_domain(
|
||||
cls, row: AttachmentRow, variants: tuple[AttachmentVariantRow, ...] = ()
|
||||
) -> Attachment:
|
||||
return Attachment(
|
||||
id=row.id,
|
||||
chat_id=row.chat_id,
|
||||
owner_id=row.owner_id,
|
||||
filename=row.filename,
|
||||
object_key=row.object_key,
|
||||
original_key=row.original_key,
|
||||
staging_key=row.staging_key,
|
||||
pipeline_version=row.pipeline_version,
|
||||
content_type=row.content_type,
|
||||
size_bytes=row.size_bytes,
|
||||
sha256=row.sha256,
|
||||
@@ -32,6 +61,7 @@ class AttachmentRepository:
|
||||
error_code=row.error_code,
|
||||
created_at=row.created_at,
|
||||
updated_at=row.updated_at,
|
||||
variants=tuple(cls._variant(item) for item in variants),
|
||||
)
|
||||
|
||||
async def chat_exists_for_member(self, chat_id: UUID, user_id: UUID) -> bool:
|
||||
@@ -43,19 +73,21 @@ class AttachmentRepository:
|
||||
)
|
||||
return found is not None
|
||||
|
||||
async def create(
|
||||
async def create_queued(
|
||||
self,
|
||||
*,
|
||||
attachment_id: UUID,
|
||||
job_id: UUID,
|
||||
chat_id: UUID,
|
||||
owner_id: UUID,
|
||||
filename: str,
|
||||
object_key: str,
|
||||
staging_key: str,
|
||||
content_type: str,
|
||||
size_bytes: int,
|
||||
sha256: str,
|
||||
width: int,
|
||||
height: int,
|
||||
max_attempts: int,
|
||||
now: datetime,
|
||||
) -> Attachment:
|
||||
row = AttachmentRow(
|
||||
@@ -63,23 +95,67 @@ class AttachmentRepository:
|
||||
chat_id=chat_id,
|
||||
owner_id=owner_id,
|
||||
filename=filename,
|
||||
object_key=object_key,
|
||||
original_key=None,
|
||||
staging_key=staging_key,
|
||||
pipeline_version=1,
|
||||
content_type=content_type,
|
||||
size_bytes=size_bytes,
|
||||
sha256=sha256,
|
||||
width=width,
|
||||
height=height,
|
||||
status='ready',
|
||||
status='queued',
|
||||
error_code=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
job = AttachmentJobRow(
|
||||
id=job_id,
|
||||
attachment_id=attachment_id,
|
||||
pipeline_version=1,
|
||||
state='queued',
|
||||
attempt=1,
|
||||
max_attempts=max_attempts,
|
||||
next_attempt_at=now,
|
||||
lease_owner=None,
|
||||
lease_until=None,
|
||||
last_error_code=None,
|
||||
replay_count=0,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
outbox = OutboxEventRow(
|
||||
id=uuid4(),
|
||||
event_type='attachment.process.v1',
|
||||
job_id=job_id,
|
||||
payload=job_payload(job_id, attachment_id, 1, 1, now),
|
||||
available_at=now,
|
||||
published_at=None,
|
||||
claim_owner=None,
|
||||
claim_until=None,
|
||||
publish_attempts=0,
|
||||
last_error_code=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
async with self._sessions() as session:
|
||||
async with session.begin():
|
||||
session.add(row)
|
||||
session.add_all((row, job, outbox))
|
||||
await session.flush()
|
||||
return self._to_domain(row)
|
||||
|
||||
async def _variants(
|
||||
self, session: AsyncSession, attachment_id: UUID, pipeline_version: int
|
||||
) -> tuple[AttachmentVariantRow, ...]:
|
||||
rows = await session.scalars(
|
||||
select(AttachmentVariantRow)
|
||||
.where(
|
||||
AttachmentVariantRow.attachment_id == attachment_id,
|
||||
AttachmentVariantRow.pipeline_version == pipeline_version,
|
||||
)
|
||||
.order_by(AttachmentVariantRow.name)
|
||||
)
|
||||
return tuple(rows)
|
||||
|
||||
async def get_for_member(self, attachment_id: UUID, user_id: UUID) -> Attachment | None:
|
||||
async with self._sessions() as session:
|
||||
row = await session.scalar(
|
||||
@@ -90,4 +166,8 @@ class AttachmentRepository:
|
||||
ChatMemberRow.user_id == user_id,
|
||||
)
|
||||
)
|
||||
return self._to_domain(row) if row is not None else None
|
||||
if row is None:
|
||||
return None
|
||||
return self._to_domain(
|
||||
row, await self._variants(session, row.id, row.pipeline_version)
|
||||
)
|
||||
|
||||
@@ -0,0 +1,358 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from sqlalchemy import or_, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from ..domain.models import Attachment, AttachmentJob, AttachmentVariant, OutboxEvent
|
||||
from ..models.attachment import AttachmentRow, AttachmentVariantRow
|
||||
from ..models.job import AttachmentJobRow, JobEventRow, OutboxEventRow
|
||||
from .attachments import AttachmentRepository, job_payload
|
||||
|
||||
|
||||
class JobRepository:
|
||||
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sessions = sessions
|
||||
|
||||
@staticmethod
|
||||
def _job(row: AttachmentJobRow) -> AttachmentJob:
|
||||
return AttachmentJob(
|
||||
id=row.id,
|
||||
attachment_id=row.attachment_id,
|
||||
pipeline_version=row.pipeline_version,
|
||||
state=row.state,
|
||||
attempt=row.attempt,
|
||||
max_attempts=row.max_attempts,
|
||||
next_attempt_at=row.next_attempt_at,
|
||||
lease_owner=row.lease_owner,
|
||||
lease_until=row.lease_until,
|
||||
last_error_code=row.last_error_code,
|
||||
replay_count=row.replay_count,
|
||||
created_at=row.created_at,
|
||||
updated_at=row.updated_at,
|
||||
)
|
||||
|
||||
async def claim(
|
||||
self,
|
||||
job_id: UUID,
|
||||
attempt: int,
|
||||
worker_id: str,
|
||||
now: datetime,
|
||||
lease_seconds: int,
|
||||
) -> tuple[AttachmentJob, Attachment] | None:
|
||||
async with self._sessions() as session:
|
||||
async with session.begin():
|
||||
row = await session.scalar(
|
||||
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
|
||||
)
|
||||
if row is None or row.state != 'queued' or row.attempt != attempt:
|
||||
return None
|
||||
attachment_row = await session.get(AttachmentRow, row.attachment_id)
|
||||
if attachment_row is None:
|
||||
return None
|
||||
row.state = 'processing'
|
||||
row.lease_owner = worker_id
|
||||
row.lease_until = now + timedelta(seconds=lease_seconds)
|
||||
row.updated_at = now
|
||||
attachment_row.status = 'processing'
|
||||
attachment_row.updated_at = now
|
||||
session.add(
|
||||
JobEventRow(
|
||||
job_id=row.id,
|
||||
kind='claimed',
|
||||
attempt=row.attempt,
|
||||
error_code=None,
|
||||
created_at=now,
|
||||
)
|
||||
)
|
||||
await session.flush()
|
||||
return self._job(row), AttachmentRepository._to_domain(attachment_row)
|
||||
|
||||
async def state(self, job_id: UUID) -> AttachmentJob | None:
|
||||
async with self._sessions() as session:
|
||||
row = await session.get(AttachmentJobRow, job_id)
|
||||
return self._job(row) if row is not None else None
|
||||
|
||||
async def id_for_attachment(self, attachment_id: UUID) -> UUID | None:
|
||||
async with self._sessions() as session:
|
||||
return await session.scalar(
|
||||
select(AttachmentJobRow.id).where(
|
||||
AttachmentJobRow.attachment_id == attachment_id
|
||||
)
|
||||
)
|
||||
|
||||
async def ready_effect_complete(self, job_id: UUID) -> bool:
|
||||
async with self._sessions() as session:
|
||||
job = await session.get(AttachmentJobRow, job_id)
|
||||
if job is None or job.state != 'ready':
|
||||
return False
|
||||
attachment = await session.get(AttachmentRow, job.attachment_id)
|
||||
if attachment is None or not attachment.original_key:
|
||||
return False
|
||||
names = set(
|
||||
await session.scalars(
|
||||
select(AttachmentVariantRow.name).where(
|
||||
AttachmentVariantRow.attachment_id == job.attachment_id,
|
||||
AttachmentVariantRow.pipeline_version == job.pipeline_version,
|
||||
)
|
||||
)
|
||||
)
|
||||
return names == {'thumbnail', 'crop', 'optimized'}
|
||||
|
||||
async def complete(
|
||||
self,
|
||||
job_id: UUID,
|
||||
worker_id: str,
|
||||
original_key: str,
|
||||
variants: tuple[AttachmentVariant, ...],
|
||||
now: datetime,
|
||||
) -> bool:
|
||||
async with self._sessions() as session:
|
||||
async with session.begin():
|
||||
job = await session.scalar(
|
||||
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
|
||||
)
|
||||
if job is None or job.state == 'ready':
|
||||
return job is not None
|
||||
if job.state != 'processing' or job.lease_owner != worker_id:
|
||||
return False
|
||||
attachment = await session.get(AttachmentRow, job.attachment_id)
|
||||
if attachment is None:
|
||||
return False
|
||||
for variant in variants:
|
||||
statement = (
|
||||
insert(AttachmentVariantRow)
|
||||
.values(
|
||||
attachment_id=attachment.id,
|
||||
pipeline_version=job.pipeline_version,
|
||||
name=variant.name,
|
||||
object_key=variant.object_key,
|
||||
content_type=variant.content_type,
|
||||
size_bytes=variant.size_bytes,
|
||||
width=variant.width,
|
||||
height=variant.height,
|
||||
sha256=variant.sha256,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
.on_conflict_do_update(
|
||||
index_elements=[
|
||||
AttachmentVariantRow.attachment_id,
|
||||
AttachmentVariantRow.pipeline_version,
|
||||
AttachmentVariantRow.name,
|
||||
],
|
||||
set_={
|
||||
'object_key': variant.object_key,
|
||||
'content_type': variant.content_type,
|
||||
'size_bytes': variant.size_bytes,
|
||||
'width': variant.width,
|
||||
'height': variant.height,
|
||||
'sha256': variant.sha256,
|
||||
'updated_at': now,
|
||||
},
|
||||
)
|
||||
)
|
||||
await session.execute(statement)
|
||||
attachment.original_key = original_key
|
||||
attachment.status = 'ready'
|
||||
attachment.error_code = None
|
||||
attachment.updated_at = now
|
||||
job.state = 'ready'
|
||||
job.lease_owner = None
|
||||
job.lease_until = None
|
||||
job.last_error_code = None
|
||||
job.updated_at = now
|
||||
session.add(
|
||||
JobEventRow(
|
||||
job_id=job.id,
|
||||
kind='ready',
|
||||
attempt=job.attempt,
|
||||
error_code=None,
|
||||
created_at=now,
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
||||
async def retry_or_fail(
|
||||
self,
|
||||
job_id: UUID,
|
||||
worker_id: str | None,
|
||||
error_code: str,
|
||||
permanent: bool,
|
||||
available_at: datetime,
|
||||
now: datetime,
|
||||
) -> str:
|
||||
async with self._sessions() as session:
|
||||
async with session.begin():
|
||||
job = await session.scalar(
|
||||
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
|
||||
)
|
||||
if job is None or job.state in {'ready', 'failed'}:
|
||||
return job.state if job is not None else 'missing'
|
||||
if job.state != 'processing':
|
||||
return job.state
|
||||
if worker_id is not None:
|
||||
if job.lease_owner != worker_id:
|
||||
return job.state
|
||||
elif job.lease_until is None or job.lease_until >= now:
|
||||
return job.state
|
||||
attachment = await session.get(AttachmentRow, job.attachment_id)
|
||||
exhausted = job.attempt >= job.max_attempts
|
||||
if permanent or exhausted:
|
||||
job.state = 'failed'
|
||||
attachment.status = 'failed'
|
||||
attachment.error_code = error_code if permanent else 'retry_exhausted'
|
||||
kind = 'failed'
|
||||
else:
|
||||
job.attempt += 1
|
||||
job.state = 'queued'
|
||||
job.next_attempt_at = available_at
|
||||
attachment.status = 'queued'
|
||||
attachment.error_code = None
|
||||
session.add(
|
||||
OutboxEventRow(
|
||||
id=uuid4(),
|
||||
event_type='attachment.process.v1',
|
||||
job_id=job.id,
|
||||
payload=job_payload(
|
||||
job.id,
|
||||
job.attachment_id,
|
||||
job.pipeline_version,
|
||||
job.attempt,
|
||||
now,
|
||||
),
|
||||
available_at=available_at,
|
||||
published_at=None,
|
||||
claim_owner=None,
|
||||
claim_until=None,
|
||||
publish_attempts=0,
|
||||
last_error_code=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
kind = 'retry_scheduled'
|
||||
job.last_error_code = error_code
|
||||
job.lease_owner = None
|
||||
job.lease_until = None
|
||||
job.updated_at = now
|
||||
attachment.updated_at = now
|
||||
session.add(
|
||||
JobEventRow(
|
||||
job_id=job.id,
|
||||
kind=kind,
|
||||
attempt=job.attempt,
|
||||
error_code=error_code,
|
||||
created_at=now,
|
||||
)
|
||||
)
|
||||
await session.flush()
|
||||
return job.state
|
||||
|
||||
async def recover_expired(self, now: datetime) -> int:
|
||||
async with self._sessions() as session:
|
||||
ids = tuple(
|
||||
await session.scalars(
|
||||
select(AttachmentJobRow.id).where(
|
||||
AttachmentJobRow.state == 'processing',
|
||||
AttachmentJobRow.lease_until < now,
|
||||
)
|
||||
)
|
||||
)
|
||||
recovered = 0
|
||||
for job_id in ids:
|
||||
state = await self.retry_or_fail(
|
||||
job_id, None, 'lease_expired', False, now, now
|
||||
)
|
||||
recovered += state in {'queued', 'failed'}
|
||||
return recovered
|
||||
|
||||
async def replay(self, job_id: UUID, now: datetime, dry_run: bool = False) -> bool:
|
||||
async with self._sessions() as session:
|
||||
async with session.begin():
|
||||
job = await session.scalar(
|
||||
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
|
||||
)
|
||||
if job is None or job.state != 'failed':
|
||||
return False
|
||||
attachment = await session.get(AttachmentRow, job.attachment_id)
|
||||
if attachment is None or not attachment.staging_key:
|
||||
return False
|
||||
if dry_run:
|
||||
return True
|
||||
job.state = 'queued'
|
||||
job.attempt += 1
|
||||
job.max_attempts = job.attempt + 4
|
||||
job.replay_count += 1
|
||||
job.last_error_code = None
|
||||
job.next_attempt_at = now
|
||||
job.updated_at = now
|
||||
attachment.status = 'queued'
|
||||
attachment.error_code = None
|
||||
attachment.updated_at = now
|
||||
session.add_all(
|
||||
(
|
||||
OutboxEventRow(
|
||||
id=uuid4(), event_type='attachment.process.v1', job_id=job.id,
|
||||
payload=job_payload(job.id, job.attachment_id, job.pipeline_version, job.attempt, now),
|
||||
available_at=now, published_at=None, claim_owner=None, claim_until=None,
|
||||
publish_attempts=0, last_error_code=None, created_at=now, updated_at=now,
|
||||
),
|
||||
JobEventRow(job_id=job.id, kind='replayed', attempt=job.attempt, error_code=None, created_at=now),
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
class OutboxRepository:
|
||||
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sessions = sessions
|
||||
|
||||
async def claim(self, owner: str, now: datetime, lease_seconds: int, limit: int = 20) -> tuple[OutboxEvent, ...]:
|
||||
async with self._sessions() as session:
|
||||
async with session.begin():
|
||||
rows = tuple(
|
||||
await session.scalars(
|
||||
select(OutboxEventRow)
|
||||
.where(
|
||||
OutboxEventRow.published_at.is_(None),
|
||||
OutboxEventRow.available_at <= now,
|
||||
or_(OutboxEventRow.claim_until.is_(None), OutboxEventRow.claim_until < now),
|
||||
)
|
||||
.order_by(OutboxEventRow.available_at, OutboxEventRow.created_at)
|
||||
.limit(limit)
|
||||
.with_for_update(skip_locked=True)
|
||||
)
|
||||
)
|
||||
until = now + timedelta(seconds=lease_seconds)
|
||||
for row in rows:
|
||||
row.claim_owner = owner
|
||||
row.claim_until = until
|
||||
row.publish_attempts += 1
|
||||
row.updated_at = now
|
||||
return tuple(OutboxEvent(row.id, row.job_id, row.payload, row.available_at) for row in rows)
|
||||
|
||||
async def published(self, event_id: UUID, owner: str, now: datetime) -> None:
|
||||
async with self._sessions() as session:
|
||||
async with session.begin():
|
||||
await session.execute(
|
||||
update(OutboxEventRow)
|
||||
.where(
|
||||
OutboxEventRow.id == event_id,
|
||||
OutboxEventRow.claim_owner == owner,
|
||||
OutboxEventRow.published_at.is_(None),
|
||||
)
|
||||
.values(published_at=now, claim_owner=None, claim_until=None, last_error_code=None, updated_at=now)
|
||||
)
|
||||
|
||||
async def failed(self, event_id: UUID, owner: str, code: str, now: datetime) -> None:
|
||||
async with self._sessions() as session:
|
||||
async with session.begin():
|
||||
await session.execute(
|
||||
update(OutboxEventRow)
|
||||
.where(OutboxEventRow.id == event_id, OutboxEventRow.claim_owner == owner)
|
||||
.values(claim_owner=None, claim_until=None, last_error_code=code, updated_at=now)
|
||||
)
|
||||
@@ -77,8 +77,8 @@ class MessageRepository:
|
||||
}
|
||||
if set(found) != set(attachment_ids):
|
||||
raise NotFound('вложение не найдено')
|
||||
if any(status != 'ready' for status in found.values()):
|
||||
raise Conflict('оригинал вложения недоступен')
|
||||
if any(status == 'failed' for status in found.values()):
|
||||
raise Conflict('неуспешное вложение нельзя прикрепить')
|
||||
|
||||
statement = (
|
||||
insert(MessageRow)
|
||||
|
||||
+31
-11
@@ -67,11 +67,13 @@ class AttachmentService:
|
||||
storage: S3Storage,
|
||||
max_upload_bytes: int,
|
||||
max_image_pixels: int,
|
||||
job_max_attempts: int,
|
||||
) -> None:
|
||||
self._repository = repository
|
||||
self._storage = storage
|
||||
self._max_upload_bytes = max_upload_bytes
|
||||
self._max_image_pixels = max_image_pixels
|
||||
self._job_max_attempts = job_max_attempts
|
||||
|
||||
async def upload(
|
||||
self,
|
||||
@@ -106,28 +108,31 @@ class AttachmentService:
|
||||
self._max_image_pixels,
|
||||
)
|
||||
attachment_id = uuid4()
|
||||
object_key = f'originals/{attachment_id}'
|
||||
await self._storage.put(object_key, source, size, actual_type)
|
||||
job_id = uuid4()
|
||||
staging_key = f'staging/{attachment_id}/original'
|
||||
await self._storage.put(staging_key, source, size, actual_type)
|
||||
now = datetime.now(timezone.utc)
|
||||
try:
|
||||
return await self._repository.create(
|
||||
return await self._repository.create_queued(
|
||||
attachment_id=attachment_id,
|
||||
job_id=job_id,
|
||||
chat_id=chat_id,
|
||||
owner_id=actor.id,
|
||||
filename=filename,
|
||||
object_key=object_key,
|
||||
staging_key=staging_key,
|
||||
content_type=actual_type,
|
||||
size_bytes=size,
|
||||
sha256=digest.hexdigest(),
|
||||
width=width,
|
||||
height=height,
|
||||
max_attempts=self._job_max_attempts,
|
||||
now=now,
|
||||
)
|
||||
except Exception:
|
||||
try:
|
||||
await self._storage.remove(object_key)
|
||||
await self._storage.remove(staging_key)
|
||||
except Unavailable:
|
||||
logger.warning('failed to compensate orphan object key=%s', object_key)
|
||||
logger.warning('failed to compensate staging object key=%s', staging_key)
|
||||
raise
|
||||
|
||||
async def get(self, actor: User, attachment_id: UUID) -> Attachment:
|
||||
@@ -136,12 +141,27 @@ class AttachmentService:
|
||||
raise NotFound('вложение не найдено')
|
||||
return attachment
|
||||
|
||||
async def content(self, actor: User, attachment_id: UUID) -> tuple[Attachment, ObjectStream]:
|
||||
async def content(
|
||||
self, actor: User, attachment_id: UUID, variant: str
|
||||
) -> tuple[Attachment, ObjectStream, str, int]:
|
||||
attachment = await self.get(actor, attachment_id)
|
||||
if attachment.status != 'ready':
|
||||
raise Conflict('оригинал вложения недоступен')
|
||||
raise Conflict('вложение ещё не готово')
|
||||
if variant == 'original':
|
||||
key = attachment.original_key
|
||||
content_type = attachment.content_type
|
||||
size_bytes = attachment.size_bytes
|
||||
else:
|
||||
found = next((item for item in attachment.variants if item.name == variant), None)
|
||||
if found is None:
|
||||
raise NotFound('вариант вложения не найден')
|
||||
key = found.object_key
|
||||
content_type = found.content_type
|
||||
size_bytes = found.size_bytes
|
||||
if key is None:
|
||||
raise Unavailable('объект вложения отсутствует в метаданных')
|
||||
try:
|
||||
stream = await self._storage.get(attachment.object_key)
|
||||
stream = await self._storage.get(key)
|
||||
except FileNotFoundError as error:
|
||||
raise Unavailable('оригинал вложения отсутствует в хранилище') from error
|
||||
return attachment, stream
|
||||
raise Unavailable('объект вложения отсутствует в хранилище') from error
|
||||
return attachment, stream, content_type, size_bytes
|
||||
|
||||
@@ -5,6 +5,7 @@ import certifi
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import BinaryIO, Iterator
|
||||
from io import BytesIO
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from minio import Minio
|
||||
@@ -57,6 +58,7 @@ class S3Storage:
|
||||
secure=parsed.scheme == 'https',
|
||||
region=region,
|
||||
http_client=PoolManager(
|
||||
maxsize=8,
|
||||
timeout=Timeout(connect=2, read=5),
|
||||
retries=Retry(total=1, connect=1, read=1, backoff_factor=0.1),
|
||||
cert_reqs='CERT_REQUIRED',
|
||||
@@ -102,6 +104,18 @@ class S3Storage:
|
||||
self._translate(error)
|
||||
return ObjectStream(response)
|
||||
|
||||
async def read(self, key: str, max_bytes: int) -> bytes:
|
||||
stream = await self.get(key)
|
||||
result = bytearray()
|
||||
for chunk in stream.chunks():
|
||||
result.extend(chunk)
|
||||
if len(result) > max_bytes:
|
||||
raise ValueError('stored object exceeds configured byte limit')
|
||||
return bytes(result)
|
||||
|
||||
async def put_bytes(self, key: str, content: bytes, content_type: str) -> None:
|
||||
await self.put(key, BytesIO(content), len(content), content_type)
|
||||
|
||||
async def exists(self, key: str) -> bool:
|
||||
try:
|
||||
await asyncio.to_thread(self._client.stat_object, self.bucket, key)
|
||||
|
||||
@@ -25,7 +25,11 @@ async def cleanup(apply: bool, grace_seconds: int) -> int:
|
||||
try:
|
||||
sessions = create_session_factory(engine)
|
||||
async with sessions() as session:
|
||||
known = set(await session.scalars(select(AttachmentRow.object_key)))
|
||||
known = set(
|
||||
key
|
||||
for key in await session.scalars(select(AttachmentRow.original_key))
|
||||
if key is not None
|
||||
)
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
|
||||
removed = 0
|
||||
for item in await storage.list('originals/'):
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..config import load_settings
|
||||
from ..db import create_engine, create_session_factory
|
||||
from ..models.attachment import AttachmentRow
|
||||
from ..storage import S3Storage
|
||||
|
||||
|
||||
async def cleanup(apply: bool, retention_seconds: int) -> int:
|
||||
settings = load_settings()
|
||||
engine = create_engine(settings.database_url)
|
||||
sessions = create_session_factory(engine)
|
||||
storage = S3Storage(
|
||||
settings.s3_endpoint,
|
||||
settings.s3_access_key_id,
|
||||
settings.s3_secret_access_key,
|
||||
settings.s3_bucket,
|
||||
settings.s3_region,
|
||||
)
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(seconds=retention_seconds)
|
||||
try:
|
||||
async with sessions() as session:
|
||||
active = set(
|
||||
key
|
||||
for key in await session.scalars(
|
||||
select(AttachmentRow.staging_key).where(
|
||||
AttachmentRow.staging_key.is_not(None),
|
||||
AttachmentRow.status.in_(('queued', 'processing')),
|
||||
)
|
||||
)
|
||||
if key is not None
|
||||
)
|
||||
removed = 0
|
||||
for item in await storage.list('staging/'):
|
||||
if item.key in active or item.last_modified > cutoff:
|
||||
continue
|
||||
print(f'{"DELETE" if apply else "WOULD_DELETE"} key={item.key}')
|
||||
if apply:
|
||||
await storage.remove(item.key)
|
||||
removed += 1
|
||||
print(f'summary active={len(active)} removed={removed} mode={"apply" if apply else "dry-run"}')
|
||||
return 0
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description='Clean inactive durable staging objects')
|
||||
mode = parser.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument('--dry-run', action='store_true')
|
||||
mode.add_argument('--apply', action='store_true')
|
||||
parser.add_argument('--retention-seconds', type=int)
|
||||
args = parser.parse_args()
|
||||
settings = load_settings()
|
||||
retention = args.retention_seconds or settings.staging_retention_seconds
|
||||
if retention < 60:
|
||||
parser.error('retention must be at least 60 seconds')
|
||||
raise SystemExit(asyncio.run(cleanup(args.apply, retention)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -74,7 +74,7 @@ async def _load_attachments(
|
||||
await session.execute(
|
||||
select(
|
||||
AttachmentRow.id,
|
||||
AttachmentRow.object_key,
|
||||
AttachmentRow.original_key,
|
||||
AttachmentRow.status,
|
||||
AttachmentRow.updated_at,
|
||||
)
|
||||
@@ -88,7 +88,7 @@ async def _metadata_key_exists(
|
||||
) -> bool:
|
||||
async with sessions() as session:
|
||||
attachment_id = await session.scalar(
|
||||
select(AttachmentRow.id).where(AttachmentRow.object_key == object_key).limit(1)
|
||||
select(AttachmentRow.id).where(AttachmentRow.original_key == object_key).limit(1)
|
||||
)
|
||||
return attachment_id is not None
|
||||
|
||||
@@ -165,7 +165,7 @@ async def reconcile(
|
||||
update(AttachmentRow)
|
||||
.where(
|
||||
AttachmentRow.id == attachment.id,
|
||||
AttachmentRow.object_key == attachment.object_key,
|
||||
AttachmentRow.original_key == attachment.object_key,
|
||||
AttachmentRow.status == 'ready',
|
||||
AttachmentRow.updated_at == attachment.updated_at,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
from datetime import datetime, timezone
|
||||
from uuid import UUID
|
||||
|
||||
from ..config import load_settings
|
||||
from ..db import create_engine, create_session_factory
|
||||
from ..repositories import JobRepository
|
||||
|
||||
|
||||
async def replay(job_id: UUID | None, attachment_id: UUID | None, dry_run: bool) -> int:
|
||||
settings = load_settings()
|
||||
engine = create_engine(settings.database_url)
|
||||
try:
|
||||
repository = JobRepository(create_session_factory(engine))
|
||||
if job_id is None:
|
||||
assert attachment_id is not None
|
||||
job_id = await repository.id_for_attachment(attachment_id)
|
||||
if job_id is None:
|
||||
print(f'REJECTED attachment_id={attachment_id} reason=job_not_found')
|
||||
return 2
|
||||
accepted = await repository.replay(job_id, datetime.now(timezone.utc), dry_run)
|
||||
if not accepted:
|
||||
print(f'REJECTED job_id={job_id} reason=not_failed_or_staging_missing')
|
||||
return 2
|
||||
print(f'{"WOULD_REPLAY" if dry_run else "REPLAYED"} job_id={job_id}')
|
||||
return 0
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description='Safely replay one failed attachment job')
|
||||
target = parser.add_mutually_exclusive_group(required=True)
|
||||
target.add_argument('--job-id', type=UUID)
|
||||
target.add_argument('--attachment-id', type=UUID)
|
||||
parser.add_argument('--dry-run', action='store_true')
|
||||
args = parser.parse_args()
|
||||
raise SystemExit(asyncio.run(replay(args.job_id, args.attachment_id, args.dry_run)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -50,7 +50,9 @@ async def prepare(size: int) -> None:
|
||||
'chat_id': CHAT_ID,
|
||||
'owner_id': USER_ID,
|
||||
'filename': f'benchmark-{index}.png',
|
||||
'object_key': f'{OBJECT_PREFIX}{index}',
|
||||
'original_key': f'{OBJECT_PREFIX}{index}',
|
||||
'staging_key': None,
|
||||
'pipeline_version': 1,
|
||||
'content_type': 'image/png',
|
||||
'size_bytes': 1,
|
||||
'sha256': '0' * 64,
|
||||
@@ -80,7 +82,7 @@ async def prepare(size: int) -> None:
|
||||
for offset in range(0, size, 500):
|
||||
await session.execute(insert(AttachmentRow), rows[offset : offset + 500])
|
||||
|
||||
await _put_objects(storage, [row['object_key'] for row in rows])
|
||||
await _put_objects(storage, [row['original_key'] for row in rows])
|
||||
print(f'prepared metadata={size} objects={size}')
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
@@ -20,6 +20,17 @@ services:
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
|
||||
broker:
|
||||
image: rabbitmq:4.1.4-alpine
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: messenger_test
|
||||
RABBITMQ_DEFAULT_PASS: messenger_test
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 2s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
|
||||
s3:
|
||||
image: cgr.dev/chainguard/minio@sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597
|
||||
user: "0"
|
||||
@@ -69,6 +80,7 @@ services:
|
||||
SESSION_TTL_SECONDS: 3600
|
||||
S3_ACCESS_KEY_ID: messenger-api
|
||||
S3_SECRET_ACCESS_KEY: test-minio-api-password
|
||||
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
@@ -76,6 +88,8 @@ services:
|
||||
|
||||
api:
|
||||
build: .
|
||||
ports:
|
||||
- "127.0.0.1:18080:8080"
|
||||
environment:
|
||||
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
@@ -87,6 +101,8 @@ services:
|
||||
S3_SECRET_ACCESS_KEY: test-minio-api-password
|
||||
MAX_UPLOAD_BYTES: 10485760
|
||||
MAX_IMAGE_PIXELS: 40000000
|
||||
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
|
||||
JOB_MAX_ATTEMPTS: 5
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
@@ -100,6 +116,67 @@ services:
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
|
||||
publisher:
|
||||
build: .
|
||||
command: ["python", "-m", "app.jobs.publisher"]
|
||||
environment:
|
||||
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
SESSION_TTL_SECONDS: 3600
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID: messenger-api
|
||||
S3_SECRET_ACCESS_KEY: test-minio-api-password
|
||||
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
|
||||
worker:
|
||||
build: .
|
||||
command: ["python", "-m", "app.jobs.worker"]
|
||||
environment:
|
||||
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
SESSION_TTL_SECONDS: 3600
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID: messenger-api
|
||||
S3_SECRET_ACCESS_KEY: test-minio-api-password
|
||||
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
|
||||
WORKER_CONCURRENCY: 2
|
||||
WORKER_PREFETCH: 2
|
||||
JOB_MAX_ATTEMPTS: 5
|
||||
JOB_TIMEOUT_SECONDS: 60
|
||||
JOB_LEASE_SECONDS: 90
|
||||
RETRY_BASE_SECONDS: 1
|
||||
RETRY_MAX_SECONDS: 4
|
||||
STAGING_RETENTION_SECONDS: 86400
|
||||
THUMBNAIL_JPEG_QUALITY: 85
|
||||
CROP_JPEG_QUALITY: 85
|
||||
OPTIMIZED_WEBP_QUALITY: 82
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
|
||||
tests:
|
||||
build:
|
||||
context: .
|
||||
@@ -114,6 +191,7 @@ services:
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID: messenger-api
|
||||
S3_SECRET_ACCESS_KEY: test-minio-api-password
|
||||
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
|
||||
@@ -28,6 +28,21 @@ services:
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
|
||||
broker:
|
||||
image: rabbitmq:4.1.4-alpine
|
||||
cpus: "0.5"
|
||||
mem_limit: 384m
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: ${BROKER_USER:-messenger}
|
||||
RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:-change-me-rabbitmq}
|
||||
volumes:
|
||||
- broker_data:/var/lib/rabbitmq
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
|
||||
s3:
|
||||
image: cgr.dev/chainguard/minio@sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597
|
||||
user: "0"
|
||||
@@ -85,6 +100,7 @@ services:
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
|
||||
BROKER_URL: ${BROKER_URL}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
@@ -106,6 +122,8 @@ services:
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
BROKER_URL: ${BROKER_URL}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
expose:
|
||||
- "8080"
|
||||
deploy:
|
||||
@@ -124,6 +142,79 @@ services:
|
||||
retries: 5
|
||||
stop_grace_period: 10s
|
||||
|
||||
publisher:
|
||||
build: .
|
||||
command: ["python", "-m", "app.jobs.publisher"]
|
||||
cpus: "0.25"
|
||||
mem_limit: 192m
|
||||
environment:
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
REDIS_URL: ${REDIS_URL}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
|
||||
BROKER_URL: ${BROKER_URL}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90}
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 10s
|
||||
|
||||
worker:
|
||||
build: .
|
||||
command: ["python", "-m", "app.jobs.worker"]
|
||||
cpus: "1.0"
|
||||
mem_limit: 768m
|
||||
environment:
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
REDIS_URL: ${REDIS_URL}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
BROKER_URL: ${BROKER_URL}
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
WORKER_PREFETCH: ${WORKER_PREFETCH:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90}
|
||||
RETRY_BASE_SECONDS: ${RETRY_BASE_SECONDS:-2}
|
||||
RETRY_MAX_SECONDS: ${RETRY_MAX_SECONDS:-60}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
THUMBNAIL_JPEG_QUALITY: ${THUMBNAIL_JPEG_QUALITY:-85}
|
||||
CROP_JPEG_QUALITY: ${CROP_JPEG_QUALITY:-85}
|
||||
OPTIMIZED_WEBP_QUALITY: ${OPTIMIZED_WEBP_QUALITY:-82}
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 70s
|
||||
|
||||
lb:
|
||||
image: haproxy:3.2.26-alpine
|
||||
cpus: "0.25"
|
||||
@@ -146,3 +237,4 @@ volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
s3_data:
|
||||
broker_data:
|
||||
|
||||
@@ -7,3 +7,4 @@ redis==5.2.1
|
||||
argon2-cffi==25.1.0
|
||||
minio==7.2.20
|
||||
Pillow==13.0.0
|
||||
aio-pika==9.5.5
|
||||
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
cd "$(dirname -- "$script_dir")"
|
||||
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
|
||||
set +e
|
||||
docker compose -f "$compose_file" --env-file .env run --rm \
|
||||
-e FAULT_PUBLISHER_AFTER_CONFIRM=1 \
|
||||
publisher python -m app.jobs.publisher --once >/dev/null
|
||||
status=$?
|
||||
set -e
|
||||
if [ "$status" -ne 91 ]; then
|
||||
printf '%s\n' "publisher fault expected exit 91, got $status" >&2
|
||||
exit 1
|
||||
fi
|
||||
docker compose -f "$compose_file" --env-file .env up -d publisher >/dev/null
|
||||
Executable
+26
@@ -0,0 +1,26 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
if [ "$#" -ne 1 ]; then
|
||||
printf '%s\n' 'usage: crash-worker.sh objects|commit|permanent' >&2
|
||||
exit 2
|
||||
fi
|
||||
case "$1" in
|
||||
objects) variable=FAULT_WORKER_AFTER_OBJECTS; expected=92 ;;
|
||||
commit) variable=FAULT_WORKER_AFTER_COMMIT; expected=93 ;;
|
||||
permanent) variable=FAULT_WORKER_PERMANENT; expected=94 ;;
|
||||
*) printf '%s\n' 'mode must be objects, commit or permanent' >&2; exit 2 ;;
|
||||
esac
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
cd "$(dirname -- "$script_dir")"
|
||||
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
|
||||
set +e
|
||||
docker compose -f "$compose_file" --env-file .env run --rm \
|
||||
-e "$variable=1" -e JOB_LEASE_SECONDS=3 \
|
||||
worker python -m app.jobs.worker >/dev/null
|
||||
status=$?
|
||||
set -e
|
||||
if [ "$status" -ne "$expected" ]; then
|
||||
printf '%s\n' "worker fault expected exit $expected, got $status" >&2
|
||||
exit 1
|
||||
fi
|
||||
docker compose -f "$compose_file" --env-file .env up -d worker >/dev/null
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
if [ "$#" -ne 1 ]; then
|
||||
printf '%s\n' 'usage: replay-attachment.sh ATTACHMENT_ID' >&2
|
||||
exit 2
|
||||
fi
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
cd "$(dirname -- "$script_dir")"
|
||||
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
|
||||
docker compose -f "$compose_file" --env-file .env run --rm \
|
||||
publisher python -m app.tools.replay_failed --attachment-id "$1"
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
cd "$(dirname -- "$script_dir")"
|
||||
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
|
||||
docker compose -f "$compose_file" --env-file .env restart api >/dev/null
|
||||
if [ "$compose_file" = "compose.yaml" ]; then
|
||||
docker compose -f "$compose_file" --env-file .env up -d --wait --wait-timeout 60 api lb >/dev/null
|
||||
else
|
||||
docker compose -f "$compose_file" --env-file .env up -d api >/dev/null
|
||||
fi
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
cd "$(dirname -- "$script_dir")"
|
||||
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
|
||||
docker compose -f "$compose_file" --env-file .env restart broker >/dev/null
|
||||
+4
-4
@@ -6,7 +6,7 @@ project_dir=$(dirname -- "$script_dir")
|
||||
cd "$project_dir"
|
||||
|
||||
# Останавливаем публичный маршрут до перезапуска общих хранилищ.
|
||||
docker compose --env-file .env stop lb api >/dev/null
|
||||
docker compose --env-file .env restart postgres redis s3 >/dev/null
|
||||
docker compose --env-file .env up -d --wait --wait-timeout 60 postgres redis s3 >/dev/null
|
||||
docker compose --env-file .env up -d --wait --wait-timeout 60 api lb >/dev/null
|
||||
docker compose --env-file .env stop lb api publisher worker >/dev/null
|
||||
docker compose --env-file .env restart postgres redis s3 broker >/dev/null
|
||||
docker compose --env-file .env up -d --wait --wait-timeout 60 postgres redis s3 broker >/dev/null
|
||||
docker compose --env-file .env up -d --wait --wait-timeout 60 publisher worker api lb >/dev/null
|
||||
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
cd "$(dirname -- "$script_dir")"
|
||||
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
|
||||
docker compose -f "$compose_file" --env-file .env up -d publisher >/dev/null
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
project_dir=$(dirname -- "$script_dir")
|
||||
cd "$project_dir"
|
||||
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
|
||||
docker compose -f "$compose_file" --env-file .env up -d worker >/dev/null
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
cd "$(dirname -- "$script_dir")"
|
||||
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
|
||||
docker compose -f "$compose_file" --env-file .env stop publisher >/dev/null
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
project_dir=$(dirname -- "$script_dir")
|
||||
cd "$project_dir"
|
||||
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
|
||||
docker compose -f "$compose_file" --env-file .env stop worker >/dev/null
|
||||
@@ -22,12 +22,13 @@ def main():
|
||||
uploaded = client.call(
|
||||
'POST',
|
||||
f'/api/v1/chats/{chat["id"]}/attachments?filename=failover.png',
|
||||
201,
|
||||
202,
|
||||
actor=alice,
|
||||
raw=content,
|
||||
headers={'Content-Type': 'image/png'},
|
||||
schema='Attachment',
|
||||
)
|
||||
ready = client.wait_ready(bob, uploaded['json'])
|
||||
target = uploaded['headers']['X-Instance-Id']
|
||||
env = dict(os.environ)
|
||||
env['TARGET_INSTANCE_ID'] = target
|
||||
@@ -40,7 +41,7 @@ def main():
|
||||
try:
|
||||
downloaded = client.call(
|
||||
'GET',
|
||||
f'/api/v1/attachments/{uploaded["json"]["id"]}/content',
|
||||
f'/api/v1/attachments/{ready["id"]}/content',
|
||||
actor=bob,
|
||||
)
|
||||
if downloaded['headers']['X-Instance-Id'] != target:
|
||||
|
||||
Executable
+47
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
"""A persistent broker volume keeps a confirmed job across broker restart."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def executable(value):
|
||||
path = Path(value).resolve()
|
||||
check(path.is_file() and os.access(path, os.X_OK), str(path) + ': not executable')
|
||||
return path
|
||||
|
||||
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument('--stop-worker-script', required=True)
|
||||
parser.add_argument('--restart-broker-script', required=True)
|
||||
parser.add_argument('--start-worker-script', required=True)
|
||||
|
||||
args = arguments('RabbitMQ persistent delivery after restart', configure)
|
||||
stop_worker = executable(args.stop_worker_script)
|
||||
restart_broker = executable(args.restart_broker_script)
|
||||
start_worker = executable(args.start_worker_script)
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, _eve, chat = client.bootstrap()
|
||||
subprocess.run([str(stop_worker)], check=True)
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
time.sleep(2)
|
||||
queued = client.call(
|
||||
'GET', f'/api/v1/attachments/{attachment["id"]}', actor=bob, schema='Attachment'
|
||||
)['json']
|
||||
check(queued['status'] == 'queued', 'worker processed job while expected stopped')
|
||||
subprocess.run([str(restart_broker)], check=True)
|
||||
subprocess.run([str(start_worker)], check=True)
|
||||
client.wait_ready(bob, attachment)
|
||||
downloaded = client.call(
|
||||
'GET', f'/api/v1/attachments/{attachment["id"]}/content', actor=bob
|
||||
)['bytes']
|
||||
check(downloaded == original, 'job/original lost across broker restart')
|
||||
print('PASS: confirmed durable message survived broker restart with its volume')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Executable
+58
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prove permanent failure, DLQ routing and guarded operator replay."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def executable(value):
|
||||
path = Path(value).resolve()
|
||||
check(path.is_file() and os.access(path, os.X_OK), str(path) + ': not executable')
|
||||
return path
|
||||
|
||||
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument('--stop-worker-script', required=True)
|
||||
parser.add_argument('--crash-worker-script', required=True)
|
||||
parser.add_argument('--replay-script', required=True)
|
||||
|
||||
args = arguments('Failed registry, DLQ and guarded replay', configure)
|
||||
stop = executable(args.stop_worker_script)
|
||||
crash = executable(args.crash_worker_script)
|
||||
replay = executable(args.replay_script)
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, _eve, chat = client.bootstrap()
|
||||
subprocess.run([str(stop)], check=True)
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
subprocess.run([str(crash), 'permanent'], check=True)
|
||||
|
||||
deadline = time.monotonic() + 20
|
||||
while True:
|
||||
failed = client.call(
|
||||
'GET', f'/api/v1/attachments/{attachment["id"]}', actor=bob,
|
||||
schema='Attachment',
|
||||
)['json']
|
||||
if failed['status'] == 'failed':
|
||||
break
|
||||
check(time.monotonic() < deadline, 'attachment did not reach failed')
|
||||
time.sleep(0.2)
|
||||
check(failed['error_code'] == 'fault_injected', 'unsafe or unexpected error code')
|
||||
|
||||
subprocess.run([str(replay), attachment['id']], check=True)
|
||||
ready = client.wait_ready(bob, attachment)
|
||||
check(len(ready['variants']) == 3, 'replay produced incomplete variants')
|
||||
content = client.call(
|
||||
'GET', f'/api/v1/attachments/{attachment["id"]}/content', actor=bob
|
||||
)['bytes']
|
||||
check(content == original, 'replay changed original')
|
||||
duplicate = subprocess.run([str(replay), attachment['id']], check=False)
|
||||
check(duplicate.returncode != 0, 'ready job was replayed again')
|
||||
print('PASS: permanent failure reached failed/DLQ and guarded replay reached ready')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prove that a committed outbox survives publisher downtime."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument('--stop-publisher-script', required=True)
|
||||
parser.add_argument('--start-publisher-script', required=True)
|
||||
|
||||
args = arguments('Transactional outbox recovery', configure)
|
||||
stop = Path(args.stop_publisher_script).resolve()
|
||||
start = Path(args.start_publisher_script).resolve()
|
||||
check(all(path.is_file() and os.access(path, os.X_OK) for path in (stop, start)), 'scripts must be executable')
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, _eve, chat = client.bootstrap()
|
||||
subprocess.run([str(stop)], check=True)
|
||||
attachment, _content = client.attachment(alice, chat)
|
||||
time.sleep(1)
|
||||
queued = client.call(
|
||||
'GET', f'/api/v1/attachments/{attachment["id"]}', actor=bob, schema='Attachment'
|
||||
)['json']
|
||||
check(queued['status'] == 'queued', 'job escaped outbox while publisher was stopped')
|
||||
subprocess.run([str(start)], check=True)
|
||||
ready = client.wait_ready(bob, attachment)
|
||||
check(ready['status'] == 'ready', 'outbox did not recover committed job')
|
||||
print('PASS: COMMIT-before-publish recovered without a new upload')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Executable
+45
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Grade-4 black-box check for every promised image variant."""
|
||||
import hashlib
|
||||
|
||||
from client import Client, arguments, check, run
|
||||
from image_probe import jpeg_size
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Verify thumbnail, crop and optimized variants')
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, _eve, chat = client.bootstrap()
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
ready = client.wait_ready(bob, attachment)
|
||||
variants = {item['name']: item for item in ready['variants']}
|
||||
check(set(variants) == {'thumbnail', 'crop', 'optimized'}, 'grade-4 variants missing')
|
||||
expected = {
|
||||
'thumbnail': ('image/jpeg', (256, 160)),
|
||||
'crop': ('image/jpeg', (128, 128)),
|
||||
'optimized': ('image/webp', (320, 200)),
|
||||
}
|
||||
for name, (content_type, dimensions) in expected.items():
|
||||
response = client.call(
|
||||
'GET',
|
||||
f'/api/v1/attachments/{ready["id"]}/content?variant={name}',
|
||||
actor=bob,
|
||||
)
|
||||
metadata = variants[name]
|
||||
check(response['headers'].get_content_type() == content_type, name + ': MIME mismatch')
|
||||
check((metadata['width'], metadata['height']) == dimensions, name + ': metadata dimensions')
|
||||
check(len(response['bytes']) == metadata['size_bytes'], name + ': size mismatch')
|
||||
check(hashlib.sha256(response['bytes']).hexdigest() == metadata['sha256'], name + ': hash mismatch')
|
||||
if content_type == 'image/jpeg':
|
||||
check(jpeg_size(response['bytes']) == dimensions, name + ': actual dimensions')
|
||||
else:
|
||||
check(response['bytes'][:4] == b'RIFF' and response['bytes'][8:12] == b'WEBP', 'optimized is not WebP')
|
||||
downloaded = client.call(
|
||||
'GET', f'/api/v1/attachments/{ready["id"]}/content', actor=bob
|
||||
)['bytes']
|
||||
check(downloaded == original, 'original bytes changed')
|
||||
print('PASS: original plus thumbnail/crop/optimized variants are valid')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Crash after publisher confirm creates a harmless duplicate delivery."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def executable(value):
|
||||
path = Path(value).resolve()
|
||||
check(path.is_file() and os.access(path, os.X_OK), str(path) + ': not executable')
|
||||
return path
|
||||
|
||||
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument('--stop-publisher-script', required=True)
|
||||
parser.add_argument('--crash-publisher-script', required=True)
|
||||
|
||||
args = arguments('Crash after publisher confirm', configure)
|
||||
stop = executable(args.stop_publisher_script)
|
||||
crash = executable(args.crash_publisher_script)
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, _eve, chat = client.bootstrap()
|
||||
subprocess.run([str(stop)], check=True)
|
||||
attachment, _original = client.attachment(alice, chat)
|
||||
subprocess.run([str(crash)], check=True)
|
||||
ready = client.wait_ready(bob, attachment)
|
||||
names = [item['name'] for item in ready['variants']]
|
||||
check(sorted(names) == ['crop', 'optimized', 'thumbnail'], 'duplicate or missing variants')
|
||||
print('PASS: confirm-before-outbox-mark duplicate produced one logical result')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -16,7 +16,7 @@ def scenario():
|
||||
return client, alice, bob, eve, chat
|
||||
|
||||
|
||||
def upload(client, actor, chat, content, *, filename='sample.png', content_type='image/png', expected=201):
|
||||
def upload(client, actor, chat, content, *, filename='sample.png', content_type='image/png', expected=202):
|
||||
return client.call(
|
||||
'POST',
|
||||
f'/api/v1/chats/{chat["id"]}/attachments?filename={quote(filename, safe="")}',
|
||||
@@ -24,7 +24,7 @@ def upload(client, actor, chat, content, *, filename='sample.png', content_type=
|
||||
actor=actor,
|
||||
raw=content,
|
||||
headers={'Content-Type': content_type},
|
||||
schema='Attachment' if expected == 201 else None,
|
||||
schema='Attachment' if expected == 202 else None,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from io import BytesIO
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from app.jobs.pipeline import PermanentImageError, process_image
|
||||
|
||||
|
||||
def encoded(width: int, height: int, mode: str = 'RGBA') -> bytes:
|
||||
image = Image.new(mode, (width, height), (20, 80, 160, 128) if mode == 'RGBA' else 'navy')
|
||||
output = BytesIO()
|
||||
image.save(output, 'PNG', pnginfo=None)
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
def run(source: bytes):
|
||||
return process_image(
|
||||
source,
|
||||
'image/png',
|
||||
hashlib.sha256(source).hexdigest(),
|
||||
40_000_000,
|
||||
85,
|
||||
85,
|
||||
82,
|
||||
)
|
||||
|
||||
|
||||
def test_pipeline_creates_all_grade4_variants() -> None:
|
||||
"""Create every grade-4 variant with the required format, size and checksum."""
|
||||
variants = {item.name: item for item in run(encoded(320, 200))}
|
||||
assert set(variants) == {'thumbnail', 'crop', 'optimized'}
|
||||
assert (variants['thumbnail'].width, variants['thumbnail'].height) == (256, 160)
|
||||
assert variants['thumbnail'].content_type == 'image/jpeg'
|
||||
assert (variants['crop'].width, variants['crop'].height) == (128, 128)
|
||||
assert variants['crop'].content_type == 'image/jpeg'
|
||||
assert (variants['optimized'].width, variants['optimized'].height) == (320, 200)
|
||||
assert variants['optimized'].content_type == 'image/webp'
|
||||
for item in variants.values():
|
||||
assert hashlib.sha256(item.content).hexdigest() == item.sha256
|
||||
|
||||
|
||||
def test_thumbnail_and_optimized_do_not_upscale_small_image() -> None:
|
||||
"""Keep small images at source size except for the required 128x128 crop."""
|
||||
variants = {item.name: item for item in run(encoded(40, 20, 'RGB'))}
|
||||
assert (variants['thumbnail'].width, variants['thumbnail'].height) == (40, 20)
|
||||
assert (variants['optimized'].width, variants['optimized'].height) == (40, 20)
|
||||
assert (variants['crop'].width, variants['crop'].height) == (128, 128)
|
||||
|
||||
|
||||
def test_jpeg_variants_composite_alpha_on_white() -> None:
|
||||
"""Flatten transparent input before encoding JPEG variants without an alpha channel."""
|
||||
variants = {item.name: item for item in run(encoded(2, 2))}
|
||||
with Image.open(BytesIO(variants['thumbnail'].content)) as image:
|
||||
assert image.mode == 'RGB'
|
||||
|
||||
|
||||
def test_checksum_mismatch_is_permanent() -> None:
|
||||
"""Reject changed staging bytes as a permanent checksum error."""
|
||||
source = encoded(20, 20)
|
||||
with pytest.raises(PermanentImageError, match='checksum_mismatch'):
|
||||
process_image(source, 'image/png', '0' * 64, 40_000_000, 85, 85, 82)
|
||||
|
||||
|
||||
def test_exif_orientation_is_applied_and_removed() -> None:
|
||||
"""Apply EXIF orientation before resizing and omit metadata from generated output."""
|
||||
image = Image.new('RGB', (320, 200), 'red')
|
||||
exif = Image.Exif()
|
||||
exif[274] = 6
|
||||
output = BytesIO()
|
||||
image.save(output, 'JPEG', exif=exif)
|
||||
source = output.getvalue()
|
||||
variants = {
|
||||
item.name: item
|
||||
for item in process_image(
|
||||
source,
|
||||
'image/jpeg',
|
||||
hashlib.sha256(source).hexdigest(),
|
||||
40_000_000,
|
||||
85,
|
||||
85,
|
||||
82,
|
||||
)
|
||||
}
|
||||
assert (variants['thumbnail'].width, variants['thumbnail'].height) == (160, 256)
|
||||
with Image.open(BytesIO(variants['thumbnail'].content)) as transformed:
|
||||
assert not transformed.getexif()
|
||||
Executable
+41
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fault injection around S3 effects, DB commit and broker ack."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def executable(value):
|
||||
path = Path(value).resolve()
|
||||
check(path.is_file() and os.access(path, os.X_OK), str(path) + ': not executable')
|
||||
return path
|
||||
|
||||
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument('--stop-worker-script', required=True)
|
||||
parser.add_argument('--crash-worker-script', required=True)
|
||||
parser.add_argument('--mode', choices=('objects', 'commit'), required=True)
|
||||
|
||||
args = arguments('Worker crash around durable effects', configure)
|
||||
stop = executable(args.stop_worker_script)
|
||||
crash = executable(args.crash_worker_script)
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, _eve, chat = client.bootstrap()
|
||||
subprocess.run([str(stop)], check=True)
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
subprocess.run([str(crash), args.mode], check=True)
|
||||
ready = client.wait_ready(bob, attachment)
|
||||
names = sorted(item['name'] for item in ready['variants'])
|
||||
check(names == ['crop', 'optimized', 'thumbnail'], 'duplicate or missing variants')
|
||||
downloaded = client.call(
|
||||
'GET', f'/api/v1/attachments/{attachment["id"]}/content', actor=bob
|
||||
)['bytes']
|
||||
check(downloaded == original, 'original changed after crash recovery')
|
||||
print('PASS: worker crash mode=%s recovered one logical result' % args.mode)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Executable
+61
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Required scenario: accepted work survives worker and API downtime."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def executable(value):
|
||||
path = Path(value).expanduser().resolve()
|
||||
check(path.is_file() and os.access(path, os.X_OK), str(path) + ': not executable')
|
||||
return path
|
||||
|
||||
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument('--stop-worker-script', required=True)
|
||||
parser.add_argument('--restart-api-script', required=True)
|
||||
parser.add_argument('--start-worker-script', required=True)
|
||||
|
||||
args = arguments('Worker outage and durable accepted job', configure)
|
||||
stop_worker = executable(args.stop_worker_script)
|
||||
restart_api = executable(args.restart_api_script)
|
||||
start_worker = executable(args.start_worker_script)
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, _eve, chat = client.bootstrap()
|
||||
subprocess.run([str(stop_worker)], check=True)
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
message = client.message(alice, chat, '', attachments=[attachment['id']])['json']
|
||||
check(message['attachment_ids'] == [attachment['id']], 'queued attachment was not linkable')
|
||||
time.sleep(1)
|
||||
client.call(
|
||||
'GET', f'/api/v1/attachments/{attachment["id"]}/content', 409, actor=bob
|
||||
)
|
||||
subprocess.run([str(restart_api)], check=True)
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
try:
|
||||
client.call('GET', '/health/ready', schema='Health')
|
||||
break
|
||||
except Exception:
|
||||
check(time.monotonic() < deadline, 'API did not recover after restart')
|
||||
time.sleep(0.3)
|
||||
persisted = client.call(
|
||||
'GET', f'/api/v1/attachments/{attachment["id"]}', actor=bob, schema='Attachment'
|
||||
)['json']
|
||||
check(persisted['status'] in {'queued', 'processing'}, 'accepted job disappeared after API restart')
|
||||
subprocess.run([str(start_worker)], check=True)
|
||||
ready = client.wait_ready(bob, attachment)
|
||||
downloaded = client.call(
|
||||
'GET', f'/api/v1/attachments/{attachment["id"]}/content', actor=bob
|
||||
)['bytes']
|
||||
check(downloaded == original, 'original changed after worker recovery')
|
||||
check(ready['status'] == 'ready', 'job did not become ready')
|
||||
print('PASS: worker stopped -> 202 -> message -> API restart -> worker -> ready')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Reference in New Issue
Block a user