Реализовать асинхронную обработку вложений

This commit is contained in:
2026-11-17 18:21:09 +03:00
parent 4d54ba3848
commit e8becaf761
51 changed files with 2234 additions and 58 deletions
+10
View File
@@ -20,15 +20,25 @@ MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
WORKER_PREFETCH=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
JOB_LEASE_SECONDS=90
RETRY_BASE_SECONDS=2
RETRY_MAX_SECONDS=60
STAGING_RETENTION_SECONDS=86400
THUMBNAIL_JPEG_QUALITY=85
CROP_JPEG_QUALITY=85
OPTIMIZED_WEBP_QUALITY=82
POSTGRES_USER=messenger
POSTGRES_PASSWORD=change-me-local
POSTGRES_DB=messenger
DATABASE_URL=postgresql+asyncpg://messenger:change-me-local@postgres:5432/messenger
REDIS_URL=redis://redis:6379/0
BROKER_USER=messenger
BROKER_PASSWORD=change-me-rabbitmq
BROKER_URL=amqp://messenger:change-me-rabbitmq@broker:5672/
MINIO_ROOT_USER=local-minio-admin
MINIO_ROOT_PASSWORD=change-me-minio-root
+56 -1
View File
@@ -4,7 +4,7 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test test-attachments test-pipeline test-pipeline-smoke test-outbox test-publisher-confirm-crash test-worker-crash test-failed-replay test-worker-outage test-broker-persistence test-docker load test-persistence test-failover replay-failed cleanup-staging
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -18,6 +18,54 @@ validate:
test:
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-attachments:
$(PYTHON) -m pytest -q tests/test_attachments.py
test-pipeline:
$(PYTHON) -m pytest -q tests/test_pipeline.py
test-pipeline-smoke:
$(PYTHON) tests/pipeline_smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-outbox:
$(PYTHON) tests/outbox_recovery.py --base-url "$(BASE_URL)" \
--stop-publisher-script scripts/stop-publisher.sh \
--start-publisher-script scripts/start-publisher.sh $(TLS_ARGS)
test-publisher-confirm-crash:
$(PYTHON) tests/publisher_confirm_crash.py --base-url "$(BASE_URL)" \
--stop-publisher-script scripts/stop-publisher.sh \
--crash-publisher-script scripts/crash-publisher-after-confirm.sh $(TLS_ARGS)
test-worker-crash:
$(PYTHON) tests/worker_crash.py --base-url "$(BASE_URL)" \
--stop-worker-script scripts/stop-worker.sh \
--crash-worker-script scripts/crash-worker.sh --mode objects $(TLS_ARGS)
$(PYTHON) tests/worker_crash.py --base-url "$(BASE_URL)" \
--stop-worker-script scripts/stop-worker.sh \
--crash-worker-script scripts/crash-worker.sh --mode commit $(TLS_ARGS)
test-failed-replay:
$(PYTHON) tests/failed_replay.py --base-url "$(BASE_URL)" \
--stop-worker-script scripts/stop-worker.sh \
--crash-worker-script scripts/crash-worker.sh \
--replay-script scripts/replay-attachment.sh $(TLS_ARGS)
test-worker-outage:
$(PYTHON) tests/worker_outage.py --base-url "$(BASE_URL)" \
--stop-worker-script scripts/stop-worker.sh \
--restart-api-script scripts/restart-api.sh \
--start-worker-script scripts/start-worker.sh $(TLS_ARGS)
test-broker-persistence:
$(PYTHON) tests/broker_persistence.py --base-url "$(BASE_URL)" \
--stop-worker-script scripts/stop-worker.sh \
--restart-broker-script scripts/restart-broker.sh \
--start-worker-script scripts/start-worker.sh $(TLS_ARGS)
test-docker:
docker compose -f compose.tests.yaml run --rm tests
load:
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
@@ -28,3 +76,10 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
replay-failed:
@test -n "$(JOB_ID)" || (echo "Set JOB_ID"; exit 2)
docker compose --env-file .env run --rm publisher python -m app.tools.replay_failed --job-id "$(JOB_ID)"
cleanup-staging:
docker compose --env-file .env run --rm worker python -m app.tools.cleanup_staging --dry-run
@@ -0,0 +1,126 @@
"""Add asynchronous attachment jobs, variants and transactional outbox.
Revision ID: 20261116_0004
Revises: 20261030_0003
Create Date: 2026-11-16
"""
from alembic import op
import sqlalchemy as sa
revision = '20261116_0004'
down_revision = '20261030_0003'
branch_labels = None
depends_on = None
def upgrade() -> None:
op.alter_column('attachments', 'object_key', new_column_name='original_key')
op.alter_column('attachments', 'original_key', nullable=True)
op.add_column('attachments', sa.Column('staging_key', sa.String(length=255), nullable=True))
op.add_column(
'attachments',
sa.Column('pipeline_version', sa.Integer(), nullable=False, server_default='1'),
)
op.create_unique_constraint('attachments_staging_key_key', 'attachments', ['staging_key'])
op.create_table(
'attachment_variants',
sa.Column('attachment_id', sa.Uuid(), nullable=False),
sa.Column('pipeline_version', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=16), nullable=False),
sa.Column('object_key', sa.String(length=255), nullable=False),
sa.Column('content_type', sa.String(length=32), nullable=False),
sa.Column('size_bytes', sa.BigInteger(), nullable=False),
sa.Column('width', sa.Integer(), nullable=False),
sa.Column('height', sa.Integer(), nullable=False),
sa.Column('sha256', sa.String(length=64), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint("name IN ('thumbnail', 'crop', 'optimized')", name='variant_name_check'),
sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('attachment_id', 'pipeline_version', 'name'),
sa.UniqueConstraint('object_key', name='attachment_variants_object_key_key'),
)
op.create_table(
'attachment_jobs',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('attachment_id', sa.Uuid(), nullable=False),
sa.Column('pipeline_version', sa.Integer(), nullable=False),
sa.Column('state', sa.String(length=16), nullable=False),
sa.Column('attempt', sa.Integer(), nullable=False),
sa.Column('max_attempts', sa.Integer(), nullable=False),
sa.Column('next_attempt_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('lease_owner', sa.String(length=128), nullable=True),
sa.Column('lease_until', sa.DateTime(timezone=True), nullable=True),
sa.Column('last_error_code', sa.String(length=64), nullable=True),
sa.Column('replay_count', sa.Integer(), nullable=False, server_default='0'),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint(
"state IN ('queued', 'processing', 'ready', 'failed')",
name='attachment_job_state_check',
),
sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint(
'attachment_id', 'pipeline_version', name='attachment_job_attachment_version_key'
),
)
op.create_index(
'attachment_jobs_lease_idx',
'attachment_jobs',
['state', 'lease_until'],
)
op.create_table(
'outbox_events',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('event_type', sa.String(length=64), nullable=False),
sa.Column('job_id', sa.Uuid(), nullable=False),
sa.Column('payload', sa.JSON(), nullable=False),
sa.Column('available_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('published_at', sa.DateTime(timezone=True), nullable=True),
sa.Column('claim_owner', sa.String(length=128), nullable=True),
sa.Column('claim_until', sa.DateTime(timezone=True), nullable=True),
sa.Column('publish_attempts', sa.Integer(), nullable=False, server_default='0'),
sa.Column('last_error_code', sa.String(length=64), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(['job_id'], ['attachment_jobs.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id'),
)
op.create_index(
'outbox_events_pending_idx',
'outbox_events',
['published_at', 'available_at'],
)
op.create_table(
'job_events',
sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False),
sa.Column('job_id', sa.Uuid(), nullable=False),
sa.Column('kind', sa.String(length=32), nullable=False),
sa.Column('attempt', sa.Integer(), nullable=False),
sa.Column('error_code', sa.String(length=64), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(['job_id'], ['attachment_jobs.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id'),
)
op.create_index('job_events_job_created_idx', 'job_events', ['job_id', 'created_at'])
def downgrade() -> None:
op.drop_index('job_events_job_created_idx', table_name='job_events')
op.drop_table('job_events')
op.drop_index('outbox_events_pending_idx', table_name='outbox_events')
op.drop_table('outbox_events')
op.drop_index('attachment_jobs_lease_idx', table_name='attachment_jobs')
op.drop_table('attachment_jobs')
op.drop_table('attachment_variants')
op.drop_constraint('attachments_staging_key_key', 'attachments', type_='unique')
op.drop_column('attachments', 'pipeline_version')
op.drop_column('attachments', 'staging_key')
op.alter_column('attachments', 'original_key', nullable=False)
op.alter_column('attachments', 'original_key', new_column_name='object_key')
+6 -8
View File
@@ -14,7 +14,7 @@ router = APIRouter(tags=['attachments'])
@router.post(
'/api/v1/chats/{chat_id}/attachments',
status_code=201,
status_code=202,
response_model=AttachmentOut,
)
async def upload(
@@ -51,16 +51,14 @@ async def content(
actor: User = Depends(current_user),
attachments: AttachmentService = Depends(attachment_service),
) -> StreamingResponse:
if variant != 'original':
from ...domain.errors import NotFound
raise NotFound('вариант вложения не найден')
attachment, stream = await attachments.content(actor, attachment_id)
attachment, stream, content_type, size_bytes = await attachments.content(
actor, attachment_id, variant
)
return StreamingResponse(
stream.chunks(),
media_type=attachment.content_type,
media_type=content_type,
headers={
'Content-Length': str(attachment.size_bytes),
'Content-Length': str(size_bytes),
'Content-Disposition': f'inline; filename="attachment-{attachment.id}"',
},
)
+21 -2
View File
@@ -134,6 +134,15 @@ class MessageOut(Strict):
)
class VariantOut(Strict):
name: str
content_type: str
size_bytes: int
width: int
height: int
sha256: str
class AttachmentOut(Strict):
id: UUID
chat_id: UUID
@@ -143,7 +152,7 @@ class AttachmentOut(Strict):
size_bytes: int
sha256: str
status: str
variants: list = Field(default_factory=list)
variants: list[VariantOut] = Field(default_factory=list)
error_code: str | None
created_at: str
updated_at: str
@@ -159,7 +168,17 @@ class AttachmentOut(Strict):
size_bytes=attachment.size_bytes,
sha256=attachment.sha256,
status=attachment.status,
variants=[],
variants=[
VariantOut(
name=item.name,
content_type=item.content_type,
size_bytes=item.size_bytes,
width=item.width,
height=item.height,
sha256=item.sha256,
)
for item in attachment.variants
],
error_code=attachment.error_code,
created_at=to_rfc3339(attachment.created_at),
updated_at=to_rfc3339(attachment.updated_at),
+31
View File
@@ -27,6 +27,18 @@ class Settings:
s3_secret_access_key: str
max_upload_bytes: int
max_image_pixels: int
broker_url: str
worker_concurrency: int
worker_prefetch: int
job_max_attempts: int
job_timeout_seconds: int
job_lease_seconds: int
retry_base_seconds: int
retry_max_seconds: int
staging_retention_seconds: int
thumbnail_jpeg_quality: int
crop_jpeg_quality: int
optimized_webp_quality: int
def _positive_int(name: str, default: str) -> int:
@@ -39,6 +51,13 @@ def _positive_int(name: str, default: str) -> int:
return value
def _quality(name: str, default: str) -> int:
value = _positive_int(name, default)
if value > 100:
raise RuntimeError(f'{name} должен быть не больше 100')
return value
def load_settings() -> Settings:
return Settings(
database_url=_read_value('DATABASE_URL', 'DATABASE_URL_FILE'),
@@ -51,4 +70,16 @@ def load_settings() -> Settings:
s3_secret_access_key=_read_value('S3_SECRET_ACCESS_KEY', 'S3_SECRET_ACCESS_KEY_FILE'),
max_upload_bytes=_positive_int('MAX_UPLOAD_BYTES', '10485760'),
max_image_pixels=_positive_int('MAX_IMAGE_PIXELS', '40000000'),
broker_url=_read_value('BROKER_URL', 'BROKER_URL_FILE'),
worker_concurrency=_positive_int('WORKER_CONCURRENCY', '2'),
worker_prefetch=_positive_int('WORKER_PREFETCH', '2'),
job_max_attempts=_positive_int('JOB_MAX_ATTEMPTS', '5'),
job_timeout_seconds=_positive_int('JOB_TIMEOUT_SECONDS', '60'),
job_lease_seconds=_positive_int('JOB_LEASE_SECONDS', '90'),
retry_base_seconds=_positive_int('RETRY_BASE_SECONDS', '2'),
retry_max_seconds=_positive_int('RETRY_MAX_SECONDS', '60'),
staging_retention_seconds=_positive_int('STAGING_RETENTION_SECONDS', '86400'),
thumbnail_jpeg_quality=_quality('THUMBNAIL_JPEG_QUALITY', '85'),
crop_jpeg_quality=_quality('CROP_JPEG_QUALITY', '85'),
optimized_webp_quality=_quality('OPTIMIZED_WEBP_QUALITY', '82'),
)
+40 -1
View File
@@ -39,7 +39,9 @@ class Attachment:
chat_id: UUID
owner_id: UUID
filename: str
object_key: str
original_key: str | None
staging_key: str | None
pipeline_version: int
content_type: str
size_bytes: int
sha256: str
@@ -49,6 +51,43 @@ class Attachment:
error_code: str | None
created_at: datetime
updated_at: datetime
variants: tuple['AttachmentVariant', ...] = ()
@dataclass(frozen=True, slots=True)
class AttachmentVariant:
name: str
object_key: str
content_type: str
size_bytes: int
width: int
height: int
sha256: str
@dataclass(frozen=True, slots=True)
class AttachmentJob:
id: UUID
attachment_id: UUID
pipeline_version: int
state: str
attempt: int
max_attempts: int
next_attempt_at: datetime
lease_owner: str | None
lease_until: datetime | None
last_error_code: str | None
replay_count: int
created_at: datetime
updated_at: datetime
@dataclass(frozen=True, slots=True)
class OutboxEvent:
id: UUID
job_id: UUID
payload: dict
available_at: datetime
@dataclass(frozen=True, slots=True)
+1
View File
@@ -0,0 +1 @@
"""Durable attachment job publisher and worker."""
+56
View File
@@ -0,0 +1,56 @@
from __future__ import annotations
import asyncio
import logging
import aio_pika
from aio_pika import DeliveryMode, ExchangeType
MAIN_EXCHANGE = 'attachments'
MAIN_QUEUE = 'attachments.process'
ROUTING_KEY = 'attachment.process.v1'
DEAD_EXCHANGE = 'attachments.dead'
DEAD_QUEUE = 'attachments.failed'
logger = logging.getLogger('app.broker')
async def connect(url: str) -> aio_pika.RobustConnection:
delay = 0.5
while True:
try:
return await aio_pika.connect_robust(url, timeout=5)
except (OSError, aio_pika.exceptions.AMQPException):
logger.warning('broker is not ready; reconnecting in %.1fs', delay)
await asyncio.sleep(delay)
delay = min(delay * 2, 5.0)
async def topology(channel: aio_pika.abc.AbstractChannel):
dead_exchange = await channel.declare_exchange(
DEAD_EXCHANGE, ExchangeType.DIRECT, durable=True
)
dead_queue = await channel.declare_queue(DEAD_QUEUE, durable=True)
await dead_queue.bind(dead_exchange, ROUTING_KEY)
exchange = await channel.declare_exchange(
MAIN_EXCHANGE, ExchangeType.DIRECT, durable=True
)
queue = await channel.declare_queue(
MAIN_QUEUE,
durable=True,
arguments={
'x-dead-letter-exchange': DEAD_EXCHANGE,
'x-dead-letter-routing-key': ROUTING_KEY,
},
)
await queue.bind(exchange, ROUTING_KEY)
return exchange, queue
def message(body: bytes, event_id: str) -> aio_pika.Message:
return aio_pika.Message(
body=body,
content_type='application/json',
delivery_mode=DeliveryMode.PERSISTENT,
message_id=event_id,
type=ROUTING_KEY,
)
+98
View File
@@ -0,0 +1,98 @@
from __future__ import annotations
import hashlib
from dataclasses import dataclass
from io import BytesIO
from PIL import Image, ImageOps, UnidentifiedImageError
class PermanentImageError(Exception):
def __init__(self, code: str) -> None:
super().__init__(code)
self.code = code
@dataclass(frozen=True, slots=True)
class GeneratedVariant:
name: str
content_type: str
width: int
height: int
sha256: str
content: bytes
def _jpeg_image(image: Image.Image) -> Image.Image:
if image.mode in {'RGBA', 'LA'} or 'transparency' in image.info:
rgba = image.convert('RGBA')
background = Image.new('RGBA', rgba.size, 'white')
background.alpha_composite(rgba)
return background.convert('RGB')
return image.convert('RGB')
def _encode(image: Image.Image, fmt: str, **options) -> bytes:
output = BytesIO()
image.save(output, format=fmt, **options)
return output.getvalue()
def _variant(name: str, content_type: str, image: Image.Image, content: bytes) -> GeneratedVariant:
return GeneratedVariant(
name=name,
content_type=content_type,
width=image.width,
height=image.height,
sha256=hashlib.sha256(content).hexdigest(),
content=content,
)
def process_image(
source: bytes,
declared_type: str,
expected_sha256: str,
max_pixels: int,
thumbnail_quality: int,
crop_quality: int,
webp_quality: int,
) -> tuple[GeneratedVariant, ...]:
if hashlib.sha256(source).hexdigest() != expected_sha256:
raise PermanentImageError('checksum_mismatch')
try:
with Image.open(BytesIO(source)) as opened:
actual = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}.get(opened.format or '')
if actual != declared_type:
raise PermanentImageError('unsupported_media_type')
if opened.width * opened.height > max_pixels:
raise PermanentImageError('image_too_large')
opened.load()
oriented = ImageOps.exif_transpose(opened)
base = oriented.copy()
except PermanentImageError:
raise
except (Image.DecompressionBombError, UnidentifiedImageError, OSError, SyntaxError) as error:
raise PermanentImageError('invalid_image') from error
thumbnail = _jpeg_image(base.copy())
thumbnail.thumbnail((256, 256), Image.Resampling.LANCZOS, reducing_gap=3.0)
thumbnail_bytes = _encode(
thumbnail, 'JPEG', quality=thumbnail_quality, optimize=True, progressive=True
)
crop_source = _jpeg_image(base.copy())
crop = ImageOps.fit(
crop_source, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)
)
crop_bytes = _encode(crop, 'JPEG', quality=crop_quality, optimize=True, progressive=True)
optimized = base.copy()
optimized.thumbnail((1280, 1280), Image.Resampling.LANCZOS, reducing_gap=3.0)
optimized_bytes = _encode(optimized, 'WEBP', quality=webp_quality, method=6)
return (
_variant('thumbnail', 'image/jpeg', thumbnail, thumbnail_bytes),
_variant('crop', 'image/jpeg', crop, crop_bytes),
_variant('optimized', 'image/webp', optimized, optimized_bytes),
)
+64
View File
@@ -0,0 +1,64 @@
from __future__ import annotations
import argparse
import asyncio
import json
import logging
import os
import socket
from datetime import datetime, timezone
from ..config import load_settings
from ..repositories import JobRepository, OutboxRepository
from .broker import ROUTING_KEY, connect, message, topology
from .runtime import configure_logging, dependencies
logger = logging.getLogger('app.publisher')
async def run(once: bool = False) -> None:
settings = load_settings()
engine, sessions, _storage = dependencies(settings)
outbox = OutboxRepository(sessions)
jobs = JobRepository(sessions)
owner = f'{socket.gethostname()}:{os.getpid()}'
connection = await connect(settings.broker_url)
try:
channel = await connection.channel(publisher_confirms=True)
exchange, _queue = await topology(channel)
while True:
now = datetime.now(timezone.utc)
await jobs.recover_expired(now)
events = await outbox.claim(owner, now, lease_seconds=30)
for event in events:
try:
body = json.dumps(event.payload, separators=(',', ':')).encode()
await exchange.publish(
message(body, str(event.id)), routing_key=ROUTING_KEY, mandatory=True
)
if os.getenv('FAULT_PUBLISHER_AFTER_CONFIRM') == '1':
os._exit(91)
await outbox.published(event.id, owner, datetime.now(timezone.utc))
except Exception:
logger.exception('outbox publish failed event_id=%s', event.id)
await outbox.failed(
event.id, owner, 'broker_unavailable', datetime.now(timezone.utc)
)
if once:
return
await asyncio.sleep(0.5 if events else 1.0)
finally:
await connection.close()
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument('--once', action='store_true')
args = parser.parse_args()
configure_logging()
asyncio.run(run(args.once))
if __name__ == '__main__':
main()
+30
View File
@@ -0,0 +1,30 @@
from __future__ import annotations
import logging
import sys
from ..config import Settings
from ..db import create_engine, create_session_factory
from ..storage import S3Storage
def configure_logging() -> None:
logging.basicConfig(
level=logging.INFO,
stream=sys.stdout,
format='%(asctime)s %(levelname)s %(name)s %(message)s',
force=True,
)
def dependencies(settings: Settings):
engine = create_engine(settings.database_url)
sessions = create_session_factory(engine)
storage = S3Storage(
endpoint=settings.s3_endpoint,
access_key=settings.s3_access_key_id,
secret_key=settings.s3_secret_access_key,
bucket=settings.s3_bucket,
region=settings.s3_region,
)
return engine, sessions, storage
+274
View File
@@ -0,0 +1,274 @@
from __future__ import annotations
import asyncio
import json
import logging
import multiprocessing
import os
import random
import socket
from datetime import datetime, timedelta, timezone
from uuid import UUID
import aio_pika
from ..config import Settings, load_settings
from ..domain.errors import Unavailable
from ..domain.models import AttachmentVariant
from ..repositories import JobRepository
from .broker import connect, topology
from .pipeline import GeneratedVariant, PermanentImageError, process_image
from .runtime import configure_logging, dependencies
logger = logging.getLogger('app.worker')
class ProcessingTimeout(Exception):
pass
def _pipeline_child(connection, arguments: tuple, memory_bytes: int) -> None:
try:
if memory_bytes:
try:
import resource
resource.setrlimit(resource.RLIMIT_AS, (memory_bytes, memory_bytes))
except (ImportError, OSError, ValueError):
pass
connection.send(('ok', process_image(*arguments)))
except PermanentImageError as error:
connection.send(('permanent', error.code))
except MemoryError:
connection.send(('temporary', 'processing_memory_limit'))
except BaseException:
connection.send(('temporary', 'processing_failed'))
finally:
connection.close()
def _run_isolated(arguments: tuple, timeout: int, memory_bytes: int) -> tuple[GeneratedVariant, ...]:
context = multiprocessing.get_context('spawn')
receiver, sender = context.Pipe(duplex=False)
process = context.Process(target=_pipeline_child, args=(sender, arguments, memory_bytes))
process.start()
sender.close()
process.join(timeout)
if process.is_alive():
process.terminate()
process.join(5)
raise ProcessingTimeout('processing_timeout')
if not receiver.poll():
raise ProcessingTimeout('processing_failed')
kind, result = receiver.recv()
receiver.close()
if kind == 'ok':
return result
if kind == 'permanent':
raise PermanentImageError(result)
raise ProcessingTimeout(result)
class Worker:
def __init__(self, settings: Settings, jobs: JobRepository, storage) -> None:
self.settings = settings
self.jobs = jobs
self.storage = storage
self.worker_id = f'{socket.gethostname()}:{os.getpid()}'
self.semaphore = asyncio.Semaphore(settings.worker_concurrency)
def _delay(self, attempt: int) -> float:
base = min(
self.settings.retry_max_seconds,
self.settings.retry_base_seconds * (2 ** max(0, attempt - 1)),
)
return base + random.uniform(0, base)
async def _handle_failure(
self,
incoming: aio_pika.IncomingMessage,
job_id: UUID,
attempt: int,
code: str,
permanent: bool,
) -> None:
now = datetime.now(timezone.utc)
available_at = now + timedelta(seconds=self._delay(attempt))
try:
state = await self.jobs.retry_or_fail(
job_id,
self.worker_id,
code,
permanent,
available_at,
now,
)
except Exception:
logger.exception('failed to persist job failure job_id=%s', job_id)
await incoming.nack(requeue=True)
return
if state == 'failed':
await incoming.reject(requeue=False)
else:
await incoming.ack()
async def handle(self, incoming: aio_pika.IncomingMessage) -> None:
async with self.semaphore:
try:
payload = json.loads(incoming.body)
if payload.get('schema_version') != 1 or payload.get('type') != 'attachment.process.v1':
await incoming.reject(requeue=False)
return
job_id = UUID(payload['job_id'])
attachment_id = UUID(payload['attachment_id'])
attempt = int(payload['attempt'])
pipeline_version = int(payload['pipeline_version'])
except (ValueError, TypeError, KeyError, json.JSONDecodeError):
await incoming.reject(requeue=False)
return
current = await self.jobs.state(job_id)
if current is None:
await incoming.reject(requeue=False)
return
if current.attachment_id != attachment_id or current.pipeline_version != pipeline_version:
await incoming.reject(requeue=False)
return
if current.state == 'ready':
if await self.jobs.ready_effect_complete(job_id):
await incoming.ack()
else:
await incoming.reject(requeue=False)
return
if current.state == 'failed':
await incoming.reject(requeue=False)
return
now = datetime.now(timezone.utc)
claimed = await self.jobs.claim(
job_id, attempt, self.worker_id, now, self.settings.job_lease_seconds
)
if claimed is None:
await incoming.ack()
return
job, attachment = claimed
if os.getenv('FAULT_WORKER_PERMANENT') == '1':
await self._handle_failure(
incoming, job.id, job.attempt, 'fault_injected', True
)
os._exit(94)
if not attachment.staging_key:
await self._handle_failure(
incoming, job.id, job.attempt, 'staging_missing', True
)
return
try:
source = await self.storage.read(
attachment.staging_key, self.settings.max_upload_bytes
)
arguments = (
source,
attachment.content_type,
attachment.sha256,
self.settings.max_image_pixels,
self.settings.thumbnail_jpeg_quality,
self.settings.crop_jpeg_quality,
self.settings.optimized_webp_quality,
)
generated = await asyncio.to_thread(
_run_isolated,
arguments,
self.settings.job_timeout_seconds,
512 * 1024 * 1024,
)
original_key = f'originals/{attachment.id}/v{job.pipeline_version}/original'
await self.storage.put_bytes(original_key, source, attachment.content_type)
variants: list[AttachmentVariant] = []
suffixes = {'thumbnail': 'jpg', 'crop': 'jpg', 'optimized': 'webp'}
for item in generated:
key = (
f'variants/{attachment.id}/v{job.pipeline_version}/'
f'{item.name}.{suffixes[item.name]}'
)
await self.storage.put_bytes(key, item.content, item.content_type)
variants.append(
AttachmentVariant(
name=item.name,
object_key=key,
content_type=item.content_type,
size_bytes=len(item.content),
width=item.width,
height=item.height,
sha256=item.sha256,
)
)
if os.getenv('FAULT_WORKER_AFTER_OBJECTS') == '1':
os._exit(92)
completed = await self.jobs.complete(
job.id,
self.worker_id,
original_key,
tuple(variants),
datetime.now(timezone.utc),
)
if not completed:
await incoming.nack(requeue=True)
return
if os.getenv('FAULT_WORKER_AFTER_COMMIT') == '1':
os._exit(93)
try:
await self.storage.remove(attachment.staging_key)
except Unavailable:
logger.warning('staging cleanup deferred attachment_id=%s', attachment.id)
await incoming.ack()
except PermanentImageError as error:
await self._handle_failure(
incoming, job.id, job.attempt, error.code, True
)
except ProcessingTimeout as error:
await self._handle_failure(
incoming, job.id, job.attempt, str(error), False
)
except (Unavailable, OSError, ValueError):
logger.exception('temporary processing failure job_id=%s', job.id)
await self._handle_failure(
incoming, job.id, job.attempt, 'storage_unavailable', False
)
except Exception:
logger.exception('unexpected processing failure job_id=%s', job.id)
await self._handle_failure(
incoming, job.id, job.attempt, 'processing_failed', False
)
async def run() -> None:
settings = load_settings()
engine, sessions, storage = dependencies(settings)
jobs = JobRepository(sessions)
worker = Worker(settings, jobs, storage)
connection = await connect(settings.broker_url)
try:
channel = await connection.channel()
await channel.set_qos(prefetch_count=settings.worker_prefetch)
_exchange, queue = await topology(channel)
await queue.consume(worker.handle, no_ack=False)
logger.info(
'worker started id=%s concurrency=%d prefetch=%d',
worker.worker_id,
settings.worker_concurrency,
settings.worker_prefetch,
)
await asyncio.Future()
finally:
await connection.close()
await engine.dispose()
def main() -> None:
configure_logging()
asyncio.run(run())
if __name__ == '__main__':
main()
+5 -4
View File
@@ -24,7 +24,7 @@ from .storage import S3Storage
logger = logging.getLogger('app')
DESCRIPTION = 'PostgreSQL хранит метаданные, Redis — сессии, приватный S3 — оригиналы вложений.'
DESCRIPTION = 'PostgreSQL хранит задания, RabbitMQ доставляет их worker, S3 хранит изображения.'
def configure_logging() -> None:
@@ -69,6 +69,7 @@ def create_app() -> FastAPI:
storage,
settings.max_upload_bytes,
settings.max_image_pixels,
settings.job_max_attempts,
),
users=UserService(user_repository),
chats=ChatService(chat_repository),
@@ -79,7 +80,7 @@ def create_app() -> FastAPI:
application.state.redis = redis
application.state.storage = storage
application.state.ready = True
logger.info('startup storage=postgres+s3 redis_sessions=true')
logger.info('startup storage=postgres+s3 redis_sessions=true async_attachments=true')
yield
finally:
application.state.ready = False
@@ -88,8 +89,8 @@ def create_app() -> FastAPI:
logger.info('shutdown complete')
api = FastAPI(
title='MTUSI Messenger. LR 4',
version='1.4.0',
title='MTUSI Messenger. LR 5',
version='1.5.0',
description=DESCRIPTION,
lifespan=lifespan,
)
+6 -1
View File
@@ -1,13 +1,18 @@
from .attachment import AttachmentRow, MessageAttachmentRow
from .attachment import AttachmentRow, AttachmentVariantRow, MessageAttachmentRow
from .chat import ChatMemberRow, ChatRow
from .message import MessageRow
from .user import UserRow
from .job import AttachmentJobRow, JobEventRow, OutboxEventRow
__all__ = [
'AttachmentRow',
'AttachmentVariantRow',
'AttachmentJobRow',
'ChatMemberRow',
'ChatRow',
'MessageAttachmentRow',
'MessageRow',
'UserRow',
'JobEventRow',
'OutboxEventRow',
]
+23 -2
View File
@@ -14,14 +14,17 @@ class AttachmentRow(Base):
__tablename__ = 'attachments'
__table_args__ = (
Index('attachments_chat_created_idx', 'chat_id', 'created_at', 'id'),
UniqueConstraint('object_key', name='attachments_object_key_key'),
UniqueConstraint('original_key', name='attachments_object_key_key'),
UniqueConstraint('staging_key', name='attachments_staging_key_key'),
)
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), nullable=False)
owner_id: Mapped[UUID] = mapped_column(ForeignKey('users.id'), nullable=False)
filename: Mapped[str] = mapped_column(String(255), nullable=False)
object_key: Mapped[str] = mapped_column(String(255), nullable=False)
original_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
staging_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
pipeline_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
content_type: Mapped[str] = mapped_column(String(32), nullable=False)
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
sha256: Mapped[str] = mapped_column(String(64), nullable=False)
@@ -33,6 +36,24 @@ class AttachmentRow(Base):
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class AttachmentVariantRow(Base):
__tablename__ = 'attachment_variants'
attachment_id: Mapped[UUID] = mapped_column(
ForeignKey('attachments.id', ondelete='CASCADE'), primary_key=True
)
pipeline_version: Mapped[int] = mapped_column(Integer, primary_key=True)
name: Mapped[str] = mapped_column(String(16), primary_key=True)
object_key: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
content_type: Mapped[str] = mapped_column(String(32), nullable=False)
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
width: Mapped[int] = mapped_column(Integer, nullable=False)
height: Mapped[int] = mapped_column(Integer, nullable=False)
sha256: Mapped[str] = mapped_column(String(64), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class MessageAttachmentRow(Base):
__tablename__ = 'message_attachments'
__table_args__ = (
+70
View File
@@ -0,0 +1,70 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer, JSON, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.types import Uuid
from ..db.postgres import Base
class AttachmentJobRow(Base):
__tablename__ = 'attachment_jobs'
__table_args__ = (
UniqueConstraint(
'attachment_id', 'pipeline_version', name='attachment_job_attachment_version_key'
),
Index('attachment_jobs_lease_idx', 'state', 'lease_until'),
)
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
attachment_id: Mapped[UUID] = mapped_column(
ForeignKey('attachments.id', ondelete='CASCADE'), nullable=False
)
pipeline_version: Mapped[int] = mapped_column(Integer, nullable=False)
state: Mapped[str] = mapped_column(String(16), nullable=False)
attempt: Mapped[int] = mapped_column(Integer, nullable=False)
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False)
next_attempt_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True)
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
replay_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class OutboxEventRow(Base):
__tablename__ = 'outbox_events'
__table_args__ = (Index('outbox_events_pending_idx', 'published_at', 'available_at'),)
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
event_type: Mapped[str] = mapped_column(String(64), nullable=False)
job_id: Mapped[UUID] = mapped_column(
ForeignKey('attachment_jobs.id', ondelete='CASCADE'), nullable=False
)
payload: Mapped[dict] = mapped_column(JSON, nullable=False)
available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
claim_owner: Mapped[str | None] = mapped_column(String(128), nullable=True)
claim_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
publish_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class JobEventRow(Base):
__tablename__ = 'job_events'
__table_args__ = (Index('job_events_job_created_idx', 'job_id', 'created_at'),)
id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True)
job_id: Mapped[UUID] = mapped_column(
ForeignKey('attachment_jobs.id', ondelete='CASCADE'), nullable=False
)
kind: Mapped[str] = mapped_column(String(32), nullable=False)
attempt: Mapped[int] = mapped_column(Integer, nullable=False)
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+5 -1
View File
@@ -2,5 +2,9 @@ from .attachments import AttachmentRepository
from .chats import ChatRepository
from .messages import MessageRepository
from .users import UserRepository
from .jobs import JobRepository, OutboxRepository
__all__ = ['AttachmentRepository', 'ChatRepository', 'MessageRepository', 'UserRepository']
__all__ = [
'AttachmentRepository', 'ChatRepository', 'JobRepository', 'MessageRepository',
'OutboxRepository', 'UserRepository',
]
+91 -11
View File
@@ -1,14 +1,27 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from uuid import UUID, uuid4
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ..domain.models import Attachment
from ..models.attachment import AttachmentRow
from ..domain.models import Attachment, AttachmentVariant
from ..models.attachment import AttachmentRow, AttachmentVariantRow
from ..models.chat import ChatMemberRow, ChatRow
from ..models.job import AttachmentJobRow, OutboxEventRow
def job_payload(job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int, now: datetime) -> dict:
return {
'schema_version': 1,
'job_id': str(job_id),
'type': 'attachment.process.v1',
'attachment_id': str(attachment_id),
'pipeline_version': pipeline_version,
'attempt': attempt,
'created_at': now.isoformat().replace('+00:00', 'Z'),
}
class AttachmentRepository:
@@ -16,13 +29,29 @@ class AttachmentRepository:
self._sessions = sessions
@staticmethod
def _to_domain(row: AttachmentRow) -> Attachment:
def _variant(row: AttachmentVariantRow) -> AttachmentVariant:
return AttachmentVariant(
name=row.name,
object_key=row.object_key,
content_type=row.content_type,
size_bytes=row.size_bytes,
width=row.width,
height=row.height,
sha256=row.sha256,
)
@classmethod
def _to_domain(
cls, row: AttachmentRow, variants: tuple[AttachmentVariantRow, ...] = ()
) -> Attachment:
return Attachment(
id=row.id,
chat_id=row.chat_id,
owner_id=row.owner_id,
filename=row.filename,
object_key=row.object_key,
original_key=row.original_key,
staging_key=row.staging_key,
pipeline_version=row.pipeline_version,
content_type=row.content_type,
size_bytes=row.size_bytes,
sha256=row.sha256,
@@ -32,6 +61,7 @@ class AttachmentRepository:
error_code=row.error_code,
created_at=row.created_at,
updated_at=row.updated_at,
variants=tuple(cls._variant(item) for item in variants),
)
async def chat_exists_for_member(self, chat_id: UUID, user_id: UUID) -> bool:
@@ -43,19 +73,21 @@ class AttachmentRepository:
)
return found is not None
async def create(
async def create_queued(
self,
*,
attachment_id: UUID,
job_id: UUID,
chat_id: UUID,
owner_id: UUID,
filename: str,
object_key: str,
staging_key: str,
content_type: str,
size_bytes: int,
sha256: str,
width: int,
height: int,
max_attempts: int,
now: datetime,
) -> Attachment:
row = AttachmentRow(
@@ -63,23 +95,67 @@ class AttachmentRepository:
chat_id=chat_id,
owner_id=owner_id,
filename=filename,
object_key=object_key,
original_key=None,
staging_key=staging_key,
pipeline_version=1,
content_type=content_type,
size_bytes=size_bytes,
sha256=sha256,
width=width,
height=height,
status='ready',
status='queued',
error_code=None,
created_at=now,
updated_at=now,
)
job = AttachmentJobRow(
id=job_id,
attachment_id=attachment_id,
pipeline_version=1,
state='queued',
attempt=1,
max_attempts=max_attempts,
next_attempt_at=now,
lease_owner=None,
lease_until=None,
last_error_code=None,
replay_count=0,
created_at=now,
updated_at=now,
)
outbox = OutboxEventRow(
id=uuid4(),
event_type='attachment.process.v1',
job_id=job_id,
payload=job_payload(job_id, attachment_id, 1, 1, now),
available_at=now,
published_at=None,
claim_owner=None,
claim_until=None,
publish_attempts=0,
last_error_code=None,
created_at=now,
updated_at=now,
)
async with self._sessions() as session:
async with session.begin():
session.add(row)
session.add_all((row, job, outbox))
await session.flush()
return self._to_domain(row)
async def _variants(
self, session: AsyncSession, attachment_id: UUID, pipeline_version: int
) -> tuple[AttachmentVariantRow, ...]:
rows = await session.scalars(
select(AttachmentVariantRow)
.where(
AttachmentVariantRow.attachment_id == attachment_id,
AttachmentVariantRow.pipeline_version == pipeline_version,
)
.order_by(AttachmentVariantRow.name)
)
return tuple(rows)
async def get_for_member(self, attachment_id: UUID, user_id: UUID) -> Attachment | None:
async with self._sessions() as session:
row = await session.scalar(
@@ -90,4 +166,8 @@ class AttachmentRepository:
ChatMemberRow.user_id == user_id,
)
)
return self._to_domain(row) if row is not None else None
if row is None:
return None
return self._to_domain(
row, await self._variants(session, row.id, row.pipeline_version)
)
+358
View File
@@ -0,0 +1,358 @@
from __future__ import annotations
from datetime import datetime, timedelta
from uuid import UUID, uuid4
from sqlalchemy import or_, select, update
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ..domain.models import Attachment, AttachmentJob, AttachmentVariant, OutboxEvent
from ..models.attachment import AttachmentRow, AttachmentVariantRow
from ..models.job import AttachmentJobRow, JobEventRow, OutboxEventRow
from .attachments import AttachmentRepository, job_payload
class JobRepository:
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
self._sessions = sessions
@staticmethod
def _job(row: AttachmentJobRow) -> AttachmentJob:
return AttachmentJob(
id=row.id,
attachment_id=row.attachment_id,
pipeline_version=row.pipeline_version,
state=row.state,
attempt=row.attempt,
max_attempts=row.max_attempts,
next_attempt_at=row.next_attempt_at,
lease_owner=row.lease_owner,
lease_until=row.lease_until,
last_error_code=row.last_error_code,
replay_count=row.replay_count,
created_at=row.created_at,
updated_at=row.updated_at,
)
async def claim(
self,
job_id: UUID,
attempt: int,
worker_id: str,
now: datetime,
lease_seconds: int,
) -> tuple[AttachmentJob, Attachment] | None:
async with self._sessions() as session:
async with session.begin():
row = await session.scalar(
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
)
if row is None or row.state != 'queued' or row.attempt != attempt:
return None
attachment_row = await session.get(AttachmentRow, row.attachment_id)
if attachment_row is None:
return None
row.state = 'processing'
row.lease_owner = worker_id
row.lease_until = now + timedelta(seconds=lease_seconds)
row.updated_at = now
attachment_row.status = 'processing'
attachment_row.updated_at = now
session.add(
JobEventRow(
job_id=row.id,
kind='claimed',
attempt=row.attempt,
error_code=None,
created_at=now,
)
)
await session.flush()
return self._job(row), AttachmentRepository._to_domain(attachment_row)
async def state(self, job_id: UUID) -> AttachmentJob | None:
async with self._sessions() as session:
row = await session.get(AttachmentJobRow, job_id)
return self._job(row) if row is not None else None
async def id_for_attachment(self, attachment_id: UUID) -> UUID | None:
async with self._sessions() as session:
return await session.scalar(
select(AttachmentJobRow.id).where(
AttachmentJobRow.attachment_id == attachment_id
)
)
async def ready_effect_complete(self, job_id: UUID) -> bool:
async with self._sessions() as session:
job = await session.get(AttachmentJobRow, job_id)
if job is None or job.state != 'ready':
return False
attachment = await session.get(AttachmentRow, job.attachment_id)
if attachment is None or not attachment.original_key:
return False
names = set(
await session.scalars(
select(AttachmentVariantRow.name).where(
AttachmentVariantRow.attachment_id == job.attachment_id,
AttachmentVariantRow.pipeline_version == job.pipeline_version,
)
)
)
return names == {'thumbnail', 'crop', 'optimized'}
async def complete(
self,
job_id: UUID,
worker_id: str,
original_key: str,
variants: tuple[AttachmentVariant, ...],
now: datetime,
) -> bool:
async with self._sessions() as session:
async with session.begin():
job = await session.scalar(
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
)
if job is None or job.state == 'ready':
return job is not None
if job.state != 'processing' or job.lease_owner != worker_id:
return False
attachment = await session.get(AttachmentRow, job.attachment_id)
if attachment is None:
return False
for variant in variants:
statement = (
insert(AttachmentVariantRow)
.values(
attachment_id=attachment.id,
pipeline_version=job.pipeline_version,
name=variant.name,
object_key=variant.object_key,
content_type=variant.content_type,
size_bytes=variant.size_bytes,
width=variant.width,
height=variant.height,
sha256=variant.sha256,
created_at=now,
updated_at=now,
)
.on_conflict_do_update(
index_elements=[
AttachmentVariantRow.attachment_id,
AttachmentVariantRow.pipeline_version,
AttachmentVariantRow.name,
],
set_={
'object_key': variant.object_key,
'content_type': variant.content_type,
'size_bytes': variant.size_bytes,
'width': variant.width,
'height': variant.height,
'sha256': variant.sha256,
'updated_at': now,
},
)
)
await session.execute(statement)
attachment.original_key = original_key
attachment.status = 'ready'
attachment.error_code = None
attachment.updated_at = now
job.state = 'ready'
job.lease_owner = None
job.lease_until = None
job.last_error_code = None
job.updated_at = now
session.add(
JobEventRow(
job_id=job.id,
kind='ready',
attempt=job.attempt,
error_code=None,
created_at=now,
)
)
return True
async def retry_or_fail(
self,
job_id: UUID,
worker_id: str | None,
error_code: str,
permanent: bool,
available_at: datetime,
now: datetime,
) -> str:
async with self._sessions() as session:
async with session.begin():
job = await session.scalar(
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
)
if job is None or job.state in {'ready', 'failed'}:
return job.state if job is not None else 'missing'
if job.state != 'processing':
return job.state
if worker_id is not None:
if job.lease_owner != worker_id:
return job.state
elif job.lease_until is None or job.lease_until >= now:
return job.state
attachment = await session.get(AttachmentRow, job.attachment_id)
exhausted = job.attempt >= job.max_attempts
if permanent or exhausted:
job.state = 'failed'
attachment.status = 'failed'
attachment.error_code = error_code if permanent else 'retry_exhausted'
kind = 'failed'
else:
job.attempt += 1
job.state = 'queued'
job.next_attempt_at = available_at
attachment.status = 'queued'
attachment.error_code = None
session.add(
OutboxEventRow(
id=uuid4(),
event_type='attachment.process.v1',
job_id=job.id,
payload=job_payload(
job.id,
job.attachment_id,
job.pipeline_version,
job.attempt,
now,
),
available_at=available_at,
published_at=None,
claim_owner=None,
claim_until=None,
publish_attempts=0,
last_error_code=None,
created_at=now,
updated_at=now,
)
)
kind = 'retry_scheduled'
job.last_error_code = error_code
job.lease_owner = None
job.lease_until = None
job.updated_at = now
attachment.updated_at = now
session.add(
JobEventRow(
job_id=job.id,
kind=kind,
attempt=job.attempt,
error_code=error_code,
created_at=now,
)
)
await session.flush()
return job.state
async def recover_expired(self, now: datetime) -> int:
async with self._sessions() as session:
ids = tuple(
await session.scalars(
select(AttachmentJobRow.id).where(
AttachmentJobRow.state == 'processing',
AttachmentJobRow.lease_until < now,
)
)
)
recovered = 0
for job_id in ids:
state = await self.retry_or_fail(
job_id, None, 'lease_expired', False, now, now
)
recovered += state in {'queued', 'failed'}
return recovered
async def replay(self, job_id: UUID, now: datetime, dry_run: bool = False) -> bool:
async with self._sessions() as session:
async with session.begin():
job = await session.scalar(
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
)
if job is None or job.state != 'failed':
return False
attachment = await session.get(AttachmentRow, job.attachment_id)
if attachment is None or not attachment.staging_key:
return False
if dry_run:
return True
job.state = 'queued'
job.attempt += 1
job.max_attempts = job.attempt + 4
job.replay_count += 1
job.last_error_code = None
job.next_attempt_at = now
job.updated_at = now
attachment.status = 'queued'
attachment.error_code = None
attachment.updated_at = now
session.add_all(
(
OutboxEventRow(
id=uuid4(), event_type='attachment.process.v1', job_id=job.id,
payload=job_payload(job.id, job.attachment_id, job.pipeline_version, job.attempt, now),
available_at=now, published_at=None, claim_owner=None, claim_until=None,
publish_attempts=0, last_error_code=None, created_at=now, updated_at=now,
),
JobEventRow(job_id=job.id, kind='replayed', attempt=job.attempt, error_code=None, created_at=now),
)
)
return True
class OutboxRepository:
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
self._sessions = sessions
async def claim(self, owner: str, now: datetime, lease_seconds: int, limit: int = 20) -> tuple[OutboxEvent, ...]:
async with self._sessions() as session:
async with session.begin():
rows = tuple(
await session.scalars(
select(OutboxEventRow)
.where(
OutboxEventRow.published_at.is_(None),
OutboxEventRow.available_at <= now,
or_(OutboxEventRow.claim_until.is_(None), OutboxEventRow.claim_until < now),
)
.order_by(OutboxEventRow.available_at, OutboxEventRow.created_at)
.limit(limit)
.with_for_update(skip_locked=True)
)
)
until = now + timedelta(seconds=lease_seconds)
for row in rows:
row.claim_owner = owner
row.claim_until = until
row.publish_attempts += 1
row.updated_at = now
return tuple(OutboxEvent(row.id, row.job_id, row.payload, row.available_at) for row in rows)
async def published(self, event_id: UUID, owner: str, now: datetime) -> None:
async with self._sessions() as session:
async with session.begin():
await session.execute(
update(OutboxEventRow)
.where(
OutboxEventRow.id == event_id,
OutboxEventRow.claim_owner == owner,
OutboxEventRow.published_at.is_(None),
)
.values(published_at=now, claim_owner=None, claim_until=None, last_error_code=None, updated_at=now)
)
async def failed(self, event_id: UUID, owner: str, code: str, now: datetime) -> None:
async with self._sessions() as session:
async with session.begin():
await session.execute(
update(OutboxEventRow)
.where(OutboxEventRow.id == event_id, OutboxEventRow.claim_owner == owner)
.values(claim_owner=None, claim_until=None, last_error_code=code, updated_at=now)
)
+2 -2
View File
@@ -77,8 +77,8 @@ class MessageRepository:
}
if set(found) != set(attachment_ids):
raise NotFound('вложение не найдено')
if any(status != 'ready' for status in found.values()):
raise Conflict('оригинал вложения недоступен')
if any(status == 'failed' for status in found.values()):
raise Conflict('неуспешное вложение нельзя прикрепить')
statement = (
insert(MessageRow)
+31 -11
View File
@@ -67,11 +67,13 @@ class AttachmentService:
storage: S3Storage,
max_upload_bytes: int,
max_image_pixels: int,
job_max_attempts: int,
) -> None:
self._repository = repository
self._storage = storage
self._max_upload_bytes = max_upload_bytes
self._max_image_pixels = max_image_pixels
self._job_max_attempts = job_max_attempts
async def upload(
self,
@@ -106,28 +108,31 @@ class AttachmentService:
self._max_image_pixels,
)
attachment_id = uuid4()
object_key = f'originals/{attachment_id}'
await self._storage.put(object_key, source, size, actual_type)
job_id = uuid4()
staging_key = f'staging/{attachment_id}/original'
await self._storage.put(staging_key, source, size, actual_type)
now = datetime.now(timezone.utc)
try:
return await self._repository.create(
return await self._repository.create_queued(
attachment_id=attachment_id,
job_id=job_id,
chat_id=chat_id,
owner_id=actor.id,
filename=filename,
object_key=object_key,
staging_key=staging_key,
content_type=actual_type,
size_bytes=size,
sha256=digest.hexdigest(),
width=width,
height=height,
max_attempts=self._job_max_attempts,
now=now,
)
except Exception:
try:
await self._storage.remove(object_key)
await self._storage.remove(staging_key)
except Unavailable:
logger.warning('failed to compensate orphan object key=%s', object_key)
logger.warning('failed to compensate staging object key=%s', staging_key)
raise
async def get(self, actor: User, attachment_id: UUID) -> Attachment:
@@ -136,12 +141,27 @@ class AttachmentService:
raise NotFound('вложение не найдено')
return attachment
async def content(self, actor: User, attachment_id: UUID) -> tuple[Attachment, ObjectStream]:
async def content(
self, actor: User, attachment_id: UUID, variant: str
) -> tuple[Attachment, ObjectStream, str, int]:
attachment = await self.get(actor, attachment_id)
if attachment.status != 'ready':
raise Conflict('оригинал вложения недоступен')
raise Conflict('вложение ещё не готово')
if variant == 'original':
key = attachment.original_key
content_type = attachment.content_type
size_bytes = attachment.size_bytes
else:
found = next((item for item in attachment.variants if item.name == variant), None)
if found is None:
raise NotFound('вариант вложения не найден')
key = found.object_key
content_type = found.content_type
size_bytes = found.size_bytes
if key is None:
raise Unavailable('объект вложения отсутствует в метаданных')
try:
stream = await self._storage.get(attachment.object_key)
stream = await self._storage.get(key)
except FileNotFoundError as error:
raise Unavailable('оригинал вложения отсутствует в хранилище') from error
return attachment, stream
raise Unavailable('объект вложения отсутствует в хранилище') from error
return attachment, stream, content_type, size_bytes
+14
View File
@@ -5,6 +5,7 @@ import certifi
from dataclasses import dataclass
from datetime import datetime
from typing import BinaryIO, Iterator
from io import BytesIO
from urllib.parse import urlparse
from minio import Minio
@@ -57,6 +58,7 @@ class S3Storage:
secure=parsed.scheme == 'https',
region=region,
http_client=PoolManager(
maxsize=8,
timeout=Timeout(connect=2, read=5),
retries=Retry(total=1, connect=1, read=1, backoff_factor=0.1),
cert_reqs='CERT_REQUIRED',
@@ -102,6 +104,18 @@ class S3Storage:
self._translate(error)
return ObjectStream(response)
async def read(self, key: str, max_bytes: int) -> bytes:
stream = await self.get(key)
result = bytearray()
for chunk in stream.chunks():
result.extend(chunk)
if len(result) > max_bytes:
raise ValueError('stored object exceeds configured byte limit')
return bytes(result)
async def put_bytes(self, key: str, content: bytes, content_type: str) -> None:
await self.put(key, BytesIO(content), len(content), content_type)
async def exists(self, key: str) -> bool:
try:
await asyncio.to_thread(self._client.stat_object, self.bucket, key)
+5 -1
View File
@@ -25,7 +25,11 @@ async def cleanup(apply: bool, grace_seconds: int) -> int:
try:
sessions = create_session_factory(engine)
async with sessions() as session:
known = set(await session.scalars(select(AttachmentRow.object_key)))
known = set(
key
for key in await session.scalars(select(AttachmentRow.original_key))
if key is not None
)
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
removed = 0
for item in await storage.list('originals/'):
+68
View File
@@ -0,0 +1,68 @@
from __future__ import annotations
import argparse
import asyncio
from datetime import datetime, timedelta, timezone
from sqlalchemy import select
from ..config import load_settings
from ..db import create_engine, create_session_factory
from ..models.attachment import AttachmentRow
from ..storage import S3Storage
async def cleanup(apply: bool, retention_seconds: int) -> int:
settings = load_settings()
engine = create_engine(settings.database_url)
sessions = create_session_factory(engine)
storage = S3Storage(
settings.s3_endpoint,
settings.s3_access_key_id,
settings.s3_secret_access_key,
settings.s3_bucket,
settings.s3_region,
)
cutoff = datetime.now(timezone.utc) - timedelta(seconds=retention_seconds)
try:
async with sessions() as session:
active = set(
key
for key in await session.scalars(
select(AttachmentRow.staging_key).where(
AttachmentRow.staging_key.is_not(None),
AttachmentRow.status.in_(('queued', 'processing')),
)
)
if key is not None
)
removed = 0
for item in await storage.list('staging/'):
if item.key in active or item.last_modified > cutoff:
continue
print(f'{"DELETE" if apply else "WOULD_DELETE"} key={item.key}')
if apply:
await storage.remove(item.key)
removed += 1
print(f'summary active={len(active)} removed={removed} mode={"apply" if apply else "dry-run"}')
return 0
finally:
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(description='Clean inactive durable staging objects')
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument('--dry-run', action='store_true')
mode.add_argument('--apply', action='store_true')
parser.add_argument('--retention-seconds', type=int)
args = parser.parse_args()
settings = load_settings()
retention = args.retention_seconds or settings.staging_retention_seconds
if retention < 60:
parser.error('retention must be at least 60 seconds')
raise SystemExit(asyncio.run(cleanup(args.apply, retention)))
if __name__ == '__main__':
main()
+3 -3
View File
@@ -74,7 +74,7 @@ async def _load_attachments(
await session.execute(
select(
AttachmentRow.id,
AttachmentRow.object_key,
AttachmentRow.original_key,
AttachmentRow.status,
AttachmentRow.updated_at,
)
@@ -88,7 +88,7 @@ async def _metadata_key_exists(
) -> bool:
async with sessions() as session:
attachment_id = await session.scalar(
select(AttachmentRow.id).where(AttachmentRow.object_key == object_key).limit(1)
select(AttachmentRow.id).where(AttachmentRow.original_key == object_key).limit(1)
)
return attachment_id is not None
@@ -165,7 +165,7 @@ async def reconcile(
update(AttachmentRow)
.where(
AttachmentRow.id == attachment.id,
AttachmentRow.object_key == attachment.object_key,
AttachmentRow.original_key == attachment.object_key,
AttachmentRow.status == 'ready',
AttachmentRow.updated_at == attachment.updated_at,
)
+45
View File
@@ -0,0 +1,45 @@
from __future__ import annotations
import argparse
import asyncio
from datetime import datetime, timezone
from uuid import UUID
from ..config import load_settings
from ..db import create_engine, create_session_factory
from ..repositories import JobRepository
async def replay(job_id: UUID | None, attachment_id: UUID | None, dry_run: bool) -> int:
settings = load_settings()
engine = create_engine(settings.database_url)
try:
repository = JobRepository(create_session_factory(engine))
if job_id is None:
assert attachment_id is not None
job_id = await repository.id_for_attachment(attachment_id)
if job_id is None:
print(f'REJECTED attachment_id={attachment_id} reason=job_not_found')
return 2
accepted = await repository.replay(job_id, datetime.now(timezone.utc), dry_run)
if not accepted:
print(f'REJECTED job_id={job_id} reason=not_failed_or_staging_missing')
return 2
print(f'{"WOULD_REPLAY" if dry_run else "REPLAYED"} job_id={job_id}')
return 0
finally:
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(description='Safely replay one failed attachment job')
target = parser.add_mutually_exclusive_group(required=True)
target.add_argument('--job-id', type=UUID)
target.add_argument('--attachment-id', type=UUID)
parser.add_argument('--dry-run', action='store_true')
args = parser.parse_args()
raise SystemExit(asyncio.run(replay(args.job_id, args.attachment_id, args.dry_run)))
if __name__ == '__main__':
main()
+4 -2
View File
@@ -50,7 +50,9 @@ async def prepare(size: int) -> None:
'chat_id': CHAT_ID,
'owner_id': USER_ID,
'filename': f'benchmark-{index}.png',
'object_key': f'{OBJECT_PREFIX}{index}',
'original_key': f'{OBJECT_PREFIX}{index}',
'staging_key': None,
'pipeline_version': 1,
'content_type': 'image/png',
'size_bytes': 1,
'sha256': '0' * 64,
@@ -80,7 +82,7 @@ async def prepare(size: int) -> None:
for offset in range(0, size, 500):
await session.execute(insert(AttachmentRow), rows[offset : offset + 500])
await _put_objects(storage, [row['object_key'] for row in rows])
await _put_objects(storage, [row['original_key'] for row in rows])
print(f'prepared metadata={size} objects={size}')
finally:
await engine.dispose()
+78
View File
@@ -20,6 +20,17 @@ services:
timeout: 2s
retries: 30
broker:
image: rabbitmq:4.1.4-alpine
environment:
RABBITMQ_DEFAULT_USER: messenger_test
RABBITMQ_DEFAULT_PASS: messenger_test
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 2s
timeout: 5s
retries: 30
s3:
image: cgr.dev/chainguard/minio@sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597
user: "0"
@@ -69,6 +80,7 @@ services:
SESSION_TTL_SECONDS: 3600
S3_ACCESS_KEY_ID: messenger-api
S3_SECRET_ACCESS_KEY: test-minio-api-password
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
depends_on:
postgres:
condition: service_healthy
@@ -76,6 +88,8 @@ services:
api:
build: .
ports:
- "127.0.0.1:18080:8080"
environment:
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
REDIS_URL: redis://redis:6379/0
@@ -87,6 +101,8 @@ services:
S3_SECRET_ACCESS_KEY: test-minio-api-password
MAX_UPLOAD_BYTES: 10485760
MAX_IMAGE_PIXELS: 40000000
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
JOB_MAX_ATTEMPTS: 5
depends_on:
migrate:
condition: service_completed_successfully
@@ -100,6 +116,67 @@ services:
timeout: 2s
retries: 30
publisher:
build: .
command: ["python", "-m", "app.jobs.publisher"]
environment:
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
REDIS_URL: redis://redis:6379/0
SESSION_TTL_SECONDS: 3600
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: messenger-api
S3_SECRET_ACCESS_KEY: test-minio-api-password
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
healthcheck:
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
interval: 5s
timeout: 3s
retries: 10
worker:
build: .
command: ["python", "-m", "app.jobs.worker"]
environment:
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
REDIS_URL: redis://redis:6379/0
SESSION_TTL_SECONDS: 3600
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: messenger-api
S3_SECRET_ACCESS_KEY: test-minio-api-password
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
WORKER_CONCURRENCY: 2
WORKER_PREFETCH: 2
JOB_MAX_ATTEMPTS: 5
JOB_TIMEOUT_SECONDS: 60
JOB_LEASE_SECONDS: 90
RETRY_BASE_SECONDS: 1
RETRY_MAX_SECONDS: 4
STAGING_RETENTION_SECONDS: 86400
THUMBNAIL_JPEG_QUALITY: 85
CROP_JPEG_QUALITY: 85
OPTIMIZED_WEBP_QUALITY: 82
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
interval: 5s
timeout: 3s
retries: 10
tests:
build:
context: .
@@ -114,6 +191,7 @@ services:
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: messenger-api
S3_SECRET_ACCESS_KEY: test-minio-api-password
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
depends_on:
api:
condition: service_healthy
+92
View File
@@ -28,6 +28,21 @@ services:
timeout: 3s
retries: 20
broker:
image: rabbitmq:4.1.4-alpine
cpus: "0.5"
mem_limit: 384m
environment:
RABBITMQ_DEFAULT_USER: ${BROKER_USER:-messenger}
RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:-change-me-rabbitmq}
volumes:
- broker_data:/var/lib/rabbitmq
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 5s
timeout: 5s
retries: 20
s3:
image: cgr.dev/chainguard/minio@sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597
user: "0"
@@ -85,6 +100,7 @@ services:
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
BROKER_URL: ${BROKER_URL}
depends_on:
postgres:
condition: service_healthy
@@ -106,6 +122,8 @@ services:
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
BROKER_URL: ${BROKER_URL}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
expose:
- "8080"
deploy:
@@ -124,6 +142,79 @@ services:
retries: 5
stop_grace_period: 10s
publisher:
build: .
command: ["python", "-m", "app.jobs.publisher"]
cpus: "0.25"
mem_limit: 192m
environment:
DATABASE_URL: ${DATABASE_URL}
REDIS_URL: ${REDIS_URL}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
BROKER_URL: ${BROKER_URL}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90}
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
healthcheck:
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
interval: 10s
timeout: 3s
retries: 3
restart: unless-stopped
stop_grace_period: 10s
worker:
build: .
command: ["python", "-m", "app.jobs.worker"]
cpus: "1.0"
mem_limit: 768m
environment:
DATABASE_URL: ${DATABASE_URL}
REDIS_URL: ${REDIS_URL}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
BROKER_URL: ${BROKER_URL}
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
WORKER_PREFETCH: ${WORKER_PREFETCH:-2}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90}
RETRY_BASE_SECONDS: ${RETRY_BASE_SECONDS:-2}
RETRY_MAX_SECONDS: ${RETRY_MAX_SECONDS:-60}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
THUMBNAIL_JPEG_QUALITY: ${THUMBNAIL_JPEG_QUALITY:-85}
CROP_JPEG_QUALITY: ${CROP_JPEG_QUALITY:-85}
OPTIMIZED_WEBP_QUALITY: ${OPTIMIZED_WEBP_QUALITY:-82}
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
interval: 10s
timeout: 3s
retries: 3
restart: unless-stopped
stop_grace_period: 70s
lb:
image: haproxy:3.2.26-alpine
cpus: "0.25"
@@ -146,3 +237,4 @@ volumes:
postgres_data:
redis_data:
s3_data:
broker_data:
+1
View File
@@ -7,3 +7,4 @@ redis==5.2.1
argon2-cffi==25.1.0
minio==7.2.20
Pillow==13.0.0
aio-pika==9.5.5
+16
View File
@@ -0,0 +1,16 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
cd "$(dirname -- "$script_dir")"
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
set +e
docker compose -f "$compose_file" --env-file .env run --rm \
-e FAULT_PUBLISHER_AFTER_CONFIRM=1 \
publisher python -m app.jobs.publisher --once >/dev/null
status=$?
set -e
if [ "$status" -ne 91 ]; then
printf '%s\n' "publisher fault expected exit 91, got $status" >&2
exit 1
fi
docker compose -f "$compose_file" --env-file .env up -d publisher >/dev/null
+26
View File
@@ -0,0 +1,26 @@
#!/bin/sh
set -eu
if [ "$#" -ne 1 ]; then
printf '%s\n' 'usage: crash-worker.sh objects|commit|permanent' >&2
exit 2
fi
case "$1" in
objects) variable=FAULT_WORKER_AFTER_OBJECTS; expected=92 ;;
commit) variable=FAULT_WORKER_AFTER_COMMIT; expected=93 ;;
permanent) variable=FAULT_WORKER_PERMANENT; expected=94 ;;
*) printf '%s\n' 'mode must be objects, commit or permanent' >&2; exit 2 ;;
esac
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
cd "$(dirname -- "$script_dir")"
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
set +e
docker compose -f "$compose_file" --env-file .env run --rm \
-e "$variable=1" -e JOB_LEASE_SECONDS=3 \
worker python -m app.jobs.worker >/dev/null
status=$?
set -e
if [ "$status" -ne "$expected" ]; then
printf '%s\n' "worker fault expected exit $expected, got $status" >&2
exit 1
fi
docker compose -f "$compose_file" --env-file .env up -d worker >/dev/null
+11
View File
@@ -0,0 +1,11 @@
#!/bin/sh
set -eu
if [ "$#" -ne 1 ]; then
printf '%s\n' 'usage: replay-attachment.sh ATTACHMENT_ID' >&2
exit 2
fi
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
cd "$(dirname -- "$script_dir")"
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
docker compose -f "$compose_file" --env-file .env run --rm \
publisher python -m app.tools.replay_failed --attachment-id "$1"
+11
View File
@@ -0,0 +1,11 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
cd "$(dirname -- "$script_dir")"
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
docker compose -f "$compose_file" --env-file .env restart api >/dev/null
if [ "$compose_file" = "compose.yaml" ]; then
docker compose -f "$compose_file" --env-file .env up -d --wait --wait-timeout 60 api lb >/dev/null
else
docker compose -f "$compose_file" --env-file .env up -d api >/dev/null
fi
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
cd "$(dirname -- "$script_dir")"
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
docker compose -f "$compose_file" --env-file .env restart broker >/dev/null
+4 -4
View File
@@ -6,7 +6,7 @@ project_dir=$(dirname -- "$script_dir")
cd "$project_dir"
# Останавливаем публичный маршрут до перезапуска общих хранилищ.
docker compose --env-file .env stop lb api >/dev/null
docker compose --env-file .env restart postgres redis s3 >/dev/null
docker compose --env-file .env up -d --wait --wait-timeout 60 postgres redis s3 >/dev/null
docker compose --env-file .env up -d --wait --wait-timeout 60 api lb >/dev/null
docker compose --env-file .env stop lb api publisher worker >/dev/null
docker compose --env-file .env restart postgres redis s3 broker >/dev/null
docker compose --env-file .env up -d --wait --wait-timeout 60 postgres redis s3 broker >/dev/null
docker compose --env-file .env up -d --wait --wait-timeout 60 publisher worker api lb >/dev/null
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
cd "$(dirname -- "$script_dir")"
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
docker compose -f "$compose_file" --env-file .env up -d publisher >/dev/null
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
project_dir=$(dirname -- "$script_dir")
cd "$project_dir"
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
docker compose -f "$compose_file" --env-file .env up -d worker >/dev/null
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
cd "$(dirname -- "$script_dir")"
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
docker compose -f "$compose_file" --env-file .env stop publisher >/dev/null
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
project_dir=$(dirname -- "$script_dir")
cd "$project_dir"
compose_file=${COMPOSE_FILE_PATH:-compose.yaml}
docker compose -f "$compose_file" --env-file .env stop worker >/dev/null
+3 -2
View File
@@ -22,12 +22,13 @@ def main():
uploaded = client.call(
'POST',
f'/api/v1/chats/{chat["id"]}/attachments?filename=failover.png',
201,
202,
actor=alice,
raw=content,
headers={'Content-Type': 'image/png'},
schema='Attachment',
)
ready = client.wait_ready(bob, uploaded['json'])
target = uploaded['headers']['X-Instance-Id']
env = dict(os.environ)
env['TARGET_INSTANCE_ID'] = target
@@ -40,7 +41,7 @@ def main():
try:
downloaded = client.call(
'GET',
f'/api/v1/attachments/{uploaded["json"]["id"]}/content',
f'/api/v1/attachments/{ready["id"]}/content',
actor=bob,
)
if downloaded['headers']['X-Instance-Id'] != target:
+47
View File
@@ -0,0 +1,47 @@
#!/usr/bin/env python3
"""A persistent broker volume keeps a confirmed job across broker restart."""
import os
from pathlib import Path
import subprocess
import time
from client import Client, arguments, check, run
def executable(value):
path = Path(value).resolve()
check(path.is_file() and os.access(path, os.X_OK), str(path) + ': not executable')
return path
def main():
def configure(parser):
parser.add_argument('--stop-worker-script', required=True)
parser.add_argument('--restart-broker-script', required=True)
parser.add_argument('--start-worker-script', required=True)
args = arguments('RabbitMQ persistent delivery after restart', configure)
stop_worker = executable(args.stop_worker_script)
restart_broker = executable(args.restart_broker_script)
start_worker = executable(args.start_worker_script)
client = Client(args.base_url, args.ca_file)
alice, bob, _eve, chat = client.bootstrap()
subprocess.run([str(stop_worker)], check=True)
attachment, original = client.attachment(alice, chat)
time.sleep(2)
queued = client.call(
'GET', f'/api/v1/attachments/{attachment["id"]}', actor=bob, schema='Attachment'
)['json']
check(queued['status'] == 'queued', 'worker processed job while expected stopped')
subprocess.run([str(restart_broker)], check=True)
subprocess.run([str(start_worker)], check=True)
client.wait_ready(bob, attachment)
downloaded = client.call(
'GET', f'/api/v1/attachments/{attachment["id"]}/content', actor=bob
)['bytes']
check(downloaded == original, 'job/original lost across broker restart')
print('PASS: confirmed durable message survived broker restart with its volume')
if __name__ == '__main__':
run(main)
+58
View File
@@ -0,0 +1,58 @@
#!/usr/bin/env python3
"""Prove permanent failure, DLQ routing and guarded operator replay."""
import os
from pathlib import Path
import subprocess
import time
from client import Client, arguments, check, run
def executable(value):
path = Path(value).resolve()
check(path.is_file() and os.access(path, os.X_OK), str(path) + ': not executable')
return path
def main():
def configure(parser):
parser.add_argument('--stop-worker-script', required=True)
parser.add_argument('--crash-worker-script', required=True)
parser.add_argument('--replay-script', required=True)
args = arguments('Failed registry, DLQ and guarded replay', configure)
stop = executable(args.stop_worker_script)
crash = executable(args.crash_worker_script)
replay = executable(args.replay_script)
client = Client(args.base_url, args.ca_file)
alice, bob, _eve, chat = client.bootstrap()
subprocess.run([str(stop)], check=True)
attachment, original = client.attachment(alice, chat)
subprocess.run([str(crash), 'permanent'], check=True)
deadline = time.monotonic() + 20
while True:
failed = client.call(
'GET', f'/api/v1/attachments/{attachment["id"]}', actor=bob,
schema='Attachment',
)['json']
if failed['status'] == 'failed':
break
check(time.monotonic() < deadline, 'attachment did not reach failed')
time.sleep(0.2)
check(failed['error_code'] == 'fault_injected', 'unsafe or unexpected error code')
subprocess.run([str(replay), attachment['id']], check=True)
ready = client.wait_ready(bob, attachment)
check(len(ready['variants']) == 3, 'replay produced incomplete variants')
content = client.call(
'GET', f'/api/v1/attachments/{attachment["id"]}/content', actor=bob
)['bytes']
check(content == original, 'replay changed original')
duplicate = subprocess.run([str(replay), attachment['id']], check=False)
check(duplicate.returncode != 0, 'ready job was replayed again')
print('PASS: permanent failure reached failed/DLQ and guarded replay reached ready')
if __name__ == '__main__':
run(main)
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env python3
"""Prove that a committed outbox survives publisher downtime."""
import os
from pathlib import Path
import subprocess
import time
from client import Client, arguments, check, run
def main():
def configure(parser):
parser.add_argument('--stop-publisher-script', required=True)
parser.add_argument('--start-publisher-script', required=True)
args = arguments('Transactional outbox recovery', configure)
stop = Path(args.stop_publisher_script).resolve()
start = Path(args.start_publisher_script).resolve()
check(all(path.is_file() and os.access(path, os.X_OK) for path in (stop, start)), 'scripts must be executable')
client = Client(args.base_url, args.ca_file)
alice, bob, _eve, chat = client.bootstrap()
subprocess.run([str(stop)], check=True)
attachment, _content = client.attachment(alice, chat)
time.sleep(1)
queued = client.call(
'GET', f'/api/v1/attachments/{attachment["id"]}', actor=bob, schema='Attachment'
)['json']
check(queued['status'] == 'queued', 'job escaped outbox while publisher was stopped')
subprocess.run([str(start)], check=True)
ready = client.wait_ready(bob, attachment)
check(ready['status'] == 'ready', 'outbox did not recover committed job')
print('PASS: COMMIT-before-publish recovered without a new upload')
if __name__ == '__main__':
run(main)
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env python3
"""Grade-4 black-box check for every promised image variant."""
import hashlib
from client import Client, arguments, check, run
from image_probe import jpeg_size
def main():
args = arguments('Verify thumbnail, crop and optimized variants')
client = Client(args.base_url, args.ca_file)
alice, bob, _eve, chat = client.bootstrap()
attachment, original = client.attachment(alice, chat)
ready = client.wait_ready(bob, attachment)
variants = {item['name']: item for item in ready['variants']}
check(set(variants) == {'thumbnail', 'crop', 'optimized'}, 'grade-4 variants missing')
expected = {
'thumbnail': ('image/jpeg', (256, 160)),
'crop': ('image/jpeg', (128, 128)),
'optimized': ('image/webp', (320, 200)),
}
for name, (content_type, dimensions) in expected.items():
response = client.call(
'GET',
f'/api/v1/attachments/{ready["id"]}/content?variant={name}',
actor=bob,
)
metadata = variants[name]
check(response['headers'].get_content_type() == content_type, name + ': MIME mismatch')
check((metadata['width'], metadata['height']) == dimensions, name + ': metadata dimensions')
check(len(response['bytes']) == metadata['size_bytes'], name + ': size mismatch')
check(hashlib.sha256(response['bytes']).hexdigest() == metadata['sha256'], name + ': hash mismatch')
if content_type == 'image/jpeg':
check(jpeg_size(response['bytes']) == dimensions, name + ': actual dimensions')
else:
check(response['bytes'][:4] == b'RIFF' and response['bytes'][8:12] == b'WEBP', 'optimized is not WebP')
downloaded = client.call(
'GET', f'/api/v1/attachments/{ready["id"]}/content', actor=bob
)['bytes']
check(downloaded == original, 'original bytes changed')
print('PASS: original plus thumbnail/crop/optimized variants are valid')
if __name__ == '__main__':
run(main)
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env python3
"""Crash after publisher confirm creates a harmless duplicate delivery."""
import os
from pathlib import Path
import subprocess
from client import Client, arguments, check, run
def executable(value):
path = Path(value).resolve()
check(path.is_file() and os.access(path, os.X_OK), str(path) + ': not executable')
return path
def main():
def configure(parser):
parser.add_argument('--stop-publisher-script', required=True)
parser.add_argument('--crash-publisher-script', required=True)
args = arguments('Crash after publisher confirm', configure)
stop = executable(args.stop_publisher_script)
crash = executable(args.crash_publisher_script)
client = Client(args.base_url, args.ca_file)
alice, bob, _eve, chat = client.bootstrap()
subprocess.run([str(stop)], check=True)
attachment, _original = client.attachment(alice, chat)
subprocess.run([str(crash)], check=True)
ready = client.wait_ready(bob, attachment)
names = [item['name'] for item in ready['variants']]
check(sorted(names) == ['crop', 'optimized', 'thumbnail'], 'duplicate or missing variants')
print('PASS: confirm-before-outbox-mark duplicate produced one logical result')
if __name__ == '__main__':
run(main)
+2 -2
View File
@@ -16,7 +16,7 @@ def scenario():
return client, alice, bob, eve, chat
def upload(client, actor, chat, content, *, filename='sample.png', content_type='image/png', expected=201):
def upload(client, actor, chat, content, *, filename='sample.png', content_type='image/png', expected=202):
return client.call(
'POST',
f'/api/v1/chats/{chat["id"]}/attachments?filename={quote(filename, safe="")}',
@@ -24,7 +24,7 @@ def upload(client, actor, chat, content, *, filename='sample.png', content_type=
actor=actor,
raw=content,
headers={'Content-Type': content_type},
schema='Attachment' if expected == 201 else None,
schema='Attachment' if expected == 202 else None,
)
+89
View File
@@ -0,0 +1,89 @@
from __future__ import annotations
import hashlib
from io import BytesIO
import pytest
from PIL import Image
from app.jobs.pipeline import PermanentImageError, process_image
def encoded(width: int, height: int, mode: str = 'RGBA') -> bytes:
image = Image.new(mode, (width, height), (20, 80, 160, 128) if mode == 'RGBA' else 'navy')
output = BytesIO()
image.save(output, 'PNG', pnginfo=None)
return output.getvalue()
def run(source: bytes):
return process_image(
source,
'image/png',
hashlib.sha256(source).hexdigest(),
40_000_000,
85,
85,
82,
)
def test_pipeline_creates_all_grade4_variants() -> None:
"""Create every grade-4 variant with the required format, size and checksum."""
variants = {item.name: item for item in run(encoded(320, 200))}
assert set(variants) == {'thumbnail', 'crop', 'optimized'}
assert (variants['thumbnail'].width, variants['thumbnail'].height) == (256, 160)
assert variants['thumbnail'].content_type == 'image/jpeg'
assert (variants['crop'].width, variants['crop'].height) == (128, 128)
assert variants['crop'].content_type == 'image/jpeg'
assert (variants['optimized'].width, variants['optimized'].height) == (320, 200)
assert variants['optimized'].content_type == 'image/webp'
for item in variants.values():
assert hashlib.sha256(item.content).hexdigest() == item.sha256
def test_thumbnail_and_optimized_do_not_upscale_small_image() -> None:
"""Keep small images at source size except for the required 128x128 crop."""
variants = {item.name: item for item in run(encoded(40, 20, 'RGB'))}
assert (variants['thumbnail'].width, variants['thumbnail'].height) == (40, 20)
assert (variants['optimized'].width, variants['optimized'].height) == (40, 20)
assert (variants['crop'].width, variants['crop'].height) == (128, 128)
def test_jpeg_variants_composite_alpha_on_white() -> None:
"""Flatten transparent input before encoding JPEG variants without an alpha channel."""
variants = {item.name: item for item in run(encoded(2, 2))}
with Image.open(BytesIO(variants['thumbnail'].content)) as image:
assert image.mode == 'RGB'
def test_checksum_mismatch_is_permanent() -> None:
"""Reject changed staging bytes as a permanent checksum error."""
source = encoded(20, 20)
with pytest.raises(PermanentImageError, match='checksum_mismatch'):
process_image(source, 'image/png', '0' * 64, 40_000_000, 85, 85, 82)
def test_exif_orientation_is_applied_and_removed() -> None:
"""Apply EXIF orientation before resizing and omit metadata from generated output."""
image = Image.new('RGB', (320, 200), 'red')
exif = Image.Exif()
exif[274] = 6
output = BytesIO()
image.save(output, 'JPEG', exif=exif)
source = output.getvalue()
variants = {
item.name: item
for item in process_image(
source,
'image/jpeg',
hashlib.sha256(source).hexdigest(),
40_000_000,
85,
85,
82,
)
}
assert (variants['thumbnail'].width, variants['thumbnail'].height) == (160, 256)
with Image.open(BytesIO(variants['thumbnail'].content)) as transformed:
assert not transformed.getexif()
+41
View File
@@ -0,0 +1,41 @@
#!/usr/bin/env python3
"""Fault injection around S3 effects, DB commit and broker ack."""
import os
from pathlib import Path
import subprocess
from client import Client, arguments, check, run
def executable(value):
path = Path(value).resolve()
check(path.is_file() and os.access(path, os.X_OK), str(path) + ': not executable')
return path
def main():
def configure(parser):
parser.add_argument('--stop-worker-script', required=True)
parser.add_argument('--crash-worker-script', required=True)
parser.add_argument('--mode', choices=('objects', 'commit'), required=True)
args = arguments('Worker crash around durable effects', configure)
stop = executable(args.stop_worker_script)
crash = executable(args.crash_worker_script)
client = Client(args.base_url, args.ca_file)
alice, bob, _eve, chat = client.bootstrap()
subprocess.run([str(stop)], check=True)
attachment, original = client.attachment(alice, chat)
subprocess.run([str(crash), args.mode], check=True)
ready = client.wait_ready(bob, attachment)
names = sorted(item['name'] for item in ready['variants'])
check(names == ['crop', 'optimized', 'thumbnail'], 'duplicate or missing variants')
downloaded = client.call(
'GET', f'/api/v1/attachments/{attachment["id"]}/content', actor=bob
)['bytes']
check(downloaded == original, 'original changed after crash recovery')
print('PASS: worker crash mode=%s recovered one logical result' % args.mode)
if __name__ == '__main__':
run(main)
+61
View File
@@ -0,0 +1,61 @@
#!/usr/bin/env python3
"""Required scenario: accepted work survives worker and API downtime."""
import os
from pathlib import Path
import subprocess
import time
from client import Client, arguments, check, run
def executable(value):
path = Path(value).expanduser().resolve()
check(path.is_file() and os.access(path, os.X_OK), str(path) + ': not executable')
return path
def main():
def configure(parser):
parser.add_argument('--stop-worker-script', required=True)
parser.add_argument('--restart-api-script', required=True)
parser.add_argument('--start-worker-script', required=True)
args = arguments('Worker outage and durable accepted job', configure)
stop_worker = executable(args.stop_worker_script)
restart_api = executable(args.restart_api_script)
start_worker = executable(args.start_worker_script)
client = Client(args.base_url, args.ca_file)
alice, bob, _eve, chat = client.bootstrap()
subprocess.run([str(stop_worker)], check=True)
attachment, original = client.attachment(alice, chat)
message = client.message(alice, chat, '', attachments=[attachment['id']])['json']
check(message['attachment_ids'] == [attachment['id']], 'queued attachment was not linkable')
time.sleep(1)
client.call(
'GET', f'/api/v1/attachments/{attachment["id"]}/content', 409, actor=bob
)
subprocess.run([str(restart_api)], check=True)
deadline = time.monotonic() + 60
while True:
try:
client.call('GET', '/health/ready', schema='Health')
break
except Exception:
check(time.monotonic() < deadline, 'API did not recover after restart')
time.sleep(0.3)
persisted = client.call(
'GET', f'/api/v1/attachments/{attachment["id"]}', actor=bob, schema='Attachment'
)['json']
check(persisted['status'] in {'queued', 'processing'}, 'accepted job disappeared after API restart')
subprocess.run([str(start_worker)], check=True)
ready = client.wait_ready(bob, attachment)
downloaded = client.call(
'GET', f'/api/v1/attachments/{attachment["id"]}/content', actor=bob
)['bytes']
check(downloaded == original, 'original changed after worker recovery')
check(ready['status'] == 'ready', 'job did not become ready')
print('PASS: worker stopped -> 202 -> message -> API restart -> worker -> ready')
if __name__ == '__main__':
run(main)