diff --git a/.env.example b/.env.example index 4e38f4a..c215a69 100644 --- a/.env.example +++ b/.env.example @@ -1,30 +1,16 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 +# Compose supplies internal PostgreSQL, Redis, broker and S3 addresses. APP_PORT=8080 -APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 +JOB_MEMORY_MB=768 STAGING_RETENTION_SECONDS=86400 -# Local settings used by compose.yaml (application carried over from lab4). +# Choose credentials locally. POSTGRES_DB=messenger POSTGRES_USER=messenger POSTGRES_PASSWORD=replace-with-local-password diff --git a/compose.yaml b/compose.yaml index a9999cd..0f26a40 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,6 +1,5 @@ x-api: &api image: mtusi2026lab5-api:local - build: . dns_opt: - timeout:1 - attempts:1 @@ -23,14 +22,66 @@ x-api: &api S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env} MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + BROKER_URL: redis://broker:6379/0 + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_MEMORY_MB: ${JOB_MEMORY_MB:-768} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} networks: [backend] cpus: 1.0 mem_limit: 512m stop_grace_period: 10s services: + broker: + image: redis:7.4.2-bookworm + command: [redis-server, --appendonly, 'yes', --appendfsync, always, --maxmemory, 128mb, --maxmemory-policy, noeviction] + volumes: + - broker_data:/data + networks: [backend] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 2s + retries: 30 + cpus: 0.5 + mem_limit: 256m + + dispatcher: + <<: *api + command: [python, -m, src.jobs, dispatch] + healthcheck: + disable: true + depends_on: + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + restart: unless-stopped + mem_limit: 128m + + worker: + <<: *api + command: [python, -m, src.worker] + init: true + healthcheck: + disable: true + depends_on: + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + s3-init: + condition: service_completed_successfully + restart: unless-stopped + cpus: 2.0 + mem_limit: 2g + pids_limit: 64 + api: <<: *api + build: . deploy: replicas: 2 depends_on: @@ -170,6 +221,7 @@ networks: backend: internal: true volumes: + broker_data: postgres_data: redis_data: s3_data: diff --git a/infra/async.md b/infra/async.md new file mode 100644 index 0000000..868f247 --- /dev/null +++ b/infra/async.md @@ -0,0 +1,70 @@ +# Локальный pipeline lab5 + +Запуск: скопировать `.env.example` в `.env`, задать локальные пароли, +выполнить `docker compose up -d --build`. API: `http://localhost:8080`. +Миграции запускаются сервисом `migrate` перед API/worker. + +Upload проверяет формат, bytes и пиксели, сохраняет неизменный оригинал +в приватный S3 `staging/`, затем одной транзакцией записывает вложение и job. +Ответ 202 содержит queued. Изменение размера выполняется только worker. +Job в PostgreSQL одновременно служит outbox: dispatcher отправляет его +в отдельный Redis Streams, после ответа XADD отмечает публикацию. +Redis использует volume, AOF и `appendfsync always`, eviction отключён. +Сбой до/после XADD даёт повторную доставку, а не потерю принятого задания. +Readiness API зависит от PostgreSQL, сессионного Redis и S3; +недоступность broker/worker не мешает сохранять работу в outbox. + +Worker: два потребителя, у каждого не больше одной выполняемой задачи. +Каждая попытка работает в отдельном процессе: timeout 60 секунд, +лимит адресного пространства 768 MiB, контейнер 2 GiB. +Настройки: `WORKER_CONCURRENCY` (1–4), `JOB_TIMEOUT_SECONDS`, +`JOB_MEMORY_MB`, `JOB_MAX_ATTEMPTS` (по умолчанию 5). +При повышении concurrency нужно соответственно увеличить память контейнера. +Lease равен timeout + 15 секунд. Dispatcher возвращает просроченный +processing в очередь. Токен lease и блокировка job защищают публикацию +от устаревшей попытки. ACK выполняется после устойчивого результата +либо сохранённого решения о retry/failed. Pending читаются через XAUTOCLAIM. + +Оригинал публикуется в `originals/`, варианты — по стабильным ключам +`variants//v1/`. Готовность и метаданные трёх +вариантов фиксируются одной транзакцией после S3. Дубль ready — no-op. +JPEG thumbnail: вписывание в 256×256, округление вниз, без увеличения. +JPEG crop: центрированный квадрат 128×128. JPEG quality=85. +WebP optimized: вписывание в 1280×1280 без увеличения, quality=80. +Применяется EXIF orientation, прозрачность — на белом; EXIF/ICC удаляются. +Оригинал остаётся побайтно неизменным. + +Временные ошибки получают backoff `min(60, 2^failures)` секунд с jitter +0.5–1.5. Повреждённое изображение/checksum — `invalid_image`, нехватка +памяти — `resource_limit`, без retry. Исчерпание попыток — `retry_exhausted`. +Последняя причина ошибки и время следующей попытки сохраняются в PostgreSQL. +Реестр failed и replay доступны только локальному оператору через Compose: + +```sh +docker compose exec api python -m src.jobs failed +docker compose exec api python -m src.jobs replay JOB_UUID --reason "причина повтора" +``` + +Replay допускается только для failed с доступным staging, сохраняет audit, +job_id и общий номер попытки, выдаёт новый бюджет retry. Ready повторять нельзя. +Cleanup запускается раз в минуту: staging ready удаляется после публикации; +failed хранится `STAGING_RETENTION_SECONDS` (86400) после ошибки. +Частично записанные конечные объекты failed очищаются после того же срока. +Бесхозные объекты удаляются после того же retention. Queued/processing +не удаляются независимо от возраста. После удаления failed staging нужен +новый upload. Реестр ошибок остаётся в PostgreSQL. + +Проверки (из корня проекта): + +```sh +python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080 +python -X utf8 student-tests/pipeline.py +python -X utf8 student-tests/edge_cases.py +docker compose run --rm --no-deps -v ./student-tests:/checks:ro worker python -m unittest discover -s /checks +``` + +Pipeline-тест останавливает/возвращает сервисы только этого Compose-проекта, +создаёт синтетические данные, проверяет разрыв COMMIT/publish, рестарт broker, +crash после S3 и после COMMIT до ack, lease, дубли, retry/replay и cleanup. +Точки `JOB_FAULT=after_objects|after_commit` используются только тестом, +в обычной конфигурации выключены. Контракты и публичные тесты не изменены. diff --git a/infra/s3-policy.json b/infra/s3-policy.json index 41bcff8..8e77546 100644 --- a/infra/s3-policy.json +++ b/infra/s3-policy.json @@ -9,7 +9,9 @@ { "Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], - "Resource": ["arn:aws:s3:::messenger-attachments/originals/*"] + "Resource": ["arn:aws:s3:::messenger-attachments/originals/*", + "arn:aws:s3:::messenger-attachments/staging/*", + "arn:aws:s3:::messenger-attachments/variants/*"] } ] } diff --git a/migrations/005_attachment_jobs.sql b/migrations/005_attachment_jobs.sql new file mode 100644 index 0000000..1e5e88c --- /dev/null +++ b/migrations/005_attachment_jobs.sql @@ -0,0 +1,23 @@ +ALTER TABLE attachments + ADD COLUMN updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + ADD COLUMN status text NOT NULL DEFAULT 'ready' + CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + ADD COLUMN staging_key text, + ADD COLUMN variants jsonb NOT NULL DEFAULT '[]', + ADD COLUMN error_code text; +ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued'; + +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id), + pipeline_version bigint NOT NULL DEFAULT 1 CHECK (pipeline_version = 1), + attempt bigint NOT NULL DEFAULT 0, + failures bigint NOT NULL DEFAULT 0, + available_at timestamptz NOT NULL DEFAULT now(), + published_at timestamptz, + lease_until timestamptz, + lease_token uuid, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + replay_audit jsonb NOT NULL DEFAULT '[]' +); +CREATE INDEX attachment_jobs_due_idx ON attachment_jobs(available_at); diff --git a/migrations/006_job_error.sql b/migrations/006_job_error.sql new file mode 100644 index 0000000..f721d2e --- /dev/null +++ b/migrations/006_job_error.sql @@ -0,0 +1 @@ +ALTER TABLE attachment_jobs ADD COLUMN last_error text; diff --git a/src/attachments.py b/src/attachments.py index 9f86243..2f1806a 100644 --- a/src/attachments.py +++ b/src/attachments.py @@ -6,7 +6,7 @@ from PIL import Image, UnidentifiedImageError from sqlalchemy import func, select from .errors import APIError -from .models import Attachment, ChatMember +from .models import Attachment, AttachmentJob, ChatMember from .object_storage import BUCKET, PREFIX, s3_client from .store import ensure_uuid, serialize, store @@ -48,25 +48,28 @@ def lock_object(session, key): def metadata(attachment): result = serialize(attachment, 'id', 'chat_id', 'owner_id', 'filename', - 'content_type', 'size_bytes', 'sha256', 'created_at') - return {**result, 'updated_at': result['created_at'], 'status': 'ready', - 'variants': [], 'error_code': None} + 'content_type', 'size_bytes', 'sha256', 'created_at', + 'updated_at', 'status', 'error_code') + return {**result, 'variants': [{k: v for k, v in item.items() if k != 'object_key'} + for item in attachment.variants]} def save_attachment(source, chat_id, actor_id, filename, content_type, size, digest): validate_image(source, content_type) attachment_id = str(uuid4()) key = PREFIX + attachment_id + staging_key = 'staging/' + attachment_id with store.sessions.begin() as session: store._chat(session, chat_id, actor_id) - lock_object(session, key) - s3_client().put_object(Bucket=BUCKET, Key=key, Body=source, + lock_object(session, staging_key) + s3_client().put_object(Bucket=BUCKET, Key=staging_key, Body=source, ContentLength=size, ContentType=content_type) attachment = Attachment(id=attachment_id, chat_id=chat_id, owner_id=actor_id, object_key=key, filename=filename, content_type=content_type, - size_bytes=size, sha256=digest) + size_bytes=size, sha256=digest, staging_key=staging_key) session.add(attachment) session.flush() + session.add(AttachmentJob(attachment_id=attachment_id)) result = metadata(attachment) # An uncertain COMMIT is not followed by DELETE: cleanup checks the DB later. return result diff --git a/src/cleanup.py b/src/cleanup.py index ac1297e..0d5eeb3 100644 --- a/src/cleanup.py +++ b/src/cleanup.py @@ -1,6 +1,7 @@ """Remove old S3 objects without committed metadata. Never delete on a DB error.""" import argparse import logging +import os import time from datetime import datetime, timedelta, timezone @@ -10,23 +11,47 @@ from sqlalchemy.orm import sessionmaker from .attachments import lock_object from .database import create_db_engine from .models import Attachment -from .object_storage import BUCKET, PREFIX, s3_client +from .object_storage import BUCKET, s3_client -GRACE = timedelta(hours=1) +GRACE = timedelta(seconds=max(1, int(os.getenv('STAGING_RETENTION_SECONDS', '86400')))) def cleanup(sessions, client, now=None): cutoff = (now or datetime.now(timezone.utc)) - GRACE deleted = 0 - for page in client.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix=PREFIX): + for page in client.get_paginator('list_objects_v2').paginate(Bucket=BUCKET): for obj in page.get('Contents', []): - if obj['LastModified'] >= cutoff: - continue key = obj['Key'] + if not key.startswith(('staging/', 'originals/', 'variants/')): + continue with sessions.begin() as session: lock_object(session, key) - if session.scalar(select(Attachment.id).where(Attachment.object_key == key)): - continue + if key.startswith('staging/'): + attachment = session.scalar(select(Attachment).where( + Attachment.staging_key == key).with_for_update()) + if attachment: + if attachment.status in ('queued', 'processing'): + continue + if attachment.status == 'failed' and attachment.updated_at >= cutoff: + continue + client.delete_object(Bucket=BUCKET, Key=key) + attachment.staging_key = None + deleted += 1 + continue + else: + # All final keys belong to a server-generated attachment UUID. + from uuid import UUID + try: + attachment_id = str(UUID(key.split('/')[1])) + except (ValueError, IndexError): + continue + attachment = session.get(Attachment, attachment_id, with_for_update=True) + if attachment: + if attachment.status != 'failed' or attachment.updated_at >= cutoff: + continue + client.delete_object(Bucket=BUCKET, Key=key) + deleted += 1 + continue # Recheck age after taking the same lock as the uploader. if client.head_object(Bucket=BUCKET, Key=key)['LastModified'] >= cutoff: continue @@ -52,7 +77,7 @@ def main(): raise SystemExit(1) from None if args.once: break - time.sleep(3600) + time.sleep(60) finally: engine.dispose() diff --git a/src/image_processor.py b/src/image_processor.py new file mode 100644 index 0000000..019eb9a --- /dev/null +++ b/src/image_processor.py @@ -0,0 +1,116 @@ +"""One isolated process per attempt; quality: JPEG 85, WebP 80; no EXIF/ICC.""" +import hashlib +import io +import os +import sys + +from PIL import Image, ImageOps, UnidentifiedImageError +from sqlalchemy.orm import sessionmaker + +from .attachments import MAX_BYTES, MAX_PIXELS +from .database import create_db_engine +from .jobs import now +from .models import Attachment, AttachmentJob +from .object_storage import BUCKET, s3_client + + +def fit_size(size, bound): + width, height = size + if max(size) <= bound: + return size + return max(1, width * bound // max(size)), max(1, height * bound // max(size)) + + +def variants(data): + with Image.open(io.BytesIO(data), formats=['PNG', 'JPEG']) as original: + if original.width * original.height > MAX_PIXELS: + raise ValueError('pixel limit') + original.load() + oriented = ImageOps.exif_transpose(original) + rgba = oriented.convert('RGBA') + rgb = Image.new('RGB', rgba.size, 'white') + rgb.paste(rgba, mask=rgba.getchannel('A')) + rgba.close() + oriented.close() + rgb.info.clear() + try: + for name, image, fmt, mime, quality in ( + ('thumbnail', rgb.resize(fit_size(rgb.size, 256), Image.Resampling.LANCZOS), 'JPEG', 'image/jpeg', 85), + ('crop', ImageOps.fit(rgb, (128, 128), Image.Resampling.LANCZOS), 'JPEG', 'image/jpeg', 85), + ('optimized', rgb.resize(fit_size(rgb.size, 1280), Image.Resampling.LANCZOS), 'WEBP', 'image/webp', 80), + ): + with image, io.BytesIO() as output: + image.save(output, format=fmt, quality=quality) + blob = output.getvalue() + yield {'name': name, 'content_type': mime, 'width': image.width, + 'height': image.height, 'size_bytes': len(blob), + 'sha256': hashlib.sha256(blob).hexdigest()}, blob + finally: + rgb.close() + + +def process(sessions, job_id, token): + with sessions() as session: + job = session.get(AttachmentJob, job_id) + if job.lease_token != token: + return + attachment = session.get(Attachment, job.attachment_id) + staging_key = attachment.staging_key + client = s3_client() + response = client.get_object(Bucket=BUCKET, Key=staging_key) + with response['Body'] as body: + data = body.read(MAX_BYTES + 1) + if len(data) != attachment.size_bytes or hashlib.sha256(data).hexdigest() != attachment.sha256: + raise ValueError('staging checksum mismatch') + try: + rendered = list(variants(data)) + except OSError as exc: + raise ValueError('invalid image') from exc + # Job lock fences the final writes, cleanup and publication against lease recovery. + # Work before this transaction is CPU-only. Keys are stable across retries. + with sessions.begin() as session: + job = session.get(AttachmentJob, job_id, with_for_update=True) + if job.lease_token != token or job.lease_until <= now(): + return + current = session.get(Attachment, job.attachment_id, with_for_update=True) + if current.status != 'processing': + return + client.put_object(Bucket=BUCKET, Key=current.object_key, Body=data, + ContentType=current.content_type) + saved = [] + for meta, blob in rendered: + key = f'variants/{current.id}/v{job.pipeline_version}/{meta["name"]}' + client.put_object(Bucket=BUCKET, Key=key, Body=blob, ContentType=meta['content_type']) + saved.append({**meta, 'object_key': key}) + # Test-only crash injection; never enabled by compose.yaml. + if os.getenv('JOB_FAULT') == 'after_objects': + os._exit(91) + current.variants = saved + current.status, current.error_code = 'ready', None + current.updated_at = now() + job.lease_token = job.lease_until = None + if os.getenv('JOB_FAULT') == 'after_commit': + os._exit(92) + + +def main(): + # Linux worker: bounded address space as well as the container memory limit. + import resource + memory = int(os.getenv('JOB_MEMORY_MB', '768')) * 1024 * 1024 + resource.setrlimit(resource.RLIMIT_AS, (memory, memory)) + engine = create_db_engine() + try: + process(sessionmaker(engine, expire_on_commit=False), sys.argv[1], sys.argv[2]) + except (UnidentifiedImageError, Image.DecompressionBombError, ValueError, SyntaxError): + return 20 + except MemoryError: + return 21 + except Exception: + return 22 # No credentials or decoder/database internals in logs. + finally: + engine.dispose() + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/jobs.py b/src/jobs.py new file mode 100644 index 0000000..c8115ea --- /dev/null +++ b/src/jobs.py @@ -0,0 +1,176 @@ +"""Small PostgreSQL outbox + Redis Streams queue; PostgreSQL owns job state.""" +import argparse +import json +import logging +import os +import random +import time +from datetime import datetime, timedelta, timezone +from uuid import UUID, uuid4 + +from redis import Redis +from redis.exceptions import ResponseError +from sqlalchemy import and_, or_, select +from sqlalchemy.orm import sessionmaker + +from .database import create_db_engine +from .models import Attachment, AttachmentJob +from .object_storage import BUCKET, s3_client + +STREAM = 'attachment-jobs' +GROUP = 'image-workers' +MAX_ATTEMPTS = max(1, int(os.getenv('JOB_MAX_ATTEMPTS', '5'))) +TIMEOUT = max(1, int(os.getenv('JOB_TIMEOUT_SECONDS', '60'))) +LEASE_SECONDS = TIMEOUT + 15 + + +def now(): + return datetime.now(timezone.utc) + + +def broker(): + return Redis.from_url(os.getenv('BROKER_URL', 'redis://broker:6379/0'), + decode_responses=True, socket_connect_timeout=2, socket_timeout=5) + + +def ensure_group(client): + try: + client.xgroup_create(STREAM, GROUP, id='0', mkstream=True) + except ResponseError as exc: + if not str(exc).startswith('BUSYGROUP'): + raise + + +def envelope(job): + return {'schema_version': 1, 'job_id': job.id, 'type': 'attachment.process.v1', + 'attachment_id': job.attachment_id, 'pipeline_version': job.pipeline_version, + 'attempt': job.attempt + 1, + 'created_at': job.created_at.isoformat().replace('+00:00', 'Z')} + + +def fail(session, job, attachment, code, permanent=False): + job.failures += 1 + job.last_error = code + job.lease_token = job.lease_until = job.published_at = None + attachment.updated_at = now() + if permanent or job.failures >= MAX_ATTEMPTS: + attachment.status = 'failed' + attachment.error_code = code if permanent else 'retry_exhausted' + else: + attachment.status = 'queued' + attachment.error_code = None + delay = min(60, 2 ** min(job.failures, 6)) * random.uniform(0.5, 1.5) + job.available_at = now() + timedelta(seconds=delay) + logging.info('job=%s attempt=%s result=%s code=%s', job.id, job.attempt, attachment.status, code) + + +def dispatch_once(sessions, client): + # A crash after XADD but before COMMIT deliberately results in redelivery. + with sessions.begin() as session: + jobs = list(session.scalars(select(AttachmentJob).join(Attachment).where( + or_(and_(Attachment.status == 'queued', AttachmentJob.published_at.is_(None)), + and_(Attachment.status == 'processing', AttachmentJob.lease_until <= now())), + AttachmentJob.available_at <= now()).order_by(AttachmentJob.available_at) + .with_for_update(skip_locked=True, of=AttachmentJob).limit(50))) + for job in jobs: + attachment = session.get(Attachment, job.attachment_id, with_for_update=True) + if attachment.status == 'processing': + if job.lease_until <= now(): + fail(session, job, attachment, 'lease_expired') + continue + if job.published_at is None: + client.xadd(STREAM, {'job': json.dumps(envelope(job))}) + job.published_at = now() + + +def claim(sessions, message): + if (message.get('schema_version') != 1 or message.get('type') != 'attachment.process.v1' + or message.get('pipeline_version') != 1): + raise ValueError('unsupported job envelope') + job_id = str(UUID(message['job_id'])) + with sessions.begin() as session: + job = session.get(AttachmentJob, job_id, with_for_update=True) + if job is None or job.attachment_id != message.get('attachment_id'): + raise ValueError('unknown job') + attachment = session.get(Attachment, job.attachment_id, with_for_update=True) + if attachment.status != 'queued' or job.available_at > now(): + return None + job.attempt += 1 + job.lease_token = str(uuid4()) + job.lease_until = now() + timedelta(seconds=LEASE_SECONDS) + attachment.status = 'processing' + attachment.updated_at = now() + return job.id, job.lease_token + + +def record_failure(sessions, job_id, token, code, permanent=False): + with sessions.begin() as session: + job = session.get(AttachmentJob, job_id, with_for_update=True) + if job.lease_token == token: + attachment = session.get(Attachment, job.attachment_id, with_for_update=True) + fail(session, job, attachment, code, permanent) + + +def replay(sessions, job_id, reason): + if not reason.strip() or len(reason) > 200: + raise ValueError('Give a nonempty reason of at most 200 characters') + with sessions.begin() as session: + job = session.get(AttachmentJob, str(UUID(job_id)), with_for_update=True) + if job is None: + raise ValueError('Job not found') + attachment = session.get(Attachment, job.attachment_id, with_for_update=True) + if attachment.status != 'failed': + raise ValueError('Replay is allowed only for failed jobs') + if not attachment.staging_key: + raise ValueError('Staging retention expired; upload the file again') + s3_client().head_object(Bucket=BUCKET, Key=attachment.staging_key) + job.replay_audit = job.replay_audit + [{'at': now().isoformat(), 'reason': reason, + 'after_attempt': job.attempt, + 'last_error': job.last_error}] + job.failures = 0 + job.available_at = now() + job.published_at = job.lease_until = job.lease_token = None + attachment.status, attachment.error_code = 'queued', None + attachment.updated_at = now() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest='command', required=True) + sub.add_parser('dispatch') + sub.add_parser('failed') + retry = sub.add_parser('replay') + retry.add_argument('job_id') + retry.add_argument('--reason', required=True) + args = parser.parse_args() + logging.basicConfig(level=logging.INFO) + engine = create_db_engine() + sessions = sessionmaker(engine, expire_on_commit=False) + try: + if args.command == 'failed': + with sessions() as session: + for job, att in session.execute(select(AttachmentJob, Attachment).join( + Attachment, Attachment.id == AttachmentJob.attachment_id) + .where(Attachment.status == 'failed')): + print(json.dumps({'job_id': job.id, 'attachment_id': att.id, + 'attempt': job.attempt, 'error_code': att.error_code, + 'last_error': job.last_error, + 'replay_available': bool(att.staging_key)})) + elif args.command == 'replay': + replay(sessions, args.job_id, args.reason) + print('Job queued for replay') + else: + client = broker() + while True: + try: + ensure_group(client) + dispatch_once(sessions, client) + except Exception as exc: + logging.warning('Dispatch deferred: %s', type(exc).__name__) + time.sleep(1) + finally: + engine.dispose() + + +if __name__ == '__main__': + main() diff --git a/src/models.py b/src/models.py index 7e1582b..596f898 100644 --- a/src/models.py +++ b/src/models.py @@ -3,7 +3,7 @@ from uuid import uuid4 from sqlalchemy import BigInteger, CheckConstraint, DateTime, ForeignKey, Index, Text, Uuid, func from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column -from sqlalchemy.dialects.postgresql import ARRAY +from sqlalchemy.dialects.postgresql import ARRAY, JSONB class Base(DeclarativeBase): @@ -74,6 +74,28 @@ class Attachment(Base): size_bytes: Mapped[int] = mapped_column(BigInteger) sha256: Mapped[str] = mapped_column(Text) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp()) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp()) + status: Mapped[str] = mapped_column(Text, server_default='queued') + staging_key: Mapped[str | None] = mapped_column(Text) + variants: Mapped[list] = mapped_column(JSONB, server_default='[]') + error_code: Mapped[str | None] = mapped_column(Text) + + +class AttachmentJob(Base): + """The job row is also the transactional outbox and failed-job registry.""" + __tablename__ = 'attachment_jobs' + id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4())) + attachment_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('attachments.id'), unique=True) + pipeline_version: Mapped[int] = mapped_column(BigInteger, server_default='1') + attempt: Mapped[int] = mapped_column(BigInteger, server_default='0') + failures: Mapped[int] = mapped_column(BigInteger, server_default='0') + last_error: Mapped[str | None] = mapped_column(Text) + available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + lease_token: Mapped[str | None] = mapped_column(Uuid(as_uuid=False)) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp()) + replay_audit: Mapped[list] = mapped_column(JSONB, server_default='[]') class IdempotencyRecord(Base): diff --git a/src/routers/attachments.py b/src/routers/attachments.py index 57007b6..f5eb354 100644 --- a/src/routers/attachments.py +++ b/src/routers/attachments.py @@ -20,7 +20,7 @@ router = APIRouter(prefix='/api/v1', tags=['attachments']) decode_slot = asyncio.Semaphore(1) -@router.post('/chats/{chat_id}/attachments', status_code=201) +@router.post('/chats/{chat_id}/attachments', status_code=202) async def upload(chat_id: str, request: Request, filename: str = Query(...), user: dict = Depends(current_user)): chat_id = ensure_uuid(chat_id) @@ -67,9 +67,15 @@ def attachment_info(attachment_id: str, user: dict = Depends(current_user)): @router.get('/attachments/{attachment_id}/content') def download(attachment_id: str, variant: str = 'original', user: dict = Depends(current_user)): attachment = get_attachment(attachment_id, user['id']) + if attachment.status != 'ready': + raise APIError(409, 'conflict', 'Вложение ещё не готово') + key, content_type, size = attachment.object_key, attachment.content_type, attachment.size_bytes if variant != 'original': - raise APIError(404, 'not_found', 'Вариант не найден') - body = s3_client().get_object(Bucket=BUCKET, Key=attachment.object_key)['Body'] + item = next((v for v in attachment.variants if v['name'] == variant), None) + if item is None: + raise APIError(404, 'not_found', 'Вариант не найден') + key, content_type, size = item['object_key'], item['content_type'], item['size_bytes'] + body = s3_client().get_object(Bucket=BUCKET, Key=key)['Body'] def chunks(): try: @@ -77,8 +83,8 @@ def download(attachment_id: str, variant: str = 'original', user: dict = Depends finally: body.close() - return StreamingResponse(chunks(), media_type=attachment.content_type, - headers={'Content-Length': str(attachment.size_bytes), + return StreamingResponse(chunks(), media_type=content_type, + headers={'Content-Length': str(size), 'Content-Disposition': "attachment; filename*=UTF-8''" + quote(attachment.filename, safe=''), 'X-Content-Type-Options': 'nosniff'}, background=BackgroundTask(body.close)) diff --git a/src/store.py b/src/store.py index 48d2a22..b273f4c 100644 --- a/src/store.py +++ b/src/store.py @@ -136,8 +136,9 @@ class PostgresStore: with self.sessions.begin() as session: self._chat(session, chat_id, actor_id) if attachment_ids: - found = list(session.scalars(select(Attachment.id).where( - Attachment.id.in_(attachment_ids), Attachment.chat_id == chat_id))) + found = list(session.scalars(select(Attachment).where( + Attachment.id.in_(attachment_ids), Attachment.chat_id == chat_id) + .order_by(Attachment.id).with_for_update())) if len(found) != len(attachment_ids): raise APIError(404, 'not_found', 'Объект не найден') if idempotency_key is not None: @@ -150,6 +151,8 @@ class PostgresStore: if previous.text != text or previous.attachment_ids != attachment_ids: raise APIError(409, 'conflict', 'Ключ уже использован для другого сообщения') return serialize(previous, *MESSAGE_FIELDS) + if attachment_ids and any(item.status == 'failed' for item in found): + raise APIError(409, 'conflict', 'Обработка вложения завершилась ошибкой') session.execute(update(Chat).where(Chat.id == chat_id).values( history_version=Chat.history_version + 1)) message = Message(chat_id=chat_id, sender_id=actor_id, text=text, attachment_ids=attachment_ids) diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 0000000..0708885 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,76 @@ +"""Bounded consumers, one unacknowledged message per consumer, hard process timeout.""" +import json +import logging +import os +import subprocess +import sys +import time +from concurrent.futures import ThreadPoolExecutor +from uuid import uuid4 + +from sqlalchemy.orm import sessionmaker + +from .database import create_db_engine +from .jobs import GROUP, STREAM, LEASE_SECONDS, TIMEOUT, broker, claim, ensure_group, record_failure + + +def handle(sessions, client, message_id, fields): + try: + claimed = claim(sessions, json.loads(fields['job'])) + except (ValueError, KeyError, TypeError): + logging.error('Invalid broker envelope: id=%s', message_id) + claimed = None + if claimed: + job_id, token = claimed + try: + result = subprocess.run([sys.executable, '-m', 'src.image_processor', job_id, token], + timeout=TIMEOUT, check=False) + code = result.returncode + except subprocess.TimeoutExpired: + code = 23 + if code in (91, 92): + # Simulate a lost worker before ack: leave the Redis pending entry and lease. + return + if code: + record_failure(sessions, job_id, token, + {20: 'invalid_image', 21: 'resource_limit', 23: 'processing_timeout'} + .get(code, 'processing_failed'), permanent=code in (20, 21)) + # Only after a durable ready/failed/retry (or an existing durable lease). + with client.pipeline(transaction=True) as pipe: + pipe.xack(STREAM, GROUP, message_id) + pipe.xdel(STREAM, message_id) + pipe.execute() + + +def consume(sessions): + client = broker() + consumer = str(uuid4()) + cursor = '0-0' + while True: + try: + ensure_group(client) + pending = client.xautoclaim(STREAM, GROUP, consumer, LEASE_SECONDS * 1000, + start_id=cursor, count=1) + cursor, messages = pending[0], pending[1] + if not messages: + batches = client.xreadgroup(GROUP, consumer, {STREAM: '>'}, count=1, block=1000) + messages = batches[0][1] if batches else [] + for message_id, fields in messages: + handle(sessions, client, message_id, fields) + except Exception as exc: + logging.warning('Worker deferred: %s', type(exc).__name__) + time.sleep(1) + + +def main(): + logging.basicConfig(level=logging.INFO) + engine = create_db_engine() + sessions = sessionmaker(engine, expire_on_commit=False) + concurrency = min(4, max(1, int(os.getenv('WORKER_CONCURRENCY', '2')))) + with ThreadPoolExecutor(max_workers=concurrency) as pool: + for task in [pool.submit(consume, sessions) for _ in range(concurrency)]: + task.result() + + +if __name__ == '__main__': + main() diff --git a/student-tests/edge_cases.py b/student-tests/edge_cases.py new file mode 100644 index 0000000..a3d750f --- /dev/null +++ b/student-tests/edge_cases.py @@ -0,0 +1,97 @@ +"""Additional real-container checks: validation, permanent failure, timeout, retention.""" +import os + +from pipeline import Client, check, compose, remote, state, until + + +def main(): + c = Client(os.getenv('BASE_URL', 'http://127.0.0.1:8080')) + alice, bob, eve, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + '/attachments' + c.call('POST', path + '?filename=bad.png', 415, actor=alice, raw=b'not a PNG', + headers={'Content-Type': 'image/png'}) + c.call('POST', path + '?filename=../bad.png', 400, actor=alice, raw=b'bad', + headers={'Content-Type': 'image/png'}) + try: + compose('stop', 'worker', 'cleanup') + att, data = c.attachment(alice, chat) + aid = att['id'] + before = state(aid) + # A queued record cannot be collected even when its staging is old. + remote(f''' +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace +from src.cleanup import cleanup +client = s3_client() +class SingleObject: + def __getattr__(self, name): + return getattr(client, name) + def get_paginator(self, name): + obj = dict(client.head_object(Bucket=BUCKET, Key={before['staging']!r}), Key={before['staging']!r}) + return SimpleNamespace(paginate=lambda **kw: [{{'Contents': [obj]}}]) +cleanup(sessions, SingleObject(), now=datetime.now(timezone.utc) + timedelta(days=2)) +s3_client().head_object(Bucket=BUCKET, Key={before['staging']!r}) +''') + # Force a real checksum failure after upload, without changing the public contract. + remote(f"s3_client().put_object(Bucket=BUCKET, Key={before['staging']!r}, Body=b'corrupted')\n") + key = 'queued-before-failure-' + aid + message = c.message(alice, chat, '', key=key, attachments=[aid])['json'] + compose('start', 'worker') + until(lambda: state(aid)['status'] == 'failed') + check(state(aid)['attempt'] == 1 and state(aid)['error'] == 'invalid_image', 'permanent error retried') + c.call('GET', '/api/v1/attachments/' + aid + '/content', 409, actor=bob) + check(c.message(alice, chat, '', key=key, attachments=[aid])['json'] == message, + 'failed processing broke message idempotency') + # Expire this test job's failed retention; cleanup must remove its staging. + remote(f''' +from datetime import timedelta +from src.jobs import now, replay +from src.cleanup import cleanup +with sessions.begin() as session: + a = session.get(Attachment, {aid!r}) + a.updated_at = now() - timedelta(days=2) +cleanup(sessions, s3_client()) +try: + replay(sessions, {before['job_id']!r}, 'expired staging must reject') +except ValueError: + pass +else: + raise AssertionError('Replay accepted missing staging') +''') + check(state(aid)['staging'] is None, 'failed staging retention did not expire') + print('PASS: upload validation, active staging retention, permanent failure, message replay, expired staging', flush=True) + + compose('stop', 'worker') + att, _ = c.attachment(alice, chat) + aid = att['id'] + until(lambda: state(aid)['published']) + # Substitute a real sleeping child for the decoder, keeping the real timeout/kill path. + remote(f''' +import subprocess, sys +import src.worker as worker +from unittest.mock import patch +real_run = subprocess.run +def stalled(*args, **kwargs): + return real_run([sys.executable, '-c', 'import time; time.sleep(30)'], timeout=1) +with sessions() as session: + j = session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == {aid!r})) + fields = {{'job': json.dumps(envelope(j))}} +with patch.object(worker.subprocess, 'run', stalled): + worker.handle(sessions, broker(), '0-0', fields) +''') + check(state(aid)['status'] == 'queued' and state(aid)['failures'] == 1, 'timeout did not schedule retry') + remote(f''' +with sessions() as session: + j = session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == {aid!r})) + assert j.last_error == 'processing_timeout' +''') + compose('start', 'worker') + c.wait_ready(bob, att) + check(state(aid)['attempt'] == 2, 'timeout recovery did not complete') + print('PASS: real subprocess timeout, termination and successful retry', flush=True) + finally: + compose('start', 'worker', 'cleanup') + + +if __name__ == '__main__': + main() diff --git a/student-tests/pipeline.py b/student-tests/pipeline.py new file mode 100644 index 0000000..0110dde --- /dev/null +++ b/student-tests/pipeline.py @@ -0,0 +1,197 @@ +"""Integration/fault checks for the local lab5 Compose project; restores stopped services.""" +import base64 +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check + + +def compose(*args, input=None): + result = subprocess.run(['docker', 'compose', *args], cwd=ROOT, input=input, + text=True, encoding='utf-8', capture_output=True) + if result.returncode: + raise AssertionError('Compose failed: ' + ' '.join(args[:3]) + '\n' + result.stderr) + return result.stdout + + +PRELUDE = ''' +import json +from sqlalchemy import select +from sqlalchemy.orm import sessionmaker +from src.database import create_db_engine +from src.models import Attachment, AttachmentJob +from src.jobs import broker, envelope, GROUP, STREAM +from src.object_storage import BUCKET, s3_client +engine = create_db_engine() +sessions = sessionmaker(engine, expire_on_commit=False) +''' + + +def remote(code, fault=None): + options = ['exec', '-T'] + if fault: + options += ['-e', 'JOB_FAULT=' + fault] + return compose(*options, 'api', 'python', '-', input=PRELUDE + code).strip() + + +def state(attachment_id): + return json.loads(remote(f''' +with sessions() as session: + a = session.get(Attachment, {attachment_id!r}) + j = session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == a.id)) + print(json.dumps(dict(status=a.status, error=a.error_code, job_id=j.id, + attempt=j.attempt, failures=j.failures, published=bool(j.published_at), + staging=a.staging_key, variants=a.variants, audit=j.replay_audit))) +''')) + + +def until(fn, timeout=130): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + result = fn() + if result: + return result + time.sleep(1) + raise AssertionError('Timed out waiting for durable state') + + +def ready(client): + def probe(): + try: + return client.call('GET', '/health/ready')['status'] == 200 + except Exception: + return False + until(probe, 40) + + +def inject(attachment_id, fault): + # Use a real consumer-group delivery, leaving the message pending at the crash point. + return remote(f''' +from src.worker import handle +client = broker() +with sessions() as session: + j = session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == {attachment_id!r})) + target = j.id +for _ in range(100): + batches = client.xreadgroup(GROUP, 'fault-test', {{STREAM: '>'}}, count=1, block=1000) + if batches: + mid, fields = batches[0][1][0] + handle(sessions, client, mid, fields) + if json.loads(fields['job'])['job_id'] == target: + print(mid) + break +else: + raise AssertionError('No delivery for target job') +''', fault=fault) + + +def main(): + client = Client(os.getenv('BASE_URL', 'http://127.0.0.1:8080')) + alice, bob, eve, chat = client.bootstrap() + try: + compose('stop', 'worker', 'dispatcher', 'cleanup') + attachment, content = client.attachment(alice, chat) + aid = attachment['id'] + check(attachment['status'] == 'queued', 'upload must return queued') + check(not state(aid)['published'], 'outbox was already published') + client.message(alice, chat, '', attachments=[aid]) + client.call('GET', f'/api/v1/attachments/{aid}/content', 409, actor=bob) + compose('restart', 'api') + ready(client) + check(state(aid)['status'] == 'queued', 'API restart lost upload') + compose('start', 'dispatcher') + until(lambda: state(aid)['published']) + compose('restart', 'broker') + compose('start', 'worker') + result = client.wait_ready(bob, attachment) + check({v['name'] for v in result['variants']} == {'thumbnail', 'crop', 'optimized'}, 'missing variants') + for variant in result['variants']: + download = client.call('GET', f'/api/v1/attachments/{aid}/content?variant=' + variant['name'], actor=bob) + check(hashlib.sha256(download['bytes']).hexdigest() == variant['sha256'], 'variant checksum mismatch') + check(len(download['bytes']) == variant['size_bytes'], 'variant size mismatch') + check(download['headers'].get_content_type() == variant['content_type'], 'variant MIME mismatch') + check(client.call('GET', f'/api/v1/attachments/{aid}/content', actor=bob)['bytes'] == content, 'original changed') + print('PASS: outbox COMMIT/publish gap, stopped worker, API and broker restart', flush=True) + + for fault in ('after_objects', 'after_commit'): + compose('stop', 'worker') + att, _ = client.attachment(alice, chat) + aid = att['id'] + until(lambda: state(aid)['published']) + pending_id = inject(aid, fault) + snapshot = state(aid) + check(snapshot['status'] == ('processing' if fault == 'after_objects' else 'ready'), 'wrong crash state') + compose('start', 'worker') + until(lambda: state(aid)['status'] == 'ready') + until(lambda: remote(f"print(int(not broker().xpending_range(STREAM, GROUP, {pending_id!r}, {pending_id!r}, 1)))\n") == '1') + if fault == 'after_objects': + check(state(aid)['attempt'] == 2, 'expired lease was not retried') + else: + time.sleep(2) + check(state(aid)['attempt'] == 1, 'ready job was reprocessed') + # Force duplicate delivery; stable object keys and metadata must remain identical. + before = state(aid) + remote(f''' +with sessions() as session: + j = session.get(AttachmentJob, {before['job_id']!r}) + broker().xadd(STREAM, {{'job': json.dumps(envelope(j))}}) +''') + time.sleep(2) + after = state(aid) + check(before['variants'] == after['variants'] and before['attempt'] == after['attempt'], 'duplicate changed result') + count = remote(f"print(s3_client().list_objects_v2(Bucket=BUCKET, Prefix='variants/{aid}/')['KeyCount'])\n") + check(count == '3', 'redelivery produced extra S3 objects') + print('PASS: crash ' + fault + ', redelivery and stable variants', flush=True) + + compose('stop', 'worker') + att, content = client.attachment(alice, chat) + aid = att['id'] + until(lambda: state(aid)['published']) + # Remove only this test's staging object to cause a real temporary S3 error. + staging = state(aid)['staging'] + remote(f"s3_client().delete_object(Bucket=BUCKET, Key={staging!r})\n") + compose('start', 'worker') + until(lambda: state(aid)['status'] == 'failed', 150) + failed = state(aid) + check(failed['error'] == 'retry_exhausted' and failed['attempt'] == 5, 'retry budget incorrect') + client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, + body={'text': '', 'attachment_ids': [aid]}, headers={'Idempotency-Key': aid}) + registry = compose('exec', '-T', 'api', 'python', '-m', 'src.jobs', 'failed') + check(failed['job_id'] in registry, 'failed job not visible') + remote(f''' +import base64 +s3_client().put_object(Bucket=BUCKET, Key={staging!r}, + Body=base64.b64decode({base64.b64encode(content).decode()!r}), ContentType='image/png') +''') + compose('exec', '-T', 'api', 'python', '-m', 'src.jobs', 'replay', failed['job_id'], + '--reason', 'synthetic test: restored staging') + client.wait_ready(bob, att) + check(len(state(aid)['audit']) == 1 and state(aid)['attempt'] == 6, 'replay lost audit or attempt') + remote(f''' +from src.jobs import replay +try: + replay(sessions, {failed['job_id']!r}, 'must reject ready') +except ValueError: + pass +else: + raise AssertionError('Replayed a ready job') +''') + print('PASS: bounded retries, failed registry, explicit audited replay, ready replay rejected', flush=True) + + compose('exec', '-T', 'api', 'python', '-m', 'src.cleanup', '--once') + check(state(aid)['staging'] is None, 'completed staging was not cleaned') + check(client.call('GET', f'/api/v1/attachments/{aid}/content', actor=bob)['bytes'] == content, 'cleanup deleted original') + print('PASS: staging cleanup preserves ready objects', flush=True) + finally: + compose('start', 'broker', 'dispatcher', 'worker', 'cleanup') + + +if __name__ == '__main__': + main() diff --git a/student-tests/test_images.py b/student-tests/test_images.py new file mode 100644 index 0000000..c644cc5 --- /dev/null +++ b/student-tests/test_images.py @@ -0,0 +1,61 @@ +"""Run inside the application image: python -m unittest discover -s /checks.""" +import hashlib +import io +import unittest +from unittest.mock import patch + +from PIL import Image + +from src.image_processor import variants + + +class ImageTests(unittest.TestCase): + def render(self, image, fmt='PNG', **kwargs): + source = io.BytesIO() + image.save(source, format=fmt, **kwargs) + results = list(variants(source.getvalue())) + self.assertEqual([meta['name'] for meta, _ in results], ['thumbnail', 'crop', 'optimized']) + decoded = {} + for meta, data in results: + self.assertEqual(meta['sha256'], hashlib.sha256(data).hexdigest()) + self.assertEqual(meta['size_bytes'], len(data)) + with Image.open(io.BytesIO(data)) as item: + item.load() + self.assertEqual(item.size, (meta['width'], meta['height'])) + self.assertFalse(item.getexif()) + self.assertNotIn('icc_profile', item.info) + decoded[meta['name']] = item.copy() + return decoded + + def test_dimensions_and_floor_rounding(self): + images = self.render(Image.new('RGB', (321, 200), 'red')) + self.assertEqual(images['thumbnail'].size, (256, 159)) + self.assertEqual(images['crop'].size, (128, 128)) + self.assertEqual(images['optimized'].size, (321, 200)) + + def test_no_upscale_and_white_alpha(self): + images = self.render(Image.new('RGBA', (20, 10), (255, 0, 0, 0))) + self.assertEqual(images['thumbnail'].size, (20, 10)) + self.assertEqual(images['optimized'].size, (20, 10)) + self.assertEqual(images['thumbnail'].getpixel((5, 5)), (255, 255, 255)) + + def test_exif_orientation_and_metadata_removal(self): + exif = Image.Exif() + exif[274] = 6 + exif[270] = 'private description' + images = self.render(Image.new('RGB', (320, 200), 'blue'), 'JPEG', exif=exif) + self.assertEqual(images['thumbnail'].size, (160, 256)) + self.assertEqual(images['optimized'].size, (200, 320)) + + def test_optimized_bound(self): + images = self.render(Image.new('RGB', (2000, 1000), 'green')) + self.assertEqual(images['optimized'].size, (1280, 640)) + + def test_worker_pixel_limit(self): + with patch('src.image_processor.MAX_PIXELS', 100): + with self.assertRaises(ValueError): + self.render(Image.new('RGB', (11, 10))) + + +if __name__ == '__main__': + unittest.main()